From 1aac3a15b190c337c2e65e50d60d848211efb012 Mon Sep 17 00:00:00 2001 From: Karen Zeng Date: Tue, 23 May 2023 17:59:16 -0400 Subject: [PATCH] Defined get_price function & modified timestamp --- __pycache__/fp2_14.cpython-311.pyc | Bin 0 -> 11872 bytes fp2_14.py | 433 ++++++++++++++--------------- test_fp2_14.py | 63 +++++ 3 files changed, 276 insertions(+), 220 deletions(-) create mode 100644 __pycache__/fp2_14.cpython-311.pyc create mode 100644 test_fp2_14.py diff --git a/__pycache__/fp2_14.cpython-311.pyc b/__pycache__/fp2_14.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ce9486dc59a48ddaf475686c5596b589e650abf4 GIT binary patch literal 11872 zcmdryTWlNGm6uNq-=Zi{ltjs6>t#wJsi!SF>xXR1uwzGd{7OU1p+}sdM49B!GsDPc z$y6<}C{q}y3lylaO|ima(^a+(*1$GEKb!3aD4PA4A*K*9fPerC1Dih;X#m4NyXVgE zDa%dvfuh*q<-K?AJ@?#m&pqcp&Y6F3IIIYs+x`^2{}4j|j$f*esgn6JX+Y>Fh(@$7 ziBh_VPKWdQq&{Vc7}UHWX-t_SCN*zNno}*27Bz255-Cf>qUOy>Ys3oe%t>3s25C#u zo^nJSl{#m{1!Y9CHRX=D)iO)6E#--{SL!<=MzyUs;#JemNT-_mB0fE0T&rDGqL&eE z{X5-b+s}0f{Q~~l6X^!19@@@y{}la9-(06dP}lNv0Q^N297X($f1|hd+K{0gd$cyK z_EkX`|6^w}L}VZBijZ_`WIyzyTRl*P+|^@0)4wrLt!qpflG*=rJx~XK?THM~?#LiB zxG_{MX-w(1=Kg6f2kM(tc>)L+^vwwhCIMJlucklZiAle=o)`G(%&6HD-zC5^pmcytG`(2njaLLsoz4{sJWSYl#%YW`%it&9mvoV1@e0gg!?($HCCXKm1^ttbV8`)ZC9xz!vl_NS^y+$k|wVT zl#s=Qw#3+EHkIac9j7^l0zycNq$z=lb5x2sO6DBFAf4c;r6dzv&SY@|RvTo_7S9AH z$0w)8rzdA}es}>!@i8u;RtFc(UJT|EnHbQ`=Ta+zkl~Mp!)t46A)dOw#xP7M#-_qd zI-Jo)8a~Hnc_w(4WobTmHo>KW*VuSA7@nP(I6QMg;HVfAP0+_;N2cOa^aK^09jB&) z(+tkh(-T2zdVG>%rl%%m55=GB!e9I+JYV1iG*(-K;Kc}kSWD5D{{f}2qUs8|ypGoA z90;j7LoH{lQ_$O=Rq8BA00h-%qY-V_ zZElwKtEe9d>OP^tteUH^t1Y!0QO$uySqq>^G*K|TV@MNPx~8k@*9)Vz>>8`}|It`& zpv`t_9wxy`!gGZYvRs-JRv1!X?g?Z<%f}NOFOW=Zf@X(Wr4@jWMWCHqU<0J;)$4H(fvwR}GOv0idP>D2uFaT5~lVpk#Vwko%RV0I{ zOvF`Ocw7xt&=W;strR$n4Djy)SYmLwiWDpUGVV$Usi>~8m`cDcK%)fEw8j8U49RCe z69F$qO*6?vDj_f&3EaS_mzd>5I*oBsX_{o<&+bLoSf!fJa+sA2#|-l%n@-+`sL}(V zinCCQ@lbKKi)2zItUf-zI(_vbm64T+n!iV-GD%p&fQfv91}cR$n3LLk^4dhlD43?; zQkEAm-F!A4PuyepqiSCyIYOR;MW5lKJgl19q6q>3>u)qVF*!9o^L1dSsMwul4i;WI z8l5|J`pmp0M6cm_RQaUw{AImV#DD?;1ZgA_ctBW2DOJRUsaEA2;qb`KT5mZ!WMhuN>7sJKOpA*$t0v8)gSX{@})Gn3JY=}_jr(u%3A3?qse z!tDoaBk<>s!Pc^lb`BypxqiNEbUf^njQiA_=pK`dV_Kn4GImu8gQ9m>bO*H{8H4NR zcl1V+W4qP6)!JWb?f>A^-=Et|NUgJS>uk|fcC~N04wPI6HjYWI3E4Fv5);q1ZJq}w zWZR%@`>P$KGdb!}nXng)+V2@WI|iu5seLcWpKMPR=vO9~D zd_S=9{UdW;^oh4??ug-&X%oc%(styu9sS1MbvkVL&4>x&7=YCtyt5kM|K$!uKS8kD zH|+32?FImQ^se1}qoJ;b4GFdV8KKqMUcdKVPV3{lZbr?gfC3U+bJ=(?yt1BC?sYQ~76^5ZzE$6&y ziwx{%f?|p#6R|sRB(*hQ=ka`z90@Up^+=5(kl`Od1PY}$flhe)KRCNNCwW7%H?-xQ zE_tUvz9P?D{r&YfTm1r>LIzV6+ynqv z-UT)tAnKCD4FXzj2uH&>!X&B!!{IwM$ca6zZQZIEHVR4*L*i%&{(Kf9EGTA^<7wx> z2RAlfmpa38XLze~rqns}@sfP#+V5|COTO_INC>4{#)5J~w}J)bZ~_Yl#E|!`;ylDf z`{zXa7SUZIx+TId6MnH8c1r}WNksQtANr(EKiB^fiGcq8;w0kFurz5>6WUX1us~FV zM->y0B~3nZtudCplVGCSE37@-0I&~)!O>LFAPou}YSbpGm~i6&eh2=10U|6GEwFz0 z20w^zQj%|6_Kk1(4wZa|w(U;YKJwA2$LGXDadC86vaiVYl|LBtUDiJ$h>DJ{%*f&t zo#Pw55J8ZL37Np#Zkg!VB6>PJe>(XcY>y zXSKF(?Y%R>{sj=fs6iEUje8ZX&+GQuT`YJ<#7lSO*{$#brs!pLaLcYfg0q@~oz&2L zFW!6q5DxeQ4v??46%Wc*`yHy4E}<<2}66{z1P}{PwEBt zJ8rP(scGG!_dlM4j$OQJ%eQ^QdGO>tFOgku^Y{zx0O_ytt1+cJn;~hRD$5;tPrg0h z(IlZ>;gx#Jy00g{yRXiCC*4)|8PMJJe8Jbq$R)3 z)knBkFI@)9IsAR_x5D2Be>?mM_*<5B^#1pWT_f&p!if*)>LXmP<`BwvFYD<8kNcac zQLW$28gqD;#tiJOF-?3M-1|gB4;6YEX^82odUmy zGzm@)urShKXXIL8D7hoM4ycur>56JbtR7Aq&#D^0&52vu0e2v*o@A5M5|a$a*<_Mk z14ChuPb`7!Y5C~UL^_idawY;lIe)lv&b^6^M_?sBMh=5b4eWWT;aiHmV!Mo~jvo^? zVUo{vql*7}ntGc`Ko?2oG{p%^LK?fDu%(&dau#ghjNd0yOpZ`}MVGPyhu8nykV;zJ z{HK}s;O?JFq^n*iY^k$=5t!l5U3=|fE>QJSWwKmsh2nut2N&tl0J(T@aRBEQLyPpc zCkrP6IdXv|0g>v{0_c?(K$8{IbsSqzbK^A&9l5pwR&Q(s#75MHR%#!WR^YuF42LzB z)XkH^{S0$#n95|pXrP);!%2E{m1on#x1Q_1{ajc00?s^0F|+YFnD7)6*ciae_R=A% z)nFO~p!$%n!6ZD_4RClb%Q>>(fzu#hS^|2d1#H3K!{QZtqo<6+M~iFtcA8iQ>k4?e z(lm1~*LIo$6D>F;szyl|P48vg9SOHsQe_ClTuuJSoNHHgAfU9Q7@nt=8O4ZwdSE_f zu&8p_g-k=o=I4$Vt9p*mI;T>1CRVpTxUc~@rR<09D6+Oo&W}cy7z7;dJ zeP?(MZ=xJtj5)k@ay=N%QkzZ14E7utv|_~bp_q~^1zc+3m`oC!4T`nGt|f+3Y{0`b zyOv~V7?fFKsN!zqXB1m$6@7+QEN5WMXR#ksu~xi{z{3zDCEGg|3aj$`@CTTOrEQ2qZ!yiI$cxO>` zjN$_1n58uW?bQOjTqvIETO+~lY9XF$jpSM@b$VJV0E(juscM1JR;RJnUkfle%CT$Q zQJ{#&#{g_8f7zp2?Gz$LC6h}qwHyoZKf(>aG~%oG=PvJq@5rwHE!RlNHL`hFa!tvu zsr7SDjTX^5vOz!uK{AfY#!=BYx?`Nzo4h;v9*g~}S!8K{&+))fbZist?>Qehi%z(X zgB9@5=7iJ{lskf39aE)_sm5FUU2&9??7VE};TG@3w|K9$Xx$-@1?+pi65*4GZkgy7 ziEcpEao~fY&HYmQnA|?L)jnBjpKL_6D30Ee>~G2Tw*XZKM%7_0S}>}%AG&_vl87#u z=n{#p&k46?33zl-A_6iI5Q#v&?3_dl%EX{Z3_k7b-Rc}Eb&h;A`FK{Gxhi#DlRK{! z?c0CSTqQ;e;`Mi=L*J1PeFqz;=2QdK+@1!i9is=>Hi9f28n?CykH$Y3>%Q>jJ5t|q zx$ijGHp~9-mVc(?pLwz@`H#!~lH zy2{>wIP%J`E{P)-WbehI?WuQ29G;iFXJzl%qOEN6J{m3A21VOoxqF~^0hqLpEZWPx zeNgA`g$dAGEU)Ue`vY73hf4j2KIVRTPn=6g{i|~S>Q?`~QvbajSbTN!*akOn77r0( z(N30K{X59uc%|HR=An7p*Zsa_V@M1flYGZz-|>g0^4P>bUHr$3A1*vzkXi!|?T_v} zd$#TB!~z$PT_?d4VmVZgw(V}QZCbL=$o3iWg@8RI+b$mle^u8QtIf{+`gO7G90bXJ zUbdeXUkJE4I9Q-9WWKVZp8Z?h!BThdqov0y;%rpvzAbm(20nHUZFNqUI;Wr9lsf0+ z&N<*x-^fK!ljj(^lkw>ed za|D7!jLF29NQ^xt+O~+k64AFg{5T{Aj{N@BnBP1t{ zui=DTsKg*Po=Zj$8;FZ7+b;Km`#&4n7#6!me$_6zj!UlNvI_+N__MFT&D61jjIhSk zB6Va#8xx{)2!ccm%fzrq46DVnn=7Jo7J@_^k%=QBairY3zi8TabrfM;s{Yi;w*XSe z@Qbkc{qwn&*MsP{ugo2MbpZXYUk5M08|b=l)bP7wt_yC%XLb{meCBpQ$!C3S7iJ8f zjjJUyCR}oKeiEvsp!G_~ARQjN5-^ndb&xLwOb}n=W}yc-n%Gm+y=djEz;nlOA$;k> zF?X{JFXTF|XW;no!lyyKX*jTH=L`5gK;Z7z9NPGMfWTqJzRa@AN%+zy$!2K|8!DCV zN)eSwz~0HhS6gx?H z-PfD|7ciVrfN!ysHoW_1!LywfLh&qo4Z`poc9<#V418-q@!S>s(z^#vKAXv~oDj+; z)B}^!{c;!%8z@yzL<5xOe{LKwzJe(PX01w^ab+fRpSy~i-oTN1O2Y<7#ZJ+5l;W1N z*!QoPPpe$eyl41ivkGo{e6ndi2Vvi}(t1_?lN4LKGt|R~(gEJ_BE6Fp3*KF;MSOHR*$6G7n$Sq?ixUm(T@L4+YPOzXBg$Hpv|4 zegtLsq|RGFpkSHQ>B{JU*c{5}u-F{Rs84JTWi%r;hcX%!n?o7d*41A_=0zBfh$>;P zeou<1DU{KEu^P(AvabHh$hEHiY8~xZk=0vld*Fw)Y2S6cmNmxK8+dzz7}hC>*Ue>Q zS~rR2p+|9uAV_FfM#CZ+-Z3LvTXFckz=ObgOBuCnq4pALe;Ad}po|7ZG+4HD6}x0h z7c6m>5y>(lTSnH+(8}4fZY{S@ipX7d3>V*&9mA3%AUguMv~R0@ywpDaq+2|;AhlnT z+b;nav}t);w{9pqT7UTF58iypzxURIw;r969s4%)8}1Fg?C4*&evZsC>Jm{`8JWLt z{hqZr^0jX`V}Q!rbJIj@+`@orS*$tmwWx3 z`td~Bci?Ybf8%;G_Q)mqPRPC!MMv2~%AUz0Q6!#~Z9TBNzwntv?c)eDB3gHl$>c1% zyB_);9o(3Be^hc0$?l<|xeVK>Npyxb;t)ZQh_FnAMKye`)g9kVZqkpFAEqCtMdxt{ t5^+K%PKd+_XuQ)4W7z5IfPo{3_%=|9Pwl<)Bk0o+)BL32(+M5K{{=QgLjM2& literal 0 HcmV?d00001 diff --git a/fp2_14.py b/fp2_14.py index 78bbadd..c6cf9cf 100644 --- a/fp2_14.py +++ b/fp2_14.py @@ -11,259 +11,241 @@ from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC import selenium.common.exceptions import re -import datetime +from datetime import datetime import h5py +import sys +import ntplib +import arrow # This fp2_14.py is used to load the new price and the timestamp, and append such data in an existing key in the HDF5 file "product1.h5" -# Create an empty dataframe to store the prices -df_new = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status']) -print(f"Created a dataframe: {df_new}") +def get_product_prices(project_id, subscription_name, url): + # Create an empty dataframe to store the prices + df_new = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status']) + print(f"Created a dataframe: {df_new}") -# Configure the Selenium driver options -options = Options() -options.add_argument('--disable-gpu') -driver = webdriver.Chrome(options=options) + # Configure the Selenium driver options + options = Options() + options.add_argument('--disable-gpu') + driver = webdriver.Chrome(options=options) -# Set up Pub/Sub subscription -project_id = "fp-20230425" -subscription_name = "fp-subscription-NFL-zipcodes" -subscriber = pubsub_v1.SubscriberClient() -subscription_path = subscriber.subscription_path(project_id, subscription_name) - -# Load the product page and extract the __NEXT_DATA__ data -url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f' -product_id = url.split("/")[-1][:5] -# product_id = url.split("/")[-1] -driver.delete_all_cookies -driver.get(url) -# time.sleep(5) - -def get_element_text(class_name_prefix): + # Set up Pub/Sub subscription ''' - Return the text in the first element with a given class name. - - Note that XPath doesn't have a function to check if a given word is present - in a list of words, so using contains() will match a class name even if - the class name is a substring of the class name being matched against. - To prevent this, we use spaces as a delimiter at the beginning and end of - the class name, and ensure there's only one space before and after all - class names in the class name list. - - Example: consider two class names with cache-busting suffixes: - * Header_soldStatusNotice__12345 - * Header_soldStatusNoticeBackground__ABCDE - - Matching against Header_soldStatusNotice using a naive implementation with - `contains()` would match both class names, even though only the first one - is the correct match. + project_id = "fp-20230425" + subscription_name = "fp-subscription-NFL-zipcodes" ''' - # Construct xpath selector. - xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... - xpath_str += "concat(' ', " # Step 2: a leading space, followed by... - xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces... - xpath_str += "' '), " # Step 4: a trailing space... - xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side. - try: - match = driver.find_element(By.XPATH, xpath_str) - return match.get_attribute('textContent') - except selenium.common.exceptions.NoSuchElementException as nsee: - pass - - # We didn't find a match, try again by doing a prefix match and assume - # the full class name ends with two underscores followed by a cache- - # busting string. - xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... - xpath_str += "concat(' ', " # Step 2: a leading space, followed by... - xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces... - xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores. - try: - match = driver.find_element(By.XPATH, xpath_str) - return match.get_attribute('textContent') - except selenium.common.exceptions.NoSuchElementException as nsee: - pass - - # No match, return empty string. - return '' + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path(project_id, subscription_name) -# time.sleep(2) -# Close the cookies setting window -try: - # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button") - cookie_button = WebDriverWait(driver, 2).until( - EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")) - ) - print(f"cookie_button = {cookie_button}") - cookie_button.click() -except: - pass + # Load the product page and extract the __NEXT_DATA__ data + # url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f' + product_id = url.split("/")[-1][:5] + # product_id = url.split("/")[-1] + driver.delete_all_cookies + driver.get(url) -# time.sleep(1) -# Clear any modal dialog -try: - # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]") - modal_close = WebDriverWait(driver, 2).until( - EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")) - ) - print(f"modal_close = {modal_close}") - modal_close.click() - #time.sleep(2) -except: - pass + def get_element_text(class_name_prefix): + ''' + Return the text in the first element with a given class name. -# Define callback function for Pub/Sub messages -def callback(message): - # Extract zip code from message data - zip_code = message.message.data.decode('utf-8') + Note that XPath doesn't have a function to check if a given word is present + in a list of words, so using contains() will match a class name even if + the class name is a substring of the class name being matched against. + To prevent this, we use spaces as a delimiter at the beginning and end of + the class name, and ensure there's only one space before and after all + class names in the class name list. - print(f"Received zip code: {zip_code}") + Example: consider two class names with cache-busting suffixes: + * Header_soldStatusNotice__12345 + * Header_soldStatusNoticeBackground__ABCDE - try: + Matching against Header_soldStatusNotice using a naive implementation with + `contains()` would match both class names, even though only the first one + is the correct match. + ''' + # Construct xpath selector. + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces... + xpath_str += "' '), " # Step 4: a trailing space... + xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side. try: - # time.sleep(2) - # Clear any modal dialog - # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]") - modal_close = WebDriverWait(driver, 2).until( - EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")) - ) - # print(f"modal_close = {modal_close}") + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + + # We didn't find a match, try again by doing a prefix match and assume + # the full class name ends with two underscores followed by a cache- + # busting string. + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces... + xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores. + try: + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + + # No match, return empty string. + return '' + + + def modal_close(): + try: + modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))) + print(f"modal_close = {modal_close}") modal_close.click() - # time.sleep(2) except: pass - + def notice_close(): try: - # time.sleep(1) # Clear any notice - # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button") - notice_close = WebDriverWait(driver, 5).until( - EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")) - ) - # print(f"notice_close = {notice_close}") - notice_close.click() - # time.sleep(1) - except: - pass - - # Enter zip code in the input field and click the "Check" button - # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]') - zip_button = WebDriverWait(driver, 5).until( - EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')) - ) - # print(f"{zip_button:}") - zip_button.click() - # time.sleep(1) - - # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input') - zip_input = WebDriverWait(driver, 2).until( - EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input')) - ) - # print(f"{zip_input:}") - zip_input.click() - # print("clearing zip_input") - zip_input.send_keys("\b\b\b\b\b") - # print("sending zip code") - zip_input.send_keys(zip_code) - - check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']") - check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']"))) - # print(f"{check_button:}") - check_button.click() - # time.sleep(2) - - try: - # time.sleep(2) - # Clear any notice - # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button") notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))) # print(f"notice_close = {notice_close}") notice_close.click() - #time.sleep(2) except: pass - try: - # Clear any modal dialog - modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))) - # print(f"modal_close = {modal_close}") - modal_close.click() + def cookie_close(): + try: + cookie_button = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")) + ) + print(f"cookie_button = {cookie_button}") + cookie_button.click() except: pass - # Record sales count, if any - sales_count_element = get_element_text('Header_count') - count_start_index = sales_count_element.index('sold ') + len('sold ') - count_end_index = sales_count_element.index('+') - sales_count = sales_count_element[count_start_index:count_end_index] - # Record any sold status notice - sold_status = get_element_text('UnavailableCartbtn_container') - if sold_status == 'Notify me': - sold_status = 'Sold Out' + cookie_close() + modal_close() + # Define callback function for Pub/Sub messages + def callback(message): + # Extract zip code from message data + zip_code = message.message.data.decode('utf-8') + + print(f"Received zip code: {zip_code}") - # Extract the price of rice crackers for this zip code - # Check if there is a minimum order quantity - min_quantity: float = 0 - total_price: float = 0 - unit_price: float = 0 try: - minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML') - if minimum_quantity_str: - regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)" - match = re.search(regexp,minimum_quantity_str) - min_quantity, total_price = match.groups() - except selenium.common.exceptions.NoSuchElementException as nsee: - print("No minimum purhcase string found") + + modal_close() + notice_close() + + # Enter zip code in the input field and click the "Check" button + zip_button = WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')) + ) + # print(f"{zip_button:}") + zip_button.click() + + # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input') + zip_input = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input')) + ) + # print(f"{zip_input:}") + zip_input.click() + # print("clearing zip_input") + zip_input.send_keys("\b\b\b\b\b") + # print("sending zip code") + zip_input.send_keys(zip_code) + + check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']") + check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']"))) + # print(f"{check_button:}") + check_button.click() + + notice_close() + modal_close() + + # Record sales count, if any + sales_count_element = get_element_text('Header_count') + count_start_index = sales_count_element.index('sold ') + len('sold ') + count_end_index = sales_count_element.index('+') + sales_count = sales_count_element[count_start_index:count_end_index] + # Record any sold status notice + sold_status = get_element_text('UnavailableCartbtn_container') + if sold_status == 'Notify me': + sold_status = 'Sold Out' + + # Extract the price of rice crackers for this zip code + # Check if there is a minimum order quantity + min_quantity: float = 0 + total_price: float = 0 + unit_price: float = 0 + try: + minimum_quantity_str = WebDriverWait(driver, 5).until(EC.presence_of_element_located(((By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML')))) + if minimum_quantity_str: + regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)" + match = re.search(regexp,minimum_quantity_str) + min_quantity, total_price = match.groups() + except selenium.common.exceptions.NoSuchElementException as nsee: + print("No minimum purhcase string found") + except Exception as e: + print(f"Exception when finding minimum purchase string: {e}") + + + + ld_json_element = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//script[@type='application/ld+json']"))) + ld_json_text = ld_json_element.get_attribute('innerHTML') + ld_json_start_index = ld_json_text.index('{') + ld_json_end_index = ld_json_text.rindex('}') + 1 + ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index] + ld_json = json.loads(ld_json_json) + ld_json_price = ld_json['offers']['price'] + # Print the price + print(ld_json_price) + + unit_price_element = WebDriverWait(driver, 5).until(EC.presence_of_element_located(((By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText')))) + unit_price = unit_price_element.replace("$", "") + print(f"unit_price = {unit_price}" ) + + subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message + # Append the zip code and price to the dataframe + #append_to_df(zip_code, price) + new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]}) + df_new = pd.concat([df_new, new_row], ignore_index=True) + return df_new + #print(f"Recorded the zip code and price: {zip_code},{price}") + #print(f"Updated a dataframe: {df_new}") + except Exception as e: - print(f"Exception when finding minimum purchase string: {e}") - + print(f"Caught exception: {e}") + print(f"Price not found for zip code {zip_code}") + return df_new + + # Start receiving messages (sync version) + while True: + pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path) + messages = [] + try: + pull_response = subscriber.pull(request = pull_request) + messages = pull_response.received_messages + except DeadlineExceeded: + pass + + if messages: + df_new = callback(messages[0]) + print(f"Updated a dataframe: {df_new}") + else: + print(f"no messages, trying again at {time.asctime()}") + break + return df_new + +''' +If using sys to get the function, please remove the following part of code. +''' +if __name__ == '__main__': + project_id = input("Enter the project ID: ") + subscription_name = input("Enter the subscription name: ") + url = input("Enter the URL: ") + + get_product_prices(project_id, subscription_name, url) - ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']") - ld_json_text = ld_json_element.get_attribute('innerHTML') - ld_json_start_index = ld_json_text.index('{') - ld_json_end_index = ld_json_text.rindex('}') + 1 - ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index] - ld_json = json.loads(ld_json_json) - ld_json_price = ld_json['offers']['price'] - # Print the price - print(ld_json_price) - - unit_price_element = driver.find_element(By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText') - unit_price = unit_price_element.replace("$", "") - print(f"unit_price = {unit_price}" ) - - subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message - # Append the zip code and price to the dataframe - #append_to_df(zip_code, price) - new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]}) - return pd.concat([df_new, new_row], ignore_index=True) - #print(f"Recorded the zip code and price: {zip_code},{price}") - #print(f"Updated a dataframe: {df_new}") - - except Exception as e: - print(f"Caught exception: {e}") - print(f"Price not found for zip code {zip_code}") - price = 'Price not found' - return df_new - -# Start receiving messages (sync version) -while True: - pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path) - messages = [] - try: - pull_response = subscriber.pull(request = pull_request) - messages = pull_response.received_messages - except DeadlineExceeded: - pass - - if messages: - df_new = callback(messages[0]) - print(f"Updated a dataframe: {df}") - else: - print(f"no messages, trying again at {time.asctime()}") - break +df_new = get_product_prices("fp-20230425", "fp-subscription-NFL-zipcodes", "https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119") # Now we have the new dataframe "df" recording the new price listed in weee's page # Load such data in the existing HDF5 file @@ -274,7 +256,18 @@ with h5py.File(file_path, 'a') as file: df = pd.read_hdf(file, key='df') # Add timestamp to the new dataframe "df_new" - timestamp = datetime.datetime.now() + # timestamp = datetime.now() + ''' + To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that + provides time synchronization with a reliable time server. + Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a + sensible and human-friendly approach to format and convert the timestamps. + ''' + # Fetch the time from an NTP server + ntp_client = ntplib.NTPClient() + response = ntp_client.request('pool.ntp.org') + ntp_time = datetime.fromtimestamp(response.tx_time) + timestamp = arrow.get(ntp_time).to('local') df_new['Timestamp'] = timestamp # Append 'df_new' to the existing df stored in the key @@ -284,4 +277,4 @@ with h5py.File(file_path, 'a') as file: # REPLACE KEY FOR DIFFERENT PRODUCT df_update.to_hdf(file, key='df', mode='a') -print("Data appended and saved successfully.") \ No newline at end of file +print("Data appended and saved successfully.") diff --git a/test_fp2_14.py b/test_fp2_14.py new file mode 100644 index 0000000..5fd00b6 --- /dev/null +++ b/test_fp2_14.py @@ -0,0 +1,63 @@ +from fp2_14 import get_product_prices +import h5py +import pandas as pd +import datetime +import sys +from datetime import datetime +import ntplib +import arrow + +project_id = "fp-20230425" +subscription_name = "fp-subscription-NFL-zipcodes" +url = "https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f" + +df_new = get_product_prices(project_id, subscription_name, url) + +''' +# When using sys to call the 'get_product_prices' function + +if __name__ == '__main__': + # Check if the required command-line arguments are provided + if len(sys.argv) < 4: + print("Usage: python test_fp2_14.py project_id subscription_name url") + sys.exit(1) + + project_id = sys.argv[1] + subscription_name = sys.argv[2] + url = sys.argv[3:] + + get_product_prices(project_id, subscription_name, url) +''' + + +# Now we have the new dataframe "df" recording the new price listed in weee's page +# Load such data in the existing HDF5 file +file_path = 'product1.h5' +with h5py.File(file_path, 'a') as file: + # Load the existing dataframe 'df' from the HDF5 file + # REPLACE KEY TO GET THE EXISTING DATAFRAME FOR EACH DIFFERENT PRODUCT + df = pd.read_hdf(file, key='df') + + # Add timestamp to the new dataframe "df_new" + # timestamp = datetime.now() + ''' + To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that + provides time synchronization with a reliable time server. + Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a + sensible and human-friendly approach to format and convert the timestamps. + ''' + # Fetch the time from an NTP server + ntp_client = ntplib.NTPClient() + response = ntp_client.request('pool.ntp.org') + ntp_time = datetime.fromtimestamp(response.tx_time) + timestamp = arrow.get(ntp_time).to('local') + df_new['Timestamp'] = timestamp + + # Append 'df_new' to the existing df stored in the key + df_update = df.append(df_new, ignore_index=True) + + # Save 'df2' to the HDF5 file with the key 'df' + # REPLACE KEY FOR DIFFERENT PRODUCT + df_update.to_hdf(file, key='df', mode='a') + +print("Data appended and saved successfully.") \ No newline at end of file