From 0e909fc3f90c6924bbb48af8f81c68ce8437744b Mon Sep 17 00:00:00 2001 From: Karen Zeng Date: Wed, 17 May 2023 15:00:37 -0400 Subject: [PATCH] Modified the sold out status checking --- fp2_13.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/fp2_13.py b/fp2_13.py index 356f587..22ead1f 100644 --- a/fp2_13.py +++ b/fp2_13.py @@ -16,7 +16,7 @@ import re # os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json" # Create an empty dataframe to store the prices -df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price']) +df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status']) print(f"Created a dataframe: {df}") # Configure the Selenium driver options @@ -31,13 +31,61 @@ subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path(project_id, subscription_name) # Load the product page and extract the __NEXT_DATA__ data -url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119' -# product_id = url.split("/")[-1][:5] -product_id = url.split("/")[-1] +url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f' +product_id = url.split("/")[-1][:5] +# product_id = url.split("/")[-1] driver.delete_all_cookies driver.get(url) # time.sleep(5) +def get_element_text(class_name_prefix): + ''' + Return the text in the first element with a given class name. + + Note that XPath doesn't have a function to check if a given word is present + in a list of words, so using contains() will match a class name even if + the class name is a substring of the class name being matched against. + To prevent this, we use spaces as a delimiter at the beginning and end of + the class name, and ensure there's only one space before and after all + class names in the class name list. + + Example: consider two class names with cache-busting suffixes: + * Header_soldStatusNotice__12345 + * Header_soldStatusNoticeBackground__ABCDE + + Matching against Header_soldStatusNotice using a naive implementation with + `contains()` would match both class names, even though only the first one + is the correct match. + ''' + # Construct xpath selector. + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces... + xpath_str += "' '), " # Step 4: a trailing space... + xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side. + try: + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + + # We didn't find a match, try again by doing a prefix match and assume + # the full class name ends with two underscores followed by a cache- + # busting string. + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces... + xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores. + try: + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + + # No match, return empty string. + return '' + + # time.sleep(2) # Close the cookies setting window try: @@ -143,6 +191,15 @@ def callback(message): except: pass + # Record sales count, if any + sales_count_element = get_element_text('Header_count') + count_start_index = sales_count_element.index('sold ') + len('sold ') + count_end_index = sales_count_element.index('+') + sales_count = sales_count_element[count_start_index:count_end_index] + # Record any sold status notice + sold_status = get_element_text('UnavailableCartbtn_container') + if sold_status == 'Notify me': + sold_status = 'Sold Out' # Extract the price of rice crackers for this zip code # Check if there is a minimum order quantity @@ -179,7 +236,7 @@ def callback(message): subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message # Append the zip code and price to the dataframe #append_to_df(zip_code, price) - new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price]}) + new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]}) return pd.concat([df, new_row], ignore_index=True) #print(f"Recorded the zip code and price: {zip_code},{price}") #print(f"Updated a dataframe: {df}") @@ -208,7 +265,7 @@ while True: break # df.to_hdf('product1.h5', key='df',mode='w') -df.to_hdf('product1.h5', key='df',mode='a') +# df.to_hdf('product1.h5', key='df',mode='a') # Open the HDF5 file in a separate window # os.startfile('product1.h5') \ No newline at end of file