From 92d7e745e141c94404a193b49c1c3d3bdd452bfc Mon Sep 17 00:00:00 2001 From: cheezedoodle2 Date: Tue, 16 May 2023 15:55:20 -0700 Subject: [PATCH] Factored out element search logic, added scraping out-of-stock messages and sales figures --- fp2_11.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/fp2_11.py b/fp2_11.py index 22d644b..52d4493 100644 --- a/fp2_11.py +++ b/fp2_11.py @@ -19,7 +19,7 @@ from selenium.webdriver.support import expected_conditions as EC # os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json" # Create an empty dataframe to store the prices -df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Displayed Price', 'ld+json Price', 'Minimum Quantity', 'Total Price']) +df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Displayed Price', 'ld+json Price', 'Minimum Quantity', 'Total Price', 'Sales Count', 'Sold Status']) print(f"Created a dataframe: {df}") # Configure the Selenium driver options @@ -94,9 +94,58 @@ def close_modals(): print("Purchase option change dialog close button not found") except Exception as e: print(f"Exception when trying to close purchase option change dialog: {e}") - -# Define callback function for Pub/Sub messages + +def get_element_text(class_name_prefix): + ''' + Return the text in the first element with a given class name. + + Note that XPath doesn't have a function to check if a given word is present + in a list of words, so using contains() will match a class name even if + the class name is a substring of the class name being matched against. + To prevent this, we use spaces as a delimiter at the beginning and end of + the class name, and ensure there's only one space before and after all + class names in the class name list. + + Example: consider two class names with cache-busting suffixes: + * Header_soldStatusNotice__12345 + * Header_soldStatusNoticeBackground__ABCDE + + Matching against Header_soldStatusNotice using a naive implementation with + `contains()` would match both class names, even though only the first one + is the correct match. + ''' + # Construct xpath selector. + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces... + xpath_str += "' '), " # Step 4: a trailing space... + xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side. + try: + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + + # We didn't find a match, try again by doing a prefix match and assume + # the full class name ends with two underscores followed by a cache- + # busting string. + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces... + xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores. + try: + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + + # No match, return empty string. + return '' + def callback(message): + ''' + Callback function for Pub/Sub messages + ''' # Extract zip code from message data zip_code = message.message.data.decode('utf-8') @@ -215,11 +264,16 @@ def callback(message): except: pass + # Record sales count, if any + sales_count = get_element_text('Header_count') + # Record any sold status notice + sold_status = get_element_text('soldStatusNotice') + # Check if there's a minimum order quantity min_quantity: float = 0 total_price: float = 0 try: - minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML') + minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' minQuantity ')]").get_attribute('innerHTML') if minimum_quantity_str: regexp="Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)" match = re.search(regexp, minimum_quantity_str) @@ -231,7 +285,7 @@ def callback(message): print(f"Exception when finding minimum purchase string: {e}") # Get the price as displayed on the web page - displayed_price = driver.find_element(By.XPATH, "//div[contains(@class, 'price_price')]").get_attribute('innerText').replace('$', '') + displayed_price = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' price_price ')]").get_attribute('innerText').replace('$', '') print(f"displayed_price = {displayed_price}") # Extract the price of rice crackers for this zip code @@ -260,7 +314,9 @@ def callback(message): 'Displayed Price': [displayed_price], 'ld+json Price': [ld_json_price], 'Minimum Quantity': [min_quantity], - 'Total Price': [total_price]}) + 'Total Price': [total_price], + 'Sales Count': [sales_count], + 'Sold Status': [sold_status]}) return pd.concat([df, new_row], ignore_index=True) #print(f"Recorded the zip code and price: {zip_code},{price}") #print(f"Updated a dataframe: {df}")