fp2_11.py

@@ -19,7 +19,7 @@ from selenium.webdriver.support import expected_conditions as EC
 # os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
 
 # Create an empty dataframe to store the prices
-df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Displayed Price', 'ld+json Price', 'Minimum Quantity', 'Total Price'])
+df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Displayed Price', 'ld+json Price', 'Minimum Quantity', 'Total Price', 'Sales Count', 'Sold Status'])
 print(f"Created a dataframe: {df}")
 
 # Configure the Selenium driver options
@@ -95,8 +95,57 @@ def close_modals():
     except Exception as e:
         print(f"Exception when trying to close purchase option change dialog: {e}")
 
-# Define callback function for Pub/Sub messages
+def get_element_text(class_name_prefix):
+    '''
+    Return the text in the first element with a given class name.
+
+    Note that XPath doesn't have a function to check if a given word is present
+    in a list of words, so using contains() will match a class name even if
+    the class name is a substring of the class name being matched against.
+    To prevent this, we use spaces as a delimiter at the beginning and end of
+    the class name, and ensure there's only one space before and after all
+    class names in the class name list.
+
+    Example: consider two class names with cache-busting suffixes:
+      * Header_soldStatusNotice__12345
+      * Header_soldStatusNoticeBackground__ABCDE
+
+    Matching against Header_soldStatusNotice using a naive implementation with
+    `contains()` would match both class names, even though only the first one
+    is the correct match. 
+    '''
+    # Construct xpath selector.  
+    xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+    xpath_str += "' '), "                     # Step 4: a trailing space...
+    xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # We didn't find a match, try again by doing a prefix match and assume
+    # the full class name ends with two underscores followed by a cache-
+    # busting string.    
+    xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+    xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # No match, return empty string.
+    return ''
+
 def callback(message):
+    '''
+    Callback function for Pub/Sub messages
+    '''
     # Extract zip code from message data
     zip_code = message.message.data.decode('utf-8')
 
@@ -215,11 +264,16 @@ def callback(message):
         except:
             pass
 
+        # Record sales count, if any
+        sales_count = get_element_text('Header_count')
+        # Record any sold status notice
+        sold_status = get_element_text('soldStatusNotice')
+
         # Check if there's a minimum order quantity
         min_quantity: float = 0
         total_price: float = 0
         try:
-            minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML')
+            minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' minQuantity ')]").get_attribute('innerHTML')
             if minimum_quantity_str:
                 regexp="Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
                 match = re.search(regexp, minimum_quantity_str)
@@ -231,7 +285,7 @@ def callback(message):
             print(f"Exception when finding minimum purchase string: {e}")
 
         # Get the price as displayed on the web page
-        displayed_price = driver.find_element(By.XPATH, "//div[contains(@class, 'price_price')]").get_attribute('innerText').replace('$', '')
+        displayed_price = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' price_price ')]").get_attribute('innerText').replace('$', '')
         print(f"displayed_price = {displayed_price}")
 
         # Extract the price of rice crackers for this zip code     
@@ -260,7 +314,9 @@ def callback(message):
             'Displayed Price': [displayed_price],
             'ld+json Price': [ld_json_price],
             'Minimum Quantity': [min_quantity],
-            'Total Price': [total_price]})
+            'Total Price': [total_price],
+            'Sales Count': [sales_count],
+            'Sold Status': [sold_status]})
         return pd.concat([df, new_row], ignore_index=True)
         #print(f"Recorded the zip code and price: {zip_code},{price}")
         #print(f"Updated a dataframe: {df}")