Update 'fp2_15_Sloetjes.py'

NFL Zip Code.py

@@ -8,7 +8,7 @@ from google.cloud import pubsub_v1
 
 # In[2]:
 # Set the project ID and topic name
-project_id = 'svig-384801'
+project_id = 'price-extraction-project'
 topic_name = 'nfl-zipcodes'
 
 
@@ -82,12 +82,12 @@ from selenium.webdriver.support import expected_conditions as EC
 from google.cloud import pubsub_v1
 
 # Set the project ID and subscription name
-project_id = 'svig-384801'
+project_id = 'price-extraction-project'
 topic_name = 'nfl-zipcodes'
-subscription_name = 'nfl-zipcodes-sub'
+subscription_name = 'sloetjes'
 
 # Set the path to your ChromeDriver executable
-chrome_driver_path = "/usr/local/bin/chromedriver"
+chrome_driver_path = "/Users/sloetjeswong/Downloads/chromedriver_mac_arm64/chromedriver"
 
 # Set up a Google Cloud Pub/Sub subscriber
 subscriber = pubsub_v1.SubscriberClient()
@@ -95,7 +95,7 @@ subscription_path = subscriber.subscription_path(project_id, subscription_name)
 
 # Set up a ChromeDriver instance
 chrome_options = webdriver.ChromeOptions()
-#chrome_options.add_argument("--headless")  # Run Chrome in headless mode
+chrome_options.add_argument("--headless")  # Run Chrome in headless mode
 chrome_service = Service(chrome_driver_path)
 chrome_service.start()
 driver = webdriver.Remote(chrome_service.service_url, options=chrome_options)
@@ -111,7 +111,6 @@ def callback(message):
 
     # Wait for the price to load
     price_xpath = "//span[@class='price']"
-    print(EC.presence_of_element_located((By.XPATH, price_xpath)))
     price_element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, price_xpath)))
 
     # Extract the price and print it
@@ -130,5 +129,3 @@ streaming_pull_future.result()
 while True:
     time.sleep(60)
 
-
-# %%

SQLAlchemy.py

@@ -108,7 +108,6 @@ if len(sys.argv) < 2:
     app.run()
 
 
-
 command = sys.argv[1]
 
 if command == "create":

fp2_15_Sloetjes.py

@@ -0,0 +1,277 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import selenium.common.exceptions
+import re
+from datetime import datetime
+import arrow
+import subprocess
+import concurrent.futures
+
+# This script is used to load the new price and the timestamp and append such data to an existing key in the HDF5 file "product1.h5"
+
+def get_element_text(driver, class_name_prefix):
+    '''
+    Return the text in the first element with a given class name.
+
+    Note that XPath doesn't have a function to check if a given word is present
+    in a list of words, so using contains() will match a class name even if
+    the class name is a substring of the class name being matched against.
+    To prevent this, we use spaces as a delimiter at the beginning and end of
+    the class name, and ensure there's only one space before and after all
+    class names in the class name list.
+
+    Example: consider two class names with cache-busting suffixes:
+    * Header_soldStatusNotice__12345
+    * Header_soldStatusNoticeBackground__ABCDE
+
+    Matching against Header_soldStatusNotice using a naive implementation with
+    `contains()` would match both class names, even though only the first one
+    is the correct match.
+    '''
+    # Construct xpath selector.
+    xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+    xpath_str += "' '), "                     # Step 4: a trailing space...
+    xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # We didn't find a match, try again by doing a prefix match and assume
+    # the full class name ends with two underscores followed by a cache-
+    # busting string.
+    xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+    xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # No match, return empty string.
+    return ''
+
+
+def scrape_product_price(project_id, subscription_name, url):
+    # Create an empty dataframe to store the prices
+    df_new = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
+    print(f"Created a dataframe: {df_new}")
+
+    # Configure the Selenium driver options
+    options = Options()
+    options.add_argument('--disable-gpu')
+    driver = webdriver.Chrome(options=options)
+
+    # Set up Pub/Sub subscription
+    subscriber = pubsub_v1.SubscriberClient()
+    subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+    # Load the product page and extract the __NEXT_DATA__ data
+    product_id = url.split("/")[-1][:5]
+    driver.delete_all_cookies()
+    driver.get(url)
+
+    # Close the cookies setting window
+    try:
+        cookie_button = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+        )
+        print(f"cookie_button = {cookie_button}")
+        cookie_button.click()
+    except:
+        pass
+
+    # Clear any modal dialog
+    try:
+        modal_close = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+        )
+        print(f"modal_close = {modal_close}")
+        modal_close.click()
+    except:
+        pass
+
+    def callback(message):
+        # Extract zip code from message data
+        zip_code = message.message.data.decode('utf-8')
+        print(f"Received zip code: {zip_code}")
+
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+            )
+            print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+        try:
+            # Clear any notice
+            notice_close = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
+            )
+            print(f"notice_close = {notice_close}")
+            notice_close.click()
+        except:
+            pass
+
+        # Enter zip code in the input field and click the "Check" button
+        zip_button = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+        )
+        zip_button.click()
+
+        zip_input = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+        )
+        zip_input.click()
+        zip_input.send_keys("\b\b\b\b\b")
+        zip_input.send_keys(zip_code)
+
+        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+        check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//button[text() = 'Confirm']")))
+        check_button.click()
+
+        # Record sales count, if any
+        sales_count_element = get_element_text(driver, 'Header_count')
+        count_start_index = sales_count_element.index('sold ') + len('sold ')
+        count_end_index = sales_count_element.index('+')
+        sales_count = sales_count_element[count_start_index:count_end_index]
+
+        # Record any sold status notice
+        sold_status = get_element_text(driver, 'UnavailableCartbtn_container')
+        if sold_status == 'Notify me':
+            sold_status = 'Sold Out'
+
+        # Extract the price of the product for this zip code
+        min_quantity = 0
+        total_price = 0
+        unit_price = 0
+        try:
+            minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]")
+            if minimum_quantity_str:
+                regexp = r"Minimum purchase: (\d+) \(\$([\d.]+)"
+                match = re.search(regexp, minimum_quantity_str.get_attribute('innerHTML'))
+                if match:
+                    min_quantity, total_price = match.groups()
+        except selenium.common.exceptions.NoSuchElementException:
+            print("No minimum purchase string found")
+        except Exception as e:
+            print(f"Exception when finding minimum purchase string: {e}")
+
+        ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        ld_json_text = ld_json_element.get_attribute('innerHTML')
+        ld_json_start_index = ld_json_text.index('{')
+        ld_json_end_index = ld_json_text.rindex('}') + 1
+        ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
+        ld_json = json.loads(ld_json_json)
+        ld_json_price = ld_json['offers']['price']
+        print(ld_json_price)
+
+        unit_price_element = driver.find_element(By.XPATH, "//div[contains(@class, 'price_price')]")
+        unit_price = unit_price_element.get_attribute('innerText').replace("$", "")
+        print(f"unit_price = {unit_price}")
+
+        subscriber.acknowledge(ack_ids=[message.ack_id], subscription=subscription_path)  # acknowledge receipt of message
+
+        new_row = pd.DataFrame({
+            'Product ID': [product_id],
+            'Zip Code': [zip_code],
+            'Shown Price': [ld_json_price],
+            'Minimum Quantity': [min_quantity],
+            'Total Price': [total_price],
+            'Unit Price': [unit_price],
+            'Sales Count': [sales_count],
+            'Sold Status': [sold_status]
+        })
+        return pd.concat([df_new, new_row], ignore_index=True)
+
+    while True:
+        pull_request = pubsub_v1.types.PullRequest(max_messages=1, subscription=subscription_path)
+        messages = []
+        try:
+            pull_response = subscriber.pull(request=pull_request)
+            messages = pull_response.received_messages
+        except DeadlineExceeded:
+            pass
+
+        if messages:
+            df_new = callback(messages[0])
+            print(f"Updated dataframe: {df_new}")
+        else:
+            print(f"No messages, trying again at {time.asctime()}")
+            break
+
+    return df_new
+
+
+def scrape_multiple_products(project_id, subscription_name, url_list):
+    df_list = []
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        futures = []
+        for url in url_list:
+            future = executor.submit(scrape_product_price, project_id, subscription_name, url)
+            futures.append(future)
+
+        for future in concurrent.futures.as_completed(futures):
+            df = future.result()
+            df_list.append(df)
+            print("Scraped data for URL:", url)
+
+    return df_list
+
+
+if __name__ == '__main__':
+    project_id = input("Enter the project ID: ")
+    subscription_name = input("Enter the subscription name: ")
+    url_list = [
+        'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
+        'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
+        'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
+        'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
+        'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
+        'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
+        'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
+        'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
+        'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
+        'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
+        'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
+    ]
+
+    df_list = scrape_multiple_products(project_id, subscription_name, url_list)
+
+    # Concatenate all dataframes in the list
+    df_combined = pd.concat(df_list, ignore_index=True)
+
+    # Save the combined dataframe to HDF5 file
+    with pd.HDFStore('product1.h5') as store:
+        store.append('key', df_combined, format='table')
+        
+    # Print the combined dataframe
+    print("Combined DataFrame:")
+    print(df_combined)
+    
+    # Save the combined dataframe to a CSV file
+    df_combined.to_csv('product1.csv', index=False)
+
+    print("Data scraping and storage completed.")
+

mykey.pub

@@ -1 +1 @@
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC5JV3Ip/2IMzY0sFQtxTv8BftCNYDi0XwzzIM6JeyNBY/F887JrybhjmyKSfrlmcJGqRxDym1RQNMbEmIWRbi7cSEQPd0CTa2oX2zzuKJvpvxU7NQRocTZXjz3cwo7IkFdZt4tstMQbM6sxLoAOIRMoMpCXjx4++LKDEzgfVLEiVJQElSWXfoJW+nS2lGxFJvNityauErwYMAIwRHQkXL90/n0Qrh8mrgiRPXqtj3FwpZRx97MJA7EDpFeqziXQxdeMmZ5YhSovC+89U7jIvHLdes1HBixXBmC4YPlp8Abpk9jweYSZkdUBtbrebQDp9xDg5CA4Nl0AemHo7vWbHmX sloetjeswong@Sloetjess-Air
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCbUrTPE/8CUByUtG9rBuMP5zeF3x5sZcPmm61bcxBblvHL5Dx1WwIjujhbZ5mhxnmDqxV07MCFZwMzlPjJ7Ow9erZvaXAe5+jS2iTQqu9apfj+U3x2xEZQmoKprXUpWl8G9YDk2wxBqCn//ugkJ5+erIEopNUGPcVhACucpaCT+kld9mHtXBjdUEKUA85HhXiBkuBsv9k/VqL/oXbvHjY8Z6gFP64LcxdNASuf4Qn1wW0w3kXlQRkSoGdSxpGLztXN1v5rvetOzjA8HXixh7TYVWCc+XCa9KsY3QRk7gFyvtIsXz59NMOTgfpCnUQE2Pk9R5wuuA2fXyr+Kg+7NPkd sloetjeswong@Sloetjess-Air