diff --git a/fp2_15_Sloetjes.py b/fp2_15_Sloetjes.py new file mode 100644 index 0000000..b3617fb --- /dev/null +++ b/fp2_15_Sloetjes.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[11]: + + +from google.cloud import pubsub_v1 +from google.api_core.exceptions import DeadlineExceeded +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +import json +import time +from concurrent.futures import TimeoutError +import pandas as pd +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +import selenium.common.exceptions +import re +from datetime import datetime +import arrow +import subprocess +import concurrent.futures + +# This script is used to load the new price and the timestamp and append such data to an existing key in the HDF5 file "product1.h5" + +def get_element_text(driver, class_name_prefix): + ''' + Return the text in the first element with a given class name. + + Note that XPath doesn't have a function to check if a given word is present + in a list of words, so using contains() will match a class name even if + the class name is a substring of the class name being matched against. + To prevent this, we use spaces as a delimiter at the beginning and end of + the class name, and ensure there's only one space before and after all + class names in the class name list. + + Example: consider two class names with cache-busting suffixes: + * Header_soldStatusNotice__12345 + * Header_soldStatusNoticeBackground__ABCDE + + Matching against Header_soldStatusNotice using a naive implementation with + `contains()` would match both class names, even though only the first one + is the correct match. + ''' + # Construct xpath selector. + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces... + xpath_str += "' '), " # Step 4: a trailing space... + xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side. + try: + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + + # We didn't find a match, try again by doing a prefix match and assume + # the full class name ends with two underscores followed by a cache- + # busting string. + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces... + xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores. + try: + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + + # No match, return empty string. + return '' + + +def scrape_product_price(project_id, subscription_name, url): + # Create an empty dataframe to store the prices + df_new = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status']) + print(f"Created a dataframe: {df_new}") + + # Configure the Selenium driver options + options = Options() + options.add_argument('--disable-gpu') + driver = webdriver.Chrome(options=options) + + # Set up Pub/Sub subscription + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path(project_id, subscription_name) + + # Load the product page and extract the __NEXT_DATA__ data + product_id = url.split("/")[-1][:5] + driver.delete_all_cookies() + driver.get(url) + + # Close the cookies setting window + try: + cookie_button = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")) + ) + print(f"cookie_button = {cookie_button}") + cookie_button.click() + except: + pass + + # Clear any modal dialog + try: + modal_close = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")) + ) + print(f"modal_close = {modal_close}") + modal_close.click() + except: + pass + + def callback(message): + # Extract zip code from message data + zip_code = message.message.data.decode('utf-8') + print(f"Received zip code: {zip_code}") + + try: + # Clear any modal dialog + modal_close = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")) + ) + print(f"modal_close = {modal_close}") + modal_close.click() + except: + pass + + try: + # Clear any notice + notice_close = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")) + ) + print(f"notice_close = {notice_close}") + notice_close.click() + except: + pass + + # Enter zip code in the input field and click the "Check" button + zip_button = WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')) + ) + zip_button.click() + + zip_input = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input')) + ) + zip_input.click() + zip_input.send_keys("\b\b\b\b\b") + zip_input.send_keys(zip_code) + + check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']") + check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//button[text() = 'Confirm']"))) + check_button.click() + + # Record sales count, if any + sales_count_element = get_element_text(driver, 'Header_count') + count_start_index = sales_count_element.index('sold ') + len('sold ') + count_end_index = sales_count_element.index('+') + sales_count = sales_count_element[count_start_index:count_end_index] + + # Record any sold status notice + sold_status = get_element_text(driver, 'UnavailableCartbtn_container') + if sold_status == 'Notify me': + sold_status = 'Sold Out' + + # Extract the price of the product for this zip code + min_quantity = 0 + total_price = 0 + unit_price = 0 + try: + minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]") + if minimum_quantity_str: + regexp = r"Minimum purchase: (\d+) \(\$([\d.]+)" + match = re.search(regexp, minimum_quantity_str.get_attribute('innerHTML')) + if match: + min_quantity, total_price = match.groups() + except selenium.common.exceptions.NoSuchElementException: + print("No minimum purchase string found") + except Exception as e: + print(f"Exception when finding minimum purchase string: {e}") + + ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']") + ld_json_text = ld_json_element.get_attribute('innerHTML') + ld_json_start_index = ld_json_text.index('{') + ld_json_end_index = ld_json_text.rindex('}') + 1 + ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index] + ld_json = json.loads(ld_json_json) + ld_json_price = ld_json['offers']['price'] + print(ld_json_price) + + unit_price_element = driver.find_element(By.XPATH, "//div[contains(@class, 'price_price')]") + unit_price = unit_price_element.get_attribute('innerText').replace("$", "") + print(f"unit_price = {unit_price}") + + subscriber.acknowledge(ack_ids=[message.ack_id], subscription=subscription_path) # acknowledge receipt of message + + new_row = pd.DataFrame({ + 'Product ID': [product_id], + 'Zip Code': [zip_code], + 'Shown Price': [ld_json_price], + 'Minimum Quantity': [min_quantity], + 'Total Price': [total_price], + 'Unit Price': [unit_price], + 'Sales Count': [sales_count], + 'Sold Status': [sold_status] + }) + return pd.concat([df_new, new_row], ignore_index=True) + + while True: + pull_request = pubsub_v1.types.PullRequest(max_messages=1, subscription=subscription_path) + messages = [] + try: + pull_response = subscriber.pull(request=pull_request) + messages = pull_response.received_messages + except DeadlineExceeded: + pass + + if messages: + df_new = callback(messages[0]) + print(f"Updated dataframe: {df_new}") + else: + print(f"No messages, trying again at {time.asctime()}") + break + + return df_new + + +def scrape_multiple_products(project_id, subscription_name, url_list): + df_list = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + futures = [] + for url in url_list: + future = executor.submit(scrape_product_price, project_id, subscription_name, url) + futures.append(future) + + for future in concurrent.futures.as_completed(futures): + df = future.result() + df_list.append(df) + print("Scraped data for URL:", url) + + return df_list + + +if __name__ == '__main__': + project_id = input("Enter the project ID: ") + subscription_name = input("Enter the subscription name: ") + url_list = [ + 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119', + 'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261', + 'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467', + 'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01', + 'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224', + 'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02', + 'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01', + 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f', + 'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353', + 'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03', + 'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7' + ] + + df_list = scrape_multiple_products(project_id, subscription_name, url_list) + + # Concatenate all dataframes in the list + df_combined = pd.concat(df_list, ignore_index=True) + + # Save the combined dataframe to HDF5 file + with pd.HDFStore('product1.h5') as store: + store.append('key', df_combined, format='table') + + # Print the combined dataframe + print("Combined DataFrame:") + print(df_combined) + + # Save the combined dataframe to a CSV file + df_combined.to_csv('product1.csv', index=False) + + print("Data scraping and storage completed.") +