python-bootcamp/fp2_15_Sloetjes.py

#!/usr/bin/env python
# coding: utf-8


from google.cloud import pubsub_v1
from google.api_core.exceptions import DeadlineExceeded
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
import json
import time
from concurrent.futures import TimeoutError
import pandas as pd
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import selenium.common.exceptions
import re
from datetime import datetime
import arrow
import subprocess
import concurrent.futures

# This script is used to load the new price and the timestamp and append such data to an existing key in the HDF5 file "product1.h5"

def get_element_text(driver, class_name_prefix):
    '''
    Return the text in the first element with a given class name.

    Note that XPath doesn't have a function to check if a given word is present
    in a list of words, so using contains() will match a class name even if
    the class name is a substring of the class name being matched against.
    To prevent this, we use spaces as a delimiter at the beginning and end of
    the class name, and ensure there's only one space before and after all
    class names in the class name list.

    Example: consider two class names with cache-busting suffixes:
    * Header_soldStatusNotice__12345
    * Header_soldStatusNoticeBackground__ABCDE

    Matching against Header_soldStatusNotice using a naive implementation with
    `contains()` would match both class names, even though only the first one
    is the correct match.
    '''
    # Construct xpath selector.
    xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
    xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
    xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
    xpath_str += "' '), "                     # Step 4: a trailing space...
    xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
    try:
        match = driver.find_element(By.XPATH, xpath_str)
        return match.get_attribute('textContent')
    except selenium.common.exceptions.NoSuchElementException as nsee:
        pass

    # We didn't find a match, try again by doing a prefix match and assume
    # the full class name ends with two underscores followed by a cache-
    # busting string.
    xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
    xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
    xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
    xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
    try:
        match = driver.find_element(By.XPATH, xpath_str)
        return match.get_attribute('textContent')
    except selenium.common.exceptions.NoSuchElementException as nsee:
        pass

    # No match, return empty string.
    return ''


def scrape_product_price(project_id, subscription_name, url):
    # Create an empty dataframe to store the prices
    df_new = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
    print(f"Created a dataframe: {df_new}")

    # Configure the Selenium driver options
    options = Options()
    options.add_argument('--disable-gpu')
    driver = webdriver.Chrome(options=options)

    # Set up Pub/Sub subscription
    subscriber = pubsub_v1.SubscriberClient()
    subscription_path = subscriber.subscription_path(project_id, subscription_name)

    # Load the product page and extract the __NEXT_DATA__ data
    product_id = url.split("/")[-1][:5]
    driver.delete_all_cookies()
    driver.get(url)

    # Close the cookies setting window
    try:
        cookie_button = WebDriverWait(driver, 2).until(
            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
        )
        print(f"cookie_button = {cookie_button}")
        cookie_button.click()
    except:
        pass

    # Clear any modal dialog
    try:
        modal_close = WebDriverWait(driver, 2).until(
            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
        )
        print(f"modal_close = {modal_close}")
        modal_close.click()
    except:
        pass

    def callback(message):
        # Extract zip code from message data
        zip_code = message.message.data.decode('utf-8')
        print(f"Received zip code: {zip_code}")

        try:
            # Clear any modal dialog
            modal_close = WebDriverWait(driver, 2).until(
                EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
            )
            print(f"modal_close = {modal_close}")
            modal_close.click()
        except:
            pass

        try:
            # Clear any notice
            notice_close = WebDriverWait(driver, 2).until(
                EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
            )
            print(f"notice_close = {notice_close}")
            notice_close.click()
        except:
            pass

        # Enter zip code in the input field and click the "Check" button
        zip_button = WebDriverWait(driver, 5).until(
            EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
        )
        zip_button.click()

        zip_input = WebDriverWait(driver, 2).until(
            EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
        )
        zip_input.click()
        zip_input.send_keys("\b\b\b\b\b")
        zip_input.send_keys(zip_code)

        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
        check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//button[text() = 'Confirm']")))
        check_button.click()

        # Record sales count, if any
        sales_count_element = get_element_text(driver, 'Header_count')
        count_start_index = sales_count_element.index('sold ') + len('sold ')
        count_end_index = sales_count_element.index('+')
        sales_count = sales_count_element[count_start_index:count_end_index]

        # Record any sold status notice
        sold_status = get_element_text(driver, 'UnavailableCartbtn_container')
        if sold_status == 'Notify me':
            sold_status = 'Sold Out'

        # Extract the price of the product for this zip code
        min_quantity = 0
        total_price = 0
        unit_price = 0
        try:
            minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]")
            if minimum_quantity_str:
                regexp = r"Minimum purchase: (\d+) \(\$([\d.]+)"
                match = re.search(regexp, minimum_quantity_str.get_attribute('innerHTML'))
                if match:
                    min_quantity, total_price = match.groups()
        except selenium.common.exceptions.NoSuchElementException:
            print("No minimum purchase string found")
        except Exception as e:
            print(f"Exception when finding minimum purchase string: {e}")

        ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
        ld_json_text = ld_json_element.get_attribute('innerHTML')
        ld_json_start_index = ld_json_text.index('{')
        ld_json_end_index = ld_json_text.rindex('}') + 1
        ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
        ld_json = json.loads(ld_json_json)
        ld_json_price = ld_json['offers']['price']
        print(ld_json_price)

        unit_price_element = driver.find_element(By.XPATH, "//div[contains(@class, 'price_price')]")
        unit_price = unit_price_element.get_attribute('innerText').replace("$", "")
        print(f"unit_price = {unit_price}")

        subscriber.acknowledge(ack_ids=[message.ack_id], subscription=subscription_path)  # acknowledge receipt of message

        new_row = pd.DataFrame({
            'Product ID': [product_id],
            'Zip Code': [zip_code],
            'Shown Price': [ld_json_price],
            'Minimum Quantity': [min_quantity],
            'Total Price': [total_price],
            'Unit Price': [unit_price],
            'Sales Count': [sales_count],
            'Sold Status': [sold_status]
        })
        return pd.concat([df_new, new_row], ignore_index=True)

    while True:
        pull_request = pubsub_v1.types.PullRequest(max_messages=1, subscription=subscription_path)
        messages = []
        try:
            pull_response = subscriber.pull(request=pull_request)
            messages = pull_response.received_messages
        except DeadlineExceeded:
            pass

        if messages:
            df_new = callback(messages[0])
            print(f"Updated dataframe: {df_new}")
        else:
            print(f"No messages, trying again at {time.asctime()}")
            break

    return df_new


def scrape_multiple_products(project_id, subscription_name, url_list):
    df_list = []
    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
        futures = []
        for url in url_list:
            future = executor.submit(scrape_product_price, project_id, subscription_name, url)
            futures.append(future)

        for future in concurrent.futures.as_completed(futures):
            df = future.result()
            df_list.append(df)
            print("Scraped data for URL:", url)

    return df_list


if __name__ == '__main__':
    project_id = input("Enter the project ID: ")
    subscription_name = input("Enter the subscription name: ")
    url_list = [
        'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
        'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
        'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
        'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
        'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
        'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
        'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
        'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
        'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
        'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
        'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
    ]

    df_list = scrape_multiple_products(project_id, subscription_name, url_list)

    # Concatenate all dataframes in the list
    df_combined = pd.concat(df_list, ignore_index=True)

    # Save the combined dataframe to HDF5 file
    with pd.HDFStore('product1.h5') as store:
        store.append('key', df_combined, format='table')
        
    # Print the combined dataframe
    print("Combined DataFrame:")
    print(df_combined)
    
    # Save the combined dataframe to a CSV file
    df_combined.to_csv('product1.csv', index=False)

    print("Data scraping and storage completed.")