import multiprocessing import subprocess import concurrent.futures from google.cloud import pubsub_v1 from google.api_core.exceptions import DeadlineExceeded from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By import json import time import os from concurrent.futures import TimeoutError import pandas as pd from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC import selenium.common.exceptions import re # Define the subset of urls url_list = [ 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119' 'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261' 'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467' 'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01' 'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224' 'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02' 'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01' 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f' 'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353' 'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03' 'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7' ] # Create a pool of worker processes # pool = multiprocessing.Pool(cpu_count) # Define the function to run the scraper def run_scraper(url): # scraping logic df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status']) print(f"Created a dataframe: {df}") options = Options() options.add_argument('--disable-gpu') driver = webdriver.Chrome(options=options) project_id = "fp-20230425" subscription_name = "fp-subscription-NFL-zipcodes" subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path(project_id, subscription_name) # product_id = url.split("/")[-1] driver.delete_all_cookies driver.get(url) def get_element_text(class_name_prefix): xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... xpath_str += "concat(' ', " # Step 2: a leading space, followed by... xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces... xpath_str += "' '), " # Step 4: a trailing space... xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side. try: match = driver.find_element(By.XPATH, xpath_str) return match.get_attribute('textContent') except selenium.common.exceptions.NoSuchElementException as nsee: pass xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... xpath_str += "concat(' ', " # Step 2: a leading space, followed by... xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces... xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores. try: match = driver.find_element(By.XPATH, xpath_str) return match.get_attribute('textContent') except selenium.common.exceptions.NoSuchElementException as nsee: pass return '' # Close the cookies setting window try: cookie_button = WebDriverWait(driver, 2).until( EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")) ) print(f"cookie_button = {cookie_button}") cookie_button.click() except: pass # Clear any modal dialog try: modal_close = WebDriverWait(driver, 2).until( EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")) ) print(f"modal_close = {modal_close}") modal_close.click() except: pass # Define callback function for Pub/Sub messages def callback(message): # Extract zip code from message data zip_code = message.message.data.decode('utf-8') print(f"Received zip code: {zip_code}") try: try: # Clear any modal dialog modal_close = WebDriverWait(driver, 2).until( EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")) ) # print(f"modal_close = {modal_close}") modal_close.click() except: pass try: # Clear any notice notice_close = WebDriverWait(driver, 5).until( EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))) # print(f"notice_close = {notice_close}") notice_close.click() except: pass # Enter zip code in the input field and click the "Check" button zip_button = WebDriverWait(driver, 5).until( EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')) ) # print(f"{zip_button:}") zip_button.click() zip_input = WebDriverWait(driver, 2).until( EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input')) ) # print(f"{zip_input:}") zip_input.click() # print("clearing zip_input") zip_input.send_keys("\b\b\b\b\b") # print("sending zip code") zip_input.send_keys(zip_code) check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']") check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']"))) # print(f"{check_button:}") check_button.click() # time.sleep(2) try: # Clear any notice notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))) # print(f"notice_close = {notice_close}") notice_close.click() except: pass try: # Clear any modal dialog modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))) # print(f"modal_close = {modal_close}") modal_close.click() except: pass # Record sales count, if any sales_count_element = get_element_text('Header_count') count_start_index = sales_count_element.index('sold ') + len('sold ') count_end_index = sales_count_element.index('+') sales_count = sales_count_element[count_start_index:count_end_index] # Record any sold status notice sold_status = get_element_text('UnavailableCartbtn_container') if sold_status == 'Notify me': sold_status = 'Sold Out' # Extract the price of rice crackers for this zip code # Check if there is a minimum order quantity min_quantity: float = 0 total_price: float = 0 unit_price: float = 0 try: minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML') if minimum_quantity_str: regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)" match = re.search(regexp,minimum_quantity_str) min_quantity, total_price = match.groups() except selenium.common.exceptions.NoSuchElementException as nsee: print("No minimum purhcase string found") except Exception as e: print(f"Exception when finding minimum purchase string: {e}") ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']") ld_json_text = ld_json_element.get_attribute('innerHTML') ld_json_start_index = ld_json_text.index('{') ld_json_end_index = ld_json_text.rindex('}') + 1 ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index] ld_json = json.loads(ld_json_json) ld_json_price = ld_json['offers']['price'] # Print the price print(ld_json_price) unit_price_element = driver.find_element(By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText') unit_price = unit_price_element.replace("$", "") print(f"unit_price = {unit_price}" ) subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message # Append the zip code and price to the dataframe #append_to_df(zip_code, price) new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]}) return pd.concat([df, new_row], ignore_index=True) #print(f"Recorded the zip code and price: {zip_code},{price}") #print(f"Updated a dataframe: {df}") except Exception as e: print(f"Caught exception: {e}") print(f"Price not found for zip code {zip_code}") price = 'Price not found' return df # Start receiving messages (sync version) while True: pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path) messages = [] try: pull_response = subscriber.pull(request = pull_request) messages = pull_response.received_messages except DeadlineExceeded: pass if messages: df = callback(messages[0]) print(f"Updated a dataframe: {df}") else: print(f"no messages, trying again at {time.asctime()}") break # subprocess.call(['python', 'fp2_13.py', url]) return run_scraper(url) if __name__ == '__main__': # Determine the number of available CPUs # cpu_count = multiprocessing.cpu_count() with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: future_to_url = {executor.submit(run_scraper, url): url for url in url_list} results = [] for future in concurrent.futures.as_completed(future_to_url): url = future_to_url[future] try: result = future.result() print(f"Scraped data for URL: {url}") results.append(result) except Exception as e: print(f"Failed to scrape data for URL: {url}\nError: {e}") ''' # Run the scraper in parallel for the subset of items results = pool.map(run_scraper, url_list) # Close the pool and wait for the processes to finish pool.close() pool.join() ''' with pd.HDFStore('product_data.h5') as store: for url, result in zip(url_list, results): # Extract the product ID from the URL product_id = url.split("/")[-1].split("?")[0] # Save the dataframe to the HDF5 file with a unique key key = f'df_{product_id}' store[key] = result # result.to_hdf('product_data.h5', key=key, mode='a')