diff --git a/Multiprocessing.py b/Multiprocessing.py new file mode 100644 index 0000000..0a5360e --- /dev/null +++ b/Multiprocessing.py @@ -0,0 +1,276 @@ +import multiprocessing +import subprocess +import concurrent.futures +from google.cloud import pubsub_v1 +from google.api_core.exceptions import DeadlineExceeded +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +import json +import time +import os +from concurrent.futures import TimeoutError +import pandas as pd +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +import selenium.common.exceptions +import re +from fp2_14 import get_product_prices + +# Define the subset of urls +url_list = [ + 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119', + 'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261', + 'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467', + 'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01', + 'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224', + 'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02', + 'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01', + 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f', + 'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353', + 'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03', + 'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7' +] + + +# Create a pool of worker processes +# pool = multiprocessing.Pool(cpu_count) + +# Define the function to run the scraper +def run_scraper(url): + # scraping logic + df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status']) + print(f"Created a dataframe: {df}") + product_id = url.split("/")[-1][:5] + + options = Options() + options.add_argument('--disable-gpu') + driver = webdriver.Chrome(options=options) + + project_id = "fp-20230425" + subscription_name = "fp-subscription-NFL-zipcodes" + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path(project_id, subscription_name) + + # product_id = url.split("/")[-1] + driver.delete_all_cookies + driver.get(url) + + def get_element_text(class_name_prefix): + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces... + xpath_str += "' '), " # Step 4: a trailing space... + xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side. + try: + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces... + xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores. + try: + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + return '' + + # Close the cookies setting window + try: + cookie_button = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")) + ) + print(f"cookie_button = {cookie_button}") + cookie_button.click() + except: + pass + + # Clear any modal dialog + try: + modal_close = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")) + ) + print(f"modal_close = {modal_close}") + modal_close.click() + except: + pass + + # Define callback function for Pub/Sub messages + def callback(message): + # Extract zip code from message data + zip_code = message.message.data.decode('utf-8') + + print(f"Received zip code: {zip_code}") + + try: + try: + # Clear any modal dialog + modal_close = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")) + ) + # print(f"modal_close = {modal_close}") + modal_close.click() + except: + pass + + + try: + + # Clear any notice + notice_close = WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))) + # print(f"notice_close = {notice_close}") + notice_close.click() + except: + pass + + # Enter zip code in the input field and click the "Check" button + zip_button = WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')) + ) + # print(f"{zip_button:}") + zip_button.click() + + zip_input = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input')) + ) + # print(f"{zip_input:}") + zip_input.click() + # print("clearing zip_input") + zip_input.send_keys("\b\b\b\b\b") + # print("sending zip code") + zip_input.send_keys(zip_code) + + check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']") + check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']"))) + # print(f"{check_button:}") + check_button.click() + # time.sleep(2) + + try: + # Clear any notice + notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))) + # print(f"notice_close = {notice_close}") + notice_close.click() + except: + pass + + try: + # Clear any modal dialog + modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))) + # print(f"modal_close = {modal_close}") + modal_close.click() + except: + pass + + # Record sales count, if any + sales_count_element = get_element_text('Header_count') + count_start_index = sales_count_element.index('sold ') + len('sold ') + count_end_index = sales_count_element.index('+') + sales_count = sales_count_element[count_start_index:count_end_index] + # Record any sold status notice + sold_status = get_element_text('UnavailableCartbtn_container') + if sold_status == 'Notify me': + sold_status = 'Sold Out' + + # Extract the price of rice crackers for this zip code + # Check if there is a minimum order quantity + min_quantity: float = 0 + total_price: float = 0 + unit_price: float = 0 + try: + minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML') + if minimum_quantity_str: + regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)" + match = re.search(regexp,minimum_quantity_str) + min_quantity, total_price = match.groups() + except selenium.common.exceptions.NoSuchElementException as nsee: + print("No minimum purhcase string found") + except Exception as e: + print(f"Exception when finding minimum purchase string: {e}") + + ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']") + ld_json_text = ld_json_element.get_attribute('innerHTML') + ld_json_start_index = ld_json_text.index('{') + ld_json_end_index = ld_json_text.rindex('}') + 1 + ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index] + ld_json = json.loads(ld_json_json) + ld_json_price = ld_json['offers']['price'] + # Print the price + print(ld_json_price) + + unit_price_element = driver.find_element(By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText') + unit_price = unit_price_element.replace("$", "") + print(f"unit_price = {unit_price}" ) + + subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message + # Append the zip code and price to the dataframe + #append_to_df(zip_code, price) + new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]}) + return pd.concat([df, new_row], ignore_index=True) + #print(f"Recorded the zip code and price: {zip_code},{price}") + #print(f"Updated a dataframe: {df}") + + except Exception as e: + print(f"Caught exception: {e}") + print(f"Price not found for zip code {zip_code}") + price = 'Price not found' + return df + + # Start receiving messages (sync version) + while True: + pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path) + messages = [] + try: + pull_response = subscriber.pull(request = pull_request) + messages = pull_response.received_messages + except DeadlineExceeded: + pass + + if messages: + df = callback(messages[0]) + print(f"Updated a dataframe: {df}") + else: + print(f"no messages, trying again at {time.asctime()}") + break + # subprocess.call(['python', 'fp2_13.py', url]) + return df + +if __name__ == '__main__': + # Determine the number of available CPUs + # cpu_count = multiprocessing.cpu_count() + + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + future_to_url = {executor.submit(run_scraper, url): url for url in url_list} + results = [] + for future in concurrent.futures.as_completed(future_to_url): + url = future_to_url[future] + try: + result = future.result() + print(f"Scraped data for URL: {url}") + results.append(result) + except Exception as e: + print(f"Failed to scrape data for URL: {url}\nError: {e}") + +''' +# Run the scraper in parallel for the subset of items +results = pool.map(run_scraper, url_list) + +# Close the pool and wait for the processes to finish +pool.close() +pool.join() +''' + +with pd.HDFStore('product_data.h5') as store: + for url, result in zip(url_list, results): + # Extract the product ID from the URL + product_id = url.split("/")[-1][:5] + + # Save the dataframe to the HDF5 file with a unique key + key = f'df_{product_id}' + store[key] = result + # result.to_hdf('product_data.h5', key=key, mode='a') \ No newline at end of file diff --git a/NFL Zip Code.py b/NFL Zip Code.py new file mode 100644 index 0000000..d1bcab0 --- /dev/null +++ b/NFL Zip Code.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: +import os +from google.cloud import pubsub_v1 + + +# In[2]: +# Set the project ID and topic name +project_id = 'svig-384801' +topic_name = 'nfl-zipcodes' + + +# In[3]: +# Create a publisher client +publisher = pubsub_v1.PublisherClient() + + +# In[4]: +# Get the topic path +topic_path = publisher.topic_path(project_id, topic_name) + + +# In[5]: +# Define the zip codes for all NFL cities +nfl_zipcodes = { + 'Arizona Cardinals': '85305', + 'Atlanta Falcons': '30363', + 'Baltimore Ravens': '21230', + 'Buffalo Bills': '14127', + 'Carolina Panthers': '28202', + 'Chicago Bears': '60605', + 'Cincinnati Bengals': '45202', + 'Cleveland Browns': '44115', + 'Dallas Cowboys': '76011', + 'Denver Broncos': '80204', + 'Detroit Lions': '48226', + 'Green Bay Packers': '54304', + 'Houston Texans': '77054', + 'Indianapolis Colts': '46225', + 'Jacksonville Jaguars': '32202', + 'Kansas City Chiefs': '64129', + 'Las Vegas Raiders': '89118', + 'Los Angeles Chargers': '92626', + 'Los Angeles Rams': '90305', + 'Miami Dolphins': '33056', + 'Minnesota Vikings': '55415', + 'New England Patriots': '02035', + 'New Orleans Saints': '70112', + 'New York Giants': '07073', + 'New York Jets': '07932', + 'Philadelphia Eagles': '19145', + 'Pittsburgh Steelers': '15212', + 'San Francisco 49ers': '95054', + 'Seattle Seahawks': '98056', + 'Tampa Bay Buccaneers': '33607', + 'Tennessee Titans': '37213', + 'Washington Football Team': '20785' +} + + +# In[6]: +# Publish each zip code to the topic +for team, zipcode in nfl_zipcodes.items(): + data = f'{zipcode}'.encode('utf-8') + publisher_future = publisher.publish(topic_path, data=data) + message_id = publisher_future.result() + print(f"message_id = {message_id}") + +print('All zip codes published.') + + +# In[ ]: +import os +import time +from selenium import webdriver +from selenium.webdriver.chrome.service import Service +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from google.cloud import pubsub_v1 + +# Set the project ID and subscription name +project_id = 'svig-384801' +topic_name = 'nfl-zipcodes' +subscription_name = 'nfl-zipcodes-sub' + +# Set the path to your ChromeDriver executable +chrome_driver_path = "/usr/local/bin/chromedriver" + +# Set up a Google Cloud Pub/Sub subscriber +subscriber = pubsub_v1.SubscriberClient() +subscription_path = subscriber.subscription_path(project_id, subscription_name) + +# Set up a ChromeDriver instance +chrome_options = webdriver.ChromeOptions() +#chrome_options.add_argument("--headless") # Run Chrome in headless mode +chrome_service = Service(chrome_driver_path) +chrome_service.start() +driver = webdriver.Remote(chrome_service.service_url, options=chrome_options) + +def callback(message): + # Get the zip code from the message + zip_code = message.data.decode("utf-8") + print(f"Received message with zip code: {zip_code}") + + # Open the product page for the rice crackers for the given zip code + product_url = f"https://www.sayweee.com/zh/product/Want-Want-Senbei-Rice-Crackers/17119?trace_id=89f7c636-05b8-4f52-abdd-74de3a83f45d&zip={zip_code}" + driver.get(product_url) + + # Wait for the price to load + price_xpath = "//span[@class='price']" + print(EC.presence_of_element_located((By.XPATH, price_xpath))) + price_element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, price_xpath))) + + # Extract the price and print it + price = price_element.text + print(f"Price for zip code {zip_code}: {price}") + + # Acknowledge the message + message.ack() + +# Subscribe to the topic +streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback) +print(f"Listening for messages on {subscription_path}...") +streaming_pull_future.result() + +# Keep the script running +while True: + time.sleep(60) + + +# %% diff --git a/NFL_Cities.csv b/NFL_Cities.csv new file mode 100644 index 0000000..8e6cb82 --- /dev/null +++ b/NFL_Cities.csv @@ -0,0 +1,33 @@ +ID,Team Name,Arena Name,Abbreviation,Conference,Division,Arena Location,ZIP Code +1,Arizona Cardinals,State Farm Stadium,ARI,NFC,West,"Glendale, Arizona",85305 +2,Atlanta Falcons,Mercedes-Benz Stadium,ATL,NFC,South,"Atlanta, Georgia",30313 +3,Baltimore Ravens,M&T Bank Stadium,BAL,AFC,North,"Baltimore, Maryland",21230 +4,Buffalo Bills,Bills Stadium,BUF,AFC,East,"Orchard Park, New York",14127 +5,Carolina Panthers,Bank of America Stadium,CAR,NFC,South,"Charlotte, North Carolina",28202 +6,Chicago Bears,Soldier Field,CHI,NFC,North,"Chicago, Illinois",60605 +7,Cincinnati Bengals,Paul Brown Stadium,CIN,AFC,North,"Cincinnati, Ohio",45202 +8,Cleveland Browns,First Energy Stadium,CLE,AFC,North,"Cleveland, Ohio",44114 +9,Dallas Cowboys,At&T Stadium,DAL,NFC,East,"Arlington, Texas",76011 +10,Denver Broncos,Empower Field at Mile High,DEN,AFC,West,"Denver, Colorado",80204 +11,Detroit Lions,Ford Field,DET,NFC,North,"Detroit, Michigan",48226 +12,Green Bay Packers,Lambeau Field,GB,NFC,North,"Green Bay, Wisconsin",54304 +13,Houston Texans,NRG Stadium,HOU,AFC,South,"Houston, Texas",77054 +14,Indianapolis Colts,Lucas Oil Stadium,IND,AFC,South,"Indianapolis, Indiana",46225 +15,Jacksonville Jaguars,TIAA Bank Field,JAX,AFC,South,"Jacksonville, Florida",32202 +16,Kansas City Chiefs,Arrowhead Stadium,KC,AFC,West,"Kansas City, Missouri",64129 +17,Las Vegas Raiders,Allegiant Stadium,LV,AFC,West,"Paradise, Nevada",89118 +18,Los Angeles Chargers,SoFi Stadium,LAC,AFC,West,"Inglewood, California",90301 +19,Los Angeles Rams,SoFi Stadium,LAR,NFC,West,"Inglewood, California",90301 +20,Miami Dolphins,Hard Rock Stadium,MIA,AFC,East,"Miami Gardens, Florida",33056 +21,Minnesota Vikings,U.S. Bank Stadium,MIN,NFC,North,"Minneapolis, Minnesota",55415 +22,New England Patriots,Gillette Stadium,NE,AFC,East,"Foxborough, Massachusetts",02035 +23,New Orleans Saints,Mercedes-Benz Superdome,NO,NFC,South,"New Orleans, Louisiana",70112 +24,New York Giants,MetLife Stadium,NYG,NFC,East,"East Rutherford, New Jersey",07073 +25,New York Jets,MetLife Stadium,NYJ,AFC,East,"East Rutherford, New Jersey",07073 +26,Philadelphia Eagles,Lincoln Financial Field,PHI,NFC,East,"Philadelphia, Pennsylvania",19148 +27,Pittsburgh Steelers,Heinz Field,PIT,AFC,North,"Pittsburgh, Pennsylvania",15212 +28,San Francisco 49ers,Levi's Stadium,SF,NFC,West,"Santa Clara, California",95054 +29,Seattle Seahawks,CenturyLink Field,SEA,NFC,West,"Seattle, Washington",98134 +30,Tampa Bay Buccaneers,Raymond James Stadium,TB,NFC,South,"Tampa, Florida",33607 +31,Tennessee Titans,Nissan Stadium,TEN,AFC,South,"Nashville, Tennessee",37213 +32,Washington Redskins,FedExField,WAS,NFC,East,"Landover, Maryland",20785 diff --git a/README.md b/README.md index e69de29..eb4d684 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,20 @@ +This branch contains a prototype for a retail price scraper for getweee.com. +To run it, install: + +* Python 3.9+, +* chrome, +* chromedriver, +* and the python depedencies: + +``` +pip install -r /path/to/requirements.txt +``` + +Once those are installed, install the GCP SDK: + +https://cloud.google.com/sdk/docs/install + +You will also have to log into GCP *twice*, once for the gcloud CLI tools, +and again for the GCP SDK libraries: + +https://googleapis.dev/python/google-api-core/latest/auth.html diff --git a/SQLAlchemy.py b/SQLAlchemy.py new file mode 100644 index 0000000..f4ecdf7 --- /dev/null +++ b/SQLAlchemy.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[ ]: + + +from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime, ForeignKey +from sqlalchemy.orm import sessionmaker, relationship, declarative_base +from flask import Flask, render_template + +# In[ ]: + + +# Create a base class for declarative models +Base = declarative_base() + +class Product(Base): + __tablename__ = 'product' + id = Column(Integer, primary_key=True) + name = Column(String) + SKU_PLU_UPC_number = Column(String) + price_histories = relationship("PriceHistory", back_populates="product") + +class Retailer(Base): + __tablename__ = 'retailer' + id = Column(Integer, primary_key=True) + name = Column(String) + products = relationship("Product", secondary="retailer_product") + +class Price(Base): + __tablename__ = 'price' + id = Column(Integer, primary_key=True) + MOQ = Column(Integer) + price = Column(Float) + +class ScraperJob(Base): + __tablename__ = 'scraper_job' + id = Column(Integer, primary_key=True) + start_time = Column(DateTime) + end_time = Column(DateTime) + parameters = Column(String) + statistics = Column(String) + +class PriceHistory(Base): + __tablename__ = 'price_history' + id = Column(Integer, primary_key=True) + location = Column(String) + zip_code = Column(Integer) + city_name = Column(String) + price_at_time = Column(DateTime) + price = Column(Float) + scraper_job_id = Column(Integer, ForeignKey('scraper_job.id')) + product_id = Column(Integer, ForeignKey('product.id')) + product = relationship("Product", back_populates="price_histories") + scraper_job = relationship("ScraperJob") + +class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + username = Column(String) + credentials = Column(String) + creation_date = Column(DateTime) + modification_date = Column(DateTime) + +class RetailerProduct(Base): + __tablename__ = 'retailer_product' + retailer_id = Column(Integer, ForeignKey('retailer.id'), primary_key=True) + product_id = Column(Integer, ForeignKey('product.id'), primary_key=True) + + +# In[ ]: + + +# Create the database engine +engine = create_engine('sqlite:///user_database.db') +Base.metadata.create_all(engine) + + +# In[ ]: + + +# Create a session factory +Session = sessionmaker(bind=engine) +session = Session() + + +# In[ ]: + + +import sys + +# Create a command-line tool that parses sys.argv to allow a user to create/edit/delete user objects. + +if len(sys.argv) < 2: + app = Flask(__name__) + + # Create a database connection + engine = create_engine('sqlite:///user_database.db') + Session = sessionmaker(bind=engine) + session = Session() + + @app.route('/') + def home(): + # Get all user objects + users = session.query(User).all() + return render_template('index.html', users=users) + + app.run() + + + +command = sys.argv[1] + +if command == "create": + if len(sys.argv) < 4: + print("Usage: python script.py create [username] [credentials]") + sys.exit(1) + + username = sys.argv[2] + credentials = sys.argv[3] + + # Create a new user object + new_user = User(username=username, credentials=credentials) + session.add(new_user) + session.commit() + print("User created successfully!") + +elif command == "edit": + if len(sys.argv) < 4: + print("Usage: python script.py edit [username] [new_credentials]") + sys.exit(1) + + username = sys.argv[2] + new_credentials = sys.argv[3] + + # Find the user object + user = session.query(User).filter_by(username=username).first() + + if user: + # Update user credentials + user.credentials = new_credentials + session.commit() + print("User edited successfully!") + else: + print("User not found!") + +elif command == "delete": + if len(sys.argv) < 3: + print("Usage: python script.py delete [username]") + sys.exit(1) + + username = sys.argv[2] + + # Find the user object + user = session.query(User).filter_by(username=username).first() + + if user: + # Delete the user object + session.delete(user) + session.commit() + print("User deleted successfully!") + else: + print("User not found!") + +else: + print("Usage: python script.py [create|edit|delete] [username] [credentials]") diff --git a/fp2_10.py b/fp2_10.py new file mode 100644 index 0000000..3a19e83 --- /dev/null +++ b/fp2_10.py @@ -0,0 +1,67 @@ +from google.cloud import pubsub_v1 +from google.api_core.exceptions import DeadlineExceeded +import csv +import os +import asyncio + +# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json" +project_id = "fp-20230425" +# project_id = "svig-384801" + +''' +topic_name = "fp-topic-NFL-zipcodes" +subscription_name = "fp-subscription-NFL-zipcodes" +subscriber = pubsub_v1.SubscriberClient() +subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}" +subscriber.delete_subscription(request={"subscription": subscription_path}) +print("Subscription deleted: {}".format(subscription_path)) + +# Delete the topic +publisher = pubsub_v1.PublisherClient() +topic_path = publisher.topic_path(project_id, topic_name) +publisher.delete_topic(request={"topic": topic_path}) +print(f"Topic '{topic_name}' deleted.") + + + +''' +# pubsub topic +publisher = pubsub_v1.PublisherClient() +topic_name = "fp-topic-NFL-zipcodes" +topic_path = publisher.topic_path(project_id, topic_name) + +# Create a Pub/Sub subscription +subscriber = pubsub_v1.SubscriberClient() +''' +subscription_name = "fp-subscription-zipcodes" +subscription_path = subscriber.subscription_path(project_id, subscription_name) +subscription = subscriber.create_subscription( + name=subscription_path, topic=topic_path +) +''' +# Publish a message to the topic +# Read the data from the csv file +# file_path = "C:/Users/Karen/Desktop/zipcodes.csv" +file_path = "zipcodes.csv" + +with open(file_path, "r") as csv_file: + reader = csv.reader(csv_file) + next(reader) # skip header row + for row in reader: + # Format the data as a string and publish it to the topic + data = ",".join(row).encode("utf-8") + try: + future = publisher.publish(topic_path, data) + print(f"Published message ID {future.result()}") + except DeadlineExceeded as de: + print(f"Caught DeadlineExceeded exception: {de}") + +''' +async def receive_messages(): + response = subscriber.pull(subscription=subscription_path, max_messages=1) + for received_message in response.received_messages: + print("Received message: {}".format(received_message.message.data)) + + +asyncio.run(receive_messages()) +''' diff --git a/fp2_11.py b/fp2_11.py new file mode 100644 index 0000000..52d4493 --- /dev/null +++ b/fp2_11.py @@ -0,0 +1,372 @@ +from google.cloud import pubsub_v1 +from google.api_core.exceptions import DeadlineExceeded +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.support.wait import WebDriverWait +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys +import selenium.common.exceptions +import json +import time +import os +import re +import sys +from concurrent.futures import TimeoutError +import pandas as pd +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC + +# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json" + +# Create an empty dataframe to store the prices +df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Displayed Price', 'ld+json Price', 'Minimum Quantity', 'Total Price', 'Sales Count', 'Sold Status']) +print(f"Created a dataframe: {df}") + +# Configure the Selenium driver options +options = Options() +options.add_argument('--disable-gpu') +driver = webdriver.Chrome(options=options) + +# Set up Pub/Sub subscription +project_id = "fp-20230425" +subscription_name = "fp-subscription-NFL-zipcodes" +subscriber = pubsub_v1.SubscriberClient() +subscription_path = subscriber.subscription_path(project_id, subscription_name) + +# Load the product page and extract the __NEXT_DATA__ data +url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119' +product_id = url.split("/")[-1] +driver.delete_all_cookies +driver.get(url) +# time.sleep(5) + +# time.sleep(2) +# Close the cookies setting window +try: + # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button") + cookie_button = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")) + ) + print(f"cookie_button = {cookie_button}") + cookie_button.click() +except: + pass + +# time.sleep(1) +# Clear any modal dialog +try: + # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]") + modal_close = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")) + ) + print(f"modal_close = {modal_close}") + modal_close.click() + #time.sleep(2) +except: + pass + + +def close_modals(): + try: + # time.sleep(2) + # Clear any modal dialog + modal_close_buttons = driver.find_elements(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]") + print(f"modal_close = {modal_close}") + for button in modal_close_buttons: + try: + button.click() + except Exception as e: + print(f"Exception when clicking on close button for element {button}: {e}") + + # time.sleep(2) + except Exception as e: + print(f"Exception when trying to close discount modal dialog: {e}") + + + try: + # time.sleep(1) + # Clear any notice + notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button") + print(f"notice_close = {notice_close}") + notice_close.click() + # time.sleep(1) + except selenium.common.exceptions.NoSuchElementException as nsee: + print("Purchase option change dialog close button not found") + except Exception as e: + print(f"Exception when trying to close purchase option change dialog: {e}") + +def get_element_text(class_name_prefix): + ''' + Return the text in the first element with a given class name. + + Note that XPath doesn't have a function to check if a given word is present + in a list of words, so using contains() will match a class name even if + the class name is a substring of the class name being matched against. + To prevent this, we use spaces as a delimiter at the beginning and end of + the class name, and ensure there's only one space before and after all + class names in the class name list. + + Example: consider two class names with cache-busting suffixes: + * Header_soldStatusNotice__12345 + * Header_soldStatusNoticeBackground__ABCDE + + Matching against Header_soldStatusNotice using a naive implementation with + `contains()` would match both class names, even though only the first one + is the correct match. + ''' + # Construct xpath selector. + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces... + xpath_str += "' '), " # Step 4: a trailing space... + xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side. + try: + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + + # We didn't find a match, try again by doing a prefix match and assume + # the full class name ends with two underscores followed by a cache- + # busting string. + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces... + xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores. + try: + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + + # No match, return empty string. + return '' + +def callback(message): + ''' + Callback function for Pub/Sub messages + ''' + # Extract zip code from message data + zip_code = message.message.data.decode('utf-8') + + print(f"Received zip code: {zip_code}") + + while True: + try: + # Enter zip code in the input field and click the "Check" button + WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')) + zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]') + # print(f"{zip_button:}") + zip_button.click() + # time.sleep(1) + + WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, '//div/label/following-sibling::input')) + zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input') + # print(f"{zip_input:}") + zip_input.click() + # print("clearing zip_input") + zip_input.send_keys("\b\b\b\b\b\b\b\b\b\b") + for i in range(1,10): + zip_input.send_keys(Keys.DELETE) + # print("sending zip code") + zip_input.send_keys(zip_code) + + WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, "//button[text() = 'Confirm']")) + check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']") + print(f"{check_button:}") + check_button.click() + # time.sleep(2) + break + except Exception as e: + print(f"Exception when changing zip code: {e}, trying to close modal dialogs") + close_modals() + + # try: + # time.sleep(2) + # # Clear any notice + # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button") + # print(f"notice_close = {notice_close}") + # notice_close.click() + # time.sleep(2) + # except: + # pass + + try: + try: + # time.sleep(2) + # Clear any modal dialog + # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]") + modal_close = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")) + ) + print(f"modal_close = {modal_close}") + modal_close.click() + # time.sleep(2) + except: + pass + + + try: + # time.sleep(1) + # Clear any notice + # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button") + notice_close = WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")) + ) + print(f"notice_close = {notice_close}") + notice_close.click() + # time.sleep(1) + except: + pass + + # Enter zip code in the input field and click the "Check" button + # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]') + zip_button = WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')) + ) + # print(f"{zip_button:}") + zip_button.click() + # time.sleep(1) + + # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input') + zip_input = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input')) + ) + # print(f"{zip_input:}") + zip_input.click() + # print("clearing zip_input") + zip_input.send_keys("\b\b\b\b\b") + # print("sending zip code") + zip_input.send_keys(zip_code) + + check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']") + check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']"))) + # print(f"{check_button:}") + check_button.click() + # time.sleep(2) + + try: + # time.sleep(2) + # Clear any notice + # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button") + notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))) + # print(f"notice_close = {notice_close}") + notice_close.click() + #time.sleep(2) + except: + pass + + try: + # Clear any modal dialog + modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))) + # print(f"modal_close = {modal_close}") + modal_close.click() + except: + pass + + # Record sales count, if any + sales_count = get_element_text('Header_count') + # Record any sold status notice + sold_status = get_element_text('soldStatusNotice') + + # Check if there's a minimum order quantity + min_quantity: float = 0 + total_price: float = 0 + try: + minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' minQuantity ')]").get_attribute('innerHTML') + if minimum_quantity_str: + regexp="Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)" + match = re.search(regexp, minimum_quantity_str) + min_quantity, total_price = match.groups() + + except selenium.common.exceptions.NoSuchElementException as nsee: + print("No minimum purchase string found") + except Exception as e: + print(f"Exception when finding minimum purchase string: {e}") + + # Get the price as displayed on the web page + displayed_price = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' price_price ')]").get_attribute('innerText').replace('$', '') + print(f"displayed_price = {displayed_price}") + + # Extract the price of rice crackers for this zip code + ''' + price_element = driver.find_element(By.XPATH, '/html/body/div[2]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div') + price = price_element.text.strip()[1:] + print(price) + ''' + + # Get the price from the ld+json element + ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']") + ld_json_text = ld_json_element.get_attribute('innerHTML') + ld_json_start_index = ld_json_text.index('{') + ld_json_end_index = ld_json_text.rindex('}') + 1 + ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index] + ld_json = json.loads(ld_json_json) + ld_json_price = ld_json['offers']['price'] + # Print the price + print(f"ld_json_price = {ld_json_price}") + + subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message + # Append the zip code and price to the dataframe + #append_to_df(zip_code, price) + new_row = pd.DataFrame({'Product ID': [product_id], + 'Zip Code': [zip_code], + 'Displayed Price': [displayed_price], + 'ld+json Price': [ld_json_price], + 'Minimum Quantity': [min_quantity], + 'Total Price': [total_price], + 'Sales Count': [sales_count], + 'Sold Status': [sold_status]}) + return pd.concat([df, new_row], ignore_index=True) + #print(f"Recorded the zip code and price: {zip_code},{price}") + #print(f"Updated a dataframe: {df}") + + except Exception as e: + print(f"Caught exception {type(e)}: {e}") + print(f"Price not found for zip code {zip_code}") + price = 'Price not found' + return df + +# Start receiving messages (sync version) +while True: + pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path) + messages = [] + try: + pull_response = subscriber.pull(request = pull_request) + messages = pull_response.received_messages + except DeadlineExceeded: + pass + + if messages: + df = callback(messages[0]) + print(f"Updated a dataframe: {df}") + df.to_hdf('product1.h5', key='df',mode='w') + else: + print(f"No messages in subscription {subscription_path} at {time.asctime()}, trying again...") + continue + +# Open the HDF5 file in a separate window +if sys.platform.startswith("win32"): + os.startfile('product1.h5') + +# Start receiving messages (async version) +#flow_control = pubsub_v1.types.FlowControl(max_messages = 1) +#streaming_pull_future = subscriber.subscribe(subscription_path, +# callback=callback, +# flow_control=flow_control) + +# print(f'Started receiving messages from subscription {subscription_path}...') + +# Wrap subscriber in a 'with' block to automatically call close() when done. +# with subscriber: +# try: + # When `timeout` is not set, result() will block indefinitely, + # unless an exception is encountered first. +# streaming_pull_future.result(timeout=timeout) +# except TimeoutError: +# streaming_pull_future.cancel() # Trigger the shutdown. +# streaming_pull_future.result() # Block until the shutdown is complete. + +# Keep the script running indefinitely to continue receiving messages +#while True: +# time.sleep(10) diff --git a/fp2_12.py b/fp2_12.py new file mode 100644 index 0000000..fdf9b0f --- /dev/null +++ b/fp2_12.py @@ -0,0 +1,199 @@ +from google.cloud import pubsub_v1 +from google.api_core.exceptions import DeadlineExceeded +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +import json +import time +from concurrent.futures import TimeoutError +import pandas as pd +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC + +# Create an empty dataframe to store the prices +df = pd.DataFrame(columns=['Zip Code', 'Price']) +print(f"Created a dataframe: {df}") + +# Configure the Selenium driver options +options = Options() +options.add_argument('--disable-gpu') +driver = webdriver.Chrome(options=options) + +# Set up Pub/Sub subscription +project_id = "fp-20230425" +subscription_name = "fp-subscription-NFL-zipcodes" +subscriber = pubsub_v1.SubscriberClient() +subscription_path = subscriber.subscription_path(project_id, subscription_name) + +# Load the product page and extract the __NEXT_DATA__ data +url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119' +driver.delete_all_cookies +driver.get(url) +# time.sleep(5) + +# time.sleep(2) +# Close the cookies setting window +try: + # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button") + cookie_button = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")) + ) + print(f"cookie_button = {cookie_button}") + cookie_button.click() +except: + pass + +# time.sleep(1) +# Clear any modal dialog +try: + # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]") + modal_close = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")) + ) + print(f"modal_close = {modal_close}") + modal_close.click() + #time.sleep(2) +except: + pass + +# Define callback function for Pub/Sub messages +def callback(message): + # Extract zip code from message data + zip_code = message.message.data.decode('utf-8') + + print(f"Received zip code: {zip_code}") + + try: + try: + # time.sleep(2) + # Clear any modal dialog + # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]") + modal_close = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")) + ) + # print(f"modal_close = {modal_close}") + modal_close.click() + # time.sleep(2) + except: + pass + + + try: + # time.sleep(1) + # Clear any notice + # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button") + notice_close = WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")) + ) + # print(f"notice_close = {notice_close}") + notice_close.click() + # time.sleep(1) + except: + pass + + # Enter zip code in the input field and click the "Check" button + # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]') + zip_button = WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')) + ) + # print(f"{zip_button:}") + zip_button.click() + # time.sleep(1) + + # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input') + zip_input = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input')) + ) + # print(f"{zip_input:}") + zip_input.click() + # print("clearing zip_input") + zip_input.send_keys("\b\b\b\b\b") + # print("sending zip code") + zip_input.send_keys(zip_code) + + check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']") + check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']"))) + # print(f"{check_button:}") + check_button.click() + # time.sleep(2) + + try: + # time.sleep(2) + # Clear any notice + # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button") + notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))) + # print(f"notice_close = {notice_close}") + notice_close.click() + #time.sleep(2) + except: + pass + + + # Extract the price of rice crackers for this zip code + + # next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']") + next_data_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, '//*[@id="__next"]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div'))) + next_data_text = next_data_element.get_attribute('nodeValue') + price = next_data_text.text + ''' + next_data_text = next_data_element.get_attribute('innerHTML') + next_data_start_index = next_data_text.index('{') + next_data_end_index = next_data_text.rindex('}') + 1 + next_data_json = next_data_text[next_data_start_index:next_data_end_index] + next_data = json.loads(next_data_json) + price = next_data['price'] + ''' + # Print the price + print(price) + + ''' + try: + min_quantity_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, '//script[@id="__NEXT_DATA__"]'))) + min_quantity_text = min_quantity_element.get_attribute('innerHTML') + min_quantity_start_index = min_quantity_text.index('{') + min_quantity_end_index = min_quantity_text.rindex('}') + 1 + min_quantity_json = min_quantity_text[min_quantity_start_index:min_quantity_end_index] + min_quantity = json.loads(min_quantity_json) + price = min_quantity['product']['price'] + except: + next_data_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//script[@type='application/ld+json']"))) + next_data_text = next_data_element.get_attribute('innerHTML') + next_data_start_index = next_data_text.index('{') + next_data_end_index = next_data_text.rindex('}') + 1 + next_data_json = next_data_text[next_data_start_index:next_data_end_index] + next_data = json.loads(next_data_json) + price = next_data['offers']['price'] + finally: + # Print the price + print(price) + ''' + + subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message + # Append the zip code and price to the dataframe + #append_to_df(zip_code, price) + new_row = pd.DataFrame({'Zip Code': [zip_code], 'Price': [price]}) + return pd.concat([df, new_row], ignore_index=True) + #print(f"Recorded the zip code and price: {zip_code},{price}") + #print(f"Updated a dataframe: {df}") + + except Exception as e: + print(f"Caught exception: {e}") + print(f"Price not found for zip code {zip_code}") + price = 'Price not found' + return df + +# Start receiving messages (sync version) +while True: + pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path) + messages = [] + try: + pull_response = subscriber.pull(request = pull_request) + messages = pull_response.received_messages + except DeadlineExceeded: + pass + + if messages: + df = callback(messages[0]) + print(f"Updated a dataframe: {df}") + else: + print(f"no messages, trying again at {time.asctime()}") \ No newline at end of file diff --git a/fp2_13.py b/fp2_13.py new file mode 100644 index 0000000..22ead1f --- /dev/null +++ b/fp2_13.py @@ -0,0 +1,271 @@ +from google.cloud import pubsub_v1 +from google.api_core.exceptions import DeadlineExceeded +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +import json +import time +import os +from concurrent.futures import TimeoutError +import pandas as pd +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +import selenium.common.exceptions +import re + +# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json" + +# Create an empty dataframe to store the prices +df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status']) +print(f"Created a dataframe: {df}") + +# Configure the Selenium driver options +options = Options() +options.add_argument('--disable-gpu') +driver = webdriver.Chrome(options=options) + +# Set up Pub/Sub subscription +project_id = "fp-20230425" +subscription_name = "fp-subscription-NFL-zipcodes" +subscriber = pubsub_v1.SubscriberClient() +subscription_path = subscriber.subscription_path(project_id, subscription_name) + +# Load the product page and extract the __NEXT_DATA__ data +url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f' +product_id = url.split("/")[-1][:5] +# product_id = url.split("/")[-1] +driver.delete_all_cookies +driver.get(url) +# time.sleep(5) + +def get_element_text(class_name_prefix): + ''' + Return the text in the first element with a given class name. + + Note that XPath doesn't have a function to check if a given word is present + in a list of words, so using contains() will match a class name even if + the class name is a substring of the class name being matched against. + To prevent this, we use spaces as a delimiter at the beginning and end of + the class name, and ensure there's only one space before and after all + class names in the class name list. + + Example: consider two class names with cache-busting suffixes: + * Header_soldStatusNotice__12345 + * Header_soldStatusNoticeBackground__ABCDE + + Matching against Header_soldStatusNotice using a naive implementation with + `contains()` would match both class names, even though only the first one + is the correct match. + ''' + # Construct xpath selector. + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces... + xpath_str += "' '), " # Step 4: a trailing space... + xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side. + try: + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + + # We didn't find a match, try again by doing a prefix match and assume + # the full class name ends with two underscores followed by a cache- + # busting string. + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces... + xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores. + try: + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + + # No match, return empty string. + return '' + + +# time.sleep(2) +# Close the cookies setting window +try: + # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button") + cookie_button = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")) + ) + print(f"cookie_button = {cookie_button}") + cookie_button.click() +except: + pass + +# time.sleep(1) +# Clear any modal dialog +try: + # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]") + modal_close = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")) + ) + print(f"modal_close = {modal_close}") + modal_close.click() + #time.sleep(2) +except: + pass + +# Define callback function for Pub/Sub messages +def callback(message): + # Extract zip code from message data + zip_code = message.message.data.decode('utf-8') + + print(f"Received zip code: {zip_code}") + + try: + try: + # time.sleep(2) + # Clear any modal dialog + # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]") + modal_close = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")) + ) + # print(f"modal_close = {modal_close}") + modal_close.click() + # time.sleep(2) + except: + pass + + + try: + # time.sleep(1) + # Clear any notice + # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button") + notice_close = WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")) + ) + # print(f"notice_close = {notice_close}") + notice_close.click() + # time.sleep(1) + except: + pass + + # Enter zip code in the input field and click the "Check" button + # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]') + zip_button = WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')) + ) + # print(f"{zip_button:}") + zip_button.click() + # time.sleep(1) + + # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input') + zip_input = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input')) + ) + # print(f"{zip_input:}") + zip_input.click() + # print("clearing zip_input") + zip_input.send_keys("\b\b\b\b\b") + # print("sending zip code") + zip_input.send_keys(zip_code) + + check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']") + check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']"))) + # print(f"{check_button:}") + check_button.click() + # time.sleep(2) + + try: + # time.sleep(2) + # Clear any notice + # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button") + notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))) + # print(f"notice_close = {notice_close}") + notice_close.click() + #time.sleep(2) + except: + pass + + try: + # Clear any modal dialog + modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))) + # print(f"modal_close = {modal_close}") + modal_close.click() + except: + pass + + # Record sales count, if any + sales_count_element = get_element_text('Header_count') + count_start_index = sales_count_element.index('sold ') + len('sold ') + count_end_index = sales_count_element.index('+') + sales_count = sales_count_element[count_start_index:count_end_index] + # Record any sold status notice + sold_status = get_element_text('UnavailableCartbtn_container') + if sold_status == 'Notify me': + sold_status = 'Sold Out' + + # Extract the price of rice crackers for this zip code + # Check if there is a minimum order quantity + min_quantity: float = 0 + total_price: float = 0 + unit_price: float = 0 + try: + minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML') + if minimum_quantity_str: + regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)" + match = re.search(regexp,minimum_quantity_str) + min_quantity, total_price = match.groups() + except selenium.common.exceptions.NoSuchElementException as nsee: + print("No minimum purhcase string found") + except Exception as e: + print(f"Exception when finding minimum purchase string: {e}") + + + + ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']") + ld_json_text = ld_json_element.get_attribute('innerHTML') + ld_json_start_index = ld_json_text.index('{') + ld_json_end_index = ld_json_text.rindex('}') + 1 + ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index] + ld_json = json.loads(ld_json_json) + ld_json_price = ld_json['offers']['price'] + # Print the price + print(ld_json_price) + + unit_price_element = driver.find_element(By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText') + unit_price = unit_price_element.replace("$", "") + print(f"unit_price = {unit_price}" ) + + subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message + # Append the zip code and price to the dataframe + #append_to_df(zip_code, price) + new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]}) + return pd.concat([df, new_row], ignore_index=True) + #print(f"Recorded the zip code and price: {zip_code},{price}") + #print(f"Updated a dataframe: {df}") + + except Exception as e: + print(f"Caught exception: {e}") + print(f"Price not found for zip code {zip_code}") + price = 'Price not found' + return df + +# Start receiving messages (sync version) +while True: + pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path) + messages = [] + try: + pull_response = subscriber.pull(request = pull_request) + messages = pull_response.received_messages + except DeadlineExceeded: + pass + + if messages: + df = callback(messages[0]) + print(f"Updated a dataframe: {df}") + else: + print(f"no messages, trying again at {time.asctime()}") + break + +# df.to_hdf('product1.h5', key='df',mode='w') +# df.to_hdf('product1.h5', key='df',mode='a') + +# Open the HDF5 file in a separate window +# os.startfile('product1.h5') \ No newline at end of file diff --git a/fp2_14.py b/fp2_14.py new file mode 100644 index 0000000..c6cf9cf --- /dev/null +++ b/fp2_14.py @@ -0,0 +1,280 @@ +from google.cloud import pubsub_v1 +from google.api_core.exceptions import DeadlineExceeded +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +import json +import time +from concurrent.futures import TimeoutError +import pandas as pd +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +import selenium.common.exceptions +import re +from datetime import datetime +import h5py +import sys +import ntplib +import arrow + +# This fp2_14.py is used to load the new price and the timestamp, and append such data in an existing key in the HDF5 file "product1.h5" + +def get_product_prices(project_id, subscription_name, url): + # Create an empty dataframe to store the prices + df_new = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status']) + print(f"Created a dataframe: {df_new}") + + # Configure the Selenium driver options + options = Options() + options.add_argument('--disable-gpu') + driver = webdriver.Chrome(options=options) + + # Set up Pub/Sub subscription + ''' + project_id = "fp-20230425" + subscription_name = "fp-subscription-NFL-zipcodes" + ''' + subscriber = pubsub_v1.SubscriberClient() + subscription_path = subscriber.subscription_path(project_id, subscription_name) + + + # Load the product page and extract the __NEXT_DATA__ data + # url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f' + product_id = url.split("/")[-1][:5] + # product_id = url.split("/")[-1] + driver.delete_all_cookies + driver.get(url) + + def get_element_text(class_name_prefix): + ''' + Return the text in the first element with a given class name. + + Note that XPath doesn't have a function to check if a given word is present + in a list of words, so using contains() will match a class name even if + the class name is a substring of the class name being matched against. + To prevent this, we use spaces as a delimiter at the beginning and end of + the class name, and ensure there's only one space before and after all + class names in the class name list. + + Example: consider two class names with cache-busting suffixes: + * Header_soldStatusNotice__12345 + * Header_soldStatusNoticeBackground__ABCDE + + Matching against Header_soldStatusNotice using a naive implementation with + `contains()` would match both class names, even though only the first one + is the correct match. + ''' + # Construct xpath selector. + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces... + xpath_str += "' '), " # Step 4: a trailing space... + xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side. + try: + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + + # We didn't find a match, try again by doing a prefix match and assume + # the full class name ends with two underscores followed by a cache- + # busting string. + xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against... + xpath_str += "concat(' ', " # Step 2: a leading space, followed by... + xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces... + xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores. + try: + match = driver.find_element(By.XPATH, xpath_str) + return match.get_attribute('textContent') + except selenium.common.exceptions.NoSuchElementException as nsee: + pass + + # No match, return empty string. + return '' + + + def modal_close(): + try: + modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))) + print(f"modal_close = {modal_close}") + modal_close.click() + except: + pass + + def notice_close(): + try: + # Clear any notice + notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))) + # print(f"notice_close = {notice_close}") + notice_close.click() + except: + pass + + def cookie_close(): + try: + cookie_button = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")) + ) + print(f"cookie_button = {cookie_button}") + cookie_button.click() + except: + pass + + cookie_close() + modal_close() + # Define callback function for Pub/Sub messages + def callback(message): + # Extract zip code from message data + zip_code = message.message.data.decode('utf-8') + + print(f"Received zip code: {zip_code}") + + try: + + modal_close() + notice_close() + + # Enter zip code in the input field and click the "Check" button + zip_button = WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')) + ) + # print(f"{zip_button:}") + zip_button.click() + + # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input') + zip_input = WebDriverWait(driver, 2).until( + EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input')) + ) + # print(f"{zip_input:}") + zip_input.click() + # print("clearing zip_input") + zip_input.send_keys("\b\b\b\b\b") + # print("sending zip code") + zip_input.send_keys(zip_code) + + check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']") + check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']"))) + # print(f"{check_button:}") + check_button.click() + + notice_close() + modal_close() + + # Record sales count, if any + sales_count_element = get_element_text('Header_count') + count_start_index = sales_count_element.index('sold ') + len('sold ') + count_end_index = sales_count_element.index('+') + sales_count = sales_count_element[count_start_index:count_end_index] + # Record any sold status notice + sold_status = get_element_text('UnavailableCartbtn_container') + if sold_status == 'Notify me': + sold_status = 'Sold Out' + + # Extract the price of rice crackers for this zip code + # Check if there is a minimum order quantity + min_quantity: float = 0 + total_price: float = 0 + unit_price: float = 0 + try: + minimum_quantity_str = WebDriverWait(driver, 5).until(EC.presence_of_element_located(((By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML')))) + if minimum_quantity_str: + regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)" + match = re.search(regexp,minimum_quantity_str) + min_quantity, total_price = match.groups() + except selenium.common.exceptions.NoSuchElementException as nsee: + print("No minimum purhcase string found") + except Exception as e: + print(f"Exception when finding minimum purchase string: {e}") + + + + ld_json_element = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//script[@type='application/ld+json']"))) + ld_json_text = ld_json_element.get_attribute('innerHTML') + ld_json_start_index = ld_json_text.index('{') + ld_json_end_index = ld_json_text.rindex('}') + 1 + ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index] + ld_json = json.loads(ld_json_json) + ld_json_price = ld_json['offers']['price'] + # Print the price + print(ld_json_price) + + unit_price_element = WebDriverWait(driver, 5).until(EC.presence_of_element_located(((By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText')))) + unit_price = unit_price_element.replace("$", "") + print(f"unit_price = {unit_price}" ) + + subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message + # Append the zip code and price to the dataframe + #append_to_df(zip_code, price) + new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]}) + df_new = pd.concat([df_new, new_row], ignore_index=True) + return df_new + #print(f"Recorded the zip code and price: {zip_code},{price}") + #print(f"Updated a dataframe: {df_new}") + + except Exception as e: + print(f"Caught exception: {e}") + print(f"Price not found for zip code {zip_code}") + return df_new + + # Start receiving messages (sync version) + while True: + pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path) + messages = [] + try: + pull_response = subscriber.pull(request = pull_request) + messages = pull_response.received_messages + except DeadlineExceeded: + pass + + if messages: + df_new = callback(messages[0]) + print(f"Updated a dataframe: {df_new}") + else: + print(f"no messages, trying again at {time.asctime()}") + break + return df_new + +''' +If using sys to get the function, please remove the following part of code. +''' +if __name__ == '__main__': + project_id = input("Enter the project ID: ") + subscription_name = input("Enter the subscription name: ") + url = input("Enter the URL: ") + + get_product_prices(project_id, subscription_name, url) + + +df_new = get_product_prices("fp-20230425", "fp-subscription-NFL-zipcodes", "https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119") + +# Now we have the new dataframe "df" recording the new price listed in weee's page +# Load such data in the existing HDF5 file +file_path = 'product1.h5' +with h5py.File(file_path, 'a') as file: + # Load the existing dataframe 'df' from the HDF5 file + # REPLACE KEY TO GET THE EXISTING DATAFRAME FOR EACH DIFFERENT PRODUCT + df = pd.read_hdf(file, key='df') + + # Add timestamp to the new dataframe "df_new" + # timestamp = datetime.now() + ''' + To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that + provides time synchronization with a reliable time server. + Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a + sensible and human-friendly approach to format and convert the timestamps. + ''' + # Fetch the time from an NTP server + ntp_client = ntplib.NTPClient() + response = ntp_client.request('pool.ntp.org') + ntp_time = datetime.fromtimestamp(response.tx_time) + timestamp = arrow.get(ntp_time).to('local') + df_new['Timestamp'] = timestamp + + # Append 'df_new' to the existing df stored in the key + df_update = df.append(df_new, ignore_index=True) + + # Save 'df2' to the HDF5 file with the key 'df' + # REPLACE KEY FOR DIFFERENT PRODUCT + df_update.to_hdf(file, key='df', mode='a') + +print("Data appended and saved successfully.") diff --git a/fp2_6.py b/fp2_6.py new file mode 100644 index 0000000..4ed9af9 --- /dev/null +++ b/fp2_6.py @@ -0,0 +1,126 @@ +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +import json +import time +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC + +# Configure the Selenium driver options +options = Options() +options.add_argument('--disable-gpu') +driver = webdriver.Chrome(options=options) + +# Define the zip codes for which to extract the price +zip_codes = ['85305', '30313', '21230', '14127','28202','60605','45202','44114','76011','80204','48226','54304','77054','46225','32202','64129','89118','90301','33056','55415','02035','70112','07073','19148','15212','95054','98134','33607','37213','20785'] + +# Load the page and extract the __NEXT_DATA__ data +#url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119' +url = 'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261' +#url = 'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467' +#url = 'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01' +#url = 'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224' +#url = 'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02' +#url = 'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01' +#url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f' +#url = 'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353?trace_id=7847dd6e-c511-48e2-a76e-a283399967b1' +#url = 'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03' + +driver.delete_all_cookies +driver.get(url) + +# Close the cookies setting window +try: + cookie_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))) + print(f"cookie_button = {cookie_button}") + cookie_button.click() +except: + pass + +# Clear any modal dialog +try: + modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))) + print(f"modal_close = {modal_close}") + modal_close.click() +except: + pass + +def callback(zip_code): + try: + try: + # Clear any modal dialog + modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))) + # print(f"modal_close = {modal_close}") + modal_close.click() + except: + pass + + + try: + # Clear any notice + notice_close = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))) + # print(f"notice_close = {notice_close}") + notice_close.click() + except: + pass + + try: + # Clear any modal dialog + modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))) + # print(f"modal_close = {modal_close}") + modal_close.click() + except: + pass + + # Enter zip code in the input field and click the "Check" button + zip_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))) + # print(f"{zip_button:}") + zip_button.click() + + zip_input = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))) + # print(f"{zip_input:}") + zip_input.click() + # print("clearing zip_input") + zip_input.send_keys("\b\b\b\b\b") + # print("sending zip code") + zip_input.send_keys(zip_code) + + try: + check_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//button[text() = 'Confirm']"))) + # print(f"{check_button:}") + except: + check_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[5]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[3]/div/button"))) + finally: + check_button.click() + + try: + # Clear any notice + notice_close = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))) + # print(f"notice_close = {notice_close}") + notice_close.click() + except: + pass + + try: + # Clear any modal dialog + modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))) + # print(f"modal_close = {modal_close}") + modal_close.click() + except: + pass + + # Extract the price of rice crackers for this zip code + try: + sold_out_notice = driver.find_element(By.XPATH, '//*[@id="soldStatusNotice"]') + print(f"No such product in this area = {zip_code, sold_out_notice}") + except: + print(f"The product is available in this area {zip_code}") + + except Exception as e: + print(f"Caught exception: {e}") + print(f"Price not found for zip code {zip_code}") + +for zip_code in zip_codes: + callback(zip_code) + +driver.quit() \ No newline at end of file diff --git a/fp3_1.py b/fp3_1.py new file mode 100644 index 0000000..9864cdc --- /dev/null +++ b/fp3_1.py @@ -0,0 +1,166 @@ +import sqlalchemy +from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Float, DateTime +from sqlalchemy.orm import declarative_base +from sqlalchemy.orm import relationship, sessionmaker +import sys +from flask import Flask, render_template + +# Create the Flask application +app = Flask(__name__) + +# Create the database engine +engine = create_engine('sqlite:///scraper_design.db') + +# Create a session factory +Session = sessionmaker(bind=engine) + +# Create a base class +Base = declarative_base() + +# Define the product class +class Product(Base): + __tablename__ = 'product' + id = Column(Integer, primary_key=True) + name = Column(String) + SKU_PLU_UPC_number = Column(String) + price_histories = relationship("PriceHistory", back_populates="product") + +# Define the retailer class +class Retailer(Base): + __tablename__ = 'retailer' + id = Column(Integer, primary_key=True) + name = Column(String) + products = relationship("Product", secondary="retailer_product") + +# Define the price class +class Price(Base): + __tablename__ = 'price' + id = Column(Integer, primary_key=True) + MOQ = Column(Integer) + price = Column(Float) + +# Define the scraperjob class +class ScraperJob(Base): + __tablename__ = 'scraper_job' + id = Column(Integer, primary_key=True) + start_time = Column(DateTime) + end_time = Column(DateTime) + parameters = Column(String) + statistics = Column(String) + +# Define the price history class +class PriceHistory(Base): + __tablename__ = 'price_history' + id = Column(Integer, primary_key=True) + location = Column(String) + zip_code = Column(Integer) + city_name = Column(String) + price_at_time = Column(DateTime) + price = Column(Float) + scraper_job_id = Column(Integer, ForeignKey('scraper_job.id')) + product_id = Column(Integer, ForeignKey('product.id')) + product = relationship("Product", back_populates="price_histories") + scraper_job = relationship("ScraperJob") + +# Define the user class +class User(Base): + __tablename__ = 'user' + id = Column(Integer, primary_key=True) + username = Column(String) + credentials = Column(String) + creation_date = Column(DateTime) + modification_date = Column(DateTime) + def __str__(self): + return f"{self.id=} {self.username=} {self.creation_date=} {self.modification_date=}" + +class RetailerProduct(Base): + __tablename__ = 'retailer_product' + retailer_id = Column(Integer, ForeignKey('retailer.id'), primary_key=True) + product_id = Column(Integer, ForeignKey('product.id'), primary_key=True) + +# Create the database tables +Base.metadata.create_all(engine) + +@app.route('/') +def homepage(): + # Create a session + session = Session() + + # Get all the classes from SQLAlchemy + classes = Base.__subclasses__() + + statement = sqlalchemy.select(User) + users = session.scalars(statement).all() + + # Render the template with the classes + return render_template('test3.html', classes=classes, users=users) + +# Create command-line arguments for the user class +def main(): + if len(sys.argv) < 2: + app.run() + + command = sys.argv[1] + + if command == "create": + if len(sys.argv) != 3: + print("Usage: python user_tool.py create [username]") + return + + username = sys.argv[2] + + # Create a new user object + user = User(username=username) + + # Add the user to the database + session = Session() + session.add(user) + session.commit() + + print("User created successfully.") + + elif command == "edit": + if len(sys.argv) != 4: + print("Usage: python user_tool.py edit [user_id] [new_username]") + return + + user_id = int(sys.argv[2]) + new_username = sys.argv[3] + + # Retrieve the user from the database + session = Session() + user = session.query(User).get(user_id) + + if user: + # Update the user object + user.username = new_username + session.commit() + print("User updated successfully.") + else: + print("User not found.") + + elif command == "delete": + if len(sys.argv) != 3: + print("Usage: python user_tool.py delete [user_id]") + return + + user_id = int(sys.argv[2]) + + # Retrieve the user from the database + session = Session() + user = session.query(User).get(user_id) + + if user: + # Delete the user from the database + session.delete(user) + session.commit() + print("User deleted successfully.") + else: + print("User not found.") + + else: + print("Usage: python user_tool.py [create|edit|delete]") + return + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/karen-zipcodes.csv b/karen-zipcodes.csv new file mode 100644 index 0000000..da8a843 --- /dev/null +++ b/karen-zipcodes.csv @@ -0,0 +1,33 @@ +ZIP Code +85305 +30313 +21230 +14127 +28202 +60605 +45202 +44114 +76011 +80204 +48226 +54304 +77054 +46225 +32202 +64129 +89118 +90301 +90301 +33056 +55415 +02035 +70112 +07073 +07073 +19148 +15212 +95054 +98134 +33607 +37213 +20785 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9fcfb3c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +google.cloud.pubsub +plotly +pandas +numpy +h5py +selenium +sqlalchemy +flask diff --git a/sloetjes-zipcodes.csv b/sloetjes-zipcodes.csv new file mode 100644 index 0000000..da8a843 --- /dev/null +++ b/sloetjes-zipcodes.csv @@ -0,0 +1,33 @@ +ZIP Code +85305 +30313 +21230 +14127 +28202 +60605 +45202 +44114 +76011 +80204 +48226 +54304 +77054 +46225 +32202 +64129 +89118 +90301 +90301 +33056 +55415 +02035 +70112 +07073 +07073 +19148 +15212 +95054 +98134 +33607 +37213 +20785 diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..ce20953 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,30 @@ + + + + Users Table + + + + + + + + + + + + + + {% for user in users %} + + + + + + + + {% endfor %} + +
IDUsernameCredentialsCreation DateModification Date
{{ user.id }}{{ user.username }}{{ user.credentials }}{{ user.creation_date }}{{ user.modification_date }}
+ + diff --git a/templates/test3.html b/templates/test3.html new file mode 100644 index 0000000..c9916d6 --- /dev/null +++ b/templates/test3.html @@ -0,0 +1,21 @@ + + + + Class Dump + + +

Class Dump

+ + +

User Dump

+ + + diff --git a/weee's products.txt b/weee's products.txt new file mode 100644 index 0000000..da235fa --- /dev/null +++ b/weee's products.txt @@ -0,0 +1,45 @@ +# Selected +# Want-Want Rice Crackers +url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119' + +# Lay's Potato Chips +url = 'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261' + +# Pocky Cookies +url = 'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467' + +# Thai Jasmine Rice +url = 'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01' + +# Japanese Medium Grain Rice +url = 'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224' + +# Millet +url = 'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02' + +# Nature's Soy Firm Tofu: https://www.sayweee.com/zh/product/Nature-s-Soy-Firm-Tofu-/19231?trace_id=ee63a29e-a1e4-4065-b213-96aa14a15521 + +# Free 2% Reduced Fat Milk: https://www.sayweee.com/zh/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353?trace_id=3030fe4c-04fc-4f8f-9257-7f6395b9e513 + +# Washington Fuji Apples: https://www.sayweee.com/zh/product/Washington-Fuji-Apples/94817?trace_id=dc4955fc-bba8-48cf-b1f9-9db3cf077eed + +# Greenlandic Sweet Prawns: https://www.sayweee.com/zh/product/Royal-Greenland-Cooked-Wild-Caught-Greenlandic-Sweet-Prawns-Frozen-1/44656?category=seafood01 + + + +# Optional +# LuoSiFen +url = 'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01' + +# Rice Noodles +url = 'https://www.sayweee.com/en/product/Sanyangyishi-Fresh-Instant-Rice-Noodles/2156?category=instant01' + +# Cup Noodles +url = 'https://www.sayweee.com/en/product/Nissin-Beef-Flavor-Cup-Noodles/102519?category=instant01' + +# Pork Sung +url = 'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?category=canned04' + +# Nature's Soy Firm Tofu: https://www.sayweee.com/zh/product/Nature-s-Soy-Firm-Tofu-/19231?trace_id=ee63a29e-a1e4-4065-b213-96aa14a15521 +# [Not available or sold out in some area] Beef Ribeye Thin Slices: https://www.sayweee.com/zh/product/Beef-Ribeye-Thin-Slices--Frozen-1/4295?category=meat02 +# Shiitake Mushrooms: https://www.sayweee.com/zh/product/Shiitake-Mushrooms/45782?trace_id=ba388941-f631-4eaa-8570-5b7d4aba64e7 \ No newline at end of file