Update 'fp2_15_Sloetjes.py'

Egg.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+import requests
+
+url = 'https://www.sayweee.com/zh/product/Vital-Farms-Pasture-Raised-Eggs--Large-18-count/40709'
+response = requests.get(url)
+html_content = response.text
+html_content
+
+
+# In[2]:
+
+
+from lxml import html
+
+tree = html.fromstring(html_content)
+script = tree.xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
+next_data = script.split('__NEXT_DATA__ = ')[1].split(';</script>')[0]
+
+
+# In[ ]:
+
+
+import json
+
+data = json.loads(next_data)
+price = data['props']['initialState']['pdp']['product']['price']
+print(price)
+

GoogleCloud.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+from google.cloud import pubsub_v1
+
+
+# In[2]:
+
+
+# Set the project ID and topic name
+project_id = "sloetjes-project1"
+topic_name = "sloetjes-project1"
+subscription_name = "sloetjes-final-project1"
+
+
+# In[3]:
+
+
+# Create a publisher client
+publisher = pubsub_v1.PublisherClient()
+
+
+# In[4]:
+
+
+# Create the fully qualified topic path
+topic_path = publisher.topic_path(project_id, topic_name)
+
+
+# In[5]:
+
+
+# Create the Pub/Sub topic
+topic = publisher.create_topic(name = topic_path)
+
+
+# In[ ]:
+
+
+# Print the topic name
+print(f"Topic created: {topic.name}")
+
+
+# In[ ]:
+
+
+# Create the subscription if it doesn't exist
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+topic_path = publisher.topic_path(project_id, topic_name)
+
+# Publish a message to the topic
+message = "Hello, World!"
+data = message.encode("utf-8")
+publisher.publish(topic_path, data=data)
+print(f"Message published: {message}")
+
+# Pull the message from the subscription
+response = subscriber.pull(subscription_path, max_messages=1)
+for received_message in response.received_messages:
+    print(f"Received message: {received_message.message.data}")
+
+    # Acknowledge the message
+    subscriber.acknowledge(subscription_path, [received_message.ack_id])

Multiprocessing.py

@@ -1,274 +0,0 @@
-import multiprocessing
-import subprocess
-import concurrent.futures
-from google.cloud import pubsub_v1
-from google.api_core.exceptions import DeadlineExceeded
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.common.by import By
-import json
-import time
-import os
-from concurrent.futures import TimeoutError
-import pandas as pd
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-import selenium.common.exceptions
-import re
-
-# Define the subset of urls
-url_list = [
-    'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
-    'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
-    'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
-    'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
-    'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
-    'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
-    'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
-    'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
-    'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
-    'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
-    'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
-]
-
-
-# Create a pool of worker processes
-# pool = multiprocessing.Pool(cpu_count)
-
-# Define the function to run the scraper
-def run_scraper(url):
-    # scraping logic
-    df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
-    print(f"Created a dataframe: {df}")
-
-    options = Options()
-    options.add_argument('--disable-gpu')
-    driver = webdriver.Chrome(options=options)
-
-    project_id = "fp-20230425"
-    subscription_name = "fp-subscription-NFL-zipcodes"
-    subscriber = pubsub_v1.SubscriberClient()
-    subscription_path = subscriber.subscription_path(project_id, subscription_name)
-
-    # product_id = url.split("/")[-1]
-    driver.delete_all_cookies
-    driver.get(url)
-
-    def get_element_text(class_name_prefix):  
-        xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
-        xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
-        xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
-        xpath_str += "' '), "                     # Step 4: a trailing space...
-        xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
-        try:
-            match = driver.find_element(By.XPATH, xpath_str)
-            return match.get_attribute('textContent')
-        except selenium.common.exceptions.NoSuchElementException as nsee:
-            pass
-   
-        xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
-        xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
-        xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
-        xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
-        try:
-            match = driver.find_element(By.XPATH, xpath_str)
-            return match.get_attribute('textContent')
-        except selenium.common.exceptions.NoSuchElementException as nsee:
-            pass
-        return ''
-
-    # Close the cookies setting window
-    try:  
-        cookie_button = WebDriverWait(driver, 2).until(
-            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
-        )
-        print(f"cookie_button = {cookie_button}")
-        cookie_button.click()
-    except:
-        pass
-
-    # Clear any modal dialog
-    try:    
-        modal_close = WebDriverWait(driver, 2).until(
-            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
-        )
-        print(f"modal_close = {modal_close}")
-        modal_close.click()
-    except:
-        pass
-
-    # Define callback function for Pub/Sub messages
-    def callback(message):
-        # Extract zip code from message data
-        zip_code = message.message.data.decode('utf-8')
-
-        print(f"Received zip code: {zip_code}")
-
-        try:
-            try:
-                # Clear any modal dialog
-                modal_close = WebDriverWait(driver, 2).until(
-                EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
-            )
-                # print(f"modal_close = {modal_close}")
-                modal_close.click()
-            except:
-                pass
-
-
-            try:
-
-                # Clear any notice
-                notice_close = WebDriverWait(driver, 5).until(
-                EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
-                # print(f"notice_close = {notice_close}")
-                notice_close.click()
-            except:
-                pass
-        
-            # Enter zip code in the input field and click the "Check" button
-            zip_button = WebDriverWait(driver, 5).until(
-                EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
-            )
-            # print(f"{zip_button:}")
-            zip_button.click()
-
-            zip_input = WebDriverWait(driver, 2).until(
-                EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
-            )
-            # print(f"{zip_input:}")
-            zip_input.click()
-            # print("clearing zip_input")
-            zip_input.send_keys("\b\b\b\b\b")
-            # print("sending zip code")
-            zip_input.send_keys(zip_code)
-
-            check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
-            check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
-            # print(f"{check_button:}")
-            check_button.click()
-            # time.sleep(2)
-
-            try:
-            # Clear any notice
-                notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
-                # print(f"notice_close = {notice_close}")
-                notice_close.click()
-            except:
-                pass
-
-            try:
-                # Clear any modal dialog
-                modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
-                # print(f"modal_close = {modal_close}")
-                modal_close.click()
-            except:
-                pass
-
-            # Record sales count, if any
-            sales_count_element = get_element_text('Header_count')
-            count_start_index = sales_count_element.index('sold ') + len('sold ')
-            count_end_index = sales_count_element.index('+')
-            sales_count = sales_count_element[count_start_index:count_end_index]
-            # Record any sold status notice
-            sold_status = get_element_text('UnavailableCartbtn_container')
-            if sold_status == 'Notify me':
-                sold_status = 'Sold Out'
-
-        # Extract the price of rice crackers for this zip code
-            # Check if there is a minimum order quantity
-            min_quantity: float = 0
-            total_price: float = 0
-            unit_price: float = 0
-            try:
-                minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML')
-                if minimum_quantity_str:
-                    regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
-                    match = re.search(regexp,minimum_quantity_str)
-                    min_quantity, total_price = match.groups()
-            except selenium.common.exceptions.NoSuchElementException as nsee:
-                print("No minimum purhcase string found")
-            except Exception as e:
-                print(f"Exception when finding minimum purchase string: {e}")
-
-            ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
-            ld_json_text = ld_json_element.get_attribute('innerHTML')
-            ld_json_start_index = ld_json_text.index('{')
-            ld_json_end_index = ld_json_text.rindex('}') + 1
-            ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
-            ld_json = json.loads(ld_json_json)
-            ld_json_price = ld_json['offers']['price']
-            # Print the price
-            print(ld_json_price)
-
-            unit_price_element = driver.find_element(By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText')
-            unit_price = unit_price_element.replace("$", "")
-            print(f"unit_price = {unit_price}" )
-
-            subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
-            # Append the zip code and price to the dataframe
-            #append_to_df(zip_code, price)
-            new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]})
-            return pd.concat([df, new_row], ignore_index=True)
-            #print(f"Recorded the zip code and price: {zip_code},{price}")
-            #print(f"Updated a dataframe: {df}")
-
-        except Exception as e:
-            print(f"Caught exception: {e}")
-            print(f"Price not found for zip code {zip_code}")
-            price = 'Price not found'
-            return df
-
-    # Start receiving messages (sync version)
-    while True: 
-        pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
-        messages = []
-        try:
-            pull_response = subscriber.pull(request = pull_request)
-            messages = pull_response.received_messages
-        except DeadlineExceeded:
-            pass
-
-        if messages:
-            df = callback(messages[0])
-            print(f"Updated a dataframe: {df}")
-        else:
-            print(f"no messages, trying again at {time.asctime()}")
-            break
-        # subprocess.call(['python', 'fp2_13.py', url])
-    return run_scraper(url)
-
-if __name__ == '__main__':
-    # Determine the number of available CPUs
-    # cpu_count = multiprocessing.cpu_count()
-
-    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
-        future_to_url = {executor.submit(run_scraper, url): url for url in url_list}
-        results = []
-        for future in concurrent.futures.as_completed(future_to_url):
-            url = future_to_url[future]
-            try:
-                result = future.result()
-                print(f"Scraped data for URL: {url}")
-                results.append(result)
-            except Exception as e:
-                print(f"Failed to scrape data for URL: {url}\nError: {e}")
-
-'''
-# Run the scraper in parallel for the subset of items
-results = pool.map(run_scraper, url_list)
-
-# Close the pool and wait for the processes to finish
-pool.close()
-pool.join()
-'''
-
-with pd.HDFStore('product_data.h5') as store:
-    for url, result in zip(url_list, results):
-        # Extract the product ID from the URL
-        product_id = url.split("/")[-1].split("?")[0]
-
-        # Save the dataframe to the HDF5 file with a unique key
-        key = f'df_{product_id}'
-        store[key] = result
-        # result.to_hdf('product_data.h5', key=key, mode='a')

NFL Zip Code.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+import os
+from google.cloud import pubsub_v1
+
+
+# In[2]:
+# Set the project ID and topic name
+project_id = 'price-extraction-project'
+topic_name = 'nfl-zipcodes'
+
+
+# In[3]:
+# Create a publisher client
+publisher = pubsub_v1.PublisherClient()
+
+
+# In[4]:
+# Get the topic path
+topic_path = publisher.topic_path(project_id, topic_name)
+
+
+# In[5]:
+# Define the zip codes for all NFL cities
+nfl_zipcodes = {
+    'Arizona Cardinals': '85305',
+    'Atlanta Falcons': '30363',
+    'Baltimore Ravens': '21230',
+    'Buffalo Bills': '14127',
+    'Carolina Panthers': '28202',
+    'Chicago Bears': '60605',
+    'Cincinnati Bengals': '45202',
+    'Cleveland Browns': '44115',
+    'Dallas Cowboys': '76011',
+    'Denver Broncos': '80204',
+    'Detroit Lions': '48226',
+    'Green Bay Packers': '54304',
+    'Houston Texans': '77054',
+    'Indianapolis Colts': '46225',
+    'Jacksonville Jaguars': '32202',
+    'Kansas City Chiefs': '64129',
+    'Las Vegas Raiders': '89118',
+    'Los Angeles Chargers': '92626',
+    'Los Angeles Rams': '90305',
+    'Miami Dolphins': '33056',
+    'Minnesota Vikings': '55415',
+    'New England Patriots': '02035',
+    'New Orleans Saints': '70112',
+    'New York Giants': '07073',
+    'New York Jets': '07932',
+    'Philadelphia Eagles': '19145',
+    'Pittsburgh Steelers': '15212',
+    'San Francisco 49ers': '95054',
+    'Seattle Seahawks': '98056',
+    'Tampa Bay Buccaneers': '33607',
+    'Tennessee Titans': '37213',
+    'Washington Football Team': '20785'
+}
+
+
+# In[6]:
+# Publish each zip code to the topic
+for team, zipcode in nfl_zipcodes.items():
+    data = f'{zipcode}'.encode('utf-8')
+    publisher_future = publisher.publish(topic_path, data=data)
+    message_id = publisher_future.result()
+    print(f"message_id = {message_id}")
+
+print('All zip codes published.')
+
+
+# In[ ]:
+import os
+import time
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from google.cloud import pubsub_v1
+
+# Set the project ID and subscription name
+project_id = 'price-extraction-project'
+topic_name = 'nfl-zipcodes'
+subscription_name = 'sloetjes'
+
+# Set the path to your ChromeDriver executable
+chrome_driver_path = "/Users/sloetjeswong/Downloads/chromedriver_mac_arm64/chromedriver"
+
+# Set up a Google Cloud Pub/Sub subscriber
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+# Set up a ChromeDriver instance
+chrome_options = webdriver.ChromeOptions()
+chrome_options.add_argument("--headless")  # Run Chrome in headless mode
+chrome_service = Service(chrome_driver_path)
+chrome_service.start()
+driver = webdriver.Remote(chrome_service.service_url, options=chrome_options)
+
+def callback(message):
+    # Get the zip code from the message
+    zip_code = message.data.decode("utf-8")
+    print(f"Received message with zip code: {zip_code}")
+
+    # Open the product page for the rice crackers for the given zip code
+    product_url = f"https://www.sayweee.com/zh/product/Want-Want-Senbei-Rice-Crackers/17119?trace_id=89f7c636-05b8-4f52-abdd-74de3a83f45d&zip={zip_code}"
+    driver.get(product_url)
+
+    # Wait for the price to load
+    price_xpath = "//span[@class='price']"
+    price_element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, price_xpath)))
+
+    # Extract the price and print it
+    price = price_element.text
+    print(f"Price for zip code {zip_code}: {price}")
+
+    # Acknowledge the message
+    message.ack()
+
+# Subscribe to the topic
+streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback)
+print(f"Listening for messages on {subscription_path}...")
+streaming_pull_future.result()
+
+# Keep the script running
+while True:
+    time.sleep(60)
+

NFL_Cities.csv

@@ -1,33 +0,0 @@
-ID,Team Name,Arena Name,Abbreviation,Conference,Division,Arena Location,ZIP Code
-1,Arizona Cardinals,State Farm Stadium,ARI,NFC,West,"Glendale, Arizona",85305
-2,Atlanta Falcons,Mercedes-Benz Stadium,ATL,NFC,South,"Atlanta, Georgia",30313
-3,Baltimore Ravens,M&T Bank Stadium,BAL,AFC,North,"Baltimore, Maryland",21230
-4,Buffalo Bills,Bills Stadium,BUF,AFC,East,"Orchard Park, New York",14127
-5,Carolina Panthers,Bank of America Stadium,CAR,NFC,South,"Charlotte, North Carolina",28202
-6,Chicago Bears,Soldier Field,CHI,NFC,North,"Chicago, Illinois",60605
-7,Cincinnati Bengals,Paul Brown Stadium,CIN,AFC,North,"Cincinnati, Ohio",45202
-8,Cleveland Browns,First Energy Stadium,CLE,AFC,North,"Cleveland, Ohio",44114
-9,Dallas Cowboys,At&T Stadium,DAL,NFC,East,"Arlington, Texas",76011
-10,Denver Broncos,Empower Field at Mile High,DEN,AFC,West,"Denver, Colorado",80204
-11,Detroit Lions,Ford Field,DET,NFC,North,"Detroit, Michigan",48226
-12,Green Bay Packers,Lambeau Field,GB,NFC,North,"Green Bay, Wisconsin",54304
-13,Houston Texans,NRG Stadium,HOU,AFC,South,"Houston, Texas",77054
-14,Indianapolis Colts,Lucas Oil Stadium,IND,AFC,South,"Indianapolis, Indiana",46225
-15,Jacksonville Jaguars,TIAA Bank Field,JAX,AFC,South,"Jacksonville, Florida",32202
-16,Kansas City Chiefs,Arrowhead Stadium,KC,AFC,West,"Kansas City, Missouri",64129
-17,Las Vegas Raiders,Allegiant Stadium,LV,AFC,West,"Paradise, Nevada",89118
-18,Los Angeles Chargers,SoFi Stadium,LAC,AFC,West,"Inglewood, California",90301
-19,Los Angeles Rams,SoFi Stadium,LAR,NFC,West,"Inglewood, California",90301
-20,Miami Dolphins,Hard Rock Stadium,MIA,AFC,East,"Miami Gardens, Florida",33056
-21,Minnesota Vikings,U.S. Bank Stadium,MIN,NFC,North,"Minneapolis, Minnesota",55415
-22,New England Patriots,Gillette Stadium,NE,AFC,East,"Foxborough, Massachusetts",02035
-23,New Orleans Saints,Mercedes-Benz Superdome,NO,NFC,South,"New Orleans, Louisiana",70112
-24,New York Giants,MetLife Stadium,NYG,NFC,East,"East Rutherford, New Jersey",07073
-25,New York Jets,MetLife Stadium,NYJ,AFC,East,"East Rutherford, New Jersey",07073
-26,Philadelphia Eagles,Lincoln Financial Field,PHI,NFC,East,"Philadelphia, Pennsylvania",19148
-27,Pittsburgh Steelers,Heinz Field,PIT,AFC,North,"Pittsburgh, Pennsylvania",15212
-28,San Francisco 49ers,Levi's Stadium,SF,NFC,West,"Santa Clara, California",95054
-29,Seattle Seahawks,CenturyLink Field,SEA,NFC,West,"Seattle, Washington",98134
-30,Tampa Bay Buccaneers,Raymond James Stadium,TB,NFC,South,"Tampa, Florida",33607
-31,Tennessee Titans,Nissan Stadium,TEN,AFC,South,"Nashville, Tennessee",37213
-32,Washington Redskins,FedExField,WAS,NFC,East,"Landover, Maryland",20785

README.md

@@ -17,27 +17,4 @@ the focus is on *working results*, not rote memorization.  Homework questions
 are simple enough that they can be reviewed interactively in 1:1 or up to 1:4
 sessions with study groups.
 
-This branch contains a prototype for a retail price scraper for getweee.com.
-To run it, install:
-
-* Python 3.9+,
-* chrome,
-* chromedriver,
-* and the python depedencies:
-
-```
-pip install -r /path/to/requirements.txt
-```
-
-Once those are installed, install the GCP SDK:
-
-https://cloud.google.com/sdk/docs/install
-
-You will also have to log into GCP *twice*, once for the gcloud CLI tools, 
-and again for the GCP SDK libraries:
-
-https://googleapis.dev/python/google-api-core/latest/auth.html
-
-
-
 

SQLAlchemy.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[ ]:
+
+
+from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime, ForeignKey
+from sqlalchemy.orm import sessionmaker, relationship, declarative_base
+from flask import Flask, render_template
+
+# In[ ]:
+
+
+# Create a base class for declarative models
+Base = declarative_base()
+
+class Product(Base):
+    __tablename__ = 'product'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    SKU_PLU_UPC_number = Column(String)
+    price_histories = relationship("PriceHistory", back_populates="product")
+
+class Retailer(Base):
+    __tablename__ = 'retailer'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    products = relationship("Product", secondary="retailer_product")
+
+class Price(Base):
+    __tablename__ = 'price'
+    id = Column(Integer, primary_key=True)
+    MOQ = Column(Integer)
+    price = Column(Float)
+
+class ScraperJob(Base):
+    __tablename__ = 'scraper_job'
+    id = Column(Integer, primary_key=True)
+    start_time = Column(DateTime)
+    end_time = Column(DateTime)
+    parameters = Column(String)
+    statistics = Column(String)
+
+class PriceHistory(Base):
+    __tablename__ = 'price_history'
+    id = Column(Integer, primary_key=True)
+    location = Column(String)
+    zip_code = Column(Integer)
+    city_name = Column(String)
+    price_at_time = Column(DateTime)
+    price = Column(Float)
+    scraper_job_id = Column(Integer, ForeignKey('scraper_job.id'))
+    product_id = Column(Integer, ForeignKey('product.id'))
+    product = relationship("Product", back_populates="price_histories")
+    scraper_job = relationship("ScraperJob")
+
+class User(Base):
+    __tablename__ = 'user'
+    id = Column(Integer, primary_key=True)
+    username = Column(String)
+    credentials = Column(String)
+    creation_date = Column(DateTime)
+    modification_date = Column(DateTime)
+
+class RetailerProduct(Base):
+    __tablename__ = 'retailer_product'
+    retailer_id = Column(Integer, ForeignKey('retailer.id'), primary_key=True)
+    product_id = Column(Integer, ForeignKey('product.id'), primary_key=True)
+
+
+# In[ ]:
+
+
+# Create the database engine
+engine = create_engine('sqlite:///user_database.db')
+Base.metadata.create_all(engine)
+
+
+# In[ ]:
+
+
+# Create a session factory
+Session = sessionmaker(bind=engine)
+session = Session()
+
+
+# In[ ]:
+
+
+import sys
+
+# Create a command-line tool that parses sys.argv to allow a user to create/edit/delete user objects.
+
+if len(sys.argv) < 2:
+    app = Flask(__name__)
+
+    # Create a database connection
+    engine = create_engine('sqlite:///user_database.db')
+    Session = sessionmaker(bind=engine)
+    session = Session()
+
+    @app.route('/')
+    def home():
+        # Get all user objects
+        users = session.query(User).all()
+        return render_template('index.html', users=users)
+
+    app.run()
+
+
+command = sys.argv[1]
+
+if command == "create":
+    if len(sys.argv) < 4:
+        print("Usage: python script.py create [username] [credentials]")
+        sys.exit(1)
+    
+    username = sys.argv[2]
+    credentials = sys.argv[3]
+
+    # Create a new user object
+    new_user = User(username=username, credentials=credentials)
+    session.add(new_user)
+    session.commit()
+    print("User created successfully!")
+
+elif command == "edit":
+    if len(sys.argv) < 4:
+        print("Usage: python script.py edit [username] [new_credentials]")
+        sys.exit(1)
+
+    username = sys.argv[2]
+    new_credentials = sys.argv[3]
+
+    # Find the user object
+    user = session.query(User).filter_by(username=username).first()
+
+    if user:
+        # Update user credentials
+        user.credentials = new_credentials
+        session.commit()
+        print("User edited successfully!")
+    else:
+        print("User not found!")
+
+elif command == "delete":
+    if len(sys.argv) < 3:
+        print("Usage: python script.py delete [username]")
+        sys.exit(1)
+
+    username = sys.argv[2]
+
+    # Find the user object
+    user = session.query(User).filter_by(username=username).first()
+
+    if user:
+        # Delete the user object
+        session.delete(user)
+        session.commit()
+        print("User deleted successfully!")
+    else:
+        print("User not found!")
+
+else:
+    print("Usage: python script.py [create|edit|delete] [username] [credentials]")

Simulation/Simulation.py

@@ -1,42 +0,0 @@
-from abc import ABC, abstractmethod
-
-class Simulation(ABC):
-    @abstractmethod
-    def start(self):
-        "Start the simulated object."
-        pass
-
-    @abstractmethod
-    def tick(self) -> bool:
-        """
-        Simulate one clock tick.
-        Returns:
-            bool: True if the simulation could advance one clock tick, or False if the object is expired.
-        """
-        pass
-
-    @abstractmethod
-    def get_uses(self):
-        """
-        Return the uses of the simulated object.  These are verb strings which are passed to the `use()` method.
-        Returns:
-            tuple: A dict of key-value pairs, where the keys are strings of
-            verbs to be passed to `use`, and the values are strings explaining
-            what the verb does in English.
-        """
-    @abstractmethod
-    def use(self, *args, **kwargs):
-        """
-        Use the simulated object.
-        Args:
-            args (tuple): A tuple of verbs to be used on the object.
-            kwargs (dict): A dictionary of key-value pairs of arguments to the verb(s).
-        Returns:
-            True if the action succeeded, False if the action did not succeed and the object needs to be expired.
-        """
-        pass
-
-    @abstractmethod
-    def stop(self):
-        "Stop the simulated object."
-        pass

Simulation/SimulationRunner.py

@@ -1,42 +0,0 @@
-from importlib import import_module
-import tkinter as tk
-import os
-import sys
-from random import choice
-from time import sleep
-from tkinter import filedialog
-from abc import ABC, abstractmethod
-
-class SimulationRunner:
-    def __init__(self, simulation) -> None:
-        self.simulation = simulation()
-        self.simulation.start()
-        uses = list(self.simulation.get_uses().keys())
-        while self.simulation.tick() and self.simulation.use(choice(uses)):
-            pass
-        self.simulation.stop()
-
-if __name__ == '__main__':
-    if len(sys.argv) == 1:
-        # Create a hidden root window
-        root = tk.Tk()
-        root.withdraw()
-
-        # Open the file chooser dialog
-        file_path = filedialog.askopenfilename()
-        print(f"Selected file: {file_path}")
-    else:
-        file_path = sys.argv[1]
-
-    # Get the python file's directory and filename
-    module_dir, module_file = os.path.split(file_path)
-    # Get the python file's name without any extension
-    module_name, _ = os.path.splitext(module_file)
-
-    # Add the module directory to sys.path so we can import it unqualified
-    sys.path.append(module_dir)
-
-    simulation = import_module(module_name)
-    # Load the simulation module's simulation class, which should be the same
-    # name as the module name.
-    simulationrunner = SimulationRunner(getattr(simulation, module_name))

View_h5_file.py

@@ -1,73 +0,0 @@
-import h5py
-import pandas as pd
-import numpy as np
-import plotly.graph_objects as go
-
-f = h5py.File('product1.h5','r')
-list(f.keys())
-for key in f.keys():
-    print(f[key])
-    df = pd.read_hdf('product1.h5', key='df')
-    df = df._append(pd.read_hdf('product1.h5', key))
-    # print(f[key].shape)
-    # print(f[key].value)
-# df = pd.read_hdf('product1.h5', key='df7')
-print(df)
-'''
-# Open the HDF5 file
-with h5py.File('product1.h5', 'r') as file:
-    dataframes=pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])  
-    for key in file.keys():
-        # Read the dataframe from the HDF5 file
-        dataframe = pd.read_hdf('product1.h5', key)
-
-        # Append the dataframe to the list
-        dataframes._append(dataframe)
-
-# Print the number of dataframes read from the HDF5 file
-print(f"Read {len(dataframes)} dataframes from the HDF5 file")
-
-item_std_devs = {}
-for dataframe in dataframes:
-    item_name = dataframe['Product ID'].iloc[0]
-    std_dev = np.std(dataframe['Unit Price'])
-    item_std_devs[item_name] = std_dev
-
-# Print the item names and their corresponding standard deviations
-for item_name, std_dev in item_std_devs.items():
-    print(f"Item: {item_name}, Std Dev: {std_dev}")
-
-# Prepare the data for the graph
-x = list(item_std_devs.keys())
-y = list(item_std_devs.values())
-
-# Create a bar graph
-fig = go.Figure(data=[go.Bar(x=x, y=y)])
-
-# Customize the graph
-fig.update_layout(
-    title="Standard Deviation of Pricing for Each Item",
-    xaxis_title="Item",
-    yaxis_title="Standard Deviation",
-)
-
-# Save the graph as an HTML file
-report_file = 'test2.html'
-fig.write_html(report_file)
-
-
-
-'''
-# Calculate the standard deviation using pandas' function
-std_dev_unit = df.groupby('Product ID')['Unit Price'].std()
-std_dev_displayed = df.groupby('Zip Code')['Shown Price'].std()
-
-# Generate the graph using Plotly
-fig = go.Figure()
-fig.add_trace(go.Bar(x=std_dev_unit.index, y=std_dev_unit.values))
-fig.update_layout(title='Standard Deviation of the Unit Pricing by Item', xaxis_title='Product ID', yaxis_title='Standard Deviation')
-
-report_file = 'test2.html'
-fig.write_html(report_file)
-
-

assessment1.py

@@ -1,81 +0,0 @@
-'''
-class Vault:
-    def __init__(self):
-        self._treasure = "gold"
-        self._attributes = dict(self.__dict__)
-        
-    def __setattr__(self, name, value):
-        if name != "_attributes" and name not in self._attributes:
-            print("attribute deleted!")
-        self._attributes[name] = value
-        super().__setattr__("_attributes", dict(self.__dict__))
-        
-    @property
-    def treasure(self):
-        if self.__dict__ != self._attributes:
-            print("attribute deleted!")
-        return self._treasure
-
-v=Vault()
-print(v.treasure)
-'''
-'''
-class Vault:
-    def __init__(self):
-        self._treasure = "gold"
-        self._original_dict = dict(self.__dict__)
-
-    def __setattr__(self, name, value):
-        if name != "_treasure" and name in self.__dict__ and self._original_dict.get(name) != value:
-            print("attribute deleted!")
-        self.__dict__[name] = value
-
-    @property
-    def treasure(self):
-        if self.__dict__ != self._original_dict:
-            print("attribute deleted!")
-        return self._treasure
-v = Vault()
-print(v.treasure)
-'''
-'''
-class Vault:
-    def __init__(self):
-        self._treasure = "gold"
-        self._attributes = dict(self.__dict__)
-
-    def __setattr__(self, name, value):
-        if name != "_attributes" and name not in self._attributes:
-            print("attribute deleted!")
-        self._attributes[name] = value
-        super().__setattr__("_attributes", dict(self.__dict__))
-
-    @property
-    def treasure(self):
-        if self.__dict__ != self._attributes:
-            print("attribute deleted!")
-        return self._treasure
-
-v = Vault()
-print(v.treasure)
-'''
-
-class Vault:
-    def __init__(self):
-        self._treasure = "gold"
-
-    def __setattr__(self, name, value):
-        if name != "_attributes" and name not in self.__dict__:
-            print("attribute deleted!")
-        self.__dict__[name] = value
-        self._attributes = dict(self.__dict__)
-
-    @property
-    def treasure(self):
-        if self.__dict__ != self._attributes:
-            print("attribute deleted!")
-        return self._treasure
-
-v = Vault()
-print(v.treasure)
-

assessment2.py

@@ -1,47 +0,0 @@
-import importlib.util
-import os
-
-def get_module_size(module_name):
-    """Get the total number of bytes in the cached .pyc file or .py file of a module or package.
-
-    Args:
-        module_name (str): The name of the module or package.
-
-    Returns:
-        int: The total number of bytes in the cached .pyc file or .py file.
-    """
-    spec = importlib.util.find_spec(module_name)
-
-    if spec is not None and spec.has_location:
-        # If the module is found and has a location, check for .pyc file first
-        pyc_file_path = spec.origin + 'c'
-        if os.path.exists(pyc_file_path):
-            return os.stat(pyc_file_path).st_size
-        else:
-            # If .pyc file doesn't exist, check for .py file
-            py_file_path = spec.origin
-            if os.path.exists(py_file_path):
-                return os.stat(py_file_path).st_size
-
-    return 0  # Return 0 if no .pyc or .py file found
-
-
-import sys
-
-def get_total_module_size():
-    """Get the total number of bytes for all modules listed in sys.modules.
-
-    Returns:
-        int: The total number of bytes for all modules.
-    """
-    total_size = 0
-    for module_name in sys.modules:
-        module_size = get_module_size(module_name)
-        total_size += module_size
-
-    return total_size
-
-if __name__ == '__main__' and __package__ is None:
-    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
-total_size = get_total_module_size()
-print("Total size of all modules in sys.modules: {} bytes".format(total_size))

fp1.py

@@ -1,65 +0,0 @@
-from google.cloud import pubsub_v1
-from google.oauth2 import service_account
-import os
-import asyncio
-
-os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
-project_id = "fp-20230425"
-
-'''
-topic_name = "fp-topic-Karen"
-subscription_name = "fp-subscription-Karen"
-subscriber = pubsub_v1.SubscriberClient()
-subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
-subscriber.delete_subscription(request={"subscription": subscription_path})
-print("Subscription deleted: {}".format(subscription_path))
-
-# Delete the topic
-publisher = pubsub_v1.PublisherClient()
-topic_path = publisher.topic_path(project_id, topic_name)
-publisher.delete_topic(request={"topic": topic_path})
-print(f"Topic '{topic_name}' deleted.")
-
-subscription_name = "fp-subscription-Karen"
-subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
-
-
-'''
-# Create a pubsub topic
-publisher = pubsub_v1.PublisherClient()
-topic_name = "fp-topic-Karen"
-topic_path = publisher.topic_path(project_id, topic_name)
-topic = publisher.create_topic(request={"name": topic_path})
-
-
-# Create a Pub/Sub subscription
-subscriber = pubsub_v1.SubscriberClient()
-subscription_name = "fp-subscription-Karen"
-subscription_path = subscriber.subscription_path(project_id, subscription_name)
-subscription = subscriber.create_subscription(
-    name=subscription_path, topic=topic_path
-)
-
-# Publish a message to the topic
-message = b"Hello!"
-publisher.publish(topic_path, data=message)
-print("Message published.")
-
-
-async def receive_messages():
-    response = subscriber.pull(subscription=subscription_path, max_messages=1)
-    for received_message in response.received_messages:
-        print("Received message: {}".format(received_message.message.data))
-       
-    
-asyncio.run(receive_messages())
-
-
-'''
-# Pull the message from the subscription
-response = subscriber.pull(subscription=subscription_path, max_messages=1)
-for received_message in response.received_messages:
-    print("Received message: {}".format(received_message.message.data))
-    # Acknowledge the message to mark it as processed
-    subscriber.acknowledge(subscription_path, [received_message.ack_id])
-'''

fp2.py

@@ -1,10 +0,0 @@
-import http.client
-
-# The product ID for the Pete-and-Gerry's organic large brown egges is 65902
-conn = http.client.HTTPSConnection("www.sayweee.com")
-payload = ''
-headers = {}
-conn.request("GET", "/en/product/Pete-and-Gerry-s-Organic-Large-Brown-Eggs/65902?category=dairy02", payload, headers)
-res = conn.getresponse()
-data = res.read()
-print(data.decode("utf-8"))

fp2_10.py

@@ -1,67 +0,0 @@
-from google.cloud import pubsub_v1
-from google.api_core.exceptions import DeadlineExceeded
-import csv
-import os
-import asyncio
-
-# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
-project_id = "fp-20230425"
-# project_id = "svig-384801"
-
-'''
-topic_name = "fp-topic-NFL-zipcodes"
-subscription_name = "fp-subscription-NFL-zipcodes"
-subscriber = pubsub_v1.SubscriberClient()
-subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
-subscriber.delete_subscription(request={"subscription": subscription_path})
-print("Subscription deleted: {}".format(subscription_path))
-
-# Delete the topic
-publisher = pubsub_v1.PublisherClient()
-topic_path = publisher.topic_path(project_id, topic_name)
-publisher.delete_topic(request={"topic": topic_path})
-print(f"Topic '{topic_name}' deleted.")
-
-
-
-'''
-# pubsub topic
-publisher = pubsub_v1.PublisherClient()
-topic_name = "fp-topic-NFL-zipcodes"
-topic_path = publisher.topic_path(project_id, topic_name)
-
-# Create a Pub/Sub subscription
-subscriber = pubsub_v1.SubscriberClient()
-'''
-subscription_name = "fp-subscription-zipcodes"
-subscription_path = subscriber.subscription_path(project_id, subscription_name)
-subscription = subscriber.create_subscription(
-    name=subscription_path, topic=topic_path
-)
-'''
-# Publish a message to the topic
-# Read the data from the csv file
-# file_path = "C:/Users/Karen/Desktop/zipcodes.csv"
-file_path = "zipcodes.csv"
-
-with open(file_path, "r") as csv_file:
-    reader = csv.reader(csv_file)
-    next(reader)  # skip header row
-    for row in reader:
-        # Format the data as a string and publish it to the topic
-        data = ",".join(row).encode("utf-8")
-        try:
-            future = publisher.publish(topic_path, data)
-            print(f"Published message ID {future.result()}")
-        except DeadlineExceeded as de:
-            print(f"Caught DeadlineExceeded exception: {de}")
-
-'''
-async def receive_messages():
-    response = subscriber.pull(subscription=subscription_path, max_messages=1)
-    for received_message in response.received_messages:
-        print("Received message: {}".format(received_message.message.data))
-       
-    
-asyncio.run(receive_messages())
-'''

fp2_11.py

@@ -1,372 +0,0 @@
-from google.cloud import pubsub_v1
-from google.api_core.exceptions import DeadlineExceeded
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.support.wait import WebDriverWait
-from selenium.webdriver.common.by import By
-from selenium.webdriver.common.keys import Keys
-import selenium.common.exceptions
-import json
-import time
-import os
-import re
-import sys
-from concurrent.futures import TimeoutError
-import pandas as pd
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-
-# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
-
-# Create an empty dataframe to store the prices
-df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Displayed Price', 'ld+json Price', 'Minimum Quantity', 'Total Price', 'Sales Count', 'Sold Status'])
-print(f"Created a dataframe: {df}")
-
-# Configure the Selenium driver options
-options = Options()
-options.add_argument('--disable-gpu')
-driver = webdriver.Chrome(options=options)
-
-# Set up Pub/Sub subscription
-project_id = "fp-20230425"
-subscription_name = "fp-subscription-NFL-zipcodes"
-subscriber = pubsub_v1.SubscriberClient()
-subscription_path = subscriber.subscription_path(project_id, subscription_name)
-
-# Load the product page and extract the __NEXT_DATA__ data
-url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
-product_id = url.split("/")[-1]
-driver.delete_all_cookies
-driver.get(url)
-# time.sleep(5)
-
-# time.sleep(2)
-# Close the cookies setting window
-try:  
-    # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")
-    cookie_button = WebDriverWait(driver, 2).until(
-        EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
-    )
-    print(f"cookie_button = {cookie_button}")
-    cookie_button.click()
-except:
-    pass
-
-# time.sleep(1)
-# Clear any modal dialog
-try:    
-    # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
-    modal_close = WebDriverWait(driver, 2).until(
-        EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
-    )
-    print(f"modal_close = {modal_close}")
-    modal_close.click()
-    #time.sleep(2)
-except:
-    pass
-
-
-def close_modals():
-    try:
-        # time.sleep(2)
-        # Clear any modal dialog
-        modal_close_buttons = driver.find_elements(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
-        print(f"modal_close = {modal_close}")
-        for button in modal_close_buttons:
-            try:
-                button.click()
-            except Exception as e:
-                print(f"Exception when clicking on close button for element {button}: {e}")
-
-        # time.sleep(2)
-    except Exception as e:
-        print(f"Exception when trying to close discount modal dialog: {e}")
-
-
-    try:
-        # time.sleep(1)
-        # Clear any notice
-        notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
-        print(f"notice_close = {notice_close}")
-        notice_close.click()
-        # time.sleep(1)
-    except selenium.common.exceptions.NoSuchElementException as nsee:
-        print("Purchase option change dialog close button not found")
-    except Exception as e:
-        print(f"Exception when trying to close purchase option change dialog: {e}")
-
-def get_element_text(class_name_prefix):
-    '''
-    Return the text in the first element with a given class name.
-
-    Note that XPath doesn't have a function to check if a given word is present
-    in a list of words, so using contains() will match a class name even if
-    the class name is a substring of the class name being matched against.
-    To prevent this, we use spaces as a delimiter at the beginning and end of
-    the class name, and ensure there's only one space before and after all
-    class names in the class name list.
-
-    Example: consider two class names with cache-busting suffixes:
-      * Header_soldStatusNotice__12345
-      * Header_soldStatusNoticeBackground__ABCDE
-
-    Matching against Header_soldStatusNotice using a naive implementation with
-    `contains()` would match both class names, even though only the first one
-    is the correct match. 
-    '''
-    # Construct xpath selector.  
-    xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
-    xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
-    xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
-    xpath_str += "' '), "                     # Step 4: a trailing space...
-    xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
-    try:
-        match = driver.find_element(By.XPATH, xpath_str)
-        return match.get_attribute('textContent')
-    except selenium.common.exceptions.NoSuchElementException as nsee:
-        pass
-
-    # We didn't find a match, try again by doing a prefix match and assume
-    # the full class name ends with two underscores followed by a cache-
-    # busting string.    
-    xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
-    xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
-    xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
-    xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
-    try:
-        match = driver.find_element(By.XPATH, xpath_str)
-        return match.get_attribute('textContent')
-    except selenium.common.exceptions.NoSuchElementException as nsee:
-        pass
-
-    # No match, return empty string.
-    return ''
-
-def callback(message):
-    '''
-    Callback function for Pub/Sub messages
-    '''
-    # Extract zip code from message data
-    zip_code = message.message.data.decode('utf-8')
-
-    print(f"Received zip code: {zip_code}")
-
-    while True:
-        try:
-            # Enter zip code in the input field and click the "Check" button
-            WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
-            zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
-            # print(f"{zip_button:}")
-            zip_button.click()
-            # time.sleep(1)
-
-            WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, '//div/label/following-sibling::input'))
-            zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
-            # print(f"{zip_input:}")
-            zip_input.click()
-            # print("clearing zip_input")
-            zip_input.send_keys("\b\b\b\b\b\b\b\b\b\b")
-            for i in range(1,10):
-                zip_input.send_keys(Keys.DELETE)
-            # print("sending zip code")
-            zip_input.send_keys(zip_code)
-
-            WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, "//button[text() = 'Confirm']"))
-            check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
-            print(f"{check_button:}")
-            check_button.click()
-            # time.sleep(2)
-            break
-        except Exception as e:
-            print(f"Exception when changing zip code: {e}, trying to close modal dialogs")
-            close_modals()
-
-        # try:
-        #     time.sleep(2)
-        #     # Clear any notice
-        #     notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
-        #     print(f"notice_close = {notice_close}")
-        #     notice_close.click()
-        #     time.sleep(2)
-        # except:
-        #     pass
-
-    try:
-        try:
-            # time.sleep(2)
-            # Clear any modal dialog
-            # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
-            modal_close = WebDriverWait(driver, 2).until(
-            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
-        )
-            print(f"modal_close = {modal_close}")
-            modal_close.click()
-            # time.sleep(2)
-        except:
-            pass
-
-
-        try:
-            # time.sleep(1)
-            # Clear any notice
-            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
-            notice_close = WebDriverWait(driver, 5).until(
-            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
-        )
-            print(f"notice_close = {notice_close}")
-            notice_close.click()
-            # time.sleep(1)
-        except:
-            pass
-        
-        # Enter zip code in the input field and click the "Check" button
-        # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
-        zip_button = WebDriverWait(driver, 5).until(
-            EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
-        )
-        # print(f"{zip_button:}")
-        zip_button.click()
-        # time.sleep(1)
-
-        # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
-        zip_input = WebDriverWait(driver, 2).until(
-            EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
-        )
-        # print(f"{zip_input:}")
-        zip_input.click()
-        # print("clearing zip_input")
-        zip_input.send_keys("\b\b\b\b\b")
-        # print("sending zip code")
-        zip_input.send_keys(zip_code)
-
-        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
-        check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
-        # print(f"{check_button:}")
-        check_button.click()
-        # time.sleep(2)
-
-        try:
-            # time.sleep(2)
-            # Clear any notice
-            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
-            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
-            # print(f"notice_close = {notice_close}")
-            notice_close.click()
-            #time.sleep(2)
-        except:
-            pass
-
-        try:
-            # Clear any modal dialog
-            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
-            # print(f"modal_close = {modal_close}")
-            modal_close.click()
-        except:
-            pass
-
-        # Record sales count, if any
-        sales_count = get_element_text('Header_count')
-        # Record any sold status notice
-        sold_status = get_element_text('soldStatusNotice')
-
-        # Check if there's a minimum order quantity
-        min_quantity: float = 0
-        total_price: float = 0
-        try:
-            minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' minQuantity ')]").get_attribute('innerHTML')
-            if minimum_quantity_str:
-                regexp="Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
-                match = re.search(regexp, minimum_quantity_str)
-                min_quantity, total_price = match.groups()
-
-        except selenium.common.exceptions.NoSuchElementException as nsee:
-            print("No minimum purchase string found")
-        except Exception as e:
-            print(f"Exception when finding minimum purchase string: {e}")
-
-        # Get the price as displayed on the web page
-        displayed_price = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' price_price ')]").get_attribute('innerText').replace('$', '')
-        print(f"displayed_price = {displayed_price}")
-
-        # Extract the price of rice crackers for this zip code     
-        '''
-        price_element = driver.find_element(By.XPATH, '/html/body/div[2]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')
-        price = price_element.text.strip()[1:]
-        print(price)
-        '''
-
-        # Get the price from the ld+json element
-        ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
-        ld_json_text = ld_json_element.get_attribute('innerHTML')
-        ld_json_start_index = ld_json_text.index('{')
-        ld_json_end_index = ld_json_text.rindex('}') + 1
-        ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
-        ld_json = json.loads(ld_json_json)
-        ld_json_price = ld_json['offers']['price']
-        # Print the price
-        print(f"ld_json_price = {ld_json_price}")
-
-        subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
-        # Append the zip code and price to the dataframe
-        #append_to_df(zip_code, price)
-        new_row = pd.DataFrame({'Product ID': [product_id],
-            'Zip Code': [zip_code],
-            'Displayed Price': [displayed_price],
-            'ld+json Price': [ld_json_price],
-            'Minimum Quantity': [min_quantity],
-            'Total Price': [total_price],
-            'Sales Count': [sales_count],
-            'Sold Status': [sold_status]})
-        return pd.concat([df, new_row], ignore_index=True)
-        #print(f"Recorded the zip code and price: {zip_code},{price}")
-        #print(f"Updated a dataframe: {df}")
-
-    except Exception as e:
-        print(f"Caught exception {type(e)}: {e}")
-        print(f"Price not found for zip code {zip_code}")
-        price = 'Price not found'
-        return df
-
-# Start receiving messages (sync version)
-while True: 
-    pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
-    messages = []
-    try:
-        pull_response = subscriber.pull(request = pull_request)
-        messages = pull_response.received_messages
-    except DeadlineExceeded:
-        pass
-
-    if messages:
-        df = callback(messages[0])
-        print(f"Updated a dataframe: {df}")
-        df.to_hdf('product1.h5', key='df',mode='w')
-    else:
-        print(f"No messages in subscription {subscription_path} at {time.asctime()}, trying again...")
-        continue
-
-# Open the HDF5 file in a separate window
-if sys.platform.startswith("win32"):
-    os.startfile('product1.h5')
-
-# Start receiving messages (async version)
-#flow_control = pubsub_v1.types.FlowControl(max_messages = 1)
-#streaming_pull_future = subscriber.subscribe(subscription_path, 
-#    callback=callback, 
-#    flow_control=flow_control)
-
-# print(f'Started receiving messages from subscription {subscription_path}...')
-
-# Wrap subscriber in a 'with' block to automatically call close() when done.
-# with subscriber:
-#    try:
-        # When `timeout` is not set, result() will block indefinitely,
-        # unless an exception is encountered first.
-#        streaming_pull_future.result(timeout=timeout)
-#    except TimeoutError:
-#        streaming_pull_future.cancel()  # Trigger the shutdown.
-#        streaming_pull_future.result()  # Block until the shutdown is complete.
-
-# Keep the script running indefinitely to continue receiving messages
-#while True:
-#    time.sleep(10)

fp2_12.py

@@ -1,199 +0,0 @@
-from google.cloud import pubsub_v1
-from google.api_core.exceptions import DeadlineExceeded
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.common.by import By
-import json
-import time
-from concurrent.futures import TimeoutError
-import pandas as pd
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-
-# Create an empty dataframe to store the prices
-df = pd.DataFrame(columns=['Zip Code', 'Price'])
-print(f"Created a dataframe: {df}")
-
-# Configure the Selenium driver options
-options = Options()
-options.add_argument('--disable-gpu')
-driver = webdriver.Chrome(options=options)
-
-# Set up Pub/Sub subscription
-project_id = "fp-20230425"
-subscription_name = "fp-subscription-NFL-zipcodes"
-subscriber = pubsub_v1.SubscriberClient()
-subscription_path = subscriber.subscription_path(project_id, subscription_name)
-
-# Load the product page and extract the __NEXT_DATA__ data
-url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
-driver.delete_all_cookies
-driver.get(url)
-# time.sleep(5)
-
-# time.sleep(2)
-# Close the cookies setting window
-try:  
-    # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")
-    cookie_button = WebDriverWait(driver, 2).until(
-        EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
-    )
-    print(f"cookie_button = {cookie_button}")
-    cookie_button.click()
-except:
-    pass
-
-# time.sleep(1)
-# Clear any modal dialog
-try:    
-    # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
-    modal_close = WebDriverWait(driver, 2).until(
-        EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
-    )
-    print(f"modal_close = {modal_close}")
-    modal_close.click()
-    #time.sleep(2)
-except:
-    pass
-
-# Define callback function for Pub/Sub messages
-def callback(message):
-    # Extract zip code from message data
-    zip_code = message.message.data.decode('utf-8')
-
-    print(f"Received zip code: {zip_code}")
-
-    try:
-        try:
-            # time.sleep(2)
-            # Clear any modal dialog
-            # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
-            modal_close = WebDriverWait(driver, 2).until(
-            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
-        )
-            # print(f"modal_close = {modal_close}")
-            modal_close.click()
-            # time.sleep(2)
-        except:
-            pass
-
-
-        try:
-            # time.sleep(1)
-            # Clear any notice
-            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
-            notice_close = WebDriverWait(driver, 5).until(
-            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
-        )
-            # print(f"notice_close = {notice_close}")
-            notice_close.click()
-            # time.sleep(1)
-        except:
-            pass
-        
-        # Enter zip code in the input field and click the "Check" button
-        # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
-        zip_button = WebDriverWait(driver, 5).until(
-            EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
-        )
-        # print(f"{zip_button:}")
-        zip_button.click()
-        # time.sleep(1)
-
-        # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
-        zip_input = WebDriverWait(driver, 2).until(
-            EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
-        )
-        # print(f"{zip_input:}")
-        zip_input.click()
-        # print("clearing zip_input")
-        zip_input.send_keys("\b\b\b\b\b")
-        # print("sending zip code")
-        zip_input.send_keys(zip_code)
-
-        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
-        check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
-        # print(f"{check_button:}")
-        check_button.click()
-        # time.sleep(2)
-
-        try:
-            # time.sleep(2)
-            # Clear any notice
-            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
-            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
-            # print(f"notice_close = {notice_close}")
-            notice_close.click()
-            #time.sleep(2)
-        except:
-            pass
-
-
-    # Extract the price of rice crackers for this zip code
-        
-        # next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
-        next_data_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, '//*[@id="__next"]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')))
-        next_data_text = next_data_element.get_attribute('nodeValue')
-        price = next_data_text.text
-        '''
-        next_data_text = next_data_element.get_attribute('innerHTML')
-        next_data_start_index = next_data_text.index('{')
-        next_data_end_index = next_data_text.rindex('}') + 1
-        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
-        next_data = json.loads(next_data_json)
-        price = next_data['price']
-        '''
-        # Print the price
-        print(price)
-
-        '''
-        try:
-            min_quantity_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, '//script[@id="__NEXT_DATA__"]')))
-            min_quantity_text = min_quantity_element.get_attribute('innerHTML')
-            min_quantity_start_index = min_quantity_text.index('{')
-            min_quantity_end_index = min_quantity_text.rindex('}') + 1
-            min_quantity_json = min_quantity_text[min_quantity_start_index:min_quantity_end_index]
-            min_quantity = json.loads(min_quantity_json)
-            price = min_quantity['product']['price']
-        except:
-            next_data_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//script[@type='application/ld+json']")))
-            next_data_text = next_data_element.get_attribute('innerHTML')
-            next_data_start_index = next_data_text.index('{')
-            next_data_end_index = next_data_text.rindex('}') + 1
-            next_data_json = next_data_text[next_data_start_index:next_data_end_index]
-            next_data = json.loads(next_data_json)        
-            price = next_data['offers']['price']
-        finally:
-            # Print the price
-            print(price)
-        '''
-
-        subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
-        # Append the zip code and price to the dataframe
-        #append_to_df(zip_code, price)
-        new_row = pd.DataFrame({'Zip Code': [zip_code], 'Price': [price]})
-        return pd.concat([df, new_row], ignore_index=True)
-        #print(f"Recorded the zip code and price: {zip_code},{price}")
-        #print(f"Updated a dataframe: {df}")
-
-    except Exception as e:
-        print(f"Caught exception: {e}")
-        print(f"Price not found for zip code {zip_code}")
-        price = 'Price not found'
-        return df
-
-# Start receiving messages (sync version)
-while True: 
-    pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
-    messages = []
-    try:
-        pull_response = subscriber.pull(request = pull_request)
-        messages = pull_response.received_messages
-    except DeadlineExceeded:
-        pass
-
-    if messages:
-        df = callback(messages[0])
-        print(f"Updated a dataframe: {df}")
-    else:
-        print(f"no messages, trying again at {time.asctime()}")

fp2_13.py

@@ -1,271 +0,0 @@
-from google.cloud import pubsub_v1
-from google.api_core.exceptions import DeadlineExceeded
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.common.by import By
-import json
-import time
-import os
-from concurrent.futures import TimeoutError
-import pandas as pd
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-import selenium.common.exceptions
-import re
-
-# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
-
-# Create an empty dataframe to store the prices
-df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
-print(f"Created a dataframe: {df}")
-
-# Configure the Selenium driver options
-options = Options()
-options.add_argument('--disable-gpu')
-driver = webdriver.Chrome(options=options)
-
-# Set up Pub/Sub subscription
-project_id = "fp-20230425"
-subscription_name = "fp-subscription-NFL-zipcodes"
-subscriber = pubsub_v1.SubscriberClient()
-subscription_path = subscriber.subscription_path(project_id, subscription_name)
-
-# Load the product page and extract the __NEXT_DATA__ data
-url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
-product_id = url.split("/")[-1][:5]
-# product_id = url.split("/")[-1]
-driver.delete_all_cookies
-driver.get(url)
-# time.sleep(5)
-
-def get_element_text(class_name_prefix):
-    '''
-    Return the text in the first element with a given class name.
-
-    Note that XPath doesn't have a function to check if a given word is present
-    in a list of words, so using contains() will match a class name even if
-    the class name is a substring of the class name being matched against.
-    To prevent this, we use spaces as a delimiter at the beginning and end of
-    the class name, and ensure there's only one space before and after all
-    class names in the class name list.
-
-    Example: consider two class names with cache-busting suffixes:
-      * Header_soldStatusNotice__12345
-      * Header_soldStatusNoticeBackground__ABCDE
-
-    Matching against Header_soldStatusNotice using a naive implementation with
-    `contains()` would match both class names, even though only the first one
-    is the correct match. 
-    '''
-    # Construct xpath selector.  
-    xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
-    xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
-    xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
-    xpath_str += "' '), "                     # Step 4: a trailing space...
-    xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
-    try:
-        match = driver.find_element(By.XPATH, xpath_str)
-        return match.get_attribute('textContent')
-    except selenium.common.exceptions.NoSuchElementException as nsee:
-        pass
-
-    # We didn't find a match, try again by doing a prefix match and assume
-    # the full class name ends with two underscores followed by a cache-
-    # busting string.    
-    xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
-    xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
-    xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
-    xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
-    try:
-        match = driver.find_element(By.XPATH, xpath_str)
-        return match.get_attribute('textContent')
-    except selenium.common.exceptions.NoSuchElementException as nsee:
-        pass
-
-    # No match, return empty string.
-    return ''
-
-
-# time.sleep(2)
-# Close the cookies setting window
-try:  
-    # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")
-    cookie_button = WebDriverWait(driver, 2).until(
-        EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
-    )
-    print(f"cookie_button = {cookie_button}")
-    cookie_button.click()
-except:
-    pass
-
-# time.sleep(1)
-# Clear any modal dialog
-try:    
-    # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
-    modal_close = WebDriverWait(driver, 2).until(
-        EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
-    )
-    print(f"modal_close = {modal_close}")
-    modal_close.click()
-    #time.sleep(2)
-except:
-    pass
-
-# Define callback function for Pub/Sub messages
-def callback(message):
-    # Extract zip code from message data
-    zip_code = message.message.data.decode('utf-8')
-
-    print(f"Received zip code: {zip_code}")
-
-    try:
-        try:
-            # time.sleep(2)
-            # Clear any modal dialog
-            # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
-            modal_close = WebDriverWait(driver, 2).until(
-            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
-        )
-            # print(f"modal_close = {modal_close}")
-            modal_close.click()
-            # time.sleep(2)
-        except:
-            pass
-
-
-        try:
-            # time.sleep(1)
-            # Clear any notice
-            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
-            notice_close = WebDriverWait(driver, 5).until(
-            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
-        )
-            # print(f"notice_close = {notice_close}")
-            notice_close.click()
-            # time.sleep(1)
-        except:
-            pass
-        
-        # Enter zip code in the input field and click the "Check" button
-        # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
-        zip_button = WebDriverWait(driver, 5).until(
-            EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
-        )
-        # print(f"{zip_button:}")
-        zip_button.click()
-        # time.sleep(1)
-
-        # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
-        zip_input = WebDriverWait(driver, 2).until(
-            EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
-        )
-        # print(f"{zip_input:}")
-        zip_input.click()
-        # print("clearing zip_input")
-        zip_input.send_keys("\b\b\b\b\b")
-        # print("sending zip code")
-        zip_input.send_keys(zip_code)
-
-        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
-        check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
-        # print(f"{check_button:}")
-        check_button.click()
-        # time.sleep(2)
-
-        try:
-            # time.sleep(2)
-            # Clear any notice
-            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
-            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
-            # print(f"notice_close = {notice_close}")
-            notice_close.click()
-            #time.sleep(2)
-        except:
-            pass
-
-        try:
-            # Clear any modal dialog
-            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
-            # print(f"modal_close = {modal_close}")
-            modal_close.click()
-        except:
-            pass
-
-        # Record sales count, if any
-        sales_count_element = get_element_text('Header_count')
-        count_start_index = sales_count_element.index('sold ') + len('sold ')
-        count_end_index = sales_count_element.index('+')
-        sales_count = sales_count_element[count_start_index:count_end_index]
-        # Record any sold status notice
-        sold_status = get_element_text('UnavailableCartbtn_container')
-        if sold_status == 'Notify me':
-            sold_status = 'Sold Out'
-
-    # Extract the price of rice crackers for this zip code
-        # Check if there is a minimum order quantity
-        min_quantity: float = 0
-        total_price: float = 0
-        unit_price: float = 0
-        try:
-            minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML')
-            if minimum_quantity_str:
-                regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
-                match = re.search(regexp,minimum_quantity_str)
-                min_quantity, total_price = match.groups()
-        except selenium.common.exceptions.NoSuchElementException as nsee:
-            print("No minimum purhcase string found")
-        except Exception as e:
-            print(f"Exception when finding minimum purchase string: {e}")
- 
-
-
-        ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
-        ld_json_text = ld_json_element.get_attribute('innerHTML')
-        ld_json_start_index = ld_json_text.index('{')
-        ld_json_end_index = ld_json_text.rindex('}') + 1
-        ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
-        ld_json = json.loads(ld_json_json)
-        ld_json_price = ld_json['offers']['price']
-        # Print the price
-        print(ld_json_price)
-
-        unit_price_element = driver.find_element(By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText')
-        unit_price = unit_price_element.replace("$", "")
-        print(f"unit_price = {unit_price}" )
-
-        subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
-        # Append the zip code and price to the dataframe
-        #append_to_df(zip_code, price)
-        new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]})
-        return pd.concat([df, new_row], ignore_index=True)
-        #print(f"Recorded the zip code and price: {zip_code},{price}")
-        #print(f"Updated a dataframe: {df}")
-
-    except Exception as e:
-        print(f"Caught exception: {e}")
-        print(f"Price not found for zip code {zip_code}")
-        price = 'Price not found'
-        return df
-
-# Start receiving messages (sync version)
-while True: 
-    pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
-    messages = []
-    try:
-        pull_response = subscriber.pull(request = pull_request)
-        messages = pull_response.received_messages
-    except DeadlineExceeded:
-        pass
-
-    if messages:
-        df = callback(messages[0])
-        print(f"Updated a dataframe: {df}")
-    else:
-        print(f"no messages, trying again at {time.asctime()}")
-        break
-
-# df.to_hdf('product1.h5', key='df',mode='w')
-# df.to_hdf('product1.h5', key='df',mode='a')
-
-# Open the HDF5 file in a separate window
-# os.startfile('product1.h5')

fp2_15_Sloetjes.py

@@ -0,0 +1,277 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import selenium.common.exceptions
+import re
+from datetime import datetime
+import arrow
+import subprocess
+import concurrent.futures
+
+# This script is used to load the new price and the timestamp and append such data to an existing key in the HDF5 file "product1.h5"
+
+def get_element_text(driver, class_name_prefix):
+    '''
+    Return the text in the first element with a given class name.
+
+    Note that XPath doesn't have a function to check if a given word is present
+    in a list of words, so using contains() will match a class name even if
+    the class name is a substring of the class name being matched against.
+    To prevent this, we use spaces as a delimiter at the beginning and end of
+    the class name, and ensure there's only one space before and after all
+    class names in the class name list.
+
+    Example: consider two class names with cache-busting suffixes:
+    * Header_soldStatusNotice__12345
+    * Header_soldStatusNoticeBackground__ABCDE
+
+    Matching against Header_soldStatusNotice using a naive implementation with
+    `contains()` would match both class names, even though only the first one
+    is the correct match.
+    '''
+    # Construct xpath selector.
+    xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+    xpath_str += "' '), "                     # Step 4: a trailing space...
+    xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # We didn't find a match, try again by doing a prefix match and assume
+    # the full class name ends with two underscores followed by a cache-
+    # busting string.
+    xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+    xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # No match, return empty string.
+    return ''
+
+
+def scrape_product_price(project_id, subscription_name, url):
+    # Create an empty dataframe to store the prices
+    df_new = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
+    print(f"Created a dataframe: {df_new}")
+
+    # Configure the Selenium driver options
+    options = Options()
+    options.add_argument('--disable-gpu')
+    driver = webdriver.Chrome(options=options)
+
+    # Set up Pub/Sub subscription
+    subscriber = pubsub_v1.SubscriberClient()
+    subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+    # Load the product page and extract the __NEXT_DATA__ data
+    product_id = url.split("/")[-1][:5]
+    driver.delete_all_cookies()
+    driver.get(url)
+
+    # Close the cookies setting window
+    try:
+        cookie_button = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+        )
+        print(f"cookie_button = {cookie_button}")
+        cookie_button.click()
+    except:
+        pass
+
+    # Clear any modal dialog
+    try:
+        modal_close = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+        )
+        print(f"modal_close = {modal_close}")
+        modal_close.click()
+    except:
+        pass
+
+    def callback(message):
+        # Extract zip code from message data
+        zip_code = message.message.data.decode('utf-8')
+        print(f"Received zip code: {zip_code}")
+
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+            )
+            print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+        try:
+            # Clear any notice
+            notice_close = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
+            )
+            print(f"notice_close = {notice_close}")
+            notice_close.click()
+        except:
+            pass
+
+        # Enter zip code in the input field and click the "Check" button
+        zip_button = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+        )
+        zip_button.click()
+
+        zip_input = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+        )
+        zip_input.click()
+        zip_input.send_keys("\b\b\b\b\b")
+        zip_input.send_keys(zip_code)
+
+        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+        check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//button[text() = 'Confirm']")))
+        check_button.click()
+
+        # Record sales count, if any
+        sales_count_element = get_element_text(driver, 'Header_count')
+        count_start_index = sales_count_element.index('sold ') + len('sold ')
+        count_end_index = sales_count_element.index('+')
+        sales_count = sales_count_element[count_start_index:count_end_index]
+
+        # Record any sold status notice
+        sold_status = get_element_text(driver, 'UnavailableCartbtn_container')
+        if sold_status == 'Notify me':
+            sold_status = 'Sold Out'
+
+        # Extract the price of the product for this zip code
+        min_quantity = 0
+        total_price = 0
+        unit_price = 0
+        try:
+            minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]")
+            if minimum_quantity_str:
+                regexp = r"Minimum purchase: (\d+) \(\$([\d.]+)"
+                match = re.search(regexp, minimum_quantity_str.get_attribute('innerHTML'))
+                if match:
+                    min_quantity, total_price = match.groups()
+        except selenium.common.exceptions.NoSuchElementException:
+            print("No minimum purchase string found")
+        except Exception as e:
+            print(f"Exception when finding minimum purchase string: {e}")
+
+        ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        ld_json_text = ld_json_element.get_attribute('innerHTML')
+        ld_json_start_index = ld_json_text.index('{')
+        ld_json_end_index = ld_json_text.rindex('}') + 1
+        ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
+        ld_json = json.loads(ld_json_json)
+        ld_json_price = ld_json['offers']['price']
+        print(ld_json_price)
+
+        unit_price_element = driver.find_element(By.XPATH, "//div[contains(@class, 'price_price')]")
+        unit_price = unit_price_element.get_attribute('innerText').replace("$", "")
+        print(f"unit_price = {unit_price}")
+
+        subscriber.acknowledge(ack_ids=[message.ack_id], subscription=subscription_path)  # acknowledge receipt of message
+
+        new_row = pd.DataFrame({
+            'Product ID': [product_id],
+            'Zip Code': [zip_code],
+            'Shown Price': [ld_json_price],
+            'Minimum Quantity': [min_quantity],
+            'Total Price': [total_price],
+            'Unit Price': [unit_price],
+            'Sales Count': [sales_count],
+            'Sold Status': [sold_status]
+        })
+        return pd.concat([df_new, new_row], ignore_index=True)
+
+    while True:
+        pull_request = pubsub_v1.types.PullRequest(max_messages=1, subscription=subscription_path)
+        messages = []
+        try:
+            pull_response = subscriber.pull(request=pull_request)
+            messages = pull_response.received_messages
+        except DeadlineExceeded:
+            pass
+
+        if messages:
+            df_new = callback(messages[0])
+            print(f"Updated dataframe: {df_new}")
+        else:
+            print(f"No messages, trying again at {time.asctime()}")
+            break
+
+    return df_new
+
+
+def scrape_multiple_products(project_id, subscription_name, url_list):
+    df_list = []
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        futures = []
+        for url in url_list:
+            future = executor.submit(scrape_product_price, project_id, subscription_name, url)
+            futures.append(future)
+
+        for future in concurrent.futures.as_completed(futures):
+            df = future.result()
+            df_list.append(df)
+            print("Scraped data for URL:", url)
+
+    return df_list
+
+
+if __name__ == '__main__':
+    project_id = input("Enter the project ID: ")
+    subscription_name = input("Enter the subscription name: ")
+    url_list = [
+        'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
+        'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
+        'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
+        'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
+        'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
+        'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
+        'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
+        'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
+        'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
+        'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
+        'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
+    ]
+
+    df_list = scrape_multiple_products(project_id, subscription_name, url_list)
+
+    # Concatenate all dataframes in the list
+    df_combined = pd.concat(df_list, ignore_index=True)
+
+    # Save the combined dataframe to HDF5 file
+    with pd.HDFStore('product1.h5') as store:
+        store.append('key', df_combined, format='table')
+        
+    # Print the combined dataframe
+    print("Combined DataFrame:")
+    print(df_combined)
+    
+    # Save the combined dataframe to a CSV file
+    df_combined.to_csv('product1.csv', index=False)
+
+    print("Data scraping and storage completed.")
+

fp2_2.py

@@ -1,32 +0,0 @@
-import http.client
-import xml.etree.ElementTree as ET
-
-conn = http.client.HTTPSConnection("www.sayweee.com")
-payload = ''
-headers = {}
-conn.request("GET", "/en/product/Pete-and-Gerry-s-Organic-Large-Brown-Eggs/65902?category=dairy02", payload, headers)
-res = conn.getresponse()
-data = res.read()
-
-# Parse the HTML page content using ElementTree
-root = ET.fromstring(data.decode('utf-8'))
-
-if root is not None:
-    # Find the script element with __NEXT_DATA__ data
-    next_data_element = None
-    for script_element in root.iter('script'):
-        if script_element.text is not None and '__NEXT_DATA__' in script_element.text:
-            next_data_element = script_element
-            break
-
-    # Extract the __NEXT_DATA__ data
-    if next_data_element is not None:
-        next_data_text = next_data_element.txt
-        next_data_start_index = next_data_text.index('{')
-        next_data_end_index = next_data_text.rindex('}') + 1
-        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
-        print(next_data_json)
-    else:
-        print('__NEXT_DATA__ element not found')
-else:
-    print('Failed to parse HTML content')

fp2_3.py

@@ -1,32 +0,0 @@
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-import json
-
-# Configure the Selenium driver options
-options = Options()
-options.add_argument('--headless')
-options.add_argument('--disable-gpu')
-driver = webdriver.Chrome(options=options)
-
-# Load the page and extract the __NEXT_DATA__ data
-url = 'https://www.sayweee.com/en/product/Pete-and-Gerry-s-Organic-Large-Brown-Eggs/65902?category=dairy02'
-driver.get(url)
-wait = WebDriverWait(driver, 10)
-next_data_element = wait.until(EC.presence_of_element_located((By.XPATH, '//script[contains(text(), "__NEXT_DATA__")]')))
-next_data_text = next_data_element.text
-next_data_start_index = next_data_text.index('{')
-next_data_end_index = next_data_text.rindex('}') + 1
-next_data_json = next_data_text[next_data_start_index:next_data_end_index]
-next_data = json.loads(next_data_json)
-
-# Extract the price from the __NEXT_DATA__ data
-price = next_data['props']['pageProps']['product']['price']
-
-# Close the Selenium driver
-driver.quit()
-
-# Print the price
-print(price)

fp2_4.py

@@ -1,43 +0,0 @@
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.common.by import By
-import json
-import time
-
-# Configure the Selenium driver options
-options = Options()
-#options.add_argument('--headless')  
-options.add_argument('--disable-gpu')
-driver = webdriver.Chrome(options=options)
-
-# Load the page and extract the __NEXT_DATA__ data
-url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
-driver.get(url)
-time.sleep(5)
-next_data_element = None
-#try:
-next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
-#except:
- #   print('__NEXT_DATA__ element not found')
-
-#time.sleep(1000)
-if next_data_element is not None:
-    next_data_text = next_data_element.get_attribute('innerHTML')
-    print(next_data_text)
-    next_data_start_index = next_data_text.index('{')
-    next_data_end_index = next_data_text.rindex('}') + 1
-    next_data_json = next_data_text[next_data_start_index:next_data_end_index]
-    next_data = json.loads(next_data_json)
-
-    # Extract the price from the __NEXT_DATA__ data
- #   price = next_data['props']['pageProps']['product']['price']
-    print(next_data)
-    price = next_data['offers']['price']
-    # Print the price
-    print(price)
-else:
-    print('__NEXT_DATA__ element not found')
-
-time.sleep(1000)
-# Close the Selenium driver
-#driver.quit()

fp2_5.py

@@ -1,33 +0,0 @@
-import re
-import json
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.common.by import By
-
-# Configure the Selenium driver options
-options = Options()
-options.add_argument('--headless')
-options.add_argument('--disable-gpu')
-driver = webdriver.Chrome(options=options)
-
-# Load the page and extract the __NEXT_DATA__ data
-url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
-driver.get(url)
-next_data_element = None
-for script_element in driver.find_elements(By.XPATH, '//script'):
-    script_text = script_element.get_attribute('innerHTML')
-    match = re.search(r'__NEXT_DATA__ = (\{.*\});', script_text)
-    if match:
-        next_data_json = match.group(1)
-        next_data = json.loads(next_data_json)
-        # Extract the price from the __NEXT_DATA__ data
-        price = next_data['props']['pageProps']['product']['price']
-        # Print the price
-        print(price)
-        break
-
-if next_data_element is None:
-    print('__NEXT_DATA__ element not found')
-
-# Close the Selenium driver
-driver.quit()

fp2_6.py

@@ -1,126 +0,0 @@
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.common.by import By
-import json
-import time
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-
-# Configure the Selenium driver options
-options = Options()
-options.add_argument('--disable-gpu')
-driver = webdriver.Chrome(options=options)
-
-# Define the zip codes for which to extract the price
-zip_codes = ['85305', '30313', '21230', '14127','28202','60605','45202','44114','76011','80204','48226','54304','77054','46225','32202','64129','89118','90301','33056','55415','02035','70112','07073','19148','15212','95054','98134','33607','37213','20785']
-
-# Load the page and extract the __NEXT_DATA__ data
-#url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
-url = 'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261'
-#url = 'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467'
-#url = 'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01'
-#url = 'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224'
-#url = 'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02'
-#url = 'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01'
-#url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
-#url = 'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353?trace_id=7847dd6e-c511-48e2-a76e-a283399967b1'
-#url = 'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03'
-
-driver.delete_all_cookies
-driver.get(url)
-
-# Close the cookies setting window
-try:  
-    cookie_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")))
-    print(f"cookie_button = {cookie_button}")
-    cookie_button.click()
-except:
-    pass
-
-# Clear any modal dialog
-try:    
-    modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
-    print(f"modal_close = {modal_close}")
-    modal_close.click()
-except:
-    pass
-
-def callback(zip_code):
-    try:
-        try:
-            # Clear any modal dialog
-            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
-            # print(f"modal_close = {modal_close}")
-            modal_close.click()
-        except:
-            pass
-
-
-        try:
-            # Clear any notice
-            notice_close = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
-            # print(f"notice_close = {notice_close}")
-            notice_close.click()
-        except:
-            pass
-
-        try:
-            # Clear any modal dialog
-            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
-            # print(f"modal_close = {modal_close}")
-            modal_close.click()
-        except:
-            pass
-
-        # Enter zip code in the input field and click the "Check" button
-        zip_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')))
-        # print(f"{zip_button:}")
-        zip_button.click()
-
-        zip_input = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input')))
-        # print(f"{zip_input:}")
-        zip_input.click()
-        # print("clearing zip_input")
-        zip_input.send_keys("\b\b\b\b\b")
-        # print("sending zip code")
-        zip_input.send_keys(zip_code)
-
-        try:
-            check_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//button[text() = 'Confirm']")))
-            # print(f"{check_button:}")
-        except:
-            check_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[5]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[3]/div/button")))
-        finally:
-            check_button.click()
-
-        try:
-            # Clear any notice
-            notice_close = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
-            # print(f"notice_close = {notice_close}")
-            notice_close.click()
-        except:
-            pass
-
-        try:
-            # Clear any modal dialog
-            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
-            # print(f"modal_close = {modal_close}")
-            modal_close.click()
-        except:
-            pass
-
-            # Extract the price of rice crackers for this zip code
-        try:
-            sold_out_notice = driver.find_element(By.XPATH, '//*[@id="soldStatusNotice"]')
-            print(f"No such product in this area = {zip_code, sold_out_notice}")
-        except:
-            print(f"The product is available in this area {zip_code}")
-    
-    except Exception as e:
-        print(f"Caught exception: {e}")
-        print(f"Price not found for zip code {zip_code}")
-
-for zip_code in zip_codes:
-    callback(zip_code)
-
-driver.quit()

fp2_7.py

@@ -1,64 +0,0 @@
-from google.cloud import pubsub_v1
-import csv
-import os
-import asyncio
-
-# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
-# project_id = "fp-20230425"
-project_id = "svig-384801"
-
-'''
-topic_name = "fp-topic-NFL-zipcodes"
-subscription_name = "fp-subscription-NFL-zipcodes"
-subscriber = pubsub_v1.SubscriberClient()
-subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
-subscriber.delete_subscription(request={"subscription": subscription_path})
-print("Subscription deleted: {}".format(subscription_path))
-
-# Delete the topic
-publisher = pubsub_v1.PublisherClient()
-topic_path = publisher.topic_path(project_id, topic_name)
-publisher.delete_topic(request={"topic": topic_path})
-print(f"Topic '{topic_name}' deleted.")
-
-'''
-
-
-# Create a pubsub topic
-publisher = pubsub_v1.PublisherClient()
-topic_name = "fp-topic-NFL-zipcodes"
-topic_path = publisher.topic_path(project_id, topic_name)
-topic = publisher.create_topic(request={"name": topic_path}, timeout=30)
-
-
-# Create a Pub/Sub subscription
-subscriber = pubsub_v1.SubscriberClient()
-subscription_name = "fp-subscription-NFL-zipcodes"
-subscription_path = subscriber.subscription_path(project_id, subscription_name)
-subscription = subscriber.create_subscription(
-    name=subscription_path, topic=topic_path
-)
-'''
-# Publish a message to the topic
-# Read the data from the csv file
-# file_path = "C:/Users/Karen/Desktop/NFL_Cities.csv"
-file_path = zipcodes.csv
-
-with open(file_path, "r") as csv_file:
-    reader = csv.reader(csv_file)
-    next(reader)  # skip header row
-    for row in reader:
-        # Format the data as a string and publish it to the topic
-        data = ",".join(row).encode("utf-8")
-        future = publisher.publish(topic_path, data)
-        print(f"Published message ID {future.result()}")
-
-
-async def receive_messages():
-    response = subscriber.pull(subscription=subscription_path, max_messages=1)
-    for received_message in response.received_messages:
-        print("Received message: {}".format(received_message.message.data))
-       
-    
-asyncio.run(receive_messages())
-'''

fp2_8.py

@@ -1,55 +0,0 @@
-import json
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.common.by import By
-import time
-from google.cloud import pubsub_v1
-import os
-
-os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
-
-options = Options()
-options.add_argument('--headless')
-options.add_argument('--disable-gpu')
-driver = webdriver.Chrome(options=options)
-
-# Set up Pub/Sub subscription
-project_id = "fp-20230425"
-subscription_name = "fp-subscription-zipcodes"
-subscriber = pubsub_v1.SubscriberClient()
-subscription_path = subscriber.subscription_path(project_id, subscription_name)
-
-def callback(message):
-    try:
-        # Get the zip code from the message
-        zip_code = message.data.decode('utf-8')
-        print(f"Received zip code: {zip_code}")
-        
-        # Load the webpage with the zip code
-        url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
-        driver.get(url)
-        driver.find_element(By.XPATH, '/html/body/div[4]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[2]/input').send_keys(zip_code)
-        driver.find_element(By.XPATH, '/html/body/div[4]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[3]/div').click()
-        
-        # Extract the price from the webpage
-        next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
-        next_data_text = next_data_element.get_attribute('innerHTML')
-        next_data_start_index = next_data_text.index('{')
-        next_data_end_index = next_data_text.rindex('}') + 1
-        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
-        next_data = json.loads(next_data_json)
-        price = next_data['offers']['price']
-        
-        # Print the result
-        print(f"Price of rice crackers in zip code {zip_code}: {price}")
-    except Exception as e:
-        print(f"Error processing message: {str(e)}")
-    finally:
-        message.ack()
-
-subscriber.subscribe(subscription_path, callback=callback)
-
-while True:
-    time.sleep(1)
-
-

fp2_9.py

@@ -1,106 +0,0 @@
-from google.cloud import pubsub_v1
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.common.by import By
-import json
-import time
-import os
-from concurrent.futures import TimeoutError
-
-# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
-
-# Configure the Selenium driver options
-options = Options()
-options.add_argument('--disable-gpu')
-driver = webdriver.Chrome(options=options)
-
-# Set up Pub/Sub subscription
-# project_id = "fp-20230425"
-project_id = "svig-384801"
-subscription_name = "fp-subscription-zipcodes"
-subscriber = pubsub_v1.SubscriberClient()
-subscription_path = subscriber.subscription_path(project_id, subscription_name)
-timeout = 5
-
-# Load the product page and extract the __NEXT_DATA__ data
-url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
-driver.get(url)
-# time.sleep(5)
-'''
-next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
-if next_data_element is not None:
-    next_data_text = next_data_element.get_attribute('innerHTML')
-    next_data_start_index = next_data_text.index('{')
-    next_data_end_index = next_data_text.rindex('}') + 1
-    next_data_json = next_data_text[next_data_start_index:next_data_end_index]
-    next_data = json.loads(next_data_json)
-else:
-    print('__NEXT_DATA__ element not found')
-'''
-    
-# Define callback function for Pub/Sub messages
-def callback(message):
-    # Extract zip code from message data
-    zip_code = message.data.decode('utf-8')
-    print(f"Received zip code: {zip_code}")
-
-    try:
-
-        time.sleep(5)
-        # Clear any modal dialog
-        modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
-        print(f"modal_close = {modal_close}")
-        modal_close.click()
-        time.sleep(2)
-
-        # Enter zip code in the input field and click the "Check" button
-        zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]/div/div/div[1]')
-        print(zip_button)
-        zip_button.click()
-        time.sleep(1)
-
-        zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
-        zip_input.clear()
-        zip_input.send_keys(zip_code)
-
-        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
-        check_button.click()
-        time.sleep(2)
-
-        # Extract the price of rice crackers for this zip code
-        '''
-        price_element = driver.find_element(By.XPATH, '/html/body/div[2]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')
-        price = price_element.text.strip()[1:]
-        print(price)
-        '''
-        next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
-        next_data_text = next_data_element.get_attribute('innerHTML')
-        next_data_start_index = next_data_text.index('{')
-        next_data_end_index = next_data_text.rindex('}') + 1
-        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
-        next_data = json.loads(next_data_json)
-        price = next_data['offers']['price']
-    # Print the price
-        print(price)
-        message.ack()  # acknowledge receipt of message
-    except:
-        print(f"Price not found for zip code {zip_code}")
-        price = 'Price not found'
-
-# Start receiving messages
-streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback)
-print(f'Started receiving messages from subscription {subscription_path}...')
-
-# Wrap subscriber in a 'with' block to automatically call close() when done.
-with subscriber:
-    try:
-        # When `timeout` is not set, result() will block indefinitely,
-        # unless an exception is encountered first.
-        streaming_pull_future.result(timeout=timeout)
-    except TimeoutError:
-        streaming_pull_future.cancel()  # Trigger the shutdown.
-        streaming_pull_future.result()  # Block until the shutdown is complete.
-
-# Keep the script running indefinitely to continue receiving messages
-while True:
-    time.sleep(10)

hello.py

@@ -1 +0,0 @@
-print("hello")

http-performance/Webpage.ipynb

@@ -0,0 +1,78 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0593e529",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from urllib.parse import urlparse\n",
+    "\n",
+    "class JPEGRequestHandler:\n",
+    "    def __init__(self, app):\n",
+    "        self.app = app\n",
+    "\n",
+    "    def __call__(self, environ, start_response):\n",
+    "        url = urlparse(environ['REQUEST_URI'])\n",
+    "        path = url.path\n",
+    "\n",
+    "        if path.endswith('.jpg'):\n",
+    "            file_path = path[1:]\n",
+    "\n",
+    "            if os.path.isfile(file_path):\n",
+    "                with open(file_path, 'rb') as file:\n",
+    "                    response_body = file.read()\n",
+    "                    status = '200 OK'\n",
+    "                    headers = [('Content-type', 'image/jpeg'),\n",
+    "                               ('Content-Length', str(len(response_body)))]\n",
+    "\n",
+    "                    start_response(status, headers)\n",
+    "                    return [response_body]\n",
+    "            else:\n",
+    "                status = '404 Not Found'\n",
+    "                headers = [('Content-type', 'text/plain'),\n",
+    "                           ('Content-Length', '0')]\n",
+    "\n",
+    "                start_response(status, headers)\n",
+    "                return []\n",
+    "        else:\n",
+    "            status = '403 Forbidden'\n",
+    "            headers = [('Content-type', 'text/plain'),\n",
+    "                       ('Content-Length', '0')]\n",
+    "\n",
+    "            start_response(status, headers)\n",
+    "            return []\n",
+    "\n",
+    "if __name__ == '__main__':\n",
+    "    from werkzeug.serving import run_simple\n",
+    "    from werkzeug.middleware.dispatcher import DispatcherMiddleware\n",
+    "\n",
+    "    app = DispatcherMiddleware(JPEGRequestHandler(app), {'/': app})\n",
+    "    run_simple('0.0.0.0', 8000, app, use_reloader=True)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

http-performance/app.py

@@ -0,0 +1,44 @@
+import os
+from urllib.parse import urlparse
+
+class JPEGRequestHandler:
+    def __init__(self, app):
+        self.app = app
+
+    def __call__(self, environ, start_response):
+        url = urlparse(environ['REQUEST_URI'])
+        path = url.path
+
+        if path.endswith('.jpg'):
+            file_path = path[1:]
+
+            if os.path.isfile(file_path):
+                with open(file_path, 'rb') as file:
+                    response_body = file.read()
+                    status = '200 OK'
+                    headers = [('Content-type', 'image/jpeg'),
+                               ('Content-Length', str(len(response_body)))]
+
+                    start_response(status, headers)
+                    return [response_body]
+            else:
+                status = '404 Not Found'
+                headers = [('Content-type', 'text/plain'),
+                           ('Content-Length', '0')]
+
+                start_response(status, headers)
+                return []
+        else:
+            status = '403 Forbidden'
+            headers = [('Content-type', 'text/plain'),
+                       ('Content-Length', '0')]
+
+            start_response(status, headers)
+            return []
+
+if __name__ == '__main__':
+    from werkzeug.serving import run_simple
+    from werkzeug.middleware.dispatcher import DispatcherMiddleware
+
+    app = DispatcherMiddleware(JPEGRequestHandler(app), {'/': app})
+    run_simple('0.0.0.0', 8000, app, use_reloader=True)

http-performance/testhh.py

@@ -1,54 +0,0 @@
-'''
-import os
-from flask import Flask, Response
-from urllib.parse import urlparse
-
-app = Flask(__name__)
-
-@app.route('/')
-def index():
-    return 'Hello, welcome to the cat picture server!'
-
-@app.route('/<path:path>')
-def serve_jpeg(path):
-    if path.endswith('.jpg'):
-        file_path = path[1:]
-        if os.path.isfile(file_path):
-            with open(file_path, 'rb') as file:
-                return Response(file.read(), content_type='image/jpeg')
-        else:
-            return 'File not found', 404
-    else:
-        return 'Only .jpg files are allowed', 403
-
-if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=8000)
-'''
-'''
-import os
-from fastapi import FastAPI
-from fastapi.responses import FileResponse
-from urllib.parse import urlparse
-
-app = FastAPI()
-
-@app.get('/')
-def index():
-    return 'Hello, welcome to the cat picture server!'
-
-@app.get('/{path:path}')
-async def serve_jpeg(path: str):
-    if path.endswith('.jpg'):
-        file_path = path[1:]
-        if os.path.isfile(file_path):
-            return FileResponse(file_path, media_type='image/jpeg')
-        else:
-            return {'detail': 'File not found'}, 404
-    else:
-        return {'detail': 'Only .jpg files are allowed'}, 403
-
-if __name__ == '__main__':
-    import uvicorn
-    uvicorn.run(app, host='0.0.0.0', port=8000)
-'''
-

hw10.py

@@ -1,14 +0,0 @@
-import sys
-
-# Iterate through the modules in sys.modules
-for module_name, module in sys.modules.items():
-    # Check if the module has a loader attribute
-    if hasattr(module, '__loader__'):
-        loader = module.__loader__
-        # Determine the type of loader
-        if type(loader).__name__ == 'SourceFileLoader':
-            print(f"Module '{module_name}' has a source loader.")
-        elif type(loader).__name__ == 'BuiltinImporter':
-            print(f"Module '{module_name}' has a built-in loader.")
-        elif type(loader).__name__ == 'ExtensionFileLoader':
-            print(f"Module '{module_name}' has an extension loader.")

hw11.py

@@ -1,12 +0,0 @@
-from cx_Freeze import setup, Executable
-
-executables = [Executable('module-inspector.py')]
-
-setup(
-    name='wow',
-    version='1.0',
-    description='Module-inspector Application',
-    executables=executables
-)
-
-

hw12.py

@@ -1,7 +0,0 @@
-i = 5
-
-def f(arg=i):
-    print(arg)
-
-i = 6
-f()

hw8.py

@@ -1,47 +0,0 @@
-import importlib.util
-import os
-
-def get_module_size(module_name):
-    """Get the total number of bytes in the cached .pyc file or .py file of a module or package.
-
-    Args:
-        module_name (str): The name of the module or package.
-
-    Returns:
-        int: The total number of bytes in the cached .pyc file or .py file.
-    """
-    spec = importlib.util.find_spec(module_name)
-
-    if spec is not None and spec.has_location:
-        # If the module is found and has a location, check for .pyc file first
-        pyc_file_path = spec.origin + 'c'
-        if os.path.exists(pyc_file_path):
-            return os.stat(pyc_file_path).st_size
-        else:
-            # If .pyc file doesn't exist, check for .py file
-            py_file_path = spec.origin
-            if os.path.exists(py_file_path):
-                return os.stat(py_file_path).st_size
-
-    return 0  # Return 0 if no .pyc or .py file found
-
-
-import sys
-
-def get_total_module_size():
-    """Get the total number of bytes for all modules listed in sys.modules.
-
-    Returns:
-        int: The total number of bytes for all modules.
-    """
-    total_size = 0
-    for module_name in sys.modules:
-        module_size = get_module_size(module_name)
-        total_size += module_size
-
-    return total_size
-
-if __name__ == '__main__' and __package__ is None:
-    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
-total_size = get_total_module_size()
-print("Total size of all modules in sys.modules: {} bytes".format(total_size))

hw9.py

@@ -1,74 +0,0 @@
-import os
-import sys
-import importlib.util
-import types
-from collections import defaultdict
-from sys import getsizeof
-
-
-def get_bytecode_size(module_name):
-    """Get the size of the cached .pyc bytecode file for a module."""
-    spec = importlib.util.find_spec(module_name)
-    if spec and spec.origin and spec.origin.endswith('.py'):
-        pyc_file = importlib.util.cache_from_source(spec.origin)
-        if os.path.exists(pyc_file):
-            return os.path.getsize(pyc_file)
-    return 0
-
-
-def get_dso_size(module_name):
-    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
-    spec = importlib.util.find_spec(module_name)
-    if spec and spec.origin and spec.origin.endswith(('.so', '.dll')):
-        return os.path.getsize(spec.origin)
-    return 0
-
-
-def generate_module_report():
-    """Generate a report summarizing the module environment."""
-    total_modules = len(sys.modules)
-    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
-    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
-    python_size = sum(
-        getsizeof(sys.modules[module_name]) - getsizeof(module_name)
-        for module_name in sys.modules
-        if isinstance(sys.modules[module_name], types.ModuleType)
-    )
-    total_size = bytecode_size + dso_size + python_size
-
-    # Calculate percentage of each type of module
-    bytecode_percent = bytecode_size / total_size * 100
-    dso_percent = dso_size / total_size * 100
-    python_percent = python_size / total_size * 100
-
-    # Generate the report
-    report = f"Total modules: {total_modules}\n"
-    report += f"Total size: {total_size} bytes\n"
-    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
-    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
-    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
-
-    # Generate histogram of module directory names
-    dir_histogram = defaultdict(int)
-    for module_name in sys.modules:
-        module_file = getattr(sys.modules[module_name], '__file__', None)
-        if module_file:
-            module_dir = os.path.dirname(module_file)
-            dir_histogram[module_dir] += 1
-
-    report += "\nModule Directory Histogram:\n"
-    for module_dir, count in dir_histogram.items():
-        report += f"{module_dir}: {'*' * count}\n"
-
-    return report
-
-
-if __name__ == "__main__":
-    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
-report = generate_module_report()
-print(report)
-
-
-
-
-

module-inspector/module-inspector.py

@@ -2,7 +2,6 @@ import sys
 import tkinter as tk
 from tkinter import ttk
 
-
 def on_treeview_heading_click(column, tree):
     data = [(tree.set(child, column), child) for child in tree.get_children("")]
     reverse = (column == tree.heading(column)["text"]) and not column.startswith("_")
@@ -39,10 +38,6 @@ tree.column('#0', width=20, stretch=False)
 
 tree.tag_configure("selected", background="#ccc")
 
-# Call the on_treeview_heading_click function when a column heading is clicked
-for col in tree["columns"]:
-    tree.heading(col, text=col, anchor=tk.W, command=lambda col=col: on_treeview_heading_click(col, tree))
-root.mainloop()
 # Call the on_treeview_heading_click function when a column heading is clicked
 for col in tree["columns"]:
     tree.heading(col, text=col, anchor=tk.W, command=lambda col=col: on_treeview_heading_click(col, tree))

mykey.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCbUrTPE/8CUByUtG9rBuMP5zeF3x5sZcPmm61bcxBblvHL5Dx1WwIjujhbZ5mhxnmDqxV07MCFZwMzlPjJ7Ow9erZvaXAe5+jS2iTQqu9apfj+U3x2xEZQmoKprXUpWl8G9YDk2wxBqCn//ugkJ5+erIEopNUGPcVhACucpaCT+kld9mHtXBjdUEKUA85HhXiBkuBsv9k/VqL/oXbvHjY8Z6gFP64LcxdNASuf4Qn1wW0w3kXlQRkSoGdSxpGLztXN1v5rvetOzjA8HXixh7TYVWCc+XCa9KsY3QRk7gFyvtIsXz59NMOTgfpCnUQE2Pk9R5wuuA2fXyr+Kg+7NPkd sloetjeswong@Sloetjess-Air

requirements.txt

@@ -1,6 +0,0 @@
-google.cloud.pubsub
-plotly
-pandas
-numpy
-h5py
-selenium

templates/index.html

@@ -0,0 +1,30 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Users Table</title>
+  </head>
+  <body>
+    <table>
+      <thead>
+        <tr>
+          <th>ID</th>
+          <th>Username</th>
+          <th>Credentials</th>
+          <th>Creation Date</th>
+          <th>Modification Date</th>
+        </tr>
+      </thead>
+      <tbody>
+        {% for user in users %}
+        <tr>
+          <td>{{ user.id }}</td>
+          <td>{{ user.username }}</td>
+          <td>{{ user.credentials }}</td>
+          <td>{{ user.creation_date }}</td>
+          <td>{{ user.modification_date }}</td>
+        </tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </body>
+</html>

test1.py

@@ -1,31 +0,0 @@
-class Count:
-    def __init__(self,mymin,mymax):
-        self.mymin=mymin
-        self.mymax=mymax    
-
-    def __getattr__(self, item):
-        self.__dict__[item]=0
-        return 0
-
-obj1 = Count(1,10)
-print(obj1.mymin)
-print(obj1.mymax)
-print(obj1.mycurrent1)
-
-class Count:
-
-    def __init__(self,mymin,mymax):
-        self.mymin=mymin
-        self.mymax=mymax
-        self.current=None
-   
-    def __getattribute__(self, item):
-        if item.startswith('cur'):
-            raise AttributeError
-        return object.__getattribute__(self,item) 
-        # or you can use ---return super().__getattribute__(item)
-
-obj1 = Count(1,10)
-print(obj1.mymin)
-print(obj1.mymax)
-print(obj1.current)

test10.py

@@ -1,14 +0,0 @@
-import sys
-
-# Iterate through the modules in sys.modules
-for module_name, module in sys.modules.items():
-    # Check if the module has a loader attribute
-    if hasattr(module, '__loader__'):
-        loader = module.__loader__
-        # Determine the type of loader
-        if type(loader).__name__ == 'SourceFileLoader':
-            print(f"Module '{module_name}' has a source loader.")
-        elif type(loader).__name__ == 'BuiltinImporter':
-            print(f"Module '{module_name}' has a built-in loader.")
-        elif type(loader).__name__ == 'ExtensionFileLoader':
-            print(f"Module '{module_name}' has an extension loader.")

test11.py

@@ -1,12 +0,0 @@
-from cx_Freeze import setup, Executable
-
-executables = [Executable('module-inspector.py')]
-
-setup(
-    name='wow',
-    version='1.0',
-    description='Module-inspector Application',
-    executables=executables
-)
-
-

test2.py

@@ -1,58 +0,0 @@
-import time
-'''import matplotlib.pyplot as plt'''
-
-# Durations to measure in seconds
-durations = [0.01, 0.1, 1.0]
-
-# Number of repetitions
-num_repetitions = 100
-
-# Lists to store jitter for each duration
-jitter_10ms = []
-jitter_100ms = []
-jitter_1s = []
-
-# Function to measure jitter
-def measure_jitter(duration):
-    start_time = time.perf_counter_ns()
-    time.sleep(duration)
-    end_time = time.perf_counter_ns()
-    return end_time - start_time
-
-# Measure jitter for each duration and repeat for num_repetitions times
-for duration in durations:
-    for _ in range(num_repetitions):
-        jitter = measure_jitter(duration)
-        if duration == 0.01:
-            jitter_10ms.append(jitter)
-        elif duration == 0.1:
-            jitter_100ms.append(jitter)
-        elif duration == 1.0:
-            jitter_1s.append(jitter)
-'''
-# Create histograms for each duration's jitter
-plt.hist(jitter_10ms, bins=10, color='r', alpha=0.5, label='10ms')
-plt.hist(jitter_100ms, bins=10, color='g', alpha=0.5, label='100ms')
-plt.hist(jitter_1s, bins=10, color='b', alpha=0.5, label='1s')
-plt.xlabel('Jitter (ns)')
-plt.ylabel('Frequency')
-plt.title('Jitter of time.sleep() for Different Durations')
-plt.legend()
-plt.show()
-'''
-
-import random
-import secrets
-
-# Create a dictionary with 1 million integer keys
-dict = {}
-
-for i in range(1000000):
-    # Generate a random 64-byte value as bytes
-    random_value = secrets.token_bytes(64)
-    # Convert the bytes value to hexadecimal representation
-    hex_value = random_value.hex()
-    # Add the key-value pair to the dictionary
-    dict[i] = hex_value
-
-print("Dictionary created with 1 million integer keys.")

test3.py

@@ -1,51 +0,0 @@
-
-class Vault:
-    def __init__(self):
-        self._treasure = "gold"
-        self._known_good_copy = self.__dict__.copy()
-
-    def __getattribute__(self, name):
-        if name == "treasure":
-            if self.__dict__ != self._known_good_copy:
-                print("Attribute deleted!")
-            return self._treasure
-        else:
-            return object.__getattribute__(self, name)
-
-    def __setattr__(self, name, value):
-        if name != "_treasure" and name != "_known_good_copy":
-            print("Attribute deletion detected!")
-            self._known_good_copy = self.__dict__.copy()
-        object.__setattr__(self, name, value)
-v = Vault()
-Vault.treasure = " haha monkeypatched"
-print(v.treasure)
-
-'''
-class Vault:
-    def __init__(self):
-        self._treasure = "gold"
-        self._known_good_copy = self.__dict__.copy()
-
-    def __get__(self, instance, owner):
-        if instance is None:
-            return self
-        else:
-            if instance.__dict__ != self._known_good_copy:
-                print("Attribute deleted!")
-            return self._treasure
-
-    def __set__(self, instance, value):
-        print("Attribute deletion detected!")
-        self._known_good_copy = instance.__dict__.copy()
-
-    def __delete__(self, instance):
-        print("Attribute deletion detected!")
-        self._known_good_copy = instance.__dict__.copy()
-
-    def __set_name__(self, owner, name):
-        self.name = name
-v = Vault()
-Vault._treasure = " haha monkeypatched"
-print(v._treasure)
-'''

test4.py

@@ -1,17 +0,0 @@
-import asyncio
-
-# Define a coroutine function
-async def greet(name):
-    print(f"Hello, {name}!")
-    await asyncio.sleep(1)
-    print(f"Goodbye, {name}!")
-
-# Call the coroutine function
-coroutine_obj = greet("Karen")
-
-# coroutine_obj is a coroutine object, not the actual execution of the coroutine
-
-# Use an event loop to run the coroutine
-loop = asyncio.get_event_loop()
-loop.run_until_complete(coroutine_obj)
-loop.close()

test5.py

@@ -1,8 +0,0 @@
-def think():
-    x=1
-    def think2():
-        nonlocal x
-        x=2
-    think2()
-    return x
-print(think())

test6.py

@@ -1,33 +0,0 @@
-'''
-import sys
-import importlib.util
-
-# Load only the sys module
-importlib.import_module('sys')
-
-# Check the type of the value associated with the 'os' key
-os_module = sys.modules['os']
-os_module_type = type(os_module)
-
-# Print the type of the value associated with the 'os' key
-print("Type of the value associated with the 'os' key in sys.modules: {}".format(os_module_type))
-'''
-'''
-import os
-print(type(os))
-'''
-
-import builtins
-
-# Access the os module from globals()
-os = globals().get('os', None)
-
-# Check if os module is available
-if os is not None:
-    # Call os.getlogin()
-    username = os.getlogin()
-    print("Username:", username)
-else:
-    print("os module not available")
-
-

test7.py

@@ -1,54 +0,0 @@
-import os
-import sys
-import importlib.util
-import types
-
-
-def get_bytecode_size(module_name):
-    """Get the size of the cached .pyc bytecode file for a module."""
-    spec = importlib.util.find_spec(module_name)
-    if spec and spec.origin.endswith('.py'):
-        pyc_file = importlib.util.cache_from_source(spec.origin)
-        if os.path.exists(pyc_file):
-            return os.path.getsize(pyc_file)
-    return 0
-
-
-def get_dso_size(module_name):
-    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
-    spec = importlib.util.find_spec(module_name)
-    if spec and spec.origin.endswith(('.so', '.dll')):
-        return os.path.getsize(spec.origin)
-    return 0
-
-
-def generate_module_report():
-    """Generate a report summarizing the module environment."""
-    total_modules = len(sys.modules)
-    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
-    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
-    python_size = sum(
-        sys.modules[module_name].__sizeof__()
-        for module_name in sys.modules
-        if isinstance(sys.modules[module_name], types.ModuleType)
-    )
-    total_size = bytecode_size + dso_size + python_size
-
-    # Calculate percentage of each type of module
-    bytecode_percent = bytecode_size / total_size * 100
-    dso_percent = dso_size / total_size * 100
-    python_percent = python_size / total_size * 100
-
-    # Generate the report
-    report = f"Total modules: {total_modules}\n"
-    report += f"Total size: {total_size} bytes\n"
-    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
-    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
-    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
-
-    return report
-
-if __name__ == '__main__' and __package__ is None:
-    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
-    report = generate_module_report()
-    print(report)

test8.py

@@ -1,259 +0,0 @@
-'''
-import os
-import sys
-import importlib.util
-import types
-import inspect
-from collections import defaultdict
-
-
-def get_bytecode_size(module_name):
-    """Get the size of the cached .pyc bytecode file for a module."""
-    spec = importlib.util.find_spec(module_name)
-    if spec and spec.origin.endswith('.py'):
-        pyc_file = importlib.util.cache_from_source(spec.origin)
-        if os.path.exists(pyc_file):
-            return os.path.getsize(pyc_file)
-    return 0
-
-
-def get_dso_size(module_name):
-    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
-    spec = importlib.util.find_spec(module_name)
-    if spec and spec.origin.endswith(('.so', '.dll')):
-        return os.path.getsize(spec.origin)
-    return 0
-
-
-def generate_module_report():
-    """Generate a report summarizing the module environment."""
-    total_modules = len(sys.modules)
-    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
-    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
-    python_size = sum(
-        sys.modules[module_name].__sizeof__()
-        for module_name in sys.modules
-        if isinstance(sys.modules[module_name], types.ModuleType)
-    )
-    total_size = bytecode_size + dso_size + python_size
-
-    # Calculate percentage of each type of module
-    bytecode_percent = bytecode_size / total_size * 100
-    dso_percent = dso_size / total_size * 100
-    python_percent = python_size / total_size * 100
-
-    # Generate the report
-    report = f"Total modules: {total_modules}\n"
-    report += f"Total size: {total_size} bytes\n"
-    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
-    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
-    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
-
-    # Generate histogram of module directory names
-    dir_histogram = defaultdict(int)
-    for module_name in sys.modules:
-        module_file = sys.modules[module_name].__file__
-        if module_file:
-            module_dir = os.path.dirname(module_file)
-            dir_histogram[module_dir] += 1
-
-    report += "\nModule Directory Histogram:\n"
-    for module_dir, count in dir_histogram.items():
-        report += f"{module_dir}: {'*' * count}\n"
-
-    return report
-
-
-if __name__ == '__main__' and __package__ is None:
-    script_path = inspect.getframeinfo(inspect.currentframe()).filename
-    __spec__ = importlib.util.spec_from_loader('__main__', None, origin=script_path)
-    report = generate_module_report()
-    print(report)
-'''
-'''
-import os
-import sys
-from importlib.util import find_spec
-
-def get_module_size(module):
-    spec = find_spec(module)
-    if spec is None:
-        return None
-    if spec.cached:
-        size = os.stat(spec.cached).st_size
-    else:
-        size = os.stat(spec.origin).st_size
-    return size
-
-def summarize_modules():
-    total_size = 0
-    byte_count = 0
-    dso_count = 0
-    py_count = 0
-    directory_histogram = {}
-    for module_name, module in sys.modules.items():
-        if module is None:
-            continue
-        size = get_module_size(module_name)
-        if size is None:
-            continue
-        total_size += size
-        if module_name.endswith('_bootstrap') or module_name.endswith('_bootstrap_external'):
-            continue
-        if module_name.startswith('_frozen_importlib'):
-            dso_count += 1
-            continue
-        if module_name.startswith('_'):
-            continue
-        if module.__file__.endswith('.pyc'):
-            byte_count += size
-        elif module.__file__.endswith(('.so', '.dll')):
-            dso_count += 1
-        else:
-            py_count += size
-        module_directory = os.path.dirname(os.path.abspath(module.__file__))
-        directory_histogram[module_directory] = directory_histogram.get(module_directory, 0) + 1
-
-    print(f"Total module size: {total_size} bytes")
-    print(f"Bytecode modules: {(byte_count/total_size)*100:.2f}% ({byte_count} bytes)")
-    print(f"DSO modules: {(dso_count/total_size)*100:.2f}% ({dso_count} modules)")
-    print(f"Python modules: {(py_count/total_size)*100:.2f}% ({py_count} bytes)")
-
-    print("Module directory histogram:")
-    for directory, count in directory_histogram.items():
-        print(f"{directory}: {'*' * count}")
-
-if __name__ == '__main__':
-    summarize_modules()
-'''
-'''
-import os
-import sys
-from importlib.util import find_spec
-
-
-def get_module_size(module):
-    spec = find_spec(module)
-    if spec is None:
-        return None
-    if spec.cached:
-        size = os.stat(spec.cached).st_size
-    else:
-        size = os.stat(spec.origin).st_size
-    return size
-
-
-def summarize_modules():
-    total_size = 0
-    byte_count = 0
-    dso_count = 0
-    py_count = 0
-    directory_histogram = {}
-    for module_name, module in sys.modules.items():
-        if module is None:
-            continue
-        size = get_module_size(module_name)
-        if size is None:
-            continue
-        total_size += size
-        if module_name.endswith('_bootstrap') or module_name.endswith('_bootstrap_external'):
-            continue
-        if module_name.startswith('_frozen_importlib'):
-            dso_count += 1
-            continue
-        if module_name.startswith('_'):
-            continue
-        if module.__file__.endswith('.pyc'):
-            byte_count += size
-        elif module.__file__.endswith(('.so', '.dll')):
-            dso_count += 1
-        else:
-            py_count += size
-        module_directory = os.path.dirname(os.path.abspath(module.__file__))
-        directory_histogram[module_directory] = directory_histogram.get(module_directory, 0) + 1
-
-    print(f"Total module size: {total_size} bytes")
-    print(f"Bytecode modules: {(byte_count/total_size)*100:.2f}% ({byte_count} bytes)")
-    print(f"DSO modules: {(dso_count/total_size)*100:.2f}% ({dso_count} modules)")
-    print(f"Python modules: {(py_count/total_size)*100:.2f}% ({py_count} bytes)")
-
-    print("Module directory histogram:")
-    for directory, count in directory_histogram.items():
-        print(f"{directory}: {'*' * count}")
-
-
-if __name__ == '__main__':
-    summarize_modules()
-'''
-
-import os
-import sys
-import importlib.util
-import types
-from collections import defaultdict
-
-
-def get_bytecode_size(module_name):
-    """Get the size of the cached .pyc bytecode file for a module."""
-    spec = importlib.util.find_spec(module_name)
-    if spec and spec.origin.endswith('.py'):
-        pyc_file = importlib.util.cache_from_source(spec.origin)
-        if os.path.exists(pyc_file):
-            return os.path.getsize(pyc_file)
-    return 0
-
-
-def get_dso_size(module_name):
-    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
-    spec = importlib.util.find_spec(module_name)
-    if spec and spec.origin.endswith(('.so', '.dll')):
-        return os.path.getsize(spec.origin)
-    return 0
-
-
-def generate_module_report():
-    """Generate a report summarizing the module environment."""
-    total_modules = len(sys.modules)
-    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
-    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
-    python_size = sum(
-        sys.modules[module_name].__sizeof__()
-        for module_name in sys.modules
-        if isinstance(sys.modules[module_name], types.ModuleType)
-    )
-    total_size = bytecode_size + dso_size + python_size
-
-    # Calculate percentage of each type of module
-    bytecode_percent = bytecode_size / total_size * 100
-    dso_percent = dso_size / total_size * 100
-    python_percent = python_size / total_size * 100
-
-    # Generate the report
-    report = f"Total modules: {total_modules}\n"
-    report += f"Total size: {total_size} bytes\n"
-    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
-    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
-    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
-
-    # Generate histogram of module directory names
-    dir_histogram = defaultdict(int)
-    for module_name in sys.modules:
-        module_file = sys.modules[module_name].__file__
-        if module_file:
-            module_dir = os.path.dirname(module_file)
-            dir_histogram[module_dir] += 1
-
-    report += "\nModule Directory Histogram:\n"
-    for module_dir, count in dir_histogram.items():
-        report += f"{module_dir}: {'*' * count}\n"
-
-    return report
-
-
-if __name__ == "__main__":
-    report = generate_module_report()
-    print(report)
-
-
-
-

test9.py

@@ -1,77 +0,0 @@
-import os
-import sys
-import importlib.util
-import types
-from collections import defaultdict
-from sys import getsizeof
-
-
-def get_bytecode_size(module_name):
-    """Get the size of the cached .pyc bytecode file for a module."""
-    spec = importlib.util.find_spec(module_name)
-    if spec and spec.origin and spec.origin.endswith('.py'):
-        pyc_file = importlib.util.cache_from_source(spec.origin)
-        if os.path.exists(pyc_file):
-            return os.path.getsize(pyc_file)
-    return 0
-
-
-def get_dso_size(module_name):
-    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
-    spec = importlib.util.find_spec(module_name)
-    if spec and spec.origin and spec.origin.endswith(('.so', '.dll')):
-        return os.path.getsize(spec.origin)
-    return 0
-
-
-def generate_module_report():
-    """Generate a report summarizing the module environment."""
-    total_modules = len(sys.modules)
-    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
-    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
-    python_size = sum(
-        getsizeof(sys.modules[module_name]) - getsizeof(module_name)
-        for module_name in sys.modules
-        if isinstance(sys.modules[module_name], types.ModuleType)
-    )
-    total_size = bytecode_size + dso_size + python_size
-
-    # Calculate percentage of each type of module
-    bytecode_percent = bytecode_size / total_size * 100
-    dso_percent = dso_size / total_size * 100
-    python_percent = python_size / total_size * 100
-
-    # Generate the report
-    report = f"Total modules: {total_modules}\n"
-    report += f"Total size: {total_size} bytes\n"
-    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
-    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
-    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
-
-    # Generate histogram of module directory names
-    dir_histogram = defaultdict(int)
-    for module_name in sys.modules:
-        module_file = getattr(sys.modules[module_name], '__file__', None)
-        if module_file:
-            module_dir = os.path.dirname(module_file)
-            dir_histogram[module_dir] += 1
-
-    report += "\nModule Directory Histogram:\n"
-    for module_dir, count in dir_histogram.items():
-        report += f"{module_dir}: {'*' * count}\n"
-
-    return report
-
-
-if __name__ == "__main__":
-    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
-report = generate_module_report()
-print(report)
-
-
-
-
-
-
-    report = generate_module_report()
-    print(report)

weee's products.txt

@@ -1,45 +0,0 @@
-# Selected
-# Want-Want Rice Crackers
-url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
-
-# Lay's Potato Chips
-url = 'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261'
-
-# Pocky Cookies
-url = 'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467'
-
-# Thai Jasmine Rice
-url = 'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01'
-
-# Japanese Medium Grain Rice
-url = 'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224'
-
-# Millet
-url = 'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02'
-
-# Nature's Soy Firm Tofu: https://www.sayweee.com/zh/product/Nature-s-Soy-Firm-Tofu-/19231?trace_id=ee63a29e-a1e4-4065-b213-96aa14a15521
-
-# Free 2% Reduced Fat Milk: https://www.sayweee.com/zh/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353?trace_id=3030fe4c-04fc-4f8f-9257-7f6395b9e513
-
-# Washington Fuji Apples: https://www.sayweee.com/zh/product/Washington-Fuji-Apples/94817?trace_id=dc4955fc-bba8-48cf-b1f9-9db3cf077eed
-
-# Greenlandic Sweet Prawns: https://www.sayweee.com/zh/product/Royal-Greenland-Cooked-Wild-Caught-Greenlandic-Sweet-Prawns-Frozen-1/44656?category=seafood01
-
-
-
-# Optional
-# LuoSiFen 
-url = 'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01'
-
-# Rice Noodles
-url = 'https://www.sayweee.com/en/product/Sanyangyishi-Fresh-Instant-Rice-Noodles/2156?category=instant01'
-
-# Cup Noodles
-url = 'https://www.sayweee.com/en/product/Nissin-Beef-Flavor-Cup-Noodles/102519?category=instant01'
-
-# Pork Sung
-url = 'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?category=canned04'
-
-# Nature's Soy Firm Tofu: https://www.sayweee.com/zh/product/Nature-s-Soy-Firm-Tofu-/19231?trace_id=ee63a29e-a1e4-4065-b213-96aa14a15521
-# [Not available or sold out in some area] Beef Ribeye Thin Slices: https://www.sayweee.com/zh/product/Beef-Ribeye-Thin-Slices--Frozen-1/4295?category=meat02
-# Shiitake Mushrooms: https://www.sayweee.com/zh/product/Shiitake-Mushrooms/45782?trace_id=ba388941-f631-4eaa-8570-5b7d4aba64e7

zipcodes.csv

@@ -1,33 +0,0 @@
-ZIP Code
-85305
-30313
-21230
-14127
-28202
-60605
-45202
-44114
-76011
-80204
-48226
-54304
-77054
-46225
-32202
-64129
-89118
-90301
-90301
-33056
-55415
-02035
-70112
-07073
-07073
-19148
-15212
-95054
-98134
-33607
-37213
-20785