Modified startup logic, added user dump

Updated list of required modules
Created object mapping
2023-05-24 14:44:55 -07:00 · 2023-05-24 14:14:43 -07:00 · 2023-05-24 17:04:30 -04:00 · 2023-05-23 17:59:16 -04:00 · 2023-05-22 17:24:46 -04:00 · 2023-05-22 17:04:22 -04:00
63 changed files with 3939 additions and 0 deletions
@@ -0,0 +1,276 @@
+import multiprocessing
+import subprocess
+import concurrent.futures
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+import os
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import selenium.common.exceptions
+import re
+from fp2_14 import get_product_prices
+
+# Define the subset of urls
+url_list = [
+    'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
+    'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
+    'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
+    'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
+    'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
+    'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
+    'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
+    'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
+    'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
+    'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
+    'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
+]
+
+
+# Create a pool of worker processes
+# pool = multiprocessing.Pool(cpu_count)
+
+# Define the function to run the scraper
+def run_scraper(url):
+    # scraping logic
+    df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
+    print(f"Created a dataframe: {df}")
+    product_id = url.split("/")[-1][:5]
+
+    options = Options()
+    options.add_argument('--disable-gpu')
+    driver = webdriver.Chrome(options=options)
+
+    project_id = "fp-20230425"
+    subscription_name = "fp-subscription-NFL-zipcodes"
+    subscriber = pubsub_v1.SubscriberClient()
+    subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+    # product_id = url.split("/")[-1]
+    driver.delete_all_cookies
+    driver.get(url)
+
+    def get_element_text(class_name_prefix):  
+        xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+        xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+        xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+        xpath_str += "' '), "                     # Step 4: a trailing space...
+        xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+        try:
+            match = driver.find_element(By.XPATH, xpath_str)
+            return match.get_attribute('textContent')
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            pass
+   
+        xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+        xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+        xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+        xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+        try:
+            match = driver.find_element(By.XPATH, xpath_str)
+            return match.get_attribute('textContent')
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            pass
+        return ''
+
+    # Close the cookies setting window
+    try:  
+        cookie_button = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+        )
+        print(f"cookie_button = {cookie_button}")
+        cookie_button.click()
+    except:
+        pass
+
+    # Clear any modal dialog
+    try:    
+        modal_close = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+        )
+        print(f"modal_close = {modal_close}")
+        modal_close.click()
+    except:
+        pass
+
+    # Define callback function for Pub/Sub messages
+    def callback(message):
+        # Extract zip code from message data
+        zip_code = message.message.data.decode('utf-8')
+
+        print(f"Received zip code: {zip_code}")
+
+        try:
+            try:
+                # Clear any modal dialog
+                modal_close = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+            )
+                # print(f"modal_close = {modal_close}")
+                modal_close.click()
+            except:
+                pass
+
+
+            try:
+
+                # Clear any notice
+                notice_close = WebDriverWait(driver, 5).until(
+                EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+                # print(f"notice_close = {notice_close}")
+                notice_close.click()
+            except:
+                pass
+        
+            # Enter zip code in the input field and click the "Check" button
+            zip_button = WebDriverWait(driver, 5).until(
+                EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+            )
+            # print(f"{zip_button:}")
+            zip_button.click()
+
+            zip_input = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+            )
+            # print(f"{zip_input:}")
+            zip_input.click()
+            # print("clearing zip_input")
+            zip_input.send_keys("\b\b\b\b\b")
+            # print("sending zip code")
+            zip_input.send_keys(zip_code)
+
+            check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+            check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+            # print(f"{check_button:}")
+            check_button.click()
+            # time.sleep(2)
+
+            try:
+            # Clear any notice
+                notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+                # print(f"notice_close = {notice_close}")
+                notice_close.click()
+            except:
+                pass
+
+            try:
+                # Clear any modal dialog
+                modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+                # print(f"modal_close = {modal_close}")
+                modal_close.click()
+            except:
+                pass
+
+            # Record sales count, if any
+            sales_count_element = get_element_text('Header_count')
+            count_start_index = sales_count_element.index('sold ') + len('sold ')
+            count_end_index = sales_count_element.index('+')
+            sales_count = sales_count_element[count_start_index:count_end_index]
+            # Record any sold status notice
+            sold_status = get_element_text('UnavailableCartbtn_container')
+            if sold_status == 'Notify me':
+                sold_status = 'Sold Out'
+
+        # Extract the price of rice crackers for this zip code
+            # Check if there is a minimum order quantity
+            min_quantity: float = 0
+            total_price: float = 0
+            unit_price: float = 0
+            try:
+                minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML')
+                if minimum_quantity_str:
+                    regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
+                    match = re.search(regexp,minimum_quantity_str)
+                    min_quantity, total_price = match.groups()
+            except selenium.common.exceptions.NoSuchElementException as nsee:
+                print("No minimum purhcase string found")
+            except Exception as e:
+                print(f"Exception when finding minimum purchase string: {e}")
+
+            ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+            ld_json_text = ld_json_element.get_attribute('innerHTML')
+            ld_json_start_index = ld_json_text.index('{')
+            ld_json_end_index = ld_json_text.rindex('}') + 1
+            ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
+            ld_json = json.loads(ld_json_json)
+            ld_json_price = ld_json['offers']['price']
+            # Print the price
+            print(ld_json_price)
+
+            unit_price_element = driver.find_element(By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText')
+            unit_price = unit_price_element.replace("$", "")
+            print(f"unit_price = {unit_price}" )
+
+            subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+            # Append the zip code and price to the dataframe
+            #append_to_df(zip_code, price)
+            new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]})
+            return pd.concat([df, new_row], ignore_index=True)
+            #print(f"Recorded the zip code and price: {zip_code},{price}")
+            #print(f"Updated a dataframe: {df}")
+
+        except Exception as e:
+            print(f"Caught exception: {e}")
+            print(f"Price not found for zip code {zip_code}")
+            price = 'Price not found'
+            return df
+
+    # Start receiving messages (sync version)
+    while True: 
+        pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+        messages = []
+        try:
+            pull_response = subscriber.pull(request = pull_request)
+            messages = pull_response.received_messages
+        except DeadlineExceeded:
+            pass
+
+        if messages:
+            df = callback(messages[0])
+            print(f"Updated a dataframe: {df}")
+        else:
+            print(f"no messages, trying again at {time.asctime()}")
+            break
+        # subprocess.call(['python', 'fp2_13.py', url])
+    return df
+
+if __name__ == '__main__':
+    # Determine the number of available CPUs
+    # cpu_count = multiprocessing.cpu_count()
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        future_to_url = {executor.submit(run_scraper, url): url for url in url_list}
+        results = []
+        for future in concurrent.futures.as_completed(future_to_url):
+            url = future_to_url[future]
+            try:
+                result = future.result()
+                print(f"Scraped data for URL: {url}")
+                results.append(result)
+            except Exception as e:
+                print(f"Failed to scrape data for URL: {url}\nError: {e}")
+
+'''
+# Run the scraper in parallel for the subset of items
+results = pool.map(run_scraper, url_list)
+
+# Close the pool and wait for the processes to finish
+pool.close()
+pool.join()
+'''
+
+with pd.HDFStore('product_data.h5') as store:
+    for url, result in zip(url_list, results):
+        # Extract the product ID from the URL
+        product_id = url.split("/")[-1][:5]
+
+        # Save the dataframe to the HDF5 file with a unique key
+        key = f'df_{product_id}'
+        store[key] = result
+        # result.to_hdf('product_data.h5', key=key, mode='a')
@@ -0,0 +1,33 @@
+ID,Team Name,Arena Name,Abbreviation,Conference,Division,Arena Location,ZIP Code
+1,Arizona Cardinals,State Farm Stadium,ARI,NFC,West,"Glendale, Arizona",85305
+2,Atlanta Falcons,Mercedes-Benz Stadium,ATL,NFC,South,"Atlanta, Georgia",30313
+3,Baltimore Ravens,M&T Bank Stadium,BAL,AFC,North,"Baltimore, Maryland",21230
+4,Buffalo Bills,Bills Stadium,BUF,AFC,East,"Orchard Park, New York",14127
+5,Carolina Panthers,Bank of America Stadium,CAR,NFC,South,"Charlotte, North Carolina",28202
+6,Chicago Bears,Soldier Field,CHI,NFC,North,"Chicago, Illinois",60605
+7,Cincinnati Bengals,Paul Brown Stadium,CIN,AFC,North,"Cincinnati, Ohio",45202
+8,Cleveland Browns,First Energy Stadium,CLE,AFC,North,"Cleveland, Ohio",44114
+9,Dallas Cowboys,At&T Stadium,DAL,NFC,East,"Arlington, Texas",76011
+10,Denver Broncos,Empower Field at Mile High,DEN,AFC,West,"Denver, Colorado",80204
+11,Detroit Lions,Ford Field,DET,NFC,North,"Detroit, Michigan",48226
+12,Green Bay Packers,Lambeau Field,GB,NFC,North,"Green Bay, Wisconsin",54304
+13,Houston Texans,NRG Stadium,HOU,AFC,South,"Houston, Texas",77054
+14,Indianapolis Colts,Lucas Oil Stadium,IND,AFC,South,"Indianapolis, Indiana",46225
+15,Jacksonville Jaguars,TIAA Bank Field,JAX,AFC,South,"Jacksonville, Florida",32202
+16,Kansas City Chiefs,Arrowhead Stadium,KC,AFC,West,"Kansas City, Missouri",64129
+17,Las Vegas Raiders,Allegiant Stadium,LV,AFC,West,"Paradise, Nevada",89118
+18,Los Angeles Chargers,SoFi Stadium,LAC,AFC,West,"Inglewood, California",90301
+19,Los Angeles Rams,SoFi Stadium,LAR,NFC,West,"Inglewood, California",90301
+20,Miami Dolphins,Hard Rock Stadium,MIA,AFC,East,"Miami Gardens, Florida",33056
+21,Minnesota Vikings,U.S. Bank Stadium,MIN,NFC,North,"Minneapolis, Minnesota",55415
+22,New England Patriots,Gillette Stadium,NE,AFC,East,"Foxborough, Massachusetts",02035
+23,New Orleans Saints,Mercedes-Benz Superdome,NO,NFC,South,"New Orleans, Louisiana",70112
+24,New York Giants,MetLife Stadium,NYG,NFC,East,"East Rutherford, New Jersey",07073
+25,New York Jets,MetLife Stadium,NYJ,AFC,East,"East Rutherford, New Jersey",07073
+26,Philadelphia Eagles,Lincoln Financial Field,PHI,NFC,East,"Philadelphia, Pennsylvania",19148
+27,Pittsburgh Steelers,Heinz Field,PIT,AFC,North,"Pittsburgh, Pennsylvania",15212
+28,San Francisco 49ers,Levi's Stadium,SF,NFC,West,"Santa Clara, California",95054
+29,Seattle Seahawks,CenturyLink Field,SEA,NFC,West,"Seattle, Washington",98134
+30,Tampa Bay Buccaneers,Raymond James Stadium,TB,NFC,South,"Tampa, Florida",33607
+31,Tennessee Titans,Nissan Stadium,TEN,AFC,South,"Nashville, Tennessee",37213
+32,Washington Redskins,FedExField,WAS,NFC,East,"Landover, Maryland",20785
@@ -17,4 +17,27 @@ the focus is on *working results*, not rote memorization.  Homework questions
 are simple enough that they can be reviewed interactively in 1:1 or up to 1:4
 sessions with study groups.

+This branch contains a prototype for a retail price scraper for getweee.com.
+To run it, install:
+
+* Python 3.9+,
+* chrome,
+* chromedriver,
+* and the python depedencies:
+
+```
+pip install -r /path/to/requirements.txt
+```
+
+Once those are installed, install the GCP SDK:
+
+https://cloud.google.com/sdk/docs/install
+
+You will also have to log into GCP *twice*, once for the gcloud CLI tools, 
+and again for the GCP SDK libraries:
+
+https://googleapis.dev/python/google-api-core/latest/auth.html
+
+
+

@@ -0,0 +1,42 @@
+from abc import ABC, abstractmethod
+
+class Simulation(ABC):
+    @abstractmethod
+    def start(self):
+        "Start the simulated object."
+        pass
+
+    @abstractmethod
+    def tick(self) -> bool:
+        """
+        Simulate one clock tick.
+        Returns:
+            bool: True if the simulation could advance one clock tick, or False if the object is expired.
+        """
+        pass
+
+    @abstractmethod
+    def get_uses(self):
+        """
+        Return the uses of the simulated object.  These are verb strings which are passed to the `use()` method.
+        Returns:
+            tuple: A dict of key-value pairs, where the keys are strings of
+            verbs to be passed to `use`, and the values are strings explaining
+            what the verb does in English.
+        """
+    @abstractmethod
+    def use(self, *args, **kwargs):
+        """
+        Use the simulated object.
+        Args:
+            args (tuple): A tuple of verbs to be used on the object.
+            kwargs (dict): A dictionary of key-value pairs of arguments to the verb(s).
+        Returns:
+            True if the action succeeded, False if the action did not succeed and the object needs to be expired.
+        """
+        pass
+
+    @abstractmethod
+    def stop(self):
+        "Stop the simulated object."
+        pass
@@ -0,0 +1,42 @@
+from importlib import import_module
+import tkinter as tk
+import os
+import sys
+from random import choice
+from time import sleep
+from tkinter import filedialog
+from abc import ABC, abstractmethod
+
+class SimulationRunner:
+    def __init__(self, simulation) -> None:
+        self.simulation = simulation()
+        self.simulation.start()
+        uses = list(self.simulation.get_uses().keys())
+        while self.simulation.tick() and self.simulation.use(choice(uses)):
+            pass
+        self.simulation.stop()
+
+if __name__ == '__main__':
+    if len(sys.argv) == 1:
+        # Create a hidden root window
+        root = tk.Tk()
+        root.withdraw()
+
+        # Open the file chooser dialog
+        file_path = filedialog.askopenfilename()
+        print(f"Selected file: {file_path}")
+    else:
+        file_path = sys.argv[1]
+
+    # Get the python file's directory and filename
+    module_dir, module_file = os.path.split(file_path)
+    # Get the python file's name without any extension
+    module_name, _ = os.path.splitext(module_file)
+
+    # Add the module directory to sys.path so we can import it unqualified
+    sys.path.append(module_dir)
+
+    simulation = import_module(module_name)
+    # Load the simulation module's simulation class, which should be the same
+    # name as the module name.
+    simulationrunner = SimulationRunner(getattr(simulation, module_name))
@@ -0,0 +1,402 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Guide\n",
+    "1. Please write your solutions at the appropriate location **# YOUR CODE HERE**\n",
+    "2. Do not modify other cells"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "# Numerical or binary attributes\n",
+    "num_or_bin= ['Income', 'Kidhome','Recency', 'MntWines', 'MntFruits', 'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts','MntGoldProds', 'NumDealsPurchases', 'NumWebPurchases',\n",
+    "       'NumCatalogPurchases', 'NumStorePurchases', 'NumWebVisitsMonth','AcceptedCmp3', 'AcceptedCmp4', 'AcceptedCmp5', 'AcceptedCmp1','AcceptedCmp2']\n",
+    "\n",
+    "# Categorical attributes\n",
+    "cat = ['Education','Marital_Status']\n",
+    "\n",
+    "# Load the data source\n",
+    "pd.set_option('display.max_columns', None)\n",
+    "df  = pd.read_csv(\"https://raw.githubusercontent.com/rpi-techfundamentals/website_fall_2021/master/site/public/marketing_campaign.csv\", delimiter='\\t')\n",
+    "df.head()"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q1. Load and preprocess the data.\n",
+    "Build and transform your dataframe `df` using the following steps before modeling. Create a new dataframe `X` that has the following properties:\n",
+    "\n",
+    "1. It should include only the columns from lists `num_or_bin` and `cat` (in the code above).\n",
+    "2. Variables in cat should be transformed to `n-1` dummy variables (where `n` is the number of categories for each variable.)\n",
+    "3. Missing variables in the `Income` table should be filled with the median of the `income` table.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "all_cols=num_or_bin+cat\n",
+    "X=pd.get_dummies(df[all_cols],columns=cat,drop_first=True)\n",
+    "X.loc[X['Income'].isnull(),'Income']=X['Income'].median()\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 1 Test Cell \n",
+    "#this tests the columns. \n",
+    "ecol = set(['Income', 'Kidhome', 'Recency', 'MntWines', 'MntFruits',\n",
+    "       'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts',\n",
+    "       'MntGoldProds', 'NumDealsPurchases', 'NumWebPurchases',\n",
+    "       'NumCatalogPurchases', 'NumStorePurchases', 'NumWebVisitsMonth',\n",
+    "       'AcceptedCmp3', 'AcceptedCmp4', 'AcceptedCmp5', 'AcceptedCmp1',\n",
+    "       'AcceptedCmp2', 'Education_Basic', 'Education_Graduation',\n",
+    "       'Education_Master', 'Education_PhD', 'Marital_Status_Alone',\n",
+    "       'Marital_Status_Divorced', 'Marital_Status_Married',\n",
+    "       'Marital_Status_Single', 'Marital_Status_Together',\n",
+    "       'Marital_Status_Widow', 'Marital_Status_YOLO'])\n",
+    "assert len(ecol-set(X.columns))==0\n",
+    "assert len(set(X.columns))-len(ecol)==0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 2 Test Cell \n",
+    "#this tests the columns. \n",
+    "assert X.shape==(2240, 30)\n",
+    "q2=X.copy()\n",
+    "q2['Response']=df['Response']\n",
+    "q2['Year_Birth']=df['Year_Birth']\n",
+    "q2['Dt_Customer']=df['Dt_Customer']\n",
+    "q2.to_csv('q2.csv',index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 3 Test Cell \n",
+    "assert len(X.loc[X['Income']==51381.5,'Income'])==24"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 4 Test Cell\n",
+    "assert X.isnull().sum().sum()==0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Reload data\n",
+    "import pandas as pd\n",
+    "df_cl=pd.read_csv('q2.csv')"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q2. Feature Creation.\n",
+    "1. In the `df_cl` dataframe, create an integer feature `age` which is `2021` minus `Year_Birth`.\n",
+    "2. In the `df_cl` dataframe, create an integer feature `holiday` which is 1 if the month of `Dt_Customer` is equal to 12 and 0 otherwise. You can get the month by accessing `mm` from the date format `dd-mm-yyyy`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "df_cl['age']=(2021-df_cl['Year_Birth']).astype(int)\n",
+    "df_cl['Dt_Customer_tmp']=df_cl['Dt_Customer'].apply(lambda x:x[3:5])\n",
+    "df_cl.loc[df_cl['Dt_Customer_tmp']=='12','holiday']=1\n",
+    "df_cl.loc[df_cl['Dt_Customer_tmp']!='12','holiday']=0\n",
+    "df_cl['holiday']=df_cl['holiday'].astype(int)\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 5 Test Cell\n",
+    "assert df_cl['age'].sum()==116915"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 6 Test Cell\n",
+    "import numpy as np\n",
+    "assert isinstance(df_cl['age'][0], (np.int64,np.int32))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 7 Test Cell\n",
+    "assert df_cl['holiday'].sum()==175"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 8 Test Cell\n",
+    "import numpy as np\n",
+    "assert isinstance(df_cl['holiday'][0], (np.int64,np.int32))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "df_cl2=pd.read_csv('dfcl2.csv')\n",
+    "y_reload=df_cl2['Response']\n",
+    "X_reload=df_cl2.drop(['Response'], axis=1)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q3. Train Test Split\n",
+    "Now split the data `X_reload`, `y_reload` intro training (50%) and testing data (50%) by using these variable names while making sure there are Make sure that there are equal number of positive cases (1's) in the train and the test set.\n",
+    "\n",
+    "`X_train`: Training feature columns\n",
+    "`X_test`: Testing feature columns\n",
+    "`y_train`: Training DV\n",
+    "`y_test`: Testing DV\n",
+    "\n",
+    "When doing so set the `random_state` parameter to 100 so you get the same split that is in the solution."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "y_reload=df_cl2['Response']\n",
+    "X_reload=df_cl2.drop(['Response'], axis=1)\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "X_train,X_test,y_train,y_test=train_test_split(X_reload,y_reload,\n",
+    "                                               train_size=0.5,\n",
+    "                                               test_size=0.5,\n",
+    "                                               random_state=100,\n",
+    "                                               stratify=y_reload)\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 9 Test Cell\n",
+    "assert X_train.shape==(1120, 31)\n",
+    "assert X_test.shape==(1120, 31)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 10 Test Cell\n",
+    "assert  y_train.sum()==y_test.sum()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "df_cl2=pd.read_csv('dfcl2.csv')\n",
+    "y_reload=df_cl2['Response']\n",
+    "X_reload=df_cl2.drop(['Response'], axis=1)\n",
+    "\n",
+    "X_train2, X_test2, y_train2, y_test2 = train_test_split(X_reload, y_reload, test_size=0.3, random_state = 100)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q4. Classification\n",
+    "Use `X_train2`, `X_test2`, `y_train2`, `y_test2` from above. Use a Gradient Boosting classifier (as instantiated below).\n",
+    "\n",
+    "Please calculate:\n",
+    "1. `test_acc` The accuracy of the classifier on the test set.\n",
+    "2. `test_roc` The area under the ROC curve on on the test set."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn import metrics\n",
+    "from sklearn.ensemble import GradientBoostingClassifier\n",
+    "#use this classifier\n",
+    "classifier = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1, random_state=0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "classifier.fit(X_train2,y_train2)\n",
+    "test_y_pred=classifier.predict(X_test2)\n",
+    "test_y_pred_prob=classifier.predict_proba(X_test2)[:,1]\n",
+    "test_acc=metrics.accuracy_score(y_test2,test_y_pred)\n",
+    "test_roc=metrics.roc_auc_score(y_test2,test_y_pred_prob)\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 11 Test Cell\n",
+    "assert round(test_acc,3)==0.860"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 12 Test Cell\n",
+    "assert round(test_roc,3)==0.846"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
@@ -0,0 +1,74 @@
+import h5py
+import pandas as pd
+import numpy as np
+import plotly.graph_objects as go
+
+f = h5py.File('product1.h5','r')
+list(f.keys())
+for key in f.keys():
+    print(f[key])
+    df = pd.read_hdf('product1.h5', key='df')
+    df = df._append(pd.read_hdf('product1.h5', key))
+    # print(f[key].shape)
+    # print(f[key].value)
+# df = pd.read_hdf('product1.h5', key='df7')
+print(df)
+'''
+# Open the HDF5 file
+with h5py.File('product1.h5', 'r') as file:
+    dataframes=pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])  
+    for key in file.keys():
+        # Read the dataframe from the HDF5 file
+        dataframe = pd.read_hdf('product1.h5', key)
+
+        # Append the dataframe to the list
+        dataframes._append(dataframe)
+
+# Print the number of dataframes read from the HDF5 file
+print(f"Read {len(dataframes)} dataframes from the HDF5 file")
+
+item_std_devs = {}
+for dataframe in dataframes:
+    item_name = dataframe['Product ID'].iloc[0]
+    std_dev = np.std(dataframe['Unit Price'])
+    item_std_devs[item_name] = std_dev
+
+# Print the item names and their corresponding standard deviations
+for item_name, std_dev in item_std_devs.items():
+    print(f"Item: {item_name}, Std Dev: {std_dev}")
+
+# Prepare the data for the graph
+x = list(item_std_devs.keys())
+y = list(item_std_devs.values())
+
+# Create a bar graph
+fig = go.Figure(data=[go.Bar(x=x, y=y)])
+
+# Customize the graph
+fig.update_layout(
+    title="Standard Deviation of Pricing for Each Item",
+    xaxis_title="Item",
+    yaxis_title="Standard Deviation",
+)
+
+# Save the graph as an HTML file
+report_file = 'test2.html'
+fig.write_html(report_file)
+
+
+
+'''
+'''
+# Calculate the standard deviation using pandas' function
+std_dev_unit = df.groupby('Product ID')['Unit Price'].std()
+std_dev_displayed = df.groupby('Zip Code')['Shown Price'].std()
+
+# Generate the graph using Plotly
+fig = go.Figure()
+fig.add_trace(go.Bar(x=std_dev_unit.index, y=std_dev_unit.values))
+fig.update_layout(title='Standard Deviation of the Unit Pricing by Item', xaxis_title='Product ID', yaxis_title='Standard Deviation')
+
+report_file = 'test2.html'
+fig.write_html(report_file)
+'''
+
@@ -0,0 +1,81 @@
+'''
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+        self._attributes = dict(self.__dict__)
+        
+    def __setattr__(self, name, value):
+        if name != "_attributes" and name not in self._attributes:
+            print("attribute deleted!")
+        self._attributes[name] = value
+        super().__setattr__("_attributes", dict(self.__dict__))
+        
+    @property
+    def treasure(self):
+        if self.__dict__ != self._attributes:
+            print("attribute deleted!")
+        return self._treasure
+
+v=Vault()
+print(v.treasure)
+'''
+'''
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+        self._original_dict = dict(self.__dict__)
+
+    def __setattr__(self, name, value):
+        if name != "_treasure" and name in self.__dict__ and self._original_dict.get(name) != value:
+            print("attribute deleted!")
+        self.__dict__[name] = value
+
+    @property
+    def treasure(self):
+        if self.__dict__ != self._original_dict:
+            print("attribute deleted!")
+        return self._treasure
+v = Vault()
+print(v.treasure)
+'''
+'''
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+        self._attributes = dict(self.__dict__)
+
+    def __setattr__(self, name, value):
+        if name != "_attributes" and name not in self._attributes:
+            print("attribute deleted!")
+        self._attributes[name] = value
+        super().__setattr__("_attributes", dict(self.__dict__))
+
+    @property
+    def treasure(self):
+        if self.__dict__ != self._attributes:
+            print("attribute deleted!")
+        return self._treasure
+
+v = Vault()
+print(v.treasure)
+'''
+
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+
+    def __setattr__(self, name, value):
+        if name != "_attributes" and name not in self.__dict__:
+            print("attribute deleted!")
+        self.__dict__[name] = value
+        self._attributes = dict(self.__dict__)
+
+    @property
+    def treasure(self):
+        if self.__dict__ != self._attributes:
+            print("attribute deleted!")
+        return self._treasure
+
+v = Vault()
+print(v.treasure)
+
@@ -0,0 +1,47 @@
+import importlib.util
+import os
+
+def get_module_size(module_name):
+    """Get the total number of bytes in the cached .pyc file or .py file of a module or package.
+
+    Args:
+        module_name (str): The name of the module or package.
+
+    Returns:
+        int: The total number of bytes in the cached .pyc file or .py file.
+    """
+    spec = importlib.util.find_spec(module_name)
+
+    if spec is not None and spec.has_location:
+        # If the module is found and has a location, check for .pyc file first
+        pyc_file_path = spec.origin + 'c'
+        if os.path.exists(pyc_file_path):
+            return os.stat(pyc_file_path).st_size
+        else:
+            # If .pyc file doesn't exist, check for .py file
+            py_file_path = spec.origin
+            if os.path.exists(py_file_path):
+                return os.stat(py_file_path).st_size
+
+    return 0  # Return 0 if no .pyc or .py file found
+
+
+import sys
+
+def get_total_module_size():
+    """Get the total number of bytes for all modules listed in sys.modules.
+
+    Returns:
+        int: The total number of bytes for all modules.
+    """
+    total_size = 0
+    for module_name in sys.modules:
+        module_size = get_module_size(module_name)
+        total_size += module_size
+
+    return total_size
+
+if __name__ == '__main__' and __package__ is None:
+    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
+total_size = get_total_module_size()
+print("Total size of all modules in sys.modules: {} bytes".format(total_size))
@@ -0,0 +1,65 @@
+from google.cloud import pubsub_v1
+from google.oauth2 import service_account
+import os
+import asyncio
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+project_id = "fp-20230425"
+
+'''
+topic_name = "fp-topic-Karen"
+subscription_name = "fp-subscription-Karen"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
+subscriber.delete_subscription(request={"subscription": subscription_path})
+print("Subscription deleted: {}".format(subscription_path))
+
+# Delete the topic
+publisher = pubsub_v1.PublisherClient()
+topic_path = publisher.topic_path(project_id, topic_name)
+publisher.delete_topic(request={"topic": topic_path})
+print(f"Topic '{topic_name}' deleted.")
+
+subscription_name = "fp-subscription-Karen"
+subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
+
+
+'''
+# Create a pubsub topic
+publisher = pubsub_v1.PublisherClient()
+topic_name = "fp-topic-Karen"
+topic_path = publisher.topic_path(project_id, topic_name)
+topic = publisher.create_topic(request={"name": topic_path})
+
+
+# Create a Pub/Sub subscription
+subscriber = pubsub_v1.SubscriberClient()
+subscription_name = "fp-subscription-Karen"
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+subscription = subscriber.create_subscription(
+    name=subscription_path, topic=topic_path
+)
+
+# Publish a message to the topic
+message = b"Hello!"
+publisher.publish(topic_path, data=message)
+print("Message published.")
+
+
+async def receive_messages():
+    response = subscriber.pull(subscription=subscription_path, max_messages=1)
+    for received_message in response.received_messages:
+        print("Received message: {}".format(received_message.message.data))
+       
+    
+asyncio.run(receive_messages())
+
+
+'''
+# Pull the message from the subscription
+response = subscriber.pull(subscription=subscription_path, max_messages=1)
+for received_message in response.received_messages:
+    print("Received message: {}".format(received_message.message.data))
+    # Acknowledge the message to mark it as processed
+    subscriber.acknowledge(subscription_path, [received_message.ack_id])
+'''
@@ -0,0 +1,10 @@
+import http.client
+
+# The product ID for the Pete-and-Gerry's organic large brown egges is 65902
+conn = http.client.HTTPSConnection("www.sayweee.com")
+payload = ''
+headers = {}
+conn.request("GET", "/en/product/Pete-and-Gerry-s-Organic-Large-Brown-Eggs/65902?category=dairy02", payload, headers)
+res = conn.getresponse()
+data = res.read()
+print(data.decode("utf-8"))
@@ -0,0 +1,67 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+import csv
+import os
+import asyncio
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+project_id = "fp-20230425"
+# project_id = "svig-384801"
+
+'''
+topic_name = "fp-topic-NFL-zipcodes"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
+subscriber.delete_subscription(request={"subscription": subscription_path})
+print("Subscription deleted: {}".format(subscription_path))
+
+# Delete the topic
+publisher = pubsub_v1.PublisherClient()
+topic_path = publisher.topic_path(project_id, topic_name)
+publisher.delete_topic(request={"topic": topic_path})
+print(f"Topic '{topic_name}' deleted.")
+
+
+
+'''
+# pubsub topic
+publisher = pubsub_v1.PublisherClient()
+topic_name = "fp-topic-NFL-zipcodes"
+topic_path = publisher.topic_path(project_id, topic_name)
+
+# Create a Pub/Sub subscription
+subscriber = pubsub_v1.SubscriberClient()
+'''
+subscription_name = "fp-subscription-zipcodes"
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+subscription = subscriber.create_subscription(
+    name=subscription_path, topic=topic_path
+)
+'''
+# Publish a message to the topic
+# Read the data from the csv file
+# file_path = "C:/Users/Karen/Desktop/zipcodes.csv"
+file_path = "zipcodes.csv"
+
+with open(file_path, "r") as csv_file:
+    reader = csv.reader(csv_file)
+    next(reader)  # skip header row
+    for row in reader:
+        # Format the data as a string and publish it to the topic
+        data = ",".join(row).encode("utf-8")
+        try:
+            future = publisher.publish(topic_path, data)
+            print(f"Published message ID {future.result()}")
+        except DeadlineExceeded as de:
+            print(f"Caught DeadlineExceeded exception: {de}")
+
+'''
+async def receive_messages():
+    response = subscriber.pull(subscription=subscription_path, max_messages=1)
+    for received_message in response.received_messages:
+        print("Received message: {}".format(received_message.message.data))
+       
+    
+asyncio.run(receive_messages())
+'''
@@ -0,0 +1,372 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+import selenium.common.exceptions
+import json
+import time
+import os
+import re
+import sys
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+
+# Create an empty dataframe to store the prices
+df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Displayed Price', 'ld+json Price', 'Minimum Quantity', 'Total Price', 'Sales Count', 'Sold Status'])
+print(f"Created a dataframe: {df}")
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+# Load the product page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+product_id = url.split("/")[-1]
+driver.delete_all_cookies
+driver.get(url)
+# time.sleep(5)
+
+# time.sleep(2)
+# Close the cookies setting window
+try:  
+    # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")
+    cookie_button = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+    )
+    print(f"cookie_button = {cookie_button}")
+    cookie_button.click()
+except:
+    pass
+
+# time.sleep(1)
+# Clear any modal dialog
+try:    
+    # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+    modal_close = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+    )
+    print(f"modal_close = {modal_close}")
+    modal_close.click()
+    #time.sleep(2)
+except:
+    pass
+
+
+def close_modals():
+    try:
+        # time.sleep(2)
+        # Clear any modal dialog
+        modal_close_buttons = driver.find_elements(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+        print(f"modal_close = {modal_close}")
+        for button in modal_close_buttons:
+            try:
+                button.click()
+            except Exception as e:
+                print(f"Exception when clicking on close button for element {button}: {e}")
+
+        # time.sleep(2)
+    except Exception as e:
+        print(f"Exception when trying to close discount modal dialog: {e}")
+
+
+    try:
+        # time.sleep(1)
+        # Clear any notice
+        notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+        print(f"notice_close = {notice_close}")
+        notice_close.click()
+        # time.sleep(1)
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        print("Purchase option change dialog close button not found")
+    except Exception as e:
+        print(f"Exception when trying to close purchase option change dialog: {e}")
+
+def get_element_text(class_name_prefix):
+    '''
+    Return the text in the first element with a given class name.
+
+    Note that XPath doesn't have a function to check if a given word is present
+    in a list of words, so using contains() will match a class name even if
+    the class name is a substring of the class name being matched against.
+    To prevent this, we use spaces as a delimiter at the beginning and end of
+    the class name, and ensure there's only one space before and after all
+    class names in the class name list.
+
+    Example: consider two class names with cache-busting suffixes:
+      * Header_soldStatusNotice__12345
+      * Header_soldStatusNoticeBackground__ABCDE
+
+    Matching against Header_soldStatusNotice using a naive implementation with
+    `contains()` would match both class names, even though only the first one
+    is the correct match. 
+    '''
+    # Construct xpath selector.  
+    xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+    xpath_str += "' '), "                     # Step 4: a trailing space...
+    xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # We didn't find a match, try again by doing a prefix match and assume
+    # the full class name ends with two underscores followed by a cache-
+    # busting string.    
+    xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+    xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # No match, return empty string.
+    return ''
+
+def callback(message):
+    '''
+    Callback function for Pub/Sub messages
+    '''
+    # Extract zip code from message data
+    zip_code = message.message.data.decode('utf-8')
+
+    print(f"Received zip code: {zip_code}")
+
+    while True:
+        try:
+            # Enter zip code in the input field and click the "Check" button
+            WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+            zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
+            # print(f"{zip_button:}")
+            zip_button.click()
+            # time.sleep(1)
+
+            WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, '//div/label/following-sibling::input'))
+            zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+            # print(f"{zip_input:}")
+            zip_input.click()
+            # print("clearing zip_input")
+            zip_input.send_keys("\b\b\b\b\b\b\b\b\b\b")
+            for i in range(1,10):
+                zip_input.send_keys(Keys.DELETE)
+            # print("sending zip code")
+            zip_input.send_keys(zip_code)
+
+            WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, "//button[text() = 'Confirm']"))
+            check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+            print(f"{check_button:}")
+            check_button.click()
+            # time.sleep(2)
+            break
+        except Exception as e:
+            print(f"Exception when changing zip code: {e}, trying to close modal dialogs")
+            close_modals()
+
+        # try:
+        #     time.sleep(2)
+        #     # Clear any notice
+        #     notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+        #     print(f"notice_close = {notice_close}")
+        #     notice_close.click()
+        #     time.sleep(2)
+        # except:
+        #     pass
+
+    try:
+        try:
+            # time.sleep(2)
+            # Clear any modal dialog
+            # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+            modal_close = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+        )
+            print(f"modal_close = {modal_close}")
+            modal_close.click()
+            # time.sleep(2)
+        except:
+            pass
+
+
+        try:
+            # time.sleep(1)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
+        )
+            print(f"notice_close = {notice_close}")
+            notice_close.click()
+            # time.sleep(1)
+        except:
+            pass
+        
+        # Enter zip code in the input field and click the "Check" button
+        # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
+        zip_button = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+        )
+        # print(f"{zip_button:}")
+        zip_button.click()
+        # time.sleep(1)
+
+        # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+        zip_input = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+        )
+        # print(f"{zip_input:}")
+        zip_input.click()
+        # print("clearing zip_input")
+        zip_input.send_keys("\b\b\b\b\b")
+        # print("sending zip code")
+        zip_input.send_keys(zip_code)
+
+        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+        check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+        # print(f"{check_button:}")
+        check_button.click()
+        # time.sleep(2)
+
+        try:
+            # time.sleep(2)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+            #time.sleep(2)
+        except:
+            pass
+
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+        # Record sales count, if any
+        sales_count = get_element_text('Header_count')
+        # Record any sold status notice
+        sold_status = get_element_text('soldStatusNotice')
+
+        # Check if there's a minimum order quantity
+        min_quantity: float = 0
+        total_price: float = 0
+        try:
+            minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' minQuantity ')]").get_attribute('innerHTML')
+            if minimum_quantity_str:
+                regexp="Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
+                match = re.search(regexp, minimum_quantity_str)
+                min_quantity, total_price = match.groups()
+
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            print("No minimum purchase string found")
+        except Exception as e:
+            print(f"Exception when finding minimum purchase string: {e}")
+
+        # Get the price as displayed on the web page
+        displayed_price = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' price_price ')]").get_attribute('innerText').replace('$', '')
+        print(f"displayed_price = {displayed_price}")
+
+        # Extract the price of rice crackers for this zip code     
+        '''
+        price_element = driver.find_element(By.XPATH, '/html/body/div[2]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')
+        price = price_element.text.strip()[1:]
+        print(price)
+        '''
+
+        # Get the price from the ld+json element
+        ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        ld_json_text = ld_json_element.get_attribute('innerHTML')
+        ld_json_start_index = ld_json_text.index('{')
+        ld_json_end_index = ld_json_text.rindex('}') + 1
+        ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
+        ld_json = json.loads(ld_json_json)
+        ld_json_price = ld_json['offers']['price']
+        # Print the price
+        print(f"ld_json_price = {ld_json_price}")
+
+        subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+        # Append the zip code and price to the dataframe
+        #append_to_df(zip_code, price)
+        new_row = pd.DataFrame({'Product ID': [product_id],
+            'Zip Code': [zip_code],
+            'Displayed Price': [displayed_price],
+            'ld+json Price': [ld_json_price],
+            'Minimum Quantity': [min_quantity],
+            'Total Price': [total_price],
+            'Sales Count': [sales_count],
+            'Sold Status': [sold_status]})
+        return pd.concat([df, new_row], ignore_index=True)
+        #print(f"Recorded the zip code and price: {zip_code},{price}")
+        #print(f"Updated a dataframe: {df}")
+
+    except Exception as e:
+        print(f"Caught exception {type(e)}: {e}")
+        print(f"Price not found for zip code {zip_code}")
+        price = 'Price not found'
+        return df
+
+# Start receiving messages (sync version)
+while True: 
+    pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+    messages = []
+    try:
+        pull_response = subscriber.pull(request = pull_request)
+        messages = pull_response.received_messages
+    except DeadlineExceeded:
+        pass
+
+    if messages:
+        df = callback(messages[0])
+        print(f"Updated a dataframe: {df}")
+        df.to_hdf('product1.h5', key='df',mode='w')
+    else:
+        print(f"No messages in subscription {subscription_path} at {time.asctime()}, trying again...")
+        continue
+
+# Open the HDF5 file in a separate window
+if sys.platform.startswith("win32"):
+    os.startfile('product1.h5')
+
+# Start receiving messages (async version)
+#flow_control = pubsub_v1.types.FlowControl(max_messages = 1)
+#streaming_pull_future = subscriber.subscribe(subscription_path, 
+#    callback=callback, 
+#    flow_control=flow_control)
+
+# print(f'Started receiving messages from subscription {subscription_path}...')
+
+# Wrap subscriber in a 'with' block to automatically call close() when done.
+# with subscriber:
+#    try:
+        # When `timeout` is not set, result() will block indefinitely,
+        # unless an exception is encountered first.
+#        streaming_pull_future.result(timeout=timeout)
+#    except TimeoutError:
+#        streaming_pull_future.cancel()  # Trigger the shutdown.
+#        streaming_pull_future.result()  # Block until the shutdown is complete.
+
+# Keep the script running indefinitely to continue receiving messages
+#while True:
+#    time.sleep(10)
@@ -0,0 +1,199 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+# Create an empty dataframe to store the prices
+df = pd.DataFrame(columns=['Zip Code', 'Price'])
+print(f"Created a dataframe: {df}")
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+# Load the product page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+driver.delete_all_cookies
+driver.get(url)
+# time.sleep(5)
+
+# time.sleep(2)
+# Close the cookies setting window
+try:  
+    # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")
+    cookie_button = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+    )
+    print(f"cookie_button = {cookie_button}")
+    cookie_button.click()
+except:
+    pass
+
+# time.sleep(1)
+# Clear any modal dialog
+try:    
+    # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+    modal_close = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+    )
+    print(f"modal_close = {modal_close}")
+    modal_close.click()
+    #time.sleep(2)
+except:
+    pass
+
+# Define callback function for Pub/Sub messages
+def callback(message):
+    # Extract zip code from message data
+    zip_code = message.message.data.decode('utf-8')
+
+    print(f"Received zip code: {zip_code}")
+
+    try:
+        try:
+            # time.sleep(2)
+            # Clear any modal dialog
+            # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+            modal_close = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+        )
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+            # time.sleep(2)
+        except:
+            pass
+
+
+        try:
+            # time.sleep(1)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
+        )
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+            # time.sleep(1)
+        except:
+            pass
+        
+        # Enter zip code in the input field and click the "Check" button
+        # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
+        zip_button = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+        )
+        # print(f"{zip_button:}")
+        zip_button.click()
+        # time.sleep(1)
+
+        # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+        zip_input = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+        )
+        # print(f"{zip_input:}")
+        zip_input.click()
+        # print("clearing zip_input")
+        zip_input.send_keys("\b\b\b\b\b")
+        # print("sending zip code")
+        zip_input.send_keys(zip_code)
+
+        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+        check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+        # print(f"{check_button:}")
+        check_button.click()
+        # time.sleep(2)
+
+        try:
+            # time.sleep(2)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+            #time.sleep(2)
+        except:
+            pass
+
+
+    # Extract the price of rice crackers for this zip code
+        
+        # next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        next_data_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, '//*[@id="__next"]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')))
+        next_data_text = next_data_element.get_attribute('nodeValue')
+        price = next_data_text.text
+        '''
+        next_data_text = next_data_element.get_attribute('innerHTML')
+        next_data_start_index = next_data_text.index('{')
+        next_data_end_index = next_data_text.rindex('}') + 1
+        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+        next_data = json.loads(next_data_json)
+        price = next_data['price']
+        '''
+        # Print the price
+        print(price)
+
+        '''
+        try:
+            min_quantity_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, '//script[@id="__NEXT_DATA__"]')))
+            min_quantity_text = min_quantity_element.get_attribute('innerHTML')
+            min_quantity_start_index = min_quantity_text.index('{')
+            min_quantity_end_index = min_quantity_text.rindex('}') + 1
+            min_quantity_json = min_quantity_text[min_quantity_start_index:min_quantity_end_index]
+            min_quantity = json.loads(min_quantity_json)
+            price = min_quantity['product']['price']
+        except:
+            next_data_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//script[@type='application/ld+json']")))
+            next_data_text = next_data_element.get_attribute('innerHTML')
+            next_data_start_index = next_data_text.index('{')
+            next_data_end_index = next_data_text.rindex('}') + 1
+            next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+            next_data = json.loads(next_data_json)        
+            price = next_data['offers']['price']
+        finally:
+            # Print the price
+            print(price)
+        '''
+
+        subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+        # Append the zip code and price to the dataframe
+        #append_to_df(zip_code, price)
+        new_row = pd.DataFrame({'Zip Code': [zip_code], 'Price': [price]})
+        return pd.concat([df, new_row], ignore_index=True)
+        #print(f"Recorded the zip code and price: {zip_code},{price}")
+        #print(f"Updated a dataframe: {df}")
+
+    except Exception as e:
+        print(f"Caught exception: {e}")
+        print(f"Price not found for zip code {zip_code}")
+        price = 'Price not found'
+        return df
+
+# Start receiving messages (sync version)
+while True: 
+    pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+    messages = []
+    try:
+        pull_response = subscriber.pull(request = pull_request)
+        messages = pull_response.received_messages
+    except DeadlineExceeded:
+        pass
+
+    if messages:
+        df = callback(messages[0])
+        print(f"Updated a dataframe: {df}")
+    else:
+        print(f"no messages, trying again at {time.asctime()}")
@@ -0,0 +1,271 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+import os
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import selenium.common.exceptions
+import re
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+
+# Create an empty dataframe to store the prices
+df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
+print(f"Created a dataframe: {df}")
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+# Load the product page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
+product_id = url.split("/")[-1][:5]
+# product_id = url.split("/")[-1]
+driver.delete_all_cookies
+driver.get(url)
+# time.sleep(5)
+
+def get_element_text(class_name_prefix):
+    '''
+    Return the text in the first element with a given class name.
+
+    Note that XPath doesn't have a function to check if a given word is present
+    in a list of words, so using contains() will match a class name even if
+    the class name is a substring of the class name being matched against.
+    To prevent this, we use spaces as a delimiter at the beginning and end of
+    the class name, and ensure there's only one space before and after all
+    class names in the class name list.
+
+    Example: consider two class names with cache-busting suffixes:
+      * Header_soldStatusNotice__12345
+      * Header_soldStatusNoticeBackground__ABCDE
+
+    Matching against Header_soldStatusNotice using a naive implementation with
+    `contains()` would match both class names, even though only the first one
+    is the correct match. 
+    '''
+    # Construct xpath selector.  
+    xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+    xpath_str += "' '), "                     # Step 4: a trailing space...
+    xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # We didn't find a match, try again by doing a prefix match and assume
+    # the full class name ends with two underscores followed by a cache-
+    # busting string.    
+    xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+    xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # No match, return empty string.
+    return ''
+
+
+# time.sleep(2)
+# Close the cookies setting window
+try:  
+    # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")
+    cookie_button = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+    )
+    print(f"cookie_button = {cookie_button}")
+    cookie_button.click()
+except:
+    pass
+
+# time.sleep(1)
+# Clear any modal dialog
+try:    
+    # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+    modal_close = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+    )
+    print(f"modal_close = {modal_close}")
+    modal_close.click()
+    #time.sleep(2)
+except:
+    pass
+
+# Define callback function for Pub/Sub messages
+def callback(message):
+    # Extract zip code from message data
+    zip_code = message.message.data.decode('utf-8')
+
+    print(f"Received zip code: {zip_code}")
+
+    try:
+        try:
+            # time.sleep(2)
+            # Clear any modal dialog
+            # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+            modal_close = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+        )
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+            # time.sleep(2)
+        except:
+            pass
+
+
+        try:
+            # time.sleep(1)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
+        )
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+            # time.sleep(1)
+        except:
+            pass
+        
+        # Enter zip code in the input field and click the "Check" button
+        # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
+        zip_button = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+        )
+        # print(f"{zip_button:}")
+        zip_button.click()
+        # time.sleep(1)
+
+        # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+        zip_input = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+        )
+        # print(f"{zip_input:}")
+        zip_input.click()
+        # print("clearing zip_input")
+        zip_input.send_keys("\b\b\b\b\b")
+        # print("sending zip code")
+        zip_input.send_keys(zip_code)
+
+        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+        check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+        # print(f"{check_button:}")
+        check_button.click()
+        # time.sleep(2)
+
+        try:
+            # time.sleep(2)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+            #time.sleep(2)
+        except:
+            pass
+
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+        # Record sales count, if any
+        sales_count_element = get_element_text('Header_count')
+        count_start_index = sales_count_element.index('sold ') + len('sold ')
+        count_end_index = sales_count_element.index('+')
+        sales_count = sales_count_element[count_start_index:count_end_index]
+        # Record any sold status notice
+        sold_status = get_element_text('UnavailableCartbtn_container')
+        if sold_status == 'Notify me':
+            sold_status = 'Sold Out'
+
+    # Extract the price of rice crackers for this zip code
+        # Check if there is a minimum order quantity
+        min_quantity: float = 0
+        total_price: float = 0
+        unit_price: float = 0
+        try:
+            minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML')
+            if minimum_quantity_str:
+                regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
+                match = re.search(regexp,minimum_quantity_str)
+                min_quantity, total_price = match.groups()
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            print("No minimum purhcase string found")
+        except Exception as e:
+            print(f"Exception when finding minimum purchase string: {e}")
+ 
+
+
+        ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        ld_json_text = ld_json_element.get_attribute('innerHTML')
+        ld_json_start_index = ld_json_text.index('{')
+        ld_json_end_index = ld_json_text.rindex('}') + 1
+        ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
+        ld_json = json.loads(ld_json_json)
+        ld_json_price = ld_json['offers']['price']
+        # Print the price
+        print(ld_json_price)
+
+        unit_price_element = driver.find_element(By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText')
+        unit_price = unit_price_element.replace("$", "")
+        print(f"unit_price = {unit_price}" )
+
+        subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+        # Append the zip code and price to the dataframe
+        #append_to_df(zip_code, price)
+        new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]})
+        return pd.concat([df, new_row], ignore_index=True)
+        #print(f"Recorded the zip code and price: {zip_code},{price}")
+        #print(f"Updated a dataframe: {df}")
+
+    except Exception as e:
+        print(f"Caught exception: {e}")
+        print(f"Price not found for zip code {zip_code}")
+        price = 'Price not found'
+        return df
+
+# Start receiving messages (sync version)
+while True: 
+    pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+    messages = []
+    try:
+        pull_response = subscriber.pull(request = pull_request)
+        messages = pull_response.received_messages
+    except DeadlineExceeded:
+        pass
+
+    if messages:
+        df = callback(messages[0])
+        print(f"Updated a dataframe: {df}")
+    else:
+        print(f"no messages, trying again at {time.asctime()}")
+        break
+
+# df.to_hdf('product1.h5', key='df',mode='w')
+# df.to_hdf('product1.h5', key='df',mode='a')
+
+# Open the HDF5 file in a separate window
+# os.startfile('product1.h5')
@@ -0,0 +1,280 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import selenium.common.exceptions
+import re
+from datetime import datetime
+import h5py
+import sys
+import ntplib
+import arrow
+
+# This fp2_14.py is used to load the new price and the timestamp, and append such data in an existing key in the HDF5 file "product1.h5"
+
+def get_product_prices(project_id, subscription_name, url):
+    # Create an empty dataframe to store the prices
+    df_new = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
+    print(f"Created a dataframe: {df_new}")
+
+    # Configure the Selenium driver options
+    options = Options()
+    options.add_argument('--disable-gpu')
+    driver = webdriver.Chrome(options=options)
+
+    # Set up Pub/Sub subscription
+    '''
+    project_id = "fp-20230425"
+    subscription_name = "fp-subscription-NFL-zipcodes"
+    '''
+    subscriber = pubsub_v1.SubscriberClient()
+    subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+
+    # Load the product page and extract the __NEXT_DATA__ data
+    # url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
+    product_id = url.split("/")[-1][:5]
+    # product_id = url.split("/")[-1]
+    driver.delete_all_cookies
+    driver.get(url)
+
+    def get_element_text(class_name_prefix):
+        '''
+        Return the text in the first element with a given class name.
+
+        Note that XPath doesn't have a function to check if a given word is present
+        in a list of words, so using contains() will match a class name even if
+        the class name is a substring of the class name being matched against.
+        To prevent this, we use spaces as a delimiter at the beginning and end of
+        the class name, and ensure there's only one space before and after all
+        class names in the class name list.
+
+        Example: consider two class names with cache-busting suffixes:
+        * Header_soldStatusNotice__12345
+        * Header_soldStatusNoticeBackground__ABCDE
+
+        Matching against Header_soldStatusNotice using a naive implementation with
+        `contains()` would match both class names, even though only the first one
+        is the correct match. 
+        '''
+        # Construct xpath selector.  
+        xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+        xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+        xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+        xpath_str += "' '), "                     # Step 4: a trailing space...
+        xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+        try:
+            match = driver.find_element(By.XPATH, xpath_str)
+            return match.get_attribute('textContent')
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            pass
+
+        # We didn't find a match, try again by doing a prefix match and assume
+        # the full class name ends with two underscores followed by a cache-
+        # busting string.    
+        xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+        xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+        xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+        xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+        try:
+            match = driver.find_element(By.XPATH, xpath_str)
+            return match.get_attribute('textContent')
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            pass
+
+        # No match, return empty string.
+        return ''
+
+
+    def modal_close():
+        try:    
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+    def notice_close():
+        try:
+            # Clear any notice
+            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+        except:
+            pass
+
+    def cookie_close():
+        try:  
+            cookie_button = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+            )
+            print(f"cookie_button = {cookie_button}")
+            cookie_button.click()
+        except:
+            pass
+
+    cookie_close()
+    modal_close()
+    # Define callback function for Pub/Sub messages
+    def callback(message):
+        # Extract zip code from message data
+        zip_code = message.message.data.decode('utf-8')
+
+        print(f"Received zip code: {zip_code}")
+
+        try:
+            
+            modal_close()
+            notice_close()
+
+            # Enter zip code in the input field and click the "Check" button
+            zip_button = WebDriverWait(driver, 5).until(
+                EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+            )
+            # print(f"{zip_button:}")
+            zip_button.click()
+
+            # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+            zip_input = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+            )
+            # print(f"{zip_input:}")
+            zip_input.click()
+            # print("clearing zip_input")
+            zip_input.send_keys("\b\b\b\b\b")
+            # print("sending zip code")
+            zip_input.send_keys(zip_code)
+
+            check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+            check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+            # print(f"{check_button:}")
+            check_button.click()
+
+            notice_close()
+            modal_close()
+
+            # Record sales count, if any
+            sales_count_element = get_element_text('Header_count')
+            count_start_index = sales_count_element.index('sold ') + len('sold ')
+            count_end_index = sales_count_element.index('+')
+            sales_count = sales_count_element[count_start_index:count_end_index]
+            # Record any sold status notice
+            sold_status = get_element_text('UnavailableCartbtn_container')
+            if sold_status == 'Notify me':
+                sold_status = 'Sold Out'
+
+        # Extract the price of rice crackers for this zip code
+            # Check if there is a minimum order quantity
+            min_quantity: float = 0
+            total_price: float = 0
+            unit_price: float = 0
+            try:
+                minimum_quantity_str = WebDriverWait(driver, 5).until(EC.presence_of_element_located(((By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML'))))
+                if minimum_quantity_str:
+                    regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
+                    match = re.search(regexp,minimum_quantity_str)
+                    min_quantity, total_price = match.groups()
+            except selenium.common.exceptions.NoSuchElementException as nsee:
+                print("No minimum purhcase string found")
+            except Exception as e:
+                print(f"Exception when finding minimum purchase string: {e}")
+    
+
+
+            ld_json_element = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//script[@type='application/ld+json']")))
+            ld_json_text = ld_json_element.get_attribute('innerHTML')
+            ld_json_start_index = ld_json_text.index('{')
+            ld_json_end_index = ld_json_text.rindex('}') + 1
+            ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
+            ld_json = json.loads(ld_json_json)
+            ld_json_price = ld_json['offers']['price']
+            # Print the price
+            print(ld_json_price)
+
+            unit_price_element = WebDriverWait(driver, 5).until(EC.presence_of_element_located(((By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText'))))
+            unit_price = unit_price_element.replace("$", "")
+            print(f"unit_price = {unit_price}" )
+
+            subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+            # Append the zip code and price to the dataframe
+            #append_to_df(zip_code, price)
+            new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]})
+            df_new = pd.concat([df_new, new_row], ignore_index=True)
+            return df_new
+            #print(f"Recorded the zip code and price: {zip_code},{price}")
+            #print(f"Updated a dataframe: {df_new}")
+
+        except Exception as e:
+            print(f"Caught exception: {e}")
+            print(f"Price not found for zip code {zip_code}")
+            return df_new
+
+    # Start receiving messages (sync version)
+    while True: 
+        pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+        messages = []
+        try:
+            pull_response = subscriber.pull(request = pull_request)
+            messages = pull_response.received_messages
+        except DeadlineExceeded:
+            pass
+
+        if messages:
+            df_new = callback(messages[0])
+            print(f"Updated a dataframe: {df_new}")
+        else:
+            print(f"no messages, trying again at {time.asctime()}")
+            break
+    return df_new
+
+'''
+If using sys to get the function, please remove the following part of code.
+'''
+if __name__ == '__main__':
+    project_id = input("Enter the project ID: ")
+    subscription_name = input("Enter the subscription name: ")
+    url = input("Enter the URL: ")
+
+    get_product_prices(project_id, subscription_name, url)
+
+
+df_new = get_product_prices("fp-20230425", "fp-subscription-NFL-zipcodes", "https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119")
+
+# Now we have the new dataframe "df" recording the new price listed in weee's page
+# Load such data in the existing HDF5 file
+file_path = 'product1.h5'
+with h5py.File(file_path, 'a') as file:
+    # Load the existing dataframe 'df' from the HDF5 file
+    # REPLACE KEY TO GET THE EXISTING DATAFRAME FOR EACH DIFFERENT PRODUCT
+    df = pd.read_hdf(file, key='df')
+
+    # Add timestamp to the new dataframe "df_new"
+    # timestamp = datetime.now()
+    '''
+    To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
+    provides time synchronization with a reliable time server.
+    Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
+    sensible and human-friendly approach to format and convert the timestamps.
+    '''
+    # Fetch the time from an NTP server
+    ntp_client = ntplib.NTPClient()
+    response = ntp_client.request('pool.ntp.org')
+    ntp_time = datetime.fromtimestamp(response.tx_time)
+    timestamp = arrow.get(ntp_time).to('local')
+    df_new['Timestamp'] = timestamp
+
+    # Append 'df_new' to the existing df stored in the key
+    df_update = df.append(df_new, ignore_index=True)
+
+    # Save 'df2' to the HDF5 file with the key 'df'
+    # REPLACE KEY FOR DIFFERENT PRODUCT
+    df_update.to_hdf(file, key='df', mode='a')
+
+print("Data appended and saved successfully.")
@@ -0,0 +1,32 @@
+import http.client
+import xml.etree.ElementTree as ET
+
+conn = http.client.HTTPSConnection("www.sayweee.com")
+payload = ''
+headers = {}
+conn.request("GET", "/en/product/Pete-and-Gerry-s-Organic-Large-Brown-Eggs/65902?category=dairy02", payload, headers)
+res = conn.getresponse()
+data = res.read()
+
+# Parse the HTML page content using ElementTree
+root = ET.fromstring(data.decode('utf-8'))
+
+if root is not None:
+    # Find the script element with __NEXT_DATA__ data
+    next_data_element = None
+    for script_element in root.iter('script'):
+        if script_element.text is not None and '__NEXT_DATA__' in script_element.text:
+            next_data_element = script_element
+            break
+
+    # Extract the __NEXT_DATA__ data
+    if next_data_element is not None:
+        next_data_text = next_data_element.txt
+        next_data_start_index = next_data_text.index('{')
+        next_data_end_index = next_data_text.rindex('}') + 1
+        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+        print(next_data_json)
+    else:
+        print('__NEXT_DATA__ element not found')
+else:
+    print('Failed to parse HTML content')
@@ -0,0 +1,32 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import json
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--headless')
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Load the page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Pete-and-Gerry-s-Organic-Large-Brown-Eggs/65902?category=dairy02'
+driver.get(url)
+wait = WebDriverWait(driver, 10)
+next_data_element = wait.until(EC.presence_of_element_located((By.XPATH, '//script[contains(text(), "__NEXT_DATA__")]')))
+next_data_text = next_data_element.text
+next_data_start_index = next_data_text.index('{')
+next_data_end_index = next_data_text.rindex('}') + 1
+next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+next_data = json.loads(next_data_json)
+
+# Extract the price from the __NEXT_DATA__ data
+price = next_data['props']['pageProps']['product']['price']
+
+# Close the Selenium driver
+driver.quit()
+
+# Print the price
+print(price)
@@ -0,0 +1,43 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+
+# Configure the Selenium driver options
+options = Options()
+#options.add_argument('--headless')  
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Load the page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+driver.get(url)
+time.sleep(5)
+next_data_element = None
+#try:
+next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+#except:
+ #   print('__NEXT_DATA__ element not found')
+
+#time.sleep(1000)
+if next_data_element is not None:
+    next_data_text = next_data_element.get_attribute('innerHTML')
+    print(next_data_text)
+    next_data_start_index = next_data_text.index('{')
+    next_data_end_index = next_data_text.rindex('}') + 1
+    next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+    next_data = json.loads(next_data_json)
+
+    # Extract the price from the __NEXT_DATA__ data
+ #   price = next_data['props']['pageProps']['product']['price']
+    print(next_data)
+    price = next_data['offers']['price']
+    # Print the price
+    print(price)
+else:
+    print('__NEXT_DATA__ element not found')
+
+time.sleep(1000)
+# Close the Selenium driver
+#driver.quit()
@@ -0,0 +1,33 @@
+import re
+import json
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--headless')
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Load the page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+driver.get(url)
+next_data_element = None
+for script_element in driver.find_elements(By.XPATH, '//script'):
+    script_text = script_element.get_attribute('innerHTML')
+    match = re.search(r'__NEXT_DATA__ = (\{.*\});', script_text)
+    if match:
+        next_data_json = match.group(1)
+        next_data = json.loads(next_data_json)
+        # Extract the price from the __NEXT_DATA__ data
+        price = next_data['props']['pageProps']['product']['price']
+        # Print the price
+        print(price)
+        break
+
+if next_data_element is None:
+    print('__NEXT_DATA__ element not found')
+
+# Close the Selenium driver
+driver.quit()
@@ -0,0 +1,126 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Define the zip codes for which to extract the price
+zip_codes = ['85305', '30313', '21230', '14127','28202','60605','45202','44114','76011','80204','48226','54304','77054','46225','32202','64129','89118','90301','33056','55415','02035','70112','07073','19148','15212','95054','98134','33607','37213','20785']
+
+# Load the page and extract the __NEXT_DATA__ data
+#url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+url = 'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261'
+#url = 'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467'
+#url = 'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01'
+#url = 'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224'
+#url = 'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02'
+#url = 'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01'
+#url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
+#url = 'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353?trace_id=7847dd6e-c511-48e2-a76e-a283399967b1'
+#url = 'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03'
+
+driver.delete_all_cookies
+driver.get(url)
+
+# Close the cookies setting window
+try:  
+    cookie_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")))
+    print(f"cookie_button = {cookie_button}")
+    cookie_button.click()
+except:
+    pass
+
+# Clear any modal dialog
+try:    
+    modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+    print(f"modal_close = {modal_close}")
+    modal_close.click()
+except:
+    pass
+
+def callback(zip_code):
+    try:
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+
+        try:
+            # Clear any notice
+            notice_close = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+        except:
+            pass
+
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+        # Enter zip code in the input field and click the "Check" button
+        zip_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')))
+        # print(f"{zip_button:}")
+        zip_button.click()
+
+        zip_input = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input')))
+        # print(f"{zip_input:}")
+        zip_input.click()
+        # print("clearing zip_input")
+        zip_input.send_keys("\b\b\b\b\b")
+        # print("sending zip code")
+        zip_input.send_keys(zip_code)
+
+        try:
+            check_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//button[text() = 'Confirm']")))
+            # print(f"{check_button:}")
+        except:
+            check_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[5]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[3]/div/button")))
+        finally:
+            check_button.click()
+
+        try:
+            # Clear any notice
+            notice_close = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+        except:
+            pass
+
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+            # Extract the price of rice crackers for this zip code
+        try:
+            sold_out_notice = driver.find_element(By.XPATH, '//*[@id="soldStatusNotice"]')
+            print(f"No such product in this area = {zip_code, sold_out_notice}")
+        except:
+            print(f"The product is available in this area {zip_code}")
+    
+    except Exception as e:
+        print(f"Caught exception: {e}")
+        print(f"Price not found for zip code {zip_code}")
+
+for zip_code in zip_codes:
+    callback(zip_code)
+
+driver.quit()
@@ -0,0 +1,64 @@
+from google.cloud import pubsub_v1
+import csv
+import os
+import asyncio
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+# project_id = "fp-20230425"
+project_id = "svig-384801"
+
+'''
+topic_name = "fp-topic-NFL-zipcodes"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
+subscriber.delete_subscription(request={"subscription": subscription_path})
+print("Subscription deleted: {}".format(subscription_path))
+
+# Delete the topic
+publisher = pubsub_v1.PublisherClient()
+topic_path = publisher.topic_path(project_id, topic_name)
+publisher.delete_topic(request={"topic": topic_path})
+print(f"Topic '{topic_name}' deleted.")
+
+'''
+
+
+# Create a pubsub topic
+publisher = pubsub_v1.PublisherClient()
+topic_name = "fp-topic-NFL-zipcodes"
+topic_path = publisher.topic_path(project_id, topic_name)
+topic = publisher.create_topic(request={"name": topic_path}, timeout=30)
+
+
+# Create a Pub/Sub subscription
+subscriber = pubsub_v1.SubscriberClient()
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+subscription = subscriber.create_subscription(
+    name=subscription_path, topic=topic_path
+)
+'''
+# Publish a message to the topic
+# Read the data from the csv file
+# file_path = "C:/Users/Karen/Desktop/NFL_Cities.csv"
+file_path = zipcodes.csv
+
+with open(file_path, "r") as csv_file:
+    reader = csv.reader(csv_file)
+    next(reader)  # skip header row
+    for row in reader:
+        # Format the data as a string and publish it to the topic
+        data = ",".join(row).encode("utf-8")
+        future = publisher.publish(topic_path, data)
+        print(f"Published message ID {future.result()}")
+
+
+async def receive_messages():
+    response = subscriber.pull(subscription=subscription_path, max_messages=1)
+    for received_message in response.received_messages:
+        print("Received message: {}".format(received_message.message.data))
+       
+    
+asyncio.run(receive_messages())
+'''
@@ -0,0 +1,55 @@
+import json
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import time
+from google.cloud import pubsub_v1
+import os
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+
+options = Options()
+options.add_argument('--headless')
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+def callback(message):
+    try:
+        # Get the zip code from the message
+        zip_code = message.data.decode('utf-8')
+        print(f"Received zip code: {zip_code}")
+        
+        # Load the webpage with the zip code
+        url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+        driver.get(url)
+        driver.find_element(By.XPATH, '/html/body/div[4]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[2]/input').send_keys(zip_code)
+        driver.find_element(By.XPATH, '/html/body/div[4]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[3]/div').click()
+        
+        # Extract the price from the webpage
+        next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        next_data_text = next_data_element.get_attribute('innerHTML')
+        next_data_start_index = next_data_text.index('{')
+        next_data_end_index = next_data_text.rindex('}') + 1
+        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+        next_data = json.loads(next_data_json)
+        price = next_data['offers']['price']
+        
+        # Print the result
+        print(f"Price of rice crackers in zip code {zip_code}: {price}")
+    except Exception as e:
+        print(f"Error processing message: {str(e)}")
+    finally:
+        message.ack()
+
+subscriber.subscribe(subscription_path, callback=callback)
+
+while True:
+    time.sleep(1)
+
+
@@ -0,0 +1,106 @@
+from google.cloud import pubsub_v1
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+import os
+from concurrent.futures import TimeoutError
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+# project_id = "fp-20230425"
+project_id = "svig-384801"
+subscription_name = "fp-subscription-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+timeout = 5
+
+# Load the product page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+driver.get(url)
+# time.sleep(5)
+'''
+next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+if next_data_element is not None:
+    next_data_text = next_data_element.get_attribute('innerHTML')
+    next_data_start_index = next_data_text.index('{')
+    next_data_end_index = next_data_text.rindex('}') + 1
+    next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+    next_data = json.loads(next_data_json)
+else:
+    print('__NEXT_DATA__ element not found')
+'''
+    
+# Define callback function for Pub/Sub messages
+def callback(message):
+    # Extract zip code from message data
+    zip_code = message.data.decode('utf-8')
+    print(f"Received zip code: {zip_code}")
+
+    try:
+
+        time.sleep(5)
+        # Clear any modal dialog
+        modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+        print(f"modal_close = {modal_close}")
+        modal_close.click()
+        time.sleep(2)
+
+        # Enter zip code in the input field and click the "Check" button
+        zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]/div/div/div[1]')
+        print(zip_button)
+        zip_button.click()
+        time.sleep(1)
+
+        zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+        zip_input.clear()
+        zip_input.send_keys(zip_code)
+
+        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+        check_button.click()
+        time.sleep(2)
+
+        # Extract the price of rice crackers for this zip code
+        '''
+        price_element = driver.find_element(By.XPATH, '/html/body/div[2]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')
+        price = price_element.text.strip()[1:]
+        print(price)
+        '''
+        next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        next_data_text = next_data_element.get_attribute('innerHTML')
+        next_data_start_index = next_data_text.index('{')
+        next_data_end_index = next_data_text.rindex('}') + 1
+        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+        next_data = json.loads(next_data_json)
+        price = next_data['offers']['price']
+    # Print the price
+        print(price)
+        message.ack()  # acknowledge receipt of message
+    except:
+        print(f"Price not found for zip code {zip_code}")
+        price = 'Price not found'
+
+# Start receiving messages
+streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback)
+print(f'Started receiving messages from subscription {subscription_path}...')
+
+# Wrap subscriber in a 'with' block to automatically call close() when done.
+with subscriber:
+    try:
+        # When `timeout` is not set, result() will block indefinitely,
+        # unless an exception is encountered first.
+        streaming_pull_future.result(timeout=timeout)
+    except TimeoutError:
+        streaming_pull_future.cancel()  # Trigger the shutdown.
+        streaming_pull_future.result()  # Block until the shutdown is complete.
+
+# Keep the script running indefinitely to continue receiving messages
+while True:
+    time.sleep(10)
@@ -0,0 +1,166 @@
+import sqlalchemy
+from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Float, DateTime
+from sqlalchemy.orm import declarative_base
+from sqlalchemy.orm import relationship, sessionmaker
+import sys
+from flask import Flask, render_template
+
+# Create the Flask application
+app = Flask(__name__)
+
+# Create the database engine
+engine = create_engine('sqlite:///scraper_design.db')
+
+# Create a session factory
+Session = sessionmaker(bind=engine)
+
+# Create a base class
+Base = declarative_base()
+
+# Define the product class
+class Product(Base):
+    __tablename__ = 'product'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    SKU_PLU_UPC_number = Column(String)
+    price_histories = relationship("PriceHistory", back_populates="product")
+
+# Define the retailer class
+class Retailer(Base):
+    __tablename__ = 'retailer'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    products = relationship("Product", secondary="retailer_product")
+
+# Define the price class
+class Price(Base):
+    __tablename__ = 'price'
+    id = Column(Integer, primary_key=True)
+    MOQ = Column(Integer)
+    price = Column(Float)
+
+# Define the scraperjob class
+class ScraperJob(Base):
+    __tablename__ = 'scraper_job'
+    id = Column(Integer, primary_key=True)
+    start_time = Column(DateTime)
+    end_time = Column(DateTime)
+    parameters = Column(String)
+    statistics = Column(String)
+
+# Define the price history class
+class PriceHistory(Base):
+    __tablename__ = 'price_history'
+    id = Column(Integer, primary_key=True)
+    location = Column(String)
+    zip_code = Column(Integer)
+    city_name = Column(String)
+    price_at_time = Column(DateTime)
+    price = Column(Float)
+    scraper_job_id = Column(Integer, ForeignKey('scraper_job.id'))
+    product_id = Column(Integer, ForeignKey('product.id'))
+    product = relationship("Product", back_populates="price_histories")
+    scraper_job = relationship("ScraperJob")
+
+# Define the user class
+class User(Base):
+    __tablename__ = 'user'
+    id = Column(Integer, primary_key=True)
+    username = Column(String)
+    credentials = Column(String)
+    creation_date = Column(DateTime)
+    modification_date = Column(DateTime)
+    def __str__(self):
+        return f"{self.id=} {self.username=} {self.creation_date=} {self.modification_date=}"
+
+class RetailerProduct(Base):
+    __tablename__ = 'retailer_product'
+    retailer_id = Column(Integer, ForeignKey('retailer.id'), primary_key=True)
+    product_id = Column(Integer, ForeignKey('product.id'), primary_key=True)
+
+# Create the database tables
+Base.metadata.create_all(engine)
+
+@app.route('/')
+def homepage():
+    # Create a session
+    session = Session()
+
+    # Get all the classes from SQLAlchemy
+    classes = Base.__subclasses__()
+
+    statement = sqlalchemy.select(User)
+    users = session.scalars(statement).all()
+
+    # Render the template with the classes
+    return render_template('test3.html', classes=classes, users=users)
+
+# Create command-line arguments for the user class
+def main():
+    if len(sys.argv) < 2:
+        app.run()
+
+    command = sys.argv[1]
+
+    if command == "create":
+        if len(sys.argv) != 3:
+            print("Usage: python user_tool.py create [username]")
+            return
+
+        username = sys.argv[2]
+
+        # Create a new user object
+        user = User(username=username)
+
+        # Add the user to the database
+        session = Session()
+        session.add(user)
+        session.commit()
+
+        print("User created successfully.")
+
+    elif command == "edit":
+        if len(sys.argv) != 4:
+            print("Usage: python user_tool.py edit [user_id] [new_username]")
+            return
+
+        user_id = int(sys.argv[2])
+        new_username = sys.argv[3]
+
+        # Retrieve the user from the database
+        session = Session()
+        user = session.query(User).get(user_id)
+
+        if user:
+            # Update the user object
+            user.username = new_username
+            session.commit()
+            print("User updated successfully.")
+        else:
+            print("User not found.")
+
+    elif command == "delete":
+        if len(sys.argv) != 3:
+            print("Usage: python user_tool.py delete [user_id]")
+            return
+
+        user_id = int(sys.argv[2])
+
+        # Retrieve the user from the database
+        session = Session()
+        user = session.query(User).get(user_id)
+
+        if user:
+            # Delete the user from the database
+            session.delete(user)
+            session.commit()
+            print("User deleted successfully.")
+        else:
+            print("User not found.")
+
+    else:
+        print("Usage: python user_tool.py [create|edit|delete]")
+        return
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1 @@
+print("hello")
@@ -0,0 +1,54 @@
+'''
+import os
+from flask import Flask, Response
+from urllib.parse import urlparse
+
+app = Flask(__name__)
+
+@app.route('/')
+def index():
+    return 'Hello, welcome to the cat picture server!'
+
+@app.route('/<path:path>')
+def serve_jpeg(path):
+    if path.endswith('.jpg'):
+        file_path = path[1:]
+        if os.path.isfile(file_path):
+            with open(file_path, 'rb') as file:
+                return Response(file.read(), content_type='image/jpeg')
+        else:
+            return 'File not found', 404
+    else:
+        return 'Only .jpg files are allowed', 403
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=8000)
+'''
+'''
+import os
+from fastapi import FastAPI
+from fastapi.responses import FileResponse
+from urllib.parse import urlparse
+
+app = FastAPI()
+
+@app.get('/')
+def index():
+    return 'Hello, welcome to the cat picture server!'
+
+@app.get('/{path:path}')
+async def serve_jpeg(path: str):
+    if path.endswith('.jpg'):
+        file_path = path[1:]
+        if os.path.isfile(file_path):
+            return FileResponse(file_path, media_type='image/jpeg')
+        else:
+            return {'detail': 'File not found'}, 404
+    else:
+        return {'detail': 'Only .jpg files are allowed'}, 403
+
+if __name__ == '__main__':
+    import uvicorn
+    uvicorn.run(app, host='0.0.0.0', port=8000)
+'''
+
@@ -0,0 +1,14 @@
+import sys
+
+# Iterate through the modules in sys.modules
+for module_name, module in sys.modules.items():
+    # Check if the module has a loader attribute
+    if hasattr(module, '__loader__'):
+        loader = module.__loader__
+        # Determine the type of loader
+        if type(loader).__name__ == 'SourceFileLoader':
+            print(f"Module '{module_name}' has a source loader.")
+        elif type(loader).__name__ == 'BuiltinImporter':
+            print(f"Module '{module_name}' has a built-in loader.")
+        elif type(loader).__name__ == 'ExtensionFileLoader':
+            print(f"Module '{module_name}' has an extension loader.")
@@ -0,0 +1,12 @@
+from cx_Freeze import setup, Executable
+
+executables = [Executable('module-inspector.py')]
+
+setup(
+    name='wow',
+    version='1.0',
+    description='Module-inspector Application',
+    executables=executables
+)
+
+
@@ -0,0 +1,7 @@
+i = 5
+
+def f(arg=i):
+    print(arg)
+
+i = 6
+f()
@@ -0,0 +1,47 @@
+import importlib.util
+import os
+
+def get_module_size(module_name):
+    """Get the total number of bytes in the cached .pyc file or .py file of a module or package.
+
+    Args:
+        module_name (str): The name of the module or package.
+
+    Returns:
+        int: The total number of bytes in the cached .pyc file or .py file.
+    """
+    spec = importlib.util.find_spec(module_name)
+
+    if spec is not None and spec.has_location:
+        # If the module is found and has a location, check for .pyc file first
+        pyc_file_path = spec.origin + 'c'
+        if os.path.exists(pyc_file_path):
+            return os.stat(pyc_file_path).st_size
+        else:
+            # If .pyc file doesn't exist, check for .py file
+            py_file_path = spec.origin
+            if os.path.exists(py_file_path):
+                return os.stat(py_file_path).st_size
+
+    return 0  # Return 0 if no .pyc or .py file found
+
+
+import sys
+
+def get_total_module_size():
+    """Get the total number of bytes for all modules listed in sys.modules.
+
+    Returns:
+        int: The total number of bytes for all modules.
+    """
+    total_size = 0
+    for module_name in sys.modules:
+        module_size = get_module_size(module_name)
+        total_size += module_size
+
+    return total_size
+
+if __name__ == '__main__' and __package__ is None:
+    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
+total_size = get_total_module_size()
+print("Total size of all modules in sys.modules: {} bytes".format(total_size))
@@ -0,0 +1,74 @@
+import os
+import sys
+import importlib.util
+import types
+from collections import defaultdict
+from sys import getsizeof
+
+
+def get_bytecode_size(module_name):
+    """Get the size of the cached .pyc bytecode file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin and spec.origin.endswith('.py'):
+        pyc_file = importlib.util.cache_from_source(spec.origin)
+        if os.path.exists(pyc_file):
+            return os.path.getsize(pyc_file)
+    return 0
+
+
+def get_dso_size(module_name):
+    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin and spec.origin.endswith(('.so', '.dll')):
+        return os.path.getsize(spec.origin)
+    return 0
+
+
+def generate_module_report():
+    """Generate a report summarizing the module environment."""
+    total_modules = len(sys.modules)
+    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
+    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
+    python_size = sum(
+        getsizeof(sys.modules[module_name]) - getsizeof(module_name)
+        for module_name in sys.modules
+        if isinstance(sys.modules[module_name], types.ModuleType)
+    )
+    total_size = bytecode_size + dso_size + python_size
+
+    # Calculate percentage of each type of module
+    bytecode_percent = bytecode_size / total_size * 100
+    dso_percent = dso_size / total_size * 100
+    python_percent = python_size / total_size * 100
+
+    # Generate the report
+    report = f"Total modules: {total_modules}\n"
+    report += f"Total size: {total_size} bytes\n"
+    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
+    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
+    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
+
+    # Generate histogram of module directory names
+    dir_histogram = defaultdict(int)
+    for module_name in sys.modules:
+        module_file = getattr(sys.modules[module_name], '__file__', None)
+        if module_file:
+            module_dir = os.path.dirname(module_file)
+            dir_histogram[module_dir] += 1
+
+    report += "\nModule Directory Histogram:\n"
+    for module_dir, count in dir_histogram.items():
+        report += f"{module_dir}: {'*' * count}\n"
+
+    return report
+
+
+if __name__ == "__main__":
+    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
+report = generate_module_report()
+print(report)
+
+
+
+
+
@@ -2,6 +2,7 @@ import sys
 import tkinter as tk
 from tkinter import ttk

+
 def on_treeview_heading_click(column, tree):
    data = [(tree.set(child, column), child) for child in tree.get_children("")]
    reverse = (column == tree.heading(column)["text"]) and not column.startswith("_")
@@ -38,6 +39,10 @@ tree.column('#0', width=20, stretch=False)

 tree.tag_configure("selected", background="#ccc")

+# Call the on_treeview_heading_click function when a column heading is clicked
+for col in tree["columns"]:
+    tree.heading(col, text=col, anchor=tk.W, command=lambda col=col: on_treeview_heading_click(col, tree))
+root.mainloop()
 # Call the on_treeview_heading_click function when a column heading is clicked
 for col in tree["columns"]:
    tree.heading(col, text=col, anchor=tk.W, command=lambda col=col: on_treeview_heading_click(col, tree))
@@ -0,0 +1,8 @@
+google.cloud.pubsub
+plotly
+pandas
+numpy
+h5py
+selenium
+sqlalchemy
+flask
@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Class Dump</title>
+</head>
+<body>
+    <h1>Class Dump</h1>
+    <ul>
+        {% for class in classes %}
+        <li>{{ class.__name__ }}</li>
+        {% endfor %}
+    </ul>
+
+    <h1>User Dump</h1>
+    <ul>
+        {% for user in users %}
+        <li>{{ user }}</li>
+        {% endfor %}
+    </ul>
+</body>
+</html>
@@ -0,0 +1,31 @@
+class Count:
+    def __init__(self,mymin,mymax):
+        self.mymin=mymin
+        self.mymax=mymax    
+
+    def __getattr__(self, item):
+        self.__dict__[item]=0
+        return 0
+
+obj1 = Count(1,10)
+print(obj1.mymin)
+print(obj1.mymax)
+print(obj1.mycurrent1)
+
+class Count:
+
+    def __init__(self,mymin,mymax):
+        self.mymin=mymin
+        self.mymax=mymax
+        self.current=None
+   
+    def __getattribute__(self, item):
+        if item.startswith('cur'):
+            raise AttributeError
+        return object.__getattribute__(self,item) 
+        # or you can use ---return super().__getattribute__(item)
+
+obj1 = Count(1,10)
+print(obj1.mymin)
+print(obj1.mymax)
+print(obj1.current)
@@ -0,0 +1,14 @@
+import sys
+
+# Iterate through the modules in sys.modules
+for module_name, module in sys.modules.items():
+    # Check if the module has a loader attribute
+    if hasattr(module, '__loader__'):
+        loader = module.__loader__
+        # Determine the type of loader
+        if type(loader).__name__ == 'SourceFileLoader':
+            print(f"Module '{module_name}' has a source loader.")
+        elif type(loader).__name__ == 'BuiltinImporter':
+            print(f"Module '{module_name}' has a built-in loader.")
+        elif type(loader).__name__ == 'ExtensionFileLoader':
+            print(f"Module '{module_name}' has an extension loader.")
@@ -0,0 +1,12 @@
+from cx_Freeze import setup, Executable
+
+executables = [Executable('module-inspector.py')]
+
+setup(
+    name='wow',
+    version='1.0',
+    description='Module-inspector Application',
+    executables=executables
+)
+
+
@@ -0,0 +1,58 @@
+import time
+'''import matplotlib.pyplot as plt'''
+
+# Durations to measure in seconds
+durations = [0.01, 0.1, 1.0]
+
+# Number of repetitions
+num_repetitions = 100
+
+# Lists to store jitter for each duration
+jitter_10ms = []
+jitter_100ms = []
+jitter_1s = []
+
+# Function to measure jitter
+def measure_jitter(duration):
+    start_time = time.perf_counter_ns()
+    time.sleep(duration)
+    end_time = time.perf_counter_ns()
+    return end_time - start_time
+
+# Measure jitter for each duration and repeat for num_repetitions times
+for duration in durations:
+    for _ in range(num_repetitions):
+        jitter = measure_jitter(duration)
+        if duration == 0.01:
+            jitter_10ms.append(jitter)
+        elif duration == 0.1:
+            jitter_100ms.append(jitter)
+        elif duration == 1.0:
+            jitter_1s.append(jitter)
+'''
+# Create histograms for each duration's jitter
+plt.hist(jitter_10ms, bins=10, color='r', alpha=0.5, label='10ms')
+plt.hist(jitter_100ms, bins=10, color='g', alpha=0.5, label='100ms')
+plt.hist(jitter_1s, bins=10, color='b', alpha=0.5, label='1s')
+plt.xlabel('Jitter (ns)')
+plt.ylabel('Frequency')
+plt.title('Jitter of time.sleep() for Different Durations')
+plt.legend()
+plt.show()
+'''
+
+import random
+import secrets
+
+# Create a dictionary with 1 million integer keys
+dict = {}
+
+for i in range(1000000):
+    # Generate a random 64-byte value as bytes
+    random_value = secrets.token_bytes(64)
+    # Convert the bytes value to hexadecimal representation
+    hex_value = random_value.hex()
+    # Add the key-value pair to the dictionary
+    dict[i] = hex_value
+
+print("Dictionary created with 1 million integer keys.")
@@ -0,0 +1,51 @@
+
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+        self._known_good_copy = self.__dict__.copy()
+
+    def __getattribute__(self, name):
+        if name == "treasure":
+            if self.__dict__ != self._known_good_copy:
+                print("Attribute deleted!")
+            return self._treasure
+        else:
+            return object.__getattribute__(self, name)
+
+    def __setattr__(self, name, value):
+        if name != "_treasure" and name != "_known_good_copy":
+            print("Attribute deletion detected!")
+            self._known_good_copy = self.__dict__.copy()
+        object.__setattr__(self, name, value)
+v = Vault()
+Vault.treasure = " haha monkeypatched"
+print(v.treasure)
+
+'''
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+        self._known_good_copy = self.__dict__.copy()
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+        else:
+            if instance.__dict__ != self._known_good_copy:
+                print("Attribute deleted!")
+            return self._treasure
+
+    def __set__(self, instance, value):
+        print("Attribute deletion detected!")
+        self._known_good_copy = instance.__dict__.copy()
+
+    def __delete__(self, instance):
+        print("Attribute deletion detected!")
+        self._known_good_copy = instance.__dict__.copy()
+
+    def __set_name__(self, owner, name):
+        self.name = name
+v = Vault()
+Vault._treasure = " haha monkeypatched"
+print(v._treasure)
+'''
@@ -0,0 +1,17 @@
+import asyncio
+
+# Define a coroutine function
+async def greet(name):
+    print(f"Hello, {name}!")
+    await asyncio.sleep(1)
+    print(f"Goodbye, {name}!")
+
+# Call the coroutine function
+coroutine_obj = greet("Karen")
+
+# coroutine_obj is a coroutine object, not the actual execution of the coroutine
+
+# Use an event loop to run the coroutine
+loop = asyncio.get_event_loop()
+loop.run_until_complete(coroutine_obj)
+loop.close()
@@ -0,0 +1,8 @@
+def think():
+    x=1
+    def think2():
+        nonlocal x
+        x=2
+    think2()
+    return x
+print(think())
@@ -0,0 +1,33 @@
+'''
+import sys
+import importlib.util
+
+# Load only the sys module
+importlib.import_module('sys')
+
+# Check the type of the value associated with the 'os' key
+os_module = sys.modules['os']
+os_module_type = type(os_module)
+
+# Print the type of the value associated with the 'os' key
+print("Type of the value associated with the 'os' key in sys.modules: {}".format(os_module_type))
+'''
+'''
+import os
+print(type(os))
+'''
+
+import builtins
+
+# Access the os module from globals()
+os = globals().get('os', None)
+
+# Check if os module is available
+if os is not None:
+    # Call os.getlogin()
+    username = os.getlogin()
+    print("Username:", username)
+else:
+    print("os module not available")
+
+
@@ -0,0 +1,54 @@
+import os
+import sys
+import importlib.util
+import types
+
+
+def get_bytecode_size(module_name):
+    """Get the size of the cached .pyc bytecode file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith('.py'):
+        pyc_file = importlib.util.cache_from_source(spec.origin)
+        if os.path.exists(pyc_file):
+            return os.path.getsize(pyc_file)
+    return 0
+
+
+def get_dso_size(module_name):
+    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith(('.so', '.dll')):
+        return os.path.getsize(spec.origin)
+    return 0
+
+
+def generate_module_report():
+    """Generate a report summarizing the module environment."""
+    total_modules = len(sys.modules)
+    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
+    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
+    python_size = sum(
+        sys.modules[module_name].__sizeof__()
+        for module_name in sys.modules
+        if isinstance(sys.modules[module_name], types.ModuleType)
+    )
+    total_size = bytecode_size + dso_size + python_size
+
+    # Calculate percentage of each type of module
+    bytecode_percent = bytecode_size / total_size * 100
+    dso_percent = dso_size / total_size * 100
+    python_percent = python_size / total_size * 100
+
+    # Generate the report
+    report = f"Total modules: {total_modules}\n"
+    report += f"Total size: {total_size} bytes\n"
+    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
+    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
+    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
+
+    return report
+
+if __name__ == '__main__' and __package__ is None:
+    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
+    report = generate_module_report()
+    print(report)
@@ -0,0 +1,259 @@
+'''
+import os
+import sys
+import importlib.util
+import types
+import inspect
+from collections import defaultdict
+
+
+def get_bytecode_size(module_name):
+    """Get the size of the cached .pyc bytecode file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith('.py'):
+        pyc_file = importlib.util.cache_from_source(spec.origin)
+        if os.path.exists(pyc_file):
+            return os.path.getsize(pyc_file)
+    return 0
+
+
+def get_dso_size(module_name):
+    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith(('.so', '.dll')):
+        return os.path.getsize(spec.origin)
+    return 0
+
+
+def generate_module_report():
+    """Generate a report summarizing the module environment."""
+    total_modules = len(sys.modules)
+    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
+    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
+    python_size = sum(
+        sys.modules[module_name].__sizeof__()
+        for module_name in sys.modules
+        if isinstance(sys.modules[module_name], types.ModuleType)
+    )
+    total_size = bytecode_size + dso_size + python_size
+
+    # Calculate percentage of each type of module
+    bytecode_percent = bytecode_size / total_size * 100
+    dso_percent = dso_size / total_size * 100
+    python_percent = python_size / total_size * 100
+
+    # Generate the report
+    report = f"Total modules: {total_modules}\n"
+    report += f"Total size: {total_size} bytes\n"
+    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
+    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
+    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
+
+    # Generate histogram of module directory names
+    dir_histogram = defaultdict(int)
+    for module_name in sys.modules:
+        module_file = sys.modules[module_name].__file__
+        if module_file:
+            module_dir = os.path.dirname(module_file)
+            dir_histogram[module_dir] += 1
+
+    report += "\nModule Directory Histogram:\n"
+    for module_dir, count in dir_histogram.items():
+        report += f"{module_dir}: {'*' * count}\n"
+
+    return report
+
+
+if __name__ == '__main__' and __package__ is None:
+    script_path = inspect.getframeinfo(inspect.currentframe()).filename
+    __spec__ = importlib.util.spec_from_loader('__main__', None, origin=script_path)
+    report = generate_module_report()
+    print(report)
+'''
+'''
+import os
+import sys
+from importlib.util import find_spec
+
+def get_module_size(module):
+    spec = find_spec(module)
+    if spec is None:
+        return None
+    if spec.cached:
+        size = os.stat(spec.cached).st_size
+    else:
+        size = os.stat(spec.origin).st_size
+    return size
+
+def summarize_modules():
+    total_size = 0
+    byte_count = 0
+    dso_count = 0
+    py_count = 0
+    directory_histogram = {}
+    for module_name, module in sys.modules.items():
+        if module is None:
+            continue
+        size = get_module_size(module_name)
+        if size is None:
+            continue
+        total_size += size
+        if module_name.endswith('_bootstrap') or module_name.endswith('_bootstrap_external'):
+            continue
+        if module_name.startswith('_frozen_importlib'):
+            dso_count += 1
+            continue
+        if module_name.startswith('_'):
+            continue
+        if module.__file__.endswith('.pyc'):
+            byte_count += size
+        elif module.__file__.endswith(('.so', '.dll')):
+            dso_count += 1
+        else:
+            py_count += size
+        module_directory = os.path.dirname(os.path.abspath(module.__file__))
+        directory_histogram[module_directory] = directory_histogram.get(module_directory, 0) + 1
+
+    print(f"Total module size: {total_size} bytes")
+    print(f"Bytecode modules: {(byte_count/total_size)*100:.2f}% ({byte_count} bytes)")
+    print(f"DSO modules: {(dso_count/total_size)*100:.2f}% ({dso_count} modules)")
+    print(f"Python modules: {(py_count/total_size)*100:.2f}% ({py_count} bytes)")
+
+    print("Module directory histogram:")
+    for directory, count in directory_histogram.items():
+        print(f"{directory}: {'*' * count}")
+
+if __name__ == '__main__':
+    summarize_modules()
+'''
+'''
+import os
+import sys
+from importlib.util import find_spec
+
+
+def get_module_size(module):
+    spec = find_spec(module)
+    if spec is None:
+        return None
+    if spec.cached:
+        size = os.stat(spec.cached).st_size
+    else:
+        size = os.stat(spec.origin).st_size
+    return size
+
+
+def summarize_modules():
+    total_size = 0
+    byte_count = 0
+    dso_count = 0
+    py_count = 0
+    directory_histogram = {}
+    for module_name, module in sys.modules.items():
+        if module is None:
+            continue
+        size = get_module_size(module_name)
+        if size is None:
+            continue
+        total_size += size
+        if module_name.endswith('_bootstrap') or module_name.endswith('_bootstrap_external'):
+            continue
+        if module_name.startswith('_frozen_importlib'):
+            dso_count += 1
+            continue
+        if module_name.startswith('_'):
+            continue
+        if module.__file__.endswith('.pyc'):
+            byte_count += size
+        elif module.__file__.endswith(('.so', '.dll')):
+            dso_count += 1
+        else:
+            py_count += size
+        module_directory = os.path.dirname(os.path.abspath(module.__file__))
+        directory_histogram[module_directory] = directory_histogram.get(module_directory, 0) + 1
+
+    print(f"Total module size: {total_size} bytes")
+    print(f"Bytecode modules: {(byte_count/total_size)*100:.2f}% ({byte_count} bytes)")
+    print(f"DSO modules: {(dso_count/total_size)*100:.2f}% ({dso_count} modules)")
+    print(f"Python modules: {(py_count/total_size)*100:.2f}% ({py_count} bytes)")
+
+    print("Module directory histogram:")
+    for directory, count in directory_histogram.items():
+        print(f"{directory}: {'*' * count}")
+
+
+if __name__ == '__main__':
+    summarize_modules()
+'''
+
+import os
+import sys
+import importlib.util
+import types
+from collections import defaultdict
+
+
+def get_bytecode_size(module_name):
+    """Get the size of the cached .pyc bytecode file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith('.py'):
+        pyc_file = importlib.util.cache_from_source(spec.origin)
+        if os.path.exists(pyc_file):
+            return os.path.getsize(pyc_file)
+    return 0
+
+
+def get_dso_size(module_name):
+    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith(('.so', '.dll')):
+        return os.path.getsize(spec.origin)
+    return 0
+
+
+def generate_module_report():
+    """Generate a report summarizing the module environment."""
+    total_modules = len(sys.modules)
+    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
+    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
+    python_size = sum(
+        sys.modules[module_name].__sizeof__()
+        for module_name in sys.modules
+        if isinstance(sys.modules[module_name], types.ModuleType)
+    )
+    total_size = bytecode_size + dso_size + python_size
+
+    # Calculate percentage of each type of module
+    bytecode_percent = bytecode_size / total_size * 100
+    dso_percent = dso_size / total_size * 100
+    python_percent = python_size / total_size * 100
+
+    # Generate the report
+    report = f"Total modules: {total_modules}\n"
+    report += f"Total size: {total_size} bytes\n"
+    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
+    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
+    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
+
+    # Generate histogram of module directory names
+    dir_histogram = defaultdict(int)
+    for module_name in sys.modules:
+        module_file = sys.modules[module_name].__file__
+        if module_file:
+            module_dir = os.path.dirname(module_file)
+            dir_histogram[module_dir] += 1
+
+    report += "\nModule Directory Histogram:\n"
+    for module_dir, count in dir_histogram.items():
+        report += f"{module_dir}: {'*' * count}\n"
+
+    return report
+
+
+if __name__ == "__main__":
+    report = generate_module_report()
+    print(report)
+
+
+
+
@@ -0,0 +1,77 @@
+import os
+import sys
+import importlib.util
+import types
+from collections import defaultdict
+from sys import getsizeof
+
+
+def get_bytecode_size(module_name):
+    """Get the size of the cached .pyc bytecode file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin and spec.origin.endswith('.py'):
+        pyc_file = importlib.util.cache_from_source(spec.origin)
+        if os.path.exists(pyc_file):
+            return os.path.getsize(pyc_file)
+    return 0
+
+
+def get_dso_size(module_name):
+    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin and spec.origin.endswith(('.so', '.dll')):
+        return os.path.getsize(spec.origin)
+    return 0
+
+
+def generate_module_report():
+    """Generate a report summarizing the module environment."""
+    total_modules = len(sys.modules)
+    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
+    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
+    python_size = sum(
+        getsizeof(sys.modules[module_name]) - getsizeof(module_name)
+        for module_name in sys.modules
+        if isinstance(sys.modules[module_name], types.ModuleType)
+    )
+    total_size = bytecode_size + dso_size + python_size
+
+    # Calculate percentage of each type of module
+    bytecode_percent = bytecode_size / total_size * 100
+    dso_percent = dso_size / total_size * 100
+    python_percent = python_size / total_size * 100
+
+    # Generate the report
+    report = f"Total modules: {total_modules}\n"
+    report += f"Total size: {total_size} bytes\n"
+    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
+    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
+    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
+
+    # Generate histogram of module directory names
+    dir_histogram = defaultdict(int)
+    for module_name in sys.modules:
+        module_file = getattr(sys.modules[module_name], '__file__', None)
+        if module_file:
+            module_dir = os.path.dirname(module_file)
+            dir_histogram[module_dir] += 1
+
+    report += "\nModule Directory Histogram:\n"
+    for module_dir, count in dir_histogram.items():
+        report += f"{module_dir}: {'*' * count}\n"
+
+    return report
+
+
+if __name__ == "__main__":
+    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
+report = generate_module_report()
+print(report)
+
+
+
+
+
+
+    report = generate_module_report()
+    print(report)
@@ -0,0 +1,63 @@
+from fp2_14 import get_product_prices
+import h5py
+import pandas as pd
+import datetime
+import sys
+from datetime import datetime
+import ntplib
+import arrow
+
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-NFL-zipcodes"
+url = "https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f"
+
+df_new = get_product_prices(project_id, subscription_name, url)
+
+'''
+# When using sys to call the 'get_product_prices' function
+
+if __name__ == '__main__':
+    # Check if the required command-line arguments are provided
+    if len(sys.argv) < 4:
+        print("Usage: python test_fp2_14.py project_id subscription_name url")
+        sys.exit(1)
+
+    project_id = sys.argv[1]
+    subscription_name = sys.argv[2]
+    url = sys.argv[3:]
+
+    get_product_prices(project_id, subscription_name, url)
+'''
+
+
+# Now we have the new dataframe "df" recording the new price listed in weee's page
+# Load such data in the existing HDF5 file
+file_path = 'product1.h5'
+with h5py.File(file_path, 'a') as file:
+    # Load the existing dataframe 'df' from the HDF5 file
+    # REPLACE KEY TO GET THE EXISTING DATAFRAME FOR EACH DIFFERENT PRODUCT
+    df = pd.read_hdf(file, key='df')
+
+    # Add timestamp to the new dataframe "df_new"
+    # timestamp = datetime.now()
+    '''
+    To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
+    provides time synchronization with a reliable time server.
+    Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
+    sensible and human-friendly approach to format and convert the timestamps.
+    '''
+    # Fetch the time from an NTP server
+    ntp_client = ntplib.NTPClient()
+    response = ntp_client.request('pool.ntp.org')
+    ntp_time = datetime.fromtimestamp(response.tx_time)
+    timestamp = arrow.get(ntp_time).to('local')
+    df_new['Timestamp'] = timestamp
+
+    # Append 'df_new' to the existing df stored in the key
+    df_update = df.append(df_new, ignore_index=True)
+
+    # Save 'df2' to the HDF5 file with the key 'df'
+    # REPLACE KEY FOR DIFFERENT PRODUCT
+    df_update.to_hdf(file, key='df', mode='a')
+
+print("Data appended and saved successfully.")
@@ -0,0 +1,45 @@
+# Selected
+# Want-Want Rice Crackers
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+
+# Lay's Potato Chips
+url = 'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261'
+
+# Pocky Cookies
+url = 'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467'
+
+# Thai Jasmine Rice
+url = 'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01'
+
+# Japanese Medium Grain Rice
+url = 'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224'
+
+# Millet
+url = 'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02'
+
+# Nature's Soy Firm Tofu: https://www.sayweee.com/zh/product/Nature-s-Soy-Firm-Tofu-/19231?trace_id=ee63a29e-a1e4-4065-b213-96aa14a15521
+
+# Free 2% Reduced Fat Milk: https://www.sayweee.com/zh/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353?trace_id=3030fe4c-04fc-4f8f-9257-7f6395b9e513
+
+# Washington Fuji Apples: https://www.sayweee.com/zh/product/Washington-Fuji-Apples/94817?trace_id=dc4955fc-bba8-48cf-b1f9-9db3cf077eed
+
+# Greenlandic Sweet Prawns: https://www.sayweee.com/zh/product/Royal-Greenland-Cooked-Wild-Caught-Greenlandic-Sweet-Prawns-Frozen-1/44656?category=seafood01
+
+
+
+# Optional
+# LuoSiFen 
+url = 'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01'
+
+# Rice Noodles
+url = 'https://www.sayweee.com/en/product/Sanyangyishi-Fresh-Instant-Rice-Noodles/2156?category=instant01'
+
+# Cup Noodles
+url = 'https://www.sayweee.com/en/product/Nissin-Beef-Flavor-Cup-Noodles/102519?category=instant01'
+
+# Pork Sung
+url = 'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?category=canned04'
+
+# Nature's Soy Firm Tofu: https://www.sayweee.com/zh/product/Nature-s-Soy-Firm-Tofu-/19231?trace_id=ee63a29e-a1e4-4065-b213-96aa14a15521
+# [Not available or sold out in some area] Beef Ribeye Thin Slices: https://www.sayweee.com/zh/product/Beef-Ribeye-Thin-Slices--Frozen-1/4295?category=meat02
+# Shiitake Mushrooms: https://www.sayweee.com/zh/product/Shiitake-Mushrooms/45782?trace_id=ba388941-f631-4eaa-8570-5b7d4aba64e7
@@ -0,0 +1,33 @@
+ZIP Code
+85305
+30313
+21230
+14127
+28202
+60605
+45202
+44114
+76011
+80204
+48226
+54304
+77054
+46225
+32202
+64129
+89118
+90301
+90301
+33056
+55415
+02035
+70112
+07073
+07073
+19148
+15212
+95054
+98134
+33607
+37213
+20785
Author	SHA1	Message	Date
cheezedoodle2	b231bf21a8	Modified startup logic, added user dump	2023-05-24 14:44:55 -07:00
cheezedoodle2	be7bb93395	Updated list of required modules	2023-05-24 14:14:43 -07:00
Karenzz	60cdddabc4	Created object mapping	2023-05-24 17:04:30 -04:00
Karenzz	1aac3a15b1	Defined get_price function & modified timestamp	2023-05-23 17:59:16 -04:00
Karenzz	93d386733a	Task designed	2023-05-22 17:24:46 -04:00
Karenzz	97d4d8eb49	Modified multiprocessing	2023-05-22 17:04:22 -04:00
Karenzz	a2fb3b29aa	Task designed	2023-05-22 17:03:54 -04:00
Karenzz	88d17d979d	Load the new price and timestamp	2023-05-22 17:03:25 -04:00
cheezedoodle2	3eb9bfb24e	Added missing commas	2023-05-19 13:50:36 -07:00
j	ea40df7f89	Update 'README.md'	2023-05-19 22:50:02 +02:00
j	8947238720	Add installation instructions	2023-05-19 21:52:16 +02:00
Karenzz	220adaf33a	Created a new dataframe to store all keys data	2023-05-19 15:32:02 -04:00
Karenzz	80bc996230	Update	2023-05-19 15:31:34 -04:00
cheezedoodle2	09fdbb8b5d	Move future.resuit() into the try block	2023-05-18 14:56:46 -07:00
cheezedoodle2	1ddc498430	Import the exception, too	2023-05-18 14:45:57 -07:00
cheezedoodle2	b8a4875e9d	Wrapped the publish call to catch DeadlineExceeded exceptions	2023-05-18 14:45:18 -07:00
cheezedoodle2	24a1f04a2f	Added package requirements and install instructions	2023-05-18 14:26:01 -07:00
Karenzz	99d2b60d5d	Modified the main scraping loop	2023-05-18 15:01:04 -04:00
Karenzz	5bcc087f63	Run the scraper in parallel	2023-05-17 15:30:40 -04:00
Karenzz	7f13d0823b	Used plotly to generate a graph	2023-05-17 15:01:04 -04:00
Karenzz	0e909fc3f9	Modified the sold out status checking	2023-05-17 15:00:37 -04:00
cheezedoodle2	92d7e745e1	Factored out element search logic, added scraping out-of-stock messages and sales figures	2023-05-16 15:55:20 -07:00
Karenzz	b844189473	Recorded 11 products' price data	2023-05-16 16:44:53 -04:00
Karenzz	811a03b85c	View the HDF5 file	2023-05-16 16:44:15 -04:00
Karenzz	27c44a1dfe	Changed the product link	2023-05-16 16:43:43 -04:00
Karenzz	6c70c91161	Closed the modal dialog and saved data in HDF5	2023-05-16 16:42:15 -04:00
cheezedoodle2	722d116e75	Added checking for quantity discounts, re-enabled element debug to help troubleshoot dialog closing	2023-05-16 10:16:24 -07:00
cheezedoodle2	d88d231f24	Replaced some timeouts with WebDriverWait to speed up execution	2023-05-15 12:13:21 -07:00
Karenzz	a925035acd	Saved the df to a HDF5 file	2023-05-15 14:56:53 -04:00
Karenzz	2ead943e3a	Cleaned up the exception and Added product_id	2023-05-11 12:50:33 -04:00
Karenzz	dddf8e3bb8	Added minimum purchase exceptions	2023-05-11 12:49:13 -04:00
Karenzz	5983b79e1a	Checked the availability of the product	2023-05-11 12:06:11 -04:00
Karenzz	7df52c95e1	Check the availability & time delay	2023-05-09 19:02:36 -04:00
Karenzz	8ed9dfbdf3	Check the availability & time delay	2023-05-09 19:01:46 -04:00
cheezedoodle2	528d134c65	Added workaround for j/python-bootcamp#3 by catching DeadlineExceeded and returning an empty message list	2023-05-08 18:29:49 -07:00
Karenzz	bf32ca3cb8	Create a dataframe to store the zipcode & price	2023-05-08 18:57:47 -04:00
cheezedoodle2	ec7131c8e2	Changed subscriber to synchronous due to only having one browser window, fixed message acknowledgment.	2023-05-05 13:59:50 -07:00
Karenzz	4930f69a01	Merge branch 'Branch_Karen' of https://gogs.gtsplab.com:5432/j/python-bootcamp into Branch_Karen	2023-05-05 15:04:40 -04:00
Karenzz	19f6b0ff66	Added better xpath selectors for the UI elements	2023-05-05 15:01:15 -04:00
Karenzz	26aad47272	Added better xpath selectors for the UI elements	2023-05-05 15:00:11 -04:00
cheezedoodle2	c96a0b40ef	Added better xpath selectors for the UI elements	2023-05-04 19:26:48 -07:00
Karenzz	8765abfb12	modified fp2_8	2023-05-04 19:19:11 -04:00
Karenzz	5a9e4a00d8	modified fp2_9	2023-05-04 19:18:52 -04:00
Karenzz	ed955015fe	modified fp2_9	2023-05-04 15:35:53 -04:00
Karenzz	da65dd3317	zipcode csv files	2023-05-03 19:09:43 -04:00
Karenzz	6d88182baa	update for the NFL zipcodes	2023-05-03 19:04:01 -04:00
Karenzz	7baaac206f	fp2_4 update	2023-05-01 12:52:50 -04:00
Karenzz	72167507f9	modified fp2_4 & fp2_5	2023-04-28 19:30:00 -04:00
Karenzz	d72087ac9f	modified fp1	2023-04-27 19:17:28 -04:00
Karenzz	1cea8f5ff2	final project update	2023-04-26 19:14:54 -04:00
Karenzz	537b20f48d	Merge branch 'Branch_Karen' of https://gogs.gtsplab.com:5432/j/python-bootcamp into Branch_Karen	2023-04-19 20:25:42 -04:00
Karenzz	9933b0b880	update	2023-04-19 20:24:03 -04:00
Karenzz	451ee7a557	for hw10	2023-04-19 20:22:09 -04:00
Karenzz	abad8cc142	for hw10	2023-04-19 20:22:09 -04:00
Karenzz	d5ee4388cb	for hw10	2023-04-19 20:22:09 -04:00
Karenzz	8eb3a5ea06	for hw9	2023-04-19 20:22:09 -04:00
Karenzz	b35c6ec71a	for hw9	2023-04-19 20:22:09 -04:00
Karenzz	1af38dcecb	fastAPI	2023-04-19 20:22:08 -04:00
Karenzz	fe732b6a7c	http-performance	2023-04-19 20:22:08 -04:00
Karenzz	0436636995	update	2023-04-19 20:22:08 -04:00
Karenzz	8c18d8e4cb	Delete 'httpserver.py'	2023-04-19 20:20:59 -04:00
Karenzz	760e95422c	for test	2023-04-19 19:11:20 -04:00
Karenzz	a9737148e6	delete	2023-04-19 19:11:20 -04:00
Karenzz	252fbee998	for hw8	2023-04-19 19:11:20 -04:00
Karenzz	9a1ce4965f	assessment1_toaster	2023-04-19 19:11:20 -04:00
Karenzz	861bd8db11	update	2023-04-19 19:11:20 -04:00
Karenzz	1ef6c3cbf8	update	2023-04-19 19:11:20 -04:00
Karenzz	2c2adc4082	HW7 jitter relationship	2023-04-19 19:11:19 -04:00
Karenzz	150ead64ca	HW7 jitter of `time.sleep()`	2023-04-19 19:11:19 -04:00
Karenzz	fd980def40	creating a new branch for Karen	2023-04-19 19:11:19 -04:00
Karenzz	443ffeec82	for hw10	2023-04-20 00:53:10 +02:00
Karenzz	f2b4cb2719	for hw10	2023-04-19 22:48:41 +02:00
Karenzz	f97d03d772	for hw10	2023-04-19 21:53:21 +02:00
Karenzz	8e3dd87df6	Merge branch 'Branch_Karen' of https://gogs.gtsplab.com:5432/j/python-bootcamp into Branch_Karen	2023-04-19 14:19:48 -04:00
Karenzz	286afab14a	update	2023-04-19 14:18:22 -04:00
Karenzz	8e3305f7a2	for hw9	2023-04-19 00:40:28 +02:00
Karenzz	9c2f7fafe0	for hw9	2023-04-18 23:50:29 +02:00
Karenzz	8cb50a71f1	fastAPI	2023-04-14 16:52:39 +02:00
Karenzz	4c48b76ca6	http-performance	2023-04-14 00:41:05 +02:00
Karenzz	31175b6cce	for hw8	2023-04-13 22:26:13 +02:00
Karenzz	4d12724964	for hw8	2023-04-13 22:26:05 +02:00
Karenzz	78670368a3	for hw8	2023-04-13 22:25:51 +02:00
Karenzz	7a44f7d4d8	for hw8	2023-04-13 22:25:37 +02:00
Karenzz	6e91afd4f0	for hw8	2023-04-13 22:25:26 +02:00
Karenzz	04ecd79a1a	for hw8	2023-04-13 22:25:12 +02:00
Karenzz	9dcd16e244	for hw8	2023-04-13 22:24:59 +02:00
Karenzz	d5ccb2caca	for hw8	2023-04-13 22:24:49 +02:00
Karenzz	236aa98458	for hw8	2023-04-13 22:24:37 +02:00
Karenzz	35613a917b	for hw8	2023-04-13 22:24:24 +02:00
Karenzz	18dd45879e	for hw8	2023-04-13 22:24:13 +02:00
Karenzz	2cab29d002	for hw8	2023-04-13 22:23:38 +02:00
Karenzz	cbf7dc40de	for hw8	2023-04-13 22:23:21 +02:00
Karenzz	1dfae3971e	for hw8	2023-04-13 22:23:06 +02:00
Karenzz	5385728ade	for hw8	2023-04-13 22:22:54 +02:00
Karenzz	b8d6e10808	for hw8	2023-04-13 22:22:36 +02:00
Karenzz	7bdd883a35	for hw8	2023-04-13 22:22:25 +02:00
Karenzz	d85ba8a9a1	for hw8	2023-04-13 22:21:54 +02:00
Karenzz	33bed5a231	for hw8	2023-04-13 22:21:33 +02:00
Karenzz	0e9bfd04ce	for hw8	2023-04-13 22:21:15 +02:00
Karenzz	7cf394fd36	for hw8	2023-04-13 22:20:56 +02:00
Karenzz	3136156b18	for hw8	2023-04-13 22:20:40 +02:00
Karenzz	ccd671437c	for hw8	2023-04-13 22:20:22 +02:00
Karenzz	c97d44d47a	for hw8	2023-04-13 22:20:00 +02:00
Karenzz	3365b21137	for hw8	2023-04-13 22:19:26 +02:00
Karenzz	54632f1f44	Merge branch 'Branch_Karen' of https://gogs.gtsplab.com:5432/j/python-bootcamp into Branch_Karen	2023-04-13 16:14:29 -04:00
Karenzz	ae4ca3d540	for test	2023-04-13 16:12:33 -04:00
Karenzz	60748462a2	delete	2023-04-13 16:10:59 -04:00
Karenzz	8c48efb1f2	Delete 'kublai10.jpg'	2023-04-13 22:01:58 +02:00
Karenzz	88b7bc9779	Delete 'kublai1.jpg'	2023-04-13 22:01:49 +02:00
Karenzz	36069dd472	Delete 'kublai0.jpg'	2023-04-13 22:01:43 +02:00
Karenzz	92d1323b99	Delete 'index.html'	2023-04-13 22:01:34 +02:00
Karenzz	69b6b7f653	Delete 'httpserver.py'	2023-04-13 22:01:27 +02:00
Karenzz	41563bbbd2	for hw8	2023-04-13 21:58:55 +02:00
Karenzz	b1450ca436	assessment1_toaster	2023-04-13 15:45:21 -04:00
Karenzz	a010090dba	update	2023-04-13 15:42:15 -04:00
Karenzz	54ca1c76c4	update	2023-04-13 15:38:06 -04:00
Karenzz	90bfaa6ae7	HW7 jitter relationship	2023-04-12 23:30:50 +02:00
Karenzz	f76493631d	HW7 jitter of `time.sleep()`	2023-04-12 23:04:23 +02:00
Karenzz	50fa94c59d	creating a new branch for Karen	2023-04-03 14:34:45 -04:00