Merged fp files

Leon_Submissions/Homework1.py

@@ -1,28 +0,0 @@
-# Question 1: 
-# The international standard for floating-point representation is 
-# called the "IEEE 754 Standard for Floating-Point Arithmetic."
-
-# Question 2: 
-# 32-bit and 64-bit formats are the most widely used ones in practice.
-
-# Question 3: 
-# My computer supports all these formats
-import numpy as np
-
-supported_formats = np.finfo(np.float32).bits, np.finfo(np.float64).bits
-print(f"Your computer supports the following floating-point formats: {supported_formats}")
-
-
-# Question 4:
-# Using the Pandas multiplication operator (df['Number1'] * df['Number2'])
-# , the result is automatically converted to a floating-point data type. 
-# On the other hand, when I multiply the numbers using the normal Python
-# multiplication operator (num1 * num2), the result remains an integer 
-# data type.
-
-# Question 5:
-# The size of a floating-point number in memory is primarily determined 
-# by the data type and not the actual value it represents. So the size 
-# of a Python floating-point number does not change as it gets larger.
-import sys
-print(sys.getsizeof(70))

Leon_Submissions/leon.py

@@ -1 +0,0 @@
-print('Hi')

Multiprocessing.py

@@ -0,0 +1,276 @@
+import multiprocessing
+import subprocess
+import concurrent.futures
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+import os
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import selenium.common.exceptions
+import re
+from fp2_14 import get_product_prices
+
+# Define the subset of urls
+url_list = [
+    'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
+    'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
+    'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
+    'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
+    'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
+    'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
+    'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
+    'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
+    'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
+    'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
+    'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
+]
+
+
+# Create a pool of worker processes
+# pool = multiprocessing.Pool(cpu_count)
+
+# Define the function to run the scraper
+def run_scraper(url):
+    # scraping logic
+    df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
+    print(f"Created a dataframe: {df}")
+    product_id = url.split("/")[-1][:5]
+
+    options = Options()
+    options.add_argument('--disable-gpu')
+    driver = webdriver.Chrome(options=options)
+
+    project_id = "fp-20230425"
+    subscription_name = "fp-subscription-NFL-zipcodes"
+    subscriber = pubsub_v1.SubscriberClient()
+    subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+    # product_id = url.split("/")[-1]
+    driver.delete_all_cookies
+    driver.get(url)
+
+    def get_element_text(class_name_prefix):  
+        xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+        xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+        xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+        xpath_str += "' '), "                     # Step 4: a trailing space...
+        xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+        try:
+            match = driver.find_element(By.XPATH, xpath_str)
+            return match.get_attribute('textContent')
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            pass
+   
+        xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+        xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+        xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+        xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+        try:
+            match = driver.find_element(By.XPATH, xpath_str)
+            return match.get_attribute('textContent')
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            pass
+        return ''
+
+    # Close the cookies setting window
+    try:  
+        cookie_button = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+        )
+        print(f"cookie_button = {cookie_button}")
+        cookie_button.click()
+    except:
+        pass
+
+    # Clear any modal dialog
+    try:    
+        modal_close = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+        )
+        print(f"modal_close = {modal_close}")
+        modal_close.click()
+    except:
+        pass
+
+    # Define callback function for Pub/Sub messages
+    def callback(message):
+        # Extract zip code from message data
+        zip_code = message.message.data.decode('utf-8')
+
+        print(f"Received zip code: {zip_code}")
+
+        try:
+            try:
+                # Clear any modal dialog
+                modal_close = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+            )
+                # print(f"modal_close = {modal_close}")
+                modal_close.click()
+            except:
+                pass
+
+
+            try:
+
+                # Clear any notice
+                notice_close = WebDriverWait(driver, 5).until(
+                EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+                # print(f"notice_close = {notice_close}")
+                notice_close.click()
+            except:
+                pass
+        
+            # Enter zip code in the input field and click the "Check" button
+            zip_button = WebDriverWait(driver, 5).until(
+                EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+            )
+            # print(f"{zip_button:}")
+            zip_button.click()
+
+            zip_input = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+            )
+            # print(f"{zip_input:}")
+            zip_input.click()
+            # print("clearing zip_input")
+            zip_input.send_keys("\b\b\b\b\b")
+            # print("sending zip code")
+            zip_input.send_keys(zip_code)
+
+            check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+            check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+            # print(f"{check_button:}")
+            check_button.click()
+            # time.sleep(2)
+
+            try:
+            # Clear any notice
+                notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+                # print(f"notice_close = {notice_close}")
+                notice_close.click()
+            except:
+                pass
+
+            try:
+                # Clear any modal dialog
+                modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+                # print(f"modal_close = {modal_close}")
+                modal_close.click()
+            except:
+                pass
+
+            # Record sales count, if any
+            sales_count_element = get_element_text('Header_count')
+            count_start_index = sales_count_element.index('sold ') + len('sold ')
+            count_end_index = sales_count_element.index('+')
+            sales_count = sales_count_element[count_start_index:count_end_index]
+            # Record any sold status notice
+            sold_status = get_element_text('UnavailableCartbtn_container')
+            if sold_status == 'Notify me':
+                sold_status = 'Sold Out'
+
+        # Extract the price of rice crackers for this zip code
+            # Check if there is a minimum order quantity
+            min_quantity: float = 0
+            total_price: float = 0
+            unit_price: float = 0
+            try:
+                minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML')
+                if minimum_quantity_str:
+                    regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
+                    match = re.search(regexp,minimum_quantity_str)
+                    min_quantity, total_price = match.groups()
+            except selenium.common.exceptions.NoSuchElementException as nsee:
+                print("No minimum purhcase string found")
+            except Exception as e:
+                print(f"Exception when finding minimum purchase string: {e}")
+
+            ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+            ld_json_text = ld_json_element.get_attribute('innerHTML')
+            ld_json_start_index = ld_json_text.index('{')
+            ld_json_end_index = ld_json_text.rindex('}') + 1
+            ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
+            ld_json = json.loads(ld_json_json)
+            ld_json_price = ld_json['offers']['price']
+            # Print the price
+            print(ld_json_price)
+
+            unit_price_element = driver.find_element(By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText')
+            unit_price = unit_price_element.replace("$", "")
+            print(f"unit_price = {unit_price}" )
+
+            subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+            # Append the zip code and price to the dataframe
+            #append_to_df(zip_code, price)
+            new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]})
+            return pd.concat([df, new_row], ignore_index=True)
+            #print(f"Recorded the zip code and price: {zip_code},{price}")
+            #print(f"Updated a dataframe: {df}")
+
+        except Exception as e:
+            print(f"Caught exception: {e}")
+            print(f"Price not found for zip code {zip_code}")
+            price = 'Price not found'
+            return df
+
+    # Start receiving messages (sync version)
+    while True: 
+        pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+        messages = []
+        try:
+            pull_response = subscriber.pull(request = pull_request)
+            messages = pull_response.received_messages
+        except DeadlineExceeded:
+            pass
+
+        if messages:
+            df = callback(messages[0])
+            print(f"Updated a dataframe: {df}")
+        else:
+            print(f"no messages, trying again at {time.asctime()}")
+            break
+        # subprocess.call(['python', 'fp2_13.py', url])
+    return df
+
+if __name__ == '__main__':
+    # Determine the number of available CPUs
+    # cpu_count = multiprocessing.cpu_count()
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        future_to_url = {executor.submit(run_scraper, url): url for url in url_list}
+        results = []
+        for future in concurrent.futures.as_completed(future_to_url):
+            url = future_to_url[future]
+            try:
+                result = future.result()
+                print(f"Scraped data for URL: {url}")
+                results.append(result)
+            except Exception as e:
+                print(f"Failed to scrape data for URL: {url}\nError: {e}")
+
+'''
+# Run the scraper in parallel for the subset of items
+results = pool.map(run_scraper, url_list)
+
+# Close the pool and wait for the processes to finish
+pool.close()
+pool.join()
+'''
+
+with pd.HDFStore('product_data.h5') as store:
+    for url, result in zip(url_list, results):
+        # Extract the product ID from the URL
+        product_id = url.split("/")[-1][:5]
+
+        # Save the dataframe to the HDF5 file with a unique key
+        key = f'df_{product_id}'
+        store[key] = result
+        # result.to_hdf('product_data.h5', key=key, mode='a')

NFL_Cities.csv

@@ -0,0 +1,33 @@
+ID,Team Name,Arena Name,Abbreviation,Conference,Division,Arena Location,ZIP Code
+1,Arizona Cardinals,State Farm Stadium,ARI,NFC,West,"Glendale, Arizona",85305
+2,Atlanta Falcons,Mercedes-Benz Stadium,ATL,NFC,South,"Atlanta, Georgia",30313
+3,Baltimore Ravens,M&T Bank Stadium,BAL,AFC,North,"Baltimore, Maryland",21230
+4,Buffalo Bills,Bills Stadium,BUF,AFC,East,"Orchard Park, New York",14127
+5,Carolina Panthers,Bank of America Stadium,CAR,NFC,South,"Charlotte, North Carolina",28202
+6,Chicago Bears,Soldier Field,CHI,NFC,North,"Chicago, Illinois",60605
+7,Cincinnati Bengals,Paul Brown Stadium,CIN,AFC,North,"Cincinnati, Ohio",45202
+8,Cleveland Browns,First Energy Stadium,CLE,AFC,North,"Cleveland, Ohio",44114
+9,Dallas Cowboys,At&T Stadium,DAL,NFC,East,"Arlington, Texas",76011
+10,Denver Broncos,Empower Field at Mile High,DEN,AFC,West,"Denver, Colorado",80204
+11,Detroit Lions,Ford Field,DET,NFC,North,"Detroit, Michigan",48226
+12,Green Bay Packers,Lambeau Field,GB,NFC,North,"Green Bay, Wisconsin",54304
+13,Houston Texans,NRG Stadium,HOU,AFC,South,"Houston, Texas",77054
+14,Indianapolis Colts,Lucas Oil Stadium,IND,AFC,South,"Indianapolis, Indiana",46225
+15,Jacksonville Jaguars,TIAA Bank Field,JAX,AFC,South,"Jacksonville, Florida",32202
+16,Kansas City Chiefs,Arrowhead Stadium,KC,AFC,West,"Kansas City, Missouri",64129
+17,Las Vegas Raiders,Allegiant Stadium,LV,AFC,West,"Paradise, Nevada",89118
+18,Los Angeles Chargers,SoFi Stadium,LAC,AFC,West,"Inglewood, California",90301
+19,Los Angeles Rams,SoFi Stadium,LAR,NFC,West,"Inglewood, California",90301
+20,Miami Dolphins,Hard Rock Stadium,MIA,AFC,East,"Miami Gardens, Florida",33056
+21,Minnesota Vikings,U.S. Bank Stadium,MIN,NFC,North,"Minneapolis, Minnesota",55415
+22,New England Patriots,Gillette Stadium,NE,AFC,East,"Foxborough, Massachusetts",02035
+23,New Orleans Saints,Mercedes-Benz Superdome,NO,NFC,South,"New Orleans, Louisiana",70112
+24,New York Giants,MetLife Stadium,NYG,NFC,East,"East Rutherford, New Jersey",07073
+25,New York Jets,MetLife Stadium,NYJ,AFC,East,"East Rutherford, New Jersey",07073
+26,Philadelphia Eagles,Lincoln Financial Field,PHI,NFC,East,"Philadelphia, Pennsylvania",19148
+27,Pittsburgh Steelers,Heinz Field,PIT,AFC,North,"Pittsburgh, Pennsylvania",15212
+28,San Francisco 49ers,Levi's Stadium,SF,NFC,West,"Santa Clara, California",95054
+29,Seattle Seahawks,CenturyLink Field,SEA,NFC,West,"Seattle, Washington",98134
+30,Tampa Bay Buccaneers,Raymond James Stadium,TB,NFC,South,"Tampa, Florida",33607
+31,Tennessee Titans,Nissan Stadium,TEN,AFC,South,"Nashville, Tennessee",37213
+32,Washington Redskins,FedExField,WAS,NFC,East,"Landover, Maryland",20785

README.md

@@ -17,4 +17,27 @@ the focus is on *working results*, not rote memorization.  Homework questions
 are simple enough that they can be reviewed interactively in 1:1 or up to 1:4
 sessions with study groups.
 
+This branch contains a prototype for a retail price scraper for getweee.com.
+To run it, install:
+
+* Python 3.9+,
+* chrome,
+* chromedriver,
+* and the python depedencies:
+
+```
+pip install -r /path/to/requirements.txt
+```
+
+Once those are installed, install the GCP SDK:
+
+https://cloud.google.com/sdk/docs/install
+
+You will also have to log into GCP *twice*, once for the gcloud CLI tools, 
+and again for the GCP SDK libraries:
+
+https://googleapis.dev/python/google-api-core/latest/auth.html
+
+
+
 

Scott_Submissions/A_BrainTeaser_byScott.py

@@ -1,100 +0,0 @@
-# A quick brain teaser
-print('What has a head and a tail but no body?')
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-# A coin

Scott_Submissions/Floating_Point_Formats.py

@@ -1,2 +0,0 @@
-import sys
-print(sys.float_info)

Scott_Submissions/HW1_Scott.rtf

@@ -1,37 +0,0 @@
-{\rtf1\ansi\ansicpg1252\cocoartf2580
-\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fnil\fcharset0 HelveticaNeue;\f1\fnil\fcharset0 Menlo-Regular;}
-{\colortbl;\red255\green255\blue255;\red0\green0\blue0;\red0\green0\blue0;}
-{\*\expandedcolortbl;;\cssrgb\c0\c0\c0\c87059;\cssrgb\c0\c0\c0\c3922;}
-\margl1440\margr1440\vieww11520\viewh8400\viewkind0
-\deftab720
-\pard\pardeftab720\partightenfactor0
-
-\f0\fs32 \cf2 \expnd0\expndtw0\kerning0
-1. What is the name of the international standard for floating-point representation?\
-	\
-	The IEEE-754 standard describes floating-point formats, a way to represent real numbers in hardware.\
-\
-\
-2. How many bits are the floating point numbers defined by that standard allowed to be?\
-\
-	The floating-point numbers defined by the international standard can be allowed to be of different lengths depending on the specific interchange format. For binary interchange, the formats can be 16 bits, 32 bits, 64 bits, and any multiple of 32 bits \uc0\u8805  128. For decimal interchange, the formats can be any multiple of 32 bits.\
-\
-\
-3. Does your computer support all those formats? Can you think of an easy way to check with only one line of Python?\
-\
-import sys\
-print(sys.float_info)\
-\
-scott@MacBook-Pro-6 python-bootcamp % python3 Floating_Point_Formats.py \
-sys.float_info(max=1.7976931348623157e+308, max_exp=1024, max_10_exp=308, min=2.2250738585072014e-308, min_exp=-1021, min_10_exp=-307, dig=15, mant_dig=53, epsilon=2.220446049250313e-16, radix=2, rounds=1)\
-\
-\
-\pard\pardeftab720\sa320\partightenfactor0
-\cf2 4. Why doesn't this code give the same answers to the two multiplications?\
-	Because they are different operations. The normal Python multiplication operator will multiply two numbers together and return the product. The Pandas multiplication operator, on the other hand, performs element-wise multiplication on the corresponding columns in a DataFrame. In other words, it multiplies the first element in the first column with the first element in the second column, the second element in the first column with the second element in the second column, and so on. \
-	Floating-point arithmetic has a limited precision due to the finite number of bits used to represent floating-point numbers in binary format. This can lead to rounding errors and loss of precision in arithmetic operations, particularly with very large or very small numbers.\
-\
-5. You can use 
-\f1\fs27\fsmilli13600 \cf2 \cb3 sys.getsizeof
-\f0\fs32 \cf2 \cb1  to see how large a Python object is. Does the size of a Python floating-point number change as it gets larger? Why or why not?\
-	The size of a Python floating-point number does not change as it gets larger. This is because Python uses a fixed number of bytes to represent floating-point numbers, regardless of their value.}

Scott_Submissions/HW2.py

@@ -1,54 +0,0 @@
-# Name three practical differences between Python strings and bytes. 
-# Using sys.getsizeof, see how the sizes may be different. 
-# Do emoji change this answer?
-
-# Encoding
-# Strings are created using Unicode character points. 
-# While bytes are used to represent actual bytes in memory, which can be encoded using ASCII or UTF-8.
-
-# Usage
-# Strings are usually used to represent text.
-# While bytes are used to represent binary data.
-
-# Memory Size 
-# Strings take up more memory than bytes, because they are Unicode-based and can contain any character.
-# While bytes are limited to binary data.
-
-
-print("---------------------------------------------")
-print("This is the Memory Size test print out:")
-
-
-import sys
-
-my_string = "Hello, world!"
-my_bytes = b'\x48\x65\x6c\x6c'
-
-print(sys.getsizeof(my_string)) # Prints 62
-print(sys.getsizeof(my_bytes)) # Prints 37
-
-print()
-print('Test the emoji:')
-
-x = 'pizza' # Prints 54
-y = b'pizza' # Prints 38
-z = '🍕' #Prints 80
-
-print(sys.getsizeof(x))
-print(sys.getsizeof(y))
-print(sys.getsizeof(z))
-
-
-print("---------------------------------------------")
-
-print("This is the String to Sequence of Numbers print out:")
-
-def String_to_SequenceofNumbers():
-    my_string = input("Enter a string: ")
-    num_sequence = [ord(c) for c in my_string]
-    print(num_sequence)
-
-if __name__ == "__main__":
-    String_to_SequenceofNumbers()
-
-print("---------------------------------------------")

Scott_Submissions/Q4.py

@@ -1,18 +0,0 @@
-import pandas as pd
-import numpy as np
-
-# Generate two random 64-bit integers
-num1 = np.random.randint(0, 10 - 1)
-num2 = np.random.randint(0, 10 - 1)
-
-# Multiply them together using the normal Python multiplication operator
-print(f"num1 * num2 = {num1 * num2}")
-
-# Create a Pandas DataFrame with the generated numbers
-data = {'Number1': [num1], 'Number2': [num2]}
-df = pd.DataFrame(data)
-
-# Multiply the numbers together using the Pandas multiplication operator
-df['Product'] = df['Number1'] * df['Number2']
-
-print(df)

Scott_Submissions/Q5.py

@@ -1,4 +0,0 @@
-import sys
-num = 99999999999
-size = sys.getsizeof(num)
-print(f"The size of the float is {size} bytes.")

Scott_Submissions/cr-example.py

@@ -1,7 +0,0 @@
-import time
-
-i=0
-while True:
-    print(f"\015{i=}", end='')
-    i += 1
-    time.sleep(0.2)

Scott_Submissions/test.py

@@ -1,5 +0,0 @@
-char1 = "u"
-result1 = ord(char1)
-print("ASCII value of the character {} is {}.".format(char1, result1))
-
-

Simulation/Simulation.py

@@ -0,0 +1,42 @@
+from abc import ABC, abstractmethod
+
+class Simulation(ABC):
+    @abstractmethod
+    def start(self):
+        "Start the simulated object."
+        pass
+
+    @abstractmethod
+    def tick(self) -> bool:
+        """
+        Simulate one clock tick.
+        Returns:
+            bool: True if the simulation could advance one clock tick, or False if the object is expired.
+        """
+        pass
+
+    @abstractmethod
+    def get_uses(self):
+        """
+        Return the uses of the simulated object.  These are verb strings which are passed to the `use()` method.
+        Returns:
+            tuple: A dict of key-value pairs, where the keys are strings of
+            verbs to be passed to `use`, and the values are strings explaining
+            what the verb does in English.
+        """
+    @abstractmethod
+    def use(self, *args, **kwargs):
+        """
+        Use the simulated object.
+        Args:
+            args (tuple): A tuple of verbs to be used on the object.
+            kwargs (dict): A dictionary of key-value pairs of arguments to the verb(s).
+        Returns:
+            True if the action succeeded, False if the action did not succeed and the object needs to be expired.
+        """
+        pass
+
+    @abstractmethod
+    def stop(self):
+        "Stop the simulated object."
+        pass

Simulation/SimulationRunner.py

@@ -0,0 +1,42 @@
+from importlib import import_module
+import tkinter as tk
+import os
+import sys
+from random import choice
+from time import sleep
+from tkinter import filedialog
+from abc import ABC, abstractmethod
+
+class SimulationRunner:
+    def __init__(self, simulation) -> None:
+        self.simulation = simulation()
+        self.simulation.start()
+        uses = list(self.simulation.get_uses().keys())
+        while self.simulation.tick() and self.simulation.use(choice(uses)):
+            pass
+        self.simulation.stop()
+
+if __name__ == '__main__':
+    if len(sys.argv) == 1:
+        # Create a hidden root window
+        root = tk.Tk()
+        root.withdraw()
+
+        # Open the file chooser dialog
+        file_path = filedialog.askopenfilename()
+        print(f"Selected file: {file_path}")
+    else:
+        file_path = sys.argv[1]
+
+    # Get the python file's directory and filename
+    module_dir, module_file = os.path.split(file_path)
+    # Get the python file's name without any extension
+    module_name, _ = os.path.splitext(module_file)
+
+    # Add the module directory to sys.path so we can import it unqualified
+    sys.path.append(module_dir)
+
+    simulation = import_module(module_name)
+    # Load the simulation module's simulation class, which should be the same
+    # name as the module name.
+    simulationrunner = SimulationRunner(getattr(simulation, module_name))

Task_assignment.ipynb

@@ -0,0 +1,402 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Guide\n",
+    "1. Please write your solutions at the appropriate location **# YOUR CODE HERE**\n",
+    "2. Do not modify other cells"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "# Numerical or binary attributes\n",
+    "num_or_bin= ['Income', 'Kidhome','Recency', 'MntWines', 'MntFruits', 'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts','MntGoldProds', 'NumDealsPurchases', 'NumWebPurchases',\n",
+    "       'NumCatalogPurchases', 'NumStorePurchases', 'NumWebVisitsMonth','AcceptedCmp3', 'AcceptedCmp4', 'AcceptedCmp5', 'AcceptedCmp1','AcceptedCmp2']\n",
+    "\n",
+    "# Categorical attributes\n",
+    "cat = ['Education','Marital_Status']\n",
+    "\n",
+    "# Load the data source\n",
+    "pd.set_option('display.max_columns', None)\n",
+    "df  = pd.read_csv(\"https://raw.githubusercontent.com/rpi-techfundamentals/website_fall_2021/master/site/public/marketing_campaign.csv\", delimiter='\\t')\n",
+    "df.head()"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q1. Load and preprocess the data.\n",
+    "Build and transform your dataframe `df` using the following steps before modeling. Create a new dataframe `X` that has the following properties:\n",
+    "\n",
+    "1. It should include only the columns from lists `num_or_bin` and `cat` (in the code above).\n",
+    "2. Variables in cat should be transformed to `n-1` dummy variables (where `n` is the number of categories for each variable.)\n",
+    "3. Missing variables in the `Income` table should be filled with the median of the `income` table.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "all_cols=num_or_bin+cat\n",
+    "X=pd.get_dummies(df[all_cols],columns=cat,drop_first=True)\n",
+    "X.loc[X['Income'].isnull(),'Income']=X['Income'].median()\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 1 Test Cell \n",
+    "#this tests the columns. \n",
+    "ecol = set(['Income', 'Kidhome', 'Recency', 'MntWines', 'MntFruits',\n",
+    "       'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts',\n",
+    "       'MntGoldProds', 'NumDealsPurchases', 'NumWebPurchases',\n",
+    "       'NumCatalogPurchases', 'NumStorePurchases', 'NumWebVisitsMonth',\n",
+    "       'AcceptedCmp3', 'AcceptedCmp4', 'AcceptedCmp5', 'AcceptedCmp1',\n",
+    "       'AcceptedCmp2', 'Education_Basic', 'Education_Graduation',\n",
+    "       'Education_Master', 'Education_PhD', 'Marital_Status_Alone',\n",
+    "       'Marital_Status_Divorced', 'Marital_Status_Married',\n",
+    "       'Marital_Status_Single', 'Marital_Status_Together',\n",
+    "       'Marital_Status_Widow', 'Marital_Status_YOLO'])\n",
+    "assert len(ecol-set(X.columns))==0\n",
+    "assert len(set(X.columns))-len(ecol)==0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 2 Test Cell \n",
+    "#this tests the columns. \n",
+    "assert X.shape==(2240, 30)\n",
+    "q2=X.copy()\n",
+    "q2['Response']=df['Response']\n",
+    "q2['Year_Birth']=df['Year_Birth']\n",
+    "q2['Dt_Customer']=df['Dt_Customer']\n",
+    "q2.to_csv('q2.csv',index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 3 Test Cell \n",
+    "assert len(X.loc[X['Income']==51381.5,'Income'])==24"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 4 Test Cell\n",
+    "assert X.isnull().sum().sum()==0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Reload data\n",
+    "import pandas as pd\n",
+    "df_cl=pd.read_csv('q2.csv')"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q2. Feature Creation.\n",
+    "1. In the `df_cl` dataframe, create an integer feature `age` which is `2021` minus `Year_Birth`.\n",
+    "2. In the `df_cl` dataframe, create an integer feature `holiday` which is 1 if the month of `Dt_Customer` is equal to 12 and 0 otherwise. You can get the month by accessing `mm` from the date format `dd-mm-yyyy`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "df_cl['age']=(2021-df_cl['Year_Birth']).astype(int)\n",
+    "df_cl['Dt_Customer_tmp']=df_cl['Dt_Customer'].apply(lambda x:x[3:5])\n",
+    "df_cl.loc[df_cl['Dt_Customer_tmp']=='12','holiday']=1\n",
+    "df_cl.loc[df_cl['Dt_Customer_tmp']!='12','holiday']=0\n",
+    "df_cl['holiday']=df_cl['holiday'].astype(int)\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 5 Test Cell\n",
+    "assert df_cl['age'].sum()==116915"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 6 Test Cell\n",
+    "import numpy as np\n",
+    "assert isinstance(df_cl['age'][0], (np.int64,np.int32))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 7 Test Cell\n",
+    "assert df_cl['holiday'].sum()==175"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 8 Test Cell\n",
+    "import numpy as np\n",
+    "assert isinstance(df_cl['holiday'][0], (np.int64,np.int32))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "df_cl2=pd.read_csv('dfcl2.csv')\n",
+    "y_reload=df_cl2['Response']\n",
+    "X_reload=df_cl2.drop(['Response'], axis=1)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q3. Train Test Split\n",
+    "Now split the data `X_reload`, `y_reload` intro training (50%) and testing data (50%) by using these variable names while making sure there are Make sure that there are equal number of positive cases (1's) in the train and the test set.\n",
+    "\n",
+    "`X_train`: Training feature columns\n",
+    "`X_test`: Testing feature columns\n",
+    "`y_train`: Training DV\n",
+    "`y_test`: Testing DV\n",
+    "\n",
+    "When doing so set the `random_state` parameter to 100 so you get the same split that is in the solution."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "y_reload=df_cl2['Response']\n",
+    "X_reload=df_cl2.drop(['Response'], axis=1)\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "X_train,X_test,y_train,y_test=train_test_split(X_reload,y_reload,\n",
+    "                                               train_size=0.5,\n",
+    "                                               test_size=0.5,\n",
+    "                                               random_state=100,\n",
+    "                                               stratify=y_reload)\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 9 Test Cell\n",
+    "assert X_train.shape==(1120, 31)\n",
+    "assert X_test.shape==(1120, 31)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 10 Test Cell\n",
+    "assert  y_train.sum()==y_test.sum()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "df_cl2=pd.read_csv('dfcl2.csv')\n",
+    "y_reload=df_cl2['Response']\n",
+    "X_reload=df_cl2.drop(['Response'], axis=1)\n",
+    "\n",
+    "X_train2, X_test2, y_train2, y_test2 = train_test_split(X_reload, y_reload, test_size=0.3, random_state = 100)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q4. Classification\n",
+    "Use `X_train2`, `X_test2`, `y_train2`, `y_test2` from above. Use a Gradient Boosting classifier (as instantiated below).\n",
+    "\n",
+    "Please calculate:\n",
+    "1. `test_acc` The accuracy of the classifier on the test set.\n",
+    "2. `test_roc` The area under the ROC curve on on the test set."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn import metrics\n",
+    "from sklearn.ensemble import GradientBoostingClassifier\n",
+    "#use this classifier\n",
+    "classifier = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1, random_state=0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "classifier.fit(X_train2,y_train2)\n",
+    "test_y_pred=classifier.predict(X_test2)\n",
+    "test_y_pred_prob=classifier.predict_proba(X_test2)[:,1]\n",
+    "test_acc=metrics.accuracy_score(y_test2,test_y_pred)\n",
+    "test_roc=metrics.roc_auc_score(y_test2,test_y_pred_prob)\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 11 Test Cell\n",
+    "assert round(test_acc,3)==0.860"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 12 Test Cell\n",
+    "assert round(test_roc,3)==0.846"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

View_h5_file.py

@@ -0,0 +1,74 @@
+import h5py
+import pandas as pd
+import numpy as np
+import plotly.graph_objects as go
+
+f = h5py.File('product1.h5','r')
+list(f.keys())
+for key in f.keys():
+    print(f[key])
+    df = pd.read_hdf('product1.h5', key='df')
+    df = df._append(pd.read_hdf('product1.h5', key))
+    # print(f[key].shape)
+    # print(f[key].value)
+# df = pd.read_hdf('product1.h5', key='df7')
+print(df)
+'''
+# Open the HDF5 file
+with h5py.File('product1.h5', 'r') as file:
+    dataframes=pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])  
+    for key in file.keys():
+        # Read the dataframe from the HDF5 file
+        dataframe = pd.read_hdf('product1.h5', key)
+
+        # Append the dataframe to the list
+        dataframes._append(dataframe)
+
+# Print the number of dataframes read from the HDF5 file
+print(f"Read {len(dataframes)} dataframes from the HDF5 file")
+
+item_std_devs = {}
+for dataframe in dataframes:
+    item_name = dataframe['Product ID'].iloc[0]
+    std_dev = np.std(dataframe['Unit Price'])
+    item_std_devs[item_name] = std_dev
+
+# Print the item names and their corresponding standard deviations
+for item_name, std_dev in item_std_devs.items():
+    print(f"Item: {item_name}, Std Dev: {std_dev}")
+
+# Prepare the data for the graph
+x = list(item_std_devs.keys())
+y = list(item_std_devs.values())
+
+# Create a bar graph
+fig = go.Figure(data=[go.Bar(x=x, y=y)])
+
+# Customize the graph
+fig.update_layout(
+    title="Standard Deviation of Pricing for Each Item",
+    xaxis_title="Item",
+    yaxis_title="Standard Deviation",
+)
+
+# Save the graph as an HTML file
+report_file = 'test2.html'
+fig.write_html(report_file)
+
+
+
+'''
+'''
+# Calculate the standard deviation using pandas' function
+std_dev_unit = df.groupby('Product ID')['Unit Price'].std()
+std_dev_displayed = df.groupby('Zip Code')['Shown Price'].std()
+
+# Generate the graph using Plotly
+fig = go.Figure()
+fig.add_trace(go.Bar(x=std_dev_unit.index, y=std_dev_unit.values))
+fig.update_layout(title='Standard Deviation of the Unit Pricing by Item', xaxis_title='Product ID', yaxis_title='Standard Deviation')
+
+report_file = 'test2.html'
+fig.write_html(report_file)
+'''
+

assessment1.py

@@ -0,0 +1,81 @@
+'''
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+        self._attributes = dict(self.__dict__)
+        
+    def __setattr__(self, name, value):
+        if name != "_attributes" and name not in self._attributes:
+            print("attribute deleted!")
+        self._attributes[name] = value
+        super().__setattr__("_attributes", dict(self.__dict__))
+        
+    @property
+    def treasure(self):
+        if self.__dict__ != self._attributes:
+            print("attribute deleted!")
+        return self._treasure
+
+v=Vault()
+print(v.treasure)
+'''
+'''
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+        self._original_dict = dict(self.__dict__)
+
+    def __setattr__(self, name, value):
+        if name != "_treasure" and name in self.__dict__ and self._original_dict.get(name) != value:
+            print("attribute deleted!")
+        self.__dict__[name] = value
+
+    @property
+    def treasure(self):
+        if self.__dict__ != self._original_dict:
+            print("attribute deleted!")
+        return self._treasure
+v = Vault()
+print(v.treasure)
+'''
+'''
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+        self._attributes = dict(self.__dict__)
+
+    def __setattr__(self, name, value):
+        if name != "_attributes" and name not in self._attributes:
+            print("attribute deleted!")
+        self._attributes[name] = value
+        super().__setattr__("_attributes", dict(self.__dict__))
+
+    @property
+    def treasure(self):
+        if self.__dict__ != self._attributes:
+            print("attribute deleted!")
+        return self._treasure
+
+v = Vault()
+print(v.treasure)
+'''
+
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+
+    def __setattr__(self, name, value):
+        if name != "_attributes" and name not in self.__dict__:
+            print("attribute deleted!")
+        self.__dict__[name] = value
+        self._attributes = dict(self.__dict__)
+
+    @property
+    def treasure(self):
+        if self.__dict__ != self._attributes:
+            print("attribute deleted!")
+        return self._treasure
+
+v = Vault()
+print(v.treasure)
+

assessment2.py

@@ -0,0 +1,47 @@
+import importlib.util
+import os
+
+def get_module_size(module_name):
+    """Get the total number of bytes in the cached .pyc file or .py file of a module or package.
+
+    Args:
+        module_name (str): The name of the module or package.
+
+    Returns:
+        int: The total number of bytes in the cached .pyc file or .py file.
+    """
+    spec = importlib.util.find_spec(module_name)
+
+    if spec is not None and spec.has_location:
+        # If the module is found and has a location, check for .pyc file first
+        pyc_file_path = spec.origin + 'c'
+        if os.path.exists(pyc_file_path):
+            return os.stat(pyc_file_path).st_size
+        else:
+            # If .pyc file doesn't exist, check for .py file
+            py_file_path = spec.origin
+            if os.path.exists(py_file_path):
+                return os.stat(py_file_path).st_size
+
+    return 0  # Return 0 if no .pyc or .py file found
+
+
+import sys
+
+def get_total_module_size():
+    """Get the total number of bytes for all modules listed in sys.modules.
+
+    Returns:
+        int: The total number of bytes for all modules.
+    """
+    total_size = 0
+    for module_name in sys.modules:
+        module_size = get_module_size(module_name)
+        total_size += module_size
+
+    return total_size
+
+if __name__ == '__main__' and __package__ is None:
+    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
+total_size = get_total_module_size()
+print("Total size of all modules in sys.modules: {} bytes".format(total_size))

fp1.py

@@ -0,0 +1,65 @@
+from google.cloud import pubsub_v1
+from google.oauth2 import service_account
+import os
+import asyncio
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+project_id = "fp-20230425"
+
+'''
+topic_name = "fp-topic-Karen"
+subscription_name = "fp-subscription-Karen"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
+subscriber.delete_subscription(request={"subscription": subscription_path})
+print("Subscription deleted: {}".format(subscription_path))
+
+# Delete the topic
+publisher = pubsub_v1.PublisherClient()
+topic_path = publisher.topic_path(project_id, topic_name)
+publisher.delete_topic(request={"topic": topic_path})
+print(f"Topic '{topic_name}' deleted.")
+
+subscription_name = "fp-subscription-Karen"
+subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
+
+
+'''
+# Create a pubsub topic
+publisher = pubsub_v1.PublisherClient()
+topic_name = "fp-topic-Karen"
+topic_path = publisher.topic_path(project_id, topic_name)
+topic = publisher.create_topic(request={"name": topic_path})
+
+
+# Create a Pub/Sub subscription
+subscriber = pubsub_v1.SubscriberClient()
+subscription_name = "fp-subscription-Karen"
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+subscription = subscriber.create_subscription(
+    name=subscription_path, topic=topic_path
+)
+
+# Publish a message to the topic
+message = b"Hello!"
+publisher.publish(topic_path, data=message)
+print("Message published.")
+
+
+async def receive_messages():
+    response = subscriber.pull(subscription=subscription_path, max_messages=1)
+    for received_message in response.received_messages:
+        print("Received message: {}".format(received_message.message.data))
+       
+    
+asyncio.run(receive_messages())
+
+
+'''
+# Pull the message from the subscription
+response = subscriber.pull(subscription=subscription_path, max_messages=1)
+for received_message in response.received_messages:
+    print("Received message: {}".format(received_message.message.data))
+    # Acknowledge the message to mark it as processed
+    subscriber.acknowledge(subscription_path, [received_message.ack_id])
+'''

fp2.py

@@ -0,0 +1,10 @@
+import http.client
+
+# The product ID for the Pete-and-Gerry's organic large brown egges is 65902
+conn = http.client.HTTPSConnection("www.sayweee.com")
+payload = ''
+headers = {}
+conn.request("GET", "/en/product/Pete-and-Gerry-s-Organic-Large-Brown-Eggs/65902?category=dairy02", payload, headers)
+res = conn.getresponse()
+data = res.read()
+print(data.decode("utf-8"))

fp2_10.py

@@ -0,0 +1,67 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+import csv
+import os
+import asyncio
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+project_id = "fp-20230425"
+# project_id = "svig-384801"
+
+'''
+topic_name = "fp-topic-NFL-zipcodes"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
+subscriber.delete_subscription(request={"subscription": subscription_path})
+print("Subscription deleted: {}".format(subscription_path))
+
+# Delete the topic
+publisher = pubsub_v1.PublisherClient()
+topic_path = publisher.topic_path(project_id, topic_name)
+publisher.delete_topic(request={"topic": topic_path})
+print(f"Topic '{topic_name}' deleted.")
+
+
+
+'''
+# pubsub topic
+publisher = pubsub_v1.PublisherClient()
+topic_name = "fp-topic-NFL-zipcodes"
+topic_path = publisher.topic_path(project_id, topic_name)
+
+# Create a Pub/Sub subscription
+subscriber = pubsub_v1.SubscriberClient()
+'''
+subscription_name = "fp-subscription-zipcodes"
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+subscription = subscriber.create_subscription(
+    name=subscription_path, topic=topic_path
+)
+'''
+# Publish a message to the topic
+# Read the data from the csv file
+# file_path = "C:/Users/Karen/Desktop/zipcodes.csv"
+file_path = "zipcodes.csv"
+
+with open(file_path, "r") as csv_file:
+    reader = csv.reader(csv_file)
+    next(reader)  # skip header row
+    for row in reader:
+        # Format the data as a string and publish it to the topic
+        data = ",".join(row).encode("utf-8")
+        try:
+            future = publisher.publish(topic_path, data)
+            print(f"Published message ID {future.result()}")
+        except DeadlineExceeded as de:
+            print(f"Caught DeadlineExceeded exception: {de}")
+
+'''
+async def receive_messages():
+    response = subscriber.pull(subscription=subscription_path, max_messages=1)
+    for received_message in response.received_messages:
+        print("Received message: {}".format(received_message.message.data))
+       
+    
+asyncio.run(receive_messages())
+'''

fp2_11.py

@@ -0,0 +1,372 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+import selenium.common.exceptions
+import json
+import time
+import os
+import re
+import sys
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+
+# Create an empty dataframe to store the prices
+df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Displayed Price', 'ld+json Price', 'Minimum Quantity', 'Total Price', 'Sales Count', 'Sold Status'])
+print(f"Created a dataframe: {df}")
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+# Load the product page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+product_id = url.split("/")[-1]
+driver.delete_all_cookies
+driver.get(url)
+# time.sleep(5)
+
+# time.sleep(2)
+# Close the cookies setting window
+try:  
+    # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")
+    cookie_button = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+    )
+    print(f"cookie_button = {cookie_button}")
+    cookie_button.click()
+except:
+    pass
+
+# time.sleep(1)
+# Clear any modal dialog
+try:    
+    # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+    modal_close = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+    )
+    print(f"modal_close = {modal_close}")
+    modal_close.click()
+    #time.sleep(2)
+except:
+    pass
+
+
+def close_modals():
+    try:
+        # time.sleep(2)
+        # Clear any modal dialog
+        modal_close_buttons = driver.find_elements(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+        print(f"modal_close = {modal_close}")
+        for button in modal_close_buttons:
+            try:
+                button.click()
+            except Exception as e:
+                print(f"Exception when clicking on close button for element {button}: {e}")
+
+        # time.sleep(2)
+    except Exception as e:
+        print(f"Exception when trying to close discount modal dialog: {e}")
+
+
+    try:
+        # time.sleep(1)
+        # Clear any notice
+        notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+        print(f"notice_close = {notice_close}")
+        notice_close.click()
+        # time.sleep(1)
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        print("Purchase option change dialog close button not found")
+    except Exception as e:
+        print(f"Exception when trying to close purchase option change dialog: {e}")
+
+def get_element_text(class_name_prefix):
+    '''
+    Return the text in the first element with a given class name.
+
+    Note that XPath doesn't have a function to check if a given word is present
+    in a list of words, so using contains() will match a class name even if
+    the class name is a substring of the class name being matched against.
+    To prevent this, we use spaces as a delimiter at the beginning and end of
+    the class name, and ensure there's only one space before and after all
+    class names in the class name list.
+
+    Example: consider two class names with cache-busting suffixes:
+      * Header_soldStatusNotice__12345
+      * Header_soldStatusNoticeBackground__ABCDE
+
+    Matching against Header_soldStatusNotice using a naive implementation with
+    `contains()` would match both class names, even though only the first one
+    is the correct match. 
+    '''
+    # Construct xpath selector.  
+    xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+    xpath_str += "' '), "                     # Step 4: a trailing space...
+    xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # We didn't find a match, try again by doing a prefix match and assume
+    # the full class name ends with two underscores followed by a cache-
+    # busting string.    
+    xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+    xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # No match, return empty string.
+    return ''
+
+def callback(message):
+    '''
+    Callback function for Pub/Sub messages
+    '''
+    # Extract zip code from message data
+    zip_code = message.message.data.decode('utf-8')
+
+    print(f"Received zip code: {zip_code}")
+
+    while True:
+        try:
+            # Enter zip code in the input field and click the "Check" button
+            WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+            zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
+            # print(f"{zip_button:}")
+            zip_button.click()
+            # time.sleep(1)
+
+            WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, '//div/label/following-sibling::input'))
+            zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+            # print(f"{zip_input:}")
+            zip_input.click()
+            # print("clearing zip_input")
+            zip_input.send_keys("\b\b\b\b\b\b\b\b\b\b")
+            for i in range(1,10):
+                zip_input.send_keys(Keys.DELETE)
+            # print("sending zip code")
+            zip_input.send_keys(zip_code)
+
+            WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, "//button[text() = 'Confirm']"))
+            check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+            print(f"{check_button:}")
+            check_button.click()
+            # time.sleep(2)
+            break
+        except Exception as e:
+            print(f"Exception when changing zip code: {e}, trying to close modal dialogs")
+            close_modals()
+
+        # try:
+        #     time.sleep(2)
+        #     # Clear any notice
+        #     notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+        #     print(f"notice_close = {notice_close}")
+        #     notice_close.click()
+        #     time.sleep(2)
+        # except:
+        #     pass
+
+    try:
+        try:
+            # time.sleep(2)
+            # Clear any modal dialog
+            # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+            modal_close = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+        )
+            print(f"modal_close = {modal_close}")
+            modal_close.click()
+            # time.sleep(2)
+        except:
+            pass
+
+
+        try:
+            # time.sleep(1)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
+        )
+            print(f"notice_close = {notice_close}")
+            notice_close.click()
+            # time.sleep(1)
+        except:
+            pass
+        
+        # Enter zip code in the input field and click the "Check" button
+        # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
+        zip_button = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+        )
+        # print(f"{zip_button:}")
+        zip_button.click()
+        # time.sleep(1)
+
+        # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+        zip_input = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+        )
+        # print(f"{zip_input:}")
+        zip_input.click()
+        # print("clearing zip_input")
+        zip_input.send_keys("\b\b\b\b\b")
+        # print("sending zip code")
+        zip_input.send_keys(zip_code)
+
+        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+        check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+        # print(f"{check_button:}")
+        check_button.click()
+        # time.sleep(2)
+
+        try:
+            # time.sleep(2)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+            #time.sleep(2)
+        except:
+            pass
+
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+        # Record sales count, if any
+        sales_count = get_element_text('Header_count')
+        # Record any sold status notice
+        sold_status = get_element_text('soldStatusNotice')
+
+        # Check if there's a minimum order quantity
+        min_quantity: float = 0
+        total_price: float = 0
+        try:
+            minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' minQuantity ')]").get_attribute('innerHTML')
+            if minimum_quantity_str:
+                regexp="Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
+                match = re.search(regexp, minimum_quantity_str)
+                min_quantity, total_price = match.groups()
+
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            print("No minimum purchase string found")
+        except Exception as e:
+            print(f"Exception when finding minimum purchase string: {e}")
+
+        # Get the price as displayed on the web page
+        displayed_price = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' price_price ')]").get_attribute('innerText').replace('$', '')
+        print(f"displayed_price = {displayed_price}")
+
+        # Extract the price of rice crackers for this zip code     
+        '''
+        price_element = driver.find_element(By.XPATH, '/html/body/div[2]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')
+        price = price_element.text.strip()[1:]
+        print(price)
+        '''
+
+        # Get the price from the ld+json element
+        ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        ld_json_text = ld_json_element.get_attribute('innerHTML')
+        ld_json_start_index = ld_json_text.index('{')
+        ld_json_end_index = ld_json_text.rindex('}') + 1
+        ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
+        ld_json = json.loads(ld_json_json)
+        ld_json_price = ld_json['offers']['price']
+        # Print the price
+        print(f"ld_json_price = {ld_json_price}")
+
+        subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+        # Append the zip code and price to the dataframe
+        #append_to_df(zip_code, price)
+        new_row = pd.DataFrame({'Product ID': [product_id],
+            'Zip Code': [zip_code],
+            'Displayed Price': [displayed_price],
+            'ld+json Price': [ld_json_price],
+            'Minimum Quantity': [min_quantity],
+            'Total Price': [total_price],
+            'Sales Count': [sales_count],
+            'Sold Status': [sold_status]})
+        return pd.concat([df, new_row], ignore_index=True)
+        #print(f"Recorded the zip code and price: {zip_code},{price}")
+        #print(f"Updated a dataframe: {df}")
+
+    except Exception as e:
+        print(f"Caught exception {type(e)}: {e}")
+        print(f"Price not found for zip code {zip_code}")
+        price = 'Price not found'
+        return df
+
+# Start receiving messages (sync version)
+while True: 
+    pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+    messages = []
+    try:
+        pull_response = subscriber.pull(request = pull_request)
+        messages = pull_response.received_messages
+    except DeadlineExceeded:
+        pass
+
+    if messages:
+        df = callback(messages[0])
+        print(f"Updated a dataframe: {df}")
+        df.to_hdf('product1.h5', key='df',mode='w')
+    else:
+        print(f"No messages in subscription {subscription_path} at {time.asctime()}, trying again...")
+        continue
+
+# Open the HDF5 file in a separate window
+if sys.platform.startswith("win32"):
+    os.startfile('product1.h5')
+
+# Start receiving messages (async version)
+#flow_control = pubsub_v1.types.FlowControl(max_messages = 1)
+#streaming_pull_future = subscriber.subscribe(subscription_path, 
+#    callback=callback, 
+#    flow_control=flow_control)
+
+# print(f'Started receiving messages from subscription {subscription_path}...')
+
+# Wrap subscriber in a 'with' block to automatically call close() when done.
+# with subscriber:
+#    try:
+        # When `timeout` is not set, result() will block indefinitely,
+        # unless an exception is encountered first.
+#        streaming_pull_future.result(timeout=timeout)
+#    except TimeoutError:
+#        streaming_pull_future.cancel()  # Trigger the shutdown.
+#        streaming_pull_future.result()  # Block until the shutdown is complete.
+
+# Keep the script running indefinitely to continue receiving messages
+#while True:
+#    time.sleep(10)

fp2_12.py

@@ -0,0 +1,199 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+# Create an empty dataframe to store the prices
+df = pd.DataFrame(columns=['Zip Code', 'Price'])
+print(f"Created a dataframe: {df}")
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+# Load the product page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+driver.delete_all_cookies
+driver.get(url)
+# time.sleep(5)
+
+# time.sleep(2)
+# Close the cookies setting window
+try:  
+    # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")
+    cookie_button = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+    )
+    print(f"cookie_button = {cookie_button}")
+    cookie_button.click()
+except:
+    pass
+
+# time.sleep(1)
+# Clear any modal dialog
+try:    
+    # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+    modal_close = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+    )
+    print(f"modal_close = {modal_close}")
+    modal_close.click()
+    #time.sleep(2)
+except:
+    pass
+
+# Define callback function for Pub/Sub messages
+def callback(message):
+    # Extract zip code from message data
+    zip_code = message.message.data.decode('utf-8')
+
+    print(f"Received zip code: {zip_code}")
+
+    try:
+        try:
+            # time.sleep(2)
+            # Clear any modal dialog
+            # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+            modal_close = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+        )
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+            # time.sleep(2)
+        except:
+            pass
+
+
+        try:
+            # time.sleep(1)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
+        )
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+            # time.sleep(1)
+        except:
+            pass
+        
+        # Enter zip code in the input field and click the "Check" button
+        # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
+        zip_button = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+        )
+        # print(f"{zip_button:}")
+        zip_button.click()
+        # time.sleep(1)
+
+        # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+        zip_input = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+        )
+        # print(f"{zip_input:}")
+        zip_input.click()
+        # print("clearing zip_input")
+        zip_input.send_keys("\b\b\b\b\b")
+        # print("sending zip code")
+        zip_input.send_keys(zip_code)
+
+        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+        check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+        # print(f"{check_button:}")
+        check_button.click()
+        # time.sleep(2)
+
+        try:
+            # time.sleep(2)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+            #time.sleep(2)
+        except:
+            pass
+
+
+    # Extract the price of rice crackers for this zip code
+        
+        # next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        next_data_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, '//*[@id="__next"]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')))
+        next_data_text = next_data_element.get_attribute('nodeValue')
+        price = next_data_text.text
+        '''
+        next_data_text = next_data_element.get_attribute('innerHTML')
+        next_data_start_index = next_data_text.index('{')
+        next_data_end_index = next_data_text.rindex('}') + 1
+        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+        next_data = json.loads(next_data_json)
+        price = next_data['price']
+        '''
+        # Print the price
+        print(price)
+
+        '''
+        try:
+            min_quantity_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, '//script[@id="__NEXT_DATA__"]')))
+            min_quantity_text = min_quantity_element.get_attribute('innerHTML')
+            min_quantity_start_index = min_quantity_text.index('{')
+            min_quantity_end_index = min_quantity_text.rindex('}') + 1
+            min_quantity_json = min_quantity_text[min_quantity_start_index:min_quantity_end_index]
+            min_quantity = json.loads(min_quantity_json)
+            price = min_quantity['product']['price']
+        except:
+            next_data_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//script[@type='application/ld+json']")))
+            next_data_text = next_data_element.get_attribute('innerHTML')
+            next_data_start_index = next_data_text.index('{')
+            next_data_end_index = next_data_text.rindex('}') + 1
+            next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+            next_data = json.loads(next_data_json)        
+            price = next_data['offers']['price']
+        finally:
+            # Print the price
+            print(price)
+        '''
+
+        subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+        # Append the zip code and price to the dataframe
+        #append_to_df(zip_code, price)
+        new_row = pd.DataFrame({'Zip Code': [zip_code], 'Price': [price]})
+        return pd.concat([df, new_row], ignore_index=True)
+        #print(f"Recorded the zip code and price: {zip_code},{price}")
+        #print(f"Updated a dataframe: {df}")
+
+    except Exception as e:
+        print(f"Caught exception: {e}")
+        print(f"Price not found for zip code {zip_code}")
+        price = 'Price not found'
+        return df
+
+# Start receiving messages (sync version)
+while True: 
+    pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+    messages = []
+    try:
+        pull_response = subscriber.pull(request = pull_request)
+        messages = pull_response.received_messages
+    except DeadlineExceeded:
+        pass
+
+    if messages:
+        df = callback(messages[0])
+        print(f"Updated a dataframe: {df}")
+    else:
+        print(f"no messages, trying again at {time.asctime()}")

fp2_13.py

@@ -0,0 +1,271 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+import os
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import selenium.common.exceptions
+import re
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+
+# Create an empty dataframe to store the prices
+df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
+print(f"Created a dataframe: {df}")
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+# Load the product page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
+product_id = url.split("/")[-1][:5]
+# product_id = url.split("/")[-1]
+driver.delete_all_cookies
+driver.get(url)
+# time.sleep(5)
+
+def get_element_text(class_name_prefix):
+    '''
+    Return the text in the first element with a given class name.
+
+    Note that XPath doesn't have a function to check if a given word is present
+    in a list of words, so using contains() will match a class name even if
+    the class name is a substring of the class name being matched against.
+    To prevent this, we use spaces as a delimiter at the beginning and end of
+    the class name, and ensure there's only one space before and after all
+    class names in the class name list.
+
+    Example: consider two class names with cache-busting suffixes:
+      * Header_soldStatusNotice__12345
+      * Header_soldStatusNoticeBackground__ABCDE
+
+    Matching against Header_soldStatusNotice using a naive implementation with
+    `contains()` would match both class names, even though only the first one
+    is the correct match. 
+    '''
+    # Construct xpath selector.  
+    xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+    xpath_str += "' '), "                     # Step 4: a trailing space...
+    xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # We didn't find a match, try again by doing a prefix match and assume
+    # the full class name ends with two underscores followed by a cache-
+    # busting string.    
+    xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+    xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # No match, return empty string.
+    return ''
+
+
+# time.sleep(2)
+# Close the cookies setting window
+try:  
+    # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")
+    cookie_button = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+    )
+    print(f"cookie_button = {cookie_button}")
+    cookie_button.click()
+except:
+    pass
+
+# time.sleep(1)
+# Clear any modal dialog
+try:    
+    # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+    modal_close = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+    )
+    print(f"modal_close = {modal_close}")
+    modal_close.click()
+    #time.sleep(2)
+except:
+    pass
+
+# Define callback function for Pub/Sub messages
+def callback(message):
+    # Extract zip code from message data
+    zip_code = message.message.data.decode('utf-8')
+
+    print(f"Received zip code: {zip_code}")
+
+    try:
+        try:
+            # time.sleep(2)
+            # Clear any modal dialog
+            # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+            modal_close = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+        )
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+            # time.sleep(2)
+        except:
+            pass
+
+
+        try:
+            # time.sleep(1)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
+        )
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+            # time.sleep(1)
+        except:
+            pass
+        
+        # Enter zip code in the input field and click the "Check" button
+        # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
+        zip_button = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+        )
+        # print(f"{zip_button:}")
+        zip_button.click()
+        # time.sleep(1)
+
+        # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+        zip_input = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+        )
+        # print(f"{zip_input:}")
+        zip_input.click()
+        # print("clearing zip_input")
+        zip_input.send_keys("\b\b\b\b\b")
+        # print("sending zip code")
+        zip_input.send_keys(zip_code)
+
+        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+        check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+        # print(f"{check_button:}")
+        check_button.click()
+        # time.sleep(2)
+
+        try:
+            # time.sleep(2)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+            #time.sleep(2)
+        except:
+            pass
+
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+        # Record sales count, if any
+        sales_count_element = get_element_text('Header_count')
+        count_start_index = sales_count_element.index('sold ') + len('sold ')
+        count_end_index = sales_count_element.index('+')
+        sales_count = sales_count_element[count_start_index:count_end_index]
+        # Record any sold status notice
+        sold_status = get_element_text('UnavailableCartbtn_container')
+        if sold_status == 'Notify me':
+            sold_status = 'Sold Out'
+
+    # Extract the price of rice crackers for this zip code
+        # Check if there is a minimum order quantity
+        min_quantity: float = 0
+        total_price: float = 0
+        unit_price: float = 0
+        try:
+            minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML')
+            if minimum_quantity_str:
+                regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
+                match = re.search(regexp,minimum_quantity_str)
+                min_quantity, total_price = match.groups()
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            print("No minimum purhcase string found")
+        except Exception as e:
+            print(f"Exception when finding minimum purchase string: {e}")
+ 
+
+
+        ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        ld_json_text = ld_json_element.get_attribute('innerHTML')
+        ld_json_start_index = ld_json_text.index('{')
+        ld_json_end_index = ld_json_text.rindex('}') + 1
+        ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
+        ld_json = json.loads(ld_json_json)
+        ld_json_price = ld_json['offers']['price']
+        # Print the price
+        print(ld_json_price)
+
+        unit_price_element = driver.find_element(By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText')
+        unit_price = unit_price_element.replace("$", "")
+        print(f"unit_price = {unit_price}" )
+
+        subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+        # Append the zip code and price to the dataframe
+        #append_to_df(zip_code, price)
+        new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]})
+        return pd.concat([df, new_row], ignore_index=True)
+        #print(f"Recorded the zip code and price: {zip_code},{price}")
+        #print(f"Updated a dataframe: {df}")
+
+    except Exception as e:
+        print(f"Caught exception: {e}")
+        print(f"Price not found for zip code {zip_code}")
+        price = 'Price not found'
+        return df
+
+# Start receiving messages (sync version)
+while True: 
+    pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+    messages = []
+    try:
+        pull_response = subscriber.pull(request = pull_request)
+        messages = pull_response.received_messages
+    except DeadlineExceeded:
+        pass
+
+    if messages:
+        df = callback(messages[0])
+        print(f"Updated a dataframe: {df}")
+    else:
+        print(f"no messages, trying again at {time.asctime()}")
+        break
+
+# df.to_hdf('product1.h5', key='df',mode='w')
+# df.to_hdf('product1.h5', key='df',mode='a')
+
+# Open the HDF5 file in a separate window
+# os.startfile('product1.h5')

fp2_14.py

@@ -0,0 +1,280 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import selenium.common.exceptions
+import re
+from datetime import datetime
+import h5py
+import sys
+import ntplib
+import arrow
+
+# This fp2_14.py is used to load the new price and the timestamp, and append such data in an existing key in the HDF5 file "product1.h5"
+
+def get_product_prices(project_id, subscription_name, url):
+    # Create an empty dataframe to store the prices
+    df_new = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
+    print(f"Created a dataframe: {df_new}")
+
+    # Configure the Selenium driver options
+    options = Options()
+    options.add_argument('--disable-gpu')
+    driver = webdriver.Chrome(options=options)
+
+    # Set up Pub/Sub subscription
+    '''
+    project_id = "fp-20230425"
+    subscription_name = "fp-subscription-NFL-zipcodes"
+    '''
+    subscriber = pubsub_v1.SubscriberClient()
+    subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+
+    # Load the product page and extract the __NEXT_DATA__ data
+    # url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
+    product_id = url.split("/")[-1][:5]
+    # product_id = url.split("/")[-1]
+    driver.delete_all_cookies
+    driver.get(url)
+
+    def get_element_text(class_name_prefix):
+        '''
+        Return the text in the first element with a given class name.
+
+        Note that XPath doesn't have a function to check if a given word is present
+        in a list of words, so using contains() will match a class name even if
+        the class name is a substring of the class name being matched against.
+        To prevent this, we use spaces as a delimiter at the beginning and end of
+        the class name, and ensure there's only one space before and after all
+        class names in the class name list.
+
+        Example: consider two class names with cache-busting suffixes:
+        * Header_soldStatusNotice__12345
+        * Header_soldStatusNoticeBackground__ABCDE
+
+        Matching against Header_soldStatusNotice using a naive implementation with
+        `contains()` would match both class names, even though only the first one
+        is the correct match. 
+        '''
+        # Construct xpath selector.  
+        xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+        xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+        xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+        xpath_str += "' '), "                     # Step 4: a trailing space...
+        xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+        try:
+            match = driver.find_element(By.XPATH, xpath_str)
+            return match.get_attribute('textContent')
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            pass
+
+        # We didn't find a match, try again by doing a prefix match and assume
+        # the full class name ends with two underscores followed by a cache-
+        # busting string.    
+        xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+        xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+        xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+        xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+        try:
+            match = driver.find_element(By.XPATH, xpath_str)
+            return match.get_attribute('textContent')
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            pass
+
+        # No match, return empty string.
+        return ''
+
+
+    def modal_close():
+        try:    
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+    def notice_close():
+        try:
+            # Clear any notice
+            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+        except:
+            pass
+
+    def cookie_close():
+        try:  
+            cookie_button = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+            )
+            print(f"cookie_button = {cookie_button}")
+            cookie_button.click()
+        except:
+            pass
+
+    cookie_close()
+    modal_close()
+    # Define callback function for Pub/Sub messages
+    def callback(message):
+        # Extract zip code from message data
+        zip_code = message.message.data.decode('utf-8')
+
+        print(f"Received zip code: {zip_code}")
+
+        try:
+            
+            modal_close()
+            notice_close()
+
+            # Enter zip code in the input field and click the "Check" button
+            zip_button = WebDriverWait(driver, 5).until(
+                EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+            )
+            # print(f"{zip_button:}")
+            zip_button.click()
+
+            # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+            zip_input = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+            )
+            # print(f"{zip_input:}")
+            zip_input.click()
+            # print("clearing zip_input")
+            zip_input.send_keys("\b\b\b\b\b")
+            # print("sending zip code")
+            zip_input.send_keys(zip_code)
+
+            check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+            check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+            # print(f"{check_button:}")
+            check_button.click()
+
+            notice_close()
+            modal_close()
+
+            # Record sales count, if any
+            sales_count_element = get_element_text('Header_count')
+            count_start_index = sales_count_element.index('sold ') + len('sold ')
+            count_end_index = sales_count_element.index('+')
+            sales_count = sales_count_element[count_start_index:count_end_index]
+            # Record any sold status notice
+            sold_status = get_element_text('UnavailableCartbtn_container')
+            if sold_status == 'Notify me':
+                sold_status = 'Sold Out'
+
+        # Extract the price of rice crackers for this zip code
+            # Check if there is a minimum order quantity
+            min_quantity: float = 0
+            total_price: float = 0
+            unit_price: float = 0
+            try:
+                minimum_quantity_str = WebDriverWait(driver, 5).until(EC.presence_of_element_located(((By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML'))))
+                if minimum_quantity_str:
+                    regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
+                    match = re.search(regexp,minimum_quantity_str)
+                    min_quantity, total_price = match.groups()
+            except selenium.common.exceptions.NoSuchElementException as nsee:
+                print("No minimum purhcase string found")
+            except Exception as e:
+                print(f"Exception when finding minimum purchase string: {e}")
+    
+
+
+            ld_json_element = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//script[@type='application/ld+json']")))
+            ld_json_text = ld_json_element.get_attribute('innerHTML')
+            ld_json_start_index = ld_json_text.index('{')
+            ld_json_end_index = ld_json_text.rindex('}') + 1
+            ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
+            ld_json = json.loads(ld_json_json)
+            ld_json_price = ld_json['offers']['price']
+            # Print the price
+            print(ld_json_price)
+
+            unit_price_element = WebDriverWait(driver, 5).until(EC.presence_of_element_located(((By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText'))))
+            unit_price = unit_price_element.replace("$", "")
+            print(f"unit_price = {unit_price}" )
+
+            subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+            # Append the zip code and price to the dataframe
+            #append_to_df(zip_code, price)
+            new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]})
+            df_new = pd.concat([df_new, new_row], ignore_index=True)
+            return df_new
+            #print(f"Recorded the zip code and price: {zip_code},{price}")
+            #print(f"Updated a dataframe: {df_new}")
+
+        except Exception as e:
+            print(f"Caught exception: {e}")
+            print(f"Price not found for zip code {zip_code}")
+            return df_new
+
+    # Start receiving messages (sync version)
+    while True: 
+        pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+        messages = []
+        try:
+            pull_response = subscriber.pull(request = pull_request)
+            messages = pull_response.received_messages
+        except DeadlineExceeded:
+            pass
+
+        if messages:
+            df_new = callback(messages[0])
+            print(f"Updated a dataframe: {df_new}")
+        else:
+            print(f"no messages, trying again at {time.asctime()}")
+            break
+    return df_new
+
+'''
+If using sys to get the function, please remove the following part of code.
+'''
+if __name__ == '__main__':
+    project_id = input("Enter the project ID: ")
+    subscription_name = input("Enter the subscription name: ")
+    url = input("Enter the URL: ")
+
+    get_product_prices(project_id, subscription_name, url)
+
+
+df_new = get_product_prices("fp-20230425", "fp-subscription-NFL-zipcodes", "https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119")
+
+# Now we have the new dataframe "df" recording the new price listed in weee's page
+# Load such data in the existing HDF5 file
+file_path = 'product1.h5'
+with h5py.File(file_path, 'a') as file:
+    # Load the existing dataframe 'df' from the HDF5 file
+    # REPLACE KEY TO GET THE EXISTING DATAFRAME FOR EACH DIFFERENT PRODUCT
+    df = pd.read_hdf(file, key='df')
+
+    # Add timestamp to the new dataframe "df_new"
+    # timestamp = datetime.now()
+    '''
+    To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
+    provides time synchronization with a reliable time server.
+    Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
+    sensible and human-friendly approach to format and convert the timestamps.
+    '''
+    # Fetch the time from an NTP server
+    ntp_client = ntplib.NTPClient()
+    response = ntp_client.request('pool.ntp.org')
+    ntp_time = datetime.fromtimestamp(response.tx_time)
+    timestamp = arrow.get(ntp_time).to('local')
+    df_new['Timestamp'] = timestamp
+
+    # Append 'df_new' to the existing df stored in the key
+    df_update = df.append(df_new, ignore_index=True)
+
+    # Save 'df2' to the HDF5 file with the key 'df'
+    # REPLACE KEY FOR DIFFERENT PRODUCT
+    df_update.to_hdf(file, key='df', mode='a')
+
+print("Data appended and saved successfully.")

fp2_2.py

@@ -0,0 +1,32 @@
+import http.client
+import xml.etree.ElementTree as ET
+
+conn = http.client.HTTPSConnection("www.sayweee.com")
+payload = ''
+headers = {}
+conn.request("GET", "/en/product/Pete-and-Gerry-s-Organic-Large-Brown-Eggs/65902?category=dairy02", payload, headers)
+res = conn.getresponse()
+data = res.read()
+
+# Parse the HTML page content using ElementTree
+root = ET.fromstring(data.decode('utf-8'))
+
+if root is not None:
+    # Find the script element with __NEXT_DATA__ data
+    next_data_element = None
+    for script_element in root.iter('script'):
+        if script_element.text is not None and '__NEXT_DATA__' in script_element.text:
+            next_data_element = script_element
+            break
+
+    # Extract the __NEXT_DATA__ data
+    if next_data_element is not None:
+        next_data_text = next_data_element.txt
+        next_data_start_index = next_data_text.index('{')
+        next_data_end_index = next_data_text.rindex('}') + 1
+        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+        print(next_data_json)
+    else:
+        print('__NEXT_DATA__ element not found')
+else:
+    print('Failed to parse HTML content')

fp2_3.py

@@ -0,0 +1,32 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import json
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--headless')
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Load the page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Pete-and-Gerry-s-Organic-Large-Brown-Eggs/65902?category=dairy02'
+driver.get(url)
+wait = WebDriverWait(driver, 10)
+next_data_element = wait.until(EC.presence_of_element_located((By.XPATH, '//script[contains(text(), "__NEXT_DATA__")]')))
+next_data_text = next_data_element.text
+next_data_start_index = next_data_text.index('{')
+next_data_end_index = next_data_text.rindex('}') + 1
+next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+next_data = json.loads(next_data_json)
+
+# Extract the price from the __NEXT_DATA__ data
+price = next_data['props']['pageProps']['product']['price']
+
+# Close the Selenium driver
+driver.quit()
+
+# Print the price
+print(price)

fp2_4.py

@@ -0,0 +1,43 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+
+# Configure the Selenium driver options
+options = Options()
+#options.add_argument('--headless')  
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Load the page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+driver.get(url)
+time.sleep(5)
+next_data_element = None
+#try:
+next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+#except:
+ #   print('__NEXT_DATA__ element not found')
+
+#time.sleep(1000)
+if next_data_element is not None:
+    next_data_text = next_data_element.get_attribute('innerHTML')
+    print(next_data_text)
+    next_data_start_index = next_data_text.index('{')
+    next_data_end_index = next_data_text.rindex('}') + 1
+    next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+    next_data = json.loads(next_data_json)
+
+    # Extract the price from the __NEXT_DATA__ data
+ #   price = next_data['props']['pageProps']['product']['price']
+    print(next_data)
+    price = next_data['offers']['price']
+    # Print the price
+    print(price)
+else:
+    print('__NEXT_DATA__ element not found')
+
+time.sleep(1000)
+# Close the Selenium driver
+#driver.quit()

fp2_5.py

@@ -0,0 +1,33 @@
+import re
+import json
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--headless')
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Load the page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+driver.get(url)
+next_data_element = None
+for script_element in driver.find_elements(By.XPATH, '//script'):
+    script_text = script_element.get_attribute('innerHTML')
+    match = re.search(r'__NEXT_DATA__ = (\{.*\});', script_text)
+    if match:
+        next_data_json = match.group(1)
+        next_data = json.loads(next_data_json)
+        # Extract the price from the __NEXT_DATA__ data
+        price = next_data['props']['pageProps']['product']['price']
+        # Print the price
+        print(price)
+        break
+
+if next_data_element is None:
+    print('__NEXT_DATA__ element not found')
+
+# Close the Selenium driver
+driver.quit()

fp2_6.py

@@ -0,0 +1,126 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Define the zip codes for which to extract the price
+zip_codes = ['85305', '30313', '21230', '14127','28202','60605','45202','44114','76011','80204','48226','54304','77054','46225','32202','64129','89118','90301','33056','55415','02035','70112','07073','19148','15212','95054','98134','33607','37213','20785']
+
+# Load the page and extract the __NEXT_DATA__ data
+#url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+url = 'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261'
+#url = 'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467'
+#url = 'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01'
+#url = 'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224'
+#url = 'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02'
+#url = 'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01'
+#url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
+#url = 'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353?trace_id=7847dd6e-c511-48e2-a76e-a283399967b1'
+#url = 'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03'
+
+driver.delete_all_cookies
+driver.get(url)
+
+# Close the cookies setting window
+try:  
+    cookie_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")))
+    print(f"cookie_button = {cookie_button}")
+    cookie_button.click()
+except:
+    pass
+
+# Clear any modal dialog
+try:    
+    modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+    print(f"modal_close = {modal_close}")
+    modal_close.click()
+except:
+    pass
+
+def callback(zip_code):
+    try:
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+
+        try:
+            # Clear any notice
+            notice_close = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+        except:
+            pass
+
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+        # Enter zip code in the input field and click the "Check" button
+        zip_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')))
+        # print(f"{zip_button:}")
+        zip_button.click()
+
+        zip_input = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input')))
+        # print(f"{zip_input:}")
+        zip_input.click()
+        # print("clearing zip_input")
+        zip_input.send_keys("\b\b\b\b\b")
+        # print("sending zip code")
+        zip_input.send_keys(zip_code)
+
+        try:
+            check_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//button[text() = 'Confirm']")))
+            # print(f"{check_button:}")
+        except:
+            check_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[5]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[3]/div/button")))
+        finally:
+            check_button.click()
+
+        try:
+            # Clear any notice
+            notice_close = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+        except:
+            pass
+
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+            # Extract the price of rice crackers for this zip code
+        try:
+            sold_out_notice = driver.find_element(By.XPATH, '//*[@id="soldStatusNotice"]')
+            print(f"No such product in this area = {zip_code, sold_out_notice}")
+        except:
+            print(f"The product is available in this area {zip_code}")
+    
+    except Exception as e:
+        print(f"Caught exception: {e}")
+        print(f"Price not found for zip code {zip_code}")
+
+for zip_code in zip_codes:
+    callback(zip_code)
+
+driver.quit()

fp2_7.py

@@ -0,0 +1,64 @@
+from google.cloud import pubsub_v1
+import csv
+import os
+import asyncio
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+# project_id = "fp-20230425"
+project_id = "svig-384801"
+
+'''
+topic_name = "fp-topic-NFL-zipcodes"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
+subscriber.delete_subscription(request={"subscription": subscription_path})
+print("Subscription deleted: {}".format(subscription_path))
+
+# Delete the topic
+publisher = pubsub_v1.PublisherClient()
+topic_path = publisher.topic_path(project_id, topic_name)
+publisher.delete_topic(request={"topic": topic_path})
+print(f"Topic '{topic_name}' deleted.")
+
+'''
+
+
+# Create a pubsub topic
+publisher = pubsub_v1.PublisherClient()
+topic_name = "fp-topic-NFL-zipcodes"
+topic_path = publisher.topic_path(project_id, topic_name)
+topic = publisher.create_topic(request={"name": topic_path}, timeout=30)
+
+
+# Create a Pub/Sub subscription
+subscriber = pubsub_v1.SubscriberClient()
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+subscription = subscriber.create_subscription(
+    name=subscription_path, topic=topic_path
+)
+'''
+# Publish a message to the topic
+# Read the data from the csv file
+# file_path = "C:/Users/Karen/Desktop/NFL_Cities.csv"
+file_path = zipcodes.csv
+
+with open(file_path, "r") as csv_file:
+    reader = csv.reader(csv_file)
+    next(reader)  # skip header row
+    for row in reader:
+        # Format the data as a string and publish it to the topic
+        data = ",".join(row).encode("utf-8")
+        future = publisher.publish(topic_path, data)
+        print(f"Published message ID {future.result()}")
+
+
+async def receive_messages():
+    response = subscriber.pull(subscription=subscription_path, max_messages=1)
+    for received_message in response.received_messages:
+        print("Received message: {}".format(received_message.message.data))
+       
+    
+asyncio.run(receive_messages())
+'''

fp2_8.py

@@ -0,0 +1,55 @@
+import json
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import time
+from google.cloud import pubsub_v1
+import os
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+
+options = Options()
+options.add_argument('--headless')
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+def callback(message):
+    try:
+        # Get the zip code from the message
+        zip_code = message.data.decode('utf-8')
+        print(f"Received zip code: {zip_code}")
+        
+        # Load the webpage with the zip code
+        url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+        driver.get(url)
+        driver.find_element(By.XPATH, '/html/body/div[4]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[2]/input').send_keys(zip_code)
+        driver.find_element(By.XPATH, '/html/body/div[4]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[3]/div').click()
+        
+        # Extract the price from the webpage
+        next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        next_data_text = next_data_element.get_attribute('innerHTML')
+        next_data_start_index = next_data_text.index('{')
+        next_data_end_index = next_data_text.rindex('}') + 1
+        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+        next_data = json.loads(next_data_json)
+        price = next_data['offers']['price']
+        
+        # Print the result
+        print(f"Price of rice crackers in zip code {zip_code}: {price}")
+    except Exception as e:
+        print(f"Error processing message: {str(e)}")
+    finally:
+        message.ack()
+
+subscriber.subscribe(subscription_path, callback=callback)
+
+while True:
+    time.sleep(1)
+
+

fp2_9.py

@@ -0,0 +1,106 @@
+from google.cloud import pubsub_v1
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+import os
+from concurrent.futures import TimeoutError
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+# project_id = "fp-20230425"
+project_id = "svig-384801"
+subscription_name = "fp-subscription-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+timeout = 5
+
+# Load the product page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+driver.get(url)
+# time.sleep(5)
+'''
+next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+if next_data_element is not None:
+    next_data_text = next_data_element.get_attribute('innerHTML')
+    next_data_start_index = next_data_text.index('{')
+    next_data_end_index = next_data_text.rindex('}') + 1
+    next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+    next_data = json.loads(next_data_json)
+else:
+    print('__NEXT_DATA__ element not found')
+'''
+    
+# Define callback function for Pub/Sub messages
+def callback(message):
+    # Extract zip code from message data
+    zip_code = message.data.decode('utf-8')
+    print(f"Received zip code: {zip_code}")
+
+    try:
+
+        time.sleep(5)
+        # Clear any modal dialog
+        modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+        print(f"modal_close = {modal_close}")
+        modal_close.click()
+        time.sleep(2)
+
+        # Enter zip code in the input field and click the "Check" button
+        zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]/div/div/div[1]')
+        print(zip_button)
+        zip_button.click()
+        time.sleep(1)
+
+        zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+        zip_input.clear()
+        zip_input.send_keys(zip_code)
+
+        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+        check_button.click()
+        time.sleep(2)
+
+        # Extract the price of rice crackers for this zip code
+        '''
+        price_element = driver.find_element(By.XPATH, '/html/body/div[2]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')
+        price = price_element.text.strip()[1:]
+        print(price)
+        '''
+        next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        next_data_text = next_data_element.get_attribute('innerHTML')
+        next_data_start_index = next_data_text.index('{')
+        next_data_end_index = next_data_text.rindex('}') + 1
+        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+        next_data = json.loads(next_data_json)
+        price = next_data['offers']['price']
+    # Print the price
+        print(price)
+        message.ack()  # acknowledge receipt of message
+    except:
+        print(f"Price not found for zip code {zip_code}")
+        price = 'Price not found'
+
+# Start receiving messages
+streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback)
+print(f'Started receiving messages from subscription {subscription_path}...')
+
+# Wrap subscriber in a 'with' block to automatically call close() when done.
+with subscriber:
+    try:
+        # When `timeout` is not set, result() will block indefinitely,
+        # unless an exception is encountered first.
+        streaming_pull_future.result(timeout=timeout)
+    except TimeoutError:
+        streaming_pull_future.cancel()  # Trigger the shutdown.
+        streaming_pull_future.result()  # Block until the shutdown is complete.
+
+# Keep the script running indefinitely to continue receiving messages
+while True:
+    time.sleep(10)

fp3_1.py

@@ -0,0 +1,166 @@
+import sqlalchemy
+from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Float, DateTime
+from sqlalchemy.orm import declarative_base
+from sqlalchemy.orm import relationship, sessionmaker
+import sys
+from flask import Flask, render_template
+
+# Create the Flask application
+app = Flask(__name__)
+
+# Create the database engine
+engine = create_engine('sqlite:///scraper_design.db')
+
+# Create a session factory
+Session = sessionmaker(bind=engine)
+
+# Create a base class
+Base = declarative_base()
+
+# Define the product class
+class Product(Base):
+    __tablename__ = 'product'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    SKU_PLU_UPC_number = Column(String)
+    price_histories = relationship("PriceHistory", back_populates="product")
+
+# Define the retailer class
+class Retailer(Base):
+    __tablename__ = 'retailer'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    products = relationship("Product", secondary="retailer_product")
+
+# Define the price class
+class Price(Base):
+    __tablename__ = 'price'
+    id = Column(Integer, primary_key=True)
+    MOQ = Column(Integer)
+    price = Column(Float)
+
+# Define the scraperjob class
+class ScraperJob(Base):
+    __tablename__ = 'scraper_job'
+    id = Column(Integer, primary_key=True)
+    start_time = Column(DateTime)
+    end_time = Column(DateTime)
+    parameters = Column(String)
+    statistics = Column(String)
+
+# Define the price history class
+class PriceHistory(Base):
+    __tablename__ = 'price_history'
+    id = Column(Integer, primary_key=True)
+    location = Column(String)
+    zip_code = Column(Integer)
+    city_name = Column(String)
+    price_at_time = Column(DateTime)
+    price = Column(Float)
+    scraper_job_id = Column(Integer, ForeignKey('scraper_job.id'))
+    product_id = Column(Integer, ForeignKey('product.id'))
+    product = relationship("Product", back_populates="price_histories")
+    scraper_job = relationship("ScraperJob")
+
+# Define the user class
+class User(Base):
+    __tablename__ = 'user'
+    id = Column(Integer, primary_key=True)
+    username = Column(String)
+    credentials = Column(String)
+    creation_date = Column(DateTime)
+    modification_date = Column(DateTime)
+    def __str__(self):
+        return f"{self.id=} {self.username=} {self.creation_date=} {self.modification_date=}"
+
+class RetailerProduct(Base):
+    __tablename__ = 'retailer_product'
+    retailer_id = Column(Integer, ForeignKey('retailer.id'), primary_key=True)
+    product_id = Column(Integer, ForeignKey('product.id'), primary_key=True)
+
+# Create the database tables
+Base.metadata.create_all(engine)
+
+@app.route('/')
+def homepage():
+    # Create a session
+    session = Session()
+
+    # Get all the classes from SQLAlchemy
+    classes = Base.__subclasses__()
+
+    statement = sqlalchemy.select(User)
+    users = session.scalars(statement).all()
+
+    # Render the template with the classes
+    return render_template('test3.html', classes=classes, users=users)
+
+# Create command-line arguments for the user class
+def main():
+    if len(sys.argv) < 2:
+        app.run()
+
+    command = sys.argv[1]
+
+    if command == "create":
+        if len(sys.argv) != 3:
+            print("Usage: python user_tool.py create [username]")
+            return
+
+        username = sys.argv[2]
+
+        # Create a new user object
+        user = User(username=username)
+
+        # Add the user to the database
+        session = Session()
+        session.add(user)
+        session.commit()
+
+        print("User created successfully.")
+
+    elif command == "edit":
+        if len(sys.argv) != 4:
+            print("Usage: python user_tool.py edit [user_id] [new_username]")
+            return
+
+        user_id = int(sys.argv[2])
+        new_username = sys.argv[3]
+
+        # Retrieve the user from the database
+        session = Session()
+        user = session.query(User).get(user_id)
+
+        if user:
+            # Update the user object
+            user.username = new_username
+            session.commit()
+            print("User updated successfully.")
+        else:
+            print("User not found.")
+
+    elif command == "delete":
+        if len(sys.argv) != 3:
+            print("Usage: python user_tool.py delete [user_id]")
+            return
+
+        user_id = int(sys.argv[2])
+
+        # Retrieve the user from the database
+        session = Session()
+        user = session.query(User).get(user_id)
+
+        if user:
+            # Delete the user from the database
+            session.delete(user)
+            session.commit()
+            print("User deleted successfully.")
+        else:
+            print("User not found.")
+
+    else:
+        print("Usage: python user_tool.py [create|edit|delete]")
+        return
+
+if __name__ == "__main__":
+    main()

fp_merge.py

@@ -0,0 +1,100 @@
+from fp2_14 import get_product_prices
+import h5py
+import pandas as pd
+import datetime
+import sys
+from datetime import datetime
+import ntplib
+import arrow
+import concurrent.futures
+import pandas as pd
+
+# Define the subset of urls
+url_list = [
+    'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
+    'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
+    'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
+    'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
+    'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
+    'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
+    'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
+    'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
+    'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
+    'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
+    'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
+]
+
+
+# Define the function to run the scraper
+def run_scraper(url):
+    project_id = "fp-20230425"
+    subscription_name = "fp-subscription-NFL-zipcodes"
+    df_new = get_product_prices(project_id, subscription_name, url)
+
+    # Add timestamp to the new dataframe "df_new"
+    '''
+    To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
+    provides time synchronization with a reliable time server.
+    Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
+    sensible and human-friendly approach to format and convert the timestamps.
+    '''
+    # Fetch the time from an NTP server
+    ntp_client = ntplib.NTPClient()
+    response = ntp_client.request('pool.ntp.org')
+    ntp_time = datetime.fromtimestamp(response.tx_time)
+    timestamp = arrow.get(ntp_time).to('local')
+    df_new['Timestamp'] = timestamp
+
+    # Append 'df_new' to the existing df stored in the key
+    df_update = df_new.append(df_new, ignore_index=True)
+
+    return df_update
+
+
+'''
+# When using sys to call the 'get_product_prices' function
+
+if __name__ == '__main__':
+    # Check if the required command-line arguments are provided
+    if len(sys.argv) < 4:
+        print("Usage: python test_fp2_14.py project_id subscription_name url")
+        sys.exit(1)
+
+    project_id = sys.argv[1]
+    subscription_name = sys.argv[2]
+    url = sys.argv[3:]
+
+    get_product_prices(project_id, subscription_name, url)
+'''
+
+
+if __name__ == '__main__':
+    # Determine the number of available CPUs
+    # cpu_count = multiprocessing.cpu_count()
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        future_to_url = {executor.submit(run_scraper, url): url for url in url_list}
+        results = []
+        for future in concurrent.futures.as_completed(future_to_url):
+            url = future_to_url[future]
+            try:
+                result = future.result()
+                print(f"Scraped data for URL: {url}")
+                results.append(result)
+            except Exception as e:
+                print(f"Failed to scrape data for URL: {url}\nError: {e}")
+
+
+# Now we have the new dataframe "df_update" recording the new price listed in weee's page
+# Load such data in the existing HDF5 file
+file_path = 'product1.h5'
+with pd.HDFStore(file_path) as store:
+    for url, result in zip(url_list, results):
+        # Extract the product ID from the URL
+        product_id = url.split("/")[-1][:5]
+
+        # Save the dataframe to the HDF5 file with a unique key
+        key = f'df_{product_id}'
+        store[key] = result
+        result.to_hdf('product_data.h5', key=key, mode='a')
+        print("Data appended and saved successfully.")

hello.py

@@ -0,0 +1 @@
+print("hello")

http-performance/Webpage.ipynb

@@ -1,78 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "0593e529",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "from urllib.parse import urlparse\n",
-    "\n",
-    "class JPEGRequestHandler:\n",
-    "    def __init__(self, app):\n",
-    "        self.app = app\n",
-    "\n",
-    "    def __call__(self, environ, start_response):\n",
-    "        url = urlparse(environ['REQUEST_URI'])\n",
-    "        path = url.path\n",
-    "\n",
-    "        if path.endswith('.jpg'):\n",
-    "            file_path = path[1:]\n",
-    "\n",
-    "            if os.path.isfile(file_path):\n",
-    "                with open(file_path, 'rb') as file:\n",
-    "                    response_body = file.read()\n",
-    "                    status = '200 OK'\n",
-    "                    headers = [('Content-type', 'image/jpeg'),\n",
-    "                               ('Content-Length', str(len(response_body)))]\n",
-    "\n",
-    "                    start_response(status, headers)\n",
-    "                    return [response_body]\n",
-    "            else:\n",
-    "                status = '404 Not Found'\n",
-    "                headers = [('Content-type', 'text/plain'),\n",
-    "                           ('Content-Length', '0')]\n",
-    "\n",
-    "                start_response(status, headers)\n",
-    "                return []\n",
-    "        else:\n",
-    "            status = '403 Forbidden'\n",
-    "            headers = [('Content-type', 'text/plain'),\n",
-    "                       ('Content-Length', '0')]\n",
-    "\n",
-    "            start_response(status, headers)\n",
-    "            return []\n",
-    "\n",
-    "if __name__ == '__main__':\n",
-    "    from werkzeug.serving import run_simple\n",
-    "    from werkzeug.middleware.dispatcher import DispatcherMiddleware\n",
-    "\n",
-    "    app = DispatcherMiddleware(JPEGRequestHandler(app), {'/': app})\n",
-    "    run_simple('0.0.0.0', 8000, app, use_reloader=True)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.13"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

http-performance/app.py

@@ -1,44 +0,0 @@
-import os
-from urllib.parse import urlparse
-
-class JPEGRequestHandler:
-    def __init__(self, app):
-        self.app = app
-
-    def __call__(self, environ, start_response):
-        url = urlparse(environ['REQUEST_URI'])
-        path = url.path
-
-        if path.endswith('.jpg'):
-            file_path = path[1:]
-
-            if os.path.isfile(file_path):
-                with open(file_path, 'rb') as file:
-                    response_body = file.read()
-                    status = '200 OK'
-                    headers = [('Content-type', 'image/jpeg'),
-                               ('Content-Length', str(len(response_body)))]
-
-                    start_response(status, headers)
-                    return [response_body]
-            else:
-                status = '404 Not Found'
-                headers = [('Content-type', 'text/plain'),
-                           ('Content-Length', '0')]
-
-                start_response(status, headers)
-                return []
-        else:
-            status = '403 Forbidden'
-            headers = [('Content-type', 'text/plain'),
-                       ('Content-Length', '0')]
-
-            start_response(status, headers)
-            return []
-
-if __name__ == '__main__':
-    from werkzeug.serving import run_simple
-    from werkzeug.middleware.dispatcher import DispatcherMiddleware
-
-    app = DispatcherMiddleware(JPEGRequestHandler(app), {'/': app})
-    run_simple('0.0.0.0', 8000, app, use_reloader=True)

http-performance/testhh.py

@@ -0,0 +1,54 @@
+'''
+import os
+from flask import Flask, Response
+from urllib.parse import urlparse
+
+app = Flask(__name__)
+
+@app.route('/')
+def index():
+    return 'Hello, welcome to the cat picture server!'
+
+@app.route('/<path:path>')
+def serve_jpeg(path):
+    if path.endswith('.jpg'):
+        file_path = path[1:]
+        if os.path.isfile(file_path):
+            with open(file_path, 'rb') as file:
+                return Response(file.read(), content_type='image/jpeg')
+        else:
+            return 'File not found', 404
+    else:
+        return 'Only .jpg files are allowed', 403
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=8000)
+'''
+'''
+import os
+from fastapi import FastAPI
+from fastapi.responses import FileResponse
+from urllib.parse import urlparse
+
+app = FastAPI()
+
+@app.get('/')
+def index():
+    return 'Hello, welcome to the cat picture server!'
+
+@app.get('/{path:path}')
+async def serve_jpeg(path: str):
+    if path.endswith('.jpg'):
+        file_path = path[1:]
+        if os.path.isfile(file_path):
+            return FileResponse(file_path, media_type='image/jpeg')
+        else:
+            return {'detail': 'File not found'}, 404
+    else:
+        return {'detail': 'Only .jpg files are allowed'}, 403
+
+if __name__ == '__main__':
+    import uvicorn
+    uvicorn.run(app, host='0.0.0.0', port=8000)
+'''
+

hw10.py

@@ -0,0 +1,14 @@
+import sys
+
+# Iterate through the modules in sys.modules
+for module_name, module in sys.modules.items():
+    # Check if the module has a loader attribute
+    if hasattr(module, '__loader__'):
+        loader = module.__loader__
+        # Determine the type of loader
+        if type(loader).__name__ == 'SourceFileLoader':
+            print(f"Module '{module_name}' has a source loader.")
+        elif type(loader).__name__ == 'BuiltinImporter':
+            print(f"Module '{module_name}' has a built-in loader.")
+        elif type(loader).__name__ == 'ExtensionFileLoader':
+            print(f"Module '{module_name}' has an extension loader.")

hw11.py

@@ -0,0 +1,12 @@
+from cx_Freeze import setup, Executable
+
+executables = [Executable('module-inspector.py')]
+
+setup(
+    name='wow',
+    version='1.0',
+    description='Module-inspector Application',
+    executables=executables
+)
+
+

hw12.py

@@ -0,0 +1,7 @@
+i = 5
+
+def f(arg=i):
+    print(arg)
+
+i = 6
+f()

hw8.py

@@ -0,0 +1,47 @@
+import importlib.util
+import os
+
+def get_module_size(module_name):
+    """Get the total number of bytes in the cached .pyc file or .py file of a module or package.
+
+    Args:
+        module_name (str): The name of the module or package.
+
+    Returns:
+        int: The total number of bytes in the cached .pyc file or .py file.
+    """
+    spec = importlib.util.find_spec(module_name)
+
+    if spec is not None and spec.has_location:
+        # If the module is found and has a location, check for .pyc file first
+        pyc_file_path = spec.origin + 'c'
+        if os.path.exists(pyc_file_path):
+            return os.stat(pyc_file_path).st_size
+        else:
+            # If .pyc file doesn't exist, check for .py file
+            py_file_path = spec.origin
+            if os.path.exists(py_file_path):
+                return os.stat(py_file_path).st_size
+
+    return 0  # Return 0 if no .pyc or .py file found
+
+
+import sys
+
+def get_total_module_size():
+    """Get the total number of bytes for all modules listed in sys.modules.
+
+    Returns:
+        int: The total number of bytes for all modules.
+    """
+    total_size = 0
+    for module_name in sys.modules:
+        module_size = get_module_size(module_name)
+        total_size += module_size
+
+    return total_size
+
+if __name__ == '__main__' and __package__ is None:
+    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
+total_size = get_total_module_size()
+print("Total size of all modules in sys.modules: {} bytes".format(total_size))

hw9.py

@@ -0,0 +1,74 @@
+import os
+import sys
+import importlib.util
+import types
+from collections import defaultdict
+from sys import getsizeof
+
+
+def get_bytecode_size(module_name):
+    """Get the size of the cached .pyc bytecode file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin and spec.origin.endswith('.py'):
+        pyc_file = importlib.util.cache_from_source(spec.origin)
+        if os.path.exists(pyc_file):
+            return os.path.getsize(pyc_file)
+    return 0
+
+
+def get_dso_size(module_name):
+    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin and spec.origin.endswith(('.so', '.dll')):
+        return os.path.getsize(spec.origin)
+    return 0
+
+
+def generate_module_report():
+    """Generate a report summarizing the module environment."""
+    total_modules = len(sys.modules)
+    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
+    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
+    python_size = sum(
+        getsizeof(sys.modules[module_name]) - getsizeof(module_name)
+        for module_name in sys.modules
+        if isinstance(sys.modules[module_name], types.ModuleType)
+    )
+    total_size = bytecode_size + dso_size + python_size
+
+    # Calculate percentage of each type of module
+    bytecode_percent = bytecode_size / total_size * 100
+    dso_percent = dso_size / total_size * 100
+    python_percent = python_size / total_size * 100
+
+    # Generate the report
+    report = f"Total modules: {total_modules}\n"
+    report += f"Total size: {total_size} bytes\n"
+    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
+    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
+    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
+
+    # Generate histogram of module directory names
+    dir_histogram = defaultdict(int)
+    for module_name in sys.modules:
+        module_file = getattr(sys.modules[module_name], '__file__', None)
+        if module_file:
+            module_dir = os.path.dirname(module_file)
+            dir_histogram[module_dir] += 1
+
+    report += "\nModule Directory Histogram:\n"
+    for module_dir, count in dir_histogram.items():
+        report += f"{module_dir}: {'*' * count}\n"
+
+    return report
+
+
+if __name__ == "__main__":
+    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
+report = generate_module_report()
+print(report)
+
+
+
+
+

max submission/5_26_HW.py

@@ -1,20 +0,0 @@
-# Question7:
-# Difference: 
-# 1. Byte objects are sequence of Bytes, whereas Strings are sequence of characters. 
-# 2. Byte objects are in machine readable form internally; Strings are only in human readable form. 
-# 3. strings (str) support various character encodings, such as UTF-8, ASCII, and more. Bytes (bytes), on the other hand, represent raw binary data and can hold any byte values. 
-## Emojis can affect the size difference.
-import sys
-string_size = sys.getsizeof("Hello")
-bytes_size = sys.getsizeof(b"Hello")
-emoji_size = sys.getsizeof("😀")
-print(f"String size: {string_size} bytes")
-print(f"Bytes size: {bytes_size} bytes")
-print(f"emoji size: {emoji_size} bytes")
-
-# python routine:
-def print_string_as_numbers(string):
-    for char in string:
-        print(ord(char))
-input_string = "Hello, world!"
-print_string_as_numbers(input_string)

max submission/hw1 1-5.py

@@ -1,21 +0,0 @@
-# question 1: The international standard for floating-point representation is called IEEE 754.
-
-# question 2: The IEEE 754 standard allows floating-point numbers to be represented with different bit sizes. The most commonly used formats specified by the standard are single precision (32 bits) and double precision (64 bits).
-
-# question 3: 	
-import sys
-print(sys.float_info)
-sys.float_info(max=1.7976931348623157e+308, max_exp=1024, max_10_exp=308, min=2.2250738585072014e-308, min_exp=-1021, min_10_exp=-307, dig=15, mant_dig=53, epsilon=2.220446049250313e-16, radix=2, rounds=1)
-
-# question 4: 	In the first multiplication (num1 * num2), you are using the normal Python multiplication 
-# operator, which operates on native Python integers. Python integers have arbitrary precision, meaning they can
-#  represent very large numbers accurately. However, when the result exceeds the limit of a 64-bit signed integer 
-# (2^63 - 1), an overflow may occur, and the result may not be accurate.
-# In the second multiplication (df['Number1'] * df['Number2']), you are using the Pandas multiplication operator 
-# on the columns of a DataFrame. By default, Pandas uses NumPy for numerical computations, and NumPy uses 
-# fixed-size data types. In this case, the integers in the DataFrame are most likely represented as 64-bit 
-# signed integers (int64), which have a maximum value of 2^63 - 1. If the result of the multiplication exceeds 
-# this limit, it will overflow and wrap around, leading to incorrect results.
-
-# Question5: 5.	The size of a Python floating-point number is determined by the implementation and is not influenced by the magnitude of the value it represents. So it does not change as the value of the number gets larger.
-

module-inspector/module-inspector.py

@@ -2,6 +2,7 @@ import sys
 import tkinter as tk
 from tkinter import ttk
 
+
 def on_treeview_heading_click(column, tree):
     data = [(tree.set(child, column), child) for child in tree.get_children("")]
     reverse = (column == tree.heading(column)["text"]) and not column.startswith("_")
@@ -38,6 +39,10 @@ tree.column('#0', width=20, stretch=False)
 
 tree.tag_configure("selected", background="#ccc")
 
+# Call the on_treeview_heading_click function when a column heading is clicked
+for col in tree["columns"]:
+    tree.heading(col, text=col, anchor=tk.W, command=lambda col=col: on_treeview_heading_click(col, tree))
+root.mainloop()
 # Call the on_treeview_heading_click function when a column heading is clicked
 for col in tree["columns"]:
     tree.heading(col, text=col, anchor=tk.W, command=lambda col=col: on_treeview_heading_click(col, tree))

public

@@ -0,0 +1,9 @@
+---- BEGIN SSH2 PUBLIC KEY ----
+Comment: "rsa-key-20230531"
+AAAAB3NzaC1yc2EAAAADAQABAAABAQDEZW0iQpBPPRGM7QCdc3GqefkqUqq5i727
+MEYvf7gy0h7Zvz72cvnDtnVF6MknCGVZ1S3uY+HeKUtujE3+z9tJ5zIxO+TxthJe
+8gbH9MUkElKX8qJN/nF1H4dLiW+77ARsT70rDoHayWccGGnwC/Gs9xXVF+Ni0JD5
+TKcGoye3Ps2Ni/KJAzkn78JyfrAb+Ail4DG4snCf1q/jXTuB39qYTTg94FJEqwe1
+dhPIzRiOjFT3eSmHAXdSuAXFjpXAq+bV8R2uxKsgNfvxyV4vlKwFhYTq+kHwDWX7
+RtLQUUs8mSTfbnL2LuE+T2d8sBtj9wi+gODFl5dDAVZkmuKrRgLF
+---- END SSH2 PUBLIC KEY ----

requirements.txt

@@ -0,0 +1,8 @@
+google.cloud.pubsub
+plotly
+pandas
+numpy
+h5py
+selenium
+sqlalchemy
+flask

templates/test3.html

@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Class Dump</title>
+</head>
+<body>
+    <h1>Class Dump</h1>
+    <ul>
+        {% for class in classes %}
+        <li>{{ class.__name__ }}</li>
+        {% endfor %}
+    </ul>
+
+    <h1>User Dump</h1>
+    <ul>
+        {% for user in users %}
+        <li>{{ user }}</li>
+        {% endfor %}
+    </ul>
+</body>
+</html>

test1.py

@@ -0,0 +1,31 @@
+class Count:
+    def __init__(self,mymin,mymax):
+        self.mymin=mymin
+        self.mymax=mymax    
+
+    def __getattr__(self, item):
+        self.__dict__[item]=0
+        return 0
+
+obj1 = Count(1,10)
+print(obj1.mymin)
+print(obj1.mymax)
+print(obj1.mycurrent1)
+
+class Count:
+
+    def __init__(self,mymin,mymax):
+        self.mymin=mymin
+        self.mymax=mymax
+        self.current=None
+   
+    def __getattribute__(self, item):
+        if item.startswith('cur'):
+            raise AttributeError
+        return object.__getattribute__(self,item) 
+        # or you can use ---return super().__getattribute__(item)
+
+obj1 = Count(1,10)
+print(obj1.mymin)
+print(obj1.mymax)
+print(obj1.current)

test10.py

@@ -0,0 +1,14 @@
+import sys
+
+# Iterate through the modules in sys.modules
+for module_name, module in sys.modules.items():
+    # Check if the module has a loader attribute
+    if hasattr(module, '__loader__'):
+        loader = module.__loader__
+        # Determine the type of loader
+        if type(loader).__name__ == 'SourceFileLoader':
+            print(f"Module '{module_name}' has a source loader.")
+        elif type(loader).__name__ == 'BuiltinImporter':
+            print(f"Module '{module_name}' has a built-in loader.")
+        elif type(loader).__name__ == 'ExtensionFileLoader':
+            print(f"Module '{module_name}' has an extension loader.")

test11.py

@@ -0,0 +1,12 @@
+from cx_Freeze import setup, Executable
+
+executables = [Executable('module-inspector.py')]
+
+setup(
+    name='wow',
+    version='1.0',
+    description='Module-inspector Application',
+    executables=executables
+)
+
+

test2.py

@@ -0,0 +1,58 @@
+import time
+'''import matplotlib.pyplot as plt'''
+
+# Durations to measure in seconds
+durations = [0.01, 0.1, 1.0]
+
+# Number of repetitions
+num_repetitions = 100
+
+# Lists to store jitter for each duration
+jitter_10ms = []
+jitter_100ms = []
+jitter_1s = []
+
+# Function to measure jitter
+def measure_jitter(duration):
+    start_time = time.perf_counter_ns()
+    time.sleep(duration)
+    end_time = time.perf_counter_ns()
+    return end_time - start_time
+
+# Measure jitter for each duration and repeat for num_repetitions times
+for duration in durations:
+    for _ in range(num_repetitions):
+        jitter = measure_jitter(duration)
+        if duration == 0.01:
+            jitter_10ms.append(jitter)
+        elif duration == 0.1:
+            jitter_100ms.append(jitter)
+        elif duration == 1.0:
+            jitter_1s.append(jitter)
+'''
+# Create histograms for each duration's jitter
+plt.hist(jitter_10ms, bins=10, color='r', alpha=0.5, label='10ms')
+plt.hist(jitter_100ms, bins=10, color='g', alpha=0.5, label='100ms')
+plt.hist(jitter_1s, bins=10, color='b', alpha=0.5, label='1s')
+plt.xlabel('Jitter (ns)')
+plt.ylabel('Frequency')
+plt.title('Jitter of time.sleep() for Different Durations')
+plt.legend()
+plt.show()
+'''
+
+import random
+import secrets
+
+# Create a dictionary with 1 million integer keys
+dict = {}
+
+for i in range(1000000):
+    # Generate a random 64-byte value as bytes
+    random_value = secrets.token_bytes(64)
+    # Convert the bytes value to hexadecimal representation
+    hex_value = random_value.hex()
+    # Add the key-value pair to the dictionary
+    dict[i] = hex_value
+
+print("Dictionary created with 1 million integer keys.")

test3.py

@@ -0,0 +1,51 @@
+
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+        self._known_good_copy = self.__dict__.copy()
+
+    def __getattribute__(self, name):
+        if name == "treasure":
+            if self.__dict__ != self._known_good_copy:
+                print("Attribute deleted!")
+            return self._treasure
+        else:
+            return object.__getattribute__(self, name)
+
+    def __setattr__(self, name, value):
+        if name != "_treasure" and name != "_known_good_copy":
+            print("Attribute deletion detected!")
+            self._known_good_copy = self.__dict__.copy()
+        object.__setattr__(self, name, value)
+v = Vault()
+Vault.treasure = " haha monkeypatched"
+print(v.treasure)
+
+'''
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+        self._known_good_copy = self.__dict__.copy()
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+        else:
+            if instance.__dict__ != self._known_good_copy:
+                print("Attribute deleted!")
+            return self._treasure
+
+    def __set__(self, instance, value):
+        print("Attribute deletion detected!")
+        self._known_good_copy = instance.__dict__.copy()
+
+    def __delete__(self, instance):
+        print("Attribute deletion detected!")
+        self._known_good_copy = instance.__dict__.copy()
+
+    def __set_name__(self, owner, name):
+        self.name = name
+v = Vault()
+Vault._treasure = " haha monkeypatched"
+print(v._treasure)
+'''

test4.py

@@ -0,0 +1,17 @@
+import asyncio
+
+# Define a coroutine function
+async def greet(name):
+    print(f"Hello, {name}!")
+    await asyncio.sleep(1)
+    print(f"Goodbye, {name}!")
+
+# Call the coroutine function
+coroutine_obj = greet("Karen")
+
+# coroutine_obj is a coroutine object, not the actual execution of the coroutine
+
+# Use an event loop to run the coroutine
+loop = asyncio.get_event_loop()
+loop.run_until_complete(coroutine_obj)
+loop.close()

test5.py

@@ -0,0 +1,8 @@
+def think():
+    x=1
+    def think2():
+        nonlocal x
+        x=2
+    think2()
+    return x
+print(think())

test6.py

@@ -0,0 +1,33 @@
+'''
+import sys
+import importlib.util
+
+# Load only the sys module
+importlib.import_module('sys')
+
+# Check the type of the value associated with the 'os' key
+os_module = sys.modules['os']
+os_module_type = type(os_module)
+
+# Print the type of the value associated with the 'os' key
+print("Type of the value associated with the 'os' key in sys.modules: {}".format(os_module_type))
+'''
+'''
+import os
+print(type(os))
+'''
+
+import builtins
+
+# Access the os module from globals()
+os = globals().get('os', None)
+
+# Check if os module is available
+if os is not None:
+    # Call os.getlogin()
+    username = os.getlogin()
+    print("Username:", username)
+else:
+    print("os module not available")
+
+

test7.py

@@ -0,0 +1,54 @@
+import os
+import sys
+import importlib.util
+import types
+
+
+def get_bytecode_size(module_name):
+    """Get the size of the cached .pyc bytecode file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith('.py'):
+        pyc_file = importlib.util.cache_from_source(spec.origin)
+        if os.path.exists(pyc_file):
+            return os.path.getsize(pyc_file)
+    return 0
+
+
+def get_dso_size(module_name):
+    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith(('.so', '.dll')):
+        return os.path.getsize(spec.origin)
+    return 0
+
+
+def generate_module_report():
+    """Generate a report summarizing the module environment."""
+    total_modules = len(sys.modules)
+    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
+    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
+    python_size = sum(
+        sys.modules[module_name].__sizeof__()
+        for module_name in sys.modules
+        if isinstance(sys.modules[module_name], types.ModuleType)
+    )
+    total_size = bytecode_size + dso_size + python_size
+
+    # Calculate percentage of each type of module
+    bytecode_percent = bytecode_size / total_size * 100
+    dso_percent = dso_size / total_size * 100
+    python_percent = python_size / total_size * 100
+
+    # Generate the report
+    report = f"Total modules: {total_modules}\n"
+    report += f"Total size: {total_size} bytes\n"
+    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
+    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
+    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
+
+    return report
+
+if __name__ == '__main__' and __package__ is None:
+    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
+    report = generate_module_report()
+    print(report)

test8.py

@@ -0,0 +1,259 @@
+'''
+import os
+import sys
+import importlib.util
+import types
+import inspect
+from collections import defaultdict
+
+
+def get_bytecode_size(module_name):
+    """Get the size of the cached .pyc bytecode file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith('.py'):
+        pyc_file = importlib.util.cache_from_source(spec.origin)
+        if os.path.exists(pyc_file):
+            return os.path.getsize(pyc_file)
+    return 0
+
+
+def get_dso_size(module_name):
+    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith(('.so', '.dll')):
+        return os.path.getsize(spec.origin)
+    return 0
+
+
+def generate_module_report():
+    """Generate a report summarizing the module environment."""
+    total_modules = len(sys.modules)
+    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
+    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
+    python_size = sum(
+        sys.modules[module_name].__sizeof__()
+        for module_name in sys.modules
+        if isinstance(sys.modules[module_name], types.ModuleType)
+    )
+    total_size = bytecode_size + dso_size + python_size
+
+    # Calculate percentage of each type of module
+    bytecode_percent = bytecode_size / total_size * 100
+    dso_percent = dso_size / total_size * 100
+    python_percent = python_size / total_size * 100
+
+    # Generate the report
+    report = f"Total modules: {total_modules}\n"
+    report += f"Total size: {total_size} bytes\n"
+    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
+    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
+    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
+
+    # Generate histogram of module directory names
+    dir_histogram = defaultdict(int)
+    for module_name in sys.modules:
+        module_file = sys.modules[module_name].__file__
+        if module_file:
+            module_dir = os.path.dirname(module_file)
+            dir_histogram[module_dir] += 1
+
+    report += "\nModule Directory Histogram:\n"
+    for module_dir, count in dir_histogram.items():
+        report += f"{module_dir}: {'*' * count}\n"
+
+    return report
+
+
+if __name__ == '__main__' and __package__ is None:
+    script_path = inspect.getframeinfo(inspect.currentframe()).filename
+    __spec__ = importlib.util.spec_from_loader('__main__', None, origin=script_path)
+    report = generate_module_report()
+    print(report)
+'''
+'''
+import os
+import sys
+from importlib.util import find_spec
+
+def get_module_size(module):
+    spec = find_spec(module)
+    if spec is None:
+        return None
+    if spec.cached:
+        size = os.stat(spec.cached).st_size
+    else:
+        size = os.stat(spec.origin).st_size
+    return size
+
+def summarize_modules():
+    total_size = 0
+    byte_count = 0
+    dso_count = 0
+    py_count = 0
+    directory_histogram = {}
+    for module_name, module in sys.modules.items():
+        if module is None:
+            continue
+        size = get_module_size(module_name)
+        if size is None:
+            continue
+        total_size += size
+        if module_name.endswith('_bootstrap') or module_name.endswith('_bootstrap_external'):
+            continue
+        if module_name.startswith('_frozen_importlib'):
+            dso_count += 1
+            continue
+        if module_name.startswith('_'):
+            continue
+        if module.__file__.endswith('.pyc'):
+            byte_count += size
+        elif module.__file__.endswith(('.so', '.dll')):
+            dso_count += 1
+        else:
+            py_count += size
+        module_directory = os.path.dirname(os.path.abspath(module.__file__))
+        directory_histogram[module_directory] = directory_histogram.get(module_directory, 0) + 1
+
+    print(f"Total module size: {total_size} bytes")
+    print(f"Bytecode modules: {(byte_count/total_size)*100:.2f}% ({byte_count} bytes)")
+    print(f"DSO modules: {(dso_count/total_size)*100:.2f}% ({dso_count} modules)")
+    print(f"Python modules: {(py_count/total_size)*100:.2f}% ({py_count} bytes)")
+
+    print("Module directory histogram:")
+    for directory, count in directory_histogram.items():
+        print(f"{directory}: {'*' * count}")
+
+if __name__ == '__main__':
+    summarize_modules()
+'''
+'''
+import os
+import sys
+from importlib.util import find_spec
+
+
+def get_module_size(module):
+    spec = find_spec(module)
+    if spec is None:
+        return None
+    if spec.cached:
+        size = os.stat(spec.cached).st_size
+    else:
+        size = os.stat(spec.origin).st_size
+    return size
+
+
+def summarize_modules():
+    total_size = 0
+    byte_count = 0
+    dso_count = 0
+    py_count = 0
+    directory_histogram = {}
+    for module_name, module in sys.modules.items():
+        if module is None:
+            continue
+        size = get_module_size(module_name)
+        if size is None:
+            continue
+        total_size += size
+        if module_name.endswith('_bootstrap') or module_name.endswith('_bootstrap_external'):
+            continue
+        if module_name.startswith('_frozen_importlib'):
+            dso_count += 1
+            continue
+        if module_name.startswith('_'):
+            continue
+        if module.__file__.endswith('.pyc'):
+            byte_count += size
+        elif module.__file__.endswith(('.so', '.dll')):
+            dso_count += 1
+        else:
+            py_count += size
+        module_directory = os.path.dirname(os.path.abspath(module.__file__))
+        directory_histogram[module_directory] = directory_histogram.get(module_directory, 0) + 1
+
+    print(f"Total module size: {total_size} bytes")
+    print(f"Bytecode modules: {(byte_count/total_size)*100:.2f}% ({byte_count} bytes)")
+    print(f"DSO modules: {(dso_count/total_size)*100:.2f}% ({dso_count} modules)")
+    print(f"Python modules: {(py_count/total_size)*100:.2f}% ({py_count} bytes)")
+
+    print("Module directory histogram:")
+    for directory, count in directory_histogram.items():
+        print(f"{directory}: {'*' * count}")
+
+
+if __name__ == '__main__':
+    summarize_modules()
+'''
+
+import os
+import sys
+import importlib.util
+import types
+from collections import defaultdict
+
+
+def get_bytecode_size(module_name):
+    """Get the size of the cached .pyc bytecode file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith('.py'):
+        pyc_file = importlib.util.cache_from_source(spec.origin)
+        if os.path.exists(pyc_file):
+            return os.path.getsize(pyc_file)
+    return 0
+
+
+def get_dso_size(module_name):
+    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith(('.so', '.dll')):
+        return os.path.getsize(spec.origin)
+    return 0
+
+
+def generate_module_report():
+    """Generate a report summarizing the module environment."""
+    total_modules = len(sys.modules)
+    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
+    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
+    python_size = sum(
+        sys.modules[module_name].__sizeof__()
+        for module_name in sys.modules
+        if isinstance(sys.modules[module_name], types.ModuleType)
+    )
+    total_size = bytecode_size + dso_size + python_size
+
+    # Calculate percentage of each type of module
+    bytecode_percent = bytecode_size / total_size * 100
+    dso_percent = dso_size / total_size * 100
+    python_percent = python_size / total_size * 100
+
+    # Generate the report
+    report = f"Total modules: {total_modules}\n"
+    report += f"Total size: {total_size} bytes\n"
+    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
+    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
+    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
+
+    # Generate histogram of module directory names
+    dir_histogram = defaultdict(int)
+    for module_name in sys.modules:
+        module_file = sys.modules[module_name].__file__
+        if module_file:
+            module_dir = os.path.dirname(module_file)
+            dir_histogram[module_dir] += 1
+
+    report += "\nModule Directory Histogram:\n"
+    for module_dir, count in dir_histogram.items():
+        report += f"{module_dir}: {'*' * count}\n"
+
+    return report
+
+
+if __name__ == "__main__":
+    report = generate_module_report()
+    print(report)
+
+
+
+

test9.py

@@ -0,0 +1,77 @@
+import os
+import sys
+import importlib.util
+import types
+from collections import defaultdict
+from sys import getsizeof
+
+
+def get_bytecode_size(module_name):
+    """Get the size of the cached .pyc bytecode file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin and spec.origin.endswith('.py'):
+        pyc_file = importlib.util.cache_from_source(spec.origin)
+        if os.path.exists(pyc_file):
+            return os.path.getsize(pyc_file)
+    return 0
+
+
+def get_dso_size(module_name):
+    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin and spec.origin.endswith(('.so', '.dll')):
+        return os.path.getsize(spec.origin)
+    return 0
+
+
+def generate_module_report():
+    """Generate a report summarizing the module environment."""
+    total_modules = len(sys.modules)
+    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
+    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
+    python_size = sum(
+        getsizeof(sys.modules[module_name]) - getsizeof(module_name)
+        for module_name in sys.modules
+        if isinstance(sys.modules[module_name], types.ModuleType)
+    )
+    total_size = bytecode_size + dso_size + python_size
+
+    # Calculate percentage of each type of module
+    bytecode_percent = bytecode_size / total_size * 100
+    dso_percent = dso_size / total_size * 100
+    python_percent = python_size / total_size * 100
+
+    # Generate the report
+    report = f"Total modules: {total_modules}\n"
+    report += f"Total size: {total_size} bytes\n"
+    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
+    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
+    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
+
+    # Generate histogram of module directory names
+    dir_histogram = defaultdict(int)
+    for module_name in sys.modules:
+        module_file = getattr(sys.modules[module_name], '__file__', None)
+        if module_file:
+            module_dir = os.path.dirname(module_file)
+            dir_histogram[module_dir] += 1
+
+    report += "\nModule Directory Histogram:\n"
+    for module_dir, count in dir_histogram.items():
+        report += f"{module_dir}: {'*' * count}\n"
+
+    return report
+
+
+if __name__ == "__main__":
+    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
+report = generate_module_report()
+print(report)
+
+
+
+
+
+
+    report = generate_module_report()
+    print(report)

test_fp2_14.py

@@ -0,0 +1,63 @@
+from fp2_14 import get_product_prices
+import h5py
+import pandas as pd
+import datetime
+import sys
+from datetime import datetime
+import ntplib
+import arrow
+
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-NFL-zipcodes"
+url = "https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f"
+
+df_new = get_product_prices(project_id, subscription_name, url)
+
+'''
+# When using sys to call the 'get_product_prices' function
+
+if __name__ == '__main__':
+    # Check if the required command-line arguments are provided
+    if len(sys.argv) < 4:
+        print("Usage: python test_fp2_14.py project_id subscription_name url")
+        sys.exit(1)
+
+    project_id = sys.argv[1]
+    subscription_name = sys.argv[2]
+    url = sys.argv[3:]
+
+    get_product_prices(project_id, subscription_name, url)
+'''
+
+
+# Now we have the new dataframe "df" recording the new price listed in weee's page
+# Load such data in the existing HDF5 file
+file_path = 'product1.h5'
+with h5py.File(file_path, 'a') as file:
+    # Load the existing dataframe 'df' from the HDF5 file
+    # REPLACE KEY TO GET THE EXISTING DATAFRAME FOR EACH DIFFERENT PRODUCT
+    df = pd.read_hdf(file, key='df')
+
+    # Add timestamp to the new dataframe "df_new"
+    # timestamp = datetime.now()
+    '''
+    To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
+    provides time synchronization with a reliable time server.
+    Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
+    sensible and human-friendly approach to format and convert the timestamps.
+    '''
+    # Fetch the time from an NTP server
+    ntp_client = ntplib.NTPClient()
+    response = ntp_client.request('pool.ntp.org')
+    ntp_time = datetime.fromtimestamp(response.tx_time)
+    timestamp = arrow.get(ntp_time).to('local')
+    df_new['Timestamp'] = timestamp
+
+    # Append 'df_new' to the existing df stored in the key
+    df_update = df.append(df_new, ignore_index=True)
+
+    # Save 'df2' to the HDF5 file with the key 'df'
+    # REPLACE KEY FOR DIFFERENT PRODUCT
+    df_update.to_hdf(file, key='df', mode='a')
+
+print("Data appended and saved successfully.")

weee's products.txt

@@ -0,0 +1,45 @@
+# Selected
+# Want-Want Rice Crackers
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+
+# Lay's Potato Chips
+url = 'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261'
+
+# Pocky Cookies
+url = 'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467'
+
+# Thai Jasmine Rice
+url = 'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01'
+
+# Japanese Medium Grain Rice
+url = 'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224'
+
+# Millet
+url = 'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02'
+
+# Nature's Soy Firm Tofu: https://www.sayweee.com/zh/product/Nature-s-Soy-Firm-Tofu-/19231?trace_id=ee63a29e-a1e4-4065-b213-96aa14a15521
+
+# Free 2% Reduced Fat Milk: https://www.sayweee.com/zh/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353?trace_id=3030fe4c-04fc-4f8f-9257-7f6395b9e513
+
+# Washington Fuji Apples: https://www.sayweee.com/zh/product/Washington-Fuji-Apples/94817?trace_id=dc4955fc-bba8-48cf-b1f9-9db3cf077eed
+
+# Greenlandic Sweet Prawns: https://www.sayweee.com/zh/product/Royal-Greenland-Cooked-Wild-Caught-Greenlandic-Sweet-Prawns-Frozen-1/44656?category=seafood01
+
+
+
+# Optional
+# LuoSiFen 
+url = 'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01'
+
+# Rice Noodles
+url = 'https://www.sayweee.com/en/product/Sanyangyishi-Fresh-Instant-Rice-Noodles/2156?category=instant01'
+
+# Cup Noodles
+url = 'https://www.sayweee.com/en/product/Nissin-Beef-Flavor-Cup-Noodles/102519?category=instant01'
+
+# Pork Sung
+url = 'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?category=canned04'
+
+# Nature's Soy Firm Tofu: https://www.sayweee.com/zh/product/Nature-s-Soy-Firm-Tofu-/19231?trace_id=ee63a29e-a1e4-4065-b213-96aa14a15521
+# [Not available or sold out in some area] Beef Ribeye Thin Slices: https://www.sayweee.com/zh/product/Beef-Ribeye-Thin-Slices--Frozen-1/4295?category=meat02
+# Shiitake Mushrooms: https://www.sayweee.com/zh/product/Shiitake-Mushrooms/45782?trace_id=ba388941-f631-4eaa-8570-5b7d4aba64e7

zipcodes.csv

@@ -0,0 +1,33 @@
+ZIP Code
+85305
+30313
+21230
+14127
+28202
+60605
+45202
+44114
+76011
+80204
+48226
+54304
+77054
+46225
+32202
+64129
+89118
+90301
+90301
+33056
+55415
+02035
+70112
+07073
+07073
+19148
+15212
+95054
+98134
+33607
+37213
+20785