Merged fp files

.gitignore

@@ -1,5 +0,0 @@
-.DS_Store
-geckodriver.log
-.gitignore
-*.pyc
-*/__pycache__

.vscode/launch.json

@@ -7,13 +7,6 @@
             "program": "simulation/SimulationRunner.py",
             "justMyCode": true,
             "args": "./HotdogSimulation.py"
-        },
-        {
-            "name": "Python: File",
-            "type": "python",
-            "request": "launch",
-            "program": "${file}",
-            "justMyCode": true
         }
     ]
 }

Leon_Submissions/Homework1.py

@@ -1,28 +0,0 @@
-# Question 1: 
-# The international standard for floating-point representation is 
-# called the "IEEE 754 Standard for Floating-Point Arithmetic."
-
-# Question 2: 
-# 32-bit and 64-bit formats are the most widely used ones in practice.
-
-# Question 3: 
-# My computer supports all these formats
-import numpy as np
-
-supported_formats = np.finfo(np.float32).bits, np.finfo(np.float64).bits
-print(f"Your computer supports the following floating-point formats: {supported_formats}")
-
-
-# Question 4:
-# Using the Pandas multiplication operator (df['Number1'] * df['Number2'])
-# , the result is automatically converted to a floating-point data type. 
-# On the other hand, when I multiply the numbers using the normal Python
-# multiplication operator (num1 * num2), the result remains an integer 
-# data type.
-
-# Question 5:
-# The size of a floating-point number in memory is primarily determined 
-# by the data type and not the actual value it represents. So the size 
-# of a Python floating-point number does not change as it gets larger.
-import sys
-print(sys.getsizeof(70))

Leon_Submissions/Homework2.py

@@ -1,30 +0,0 @@
-# Question 7:
-# 1. Strings in Python are sequences of Unicode characters, while bytes 
-# are sequences of raw binary data.
-# 2. Strings in Python are immutable, bytes objects are mutable 
-# 3. Strings in Python are encoded using a specific character encoding, 
-# bytes objects do not have an inherent encoding and represent raw binary 
-# data.
-import sys
-
-string_example = "Hello"
-bytes_example = b"Hello"
-
-string_size = sys.getsizeof(string_example)
-bytes_size = sys.getsizeof(bytes_example)
-
-print("Size of string:", string_size)
-print("Size of bytes:", bytes_size)
-
-# Emojis are represented using Unicode characters, which require more 
-# bytes to store compared to ASCII characters. Therefore, if you have 
-# a string that contains emojis, its size will be larger than a string 
-# without emojis. 
-
-def print_string_as_numbers(input_string):
-    for char in input_string:
-        print(ord(char), end=' ')
-
-# Example usage
-input_string = "Hello, World!"
-print_string_as_numbers(input_string)

Leon_Submissions/Homework3.py

@@ -1,36 +0,0 @@
-# Question 8: It assumes that the CSV file being read 
-# (tomato_sales_error.csv) has a column named "Tomato_Sales." 
-# If the column doesn't exist in the CSV file or if there is an issue 
-# with the file itself, an error will be raised.
-
-# Question 9: 
-import pandas as pd
-
-def read_tomato_sales(file_name):
-    data = pd.read_csv(file_name)
-
-    # Ensure required columns exist
-    required_columns = ["Store", "Region", "Tomato_Sales"]
-    missing_columns = set(required_columns) - set(data.columns)
-    if missing_columns:
-        raise ValueError(f"Invalid CSV format: Missing columns: {', '.join(missing_columns)}")
-
-    # Validate Tomato_Sales column
-    tomato_sales = data["Tomato_Sales"]
-    if not tomato_sales.dtype == int and not tomato_sales.dtype == float:
-        raise ValueError("Invalid CSV format: 'Tomato_Sales' column must contain numerical values.")
-
-    # Calculate total tomato sales
-    total_sales = tomato_sales.sum()
-    print(f"Total tomato sales: {total_sales}")
-
-# Question 10: Bytes represent raw binary data, while strings represent 
-# textual data. Using bytes allows us to handle various character 
-# encodings. By working with bytes, we can ensure that the data remains in 
-# its raw encoded form, allowing for accurate handling of different 
-# encodings.
-
-# In the interactive Python REPL (Read-Eval-Print Loop), the "_" variable 
-# is a special variable that is automatically assigned the result of the 
-# last expression evaluated. The type of the "_" variable reflects the 
-# type of the last expression evaluated in the REPL.

Leon_Submissions/leon.py

@@ -1 +0,0 @@
-print('Hi')

Multiprocessing.py

@@ -0,0 +1,276 @@
+import multiprocessing
+import subprocess
+import concurrent.futures
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+import os
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import selenium.common.exceptions
+import re
+from fp2_14 import get_product_prices
+
+# Define the subset of urls
+url_list = [
+    'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
+    'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
+    'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
+    'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
+    'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
+    'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
+    'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
+    'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
+    'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
+    'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
+    'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
+]
+
+
+# Create a pool of worker processes
+# pool = multiprocessing.Pool(cpu_count)
+
+# Define the function to run the scraper
+def run_scraper(url):
+    # scraping logic
+    df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
+    print(f"Created a dataframe: {df}")
+    product_id = url.split("/")[-1][:5]
+
+    options = Options()
+    options.add_argument('--disable-gpu')
+    driver = webdriver.Chrome(options=options)
+
+    project_id = "fp-20230425"
+    subscription_name = "fp-subscription-NFL-zipcodes"
+    subscriber = pubsub_v1.SubscriberClient()
+    subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+    # product_id = url.split("/")[-1]
+    driver.delete_all_cookies
+    driver.get(url)
+
+    def get_element_text(class_name_prefix):  
+        xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+        xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+        xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+        xpath_str += "' '), "                     # Step 4: a trailing space...
+        xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+        try:
+            match = driver.find_element(By.XPATH, xpath_str)
+            return match.get_attribute('textContent')
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            pass
+   
+        xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+        xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+        xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+        xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+        try:
+            match = driver.find_element(By.XPATH, xpath_str)
+            return match.get_attribute('textContent')
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            pass
+        return ''
+
+    # Close the cookies setting window
+    try:  
+        cookie_button = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+        )
+        print(f"cookie_button = {cookie_button}")
+        cookie_button.click()
+    except:
+        pass
+
+    # Clear any modal dialog
+    try:    
+        modal_close = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+        )
+        print(f"modal_close = {modal_close}")
+        modal_close.click()
+    except:
+        pass
+
+    # Define callback function for Pub/Sub messages
+    def callback(message):
+        # Extract zip code from message data
+        zip_code = message.message.data.decode('utf-8')
+
+        print(f"Received zip code: {zip_code}")
+
+        try:
+            try:
+                # Clear any modal dialog
+                modal_close = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+            )
+                # print(f"modal_close = {modal_close}")
+                modal_close.click()
+            except:
+                pass
+
+
+            try:
+
+                # Clear any notice
+                notice_close = WebDriverWait(driver, 5).until(
+                EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+                # print(f"notice_close = {notice_close}")
+                notice_close.click()
+            except:
+                pass
+        
+            # Enter zip code in the input field and click the "Check" button
+            zip_button = WebDriverWait(driver, 5).until(
+                EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+            )
+            # print(f"{zip_button:}")
+            zip_button.click()
+
+            zip_input = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+            )
+            # print(f"{zip_input:}")
+            zip_input.click()
+            # print("clearing zip_input")
+            zip_input.send_keys("\b\b\b\b\b")
+            # print("sending zip code")
+            zip_input.send_keys(zip_code)
+
+            check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+            check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+            # print(f"{check_button:}")
+            check_button.click()
+            # time.sleep(2)
+
+            try:
+            # Clear any notice
+                notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+                # print(f"notice_close = {notice_close}")
+                notice_close.click()
+            except:
+                pass
+
+            try:
+                # Clear any modal dialog
+                modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+                # print(f"modal_close = {modal_close}")
+                modal_close.click()
+            except:
+                pass
+
+            # Record sales count, if any
+            sales_count_element = get_element_text('Header_count')
+            count_start_index = sales_count_element.index('sold ') + len('sold ')
+            count_end_index = sales_count_element.index('+')
+            sales_count = sales_count_element[count_start_index:count_end_index]
+            # Record any sold status notice
+            sold_status = get_element_text('UnavailableCartbtn_container')
+            if sold_status == 'Notify me':
+                sold_status = 'Sold Out'
+
+        # Extract the price of rice crackers for this zip code
+            # Check if there is a minimum order quantity
+            min_quantity: float = 0
+            total_price: float = 0
+            unit_price: float = 0
+            try:
+                minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML')
+                if minimum_quantity_str:
+                    regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
+                    match = re.search(regexp,minimum_quantity_str)
+                    min_quantity, total_price = match.groups()
+            except selenium.common.exceptions.NoSuchElementException as nsee:
+                print("No minimum purhcase string found")
+            except Exception as e:
+                print(f"Exception when finding minimum purchase string: {e}")
+
+            ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+            ld_json_text = ld_json_element.get_attribute('innerHTML')
+            ld_json_start_index = ld_json_text.index('{')
+            ld_json_end_index = ld_json_text.rindex('}') + 1
+            ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
+            ld_json = json.loads(ld_json_json)
+            ld_json_price = ld_json['offers']['price']
+            # Print the price
+            print(ld_json_price)
+
+            unit_price_element = driver.find_element(By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText')
+            unit_price = unit_price_element.replace("$", "")
+            print(f"unit_price = {unit_price}" )
+
+            subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+            # Append the zip code and price to the dataframe
+            #append_to_df(zip_code, price)
+            new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]})
+            return pd.concat([df, new_row], ignore_index=True)
+            #print(f"Recorded the zip code and price: {zip_code},{price}")
+            #print(f"Updated a dataframe: {df}")
+
+        except Exception as e:
+            print(f"Caught exception: {e}")
+            print(f"Price not found for zip code {zip_code}")
+            price = 'Price not found'
+            return df
+
+    # Start receiving messages (sync version)
+    while True: 
+        pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+        messages = []
+        try:
+            pull_response = subscriber.pull(request = pull_request)
+            messages = pull_response.received_messages
+        except DeadlineExceeded:
+            pass
+
+        if messages:
+            df = callback(messages[0])
+            print(f"Updated a dataframe: {df}")
+        else:
+            print(f"no messages, trying again at {time.asctime()}")
+            break
+        # subprocess.call(['python', 'fp2_13.py', url])
+    return df
+
+if __name__ == '__main__':
+    # Determine the number of available CPUs
+    # cpu_count = multiprocessing.cpu_count()
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        future_to_url = {executor.submit(run_scraper, url): url for url in url_list}
+        results = []
+        for future in concurrent.futures.as_completed(future_to_url):
+            url = future_to_url[future]
+            try:
+                result = future.result()
+                print(f"Scraped data for URL: {url}")
+                results.append(result)
+            except Exception as e:
+                print(f"Failed to scrape data for URL: {url}\nError: {e}")
+
+'''
+# Run the scraper in parallel for the subset of items
+results = pool.map(run_scraper, url_list)
+
+# Close the pool and wait for the processes to finish
+pool.close()
+pool.join()
+'''
+
+with pd.HDFStore('product_data.h5') as store:
+    for url, result in zip(url_list, results):
+        # Extract the product ID from the URL
+        product_id = url.split("/")[-1][:5]
+
+        # Save the dataframe to the HDF5 file with a unique key
+        key = f'df_{product_id}'
+        store[key] = result
+        # result.to_hdf('product_data.h5', key=key, mode='a')

NFL_Cities.csv

@@ -0,0 +1,33 @@
+ID,Team Name,Arena Name,Abbreviation,Conference,Division,Arena Location,ZIP Code
+1,Arizona Cardinals,State Farm Stadium,ARI,NFC,West,"Glendale, Arizona",85305
+2,Atlanta Falcons,Mercedes-Benz Stadium,ATL,NFC,South,"Atlanta, Georgia",30313
+3,Baltimore Ravens,M&T Bank Stadium,BAL,AFC,North,"Baltimore, Maryland",21230
+4,Buffalo Bills,Bills Stadium,BUF,AFC,East,"Orchard Park, New York",14127
+5,Carolina Panthers,Bank of America Stadium,CAR,NFC,South,"Charlotte, North Carolina",28202
+6,Chicago Bears,Soldier Field,CHI,NFC,North,"Chicago, Illinois",60605
+7,Cincinnati Bengals,Paul Brown Stadium,CIN,AFC,North,"Cincinnati, Ohio",45202
+8,Cleveland Browns,First Energy Stadium,CLE,AFC,North,"Cleveland, Ohio",44114
+9,Dallas Cowboys,At&T Stadium,DAL,NFC,East,"Arlington, Texas",76011
+10,Denver Broncos,Empower Field at Mile High,DEN,AFC,West,"Denver, Colorado",80204
+11,Detroit Lions,Ford Field,DET,NFC,North,"Detroit, Michigan",48226
+12,Green Bay Packers,Lambeau Field,GB,NFC,North,"Green Bay, Wisconsin",54304
+13,Houston Texans,NRG Stadium,HOU,AFC,South,"Houston, Texas",77054
+14,Indianapolis Colts,Lucas Oil Stadium,IND,AFC,South,"Indianapolis, Indiana",46225
+15,Jacksonville Jaguars,TIAA Bank Field,JAX,AFC,South,"Jacksonville, Florida",32202
+16,Kansas City Chiefs,Arrowhead Stadium,KC,AFC,West,"Kansas City, Missouri",64129
+17,Las Vegas Raiders,Allegiant Stadium,LV,AFC,West,"Paradise, Nevada",89118
+18,Los Angeles Chargers,SoFi Stadium,LAC,AFC,West,"Inglewood, California",90301
+19,Los Angeles Rams,SoFi Stadium,LAR,NFC,West,"Inglewood, California",90301
+20,Miami Dolphins,Hard Rock Stadium,MIA,AFC,East,"Miami Gardens, Florida",33056
+21,Minnesota Vikings,U.S. Bank Stadium,MIN,NFC,North,"Minneapolis, Minnesota",55415
+22,New England Patriots,Gillette Stadium,NE,AFC,East,"Foxborough, Massachusetts",02035
+23,New Orleans Saints,Mercedes-Benz Superdome,NO,NFC,South,"New Orleans, Louisiana",70112
+24,New York Giants,MetLife Stadium,NYG,NFC,East,"East Rutherford, New Jersey",07073
+25,New York Jets,MetLife Stadium,NYJ,AFC,East,"East Rutherford, New Jersey",07073
+26,Philadelphia Eagles,Lincoln Financial Field,PHI,NFC,East,"Philadelphia, Pennsylvania",19148
+27,Pittsburgh Steelers,Heinz Field,PIT,AFC,North,"Pittsburgh, Pennsylvania",15212
+28,San Francisco 49ers,Levi's Stadium,SF,NFC,West,"Santa Clara, California",95054
+29,Seattle Seahawks,CenturyLink Field,SEA,NFC,West,"Seattle, Washington",98134
+30,Tampa Bay Buccaneers,Raymond James Stadium,TB,NFC,South,"Tampa, Florida",33607
+31,Tennessee Titans,Nissan Stadium,TEN,AFC,South,"Nashville, Tennessee",37213
+32,Washington Redskins,FedExField,WAS,NFC,East,"Landover, Maryland",20785

README.md

@@ -17,4 +17,27 @@ the focus is on *working results*, not rote memorization.  Homework questions
 are simple enough that they can be reviewed interactively in 1:1 or up to 1:4
 sessions with study groups.
 
+This branch contains a prototype for a retail price scraper for getweee.com.
+To run it, install:
+
+* Python 3.9+,
+* chrome,
+* chromedriver,
+* and the python depedencies:
+
+```
+pip install -r /path/to/requirements.txt
+```
+
+Once those are installed, install the GCP SDK:
+
+https://cloud.google.com/sdk/docs/install
+
+You will also have to log into GCP *twice*, once for the gcloud CLI tools, 
+and again for the GCP SDK libraries:
+
+https://googleapis.dev/python/google-api-core/latest/auth.html
+
+
+
 

Scott_Submissions/A_BrainTeaser_byScott.py

@@ -1,100 +0,0 @@
-# A quick brain teaser
-print('What has a head and a tail but no body?')
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-#
-# A coin

Scott_Submissions/BoundedInt.py

@@ -1,46 +0,0 @@
-class BoundedInt():
-    def __init__(self, i, min, max):
-        """
-        Create a bounded integer.
-
-        Args:
-            i (int): The initial value of the integer.
-            min (int): The minimum value of the integer.
-            max (int): The maximum value of the integer.
-        """
-        self.i = i
-        self.min = min
-        self.max = max
-    
-    def __int__(self):
-        """ 
-        Convert the bounded integer to int.
-
-        Returns:
-            int: The value of the bounded integer.
-        """
-        return self.i
-
-    def __add__(self, other):
-        if isinstance(other, BoundedInt):
-            if self.i + other.i > self.max:
-                raise ValueError("Addition exceeds maximum value")
-            return BoundedInt(self.i + other.i, self.min, self.max)
-        elif isinstance(other, int):
-            if self.i + other > self.max:
-                raise ValueError("Addition exceeds maximum value")
-            return BoundedInt(self.i + other, self.min, self.max)
-        else:
-            raise TypeError("Unsupported operand type")
-
-    def __sub__(self, other):
-        if isinstance(other, BoundedInt):
-            if self.i - other.i < self.min:
-                raise ValueError("Subtraction exceeds minimum value")
-            return BoundedInt(self.i - other.i, self.min, self.max)
-        elif isinstance(other, int):
-            if self.i - other < self.min:
-                raise ValueError("Subtraction exceeds minimum value")
-            return BoundedInt(self.i - other, self.min, self.max)
-        else:
-            raise TypeError("Unsupported operand type")

Scott_Submissions/BoundedIntTester.py

@@ -1,67 +0,0 @@
-from BoundedInt import BoundedInt
-
-class BoundedIntTester:
-    def __init__(self):
-        pass
-
-    def test1(self):
-        bi1 = BoundedInt(0xdeadbeef, min = (0xdeadbeef-1), max = (0xdeadbeef+1))
-        if int(bi1) == 0xdeadbeef:
-            print(f"✅ Test 1 passed!")
-        else:
-            print(f"❌ Test 1 failed, int(bi1) = {int(bi1)}, expected 0xdeadbeef")
-
-    def test2(self):
-        """
-        Test that addition works for two BoundedInts, and throws an error if the bounds are exceeded.
-        """
-        bi2 = BoundedInt(1, min = 0, max = 1)
-        bi3 = BoundedInt(1, min = 0, max = 1)
-        try:
-            bi4 = bi2 + bi3 # 1 + 1 = 2, which is more than max = 1
-            print(f"❌ Test 2 failed!")
-        except ValueError as err:
-            print(f"✅ Test 2 passed!")
-
-    def test3(self):
-        """
-        Test that addition works for two BoundedInts, and throws an error if the bounds are exceeded.
-        """
-        bi4 = BoundedInt(1, min = 0, max = 1)
-        bi5 = BoundedInt(2, min = 0, max = -1)
-        try:
-            bi6 = bi4 - bi5 # 1 - 2 = (-1), which is less than min = 0
-            print(f"❌ Test 3 failed!")
-        except ValueError as err:
-            print(f"✅ Test 3 passed!")
-
-    def test4(self):
-        """
-        Test that addition works for a BoundedInt and a regular int, and throws an error if the bounds are exceeded.
-        """
-        bi7 = BoundedInt(1, min = 0, max = 1)
-        bi8 = 2
-        try:
-            bi9 = bi7 + bi8
-            print(f"❌ Test 4 failed!")
-        except ValueError as err:
-            print(f"✅ Test 4 passed!")
-
-    def test5(self):
-        """
-        Test that subtraction works for a BoundedInt and a regular int, and throws an error if the bounds are exceeded.
-        """
-        bi10 = BoundedInt(1, min = 0, max = 1)
-        bi11 = 2
-        try:
-            bi12 = bi10 - 11
-            print(f"❌ Test 5 failed!")
-        except ValueError as err:
-            print(f"✅ Test 5 passed!")
-
-tester = BoundedIntTester()
-tester.test1()
-tester.test2()
-tester.test3()
-tester.test4()
-tester.test5()

Scott_Submissions/Floating_Point_Formats.py

@@ -1,2 +0,0 @@
-import sys
-print(sys.float_info)

Scott_Submissions/HW1_Scott.rtf

@@ -1,37 +0,0 @@
-{\rtf1\ansi\ansicpg1252\cocoartf2580
-\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fnil\fcharset0 HelveticaNeue;\f1\fnil\fcharset0 Menlo-Regular;}
-{\colortbl;\red255\green255\blue255;\red0\green0\blue0;\red0\green0\blue0;}
-{\*\expandedcolortbl;;\cssrgb\c0\c0\c0\c87059;\cssrgb\c0\c0\c0\c3922;}
-\margl1440\margr1440\vieww11520\viewh8400\viewkind0
-\deftab720
-\pard\pardeftab720\partightenfactor0
-
-\f0\fs32 \cf2 \expnd0\expndtw0\kerning0
-1. What is the name of the international standard for floating-point representation?\
-	\
-	The IEEE-754 standard describes floating-point formats, a way to represent real numbers in hardware.\
-\
-\
-2. How many bits are the floating point numbers defined by that standard allowed to be?\
-\
-	The floating-point numbers defined by the international standard can be allowed to be of different lengths depending on the specific interchange format. For binary interchange, the formats can be 16 bits, 32 bits, 64 bits, and any multiple of 32 bits \uc0\u8805  128. For decimal interchange, the formats can be any multiple of 32 bits.\
-\
-\
-3. Does your computer support all those formats? Can you think of an easy way to check with only one line of Python?\
-\
-import sys\
-print(sys.float_info)\
-\
-scott@MacBook-Pro-6 python-bootcamp % python3 Floating_Point_Formats.py \
-sys.float_info(max=1.7976931348623157e+308, max_exp=1024, max_10_exp=308, min=2.2250738585072014e-308, min_exp=-1021, min_10_exp=-307, dig=15, mant_dig=53, epsilon=2.220446049250313e-16, radix=2, rounds=1)\
-\
-\
-\pard\pardeftab720\sa320\partightenfactor0
-\cf2 4. Why doesn't this code give the same answers to the two multiplications?\
-	Because they are different operations. The normal Python multiplication operator will multiply two numbers together and return the product. The Pandas multiplication operator, on the other hand, performs element-wise multiplication on the corresponding columns in a DataFrame. In other words, it multiplies the first element in the first column with the first element in the second column, the second element in the first column with the second element in the second column, and so on. \
-	Floating-point arithmetic has a limited precision due to the finite number of bits used to represent floating-point numbers in binary format. This can lead to rounding errors and loss of precision in arithmetic operations, particularly with very large or very small numbers.\
-\
-5. You can use 
-\f1\fs27\fsmilli13600 \cf2 \cb3 sys.getsizeof
-\f0\fs32 \cf2 \cb1  to see how large a Python object is. Does the size of a Python floating-point number change as it gets larger? Why or why not?\
-	The size of a Python floating-point number does not change as it gets larger. This is because Python uses a fixed number of bytes to represent floating-point numbers, regardless of their value.}

Scott_Submissions/HW2.py

@@ -1,54 +0,0 @@
-# Name three practical differences between Python strings and bytes. 
-# Using sys.getsizeof, see how the sizes may be different. 
-# Do emoji change this answer?
-
-# Encoding
-# Strings are created using Unicode character points. 
-# While bytes are used to represent actual bytes in memory, which can be encoded using ASCII or UTF-8.
-
-# Usage
-# Strings are usually used to represent text.
-# While bytes are used to represent binary data.
-
-# Memory Size 
-# Strings take up more memory than bytes, because they are Unicode-based and can contain any character.
-# While bytes are limited to binary data.
-
-
-print("---------------------------------------------")
-print("This is the Memory Size test print out:")
-
-
-import sys
-
-my_string = "Hello, world!"
-my_bytes = b'\x48\x65\x6c\x6c'
-
-print(sys.getsizeof(my_string)) # Prints 62
-print(sys.getsizeof(my_bytes)) # Prints 37
-
-print()
-print('Test the emoji:')
-
-x = 'pizza' # Prints 54
-y = b'pizza' # Prints 38
-z = '🍕' #Prints 80
-
-print(sys.getsizeof(x))
-print(sys.getsizeof(y))
-print(sys.getsizeof(z))
-
-
-print("---------------------------------------------")
-
-print("This is the String to Sequence of Numbers print out:")
-
-def String_to_SequenceofNumbers():
-    my_string = input("Enter a string: ")
-    num_sequence = [ord(c) for c in my_string]
-    print(num_sequence)
-
-if __name__ == "__main__":
-    String_to_SequenceofNumbers()
-
-print("---------------------------------------------")

Scott_Submissions/HW3.py

@@ -1,63 +0,0 @@
-# 1. Why does ord("😀") give a different number than 0xf09f9880, which is what the hexdump of the same character yields?
-# 
-#   The ord() function returns the Unicode code point of a character as an integer, which is 128512 for the smiley face.
-#   However, the hexdump command, returns the UTF-8 encoding of character.
-#   The hexadecimal representation of 128512 is 0x1F600.
-#   The UTF-8 encoding of the "😀" character is represented by the sequence of bytes F0 9F 98 80 in hexadecimal.
-#   UTF-8 is one of the encoding schemes used to represent Unicode characters as binary data.
-#
-#
-# 8. Hypothetical Situation: If the dataframe is not formatted or cleaned, while the column storing the tomoto sales data isn't named exactly same as 'Tomato_Sales'
-#       It will raise a KeyError instead.
-#
-#
-# 9. The solution of the hypothetical situation from my question 8:
-
-def read_tomato_sales(file_name):
-    data = pd.read_csv(file_name)
-    if not pd.api.types.is_numeric_dtype(data["Tomato_Sales"]):
-        raise TypeError("Tomato_Sales column should contain only numeric values")
-    total_sales = data.iloc[:, -1].sum()  # This is the key
-    print(f"Total tomato sales: {total_sales}")
-# read_tomato_sales("tomato_sales_error.csv")
-
-
-# 10. Different human languages can use various character encodings such as UTF-8, UTF-16, or ASCII. 
-#   While using bytes, you can work with raw data without worrying about specific encodings. 
-#   This allows us to avoid encoding-related errors and ensures compatibility across different language texts.
-
-def encode_decode_data():
-    print('-' * 20)
-    data = input("Enter a sentence using any language: ")
-    print('-' * 20)
-    print("The sentence you entered is: {}".format(data))
-    print('-' * 20)
-    encoded_data = data.encode('utf-8')
-    print("This sentence after encoded is: {}".format(encoded_data))
-    print('-' * 20)
-    byte_length = len(encoded_data)
-    print("The byte length of this sentence is: {}".format(byte_length))
-    print('-' * 20)
-    decoded_data = encoded_data.decode()
-    print("This sentence after decoded is: {}".format(decoded_data))
-    print('-' * 20)
-
-if __name__ == "__main__":
-    run_function = 'function1'
-
-    if run_function == 'function1':
-        encode_decode_data()
-    elif run_function == 'function2':
-        read_tomato_sales(file_name)
-
-
-# 2.5. REPL stands for Read-Eval-Print Loop. 
-#   A REPL is an interactive programming environment that allows you to read input, evaluate expressions or statements, print output, and repeat the process. 
-#   It provides a quick way to experiment with code, test small snippets, and get immediate feedback. 
-#   The Python REPL allows you to interactively enter Python code and see the results immediately without needing to write a full script and execute it separately.
-
-
-# 3. The '_' is a convention that holds the value of the last printed expression or the result of the last evaluated expression. 
-#   The actual type of the "_" variable depends on the value assigned to it.
-
-

Scott_Submissions/HW4.py

@@ -1,147 +0,0 @@
-# 1. Annotate the following python expressions with the specific types of each literal, as well as the overall return value:
-# i = 3
-# f = 3.0
-# c = 3+3j
-# b = b'abcdefg'
-# s = 'abcdefg'
-# s = "abcdefg"
-
-i = 3
-f = 3.0
-c = 3+3j
-b = b'abcdefg'
-s1 = 'abcdefg'
-s2 = "abcdefg"
-
-def check_the_datatype():
-    print()
-    print('This is the answer for Q1:')
-    print()
-    variable_names = ['i', 'f', 'c', 'b', 's1', 's2']
-    data_list = [i, f, c, b, s1, s2]
-    for variable, data in zip(variable_names, data_list):
-        print("The type of this assigned value {} = {} is: {}".format(variable, data, type(data).__name__))
-
-
-
-# 2. For each of the types string, bytestring, integer, floating point, and imaginary literals, 
-#   construct a 5x5 table showing what happens when the character "0" or number 0 is compared to each other type
-
-from tabulate import tabulate
-from termcolor import colored
-
-from tabulate import tabulate
-from termcolor import colored
-
-def tablesQ2():
-    print()
-    print('-' * 20)
-    print()
-    print('This is the answer for Q2:')
-    print()
-    types = ["String", "Byte String", "Integer", "Floating Point", "Imaginary"]
-    literals = ['"0"', 'b"0"', '0', '0.0', '0j']
-
-    table_data = []
-
-    for type_ in types:
-        
-        row = [type_]
-
-        for literal in literals:
-            
-            if type_ == "String":
-                result = literal == '"0"'
-            elif type_ == "Byte String":
-                result = literal.encode() == b"0"
-            elif type_ == "Integer":
-                try:
-                    result = int(literal) == 0
-                except ValueError:
-                    result = False
-            elif type_ == "Floating Point":
-                try:
-                    result = float(literal) == 0.0
-                except ValueError:
-                    result = False
-            elif type_ == "Imaginary":
-                result = literal.replace('j', 'j0') == "0j"
-            color = 'green' if result else 'red'
-            colored_result = colored(result, color)
-            row.append(colored_result)
-
-        table_data.append(row)
-
-    table = tabulate(table_data, headers=["Type"] + literals, tablefmt="pipe")
-    print(table)
-
-
-
-# 3. Now do the same thing for the + and == operators for each pair of types in two other 5x5 tables.
-
-def tablesQ3():
-    print()
-    print('-' * 20)
-    print()
-    print('This is the answer for Q3:')
-    print()
-
-    operators = ["+", "=="]
-    types = ["str", "bytes", "int", "float", "complex"]
-    literals = ['"0"', 'b"0"', '0', '0.0', '0+0j']
-
-    for operator in operators:
-        print(f"Table for {operator} Operator:")
-        table_data = []
-
-        for type_ in types:
-            row = [type_]
-
-            for literal in literals:
-                try:
-                    result = eval(f"{literal} {operator} {type_}({literal})")
-                    color = 'green'
-                except TypeError:
-                    result = False
-                    color = 'red'
-
-                colored_result = colored(result, color)
-                row.append(colored_result)
-
-            table_data.append(row)
-
-        table = tabulate(table_data, headers=["Type"] + literals, tablefmt="pipe")
-        print(table)
-        print()
-
-
-
-# Write a small program that uses the AST module to open the lexer/parser example, and dumps the resulting AST. 
-# See if there are any arguments to the parse method which make the output easier to read.
-
-# By default, ast.dump() prints the AST in a compact format. 
-# However, there are two optional arguments that can enhance the readability:
-# indent: Specifies the number of spaces to use for indentation. Setting it to a positive integer (e.g., 4) indents each level of the AST by that amount.
-# annotate_fields: When set to True, adds field names as comments to the output, making it easier to identify each part of the AST.
-
-import ast
-
-def dump_ast(filename):
-    with open(filename, 'r') as file:
-        source_code = file.read()
-
-    try:
-        tree = ast.parse(source_code)
-        ast_dump = ast.dump(tree, indent=4, annotate_fields=True)
-        print(ast_dump)
-    except SyntaxError as e:
-        print(f"Syntax error in {filename}: {e}")
-
-
-
-
-if __name__ == "__main__":
-    check_the_datatype()
-    tablesQ2()
-    tablesQ3()
-    print()

Scott_Submissions/HW5.py

@@ -1,91 +0,0 @@
-# 5. What does this do? print("\007") Can you figure out why?
-
-    # It is a ASCII Bell character.
-    # It generates a beep sound or a system alert depending on the platform and environment where the code is executed.
-    # It is often used to trigger attention or alert the user in certain situations or as a way to generate a simple sound signal in text-based interfaces.
-
-    # BUT NOTHING HAPPENED ON MAC.
-
-
-
-# 6. What does this do? print("abcdefg\rhijklmnop") Can you figure out why?
-
-    # '\r' is a carriage return character
-    # When the string is printed, the carriage return character '\r' causes the cursor to return to the beginning of the line. 
-    # So when "hijklmnop" is printed, the cursor moves back to the start of the line, and the subsequent text "abcdefg" overwrites the initial characters.
-
-    # The output is gonna be 'hijklmnop'.
-
-
-
-# 7. To shut down the HTTP server afterward, using Ctrl + C to interrupt the execution of the Python program.
-
-import http.server
-import socketserver
-from urllib.parse import urlparse, parse_qs
-
-class MyRequestHandler(http.server.SimpleHTTPRequestHandler):
-    def do_GET(self):
-        self.send_response(200)
-        self.send_header('Content-type', 'text/plain')
-        self.end_headers()
-        output_string = "abcdefg\rhijklmnop"
-        self.wfile.write(output_string.encode())
-
-PORT = 8000
-Handler = MyRequestHandler
-httpd = socketserver.TCPServer(("", PORT), Handler)
-print(f"Serving on port {PORT}")
-
-httpd.serve_forever()
-
-    # The HTTP server sends the response over the network to the client, which is typically a web browser.
-    # In this function, the server sends an HTTP response with a status code of 200 and a content type of "text/plain". 
-    # The response body is set to the string "abcdefg\rhijklmnop" encoded as bytes.
-    # When the client receives this response, it interprets the \r character as a normal character and displays the entire string "abcdefg\rhijklmnop" as it is.
-
-    # The '\r' character is a carriage return character, which is a control character used to return the cursor to the beginning of the current line.
-    # The behavior of the carriage return character '\r' can depend on the platform and how the output is handled.
-
-    # For example:
-    # In Safari and Chrome, the "http://localhost:8000" shows:
-        # abcdefg
-        # hijklmnop
-    # In Macbook Terminal, the "curl http://localhost:8000" shows:
-        # hijklmnop%    
-
-
-
-# 8. Think of a way someone might try to commit a crime or hide an embezzlement scheme using the answers to the above two questions.
-
-    # The key is by manipulating or altering the output displayed on a web page versus what is actually happening in the system.
-
-    # Case 1:
-        # Show the false financial transactions to earn the cut.
-    # Case 2:
-        # Delivey the heist planning details and hide any suspicious activity.
-    # Case 3:
-        # Overwrite the previously displayed information for the false testimony purpose.
-
-
-
-# 9. Think of how you would do input validation to prevent such a crime. Replace the code REPLACE ME here so the tests pass:
-
-def validate(input_str):
-    if '\r' in input_str or '\n' in input_str:
-        raise ValueError("Crime detected!")
-
-try:
-    validate("abcdefg")
-    print("✅ test1 pass!")
-except ValueError:
-    print("❌ test1 fail!")
-
-try:
-    validate("abcdefg\rhijklmnop")
-    print("❌ test2 fail!")
-except ValueError:
-    print("✅ test2 pass!")
-
-    # Test 1 pass means there is no ValueError
-    # Test 2 pass means the ValueError has been detected

Scott_Submissions/HW6.py

@@ -1,76 +0,0 @@
-########## This file only contains the question 2, 4, and 10. The rest is in the BoundedInt.py. ##########
-
-
-
-# 2. Modify the template to pass test 1, which checks that int() called on your class will return the value of i.
-
-# class BoundedInt():
-#     def __init__(self, i, min, max):
-#         """
-#         Create a bounded integer.
-
-#         Args:
-#             i (int): The initial value of the integer.
-#             min (int): The minimum value of the integer.
-#             max (int): The maximum value of the integer.
-#         """
-#         self.i = i
-#         self.min = min
-#         self.max = max
-    
-#     def __int__(self):
-#         """
-#         Convert the bounded integer to int.
-
-#         Returns:
-#             int: The value of the bounded integer.
-#         """
-#         return self.i
-
-
-
-# 4. In that reading material, there's a sentence that starts with "For instance, to evaluate the expression x + y, "... 
-# What do you think the types of x and y are? Guess, without checking first. 
-# What are they, actually? You can use the debugger's local window to find out.
-
-    # x might be an instance of a class that has defined the __add__() method
-    # And y is the object that is compatible with the x instance for the addition operation.
-
-class MyClass:
-    def __init__(self, value):
-        self.value = value
-
-    def __add__(self, other):
-        if isinstance(other, MyClass):
-            return MyClass(self.value + other.value)
-        elif isinstance(other, int):
-            return MyClass(self.value + other)
-        else:
-            raise TypeError("Unsupported operand type.")
-
-# Example usage
-obj1 = MyClass(5)
-obj2 = MyClass(10)
-result1 = obj1 + obj2  # Calls obj1.__add__(obj2)
-print(result1.value)  # Output: 15
-
-result2 = obj1 + 3  # Calls obj1.__add__(3)
-print(result2.value)  # Output: 8
-
-
-
-# 10. Think about how you would change the template so it could allow adding a BoundedInt to an emoji, 
-#       if there was a hypothetical function sentiment(emoji) which returned a floating point number from -10.0 to 10.0 
-#       which indicated whether the emoji was happy or sad. e.g. BoundedInt(1, min = 1, max = 10) + 😁 would raise ValueError.
-
-    # By adding the conceptional function 'sentiment(emoji)' in the '__add__()' method
-    # The sentiment will analyze the happy level of the input emoji and return a floating number from a number range
-        # For example:
-            # 10 for a 😊 smiley face
-            # -10 for a frowny face
-            # 0 for a 🍕
-    # The protential code can be like this:
-'''
-        elif isinstance(other, str) and sentiment(other) < 0:
-            aise ValueError("Cannot add a BoundedInt to a sad emoji")   
-        '''

Scott_Submissions/HW7.py

@@ -1,77 +0,0 @@
-########## Some commands in pdb ##########
-# help: Display a list of available pdb commands or get help on a specific command.
-# break or b: Set a breakpoint at a specific line or function.
-# continue or c: Continue execution until the next breakpoint or the program finishes.
-# step or s: Step into the next line of code, diving into function calls.
-# next or n: Step over the next line of code, without diving into function calls.
-# return or r: Continue execution until the current function returns.
-# list or l: Show the current line of code and surrounding lines.
-# print or p: Evaluate and print the value of an expression or variable.
-# watch: Set a watchpoint on an expression or variable. The debugger will break when the value changes.
-# quit or q: Quit the debugger and stop execution.
-
-
-########## There is a main() function at the end ##########
-
-
-# 2. Provide an example script where setting a breakpoint on a particular line
-#       Then using either step over or step into will yield the same result.
-
-class Same_StepOverStepInto_Result:
-    def calculation2(self, a, b):
-        result = a + b
-        product = self.multiply(a, b)  # Step into this method call
-
-        breakpoint()  # Set a breakpoint here
-
-        print(f"The sum is: {result}")
-        print(f"The product is: {product}")
-
-    def multiply(self, x, y):
-        return x * y
-
-
-# calculator2 = Same_StepOverStepInto_Result()
-# calculator2.calculation2(6, 7)
-
-
-
-# 3. Here is another example which will provide different results from 'step' and 'next' commands
-
-class Diff_StepOverStepInto_Result:
-    def multiply(self, a, b):
-        return a * b
-
-    def calculation3(self, a, b):
-        result = a + b
-
-        breakpoint()  # Set a breakpoint here
-
-        product = self.multiply(a, b)
-
-        print(f"The sum is: {result}")
-        print(f"The product is: {product}")
-
-
-# calculator3 = Diff_StepOverStepInto_Result()
-# calculator3.calculation3(6, 7)
-
-
-
-# Choose wich Class to run
-def main():
-    choice = input("Enter 2 to run Q2 class for same result, 3 to run Q3 class for different result: ")
-
-    if choice == '2':
-        print('Here is the sample task for Question 2:')
-        calculator2 = Same_StepOverStepInto_Result()
-        calculator2.calculation2(6, 7)
-    elif choice == '3':
-        print('Here is the sample task for Question 3:')
-        calculator3 = Diff_StepOverStepInto_Result()
-        calculator3.calculation3(6, 7)
-    else:
-        print("Please only choose between 2 and 3")
-
-if __name__ == "__main__":
-    main()

Scott_Submissions/HW7_test.py

@@ -1,79 +0,0 @@
-class Vector_1:
-    def __init__(self, x, y):
-        self.x = x
-        self.y = y
-    
-    def __add__(self, other):
-        if isinstance(other, Vector_1):
-            # Add the corresponding coordinates of both vectors
-            new_x = self.x + other.x
-            new_y = self.y + other.y
-            return Vector_1(new_x, new_y)
-        else:
-            raise TypeError("Unsupported operand type: +")
-    
-    def __str__(self):
-        return f"Vector({self.x}, {self.y})"
-
-# # Create two Vector instances
-# vector1 = Vector_1(2, 3)
-# vector2 = Vector_1(4, 5)
-
-# # Add two Vector instances using the + operator
-# result1 = vector1 + vector2
-
-# # Print the result
-# print(result1)
-
-
-
-class Vector_2:
-    def __init__(self, x, y):
-        self.x = x
-        self.y = y
-    
-    def __add__(self, other):
-        if isinstance(other, Vector_2):
-            # Add the corresponding coordinates of both vectors
-            new_x = self.x + other.x
-            new_y = self.y + other.y
-            return Vector_2(new_x, new_y)
-        else:
-            raise TypeError("Unsupported operand type: +")
-    
-    # def __str__(self):
-    #     return f"Vector({self.x}, {self.y})"
-
-# # Create two Vector instances
-# vector3 = Vector_2(2, 3)
-# vector4 = Vector_2(4, 5)
-
-# # Add two Vector instances using the + operator
-# result2 = vector3 + vector4
-
-# # Print the result
-# print(result2)
-
-
-def main():
-    # Create two Vector instances
-    vector1 = Vector_1(2, 3)
-    vector2 = Vector_1(4, 5)
-    # Add two Vector instances using the + operator
-    result1 = vector1 + vector2
-    # Print the result
-    print("This is printout of the Class using __str__() method:")
-    print(result1)
-
-    # Create two Vector instances
-    vector3 = Vector_2(2, 3)
-    vector4 = Vector_2(4, 5)
-    # Add two Vector instances using the + operator
-    result2 = vector3 + vector4
-    # Print the result
-    print("This is printout of the Class without using __str__() method:")
-    print(result2) 
-
-
-if __name__ == "__main__":
-    main()

Scott_Submissions/PDB_inVSC.rtf

@@ -1,56 +0,0 @@
-{\rtf1\ansi\ansicpg1252\cocoartf2580
-\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;}
-{\colortbl;\red255\green255\blue255;}
-{\*\expandedcolortbl;;}
-\margl1440\margr1440\vieww11520\viewh8400\viewkind0
-\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0
-
-\f0\fs24 \cf0 scott@MacBook-Pro-6 Scott_Submissions % python3 HW7.py\
-Enter 2 to run Q2 class for same result, 3 to run Q3 class for different result: 1\
-Please only choose between 2 and 3\
-scott@MacBook-Pro-6 Scott_Submissions % python3 HW7.py\
-Enter 2 to run Q2 class for same result, 3 to run Q3 class for different result: 2\
-Here is the sample task for Question 2:\
-> /Users/scott/Desktop/GTSP/python-bootcamp/Scott_Submissions/HW7.py(27)calculation2()\
--> print(f"The sum is: \{result\}")\
-(Pdb) step\
-The sum is: 13\
-> /Users/scott/Desktop/GTSP/python-bootcamp/Scott_Submissions/HW7.py(28)calculation2()\
--> print(f"The product is: \{product\}")\
-(Pdb) c\
-The product is: 42\
-scott@MacBook-Pro-6 Scott_Submissions % python3 HW7.py\
-Enter 2 to run Q2 class for same result, 3 to run Q3 class for different result: 2\
-Here is the sample task for Question 2:\
-> /Users/scott/Desktop/GTSP/python-bootcamp/Scott_Submissions/HW7.py(27)calculation2()\
--> print(f"The sum is: \{result\}")\
-(Pdb) next\
-The sum is: 13\
-> /Users/scott/Desktop/GTSP/python-bootcamp/Scott_Submissions/HW7.py(28)calculation2()\
--> print(f"The product is: \{product\}")\
-(Pdb) c\
-The product is: 42\
-scott@MacBook-Pro-6 Scott_Submissions % python3 HW7.py\
-Enter 2 to run Q2 class for same result, 3 to run Q3 class for different result: 3\
-Here is the sample task for Question 3:\
-> /Users/scott/Desktop/GTSP/python-bootcamp/Scott_Submissions/HW7.py(50)calculation3()\
--> product = self.multiply(a, b)\
-(Pdb) step\
---Call--\
-> /Users/scott/Desktop/GTSP/python-bootcamp/Scott_Submissions/HW7.py(42)multiply()\
--> def multiply(self, a, b):\
-(Pdb) c\
-The sum is: 13\
-The product is: 42\
-scott@MacBook-Pro-6 Scott_Submissions % python3 HW7.py\
-Enter 2 to run Q2 class for same result, 3 to run Q3 class for different result: 3\
-Here is the sample task for Question 3:\
-> /Users/scott/Desktop/GTSP/python-bootcamp/Scott_Submissions/HW7.py(50)calculation3()\
--> product = self.multiply(a, b)\
-(Pdb) next\
-> /Users/scott/Desktop/GTSP/python-bootcamp/Scott_Submissions/HW7.py(52)calculation3()\
--> print(f"The sum is: \{result\}")\
-(Pdb) c\
-The sum is: 13\
-The product is: 42\
-scott@MacBook-Pro-6 Scott_Submissions % }

Scott_Submissions/Q4.py

@@ -1,18 +0,0 @@
-import pandas as pd
-import numpy as np
-
-# Generate two random 64-bit integers
-num1 = np.random.randint(0, 10 - 1)
-num2 = np.random.randint(0, 10 - 1)
-
-# Multiply them together using the normal Python multiplication operator
-print(f"num1 * num2 = {num1 * num2}")
-
-# Create a Pandas DataFrame with the generated numbers
-data = {'Number1': [num1], 'Number2': [num2]}
-df = pd.DataFrame(data)
-
-# Multiply the numbers together using the Pandas multiplication operator
-df['Product'] = df['Number1'] * df['Number2']
-
-print(df)

Scott_Submissions/Q5.py

@@ -1,4 +0,0 @@
-import sys
-num = 99999999999
-size = sys.getsizeof(num)
-print(f"The size of the float is {size} bytes.")

Scott_Submissions/Reading_Material_Notes.rtf

@@ -1,51 +0,0 @@
-{\rtf1\ansi\ansicpg1252\cocoartf2580
-\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fnil\fcharset0 LucidaGrande-Bold;\f1\fnil\fcharset0 LucidaGrande;}
-{\colortbl;\red255\green255\blue255;\red26\green26\blue26;\red255\green255\blue255;}
-{\*\expandedcolortbl;;\cssrgb\c13333\c13333\c13333;\cssrgb\c100000\c100000\c100000;}
-\margl1440\margr1440\vieww11520\viewh8400\viewkind0
-\deftab720
-\pard\pardeftab720\parhyphenfactor20\partightenfactor0
-
-\f0\b\fs32 \cf2 \cb3 \expnd0\expndtw0\kerning0
-CPython implementation detail:
-\f1\b0  CPython currently uses a reference-counting scheme with (optional) delayed detection of cyclically linked garbage, which collects most objects as soon as they become unreachable, but is not guaranteed to collect garbage containing circular references.\
-	\
-	What is a r\cf2 eference-counting scheme?\
-		It refers to a memory management technique where each object in a programming language implementation, such as CPython, keeps track of the number of references pointing to it. \
-		Whenever a reference is created to an object, its reference count is increased, and when a reference is deleted or goes out of scope, the reference count is decreased. \
-	\
-	What is a reference?\
-		In the context of memory management, a "reference" refers to an identifier or value that allows access to an object in a programming language. \
-		It is a way to refer to or point to the memory location where an object is stored.\
-	\
-	What is the advantage of reference-counting scheme?\
-		The advantage of the reference-counting scheme is that it allows objects to be deallocated immediately when they are no longer needed. \
-		It ensures efficient memory management because objects can be freed as soon as their reference count reaches zero, without the need for more complex garbage collection algorithms.\
-	\
-	What is a garbage?\
-		In the context of memory management, "garbage" refers to objects that are no longer reachable or used by the program but still occupy memory. \
-		This typically occurs when objects are created dynamically during program execution but are not properly deallocated or no longer accessible. \
-		Garbage collection is the process of identifying and reclaiming memory occupied by such unused objects, making it available for reuse. \
-		The reference-counting scheme used by CPython helps in automatically deallocating most objects as soon as they become unreachable. \
-		However, in the case of circular references, where objects refer to each other in a loop, additional mechanisms are required to detect and collect such cyclically linked garbage. It\'92s called "delayed detection of cyclically linked garbage" to collect and deallocate objects that are involved in cyclic references.\
-\
-\
-\
-What is a __blabla__(): method?\
-	In Python, the following are special attributes (also known as dunder attributes) that can be accessed on objects or functions.\
-	These special attributes provide additional information or access to specific properties of objects in Python. \
-\
-What is the advantages of the dunder method?\
-	Overall, dunder attributes provide flexibility, customization, and enhanced functionality to Python objects and classes. \
-	They contribute to the expressive nature of the language and enable powerful features like operator overloading and introspection.\
-\
-	Convention: Dunder attributes follow a specific naming convention, starting and ending with double underscores (__). This convention makes them easily recognizable and distinguishes them from regular attributes.\
-	Customization: Dunder attributes allow you to customize and define the behavior of objects and classes. By implementing specific dunder methods, you can change how objects behave in certain situations, such as arithmetic operations, string representation, comparisons, and more.\
-	Operator Overloading: Dunder attributes enable operator overloading, allowing objects to participate in built-in operations like addition (+), subtraction (-), equality (==), and more. This makes it possible to define meaningful operations for custom classes.\
-	Introspection: Dunder attributes provide a way to introspect and access information about objects and classes. For example, name and doc allow you to retrieve the name and documentation of an object. This introspection capability is useful for debugging, documentation generation, and code analysis.\
-	Integration with Language Features: Dunder attributes seamlessly integrate with various language features, such as iteration (with iter), context managers (with enter and exit), attribute access (with getattr, setattr), and more. This allows you to leverage the full power of the Python language in your classes.\
-	Code Readability: By following the conventions for dunder attributes, your code becomes more readable and understandable by other Python developers. They can easily recognize and understand the purpose of these special attributes.\
-\
-\
-\
-## The attached file HW7_test.py provides an example on how dunder method works. Don\'92t forget to check it out.}

Scott_Submissions/cr-example.py

@@ -1,7 +0,0 @@
-import time
-
-i=0
-while True:
-    print(f"\015{i=}", end='')
-    i += 1
-    time.sleep(0.2)

Scott_Submissions/pdb_test.py

@@ -1,28 +0,0 @@
-import pdb
-
-def calculate_average(numbers):
-    total = sum(numbers)
-    average = total / len(numbers)
-    return average
-
-def main():
-    data = [1, 2, 3, 4, 5]
-
-    pdb.set_trace()
-    result = calculate_average(data)
-    print("The average is:", result)
-
-    pdb.set_trace()  # Set a breakpoint at this line
-
-    print("Program execution resumed.")
-    # ... continue with the rest of your code
-
-if __name__ == '__main__':
-    main()
-
-
-# n or next: Execute the next line of code.
-# s or step: Step into a function call.
-# c or continue: Continue execution until the next breakpoint or the program's end.
-# p or print: Print the value of a variable.
-# q or quit: Quit the debugger and exit the program.

Scott_Submissions/test.py

@@ -1,5 +0,0 @@
-char1 = "u"
-result1 = ord(char1)
-print("ASCII value of the character {} is {}.".format(char1, result1))
-
-

Simulation/Simulation.py

@@ -0,0 +1,42 @@
+from abc import ABC, abstractmethod
+
+class Simulation(ABC):
+    @abstractmethod
+    def start(self):
+        "Start the simulated object."
+        pass
+
+    @abstractmethod
+    def tick(self) -> bool:
+        """
+        Simulate one clock tick.
+        Returns:
+            bool: True if the simulation could advance one clock tick, or False if the object is expired.
+        """
+        pass
+
+    @abstractmethod
+    def get_uses(self):
+        """
+        Return the uses of the simulated object.  These are verb strings which are passed to the `use()` method.
+        Returns:
+            tuple: A dict of key-value pairs, where the keys are strings of
+            verbs to be passed to `use`, and the values are strings explaining
+            what the verb does in English.
+        """
+    @abstractmethod
+    def use(self, *args, **kwargs):
+        """
+        Use the simulated object.
+        Args:
+            args (tuple): A tuple of verbs to be used on the object.
+            kwargs (dict): A dictionary of key-value pairs of arguments to the verb(s).
+        Returns:
+            True if the action succeeded, False if the action did not succeed and the object needs to be expired.
+        """
+        pass
+
+    @abstractmethod
+    def stop(self):
+        "Stop the simulated object."
+        pass

Simulation/SimulationRunner.py

@@ -0,0 +1,42 @@
+from importlib import import_module
+import tkinter as tk
+import os
+import sys
+from random import choice
+from time import sleep
+from tkinter import filedialog
+from abc import ABC, abstractmethod
+
+class SimulationRunner:
+    def __init__(self, simulation) -> None:
+        self.simulation = simulation()
+        self.simulation.start()
+        uses = list(self.simulation.get_uses().keys())
+        while self.simulation.tick() and self.simulation.use(choice(uses)):
+            pass
+        self.simulation.stop()
+
+if __name__ == '__main__':
+    if len(sys.argv) == 1:
+        # Create a hidden root window
+        root = tk.Tk()
+        root.withdraw()
+
+        # Open the file chooser dialog
+        file_path = filedialog.askopenfilename()
+        print(f"Selected file: {file_path}")
+    else:
+        file_path = sys.argv[1]
+
+    # Get the python file's directory and filename
+    module_dir, module_file = os.path.split(file_path)
+    # Get the python file's name without any extension
+    module_name, _ = os.path.splitext(module_file)
+
+    # Add the module directory to sys.path so we can import it unqualified
+    sys.path.append(module_dir)
+
+    simulation = import_module(module_name)
+    # Load the simulation module's simulation class, which should be the same
+    # name as the module name.
+    simulationrunner = SimulationRunner(getattr(simulation, module_name))

Task_assignment.ipynb

@@ -0,0 +1,402 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Guide\n",
+    "1. Please write your solutions at the appropriate location **# YOUR CODE HERE**\n",
+    "2. Do not modify other cells"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "# Numerical or binary attributes\n",
+    "num_or_bin= ['Income', 'Kidhome','Recency', 'MntWines', 'MntFruits', 'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts','MntGoldProds', 'NumDealsPurchases', 'NumWebPurchases',\n",
+    "       'NumCatalogPurchases', 'NumStorePurchases', 'NumWebVisitsMonth','AcceptedCmp3', 'AcceptedCmp4', 'AcceptedCmp5', 'AcceptedCmp1','AcceptedCmp2']\n",
+    "\n",
+    "# Categorical attributes\n",
+    "cat = ['Education','Marital_Status']\n",
+    "\n",
+    "# Load the data source\n",
+    "pd.set_option('display.max_columns', None)\n",
+    "df  = pd.read_csv(\"https://raw.githubusercontent.com/rpi-techfundamentals/website_fall_2021/master/site/public/marketing_campaign.csv\", delimiter='\\t')\n",
+    "df.head()"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q1. Load and preprocess the data.\n",
+    "Build and transform your dataframe `df` using the following steps before modeling. Create a new dataframe `X` that has the following properties:\n",
+    "\n",
+    "1. It should include only the columns from lists `num_or_bin` and `cat` (in the code above).\n",
+    "2. Variables in cat should be transformed to `n-1` dummy variables (where `n` is the number of categories for each variable.)\n",
+    "3. Missing variables in the `Income` table should be filled with the median of the `income` table.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "all_cols=num_or_bin+cat\n",
+    "X=pd.get_dummies(df[all_cols],columns=cat,drop_first=True)\n",
+    "X.loc[X['Income'].isnull(),'Income']=X['Income'].median()\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 1 Test Cell \n",
+    "#this tests the columns. \n",
+    "ecol = set(['Income', 'Kidhome', 'Recency', 'MntWines', 'MntFruits',\n",
+    "       'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts',\n",
+    "       'MntGoldProds', 'NumDealsPurchases', 'NumWebPurchases',\n",
+    "       'NumCatalogPurchases', 'NumStorePurchases', 'NumWebVisitsMonth',\n",
+    "       'AcceptedCmp3', 'AcceptedCmp4', 'AcceptedCmp5', 'AcceptedCmp1',\n",
+    "       'AcceptedCmp2', 'Education_Basic', 'Education_Graduation',\n",
+    "       'Education_Master', 'Education_PhD', 'Marital_Status_Alone',\n",
+    "       'Marital_Status_Divorced', 'Marital_Status_Married',\n",
+    "       'Marital_Status_Single', 'Marital_Status_Together',\n",
+    "       'Marital_Status_Widow', 'Marital_Status_YOLO'])\n",
+    "assert len(ecol-set(X.columns))==0\n",
+    "assert len(set(X.columns))-len(ecol)==0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 2 Test Cell \n",
+    "#this tests the columns. \n",
+    "assert X.shape==(2240, 30)\n",
+    "q2=X.copy()\n",
+    "q2['Response']=df['Response']\n",
+    "q2['Year_Birth']=df['Year_Birth']\n",
+    "q2['Dt_Customer']=df['Dt_Customer']\n",
+    "q2.to_csv('q2.csv',index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 3 Test Cell \n",
+    "assert len(X.loc[X['Income']==51381.5,'Income'])==24"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 4 Test Cell\n",
+    "assert X.isnull().sum().sum()==0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Reload data\n",
+    "import pandas as pd\n",
+    "df_cl=pd.read_csv('q2.csv')"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q2. Feature Creation.\n",
+    "1. In the `df_cl` dataframe, create an integer feature `age` which is `2021` minus `Year_Birth`.\n",
+    "2. In the `df_cl` dataframe, create an integer feature `holiday` which is 1 if the month of `Dt_Customer` is equal to 12 and 0 otherwise. You can get the month by accessing `mm` from the date format `dd-mm-yyyy`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "df_cl['age']=(2021-df_cl['Year_Birth']).astype(int)\n",
+    "df_cl['Dt_Customer_tmp']=df_cl['Dt_Customer'].apply(lambda x:x[3:5])\n",
+    "df_cl.loc[df_cl['Dt_Customer_tmp']=='12','holiday']=1\n",
+    "df_cl.loc[df_cl['Dt_Customer_tmp']!='12','holiday']=0\n",
+    "df_cl['holiday']=df_cl['holiday'].astype(int)\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 5 Test Cell\n",
+    "assert df_cl['age'].sum()==116915"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 6 Test Cell\n",
+    "import numpy as np\n",
+    "assert isinstance(df_cl['age'][0], (np.int64,np.int32))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 7 Test Cell\n",
+    "assert df_cl['holiday'].sum()==175"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 8 Test Cell\n",
+    "import numpy as np\n",
+    "assert isinstance(df_cl['holiday'][0], (np.int64,np.int32))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "df_cl2=pd.read_csv('dfcl2.csv')\n",
+    "y_reload=df_cl2['Response']\n",
+    "X_reload=df_cl2.drop(['Response'], axis=1)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q3. Train Test Split\n",
+    "Now split the data `X_reload`, `y_reload` intro training (50%) and testing data (50%) by using these variable names while making sure there are Make sure that there are equal number of positive cases (1's) in the train and the test set.\n",
+    "\n",
+    "`X_train`: Training feature columns\n",
+    "`X_test`: Testing feature columns\n",
+    "`y_train`: Training DV\n",
+    "`y_test`: Testing DV\n",
+    "\n",
+    "When doing so set the `random_state` parameter to 100 so you get the same split that is in the solution."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "y_reload=df_cl2['Response']\n",
+    "X_reload=df_cl2.drop(['Response'], axis=1)\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "X_train,X_test,y_train,y_test=train_test_split(X_reload,y_reload,\n",
+    "                                               train_size=0.5,\n",
+    "                                               test_size=0.5,\n",
+    "                                               random_state=100,\n",
+    "                                               stratify=y_reload)\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 9 Test Cell\n",
+    "assert X_train.shape==(1120, 31)\n",
+    "assert X_test.shape==(1120, 31)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 10 Test Cell\n",
+    "assert  y_train.sum()==y_test.sum()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "df_cl2=pd.read_csv('dfcl2.csv')\n",
+    "y_reload=df_cl2['Response']\n",
+    "X_reload=df_cl2.drop(['Response'], axis=1)\n",
+    "\n",
+    "X_train2, X_test2, y_train2, y_test2 = train_test_split(X_reload, y_reload, test_size=0.3, random_state = 100)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q4. Classification\n",
+    "Use `X_train2`, `X_test2`, `y_train2`, `y_test2` from above. Use a Gradient Boosting classifier (as instantiated below).\n",
+    "\n",
+    "Please calculate:\n",
+    "1. `test_acc` The accuracy of the classifier on the test set.\n",
+    "2. `test_roc` The area under the ROC curve on on the test set."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn import metrics\n",
+    "from sklearn.ensemble import GradientBoostingClassifier\n",
+    "#use this classifier\n",
+    "classifier = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1, random_state=0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "classifier.fit(X_train2,y_train2)\n",
+    "test_y_pred=classifier.predict(X_test2)\n",
+    "test_y_pred_prob=classifier.predict_proba(X_test2)[:,1]\n",
+    "test_acc=metrics.accuracy_score(y_test2,test_y_pred)\n",
+    "test_roc=metrics.roc_auc_score(y_test2,test_y_pred_prob)\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 11 Test Cell\n",
+    "assert round(test_acc,3)==0.860"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 12 Test Cell\n",
+    "assert round(test_roc,3)==0.846"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

View_h5_file.py

@@ -0,0 +1,74 @@
+import h5py
+import pandas as pd
+import numpy as np
+import plotly.graph_objects as go
+
+f = h5py.File('product1.h5','r')
+list(f.keys())
+for key in f.keys():
+    print(f[key])
+    df = pd.read_hdf('product1.h5', key='df')
+    df = df._append(pd.read_hdf('product1.h5', key))
+    # print(f[key].shape)
+    # print(f[key].value)
+# df = pd.read_hdf('product1.h5', key='df7')
+print(df)
+'''
+# Open the HDF5 file
+with h5py.File('product1.h5', 'r') as file:
+    dataframes=pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])  
+    for key in file.keys():
+        # Read the dataframe from the HDF5 file
+        dataframe = pd.read_hdf('product1.h5', key)
+
+        # Append the dataframe to the list
+        dataframes._append(dataframe)
+
+# Print the number of dataframes read from the HDF5 file
+print(f"Read {len(dataframes)} dataframes from the HDF5 file")
+
+item_std_devs = {}
+for dataframe in dataframes:
+    item_name = dataframe['Product ID'].iloc[0]
+    std_dev = np.std(dataframe['Unit Price'])
+    item_std_devs[item_name] = std_dev
+
+# Print the item names and their corresponding standard deviations
+for item_name, std_dev in item_std_devs.items():
+    print(f"Item: {item_name}, Std Dev: {std_dev}")
+
+# Prepare the data for the graph
+x = list(item_std_devs.keys())
+y = list(item_std_devs.values())
+
+# Create a bar graph
+fig = go.Figure(data=[go.Bar(x=x, y=y)])
+
+# Customize the graph
+fig.update_layout(
+    title="Standard Deviation of Pricing for Each Item",
+    xaxis_title="Item",
+    yaxis_title="Standard Deviation",
+)
+
+# Save the graph as an HTML file
+report_file = 'test2.html'
+fig.write_html(report_file)
+
+
+
+'''
+'''
+# Calculate the standard deviation using pandas' function
+std_dev_unit = df.groupby('Product ID')['Unit Price'].std()
+std_dev_displayed = df.groupby('Zip Code')['Shown Price'].std()
+
+# Generate the graph using Plotly
+fig = go.Figure()
+fig.add_trace(go.Bar(x=std_dev_unit.index, y=std_dev_unit.values))
+fig.update_layout(title='Standard Deviation of the Unit Pricing by Item', xaxis_title='Product ID', yaxis_title='Standard Deviation')
+
+report_file = 'test2.html'
+fig.write_html(report_file)
+'''
+

assessment1.py

@@ -0,0 +1,81 @@
+'''
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+        self._attributes = dict(self.__dict__)
+        
+    def __setattr__(self, name, value):
+        if name != "_attributes" and name not in self._attributes:
+            print("attribute deleted!")
+        self._attributes[name] = value
+        super().__setattr__("_attributes", dict(self.__dict__))
+        
+    @property
+    def treasure(self):
+        if self.__dict__ != self._attributes:
+            print("attribute deleted!")
+        return self._treasure
+
+v=Vault()
+print(v.treasure)
+'''
+'''
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+        self._original_dict = dict(self.__dict__)
+
+    def __setattr__(self, name, value):
+        if name != "_treasure" and name in self.__dict__ and self._original_dict.get(name) != value:
+            print("attribute deleted!")
+        self.__dict__[name] = value
+
+    @property
+    def treasure(self):
+        if self.__dict__ != self._original_dict:
+            print("attribute deleted!")
+        return self._treasure
+v = Vault()
+print(v.treasure)
+'''
+'''
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+        self._attributes = dict(self.__dict__)
+
+    def __setattr__(self, name, value):
+        if name != "_attributes" and name not in self._attributes:
+            print("attribute deleted!")
+        self._attributes[name] = value
+        super().__setattr__("_attributes", dict(self.__dict__))
+
+    @property
+    def treasure(self):
+        if self.__dict__ != self._attributes:
+            print("attribute deleted!")
+        return self._treasure
+
+v = Vault()
+print(v.treasure)
+'''
+
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+
+    def __setattr__(self, name, value):
+        if name != "_attributes" and name not in self.__dict__:
+            print("attribute deleted!")
+        self.__dict__[name] = value
+        self._attributes = dict(self.__dict__)
+
+    @property
+    def treasure(self):
+        if self.__dict__ != self._attributes:
+            print("attribute deleted!")
+        return self._treasure
+
+v = Vault()
+print(v.treasure)
+

assessment2.py

@@ -0,0 +1,47 @@
+import importlib.util
+import os
+
+def get_module_size(module_name):
+    """Get the total number of bytes in the cached .pyc file or .py file of a module or package.
+
+    Args:
+        module_name (str): The name of the module or package.
+
+    Returns:
+        int: The total number of bytes in the cached .pyc file or .py file.
+    """
+    spec = importlib.util.find_spec(module_name)
+
+    if spec is not None and spec.has_location:
+        # If the module is found and has a location, check for .pyc file first
+        pyc_file_path = spec.origin + 'c'
+        if os.path.exists(pyc_file_path):
+            return os.stat(pyc_file_path).st_size
+        else:
+            # If .pyc file doesn't exist, check for .py file
+            py_file_path = spec.origin
+            if os.path.exists(py_file_path):
+                return os.stat(py_file_path).st_size
+
+    return 0  # Return 0 if no .pyc or .py file found
+
+
+import sys
+
+def get_total_module_size():
+    """Get the total number of bytes for all modules listed in sys.modules.
+
+    Returns:
+        int: The total number of bytes for all modules.
+    """
+    total_size = 0
+    for module_name in sys.modules:
+        module_size = get_module_size(module_name)
+        total_size += module_size
+
+    return total_size
+
+if __name__ == '__main__' and __package__ is None:
+    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
+total_size = get_total_module_size()
+print("Total size of all modules in sys.modules: {} bytes".format(total_size))

fp1.py

@@ -0,0 +1,65 @@
+from google.cloud import pubsub_v1
+from google.oauth2 import service_account
+import os
+import asyncio
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+project_id = "fp-20230425"
+
+'''
+topic_name = "fp-topic-Karen"
+subscription_name = "fp-subscription-Karen"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
+subscriber.delete_subscription(request={"subscription": subscription_path})
+print("Subscription deleted: {}".format(subscription_path))
+
+# Delete the topic
+publisher = pubsub_v1.PublisherClient()
+topic_path = publisher.topic_path(project_id, topic_name)
+publisher.delete_topic(request={"topic": topic_path})
+print(f"Topic '{topic_name}' deleted.")
+
+subscription_name = "fp-subscription-Karen"
+subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
+
+
+'''
+# Create a pubsub topic
+publisher = pubsub_v1.PublisherClient()
+topic_name = "fp-topic-Karen"
+topic_path = publisher.topic_path(project_id, topic_name)
+topic = publisher.create_topic(request={"name": topic_path})
+
+
+# Create a Pub/Sub subscription
+subscriber = pubsub_v1.SubscriberClient()
+subscription_name = "fp-subscription-Karen"
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+subscription = subscriber.create_subscription(
+    name=subscription_path, topic=topic_path
+)
+
+# Publish a message to the topic
+message = b"Hello!"
+publisher.publish(topic_path, data=message)
+print("Message published.")
+
+
+async def receive_messages():
+    response = subscriber.pull(subscription=subscription_path, max_messages=1)
+    for received_message in response.received_messages:
+        print("Received message: {}".format(received_message.message.data))
+       
+    
+asyncio.run(receive_messages())
+
+
+'''
+# Pull the message from the subscription
+response = subscriber.pull(subscription=subscription_path, max_messages=1)
+for received_message in response.received_messages:
+    print("Received message: {}".format(received_message.message.data))
+    # Acknowledge the message to mark it as processed
+    subscriber.acknowledge(subscription_path, [received_message.ack_id])
+'''

fp2.py

@@ -0,0 +1,10 @@
+import http.client
+
+# The product ID for the Pete-and-Gerry's organic large brown egges is 65902
+conn = http.client.HTTPSConnection("www.sayweee.com")
+payload = ''
+headers = {}
+conn.request("GET", "/en/product/Pete-and-Gerry-s-Organic-Large-Brown-Eggs/65902?category=dairy02", payload, headers)
+res = conn.getresponse()
+data = res.read()
+print(data.decode("utf-8"))

fp2_10.py

@@ -0,0 +1,67 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+import csv
+import os
+import asyncio
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+project_id = "fp-20230425"
+# project_id = "svig-384801"
+
+'''
+topic_name = "fp-topic-NFL-zipcodes"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
+subscriber.delete_subscription(request={"subscription": subscription_path})
+print("Subscription deleted: {}".format(subscription_path))
+
+# Delete the topic
+publisher = pubsub_v1.PublisherClient()
+topic_path = publisher.topic_path(project_id, topic_name)
+publisher.delete_topic(request={"topic": topic_path})
+print(f"Topic '{topic_name}' deleted.")
+
+
+
+'''
+# pubsub topic
+publisher = pubsub_v1.PublisherClient()
+topic_name = "fp-topic-NFL-zipcodes"
+topic_path = publisher.topic_path(project_id, topic_name)
+
+# Create a Pub/Sub subscription
+subscriber = pubsub_v1.SubscriberClient()
+'''
+subscription_name = "fp-subscription-zipcodes"
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+subscription = subscriber.create_subscription(
+    name=subscription_path, topic=topic_path
+)
+'''
+# Publish a message to the topic
+# Read the data from the csv file
+# file_path = "C:/Users/Karen/Desktop/zipcodes.csv"
+file_path = "zipcodes.csv"
+
+with open(file_path, "r") as csv_file:
+    reader = csv.reader(csv_file)
+    next(reader)  # skip header row
+    for row in reader:
+        # Format the data as a string and publish it to the topic
+        data = ",".join(row).encode("utf-8")
+        try:
+            future = publisher.publish(topic_path, data)
+            print(f"Published message ID {future.result()}")
+        except DeadlineExceeded as de:
+            print(f"Caught DeadlineExceeded exception: {de}")
+
+'''
+async def receive_messages():
+    response = subscriber.pull(subscription=subscription_path, max_messages=1)
+    for received_message in response.received_messages:
+        print("Received message: {}".format(received_message.message.data))
+       
+    
+asyncio.run(receive_messages())
+'''

fp2_11.py

@@ -0,0 +1,372 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+import selenium.common.exceptions
+import json
+import time
+import os
+import re
+import sys
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+
+# Create an empty dataframe to store the prices
+df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Displayed Price', 'ld+json Price', 'Minimum Quantity', 'Total Price', 'Sales Count', 'Sold Status'])
+print(f"Created a dataframe: {df}")
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+# Load the product page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+product_id = url.split("/")[-1]
+driver.delete_all_cookies
+driver.get(url)
+# time.sleep(5)
+
+# time.sleep(2)
+# Close the cookies setting window
+try:  
+    # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")
+    cookie_button = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+    )
+    print(f"cookie_button = {cookie_button}")
+    cookie_button.click()
+except:
+    pass
+
+# time.sleep(1)
+# Clear any modal dialog
+try:    
+    # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+    modal_close = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+    )
+    print(f"modal_close = {modal_close}")
+    modal_close.click()
+    #time.sleep(2)
+except:
+    pass
+
+
+def close_modals():
+    try:
+        # time.sleep(2)
+        # Clear any modal dialog
+        modal_close_buttons = driver.find_elements(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+        print(f"modal_close = {modal_close}")
+        for button in modal_close_buttons:
+            try:
+                button.click()
+            except Exception as e:
+                print(f"Exception when clicking on close button for element {button}: {e}")
+
+        # time.sleep(2)
+    except Exception as e:
+        print(f"Exception when trying to close discount modal dialog: {e}")
+
+
+    try:
+        # time.sleep(1)
+        # Clear any notice
+        notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+        print(f"notice_close = {notice_close}")
+        notice_close.click()
+        # time.sleep(1)
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        print("Purchase option change dialog close button not found")
+    except Exception as e:
+        print(f"Exception when trying to close purchase option change dialog: {e}")
+
+def get_element_text(class_name_prefix):
+    '''
+    Return the text in the first element with a given class name.
+
+    Note that XPath doesn't have a function to check if a given word is present
+    in a list of words, so using contains() will match a class name even if
+    the class name is a substring of the class name being matched against.
+    To prevent this, we use spaces as a delimiter at the beginning and end of
+    the class name, and ensure there's only one space before and after all
+    class names in the class name list.
+
+    Example: consider two class names with cache-busting suffixes:
+      * Header_soldStatusNotice__12345
+      * Header_soldStatusNoticeBackground__ABCDE
+
+    Matching against Header_soldStatusNotice using a naive implementation with
+    `contains()` would match both class names, even though only the first one
+    is the correct match. 
+    '''
+    # Construct xpath selector.  
+    xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+    xpath_str += "' '), "                     # Step 4: a trailing space...
+    xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # We didn't find a match, try again by doing a prefix match and assume
+    # the full class name ends with two underscores followed by a cache-
+    # busting string.    
+    xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+    xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # No match, return empty string.
+    return ''
+
+def callback(message):
+    '''
+    Callback function for Pub/Sub messages
+    '''
+    # Extract zip code from message data
+    zip_code = message.message.data.decode('utf-8')
+
+    print(f"Received zip code: {zip_code}")
+
+    while True:
+        try:
+            # Enter zip code in the input field and click the "Check" button
+            WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+            zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
+            # print(f"{zip_button:}")
+            zip_button.click()
+            # time.sleep(1)
+
+            WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, '//div/label/following-sibling::input'))
+            zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+            # print(f"{zip_input:}")
+            zip_input.click()
+            # print("clearing zip_input")
+            zip_input.send_keys("\b\b\b\b\b\b\b\b\b\b")
+            for i in range(1,10):
+                zip_input.send_keys(Keys.DELETE)
+            # print("sending zip code")
+            zip_input.send_keys(zip_code)
+
+            WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, "//button[text() = 'Confirm']"))
+            check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+            print(f"{check_button:}")
+            check_button.click()
+            # time.sleep(2)
+            break
+        except Exception as e:
+            print(f"Exception when changing zip code: {e}, trying to close modal dialogs")
+            close_modals()
+
+        # try:
+        #     time.sleep(2)
+        #     # Clear any notice
+        #     notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+        #     print(f"notice_close = {notice_close}")
+        #     notice_close.click()
+        #     time.sleep(2)
+        # except:
+        #     pass
+
+    try:
+        try:
+            # time.sleep(2)
+            # Clear any modal dialog
+            # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+            modal_close = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+        )
+            print(f"modal_close = {modal_close}")
+            modal_close.click()
+            # time.sleep(2)
+        except:
+            pass
+
+
+        try:
+            # time.sleep(1)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
+        )
+            print(f"notice_close = {notice_close}")
+            notice_close.click()
+            # time.sleep(1)
+        except:
+            pass
+        
+        # Enter zip code in the input field and click the "Check" button
+        # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
+        zip_button = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+        )
+        # print(f"{zip_button:}")
+        zip_button.click()
+        # time.sleep(1)
+
+        # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+        zip_input = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+        )
+        # print(f"{zip_input:}")
+        zip_input.click()
+        # print("clearing zip_input")
+        zip_input.send_keys("\b\b\b\b\b")
+        # print("sending zip code")
+        zip_input.send_keys(zip_code)
+
+        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+        check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+        # print(f"{check_button:}")
+        check_button.click()
+        # time.sleep(2)
+
+        try:
+            # time.sleep(2)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+            #time.sleep(2)
+        except:
+            pass
+
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+        # Record sales count, if any
+        sales_count = get_element_text('Header_count')
+        # Record any sold status notice
+        sold_status = get_element_text('soldStatusNotice')
+
+        # Check if there's a minimum order quantity
+        min_quantity: float = 0
+        total_price: float = 0
+        try:
+            minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' minQuantity ')]").get_attribute('innerHTML')
+            if minimum_quantity_str:
+                regexp="Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
+                match = re.search(regexp, minimum_quantity_str)
+                min_quantity, total_price = match.groups()
+
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            print("No minimum purchase string found")
+        except Exception as e:
+            print(f"Exception when finding minimum purchase string: {e}")
+
+        # Get the price as displayed on the web page
+        displayed_price = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' price_price ')]").get_attribute('innerText').replace('$', '')
+        print(f"displayed_price = {displayed_price}")
+
+        # Extract the price of rice crackers for this zip code     
+        '''
+        price_element = driver.find_element(By.XPATH, '/html/body/div[2]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')
+        price = price_element.text.strip()[1:]
+        print(price)
+        '''
+
+        # Get the price from the ld+json element
+        ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        ld_json_text = ld_json_element.get_attribute('innerHTML')
+        ld_json_start_index = ld_json_text.index('{')
+        ld_json_end_index = ld_json_text.rindex('}') + 1
+        ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
+        ld_json = json.loads(ld_json_json)
+        ld_json_price = ld_json['offers']['price']
+        # Print the price
+        print(f"ld_json_price = {ld_json_price}")
+
+        subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+        # Append the zip code and price to the dataframe
+        #append_to_df(zip_code, price)
+        new_row = pd.DataFrame({'Product ID': [product_id],
+            'Zip Code': [zip_code],
+            'Displayed Price': [displayed_price],
+            'ld+json Price': [ld_json_price],
+            'Minimum Quantity': [min_quantity],
+            'Total Price': [total_price],
+            'Sales Count': [sales_count],
+            'Sold Status': [sold_status]})
+        return pd.concat([df, new_row], ignore_index=True)
+        #print(f"Recorded the zip code and price: {zip_code},{price}")
+        #print(f"Updated a dataframe: {df}")
+
+    except Exception as e:
+        print(f"Caught exception {type(e)}: {e}")
+        print(f"Price not found for zip code {zip_code}")
+        price = 'Price not found'
+        return df
+
+# Start receiving messages (sync version)
+while True: 
+    pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+    messages = []
+    try:
+        pull_response = subscriber.pull(request = pull_request)
+        messages = pull_response.received_messages
+    except DeadlineExceeded:
+        pass
+
+    if messages:
+        df = callback(messages[0])
+        print(f"Updated a dataframe: {df}")
+        df.to_hdf('product1.h5', key='df',mode='w')
+    else:
+        print(f"No messages in subscription {subscription_path} at {time.asctime()}, trying again...")
+        continue
+
+# Open the HDF5 file in a separate window
+if sys.platform.startswith("win32"):
+    os.startfile('product1.h5')
+
+# Start receiving messages (async version)
+#flow_control = pubsub_v1.types.FlowControl(max_messages = 1)
+#streaming_pull_future = subscriber.subscribe(subscription_path, 
+#    callback=callback, 
+#    flow_control=flow_control)
+
+# print(f'Started receiving messages from subscription {subscription_path}...')
+
+# Wrap subscriber in a 'with' block to automatically call close() when done.
+# with subscriber:
+#    try:
+        # When `timeout` is not set, result() will block indefinitely,
+        # unless an exception is encountered first.
+#        streaming_pull_future.result(timeout=timeout)
+#    except TimeoutError:
+#        streaming_pull_future.cancel()  # Trigger the shutdown.
+#        streaming_pull_future.result()  # Block until the shutdown is complete.
+
+# Keep the script running indefinitely to continue receiving messages
+#while True:
+#    time.sleep(10)

fp2_12.py

@@ -0,0 +1,199 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+# Create an empty dataframe to store the prices
+df = pd.DataFrame(columns=['Zip Code', 'Price'])
+print(f"Created a dataframe: {df}")
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+# Load the product page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+driver.delete_all_cookies
+driver.get(url)
+# time.sleep(5)
+
+# time.sleep(2)
+# Close the cookies setting window
+try:  
+    # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")
+    cookie_button = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+    )
+    print(f"cookie_button = {cookie_button}")
+    cookie_button.click()
+except:
+    pass
+
+# time.sleep(1)
+# Clear any modal dialog
+try:    
+    # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+    modal_close = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+    )
+    print(f"modal_close = {modal_close}")
+    modal_close.click()
+    #time.sleep(2)
+except:
+    pass
+
+# Define callback function for Pub/Sub messages
+def callback(message):
+    # Extract zip code from message data
+    zip_code = message.message.data.decode('utf-8')
+
+    print(f"Received zip code: {zip_code}")
+
+    try:
+        try:
+            # time.sleep(2)
+            # Clear any modal dialog
+            # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+            modal_close = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+        )
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+            # time.sleep(2)
+        except:
+            pass
+
+
+        try:
+            # time.sleep(1)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
+        )
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+            # time.sleep(1)
+        except:
+            pass
+        
+        # Enter zip code in the input field and click the "Check" button
+        # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
+        zip_button = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+        )
+        # print(f"{zip_button:}")
+        zip_button.click()
+        # time.sleep(1)
+
+        # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+        zip_input = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+        )
+        # print(f"{zip_input:}")
+        zip_input.click()
+        # print("clearing zip_input")
+        zip_input.send_keys("\b\b\b\b\b")
+        # print("sending zip code")
+        zip_input.send_keys(zip_code)
+
+        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+        check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+        # print(f"{check_button:}")
+        check_button.click()
+        # time.sleep(2)
+
+        try:
+            # time.sleep(2)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+            #time.sleep(2)
+        except:
+            pass
+
+
+    # Extract the price of rice crackers for this zip code
+        
+        # next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        next_data_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, '//*[@id="__next"]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')))
+        next_data_text = next_data_element.get_attribute('nodeValue')
+        price = next_data_text.text
+        '''
+        next_data_text = next_data_element.get_attribute('innerHTML')
+        next_data_start_index = next_data_text.index('{')
+        next_data_end_index = next_data_text.rindex('}') + 1
+        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+        next_data = json.loads(next_data_json)
+        price = next_data['price']
+        '''
+        # Print the price
+        print(price)
+
+        '''
+        try:
+            min_quantity_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, '//script[@id="__NEXT_DATA__"]')))
+            min_quantity_text = min_quantity_element.get_attribute('innerHTML')
+            min_quantity_start_index = min_quantity_text.index('{')
+            min_quantity_end_index = min_quantity_text.rindex('}') + 1
+            min_quantity_json = min_quantity_text[min_quantity_start_index:min_quantity_end_index]
+            min_quantity = json.loads(min_quantity_json)
+            price = min_quantity['product']['price']
+        except:
+            next_data_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//script[@type='application/ld+json']")))
+            next_data_text = next_data_element.get_attribute('innerHTML')
+            next_data_start_index = next_data_text.index('{')
+            next_data_end_index = next_data_text.rindex('}') + 1
+            next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+            next_data = json.loads(next_data_json)        
+            price = next_data['offers']['price']
+        finally:
+            # Print the price
+            print(price)
+        '''
+
+        subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+        # Append the zip code and price to the dataframe
+        #append_to_df(zip_code, price)
+        new_row = pd.DataFrame({'Zip Code': [zip_code], 'Price': [price]})
+        return pd.concat([df, new_row], ignore_index=True)
+        #print(f"Recorded the zip code and price: {zip_code},{price}")
+        #print(f"Updated a dataframe: {df}")
+
+    except Exception as e:
+        print(f"Caught exception: {e}")
+        print(f"Price not found for zip code {zip_code}")
+        price = 'Price not found'
+        return df
+
+# Start receiving messages (sync version)
+while True: 
+    pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+    messages = []
+    try:
+        pull_response = subscriber.pull(request = pull_request)
+        messages = pull_response.received_messages
+    except DeadlineExceeded:
+        pass
+
+    if messages:
+        df = callback(messages[0])
+        print(f"Updated a dataframe: {df}")
+    else:
+        print(f"no messages, trying again at {time.asctime()}")

fp2_13.py

@@ -0,0 +1,271 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+import os
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import selenium.common.exceptions
+import re
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+
+# Create an empty dataframe to store the prices
+df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
+print(f"Created a dataframe: {df}")
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+# Load the product page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
+product_id = url.split("/")[-1][:5]
+# product_id = url.split("/")[-1]
+driver.delete_all_cookies
+driver.get(url)
+# time.sleep(5)
+
+def get_element_text(class_name_prefix):
+    '''
+    Return the text in the first element with a given class name.
+
+    Note that XPath doesn't have a function to check if a given word is present
+    in a list of words, so using contains() will match a class name even if
+    the class name is a substring of the class name being matched against.
+    To prevent this, we use spaces as a delimiter at the beginning and end of
+    the class name, and ensure there's only one space before and after all
+    class names in the class name list.
+
+    Example: consider two class names with cache-busting suffixes:
+      * Header_soldStatusNotice__12345
+      * Header_soldStatusNoticeBackground__ABCDE
+
+    Matching against Header_soldStatusNotice using a naive implementation with
+    `contains()` would match both class names, even though only the first one
+    is the correct match. 
+    '''
+    # Construct xpath selector.  
+    xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+    xpath_str += "' '), "                     # Step 4: a trailing space...
+    xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # We didn't find a match, try again by doing a prefix match and assume
+    # the full class name ends with two underscores followed by a cache-
+    # busting string.    
+    xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+    xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+    xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+    xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+    try:
+        match = driver.find_element(By.XPATH, xpath_str)
+        return match.get_attribute('textContent')
+    except selenium.common.exceptions.NoSuchElementException as nsee:
+        pass
+
+    # No match, return empty string.
+    return ''
+
+
+# time.sleep(2)
+# Close the cookies setting window
+try:  
+    # cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")
+    cookie_button = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+    )
+    print(f"cookie_button = {cookie_button}")
+    cookie_button.click()
+except:
+    pass
+
+# time.sleep(1)
+# Clear any modal dialog
+try:    
+    # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+    modal_close = WebDriverWait(driver, 2).until(
+        EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+    )
+    print(f"modal_close = {modal_close}")
+    modal_close.click()
+    #time.sleep(2)
+except:
+    pass
+
+# Define callback function for Pub/Sub messages
+def callback(message):
+    # Extract zip code from message data
+    zip_code = message.message.data.decode('utf-8')
+
+    print(f"Received zip code: {zip_code}")
+
+    try:
+        try:
+            # time.sleep(2)
+            # Clear any modal dialog
+            # modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+            modal_close = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
+        )
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+            # time.sleep(2)
+        except:
+            pass
+
+
+        try:
+            # time.sleep(1)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
+        )
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+            # time.sleep(1)
+        except:
+            pass
+        
+        # Enter zip code in the input field and click the "Check" button
+        # zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
+        zip_button = WebDriverWait(driver, 5).until(
+            EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+        )
+        # print(f"{zip_button:}")
+        zip_button.click()
+        # time.sleep(1)
+
+        # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+        zip_input = WebDriverWait(driver, 2).until(
+            EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+        )
+        # print(f"{zip_input:}")
+        zip_input.click()
+        # print("clearing zip_input")
+        zip_input.send_keys("\b\b\b\b\b")
+        # print("sending zip code")
+        zip_input.send_keys(zip_code)
+
+        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+        check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+        # print(f"{check_button:}")
+        check_button.click()
+        # time.sleep(2)
+
+        try:
+            # time.sleep(2)
+            # Clear any notice
+            # notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
+            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+            #time.sleep(2)
+        except:
+            pass
+
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+        # Record sales count, if any
+        sales_count_element = get_element_text('Header_count')
+        count_start_index = sales_count_element.index('sold ') + len('sold ')
+        count_end_index = sales_count_element.index('+')
+        sales_count = sales_count_element[count_start_index:count_end_index]
+        # Record any sold status notice
+        sold_status = get_element_text('UnavailableCartbtn_container')
+        if sold_status == 'Notify me':
+            sold_status = 'Sold Out'
+
+    # Extract the price of rice crackers for this zip code
+        # Check if there is a minimum order quantity
+        min_quantity: float = 0
+        total_price: float = 0
+        unit_price: float = 0
+        try:
+            minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML')
+            if minimum_quantity_str:
+                regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
+                match = re.search(regexp,minimum_quantity_str)
+                min_quantity, total_price = match.groups()
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            print("No minimum purhcase string found")
+        except Exception as e:
+            print(f"Exception when finding minimum purchase string: {e}")
+ 
+
+
+        ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        ld_json_text = ld_json_element.get_attribute('innerHTML')
+        ld_json_start_index = ld_json_text.index('{')
+        ld_json_end_index = ld_json_text.rindex('}') + 1
+        ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
+        ld_json = json.loads(ld_json_json)
+        ld_json_price = ld_json['offers']['price']
+        # Print the price
+        print(ld_json_price)
+
+        unit_price_element = driver.find_element(By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText')
+        unit_price = unit_price_element.replace("$", "")
+        print(f"unit_price = {unit_price}" )
+
+        subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+        # Append the zip code and price to the dataframe
+        #append_to_df(zip_code, price)
+        new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]})
+        return pd.concat([df, new_row], ignore_index=True)
+        #print(f"Recorded the zip code and price: {zip_code},{price}")
+        #print(f"Updated a dataframe: {df}")
+
+    except Exception as e:
+        print(f"Caught exception: {e}")
+        print(f"Price not found for zip code {zip_code}")
+        price = 'Price not found'
+        return df
+
+# Start receiving messages (sync version)
+while True: 
+    pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+    messages = []
+    try:
+        pull_response = subscriber.pull(request = pull_request)
+        messages = pull_response.received_messages
+    except DeadlineExceeded:
+        pass
+
+    if messages:
+        df = callback(messages[0])
+        print(f"Updated a dataframe: {df}")
+    else:
+        print(f"no messages, trying again at {time.asctime()}")
+        break
+
+# df.to_hdf('product1.h5', key='df',mode='w')
+# df.to_hdf('product1.h5', key='df',mode='a')
+
+# Open the HDF5 file in a separate window
+# os.startfile('product1.h5')

fp2_14.py

@@ -0,0 +1,280 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import selenium.common.exceptions
+import re
+from datetime import datetime
+import h5py
+import sys
+import ntplib
+import arrow
+
+# This fp2_14.py is used to load the new price and the timestamp, and append such data in an existing key in the HDF5 file "product1.h5"
+
+def get_product_prices(project_id, subscription_name, url):
+    # Create an empty dataframe to store the prices
+    df_new = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
+    print(f"Created a dataframe: {df_new}")
+
+    # Configure the Selenium driver options
+    options = Options()
+    options.add_argument('--disable-gpu')
+    driver = webdriver.Chrome(options=options)
+
+    # Set up Pub/Sub subscription
+    '''
+    project_id = "fp-20230425"
+    subscription_name = "fp-subscription-NFL-zipcodes"
+    '''
+    subscriber = pubsub_v1.SubscriberClient()
+    subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+
+    # Load the product page and extract the __NEXT_DATA__ data
+    # url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
+    product_id = url.split("/")[-1][:5]
+    # product_id = url.split("/")[-1]
+    driver.delete_all_cookies
+    driver.get(url)
+
+    def get_element_text(class_name_prefix):
+        '''
+        Return the text in the first element with a given class name.
+
+        Note that XPath doesn't have a function to check if a given word is present
+        in a list of words, so using contains() will match a class name even if
+        the class name is a substring of the class name being matched against.
+        To prevent this, we use spaces as a delimiter at the beginning and end of
+        the class name, and ensure there's only one space before and after all
+        class names in the class name list.
+
+        Example: consider two class names with cache-busting suffixes:
+        * Header_soldStatusNotice__12345
+        * Header_soldStatusNoticeBackground__ABCDE
+
+        Matching against Header_soldStatusNotice using a naive implementation with
+        `contains()` would match both class names, even though only the first one
+        is the correct match. 
+        '''
+        # Construct xpath selector.  
+        xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+        xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+        xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+        xpath_str += "' '), "                     # Step 4: a trailing space...
+        xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+        try:
+            match = driver.find_element(By.XPATH, xpath_str)
+            return match.get_attribute('textContent')
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            pass
+
+        # We didn't find a match, try again by doing a prefix match and assume
+        # the full class name ends with two underscores followed by a cache-
+        # busting string.    
+        xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+        xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+        xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+        xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+        try:
+            match = driver.find_element(By.XPATH, xpath_str)
+            return match.get_attribute('textContent')
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            pass
+
+        # No match, return empty string.
+        return ''
+
+
+    def modal_close():
+        try:    
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+    def notice_close():
+        try:
+            # Clear any notice
+            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+        except:
+            pass
+
+    def cookie_close():
+        try:  
+            cookie_button = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+            )
+            print(f"cookie_button = {cookie_button}")
+            cookie_button.click()
+        except:
+            pass
+
+    cookie_close()
+    modal_close()
+    # Define callback function for Pub/Sub messages
+    def callback(message):
+        # Extract zip code from message data
+        zip_code = message.message.data.decode('utf-8')
+
+        print(f"Received zip code: {zip_code}")
+
+        try:
+            
+            modal_close()
+            notice_close()
+
+            # Enter zip code in the input field and click the "Check" button
+            zip_button = WebDriverWait(driver, 5).until(
+                EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+            )
+            # print(f"{zip_button:}")
+            zip_button.click()
+
+            # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+            zip_input = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+            )
+            # print(f"{zip_input:}")
+            zip_input.click()
+            # print("clearing zip_input")
+            zip_input.send_keys("\b\b\b\b\b")
+            # print("sending zip code")
+            zip_input.send_keys(zip_code)
+
+            check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+            check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+            # print(f"{check_button:}")
+            check_button.click()
+
+            notice_close()
+            modal_close()
+
+            # Record sales count, if any
+            sales_count_element = get_element_text('Header_count')
+            count_start_index = sales_count_element.index('sold ') + len('sold ')
+            count_end_index = sales_count_element.index('+')
+            sales_count = sales_count_element[count_start_index:count_end_index]
+            # Record any sold status notice
+            sold_status = get_element_text('UnavailableCartbtn_container')
+            if sold_status == 'Notify me':
+                sold_status = 'Sold Out'
+
+        # Extract the price of rice crackers for this zip code
+            # Check if there is a minimum order quantity
+            min_quantity: float = 0
+            total_price: float = 0
+            unit_price: float = 0
+            try:
+                minimum_quantity_str = WebDriverWait(driver, 5).until(EC.presence_of_element_located(((By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML'))))
+                if minimum_quantity_str:
+                    regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
+                    match = re.search(regexp,minimum_quantity_str)
+                    min_quantity, total_price = match.groups()
+            except selenium.common.exceptions.NoSuchElementException as nsee:
+                print("No minimum purhcase string found")
+            except Exception as e:
+                print(f"Exception when finding minimum purchase string: {e}")
+    
+
+
+            ld_json_element = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//script[@type='application/ld+json']")))
+            ld_json_text = ld_json_element.get_attribute('innerHTML')
+            ld_json_start_index = ld_json_text.index('{')
+            ld_json_end_index = ld_json_text.rindex('}') + 1
+            ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
+            ld_json = json.loads(ld_json_json)
+            ld_json_price = ld_json['offers']['price']
+            # Print the price
+            print(ld_json_price)
+
+            unit_price_element = WebDriverWait(driver, 5).until(EC.presence_of_element_located(((By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText'))))
+            unit_price = unit_price_element.replace("$", "")
+            print(f"unit_price = {unit_price}" )
+
+            subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+            # Append the zip code and price to the dataframe
+            #append_to_df(zip_code, price)
+            new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]})
+            df_new = pd.concat([df_new, new_row], ignore_index=True)
+            return df_new
+            #print(f"Recorded the zip code and price: {zip_code},{price}")
+            #print(f"Updated a dataframe: {df_new}")
+
+        except Exception as e:
+            print(f"Caught exception: {e}")
+            print(f"Price not found for zip code {zip_code}")
+            return df_new
+
+    # Start receiving messages (sync version)
+    while True: 
+        pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+        messages = []
+        try:
+            pull_response = subscriber.pull(request = pull_request)
+            messages = pull_response.received_messages
+        except DeadlineExceeded:
+            pass
+
+        if messages:
+            df_new = callback(messages[0])
+            print(f"Updated a dataframe: {df_new}")
+        else:
+            print(f"no messages, trying again at {time.asctime()}")
+            break
+    return df_new
+
+'''
+If using sys to get the function, please remove the following part of code.
+'''
+if __name__ == '__main__':
+    project_id = input("Enter the project ID: ")
+    subscription_name = input("Enter the subscription name: ")
+    url = input("Enter the URL: ")
+
+    get_product_prices(project_id, subscription_name, url)
+
+
+df_new = get_product_prices("fp-20230425", "fp-subscription-NFL-zipcodes", "https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119")
+
+# Now we have the new dataframe "df" recording the new price listed in weee's page
+# Load such data in the existing HDF5 file
+file_path = 'product1.h5'
+with h5py.File(file_path, 'a') as file:
+    # Load the existing dataframe 'df' from the HDF5 file
+    # REPLACE KEY TO GET THE EXISTING DATAFRAME FOR EACH DIFFERENT PRODUCT
+    df = pd.read_hdf(file, key='df')
+
+    # Add timestamp to the new dataframe "df_new"
+    # timestamp = datetime.now()
+    '''
+    To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
+    provides time synchronization with a reliable time server.
+    Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
+    sensible and human-friendly approach to format and convert the timestamps.
+    '''
+    # Fetch the time from an NTP server
+    ntp_client = ntplib.NTPClient()
+    response = ntp_client.request('pool.ntp.org')
+    ntp_time = datetime.fromtimestamp(response.tx_time)
+    timestamp = arrow.get(ntp_time).to('local')
+    df_new['Timestamp'] = timestamp
+
+    # Append 'df_new' to the existing df stored in the key
+    df_update = df.append(df_new, ignore_index=True)
+
+    # Save 'df2' to the HDF5 file with the key 'df'
+    # REPLACE KEY FOR DIFFERENT PRODUCT
+    df_update.to_hdf(file, key='df', mode='a')
+
+print("Data appended and saved successfully.")

fp2_2.py

@@ -0,0 +1,32 @@
+import http.client
+import xml.etree.ElementTree as ET
+
+conn = http.client.HTTPSConnection("www.sayweee.com")
+payload = ''
+headers = {}
+conn.request("GET", "/en/product/Pete-and-Gerry-s-Organic-Large-Brown-Eggs/65902?category=dairy02", payload, headers)
+res = conn.getresponse()
+data = res.read()
+
+# Parse the HTML page content using ElementTree
+root = ET.fromstring(data.decode('utf-8'))
+
+if root is not None:
+    # Find the script element with __NEXT_DATA__ data
+    next_data_element = None
+    for script_element in root.iter('script'):
+        if script_element.text is not None and '__NEXT_DATA__' in script_element.text:
+            next_data_element = script_element
+            break
+
+    # Extract the __NEXT_DATA__ data
+    if next_data_element is not None:
+        next_data_text = next_data_element.txt
+        next_data_start_index = next_data_text.index('{')
+        next_data_end_index = next_data_text.rindex('}') + 1
+        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+        print(next_data_json)
+    else:
+        print('__NEXT_DATA__ element not found')
+else:
+    print('Failed to parse HTML content')

fp2_3.py

@@ -0,0 +1,32 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import json
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--headless')
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Load the page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Pete-and-Gerry-s-Organic-Large-Brown-Eggs/65902?category=dairy02'
+driver.get(url)
+wait = WebDriverWait(driver, 10)
+next_data_element = wait.until(EC.presence_of_element_located((By.XPATH, '//script[contains(text(), "__NEXT_DATA__")]')))
+next_data_text = next_data_element.text
+next_data_start_index = next_data_text.index('{')
+next_data_end_index = next_data_text.rindex('}') + 1
+next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+next_data = json.loads(next_data_json)
+
+# Extract the price from the __NEXT_DATA__ data
+price = next_data['props']['pageProps']['product']['price']
+
+# Close the Selenium driver
+driver.quit()
+
+# Print the price
+print(price)

fp2_4.py

@@ -0,0 +1,43 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+
+# Configure the Selenium driver options
+options = Options()
+#options.add_argument('--headless')  
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Load the page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+driver.get(url)
+time.sleep(5)
+next_data_element = None
+#try:
+next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+#except:
+ #   print('__NEXT_DATA__ element not found')
+
+#time.sleep(1000)
+if next_data_element is not None:
+    next_data_text = next_data_element.get_attribute('innerHTML')
+    print(next_data_text)
+    next_data_start_index = next_data_text.index('{')
+    next_data_end_index = next_data_text.rindex('}') + 1
+    next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+    next_data = json.loads(next_data_json)
+
+    # Extract the price from the __NEXT_DATA__ data
+ #   price = next_data['props']['pageProps']['product']['price']
+    print(next_data)
+    price = next_data['offers']['price']
+    # Print the price
+    print(price)
+else:
+    print('__NEXT_DATA__ element not found')
+
+time.sleep(1000)
+# Close the Selenium driver
+#driver.quit()

fp2_5.py

@@ -0,0 +1,33 @@
+import re
+import json
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--headless')
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Load the page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+driver.get(url)
+next_data_element = None
+for script_element in driver.find_elements(By.XPATH, '//script'):
+    script_text = script_element.get_attribute('innerHTML')
+    match = re.search(r'__NEXT_DATA__ = (\{.*\});', script_text)
+    if match:
+        next_data_json = match.group(1)
+        next_data = json.loads(next_data_json)
+        # Extract the price from the __NEXT_DATA__ data
+        price = next_data['props']['pageProps']['product']['price']
+        # Print the price
+        print(price)
+        break
+
+if next_data_element is None:
+    print('__NEXT_DATA__ element not found')
+
+# Close the Selenium driver
+driver.quit()

fp2_6.py

@@ -0,0 +1,126 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Define the zip codes for which to extract the price
+zip_codes = ['85305', '30313', '21230', '14127','28202','60605','45202','44114','76011','80204','48226','54304','77054','46225','32202','64129','89118','90301','33056','55415','02035','70112','07073','19148','15212','95054','98134','33607','37213','20785']
+
+# Load the page and extract the __NEXT_DATA__ data
+#url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+url = 'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261'
+#url = 'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467'
+#url = 'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01'
+#url = 'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224'
+#url = 'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02'
+#url = 'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01'
+#url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
+#url = 'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353?trace_id=7847dd6e-c511-48e2-a76e-a283399967b1'
+#url = 'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03'
+
+driver.delete_all_cookies
+driver.get(url)
+
+# Close the cookies setting window
+try:  
+    cookie_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")))
+    print(f"cookie_button = {cookie_button}")
+    cookie_button.click()
+except:
+    pass
+
+# Clear any modal dialog
+try:    
+    modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+    print(f"modal_close = {modal_close}")
+    modal_close.click()
+except:
+    pass
+
+def callback(zip_code):
+    try:
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+
+        try:
+            # Clear any notice
+            notice_close = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+        except:
+            pass
+
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+        # Enter zip code in the input field and click the "Check" button
+        zip_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')))
+        # print(f"{zip_button:}")
+        zip_button.click()
+
+        zip_input = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input')))
+        # print(f"{zip_input:}")
+        zip_input.click()
+        # print("clearing zip_input")
+        zip_input.send_keys("\b\b\b\b\b")
+        # print("sending zip code")
+        zip_input.send_keys(zip_code)
+
+        try:
+            check_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//button[text() = 'Confirm']")))
+            # print(f"{check_button:}")
+        except:
+            check_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[5]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[3]/div/button")))
+        finally:
+            check_button.click()
+
+        try:
+            # Clear any notice
+            notice_close = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+        except:
+            pass
+
+        try:
+            # Clear any modal dialog
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            # print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+            # Extract the price of rice crackers for this zip code
+        try:
+            sold_out_notice = driver.find_element(By.XPATH, '//*[@id="soldStatusNotice"]')
+            print(f"No such product in this area = {zip_code, sold_out_notice}")
+        except:
+            print(f"The product is available in this area {zip_code}")
+    
+    except Exception as e:
+        print(f"Caught exception: {e}")
+        print(f"Price not found for zip code {zip_code}")
+
+for zip_code in zip_codes:
+    callback(zip_code)
+
+driver.quit()

fp2_7.py

@@ -0,0 +1,64 @@
+from google.cloud import pubsub_v1
+import csv
+import os
+import asyncio
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+# project_id = "fp-20230425"
+project_id = "svig-384801"
+
+'''
+topic_name = "fp-topic-NFL-zipcodes"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
+subscriber.delete_subscription(request={"subscription": subscription_path})
+print("Subscription deleted: {}".format(subscription_path))
+
+# Delete the topic
+publisher = pubsub_v1.PublisherClient()
+topic_path = publisher.topic_path(project_id, topic_name)
+publisher.delete_topic(request={"topic": topic_path})
+print(f"Topic '{topic_name}' deleted.")
+
+'''
+
+
+# Create a pubsub topic
+publisher = pubsub_v1.PublisherClient()
+topic_name = "fp-topic-NFL-zipcodes"
+topic_path = publisher.topic_path(project_id, topic_name)
+topic = publisher.create_topic(request={"name": topic_path}, timeout=30)
+
+
+# Create a Pub/Sub subscription
+subscriber = pubsub_v1.SubscriberClient()
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+subscription = subscriber.create_subscription(
+    name=subscription_path, topic=topic_path
+)
+'''
+# Publish a message to the topic
+# Read the data from the csv file
+# file_path = "C:/Users/Karen/Desktop/NFL_Cities.csv"
+file_path = zipcodes.csv
+
+with open(file_path, "r") as csv_file:
+    reader = csv.reader(csv_file)
+    next(reader)  # skip header row
+    for row in reader:
+        # Format the data as a string and publish it to the topic
+        data = ",".join(row).encode("utf-8")
+        future = publisher.publish(topic_path, data)
+        print(f"Published message ID {future.result()}")
+
+
+async def receive_messages():
+    response = subscriber.pull(subscription=subscription_path, max_messages=1)
+    for received_message in response.received_messages:
+        print("Received message: {}".format(received_message.message.data))
+       
+    
+asyncio.run(receive_messages())
+'''

fp2_8.py

@@ -0,0 +1,55 @@
+import json
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import time
+from google.cloud import pubsub_v1
+import os
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+
+options = Options()
+options.add_argument('--headless')
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+def callback(message):
+    try:
+        # Get the zip code from the message
+        zip_code = message.data.decode('utf-8')
+        print(f"Received zip code: {zip_code}")
+        
+        # Load the webpage with the zip code
+        url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+        driver.get(url)
+        driver.find_element(By.XPATH, '/html/body/div[4]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[2]/input').send_keys(zip_code)
+        driver.find_element(By.XPATH, '/html/body/div[4]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[3]/div').click()
+        
+        # Extract the price from the webpage
+        next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        next_data_text = next_data_element.get_attribute('innerHTML')
+        next_data_start_index = next_data_text.index('{')
+        next_data_end_index = next_data_text.rindex('}') + 1
+        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+        next_data = json.loads(next_data_json)
+        price = next_data['offers']['price']
+        
+        # Print the result
+        print(f"Price of rice crackers in zip code {zip_code}: {price}")
+    except Exception as e:
+        print(f"Error processing message: {str(e)}")
+    finally:
+        message.ack()
+
+subscriber.subscribe(subscription_path, callback=callback)
+
+while True:
+    time.sleep(1)
+
+

fp2_9.py

@@ -0,0 +1,106 @@
+from google.cloud import pubsub_v1
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+import os
+from concurrent.futures import TimeoutError
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+# project_id = "fp-20230425"
+project_id = "svig-384801"
+subscription_name = "fp-subscription-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+timeout = 5
+
+# Load the product page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+driver.get(url)
+# time.sleep(5)
+'''
+next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+if next_data_element is not None:
+    next_data_text = next_data_element.get_attribute('innerHTML')
+    next_data_start_index = next_data_text.index('{')
+    next_data_end_index = next_data_text.rindex('}') + 1
+    next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+    next_data = json.loads(next_data_json)
+else:
+    print('__NEXT_DATA__ element not found')
+'''
+    
+# Define callback function for Pub/Sub messages
+def callback(message):
+    # Extract zip code from message data
+    zip_code = message.data.decode('utf-8')
+    print(f"Received zip code: {zip_code}")
+
+    try:
+
+        time.sleep(5)
+        # Clear any modal dialog
+        modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
+        print(f"modal_close = {modal_close}")
+        modal_close.click()
+        time.sleep(2)
+
+        # Enter zip code in the input field and click the "Check" button
+        zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]/div/div/div[1]')
+        print(zip_button)
+        zip_button.click()
+        time.sleep(1)
+
+        zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+        zip_input.clear()
+        zip_input.send_keys(zip_code)
+
+        check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+        check_button.click()
+        time.sleep(2)
+
+        # Extract the price of rice crackers for this zip code
+        '''
+        price_element = driver.find_element(By.XPATH, '/html/body/div[2]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')
+        price = price_element.text.strip()[1:]
+        print(price)
+        '''
+        next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        next_data_text = next_data_element.get_attribute('innerHTML')
+        next_data_start_index = next_data_text.index('{')
+        next_data_end_index = next_data_text.rindex('}') + 1
+        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+        next_data = json.loads(next_data_json)
+        price = next_data['offers']['price']
+    # Print the price
+        print(price)
+        message.ack()  # acknowledge receipt of message
+    except:
+        print(f"Price not found for zip code {zip_code}")
+        price = 'Price not found'
+
+# Start receiving messages
+streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback)
+print(f'Started receiving messages from subscription {subscription_path}...')
+
+# Wrap subscriber in a 'with' block to automatically call close() when done.
+with subscriber:
+    try:
+        # When `timeout` is not set, result() will block indefinitely,
+        # unless an exception is encountered first.
+        streaming_pull_future.result(timeout=timeout)
+    except TimeoutError:
+        streaming_pull_future.cancel()  # Trigger the shutdown.
+        streaming_pull_future.result()  # Block until the shutdown is complete.
+
+# Keep the script running indefinitely to continue receiving messages
+while True:
+    time.sleep(10)

fp3_1.py

@@ -0,0 +1,166 @@
+import sqlalchemy
+from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Float, DateTime
+from sqlalchemy.orm import declarative_base
+from sqlalchemy.orm import relationship, sessionmaker
+import sys
+from flask import Flask, render_template
+
+# Create the Flask application
+app = Flask(__name__)
+
+# Create the database engine
+engine = create_engine('sqlite:///scraper_design.db')
+
+# Create a session factory
+Session = sessionmaker(bind=engine)
+
+# Create a base class
+Base = declarative_base()
+
+# Define the product class
+class Product(Base):
+    __tablename__ = 'product'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    SKU_PLU_UPC_number = Column(String)
+    price_histories = relationship("PriceHistory", back_populates="product")
+
+# Define the retailer class
+class Retailer(Base):
+    __tablename__ = 'retailer'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    products = relationship("Product", secondary="retailer_product")
+
+# Define the price class
+class Price(Base):
+    __tablename__ = 'price'
+    id = Column(Integer, primary_key=True)
+    MOQ = Column(Integer)
+    price = Column(Float)
+
+# Define the scraperjob class
+class ScraperJob(Base):
+    __tablename__ = 'scraper_job'
+    id = Column(Integer, primary_key=True)
+    start_time = Column(DateTime)
+    end_time = Column(DateTime)
+    parameters = Column(String)
+    statistics = Column(String)
+
+# Define the price history class
+class PriceHistory(Base):
+    __tablename__ = 'price_history'
+    id = Column(Integer, primary_key=True)
+    location = Column(String)
+    zip_code = Column(Integer)
+    city_name = Column(String)
+    price_at_time = Column(DateTime)
+    price = Column(Float)
+    scraper_job_id = Column(Integer, ForeignKey('scraper_job.id'))
+    product_id = Column(Integer, ForeignKey('product.id'))
+    product = relationship("Product", back_populates="price_histories")
+    scraper_job = relationship("ScraperJob")
+
+# Define the user class
+class User(Base):
+    __tablename__ = 'user'
+    id = Column(Integer, primary_key=True)
+    username = Column(String)
+    credentials = Column(String)
+    creation_date = Column(DateTime)
+    modification_date = Column(DateTime)
+    def __str__(self):
+        return f"{self.id=} {self.username=} {self.creation_date=} {self.modification_date=}"
+
+class RetailerProduct(Base):
+    __tablename__ = 'retailer_product'
+    retailer_id = Column(Integer, ForeignKey('retailer.id'), primary_key=True)
+    product_id = Column(Integer, ForeignKey('product.id'), primary_key=True)
+
+# Create the database tables
+Base.metadata.create_all(engine)
+
+@app.route('/')
+def homepage():
+    # Create a session
+    session = Session()
+
+    # Get all the classes from SQLAlchemy
+    classes = Base.__subclasses__()
+
+    statement = sqlalchemy.select(User)
+    users = session.scalars(statement).all()
+
+    # Render the template with the classes
+    return render_template('test3.html', classes=classes, users=users)
+
+# Create command-line arguments for the user class
+def main():
+    if len(sys.argv) < 2:
+        app.run()
+
+    command = sys.argv[1]
+
+    if command == "create":
+        if len(sys.argv) != 3:
+            print("Usage: python user_tool.py create [username]")
+            return
+
+        username = sys.argv[2]
+
+        # Create a new user object
+        user = User(username=username)
+
+        # Add the user to the database
+        session = Session()
+        session.add(user)
+        session.commit()
+
+        print("User created successfully.")
+
+    elif command == "edit":
+        if len(sys.argv) != 4:
+            print("Usage: python user_tool.py edit [user_id] [new_username]")
+            return
+
+        user_id = int(sys.argv[2])
+        new_username = sys.argv[3]
+
+        # Retrieve the user from the database
+        session = Session()
+        user = session.query(User).get(user_id)
+
+        if user:
+            # Update the user object
+            user.username = new_username
+            session.commit()
+            print("User updated successfully.")
+        else:
+            print("User not found.")
+
+    elif command == "delete":
+        if len(sys.argv) != 3:
+            print("Usage: python user_tool.py delete [user_id]")
+            return
+
+        user_id = int(sys.argv[2])
+
+        # Retrieve the user from the database
+        session = Session()
+        user = session.query(User).get(user_id)
+
+        if user:
+            # Delete the user from the database
+            session.delete(user)
+            session.commit()
+            print("User deleted successfully.")
+        else:
+            print("User not found.")
+
+    else:
+        print("Usage: python user_tool.py [create|edit|delete]")
+        return
+
+if __name__ == "__main__":
+    main()

fp_merge.py

@@ -0,0 +1,100 @@
+from fp2_14 import get_product_prices
+import h5py
+import pandas as pd
+import datetime
+import sys
+from datetime import datetime
+import ntplib
+import arrow
+import concurrent.futures
+import pandas as pd
+
+# Define the subset of urls
+url_list = [
+    'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
+    'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
+    'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
+    'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
+    'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
+    'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
+    'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
+    'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
+    'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
+    'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
+    'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
+]
+
+
+# Define the function to run the scraper
+def run_scraper(url):
+    project_id = "fp-20230425"
+    subscription_name = "fp-subscription-NFL-zipcodes"
+    df_new = get_product_prices(project_id, subscription_name, url)
+
+    # Add timestamp to the new dataframe "df_new"
+    '''
+    To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
+    provides time synchronization with a reliable time server.
+    Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
+    sensible and human-friendly approach to format and convert the timestamps.
+    '''
+    # Fetch the time from an NTP server
+    ntp_client = ntplib.NTPClient()
+    response = ntp_client.request('pool.ntp.org')
+    ntp_time = datetime.fromtimestamp(response.tx_time)
+    timestamp = arrow.get(ntp_time).to('local')
+    df_new['Timestamp'] = timestamp
+
+    # Append 'df_new' to the existing df stored in the key
+    df_update = df_new.append(df_new, ignore_index=True)
+
+    return df_update
+
+
+'''
+# When using sys to call the 'get_product_prices' function
+
+if __name__ == '__main__':
+    # Check if the required command-line arguments are provided
+    if len(sys.argv) < 4:
+        print("Usage: python test_fp2_14.py project_id subscription_name url")
+        sys.exit(1)
+
+    project_id = sys.argv[1]
+    subscription_name = sys.argv[2]
+    url = sys.argv[3:]
+
+    get_product_prices(project_id, subscription_name, url)
+'''
+
+
+if __name__ == '__main__':
+    # Determine the number of available CPUs
+    # cpu_count = multiprocessing.cpu_count()
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        future_to_url = {executor.submit(run_scraper, url): url for url in url_list}
+        results = []
+        for future in concurrent.futures.as_completed(future_to_url):
+            url = future_to_url[future]
+            try:
+                result = future.result()
+                print(f"Scraped data for URL: {url}")
+                results.append(result)
+            except Exception as e:
+                print(f"Failed to scrape data for URL: {url}\nError: {e}")
+
+
+# Now we have the new dataframe "df_update" recording the new price listed in weee's page
+# Load such data in the existing HDF5 file
+file_path = 'product1.h5'
+with pd.HDFStore(file_path) as store:
+    for url, result in zip(url_list, results):
+        # Extract the product ID from the URL
+        product_id = url.split("/")[-1][:5]
+
+        # Save the dataframe to the HDF5 file with a unique key
+        key = f'df_{product_id}'
+        store[key] = result
+        result.to_hdf('product_data.h5', key=key, mode='a')
+        print("Data appended and saved successfully.")

hello.py

@@ -0,0 +1 @@
+print("hello")

http-performance/Webpage.ipynb

@@ -1,78 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "0593e529",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "from urllib.parse import urlparse\n",
-    "\n",
-    "class JPEGRequestHandler:\n",
-    "    def __init__(self, app):\n",
-    "        self.app = app\n",
-    "\n",
-    "    def __call__(self, environ, start_response):\n",
-    "        url = urlparse(environ['REQUEST_URI'])\n",
-    "        path = url.path\n",
-    "\n",
-    "        if path.endswith('.jpg'):\n",
-    "            file_path = path[1:]\n",
-    "\n",
-    "            if os.path.isfile(file_path):\n",
-    "                with open(file_path, 'rb') as file:\n",
-    "                    response_body = file.read()\n",
-    "                    status = '200 OK'\n",
-    "                    headers = [('Content-type', 'image/jpeg'),\n",
-    "                               ('Content-Length', str(len(response_body)))]\n",
-    "\n",
-    "                    start_response(status, headers)\n",
-    "                    return [response_body]\n",
-    "            else:\n",
-    "                status = '404 Not Found'\n",
-    "                headers = [('Content-type', 'text/plain'),\n",
-    "                           ('Content-Length', '0')]\n",
-    "\n",
-    "                start_response(status, headers)\n",
-    "                return []\n",
-    "        else:\n",
-    "            status = '403 Forbidden'\n",
-    "            headers = [('Content-type', 'text/plain'),\n",
-    "                       ('Content-Length', '0')]\n",
-    "\n",
-    "            start_response(status, headers)\n",
-    "            return []\n",
-    "\n",
-    "if __name__ == '__main__':\n",
-    "    from werkzeug.serving import run_simple\n",
-    "    from werkzeug.middleware.dispatcher import DispatcherMiddleware\n",
-    "\n",
-    "    app = DispatcherMiddleware(JPEGRequestHandler(app), {'/': app})\n",
-    "    run_simple('0.0.0.0', 8000, app, use_reloader=True)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.13"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

http-performance/app.py

@@ -1,44 +0,0 @@
-import os
-from urllib.parse import urlparse
-
-class JPEGRequestHandler:
-    def __init__(self, app):
-        self.app = app
-
-    def __call__(self, environ, start_response):
-        url = urlparse(environ['REQUEST_URI'])
-        path = url.path
-
-        if path.endswith('.jpg'):
-            file_path = path[1:]
-
-            if os.path.isfile(file_path):
-                with open(file_path, 'rb') as file:
-                    response_body = file.read()
-                    status = '200 OK'
-                    headers = [('Content-type', 'image/jpeg'),
-                               ('Content-Length', str(len(response_body)))]
-
-                    start_response(status, headers)
-                    return [response_body]
-            else:
-                status = '404 Not Found'
-                headers = [('Content-type', 'text/plain'),
-                           ('Content-Length', '0')]
-
-                start_response(status, headers)
-                return []
-        else:
-            status = '403 Forbidden'
-            headers = [('Content-type', 'text/plain'),
-                       ('Content-Length', '0')]
-
-            start_response(status, headers)
-            return []
-
-if __name__ == '__main__':
-    from werkzeug.serving import run_simple
-    from werkzeug.middleware.dispatcher import DispatcherMiddleware
-
-    app = DispatcherMiddleware(JPEGRequestHandler(app), {'/': app})
-    run_simple('0.0.0.0', 8000, app, use_reloader=True)

http-performance/testhh.py

@@ -0,0 +1,54 @@
+'''
+import os
+from flask import Flask, Response
+from urllib.parse import urlparse
+
+app = Flask(__name__)
+
+@app.route('/')
+def index():
+    return 'Hello, welcome to the cat picture server!'
+
+@app.route('/<path:path>')
+def serve_jpeg(path):
+    if path.endswith('.jpg'):
+        file_path = path[1:]
+        if os.path.isfile(file_path):
+            with open(file_path, 'rb') as file:
+                return Response(file.read(), content_type='image/jpeg')
+        else:
+            return 'File not found', 404
+    else:
+        return 'Only .jpg files are allowed', 403
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=8000)
+'''
+'''
+import os
+from fastapi import FastAPI
+from fastapi.responses import FileResponse
+from urllib.parse import urlparse
+
+app = FastAPI()
+
+@app.get('/')
+def index():
+    return 'Hello, welcome to the cat picture server!'
+
+@app.get('/{path:path}')
+async def serve_jpeg(path: str):
+    if path.endswith('.jpg'):
+        file_path = path[1:]
+        if os.path.isfile(file_path):
+            return FileResponse(file_path, media_type='image/jpeg')
+        else:
+            return {'detail': 'File not found'}, 404
+    else:
+        return {'detail': 'Only .jpg files are allowed'}, 403
+
+if __name__ == '__main__':
+    import uvicorn
+    uvicorn.run(app, host='0.0.0.0', port=8000)
+'''
+

hw10.py

@@ -0,0 +1,14 @@
+import sys
+
+# Iterate through the modules in sys.modules
+for module_name, module in sys.modules.items():
+    # Check if the module has a loader attribute
+    if hasattr(module, '__loader__'):
+        loader = module.__loader__
+        # Determine the type of loader
+        if type(loader).__name__ == 'SourceFileLoader':
+            print(f"Module '{module_name}' has a source loader.")
+        elif type(loader).__name__ == 'BuiltinImporter':
+            print(f"Module '{module_name}' has a built-in loader.")
+        elif type(loader).__name__ == 'ExtensionFileLoader':
+            print(f"Module '{module_name}' has an extension loader.")

hw11.py

@@ -0,0 +1,12 @@
+from cx_Freeze import setup, Executable
+
+executables = [Executable('module-inspector.py')]
+
+setup(
+    name='wow',
+    version='1.0',
+    description='Module-inspector Application',
+    executables=executables
+)
+
+

hw12.py

@@ -0,0 +1,7 @@
+i = 5
+
+def f(arg=i):
+    print(arg)
+
+i = 6
+f()

hw8.py

@@ -0,0 +1,47 @@
+import importlib.util
+import os
+
+def get_module_size(module_name):
+    """Get the total number of bytes in the cached .pyc file or .py file of a module or package.
+
+    Args:
+        module_name (str): The name of the module or package.
+
+    Returns:
+        int: The total number of bytes in the cached .pyc file or .py file.
+    """
+    spec = importlib.util.find_spec(module_name)
+
+    if spec is not None and spec.has_location:
+        # If the module is found and has a location, check for .pyc file first
+        pyc_file_path = spec.origin + 'c'
+        if os.path.exists(pyc_file_path):
+            return os.stat(pyc_file_path).st_size
+        else:
+            # If .pyc file doesn't exist, check for .py file
+            py_file_path = spec.origin
+            if os.path.exists(py_file_path):
+                return os.stat(py_file_path).st_size
+
+    return 0  # Return 0 if no .pyc or .py file found
+
+
+import sys
+
+def get_total_module_size():
+    """Get the total number of bytes for all modules listed in sys.modules.
+
+    Returns:
+        int: The total number of bytes for all modules.
+    """
+    total_size = 0
+    for module_name in sys.modules:
+        module_size = get_module_size(module_name)
+        total_size += module_size
+
+    return total_size
+
+if __name__ == '__main__' and __package__ is None:
+    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
+total_size = get_total_module_size()
+print("Total size of all modules in sys.modules: {} bytes".format(total_size))

hw9.py

@@ -0,0 +1,74 @@
+import os
+import sys
+import importlib.util
+import types
+from collections import defaultdict
+from sys import getsizeof
+
+
+def get_bytecode_size(module_name):
+    """Get the size of the cached .pyc bytecode file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin and spec.origin.endswith('.py'):
+        pyc_file = importlib.util.cache_from_source(spec.origin)
+        if os.path.exists(pyc_file):
+            return os.path.getsize(pyc_file)
+    return 0
+
+
+def get_dso_size(module_name):
+    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin and spec.origin.endswith(('.so', '.dll')):
+        return os.path.getsize(spec.origin)
+    return 0
+
+
+def generate_module_report():
+    """Generate a report summarizing the module environment."""
+    total_modules = len(sys.modules)
+    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
+    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
+    python_size = sum(
+        getsizeof(sys.modules[module_name]) - getsizeof(module_name)
+        for module_name in sys.modules
+        if isinstance(sys.modules[module_name], types.ModuleType)
+    )
+    total_size = bytecode_size + dso_size + python_size
+
+    # Calculate percentage of each type of module
+    bytecode_percent = bytecode_size / total_size * 100
+    dso_percent = dso_size / total_size * 100
+    python_percent = python_size / total_size * 100
+
+    # Generate the report
+    report = f"Total modules: {total_modules}\n"
+    report += f"Total size: {total_size} bytes\n"
+    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
+    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
+    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
+
+    # Generate histogram of module directory names
+    dir_histogram = defaultdict(int)
+    for module_name in sys.modules:
+        module_file = getattr(sys.modules[module_name], '__file__', None)
+        if module_file:
+            module_dir = os.path.dirname(module_file)
+            dir_histogram[module_dir] += 1
+
+    report += "\nModule Directory Histogram:\n"
+    for module_dir, count in dir_histogram.items():
+        report += f"{module_dir}: {'*' * count}\n"
+
+    return report
+
+
+if __name__ == "__main__":
+    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
+report = generate_module_report()
+print(report)
+
+
+
+
+

lexer_parser.py

@@ -1,76 +0,0 @@
-from enum import Enum
-
-class Tokens(Enum):
-    QUOTE = 1,
-    CHAR = 2,
-    NEWLINE = 3,
-    SPACE = 4
-
-class Lexer:
-    tokens = list()
-    def __init__(self):
-        pass
-    def lex(self, str):
-        for c in str:
-            if c == "'" or c == '"':
-                self.tokens.append(Tokens.QUOTE)
-                print("Token: QUOTE")
-            elif c == "\n":
-                self.tokens.append(Tokens.NEWLINE)
-                print("Token: NEWLINE")
-            elif c == " ":
-                self.tokens.append(Tokens.SPACE)
-                print("Token: SPACE")
-            else:
-                self.tokens.append(Tokens.CHAR)
-                print("Token: CHAR")
-    def __str__(self):
-        return ",".join(Tokens(token).name for token in self.tokens)
-
-class States(Enum):
-    INITIAL = 0,
-    BEGINQUOTE = 1
-    
-class Parser:
-    def __init__(self):
-        self.state = States.INITIAL
-        self.line = 1
-
-    def transition(self, state):
-        print(f"{States(self.state).name} -> {States(state).name}")
-        self.state = state
-
-    def parse(self, tokens):
-        for token in tokens:
-            if self.state == States.INITIAL:
-                if token == Tokens.QUOTE:
-                    self.transition(States.BEGINQUOTE)
-                elif token == Tokens.NEWLINE:
-                    self.line += 1
-                continue
-            if self.state == States.BEGINQUOTE:
-                if token == Tokens.QUOTE:
-                    self.transition(States.INITIAL)
-                elif token == Tokens.CHAR or token == Tokens.SPACE:
-                    pass                    
-                elif token == Tokens.NEWLINE:
-                    raise ValueError(f"Unterminated string on line {self.line}")
-                continue
-
-if __name__ == "__main__":
-    test1 = """abcdefg
-    "abcde
-    'a'
-    bareword
-"""
-
-    lexer = Lexer()
-    lexer.lex(test1)
-    # print(lexer)
-    parser = Parser()
-    try:
-        parser.parse(lexer.tokens)
-    except ValueError as error:
-        print(error)
-
-

max submission/5_26_HW.py

@@ -1,20 +0,0 @@
-# Question7:
-# Difference: 
-# 1. Byte objects are sequence of Bytes, whereas Strings are sequence of characters. 
-# 2. Byte objects are in machine readable form internally; Strings are only in human readable form. 
-# 3. strings (str) support various character encodings, such as UTF-8, ASCII, and more. Bytes (bytes), on the other hand, represent raw binary data and can hold any byte values. 
-## Emojis can affect the size difference.
-import sys
-string_size = sys.getsizeof("Hello")
-bytes_size = sys.getsizeof(b"Hello")
-emoji_size = sys.getsizeof("😀")
-print(f"String size: {string_size} bytes")
-print(f"Bytes size: {bytes_size} bytes")
-print(f"emoji size: {emoji_size} bytes")
-
-# python routine:
-def print_string_as_numbers(string):
-    for char in string:
-        print(ord(char))
-input_string = "Hello, world!"
-print_string_as_numbers(input_string)

max submission/5_31.HW.py

@@ -1,38 +0,0 @@
-# question 8: 	If the column or data type is invalid, and we take them into calculations without noticing this, wrong number will be produced.
-# question 9：	
-def read_tomato_sales(file_name):
-        data = pd.read_csv(file_name)
-        if "Tomato_Sales" in data.columns and data["Tomato_Sales"].dtype == "int64":
-            total_sales = data["Tomato_Sales"].sum()
-            print(f"Total tomato sales: {total_sales}")
-        else:
-            raise ValueError("Invalid column or data type.")
-# question 10:	Reasons: 
-# 1. Different languages may have different character encodings, which define how characters are represented 
-# as binary data. By using bytes, you can handle various character encodings, such as UTF-8, UTF-16, or others. 
-# 2. Bytes are language-agnostic and treat text as raw binary data. 
-# 3. Bytes can handle any kind of binary data, including text data in different languages, as well as
-#  non-textual data such as images, audio, or video.
-def process_multilingual_data(data):
-    # Convert string to bytes using UTF-8 encoding
-    byte_data = data.encode('utf-8')
-
-    # Convert bytes back to string for display or further processing
-    processed_data = byte_data.decode('utf-8')
-
-    return processed_data
-
-# Example usage
-text = "This is shunan zhao"
-processed_text = process_multilingual_data(text)
-print(processed_text)
-# 1: ord("😀") gives you the Unicode code point of the character, while 0xf09f9880 represents the byte 
-# sequence in UTF-8 encoding used to represent that character.
-# 2.5: A REPL  is an interactive programming environment that allows you to enter commands or expressions, 
-# which are then evaluated and the results are printed or displayed. 
-# 3: the "_" variable is a special variable that holds the result of the last executed expression or 
-# statement. It is often referred to as the "underscore variable."
-#The type can vary based on the context and the value assigned to  in the current REPL session. For example:
-# If the last executed expression was an integer, the type of "_" would be int.
-# If the last executed expression was a string, the type of "_" would be str.
-# If the last executed expression was a list, the type of "_" would be list.

max submission/6_5_HW.py

@@ -1,53 +0,0 @@
-#question 1:
-# i = 3 Literal Type: Integer, and its value is 3.
-
-# f = 3.0 Literal Type: Float, and its value is 3.0.
-
-# c = 3+3j Literal Type: Complex, and its value is 3+3j.
-
-# b = b'abcdefg' Literal Type: Bytes, and its value is b'abcdefg'.
-
-# s = 'abcdefg' Literal Type: String, and its value is 'abcdefg'.
-
-# s = "abcdefg" Literal Type: String, and its value is "abcdefg".
-
-#question 2: 
-'''
-	'0'  	   b'0'	     0	        0.0        0j
-'0'	Equal	  |Not equal |Not equal	|Not equal |Not equal
-b'0'Not equal |Equal	 |Not equal	|Not equal |Not equal
-0	Not equal |Not equal |Equal	    |Equal	   |Equal
-0.0	Not equal |Not equal |Equal	    |Equal	   |Not Equal
-0j	Not equal |Not equal |Equal	    |Not equal |Equal
-'''
-#question 3:
-##The "+" operator case:
-'''
-	 '0'	   b'0'	      0	         0.0	    0j
-'0'	 '00'	   |TypeError |TypeError |TypeError |TypeError
-b'0' TypeError |b'00'	  |TypeError |TypeError	|TypeError
-0	 TypeError |TypeError |0         |0.0	    |0j
-0.0	 TypeError |TypeError |0.0	     |0.0	    |TypeError
-0j	 TypeError |TypeError |0j	     |TypeError	|0j
-'''
-##The "==" operator case:
-'''
-	 '0'	b'0'	0	    0.0	    0j
-'0'	 True	|False	|False	|False	|False
-b'0  False	|False	|False	|False	|False
-0	 False	|False	|False	|True	|True
-0.0	 False	|False	|True	|True	|False
-0j	 False	|False	|True	|False	|True
-'''
-#question4:
-import ast
-
-# Open the lexer/parser example file
-with open('lexer_parser_example.py', 'r') as file:
-    source_code = file.read()
-
-# Parse the source code into an AST
-tree = ast.parse(source_code)
-
-# Dump the AST
-print(ast.dump(tree, indent=4))

max submission/6_6_HW.py

@@ -1,34 +0,0 @@
-# question 5:
-# The statement print("\007") outputs an ASCII escape sequence \007, which represents the Bell control character. The reason why print("\007") 
-# produces a sound or visual alert is that when the string "\007" is printed, it is interpreted by the terminal or system as a signal to generate 
-# an audible or visual alert, as defined by the specific implementation.
-
-# question 6:
-# The code print("abcdefg\rhijklmnop") will output "hijklmnopg"
-# In Python, the '\r' character is a carriage return character, which instructs the console or terminal to move the cursor back to the beginning 
-# of the current line, without advancing to the next line. This means that anything printed after the '\r' will overwrite whatever was printed earlier 
-# on the same line.
-
-# question 7:
-# In this case, the carriage return character \r is treated as a regular character within the response content. It does not have the same effect as 
-# in the previous example where it was used in a print statement.
-
-# qustion 8:
-# If a hacker attacks a credit card system. When the customer came to buy a can of Coke, he paid 2 knives to the store, but the display in the 
-# background was "2/r500".
-
-# question 9:
-def validate(input_string):
-    if "\r" in input_string:
-        raise ValueError("Crime detected!")
-try:
-    validate("abcdefg")
-    print("✅ test1 pass!")
-except:
-    print("❌ test1 fail!")
-
-try:
-    validate("abcdefg\rhijklmnop")
-    print("❌ test2 fail!")
-except ValueError as err:
-    print("✅ test2 pass!")

max submission/6_7_HW.py

@@ -1,77 +0,0 @@
-class BoundedInt():
-    def __init__(self, i, min_val, max_val):
-        self.i = i
-        self.min = min_val
-        self.max = max_val
-
-    def __int__(self):
-        return self.i
-
-    def __sub__(self, other):
-        if isinstance(other, BoundedInt):
-            result = self.i - other.i
-        else:
-            result = self.i - other
-        if result < self.min:
-            raise ValueError("Result exceeds min value")
-        return BoundedInt(result, self.min, self.max)
-
-    def __add__(self, other):
-        if isinstance(other, BoundedInt):
-            result = self.i + other.i
-        else:
-            result = self.i + other
-        if result > self.max:
-            raise ValueError("Result exceeds max value")
-        return BoundedInt(result, self.min, self.max)
-
-    def test1(self):
-        bi1 = BoundedInt(0xdeadbeef, min_val=(0xdeadbeef - 1), max_val=(0xdeadbeef + 1))
-        if int(bi1) == 0xdeadbeef:
-            print(f"✅ Test 1 passed!")
-        else:
-            print(f"❌ Test 1 failed, int(bi1) = {int(bi1)}, expected 0xdeadbeef")
-
-    def test2(self):
-        bi2 = BoundedInt(1, min_val=0, max_val=1)
-        bi3 = BoundedInt(1, min_val=0, max_val=1)
-        try:
-            bi4 = bi2 + bi3  # 1 + 1 = 2, which exceeds max_val = 1
-            print(f"❌ Test 2 failed!")
-        except ValueError as err:
-            print(f"✅ Test 2 passed!")
-
-    def test3(self):
-        bi4 = BoundedInt(1, min_val=0, max_val=1)
-        bi5 = BoundedInt(2, min_val=0, max_val=-1)
-        try:
-            bi6 = bi4 - bi5  # 1 - 2 = -1, which is less than min_val = 0
-            print(f"❌ Test 3 failed!")
-        except ValueError as err:
-            print(f"✅ Test 3 passed!")
-
-    def test4(self):
-        bi7 = BoundedInt(1, min_val=0, max_val=1)
-        bi8 = 2
-        try:
-            bi9 = bi7 + bi8
-            print(f"❌ Test 4 failed!")
-        except ValueError as err:
-            print(f"✅ Test 4 passed!")
-
-    def test5(self):
-        bi10 = BoundedInt(1, min_val=0, max_val=1)
-        bi11 = 2
-        try:
-            bi12 = bi10 - 11
-            print(f"❌ Test 5 failed!")
-        except ValueError as err:
-            print(f"✅ Test 5 passed!")
-
-
-tester = BoundedInt(0, 0, 0)
-tester.test1()
-tester.test2()
-tester.test3()
-tester.test4()
-tester.test5()

max submission/6_9_HW.py

@@ -1,110 +0,0 @@
-# question 1: 
- #boolean context:
- #In programming, a boolean context refers to a situation where a condition or expression is evaluated as either true or false. 
- #A boolean context typically occurs in control flow statements, such as if statements or while loops, where the condition determines whether 
- #a certain block of code should be executed or repeated.
-
- #Ellipsis:
- #an ellipsis (...) is a special token that represents an ellipsis object. It is often used as a placeholder in code or data structures where 
- #you intend to fill in additional functionality or content later.
-
- #machine-level double precision floating point numbers
- #Machine-level double precision floating-point numbers, often referred to as double, are a common representation of real numbers in computer systems.
- #They are a binary format used to store and manipulate floating-point values with higher precision compared to single precision.
-
- #Callable types:
- #callable types refer to objects that can be called as functions. These objects are capable of executing code when invoked with parentheses, just 
- #like a regular function.
-
- #cell_contents:
- #the term "cell contents" refers to the value stored in a cell of a closure. A closure is a function object that retains references to variables 
- #in its enclosing lexical scope, even when the outer function has finished executing.
- #When a closure is created, each variable it references is stored in a cell, which is an internal data structure associated with the closure.
- #The cell contents refer to the value stored within the cell associated with a particular variable in a closure. It represents the value of the 
- #variable as seen by the closure, regardless of any changes made to the variable in the outer scope.
-
- #example for "When an instance method object is created by retrieving a user-defined function object from a class via one of its instances, 
- #its __self__ attribute is the instance, and the method object is said to be bound. The new method’s __func__ attribute is the original function 
- #object."
-class MyClass:
-    def my_method(self):
-        print("Hello, world!")
-
-# Creating an instance of MyClass
-my_instance = MyClass()
-
-# Retrieving the method object from the instance
-method_object = my_instance.my_method
-
-# Accessing the attributes of the method object
-print(method_object.__self__)  # Output: <__main__.MyClass object at 0x...>
-print(method_object.__func__)  # Output: <function MyClass.my_method at 0x...>
-
-# Calling the method through the method object
-method_object()  # Output: Hello, world!
-
- #async def:
- # the async def syntax is used to define an asynchronous function, also known as a coroutine. An asynchronous function allows concurrent
- # execution of multiple tasks by allowing the program to proceed to other tasks while waiting for certain operations to complete.
-import asyncio
-
-async def my_coroutine():
-    print("Coroutine started")
-    await asyncio.sleep(1)  # Simulating an asynchronous operation
-    print("Coroutine completed")
-
-# Calling the coroutine
-asyncio.run(my_coroutine())
- 
- #Static method and class method:
- #In Python, static methods and class methods are two types of methods that can be defined within a class.
- #Static methods:
- #Static methods are defined using the @staticmethod decorator or by declaring them inside the class without any special decorator.
- #Static methods are not bound to a specific instance or the class itself. They behave like regular functions but are defined within the class for organizational purposes.
- #Static methods do not have access to the instance or class attributes and cannot modify them directly.
- #Static methods can be called on the class itself or on an instance of the class.
- #Class methods:
- #Class methods are defined using the @classmethod decorator.
- #Class methods are bound to the class itself rather than the instances. They receive the class as the first argument (usually named cls) instead of the instance (self).
- #Class methods have access to the class attributes and can modify them if needed.
- #Class methods can be called on the class itself or on an instance of the class.
-
-# question 2:
-def add_numbers(a, b):
-    result = a + b
-    return result
-
-def multiply_numbers(x, y):
-    result = x * y
-    return result
-
-def main():
-    number1 = 5
-    number2 = 10
-    sum_result = add_numbers(number1, number2)
-    product_result = multiply_numbers(sum_result, number2)
-    print("Product:", product_result)
-
-if __name__ == "__main__":
-    main()
-
-# question 3:
-def foo():
-    a = 10
-    b = 20
-    return a + b
-
-def bar():
-    x = 5
-    y = 7
-    return x * y
-
-result = foo() + bar()
-print(result)
-# In this example, using "Step Over" and "Step Into" may not yield different results, but the behavior is 
-# dependent on the implementation of the functions being called.
-
-
-
-
-

max submission/hw1 1-5.py

@@ -1,21 +0,0 @@
-# question 1: The international standard for floating-point representation is called IEEE 754.
-
-# question 2: The IEEE 754 standard allows floating-point numbers to be represented with different bit sizes. The most commonly used formats specified by the standard are single precision (32 bits) and double precision (64 bits).
-
-# question 3: 	
-import sys
-print(sys.float_info)
-sys.float_info(max=1.7976931348623157e+308, max_exp=1024, max_10_exp=308, min=2.2250738585072014e-308, min_exp=-1021, min_10_exp=-307, dig=15, mant_dig=53, epsilon=2.220446049250313e-16, radix=2, rounds=1)
-
-# question 4: 	In the first multiplication (num1 * num2), you are using the normal Python multiplication 
-# operator, which operates on native Python integers. Python integers have arbitrary precision, meaning they can
-#  represent very large numbers accurately. However, when the result exceeds the limit of a 64-bit signed integer 
-# (2^63 - 1), an overflow may occur, and the result may not be accurate.
-# In the second multiplication (df['Number1'] * df['Number2']), you are using the Pandas multiplication operator 
-# on the columns of a DataFrame. By default, Pandas uses NumPy for numerical computations, and NumPy uses 
-# fixed-size data types. In this case, the integers in the DataFrame are most likely represented as 64-bit 
-# signed integers (int64), which have a maximum value of 2^63 - 1. If the result of the multiplication exceeds 
-# this limit, it will overflow and wrap around, leading to incorrect results.
-
-# Question5: 5.	The size of a Python floating-point number is determined by the implementation and is not influenced by the magnitude of the value it represents. So it does not change as the value of the number gets larger.
-

module-inspector/module-inspector.py

@@ -2,6 +2,7 @@ import sys
 import tkinter as tk
 from tkinter import ttk
 
+
 def on_treeview_heading_click(column, tree):
     data = [(tree.set(child, column), child) for child in tree.get_children("")]
     reverse = (column == tree.heading(column)["text"]) and not column.startswith("_")
@@ -42,3 +43,7 @@ tree.tag_configure("selected", background="#ccc")
 for col in tree["columns"]:
     tree.heading(col, text=col, anchor=tk.W, command=lambda col=col: on_treeview_heading_click(col, tree))
 root.mainloop()
+# Call the on_treeview_heading_click function when a column heading is clicked
+for col in tree["columns"]:
+    tree.heading(col, text=col, anchor=tk.W, command=lambda col=col: on_treeview_heading_click(col, tree))
+root.mainloop()

public

@@ -0,0 +1,9 @@
+---- BEGIN SSH2 PUBLIC KEY ----
+Comment: "rsa-key-20230531"
+AAAAB3NzaC1yc2EAAAADAQABAAABAQDEZW0iQpBPPRGM7QCdc3GqefkqUqq5i727
+MEYvf7gy0h7Zvz72cvnDtnVF6MknCGVZ1S3uY+HeKUtujE3+z9tJ5zIxO+TxthJe
+8gbH9MUkElKX8qJN/nF1H4dLiW+77ARsT70rDoHayWccGGnwC/Gs9xXVF+Ni0JD5
+TKcGoye3Ps2Ni/KJAzkn78JyfrAb+Ail4DG4snCf1q/jXTuB39qYTTg94FJEqwe1
+dhPIzRiOjFT3eSmHAXdSuAXFjpXAq+bV8R2uxKsgNfvxyV4vlKwFhYTq+kHwDWX7
+RtLQUUs8mSTfbnL2LuE+T2d8sBtj9wi+gODFl5dDAVZkmuKrRgLF
+---- END SSH2 PUBLIC KEY ----

requirements.txt

@@ -0,0 +1,8 @@
+google.cloud.pubsub
+plotly
+pandas
+numpy
+h5py
+selenium
+sqlalchemy
+flask

templates/test3.html

@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Class Dump</title>
+</head>
+<body>
+    <h1>Class Dump</h1>
+    <ul>
+        {% for class in classes %}
+        <li>{{ class.__name__ }}</li>
+        {% endfor %}
+    </ul>
+
+    <h1>User Dump</h1>
+    <ul>
+        {% for user in users %}
+        <li>{{ user }}</li>
+        {% endfor %}
+    </ul>
+</body>
+</html>

test1.py

@@ -0,0 +1,31 @@
+class Count:
+    def __init__(self,mymin,mymax):
+        self.mymin=mymin
+        self.mymax=mymax    
+
+    def __getattr__(self, item):
+        self.__dict__[item]=0
+        return 0
+
+obj1 = Count(1,10)
+print(obj1.mymin)
+print(obj1.mymax)
+print(obj1.mycurrent1)
+
+class Count:
+
+    def __init__(self,mymin,mymax):
+        self.mymin=mymin
+        self.mymax=mymax
+        self.current=None
+   
+    def __getattribute__(self, item):
+        if item.startswith('cur'):
+            raise AttributeError
+        return object.__getattribute__(self,item) 
+        # or you can use ---return super().__getattribute__(item)
+
+obj1 = Count(1,10)
+print(obj1.mymin)
+print(obj1.mymax)
+print(obj1.current)

test10.py

@@ -0,0 +1,14 @@
+import sys
+
+# Iterate through the modules in sys.modules
+for module_name, module in sys.modules.items():
+    # Check if the module has a loader attribute
+    if hasattr(module, '__loader__'):
+        loader = module.__loader__
+        # Determine the type of loader
+        if type(loader).__name__ == 'SourceFileLoader':
+            print(f"Module '{module_name}' has a source loader.")
+        elif type(loader).__name__ == 'BuiltinImporter':
+            print(f"Module '{module_name}' has a built-in loader.")
+        elif type(loader).__name__ == 'ExtensionFileLoader':
+            print(f"Module '{module_name}' has an extension loader.")

test11.py

@@ -0,0 +1,12 @@
+from cx_Freeze import setup, Executable
+
+executables = [Executable('module-inspector.py')]
+
+setup(
+    name='wow',
+    version='1.0',
+    description='Module-inspector Application',
+    executables=executables
+)
+
+

test2.py

@@ -0,0 +1,58 @@
+import time
+'''import matplotlib.pyplot as plt'''
+
+# Durations to measure in seconds
+durations = [0.01, 0.1, 1.0]
+
+# Number of repetitions
+num_repetitions = 100
+
+# Lists to store jitter for each duration
+jitter_10ms = []
+jitter_100ms = []
+jitter_1s = []
+
+# Function to measure jitter
+def measure_jitter(duration):
+    start_time = time.perf_counter_ns()
+    time.sleep(duration)
+    end_time = time.perf_counter_ns()
+    return end_time - start_time
+
+# Measure jitter for each duration and repeat for num_repetitions times
+for duration in durations:
+    for _ in range(num_repetitions):
+        jitter = measure_jitter(duration)
+        if duration == 0.01:
+            jitter_10ms.append(jitter)
+        elif duration == 0.1:
+            jitter_100ms.append(jitter)
+        elif duration == 1.0:
+            jitter_1s.append(jitter)
+'''
+# Create histograms for each duration's jitter
+plt.hist(jitter_10ms, bins=10, color='r', alpha=0.5, label='10ms')
+plt.hist(jitter_100ms, bins=10, color='g', alpha=0.5, label='100ms')
+plt.hist(jitter_1s, bins=10, color='b', alpha=0.5, label='1s')
+plt.xlabel('Jitter (ns)')
+plt.ylabel('Frequency')
+plt.title('Jitter of time.sleep() for Different Durations')
+plt.legend()
+plt.show()
+'''
+
+import random
+import secrets
+
+# Create a dictionary with 1 million integer keys
+dict = {}
+
+for i in range(1000000):
+    # Generate a random 64-byte value as bytes
+    random_value = secrets.token_bytes(64)
+    # Convert the bytes value to hexadecimal representation
+    hex_value = random_value.hex()
+    # Add the key-value pair to the dictionary
+    dict[i] = hex_value
+
+print("Dictionary created with 1 million integer keys.")

test3.py

@@ -0,0 +1,51 @@
+
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+        self._known_good_copy = self.__dict__.copy()
+
+    def __getattribute__(self, name):
+        if name == "treasure":
+            if self.__dict__ != self._known_good_copy:
+                print("Attribute deleted!")
+            return self._treasure
+        else:
+            return object.__getattribute__(self, name)
+
+    def __setattr__(self, name, value):
+        if name != "_treasure" and name != "_known_good_copy":
+            print("Attribute deletion detected!")
+            self._known_good_copy = self.__dict__.copy()
+        object.__setattr__(self, name, value)
+v = Vault()
+Vault.treasure = " haha monkeypatched"
+print(v.treasure)
+
+'''
+class Vault:
+    def __init__(self):
+        self._treasure = "gold"
+        self._known_good_copy = self.__dict__.copy()
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+        else:
+            if instance.__dict__ != self._known_good_copy:
+                print("Attribute deleted!")
+            return self._treasure
+
+    def __set__(self, instance, value):
+        print("Attribute deletion detected!")
+        self._known_good_copy = instance.__dict__.copy()
+
+    def __delete__(self, instance):
+        print("Attribute deletion detected!")
+        self._known_good_copy = instance.__dict__.copy()
+
+    def __set_name__(self, owner, name):
+        self.name = name
+v = Vault()
+Vault._treasure = " haha monkeypatched"
+print(v._treasure)
+'''

test4.py

@@ -0,0 +1,17 @@
+import asyncio
+
+# Define a coroutine function
+async def greet(name):
+    print(f"Hello, {name}!")
+    await asyncio.sleep(1)
+    print(f"Goodbye, {name}!")
+
+# Call the coroutine function
+coroutine_obj = greet("Karen")
+
+# coroutine_obj is a coroutine object, not the actual execution of the coroutine
+
+# Use an event loop to run the coroutine
+loop = asyncio.get_event_loop()
+loop.run_until_complete(coroutine_obj)
+loop.close()

test5.py

@@ -0,0 +1,8 @@
+def think():
+    x=1
+    def think2():
+        nonlocal x
+        x=2
+    think2()
+    return x
+print(think())

test6.py

@@ -0,0 +1,33 @@
+'''
+import sys
+import importlib.util
+
+# Load only the sys module
+importlib.import_module('sys')
+
+# Check the type of the value associated with the 'os' key
+os_module = sys.modules['os']
+os_module_type = type(os_module)
+
+# Print the type of the value associated with the 'os' key
+print("Type of the value associated with the 'os' key in sys.modules: {}".format(os_module_type))
+'''
+'''
+import os
+print(type(os))
+'''
+
+import builtins
+
+# Access the os module from globals()
+os = globals().get('os', None)
+
+# Check if os module is available
+if os is not None:
+    # Call os.getlogin()
+    username = os.getlogin()
+    print("Username:", username)
+else:
+    print("os module not available")
+
+

test7.py

@@ -0,0 +1,54 @@
+import os
+import sys
+import importlib.util
+import types
+
+
+def get_bytecode_size(module_name):
+    """Get the size of the cached .pyc bytecode file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith('.py'):
+        pyc_file = importlib.util.cache_from_source(spec.origin)
+        if os.path.exists(pyc_file):
+            return os.path.getsize(pyc_file)
+    return 0
+
+
+def get_dso_size(module_name):
+    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith(('.so', '.dll')):
+        return os.path.getsize(spec.origin)
+    return 0
+
+
+def generate_module_report():
+    """Generate a report summarizing the module environment."""
+    total_modules = len(sys.modules)
+    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
+    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
+    python_size = sum(
+        sys.modules[module_name].__sizeof__()
+        for module_name in sys.modules
+        if isinstance(sys.modules[module_name], types.ModuleType)
+    )
+    total_size = bytecode_size + dso_size + python_size
+
+    # Calculate percentage of each type of module
+    bytecode_percent = bytecode_size / total_size * 100
+    dso_percent = dso_size / total_size * 100
+    python_percent = python_size / total_size * 100
+
+    # Generate the report
+    report = f"Total modules: {total_modules}\n"
+    report += f"Total size: {total_size} bytes\n"
+    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
+    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
+    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
+
+    return report
+
+if __name__ == '__main__' and __package__ is None:
+    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
+    report = generate_module_report()
+    print(report)

test8.py

@@ -0,0 +1,259 @@
+'''
+import os
+import sys
+import importlib.util
+import types
+import inspect
+from collections import defaultdict
+
+
+def get_bytecode_size(module_name):
+    """Get the size of the cached .pyc bytecode file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith('.py'):
+        pyc_file = importlib.util.cache_from_source(spec.origin)
+        if os.path.exists(pyc_file):
+            return os.path.getsize(pyc_file)
+    return 0
+
+
+def get_dso_size(module_name):
+    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith(('.so', '.dll')):
+        return os.path.getsize(spec.origin)
+    return 0
+
+
+def generate_module_report():
+    """Generate a report summarizing the module environment."""
+    total_modules = len(sys.modules)
+    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
+    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
+    python_size = sum(
+        sys.modules[module_name].__sizeof__()
+        for module_name in sys.modules
+        if isinstance(sys.modules[module_name], types.ModuleType)
+    )
+    total_size = bytecode_size + dso_size + python_size
+
+    # Calculate percentage of each type of module
+    bytecode_percent = bytecode_size / total_size * 100
+    dso_percent = dso_size / total_size * 100
+    python_percent = python_size / total_size * 100
+
+    # Generate the report
+    report = f"Total modules: {total_modules}\n"
+    report += f"Total size: {total_size} bytes\n"
+    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
+    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
+    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
+
+    # Generate histogram of module directory names
+    dir_histogram = defaultdict(int)
+    for module_name in sys.modules:
+        module_file = sys.modules[module_name].__file__
+        if module_file:
+            module_dir = os.path.dirname(module_file)
+            dir_histogram[module_dir] += 1
+
+    report += "\nModule Directory Histogram:\n"
+    for module_dir, count in dir_histogram.items():
+        report += f"{module_dir}: {'*' * count}\n"
+
+    return report
+
+
+if __name__ == '__main__' and __package__ is None:
+    script_path = inspect.getframeinfo(inspect.currentframe()).filename
+    __spec__ = importlib.util.spec_from_loader('__main__', None, origin=script_path)
+    report = generate_module_report()
+    print(report)
+'''
+'''
+import os
+import sys
+from importlib.util import find_spec
+
+def get_module_size(module):
+    spec = find_spec(module)
+    if spec is None:
+        return None
+    if spec.cached:
+        size = os.stat(spec.cached).st_size
+    else:
+        size = os.stat(spec.origin).st_size
+    return size
+
+def summarize_modules():
+    total_size = 0
+    byte_count = 0
+    dso_count = 0
+    py_count = 0
+    directory_histogram = {}
+    for module_name, module in sys.modules.items():
+        if module is None:
+            continue
+        size = get_module_size(module_name)
+        if size is None:
+            continue
+        total_size += size
+        if module_name.endswith('_bootstrap') or module_name.endswith('_bootstrap_external'):
+            continue
+        if module_name.startswith('_frozen_importlib'):
+            dso_count += 1
+            continue
+        if module_name.startswith('_'):
+            continue
+        if module.__file__.endswith('.pyc'):
+            byte_count += size
+        elif module.__file__.endswith(('.so', '.dll')):
+            dso_count += 1
+        else:
+            py_count += size
+        module_directory = os.path.dirname(os.path.abspath(module.__file__))
+        directory_histogram[module_directory] = directory_histogram.get(module_directory, 0) + 1
+
+    print(f"Total module size: {total_size} bytes")
+    print(f"Bytecode modules: {(byte_count/total_size)*100:.2f}% ({byte_count} bytes)")
+    print(f"DSO modules: {(dso_count/total_size)*100:.2f}% ({dso_count} modules)")
+    print(f"Python modules: {(py_count/total_size)*100:.2f}% ({py_count} bytes)")
+
+    print("Module directory histogram:")
+    for directory, count in directory_histogram.items():
+        print(f"{directory}: {'*' * count}")
+
+if __name__ == '__main__':
+    summarize_modules()
+'''
+'''
+import os
+import sys
+from importlib.util import find_spec
+
+
+def get_module_size(module):
+    spec = find_spec(module)
+    if spec is None:
+        return None
+    if spec.cached:
+        size = os.stat(spec.cached).st_size
+    else:
+        size = os.stat(spec.origin).st_size
+    return size
+
+
+def summarize_modules():
+    total_size = 0
+    byte_count = 0
+    dso_count = 0
+    py_count = 0
+    directory_histogram = {}
+    for module_name, module in sys.modules.items():
+        if module is None:
+            continue
+        size = get_module_size(module_name)
+        if size is None:
+            continue
+        total_size += size
+        if module_name.endswith('_bootstrap') or module_name.endswith('_bootstrap_external'):
+            continue
+        if module_name.startswith('_frozen_importlib'):
+            dso_count += 1
+            continue
+        if module_name.startswith('_'):
+            continue
+        if module.__file__.endswith('.pyc'):
+            byte_count += size
+        elif module.__file__.endswith(('.so', '.dll')):
+            dso_count += 1
+        else:
+            py_count += size
+        module_directory = os.path.dirname(os.path.abspath(module.__file__))
+        directory_histogram[module_directory] = directory_histogram.get(module_directory, 0) + 1
+
+    print(f"Total module size: {total_size} bytes")
+    print(f"Bytecode modules: {(byte_count/total_size)*100:.2f}% ({byte_count} bytes)")
+    print(f"DSO modules: {(dso_count/total_size)*100:.2f}% ({dso_count} modules)")
+    print(f"Python modules: {(py_count/total_size)*100:.2f}% ({py_count} bytes)")
+
+    print("Module directory histogram:")
+    for directory, count in directory_histogram.items():
+        print(f"{directory}: {'*' * count}")
+
+
+if __name__ == '__main__':
+    summarize_modules()
+'''
+
+import os
+import sys
+import importlib.util
+import types
+from collections import defaultdict
+
+
+def get_bytecode_size(module_name):
+    """Get the size of the cached .pyc bytecode file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith('.py'):
+        pyc_file = importlib.util.cache_from_source(spec.origin)
+        if os.path.exists(pyc_file):
+            return os.path.getsize(pyc_file)
+    return 0
+
+
+def get_dso_size(module_name):
+    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin.endswith(('.so', '.dll')):
+        return os.path.getsize(spec.origin)
+    return 0
+
+
+def generate_module_report():
+    """Generate a report summarizing the module environment."""
+    total_modules = len(sys.modules)
+    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
+    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
+    python_size = sum(
+        sys.modules[module_name].__sizeof__()
+        for module_name in sys.modules
+        if isinstance(sys.modules[module_name], types.ModuleType)
+    )
+    total_size = bytecode_size + dso_size + python_size
+
+    # Calculate percentage of each type of module
+    bytecode_percent = bytecode_size / total_size * 100
+    dso_percent = dso_size / total_size * 100
+    python_percent = python_size / total_size * 100
+
+    # Generate the report
+    report = f"Total modules: {total_modules}\n"
+    report += f"Total size: {total_size} bytes\n"
+    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
+    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
+    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
+
+    # Generate histogram of module directory names
+    dir_histogram = defaultdict(int)
+    for module_name in sys.modules:
+        module_file = sys.modules[module_name].__file__
+        if module_file:
+            module_dir = os.path.dirname(module_file)
+            dir_histogram[module_dir] += 1
+
+    report += "\nModule Directory Histogram:\n"
+    for module_dir, count in dir_histogram.items():
+        report += f"{module_dir}: {'*' * count}\n"
+
+    return report
+
+
+if __name__ == "__main__":
+    report = generate_module_report()
+    print(report)
+
+
+
+

test9.py

@@ -0,0 +1,77 @@
+import os
+import sys
+import importlib.util
+import types
+from collections import defaultdict
+from sys import getsizeof
+
+
+def get_bytecode_size(module_name):
+    """Get the size of the cached .pyc bytecode file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin and spec.origin.endswith('.py'):
+        pyc_file = importlib.util.cache_from_source(spec.origin)
+        if os.path.exists(pyc_file):
+            return os.path.getsize(pyc_file)
+    return 0
+
+
+def get_dso_size(module_name):
+    """Get the size of the DSO (Dynamic Shared Object) module file for a module."""
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin and spec.origin.endswith(('.so', '.dll')):
+        return os.path.getsize(spec.origin)
+    return 0
+
+
+def generate_module_report():
+    """Generate a report summarizing the module environment."""
+    total_modules = len(sys.modules)
+    bytecode_size = sum(get_bytecode_size(module_name) for module_name in sys.modules)
+    dso_size = sum(get_dso_size(module_name) for module_name in sys.modules)
+    python_size = sum(
+        getsizeof(sys.modules[module_name]) - getsizeof(module_name)
+        for module_name in sys.modules
+        if isinstance(sys.modules[module_name], types.ModuleType)
+    )
+    total_size = bytecode_size + dso_size + python_size
+
+    # Calculate percentage of each type of module
+    bytecode_percent = bytecode_size / total_size * 100
+    dso_percent = dso_size / total_size * 100
+    python_percent = python_size / total_size * 100
+
+    # Generate the report
+    report = f"Total modules: {total_modules}\n"
+    report += f"Total size: {total_size} bytes\n"
+    report += f"Bytecode modules: {bytecode_percent:.2f}% ({bytecode_size} bytes)\n"
+    report += f"DSO modules: {dso_percent:.2f}% ({dso_size} bytes)\n"
+    report += f"Python code modules: {python_percent:.2f}% ({python_size} bytes)\n"
+
+    # Generate histogram of module directory names
+    dir_histogram = defaultdict(int)
+    for module_name in sys.modules:
+        module_file = getattr(sys.modules[module_name], '__file__', None)
+        if module_file:
+            module_dir = os.path.dirname(module_file)
+            dir_histogram[module_dir] += 1
+
+    report += "\nModule Directory Histogram:\n"
+    for module_dir, count in dir_histogram.items():
+        report += f"{module_dir}: {'*' * count}\n"
+
+    return report
+
+
+if __name__ == "__main__":
+    __spec__ = importlib.util.spec_from_file_location('__main__', __file__)
+report = generate_module_report()
+print(report)
+
+
+
+
+
+
+    report = generate_module_report()
+    print(report)

test_fp2_14.py

@@ -0,0 +1,63 @@
+from fp2_14 import get_product_prices
+import h5py
+import pandas as pd
+import datetime
+import sys
+from datetime import datetime
+import ntplib
+import arrow
+
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-NFL-zipcodes"
+url = "https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f"
+
+df_new = get_product_prices(project_id, subscription_name, url)
+
+'''
+# When using sys to call the 'get_product_prices' function
+
+if __name__ == '__main__':
+    # Check if the required command-line arguments are provided
+    if len(sys.argv) < 4:
+        print("Usage: python test_fp2_14.py project_id subscription_name url")
+        sys.exit(1)
+
+    project_id = sys.argv[1]
+    subscription_name = sys.argv[2]
+    url = sys.argv[3:]
+
+    get_product_prices(project_id, subscription_name, url)
+'''
+
+
+# Now we have the new dataframe "df" recording the new price listed in weee's page
+# Load such data in the existing HDF5 file
+file_path = 'product1.h5'
+with h5py.File(file_path, 'a') as file:
+    # Load the existing dataframe 'df' from the HDF5 file
+    # REPLACE KEY TO GET THE EXISTING DATAFRAME FOR EACH DIFFERENT PRODUCT
+    df = pd.read_hdf(file, key='df')
+
+    # Add timestamp to the new dataframe "df_new"
+    # timestamp = datetime.now()
+    '''
+    To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
+    provides time synchronization with a reliable time server.
+    Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
+    sensible and human-friendly approach to format and convert the timestamps.
+    '''
+    # Fetch the time from an NTP server
+    ntp_client = ntplib.NTPClient()
+    response = ntp_client.request('pool.ntp.org')
+    ntp_time = datetime.fromtimestamp(response.tx_time)
+    timestamp = arrow.get(ntp_time).to('local')
+    df_new['Timestamp'] = timestamp
+
+    # Append 'df_new' to the existing df stored in the key
+    df_update = df.append(df_new, ignore_index=True)
+
+    # Save 'df2' to the HDF5 file with the key 'df'
+    # REPLACE KEY FOR DIFFERENT PRODUCT
+    df_update.to_hdf(file, key='df', mode='a')
+
+print("Data appended and saved successfully.")