Merged fp files

Multiprocessing.py

@@ -15,19 +15,20 @@ from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 import selenium.common.exceptions
 import re
+from fp2_14 import get_product_prices
 
 # Define the subset of urls
 url_list = [
-    'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
-    'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261'
-    'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467'
-    'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01'
-    'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224'
-    'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02'
-    'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01'
-    'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
-    'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353'
-    'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03'
+    'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
+    'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
+    'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
+    'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
+    'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
+    'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
+    'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
+    'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
+    'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
+    'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
     'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
 ]
 
@@ -40,6 +41,7 @@ def run_scraper(url):
     # scraping logic
     df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
     print(f"Created a dataframe: {df}")
+    product_id = url.split("/")[-1][:5]
 
     options = Options()
     options.add_argument('--disable-gpu')
@@ -50,8 +52,6 @@ def run_scraper(url):
     subscriber = pubsub_v1.SubscriberClient()
     subscription_path = subscriber.subscription_path(project_id, subscription_name)
 
-    url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
-    product_id = url.split("/")[-1][:5]
     # product_id = url.split("/")[-1]
     driver.delete_all_cookies
     driver.get(url)
@@ -238,19 +238,21 @@ def run_scraper(url):
             print(f"no messages, trying again at {time.asctime()}")
             break
         # subprocess.call(['python', 'fp2_13.py', url])
-        return df
+    return df
 
 if __name__ == '__main__':
     # Determine the number of available CPUs
-    cpu_count = multiprocessing.cpu_count()
+    # cpu_count = multiprocessing.cpu_count()
 
-    with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count) as executor:
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
         future_to_url = {executor.submit(run_scraper, url): url for url in url_list}
+        results = []
         for future in concurrent.futures.as_completed(future_to_url):
             url = future_to_url[future]
             try:
                 result = future.result()
                 print(f"Scraped data for URL: {url}")
+                results.append(result)
             except Exception as e:
                 print(f"Failed to scrape data for URL: {url}\nError: {e}")
 
@@ -263,10 +265,12 @@ pool.close()
 pool.join()
 '''
 
-for url, result in zip(url_list, result):
-    # Extract the product ID from the URL
-    product_id = url.split("/")[-1].split("?")[0]
+with pd.HDFStore('product_data.h5') as store:
+    for url, result in zip(url_list, results):
+        # Extract the product ID from the URL
+        product_id = url.split("/")[-1][:5]
 
-    # Save the dataframe to the HDF5 file with a unique key
-    key = f'df_{product_id}'
-    result.to_hdf('product_data.h5', key=key, mode='a')
+        # Save the dataframe to the HDF5 file with a unique key
+        key = f'df_{product_id}'
+        store[key] = result
+        # result.to_hdf('product_data.h5', key=key, mode='a')

README.md

@@ -17,4 +17,27 @@ the focus is on *working results*, not rote memorization.  Homework questions
 are simple enough that they can be reviewed interactively in 1:1 or up to 1:4
 sessions with study groups.
 
+This branch contains a prototype for a retail price scraper for getweee.com.
+To run it, install:
+
+* Python 3.9+,
+* chrome,
+* chromedriver,
+* and the python depedencies:
+
+```
+pip install -r /path/to/requirements.txt
+```
+
+Once those are installed, install the GCP SDK:
+
+https://cloud.google.com/sdk/docs/install
+
+You will also have to log into GCP *twice*, once for the gcloud CLI tools, 
+and again for the GCP SDK libraries:
+
+https://googleapis.dev/python/google-api-core/latest/auth.html
+
+
+
 

Task_assignment.ipynb

@@ -0,0 +1,402 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Guide\n",
+    "1. Please write your solutions at the appropriate location **# YOUR CODE HERE**\n",
+    "2. Do not modify other cells"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "# Numerical or binary attributes\n",
+    "num_or_bin= ['Income', 'Kidhome','Recency', 'MntWines', 'MntFruits', 'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts','MntGoldProds', 'NumDealsPurchases', 'NumWebPurchases',\n",
+    "       'NumCatalogPurchases', 'NumStorePurchases', 'NumWebVisitsMonth','AcceptedCmp3', 'AcceptedCmp4', 'AcceptedCmp5', 'AcceptedCmp1','AcceptedCmp2']\n",
+    "\n",
+    "# Categorical attributes\n",
+    "cat = ['Education','Marital_Status']\n",
+    "\n",
+    "# Load the data source\n",
+    "pd.set_option('display.max_columns', None)\n",
+    "df  = pd.read_csv(\"https://raw.githubusercontent.com/rpi-techfundamentals/website_fall_2021/master/site/public/marketing_campaign.csv\", delimiter='\\t')\n",
+    "df.head()"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q1. Load and preprocess the data.\n",
+    "Build and transform your dataframe `df` using the following steps before modeling. Create a new dataframe `X` that has the following properties:\n",
+    "\n",
+    "1. It should include only the columns from lists `num_or_bin` and `cat` (in the code above).\n",
+    "2. Variables in cat should be transformed to `n-1` dummy variables (where `n` is the number of categories for each variable.)\n",
+    "3. Missing variables in the `Income` table should be filled with the median of the `income` table.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "all_cols=num_or_bin+cat\n",
+    "X=pd.get_dummies(df[all_cols],columns=cat,drop_first=True)\n",
+    "X.loc[X['Income'].isnull(),'Income']=X['Income'].median()\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 1 Test Cell \n",
+    "#this tests the columns. \n",
+    "ecol = set(['Income', 'Kidhome', 'Recency', 'MntWines', 'MntFruits',\n",
+    "       'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts',\n",
+    "       'MntGoldProds', 'NumDealsPurchases', 'NumWebPurchases',\n",
+    "       'NumCatalogPurchases', 'NumStorePurchases', 'NumWebVisitsMonth',\n",
+    "       'AcceptedCmp3', 'AcceptedCmp4', 'AcceptedCmp5', 'AcceptedCmp1',\n",
+    "       'AcceptedCmp2', 'Education_Basic', 'Education_Graduation',\n",
+    "       'Education_Master', 'Education_PhD', 'Marital_Status_Alone',\n",
+    "       'Marital_Status_Divorced', 'Marital_Status_Married',\n",
+    "       'Marital_Status_Single', 'Marital_Status_Together',\n",
+    "       'Marital_Status_Widow', 'Marital_Status_YOLO'])\n",
+    "assert len(ecol-set(X.columns))==0\n",
+    "assert len(set(X.columns))-len(ecol)==0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 2 Test Cell \n",
+    "#this tests the columns. \n",
+    "assert X.shape==(2240, 30)\n",
+    "q2=X.copy()\n",
+    "q2['Response']=df['Response']\n",
+    "q2['Year_Birth']=df['Year_Birth']\n",
+    "q2['Dt_Customer']=df['Dt_Customer']\n",
+    "q2.to_csv('q2.csv',index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 3 Test Cell \n",
+    "assert len(X.loc[X['Income']==51381.5,'Income'])==24"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 4 Test Cell\n",
+    "assert X.isnull().sum().sum()==0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Reload data\n",
+    "import pandas as pd\n",
+    "df_cl=pd.read_csv('q2.csv')"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q2. Feature Creation.\n",
+    "1. In the `df_cl` dataframe, create an integer feature `age` which is `2021` minus `Year_Birth`.\n",
+    "2. In the `df_cl` dataframe, create an integer feature `holiday` which is 1 if the month of `Dt_Customer` is equal to 12 and 0 otherwise. You can get the month by accessing `mm` from the date format `dd-mm-yyyy`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "df_cl['age']=(2021-df_cl['Year_Birth']).astype(int)\n",
+    "df_cl['Dt_Customer_tmp']=df_cl['Dt_Customer'].apply(lambda x:x[3:5])\n",
+    "df_cl.loc[df_cl['Dt_Customer_tmp']=='12','holiday']=1\n",
+    "df_cl.loc[df_cl['Dt_Customer_tmp']!='12','holiday']=0\n",
+    "df_cl['holiday']=df_cl['holiday'].astype(int)\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 5 Test Cell\n",
+    "assert df_cl['age'].sum()==116915"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 6 Test Cell\n",
+    "import numpy as np\n",
+    "assert isinstance(df_cl['age'][0], (np.int64,np.int32))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 7 Test Cell\n",
+    "assert df_cl['holiday'].sum()==175"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 8 Test Cell\n",
+    "import numpy as np\n",
+    "assert isinstance(df_cl['holiday'][0], (np.int64,np.int32))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "df_cl2=pd.read_csv('dfcl2.csv')\n",
+    "y_reload=df_cl2['Response']\n",
+    "X_reload=df_cl2.drop(['Response'], axis=1)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q3. Train Test Split\n",
+    "Now split the data `X_reload`, `y_reload` intro training (50%) and testing data (50%) by using these variable names while making sure there are Make sure that there are equal number of positive cases (1's) in the train and the test set.\n",
+    "\n",
+    "`X_train`: Training feature columns\n",
+    "`X_test`: Testing feature columns\n",
+    "`y_train`: Training DV\n",
+    "`y_test`: Testing DV\n",
+    "\n",
+    "When doing so set the `random_state` parameter to 100 so you get the same split that is in the solution."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "y_reload=df_cl2['Response']\n",
+    "X_reload=df_cl2.drop(['Response'], axis=1)\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "X_train,X_test,y_train,y_test=train_test_split(X_reload,y_reload,\n",
+    "                                               train_size=0.5,\n",
+    "                                               test_size=0.5,\n",
+    "                                               random_state=100,\n",
+    "                                               stratify=y_reload)\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 9 Test Cell\n",
+    "assert X_train.shape==(1120, 31)\n",
+    "assert X_test.shape==(1120, 31)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 10 Test Cell\n",
+    "assert  y_train.sum()==y_test.sum()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "df_cl2=pd.read_csv('dfcl2.csv')\n",
+    "y_reload=df_cl2['Response']\n",
+    "X_reload=df_cl2.drop(['Response'], axis=1)\n",
+    "\n",
+    "X_train2, X_test2, y_train2, y_test2 = train_test_split(X_reload, y_reload, test_size=0.3, random_state = 100)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Q4. Classification\n",
+    "Use `X_train2`, `X_test2`, `y_train2`, `y_test2` from above. Use a Gradient Boosting classifier (as instantiated below).\n",
+    "\n",
+    "Please calculate:\n",
+    "1. `test_acc` The accuracy of the classifier on the test set.\n",
+    "2. `test_roc` The area under the ROC curve on on the test set."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn import metrics\n",
+    "from sklearn.ensemble import GradientBoostingClassifier\n",
+    "#use this classifier\n",
+    "classifier = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1, random_state=0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raise NotImplementedError()\n",
+    "# YOUR CODE HERE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Answer\n",
+    "'''\n",
+    "classifier.fit(X_train2,y_train2)\n",
+    "test_y_pred=classifier.predict(X_test2)\n",
+    "test_y_pred_prob=classifier.predict_proba(X_test2)[:,1]\n",
+    "test_acc=metrics.accuracy_score(y_test2,test_y_pred)\n",
+    "test_roc=metrics.roc_auc_score(y_test2,test_y_pred_prob)\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 11 Test Cell\n",
+    "assert round(test_acc,3)==0.860"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Please do not modify/delete this cell\n",
+    "# 12 Test Cell\n",
+    "assert round(test_roc,3)==0.846"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

View_h5_file.py

@@ -7,19 +7,68 @@ f = h5py.File('product1.h5','r')
 list(f.keys())
 for key in f.keys():
     print(f[key])
+    df = pd.read_hdf('product1.h5', key='df')
+    df = df._append(pd.read_hdf('product1.h5', key))
     # print(f[key].shape)
     # print(f[key].value)
-df = pd.read_hdf('product1.h5', key='df8')
+# df = pd.read_hdf('product1.h5', key='df7')
 print(df)
+'''
+# Open the HDF5 file
+with h5py.File('product1.h5', 'r') as file:
+    dataframes=pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])  
+    for key in file.keys():
+        # Read the dataframe from the HDF5 file
+        dataframe = pd.read_hdf('product1.h5', key)
 
+        # Append the dataframe to the list
+        dataframes._append(dataframe)
+
+# Print the number of dataframes read from the HDF5 file
+print(f"Read {len(dataframes)} dataframes from the HDF5 file")
+
+item_std_devs = {}
+for dataframe in dataframes:
+    item_name = dataframe['Product ID'].iloc[0]
+    std_dev = np.std(dataframe['Unit Price'])
+    item_std_devs[item_name] = std_dev
+
+# Print the item names and their corresponding standard deviations
+for item_name, std_dev in item_std_devs.items():
+    print(f"Item: {item_name}, Std Dev: {std_dev}")
+
+# Prepare the data for the graph
+x = list(item_std_devs.keys())
+y = list(item_std_devs.values())
+
+# Create a bar graph
+fig = go.Figure(data=[go.Bar(x=x, y=y)])
+
+# Customize the graph
+fig.update_layout(
+    title="Standard Deviation of Pricing for Each Item",
+    xaxis_title="Item",
+    yaxis_title="Standard Deviation",
+)
+
+# Save the graph as an HTML file
+report_file = 'test2.html'
+fig.write_html(report_file)
+
+
+
+'''
+'''
 # Calculate the standard deviation using pandas' function
-std_dev_unit = df.groupby('Zip Code')['Unit Price'].std()
+std_dev_unit = df.groupby('Product ID')['Unit Price'].std()
 std_dev_displayed = df.groupby('Zip Code')['Shown Price'].std()
 
 # Generate the graph using Plotly
 fig = go.Figure()
 fig.add_trace(go.Bar(x=std_dev_unit.index, y=std_dev_unit.values))
-fig.update_layout(title='Standard Deviation of the Unit Pricing by Item', xaxis_title='Zip Code', yaxis_title='Standard Deviation')
+fig.update_layout(title='Standard Deviation of the Unit Pricing by Item', xaxis_title='Product ID', yaxis_title='Standard Deviation')
 
-report_file = 'test.html'
+report_file = 'test2.html'
 fig.write_html(report_file)
+'''
+

fp2_14.py

@@ -0,0 +1,280 @@
+from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+from concurrent.futures import TimeoutError
+import pandas as pd
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+import selenium.common.exceptions
+import re
+from datetime import datetime
+import h5py
+import sys
+import ntplib
+import arrow
+
+# This fp2_14.py is used to load the new price and the timestamp, and append such data in an existing key in the HDF5 file "product1.h5"
+
+def get_product_prices(project_id, subscription_name, url):
+    # Create an empty dataframe to store the prices
+    df_new = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
+    print(f"Created a dataframe: {df_new}")
+
+    # Configure the Selenium driver options
+    options = Options()
+    options.add_argument('--disable-gpu')
+    driver = webdriver.Chrome(options=options)
+
+    # Set up Pub/Sub subscription
+    '''
+    project_id = "fp-20230425"
+    subscription_name = "fp-subscription-NFL-zipcodes"
+    '''
+    subscriber = pubsub_v1.SubscriberClient()
+    subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+
+    # Load the product page and extract the __NEXT_DATA__ data
+    # url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
+    product_id = url.split("/")[-1][:5]
+    # product_id = url.split("/")[-1]
+    driver.delete_all_cookies
+    driver.get(url)
+
+    def get_element_text(class_name_prefix):
+        '''
+        Return the text in the first element with a given class name.
+
+        Note that XPath doesn't have a function to check if a given word is present
+        in a list of words, so using contains() will match a class name even if
+        the class name is a substring of the class name being matched against.
+        To prevent this, we use spaces as a delimiter at the beginning and end of
+        the class name, and ensure there's only one space before and after all
+        class names in the class name list.
+
+        Example: consider two class names with cache-busting suffixes:
+        * Header_soldStatusNotice__12345
+        * Header_soldStatusNoticeBackground__ABCDE
+
+        Matching against Header_soldStatusNotice using a naive implementation with
+        `contains()` would match both class names, even though only the first one
+        is the correct match. 
+        '''
+        # Construct xpath selector.  
+        xpath_str = "//*[contains("               # Step 1: search for a class name by matching the beginning of the class name against...
+        xpath_str += "concat(' ', "               # Step 2: a leading space, followed by...
+        xpath_str += "normalize-space(@class), "  # Step 3: the list of class names, separated by single spaces...
+        xpath_str += "' '), "                     # Step 4: a trailing space...
+        xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
+        try:
+            match = driver.find_element(By.XPATH, xpath_str)
+            return match.get_attribute('textContent')
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            pass
+
+        # We didn't find a match, try again by doing a prefix match and assume
+        # the full class name ends with two underscores followed by a cache-
+        # busting string.    
+        xpath_str = "//*[contains("                 # Step 1: search for a class name by matching the beginning of the class name against...
+        xpath_str += "concat(' ', "                 # Step 2: a leading space, followed by...
+        xpath_str += "normalize-space(@class)), "   # Step 3: the list of class names, separated by single spaces...
+        xpath_str += f"' {class_name_prefix}__')]"  # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
+        try:
+            match = driver.find_element(By.XPATH, xpath_str)
+            return match.get_attribute('textContent')
+        except selenium.common.exceptions.NoSuchElementException as nsee:
+            pass
+
+        # No match, return empty string.
+        return ''
+
+
+    def modal_close():
+        try:    
+            modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
+            print(f"modal_close = {modal_close}")
+            modal_close.click()
+        except:
+            pass
+
+    def notice_close():
+        try:
+            # Clear any notice
+            notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
+            # print(f"notice_close = {notice_close}")
+            notice_close.click()
+        except:
+            pass
+
+    def cookie_close():
+        try:  
+            cookie_button = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
+            )
+            print(f"cookie_button = {cookie_button}")
+            cookie_button.click()
+        except:
+            pass
+
+    cookie_close()
+    modal_close()
+    # Define callback function for Pub/Sub messages
+    def callback(message):
+        # Extract zip code from message data
+        zip_code = message.message.data.decode('utf-8')
+
+        print(f"Received zip code: {zip_code}")
+
+        try:
+            
+            modal_close()
+            notice_close()
+
+            # Enter zip code in the input field and click the "Check" button
+            zip_button = WebDriverWait(driver, 5).until(
+                EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
+            )
+            # print(f"{zip_button:}")
+            zip_button.click()
+
+            # zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
+            zip_input = WebDriverWait(driver, 2).until(
+                EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
+            )
+            # print(f"{zip_input:}")
+            zip_input.click()
+            # print("clearing zip_input")
+            zip_input.send_keys("\b\b\b\b\b")
+            # print("sending zip code")
+            zip_input.send_keys(zip_code)
+
+            check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
+            check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
+            # print(f"{check_button:}")
+            check_button.click()
+
+            notice_close()
+            modal_close()
+
+            # Record sales count, if any
+            sales_count_element = get_element_text('Header_count')
+            count_start_index = sales_count_element.index('sold ') + len('sold ')
+            count_end_index = sales_count_element.index('+')
+            sales_count = sales_count_element[count_start_index:count_end_index]
+            # Record any sold status notice
+            sold_status = get_element_text('UnavailableCartbtn_container')
+            if sold_status == 'Notify me':
+                sold_status = 'Sold Out'
+
+        # Extract the price of rice crackers for this zip code
+            # Check if there is a minimum order quantity
+            min_quantity: float = 0
+            total_price: float = 0
+            unit_price: float = 0
+            try:
+                minimum_quantity_str = WebDriverWait(driver, 5).until(EC.presence_of_element_located(((By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML'))))
+                if minimum_quantity_str:
+                    regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
+                    match = re.search(regexp,minimum_quantity_str)
+                    min_quantity, total_price = match.groups()
+            except selenium.common.exceptions.NoSuchElementException as nsee:
+                print("No minimum purhcase string found")
+            except Exception as e:
+                print(f"Exception when finding minimum purchase string: {e}")
+    
+
+
+            ld_json_element = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//script[@type='application/ld+json']")))
+            ld_json_text = ld_json_element.get_attribute('innerHTML')
+            ld_json_start_index = ld_json_text.index('{')
+            ld_json_end_index = ld_json_text.rindex('}') + 1
+            ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
+            ld_json = json.loads(ld_json_json)
+            ld_json_price = ld_json['offers']['price']
+            # Print the price
+            print(ld_json_price)
+
+            unit_price_element = WebDriverWait(driver, 5).until(EC.presence_of_element_located(((By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText'))))
+            unit_price = unit_price_element.replace("$", "")
+            print(f"unit_price = {unit_price}" )
+
+            subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
+            # Append the zip code and price to the dataframe
+            #append_to_df(zip_code, price)
+            new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]})
+            df_new = pd.concat([df_new, new_row], ignore_index=True)
+            return df_new
+            #print(f"Recorded the zip code and price: {zip_code},{price}")
+            #print(f"Updated a dataframe: {df_new}")
+
+        except Exception as e:
+            print(f"Caught exception: {e}")
+            print(f"Price not found for zip code {zip_code}")
+            return df_new
+
+    # Start receiving messages (sync version)
+    while True: 
+        pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
+        messages = []
+        try:
+            pull_response = subscriber.pull(request = pull_request)
+            messages = pull_response.received_messages
+        except DeadlineExceeded:
+            pass
+
+        if messages:
+            df_new = callback(messages[0])
+            print(f"Updated a dataframe: {df_new}")
+        else:
+            print(f"no messages, trying again at {time.asctime()}")
+            break
+    return df_new
+
+'''
+If using sys to get the function, please remove the following part of code.
+'''
+if __name__ == '__main__':
+    project_id = input("Enter the project ID: ")
+    subscription_name = input("Enter the subscription name: ")
+    url = input("Enter the URL: ")
+
+    get_product_prices(project_id, subscription_name, url)
+
+
+df_new = get_product_prices("fp-20230425", "fp-subscription-NFL-zipcodes", "https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119")
+
+# Now we have the new dataframe "df" recording the new price listed in weee's page
+# Load such data in the existing HDF5 file
+file_path = 'product1.h5'
+with h5py.File(file_path, 'a') as file:
+    # Load the existing dataframe 'df' from the HDF5 file
+    # REPLACE KEY TO GET THE EXISTING DATAFRAME FOR EACH DIFFERENT PRODUCT
+    df = pd.read_hdf(file, key='df')
+
+    # Add timestamp to the new dataframe "df_new"
+    # timestamp = datetime.now()
+    '''
+    To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
+    provides time synchronization with a reliable time server.
+    Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
+    sensible and human-friendly approach to format and convert the timestamps.
+    '''
+    # Fetch the time from an NTP server
+    ntp_client = ntplib.NTPClient()
+    response = ntp_client.request('pool.ntp.org')
+    ntp_time = datetime.fromtimestamp(response.tx_time)
+    timestamp = arrow.get(ntp_time).to('local')
+    df_new['Timestamp'] = timestamp
+
+    # Append 'df_new' to the existing df stored in the key
+    df_update = df.append(df_new, ignore_index=True)
+
+    # Save 'df2' to the HDF5 file with the key 'df'
+    # REPLACE KEY FOR DIFFERENT PRODUCT
+    df_update.to_hdf(file, key='df', mode='a')
+
+print("Data appended and saved successfully.")

fp3_1.py

@@ -0,0 +1,166 @@
+import sqlalchemy
+from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Float, DateTime
+from sqlalchemy.orm import declarative_base
+from sqlalchemy.orm import relationship, sessionmaker
+import sys
+from flask import Flask, render_template
+
+# Create the Flask application
+app = Flask(__name__)
+
+# Create the database engine
+engine = create_engine('sqlite:///scraper_design.db')
+
+# Create a session factory
+Session = sessionmaker(bind=engine)
+
+# Create a base class
+Base = declarative_base()
+
+# Define the product class
+class Product(Base):
+    __tablename__ = 'product'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    SKU_PLU_UPC_number = Column(String)
+    price_histories = relationship("PriceHistory", back_populates="product")
+
+# Define the retailer class
+class Retailer(Base):
+    __tablename__ = 'retailer'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    products = relationship("Product", secondary="retailer_product")
+
+# Define the price class
+class Price(Base):
+    __tablename__ = 'price'
+    id = Column(Integer, primary_key=True)
+    MOQ = Column(Integer)
+    price = Column(Float)
+
+# Define the scraperjob class
+class ScraperJob(Base):
+    __tablename__ = 'scraper_job'
+    id = Column(Integer, primary_key=True)
+    start_time = Column(DateTime)
+    end_time = Column(DateTime)
+    parameters = Column(String)
+    statistics = Column(String)
+
+# Define the price history class
+class PriceHistory(Base):
+    __tablename__ = 'price_history'
+    id = Column(Integer, primary_key=True)
+    location = Column(String)
+    zip_code = Column(Integer)
+    city_name = Column(String)
+    price_at_time = Column(DateTime)
+    price = Column(Float)
+    scraper_job_id = Column(Integer, ForeignKey('scraper_job.id'))
+    product_id = Column(Integer, ForeignKey('product.id'))
+    product = relationship("Product", back_populates="price_histories")
+    scraper_job = relationship("ScraperJob")
+
+# Define the user class
+class User(Base):
+    __tablename__ = 'user'
+    id = Column(Integer, primary_key=True)
+    username = Column(String)
+    credentials = Column(String)
+    creation_date = Column(DateTime)
+    modification_date = Column(DateTime)
+    def __str__(self):
+        return f"{self.id=} {self.username=} {self.creation_date=} {self.modification_date=}"
+
+class RetailerProduct(Base):
+    __tablename__ = 'retailer_product'
+    retailer_id = Column(Integer, ForeignKey('retailer.id'), primary_key=True)
+    product_id = Column(Integer, ForeignKey('product.id'), primary_key=True)
+
+# Create the database tables
+Base.metadata.create_all(engine)
+
+@app.route('/')
+def homepage():
+    # Create a session
+    session = Session()
+
+    # Get all the classes from SQLAlchemy
+    classes = Base.__subclasses__()
+
+    statement = sqlalchemy.select(User)
+    users = session.scalars(statement).all()
+
+    # Render the template with the classes
+    return render_template('test3.html', classes=classes, users=users)
+
+# Create command-line arguments for the user class
+def main():
+    if len(sys.argv) < 2:
+        app.run()
+
+    command = sys.argv[1]
+
+    if command == "create":
+        if len(sys.argv) != 3:
+            print("Usage: python user_tool.py create [username]")
+            return
+
+        username = sys.argv[2]
+
+        # Create a new user object
+        user = User(username=username)
+
+        # Add the user to the database
+        session = Session()
+        session.add(user)
+        session.commit()
+
+        print("User created successfully.")
+
+    elif command == "edit":
+        if len(sys.argv) != 4:
+            print("Usage: python user_tool.py edit [user_id] [new_username]")
+            return
+
+        user_id = int(sys.argv[2])
+        new_username = sys.argv[3]
+
+        # Retrieve the user from the database
+        session = Session()
+        user = session.query(User).get(user_id)
+
+        if user:
+            # Update the user object
+            user.username = new_username
+            session.commit()
+            print("User updated successfully.")
+        else:
+            print("User not found.")
+
+    elif command == "delete":
+        if len(sys.argv) != 3:
+            print("Usage: python user_tool.py delete [user_id]")
+            return
+
+        user_id = int(sys.argv[2])
+
+        # Retrieve the user from the database
+        session = Session()
+        user = session.query(User).get(user_id)
+
+        if user:
+            # Delete the user from the database
+            session.delete(user)
+            session.commit()
+            print("User deleted successfully.")
+        else:
+            print("User not found.")
+
+    else:
+        print("Usage: python user_tool.py [create|edit|delete]")
+        return
+
+if __name__ == "__main__":
+    main()

fp_merge.py

@@ -0,0 +1,100 @@
+from fp2_14 import get_product_prices
+import h5py
+import pandas as pd
+import datetime
+import sys
+from datetime import datetime
+import ntplib
+import arrow
+import concurrent.futures
+import pandas as pd
+
+# Define the subset of urls
+url_list = [
+    'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
+    'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
+    'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
+    'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
+    'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
+    'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
+    'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
+    'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
+    'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
+    'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
+    'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
+]
+
+
+# Define the function to run the scraper
+def run_scraper(url):
+    project_id = "fp-20230425"
+    subscription_name = "fp-subscription-NFL-zipcodes"
+    df_new = get_product_prices(project_id, subscription_name, url)
+
+    # Add timestamp to the new dataframe "df_new"
+    '''
+    To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
+    provides time synchronization with a reliable time server.
+    Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
+    sensible and human-friendly approach to format and convert the timestamps.
+    '''
+    # Fetch the time from an NTP server
+    ntp_client = ntplib.NTPClient()
+    response = ntp_client.request('pool.ntp.org')
+    ntp_time = datetime.fromtimestamp(response.tx_time)
+    timestamp = arrow.get(ntp_time).to('local')
+    df_new['Timestamp'] = timestamp
+
+    # Append 'df_new' to the existing df stored in the key
+    df_update = df_new.append(df_new, ignore_index=True)
+
+    return df_update
+
+
+'''
+# When using sys to call the 'get_product_prices' function
+
+if __name__ == '__main__':
+    # Check if the required command-line arguments are provided
+    if len(sys.argv) < 4:
+        print("Usage: python test_fp2_14.py project_id subscription_name url")
+        sys.exit(1)
+
+    project_id = sys.argv[1]
+    subscription_name = sys.argv[2]
+    url = sys.argv[3:]
+
+    get_product_prices(project_id, subscription_name, url)
+'''
+
+
+if __name__ == '__main__':
+    # Determine the number of available CPUs
+    # cpu_count = multiprocessing.cpu_count()
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        future_to_url = {executor.submit(run_scraper, url): url for url in url_list}
+        results = []
+        for future in concurrent.futures.as_completed(future_to_url):
+            url = future_to_url[future]
+            try:
+                result = future.result()
+                print(f"Scraped data for URL: {url}")
+                results.append(result)
+            except Exception as e:
+                print(f"Failed to scrape data for URL: {url}\nError: {e}")
+
+
+# Now we have the new dataframe "df_update" recording the new price listed in weee's page
+# Load such data in the existing HDF5 file
+file_path = 'product1.h5'
+with pd.HDFStore(file_path) as store:
+    for url, result in zip(url_list, results):
+        # Extract the product ID from the URL
+        product_id = url.split("/")[-1][:5]
+
+        # Save the dataframe to the HDF5 file with a unique key
+        key = f'df_{product_id}'
+        store[key] = result
+        result.to_hdf('product_data.h5', key=key, mode='a')
+        print("Data appended and saved successfully.")

public

@@ -0,0 +1,9 @@
+---- BEGIN SSH2 PUBLIC KEY ----
+Comment: "rsa-key-20230531"
+AAAAB3NzaC1yc2EAAAADAQABAAABAQDEZW0iQpBPPRGM7QCdc3GqefkqUqq5i727
+MEYvf7gy0h7Zvz72cvnDtnVF6MknCGVZ1S3uY+HeKUtujE3+z9tJ5zIxO+TxthJe
+8gbH9MUkElKX8qJN/nF1H4dLiW+77ARsT70rDoHayWccGGnwC/Gs9xXVF+Ni0JD5
+TKcGoye3Ps2Ni/KJAzkn78JyfrAb+Ail4DG4snCf1q/jXTuB39qYTTg94FJEqwe1
+dhPIzRiOjFT3eSmHAXdSuAXFjpXAq+bV8R2uxKsgNfvxyV4vlKwFhYTq+kHwDWX7
+RtLQUUs8mSTfbnL2LuE+T2d8sBtj9wi+gODFl5dDAVZkmuKrRgLF
+---- END SSH2 PUBLIC KEY ----

requirements.txt

@@ -4,3 +4,5 @@ pandas
 numpy
 h5py
 selenium
+sqlalchemy
+flask

templates/test3.html

@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Class Dump</title>
+</head>
+<body>
+    <h1>Class Dump</h1>
+    <ul>
+        {% for class in classes %}
+        <li>{{ class.__name__ }}</li>
+        {% endfor %}
+    </ul>
+
+    <h1>User Dump</h1>
+    <ul>
+        {% for user in users %}
+        <li>{{ user }}</li>
+        {% endfor %}
+    </ul>
+</body>
+</html>

test_fp2_14.py

@@ -0,0 +1,63 @@
+from fp2_14 import get_product_prices
+import h5py
+import pandas as pd
+import datetime
+import sys
+from datetime import datetime
+import ntplib
+import arrow
+
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-NFL-zipcodes"
+url = "https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f"
+
+df_new = get_product_prices(project_id, subscription_name, url)
+
+'''
+# When using sys to call the 'get_product_prices' function
+
+if __name__ == '__main__':
+    # Check if the required command-line arguments are provided
+    if len(sys.argv) < 4:
+        print("Usage: python test_fp2_14.py project_id subscription_name url")
+        sys.exit(1)
+
+    project_id = sys.argv[1]
+    subscription_name = sys.argv[2]
+    url = sys.argv[3:]
+
+    get_product_prices(project_id, subscription_name, url)
+'''
+
+
+# Now we have the new dataframe "df" recording the new price listed in weee's page
+# Load such data in the existing HDF5 file
+file_path = 'product1.h5'
+with h5py.File(file_path, 'a') as file:
+    # Load the existing dataframe 'df' from the HDF5 file
+    # REPLACE KEY TO GET THE EXISTING DATAFRAME FOR EACH DIFFERENT PRODUCT
+    df = pd.read_hdf(file, key='df')
+
+    # Add timestamp to the new dataframe "df_new"
+    # timestamp = datetime.now()
+    '''
+    To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
+    provides time synchronization with a reliable time server.
+    Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
+    sensible and human-friendly approach to format and convert the timestamps.
+    '''
+    # Fetch the time from an NTP server
+    ntp_client = ntplib.NTPClient()
+    response = ntp_client.request('pool.ntp.org')
+    ntp_time = datetime.fromtimestamp(response.tx_time)
+    timestamp = arrow.get(ntp_time).to('local')
+    df_new['Timestamp'] = timestamp
+
+    # Append 'df_new' to the existing df stored in the key
+    df_update = df.append(df_new, ignore_index=True)
+
+    # Save 'df2' to the HDF5 file with the key 'df'
+    # REPLACE KEY FOR DIFFERENT PRODUCT
+    df_update.to_hdf(file, key='df', mode='a')
+
+print("Data appended and saved successfully.")