From 217d8e90ab5bff30ef98f3e517a54e34457a9cf8 Mon Sep 17 00:00:00 2001
From: Karen Zeng <karen.zeng@outlook.com>
Date: Thu, 1 Jun 2023 15:31:03 -0400
Subject: [PATCH] Merged fp2_14 & multiprocessing

---
 fp_merge.py | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 100 insertions(+)
 create mode 100644 fp_merge.py

diff --git a/fp_merge.py b/fp_merge.py
new file mode 100644
index 0000000..c53290c
--- /dev/null
+++ b/fp_merge.py
@@ -0,0 +1,100 @@
+from fp2_14 import get_product_prices
+import h5py
+import pandas as pd
+import datetime
+import sys
+from datetime import datetime
+import ntplib
+import arrow
+import concurrent.futures
+import pandas as pd
+
+# Define the subset of urls
+url_list = [
+    'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
+    'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
+    'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
+    'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
+    'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
+    'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
+    'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
+    'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
+    'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
+    'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
+    'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
+]
+
+
+# Define the function to run the scraper
+def run_scraper(url):
+    project_id = "fp-20230425"
+    subscription_name = "fp-subscription-NFL-zipcodes"
+    df_new = get_product_prices(project_id, subscription_name, url)
+
+    # Add timestamp to the new dataframe "df_new"
+    '''
+    To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
+    provides time synchronization with a reliable time server.
+    Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
+    sensible and human-friendly approach to format and convert the timestamps.
+    '''
+    # Fetch the time from an NTP server
+    ntp_client = ntplib.NTPClient()
+    response = ntp_client.request('pool.ntp.org')
+    ntp_time = datetime.fromtimestamp(response.tx_time)
+    timestamp = arrow.get(ntp_time).to('local')
+    df_new['Timestamp'] = timestamp
+
+    # Append 'df_new' to the existing df stored in the key
+    df_update = df_new.append(df_new, ignore_index=True)
+
+    return df_update
+
+
+'''
+# When using sys to call the 'get_product_prices' function
+
+if __name__ == '__main__':
+    # Check if the required command-line arguments are provided
+    if len(sys.argv) < 4:
+        print("Usage: python test_fp2_14.py project_id subscription_name url")
+        sys.exit(1)
+
+    project_id = sys.argv[1]
+    subscription_name = sys.argv[2]
+    url = sys.argv[3:]
+
+    get_product_prices(project_id, subscription_name, url)
+'''
+
+
+if __name__ == '__main__':
+    # Determine the number of available CPUs
+    # cpu_count = multiprocessing.cpu_count()
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        future_to_url = {executor.submit(run_scraper, url): url for url in url_list}
+        results = []
+        for future in concurrent.futures.as_completed(future_to_url):
+            url = future_to_url[future]
+            try:
+                result = future.result()
+                print(f"Scraped data for URL: {url}")
+                results.append(result)
+            except Exception as e:
+                print(f"Failed to scrape data for URL: {url}\nError: {e}")
+
+
+# Now we have the new dataframe "df_update" recording the new price listed in weee's page
+# Load such data in the existing HDF5 file
+file_path = 'product1.h5'
+with pd.HDFStore(file_path) as store:
+    for url, result in zip(url_list, results):
+        # Extract the product ID from the URL
+        product_id = url.split("/")[-1][:5]
+
+        # Save the dataframe to the HDF5 file with a unique key
+        key = f'df_{product_id}'
+        store[key] = result
+        result.to_hdf('product_data.h5', key=key, mode='a')
+        print("Data appended and saved successfully.")
\ No newline at end of file