Created a new dataframe to store all keys data

Multiprocessing.py

@@ -50,8 +50,6 @@ def run_scraper(url):
     subscriber = pubsub_v1.SubscriberClient()
     subscription_path = subscriber.subscription_path(project_id, subscription_name)
 
-    url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
-    product_id = url.split("/")[-1][:5]
     # product_id = url.split("/")[-1]
     driver.delete_all_cookies
     driver.get(url)
@@ -238,19 +236,21 @@ def run_scraper(url):
             print(f"no messages, trying again at {time.asctime()}")
             break
         # subprocess.call(['python', 'fp2_13.py', url])
-        return df
+    return run_scraper(url)
 
 if __name__ == '__main__':
     # Determine the number of available CPUs
-    cpu_count = multiprocessing.cpu_count()
+    # cpu_count = multiprocessing.cpu_count()
 
-    with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count) as executor:
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
         future_to_url = {executor.submit(run_scraper, url): url for url in url_list}
+        results = []
         for future in concurrent.futures.as_completed(future_to_url):
             url = future_to_url[future]
             try:
                 result = future.result()
                 print(f"Scraped data for URL: {url}")
+                results.append(result)
             except Exception as e:
                 print(f"Failed to scrape data for URL: {url}\nError: {e}")
 
@@ -263,10 +263,12 @@ pool.close()
 pool.join()
 '''
 
-for url, result in zip(url_list, result):
+with pd.HDFStore('product_data.h5') as store:
+    for url, result in zip(url_list, results):
         # Extract the product ID from the URL
         product_id = url.split("/")[-1].split("?")[0]
 
         # Save the dataframe to the HDF5 file with a unique key
         key = f'df_{product_id}'
-    result.to_hdf('product_data.h5', key=key, mode='a')
+        store[key] = result
+        # result.to_hdf('product_data.h5', key=key, mode='a')

View_h5_file.py

@@ -7,19 +7,67 @@ f = h5py.File('product1.h5','r')
 list(f.keys())
 for key in f.keys():
     print(f[key])
+    df = pd.read_hdf('product1.h5', key='df')
+    df = df._append(pd.read_hdf('product1.h5', key))
     # print(f[key].shape)
     # print(f[key].value)
-df = pd.read_hdf('product1.h5', key='df8')
+# df = pd.read_hdf('product1.h5', key='df7')
 print(df)
+'''
+# Open the HDF5 file
+with h5py.File('product1.h5', 'r') as file:
+    dataframes=pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])  
+    for key in file.keys():
+        # Read the dataframe from the HDF5 file
+        dataframe = pd.read_hdf('product1.h5', key)
 
+        # Append the dataframe to the list
+        dataframes._append(dataframe)
+
+# Print the number of dataframes read from the HDF5 file
+print(f"Read {len(dataframes)} dataframes from the HDF5 file")
+
+item_std_devs = {}
+for dataframe in dataframes:
+    item_name = dataframe['Product ID'].iloc[0]
+    std_dev = np.std(dataframe['Unit Price'])
+    item_std_devs[item_name] = std_dev
+
+# Print the item names and their corresponding standard deviations
+for item_name, std_dev in item_std_devs.items():
+    print(f"Item: {item_name}, Std Dev: {std_dev}")
+
+# Prepare the data for the graph
+x = list(item_std_devs.keys())
+y = list(item_std_devs.values())
+
+# Create a bar graph
+fig = go.Figure(data=[go.Bar(x=x, y=y)])
+
+# Customize the graph
+fig.update_layout(
+    title="Standard Deviation of Pricing for Each Item",
+    xaxis_title="Item",
+    yaxis_title="Standard Deviation",
+)
+
+# Save the graph as an HTML file
+report_file = 'test2.html'
+fig.write_html(report_file)
+
+
+
+'''
 # Calculate the standard deviation using pandas' function
-std_dev_unit = df.groupby('Zip Code')['Unit Price'].std()
+std_dev_unit = df.groupby('Product ID')['Unit Price'].std()
 std_dev_displayed = df.groupby('Zip Code')['Shown Price'].std()
 
 # Generate the graph using Plotly
 fig = go.Figure()
 fig.add_trace(go.Bar(x=std_dev_unit.index, y=std_dev_unit.values))
-fig.update_layout(title='Standard Deviation of the Unit Pricing by Item', xaxis_title='Zip Code', yaxis_title='Standard Deviation')
+fig.update_layout(title='Standard Deviation of the Unit Pricing by Item', xaxis_title='Product ID', yaxis_title='Standard Deviation')
 
-report_file = 'test.html'
+report_file = 'test2.html'
 fig.write_html(report_file)
+
+