Multiprocessing.py

@@ -40,6 +40,7 @@ def run_scraper(url):
     # scraping logic
     df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
     print(f"Created a dataframe: {df}")
+    product_id = url.split("/")[-1][:5]
 
     options = Options()
     options.add_argument('--disable-gpu')
@@ -236,7 +237,7 @@ def run_scraper(url):
             print(f"no messages, trying again at {time.asctime()}")
             break
         # subprocess.call(['python', 'fp2_13.py', url])
-    return run_scraper(url)
+    return df
 
 if __name__ == '__main__':
     # Determine the number of available CPUs
@@ -266,7 +267,7 @@ pool.join()
 with pd.HDFStore('product_data.h5') as store:
     for url, result in zip(url_list, results):
         # Extract the product ID from the URL
-        product_id = url.split("/")[-1].split("?")[0]
+        product_id = url.split("/")[-1][:5]
 
         # Save the dataframe to the HDF5 file with a unique key
         key = f'df_{product_id}'

View_h5_file.py

@@ -57,6 +57,7 @@ fig.write_html(report_file)
 
 
 
+'''
 '''
 # Calculate the standard deviation using pandas' function
 std_dev_unit = df.groupby('Product ID')['Unit Price'].std()
@@ -69,5 +70,5 @@ fig.update_layout(title='Standard Deviation of the Unit Pricing by Item', xaxis_
 
 report_file = 'test2.html'
 fig.write_html(report_file)
-
+'''