Multiprocessing.py

@@ -50,8 +50,6 @@ def run_scraper(url):
     subscriber = pubsub_v1.SubscriberClient()
     subscription_path = subscriber.subscription_path(project_id, subscription_name)
 
-    url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
-    product_id = url.split("/")[-1][:5]
     # product_id = url.split("/")[-1]
     driver.delete_all_cookies
     driver.get(url)
@@ -238,19 +236,21 @@ def run_scraper(url):
             print(f"no messages, trying again at {time.asctime()}")
             break
         # subprocess.call(['python', 'fp2_13.py', url])
-        return df
+    return run_scraper(url)
 
 if __name__ == '__main__':
     # Determine the number of available CPUs
-    cpu_count = multiprocessing.cpu_count()
+    # cpu_count = multiprocessing.cpu_count()
 
-    with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count) as executor:
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
         future_to_url = {executor.submit(run_scraper, url): url for url in url_list}
+        results = []
         for future in concurrent.futures.as_completed(future_to_url):
             url = future_to_url[future]
             try:
                 result = future.result()
                 print(f"Scraped data for URL: {url}")
+                results.append(result)
             except Exception as e:
                 print(f"Failed to scrape data for URL: {url}\nError: {e}")
 
@@ -263,10 +263,12 @@ pool.close()
 pool.join()
 '''
 
-for url, result in zip(url_list, result):
+with pd.HDFStore('product_data.h5') as store:
+    for url, result in zip(url_list, results):
         # Extract the product ID from the URL
         product_id = url.split("/")[-1].split("?")[0]
 
         # Save the dataframe to the HDF5 file with a unique key
         key = f'df_{product_id}'
-    result.to_hdf('product_data.h5', key=key, mode='a')
+        store[key] = result
+        # result.to_hdf('product_data.h5', key=key, mode='a')