diff --git a/Multiprocessing.py b/Multiprocessing.py index 901899f..65fb5d1 100644 --- a/Multiprocessing.py +++ b/Multiprocessing.py @@ -50,8 +50,6 @@ def run_scraper(url): subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path(project_id, subscription_name) - url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f' - product_id = url.split("/")[-1][:5] # product_id = url.split("/")[-1] driver.delete_all_cookies driver.get(url) @@ -238,19 +236,21 @@ def run_scraper(url): print(f"no messages, trying again at {time.asctime()}") break # subprocess.call(['python', 'fp2_13.py', url]) - return df + return run_scraper(url) if __name__ == '__main__': # Determine the number of available CPUs - cpu_count = multiprocessing.cpu_count() + # cpu_count = multiprocessing.cpu_count() - with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: future_to_url = {executor.submit(run_scraper, url): url for url in url_list} + results = [] for future in concurrent.futures.as_completed(future_to_url): url = future_to_url[future] try: result = future.result() print(f"Scraped data for URL: {url}") + results.append(result) except Exception as e: print(f"Failed to scrape data for URL: {url}\nError: {e}") @@ -263,10 +263,12 @@ pool.close() pool.join() ''' -for url, result in zip(url_list, result): - # Extract the product ID from the URL - product_id = url.split("/")[-1].split("?")[0] +with pd.HDFStore('product_data.h5') as store: + for url, result in zip(url_list, results): + # Extract the product ID from the URL + product_id = url.split("/")[-1].split("?")[0] - # Save the dataframe to the HDF5 file with a unique key - key = f'df_{product_id}' - result.to_hdf('product_data.h5', key=key, mode='a') \ No newline at end of file + # Save the dataframe to the HDF5 file with a unique key + key = f'df_{product_id}' + store[key] = result + # result.to_hdf('product_data.h5', key=key, mode='a') \ No newline at end of file