SVIG/python-bootcamp
7
0
Fork 1
Code Issues Pull Requests 1 Releases Wiki Activity

Update

Browse Source
This commit is contained in:
Karenzz
2023-05-19 15:31:34 -04:00
parent 09fdbb8b5d
commit 80bc996230
1 changed files with 13 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Multiprocessing.py
+13 -11
View File
@@ -50,8 +50,6 @@ def run_scraper(url):
subscriber = pubsub_v1.SubscriberClient() subscriber = pubsub_v1.SubscriberClient()
subscription_path = subscriber.subscription_path(project_id, subscription_name) subscription_path = subscriber.subscription_path(project_id, subscription_name)
url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
product_id = url.split("/")[-1][:5]
# product_id = url.split("/")[-1] # product_id = url.split("/")[-1]
driver.delete_all_cookies driver.delete_all_cookies
driver.get(url) driver.get(url)
@@ -238,19 +236,21 @@ def run_scraper(url):
print(f"no messages, trying again at {time.asctime()}") print(f"no messages, trying again at {time.asctime()}")
break break
# subprocess.call(['python', 'fp2_13.py', url]) # subprocess.call(['python', 'fp2_13.py', url])
return df return run_scraper(url)
if __name__ == '__main__': if __name__ == '__main__':
# Determine the number of available CPUs # Determine the number of available CPUs
cpu_count = multiprocessing.cpu_count() # cpu_count = multiprocessing.cpu_count()
with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count) as executor: with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
future_to_url = {executor.submit(run_scraper, url): url for url in url_list} future_to_url = {executor.submit(run_scraper, url): url for url in url_list}
results = []
for future in concurrent.futures.as_completed(future_to_url): for future in concurrent.futures.as_completed(future_to_url):
url = future_to_url[future] url = future_to_url[future]
try: try:
result = future.result() result = future.result()
print(f"Scraped data for URL: {url}") print(f"Scraped data for URL: {url}")
results.append(result)
except Exception as e: except Exception as e:
print(f"Failed to scrape data for URL: {url}\nError: {e}") print(f"Failed to scrape data for URL: {url}\nError: {e}")
@@ -263,10 +263,12 @@ pool.close()
pool.join() pool.join()
''' '''
for url, result in zip(url_list, result): with pd.HDFStore('product_data.h5') as store:
# Extract the product ID from the URL for url, result in zip(url_list, results):
product_id = url.split("/")[-1].split("?")[0] # Extract the product ID from the URL
product_id = url.split("/")[-1].split("?")[0]
# Save the dataframe to the HDF5 file with a unique key # Save the dataframe to the HDF5 file with a unique key
key = f'df_{product_id}' key = f'df_{product_id}'
result.to_hdf('product_data.h5', key=key, mode='a') store[key] = result
# result.to_hdf('product_data.h5', key=key, mode='a')