SVIG/python-bootcamp
7
0
Fork 1
Code Issues Pull Requests 1 Releases Wiki Activity

Modified multiprocessing

Browse Source
This commit is contained in:
Karenzz
2023-05-22 17:04:22 -04:00
parent a2fb3b29aa
commit 97d4d8eb49
2 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Multiprocessing.py
+3 -2
View File
@@ -40,6 +40,7 @@ def run_scraper(url):
# scraping logic # scraping logic
df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status']) df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
print(f"Created a dataframe: {df}") print(f"Created a dataframe: {df}")
product_id = url.split("/")[-1][:5]
options = Options() options = Options()
options.add_argument('--disable-gpu') options.add_argument('--disable-gpu')
@@ -236,7 +237,7 @@ def run_scraper(url):
print(f"no messages, trying again at {time.asctime()}") print(f"no messages, trying again at {time.asctime()}")
break break
# subprocess.call(['python', 'fp2_13.py', url]) # subprocess.call(['python', 'fp2_13.py', url])
return run_scraper(url) return df
if __name__ == '__main__': if __name__ == '__main__':
# Determine the number of available CPUs # Determine the number of available CPUs
@@ -266,7 +267,7 @@ pool.join()
with pd.HDFStore('product_data.h5') as store: with pd.HDFStore('product_data.h5') as store:
for url, result in zip(url_list, results): for url, result in zip(url_list, results):
# Extract the product ID from the URL # Extract the product ID from the URL
product_id = url.split("/")[-1].split("?")[0] product_id = url.split("/")[-1][:5]
# Save the dataframe to the HDF5 file with a unique key # Save the dataframe to the HDF5 file with a unique key
key = f'df_{product_id}' key = f'df_{product_id}'
View_h5_file.py
+2 -1
View File
@@ -57,6 +57,7 @@ fig.write_html(report_file)
'''
''' '''
# Calculate the standard deviation using pandas' function # Calculate the standard deviation using pandas' function
std_dev_unit = df.groupby('Product ID')['Unit Price'].std() std_dev_unit = df.groupby('Product ID')['Unit Price'].std()
@@ -69,5 +70,5 @@ fig.update_layout(title='Standard Deviation of the Unit Pricing by Item', xaxis_
report_file = 'test2.html' report_file = 'test2.html'
fig.write_html(report_file) fig.write_html(report_file)
'''