forked from SVIG/python-bootcamp
37 lines
1.6 KiB
Python
37 lines
1.6 KiB
Python
|
import multiprocessing
|
||
|
|
import subprocess
|
||
|
|
|
||
|
|
# Define the subset of urls
|
||
|
|
url_list = [
|
||
|
|
'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
|
||
|
|
'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261'
|
||
|
|
'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467'
|
||
|
|
'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01'
|
||
|
|
'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224'
|
||
|
|
'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02'
|
||
|
|
'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01'
|
||
|
|
'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
|
||
|
|
'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353'
|
||
|
|
'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03'
|
||
|
|
'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
|
||
|
|
]
|
||
|
|
|
||
|
|
# Determine the number of available CPUs
|
||
|
|
cpu_count = multiprocessing.cpu_count()
|
||
|
|
|
||
|
|
# Create a pool of worker processes
|
||
|
|
pool = multiprocessing.Pool(cpu_count)
|
||
|
|
|
||
|
|
# Define the function to run the scraper
|
||
|
|
def run_scraper(url):
|
||
|
|
# scraping logic
|
||
|
|
subprocess.call(['python', 'fp2_13.py', url])
|
||
|
|
return run_scraper
|
||
|
|
|
||
|
|
# Run the scraper in parallel for the subset of items
|
||
|
|
results = pool.map(run_scraper, url_list)
|
||
|
|
|
||
|
|
# Close the pool and wait for the processes to finish
|
||
|
|
pool.close()
|
||
|
|
pool.join()
|