diff --git a/Multiprocessing.py b/Multiprocessing.py new file mode 100644 index 0000000..b0a035a --- /dev/null +++ b/Multiprocessing.py @@ -0,0 +1,36 @@ +import multiprocessing +import subprocess + +# Define the subset of urls +url_list = [ + 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119' + 'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261' + 'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467' + 'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01' + 'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224' + 'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02' + 'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01' + 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f' + 'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353' + 'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03' + 'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7' +] + +# Determine the number of available CPUs +cpu_count = multiprocessing.cpu_count() + +# Create a pool of worker processes +pool = multiprocessing.Pool(cpu_count) + +# Define the function to run the scraper +def run_scraper(url): + # scraping logic + subprocess.call(['python', 'fp2_13.py', url]) + return run_scraper + +# Run the scraper in parallel for the subset of items +results = pool.map(run_scraper, url_list) + +# Close the pool and wait for the processes to finish +pool.close() +pool.join()