1 changed files with 36 additions and 0 deletions
@@ -0,0 +1,36 @@
+import multiprocessing
+import subprocess
+
+# Define the subset of urls
+url_list = [
+    'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+    'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261'
+    'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467'
+    'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01'
+    'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224'
+    'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02'
+    'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01'
+    'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
+    'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353'
+    'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03'
+    'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
+]
+
+# Determine the number of available CPUs
+cpu_count = multiprocessing.cpu_count()
+
+# Create a pool of worker processes
+pool = multiprocessing.Pool(cpu_count)
+
+# Define the function to run the scraper
+def run_scraper(url):
+    # scraping logic
+    subprocess.call(['python', 'fp2_13.py', url])
+    return run_scraper
+
+# Run the scraper in parallel for the subset of items
+results = pool.map(run_scraper, url_list)
+
+# Close the pool and wait for the processes to finish
+pool.close()
+pool.join()