100 lines
4.0 KiB
Python
100 lines
4.0 KiB
Python
from fp2_14 import get_product_prices
|
|
import h5py
|
|
import pandas as pd
|
|
import datetime
|
|
import sys
|
|
from datetime import datetime
|
|
import ntplib
|
|
import arrow
|
|
import concurrent.futures
|
|
import pandas as pd
|
|
|
|
# Define the subset of urls
|
|
url_list = [
|
|
'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
|
|
'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
|
|
'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
|
|
'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
|
|
'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
|
|
'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
|
|
'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
|
|
'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
|
|
'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
|
|
'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
|
|
'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
|
|
]
|
|
|
|
|
|
# Define the function to run the scraper
|
|
def run_scraper(url):
|
|
project_id = "fp-20230425"
|
|
subscription_name = "fp-subscription-NFL-zipcodes"
|
|
df_new = get_product_prices(project_id, subscription_name, url)
|
|
|
|
# Add timestamp to the new dataframe "df_new"
|
|
'''
|
|
To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
|
|
provides time synchronization with a reliable time server.
|
|
Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
|
|
sensible and human-friendly approach to format and convert the timestamps.
|
|
'''
|
|
# Fetch the time from an NTP server
|
|
ntp_client = ntplib.NTPClient()
|
|
response = ntp_client.request('pool.ntp.org')
|
|
ntp_time = datetime.fromtimestamp(response.tx_time)
|
|
timestamp = arrow.get(ntp_time).to('local')
|
|
df_new['Timestamp'] = timestamp
|
|
|
|
# Append 'df_new' to the existing df stored in the key
|
|
df_update = df_new.append(df_new, ignore_index=True)
|
|
|
|
return df_update
|
|
|
|
|
|
'''
|
|
# When using sys to call the 'get_product_prices' function
|
|
|
|
if __name__ == '__main__':
|
|
# Check if the required command-line arguments are provided
|
|
if len(sys.argv) < 4:
|
|
print("Usage: python test_fp2_14.py project_id subscription_name url")
|
|
sys.exit(1)
|
|
|
|
project_id = sys.argv[1]
|
|
subscription_name = sys.argv[2]
|
|
url = sys.argv[3:]
|
|
|
|
get_product_prices(project_id, subscription_name, url)
|
|
'''
|
|
|
|
|
|
if __name__ == '__main__':
|
|
# Determine the number of available CPUs
|
|
# cpu_count = multiprocessing.cpu_count()
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
|
|
future_to_url = {executor.submit(run_scraper, url): url for url in url_list}
|
|
results = []
|
|
for future in concurrent.futures.as_completed(future_to_url):
|
|
url = future_to_url[future]
|
|
try:
|
|
result = future.result()
|
|
print(f"Scraped data for URL: {url}")
|
|
results.append(result)
|
|
except Exception as e:
|
|
print(f"Failed to scrape data for URL: {url}\nError: {e}")
|
|
|
|
|
|
# Now we have the new dataframe "df_update" recording the new price listed in weee's page
|
|
# Load such data in the existing HDF5 file
|
|
file_path = 'product1.h5'
|
|
with pd.HDFStore(file_path) as store:
|
|
for url, result in zip(url_list, results):
|
|
# Extract the product ID from the URL
|
|
product_id = url.split("/")[-1][:5]
|
|
|
|
# Save the dataframe to the HDF5 file with a unique key
|
|
key = f'df_{product_id}'
|
|
store[key] = result
|
|
result.to_hdf('product_data.h5', key=key, mode='a')
|
|
print("Data appended and saved successfully.") |