63 lines
2.2 KiB
Python
63 lines
2.2 KiB
Python
from fp2_14 import get_product_prices
|
|
import h5py
|
|
import pandas as pd
|
|
import datetime
|
|
import sys
|
|
from datetime import datetime
|
|
import ntplib
|
|
import arrow
|
|
|
|
project_id = "fp-20230425"
|
|
subscription_name = "fp-subscription-NFL-zipcodes"
|
|
url = "https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f"
|
|
|
|
df_new = get_product_prices(project_id, subscription_name, url)
|
|
|
|
'''
|
|
# When using sys to call the 'get_product_prices' function
|
|
|
|
if __name__ == '__main__':
|
|
# Check if the required command-line arguments are provided
|
|
if len(sys.argv) < 4:
|
|
print("Usage: python test_fp2_14.py project_id subscription_name url")
|
|
sys.exit(1)
|
|
|
|
project_id = sys.argv[1]
|
|
subscription_name = sys.argv[2]
|
|
url = sys.argv[3:]
|
|
|
|
get_product_prices(project_id, subscription_name, url)
|
|
'''
|
|
|
|
|
|
# Now we have the new dataframe "df" recording the new price listed in weee's page
|
|
# Load such data in the existing HDF5 file
|
|
file_path = 'product1.h5'
|
|
with h5py.File(file_path, 'a') as file:
|
|
# Load the existing dataframe 'df' from the HDF5 file
|
|
# REPLACE KEY TO GET THE EXISTING DATAFRAME FOR EACH DIFFERENT PRODUCT
|
|
df = pd.read_hdf(file, key='df')
|
|
|
|
# Add timestamp to the new dataframe "df_new"
|
|
# timestamp = datetime.now()
|
|
'''
|
|
To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
|
|
provides time synchronization with a reliable time server.
|
|
Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
|
|
sensible and human-friendly approach to format and convert the timestamps.
|
|
'''
|
|
# Fetch the time from an NTP server
|
|
ntp_client = ntplib.NTPClient()
|
|
response = ntp_client.request('pool.ntp.org')
|
|
ntp_time = datetime.fromtimestamp(response.tx_time)
|
|
timestamp = arrow.get(ntp_time).to('local')
|
|
df_new['Timestamp'] = timestamp
|
|
|
|
# Append 'df_new' to the existing df stored in the key
|
|
df_update = df.append(df_new, ignore_index=True)
|
|
|
|
# Save 'df2' to the HDF5 file with the key 'df'
|
|
# REPLACE KEY FOR DIFFERENT PRODUCT
|
|
df_update.to_hdf(file, key='df', mode='a')
|
|
|
|
print("Data appended and saved successfully.") |