pricer/fp_merge.py

from fp2_14 import get_product_prices
import h5py
import pandas as pd
import datetime
import sys
from datetime import datetime
import ntplib
import arrow
import concurrent.futures
import pandas as pd

# Define the subset of urls
url_list = [
    'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
    'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
    'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
    'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
    'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
    'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
    'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
    'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
    'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
    'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
    'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
]


# Define the function to run the scraper
def run_scraper(url):
    project_id = "fp-20230425"
    subscription_name = "fp-subscription-NFL-zipcodes"
    df_new = get_product_prices(project_id, subscription_name, url)

    # Add timestamp to the new dataframe "df_new"
    '''
    To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
    provides time synchronization with a reliable time server.
    Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
    sensible and human-friendly approach to format and convert the timestamps.
    '''
    # Fetch the time from an NTP server
    ntp_client = ntplib.NTPClient()
    response = ntp_client.request('pool.ntp.org')
    ntp_time = datetime.fromtimestamp(response.tx_time)
    timestamp = arrow.get(ntp_time).to('local')
    df_new['Timestamp'] = timestamp

    # Append 'df_new' to the existing df stored in the key
    df_update = df_new.append(df_new, ignore_index=True)

    return df_update


'''
# When using sys to call the 'get_product_prices' function

if __name__ == '__main__':
    # Check if the required command-line arguments are provided
    if len(sys.argv) < 4:
        print("Usage: python test_fp2_14.py project_id subscription_name url")
        sys.exit(1)

    project_id = sys.argv[1]
    subscription_name = sys.argv[2]
    url = sys.argv[3:]

    get_product_prices(project_id, subscription_name, url)
'''


if __name__ == '__main__':
    # Determine the number of available CPUs
    # cpu_count = multiprocessing.cpu_count()

    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
        future_to_url = {executor.submit(run_scraper, url): url for url in url_list}
        results = []
        for future in concurrent.futures.as_completed(future_to_url):
            url = future_to_url[future]
            try:
                result = future.result()
                print(f"Scraped data for URL: {url}")
                results.append(result)
            except Exception as e:
                print(f"Failed to scrape data for URL: {url}\nError: {e}")


# Now we have the new dataframe "df_update" recording the new price listed in weee's page
# Load such data in the existing HDF5 file
file_path = 'product1.h5'
with pd.HDFStore(file_path) as store:
    for url, result in zip(url_list, results):
        # Extract the product ID from the URL
        product_id = url.split("/")[-1][:5]

        # Save the dataframe to the HDF5 file with a unique key
        key = f'df_{product_id}'
        store[key] = result
        result.to_hdf('product_data.h5', key=key, mode='a')
        print("Data appended and saved successfully.")