From 6d88182baa7c542839d362061e4692a01b58f1fb Mon Sep 17 00:00:00 2001 From: Karen Zeng Date: Wed, 3 May 2023 19:04:01 -0400 Subject: [PATCH] update for the NFL zipcodes --- fp2_10.py | 61 +++++++++++++++++++++++++++++++++++++++++++++ fp2_6.py | 52 ++++++++++++++++++++++++++++++++++++++ fp2_7.py | 61 +++++++++++++++++++++++++++++++++++++++++++++ fp2_8.py | 55 +++++++++++++++++++++++++++++++++++++++++ fp2_9.py | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 303 insertions(+) create mode 100644 fp2_10.py create mode 100644 fp2_6.py create mode 100644 fp2_7.py create mode 100644 fp2_8.py create mode 100644 fp2_9.py diff --git a/fp2_10.py b/fp2_10.py new file mode 100644 index 0000000..b955209 --- /dev/null +++ b/fp2_10.py @@ -0,0 +1,61 @@ +from google.cloud import pubsub_v1 +import csv +import os +import asyncio + +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json" +project_id = "fp-20230425" + +''' +topic_name = "fp-topic-NFL-zipcodes" +subscription_name = "fp-subscription-NFL-zipcodes" +subscriber = pubsub_v1.SubscriberClient() +subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}" +subscriber.delete_subscription(request={"subscription": subscription_path}) +print("Subscription deleted: {}".format(subscription_path)) + +# Delete the topic +publisher = pubsub_v1.PublisherClient() +topic_path = publisher.topic_path(project_id, topic_name) +publisher.delete_topic(request={"topic": topic_path}) +print(f"Topic '{topic_name}' deleted.") + + + +''' +# pubsub topic +publisher = pubsub_v1.PublisherClient() +topic_name = "fp-topic-NFL-zipcodes" +topic_path = publisher.topic_path(project_id, topic_name) + + + +# Create a Pub/Sub subscription +subscriber = pubsub_v1.SubscriberClient() +subscription_name = "fp-subscription-zipcodes" +subscription_path = subscriber.subscription_path(project_id, subscription_name) +subscription = subscriber.create_subscription( + name=subscription_path, topic=topic_path +) + +# Publish a message to the topic +# Read the data from the csv file +file_path = "C:/Users/Karen/Desktop/zipcodes.csv" + +with open(file_path, "r") as csv_file: + reader = csv.reader(csv_file) + next(reader) # skip header row + for row in reader: + # Format the data as a string and publish it to the topic + data = ",".join(row).encode("utf-8") + future = publisher.publish(topic_path, data) + print(f"Published message ID {future.result()}") + + +async def receive_messages(): + response = subscriber.pull(subscription=subscription_path, max_messages=1) + for received_message in response.received_messages: + print("Received message: {}".format(received_message.message.data)) + + +asyncio.run(receive_messages()) \ No newline at end of file diff --git a/fp2_6.py b/fp2_6.py new file mode 100644 index 0000000..cfc09ec --- /dev/null +++ b/fp2_6.py @@ -0,0 +1,52 @@ +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +import json +import time + +# Configure the Selenium driver options +options = Options() +options.add_argument('--disable-gpu') +driver = webdriver.Chrome(options=options) + +# Define the zip codes for which to extract the price +zip_codes = ['10001', '90001', '90210'] + +# Load the page and extract the __NEXT_DATA__ data +url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119' +driver.get(url) +time.sleep(5) + +for zip_code in zip_codes: + # Enter the zip code and click the "Check" button + zip_code_input = driver.find_element(By.XPATH, '//*[@id="rowToolRef"]/div[2]/div[1]/div/div/div[1]') + zip_code_input.clear() + zip_code_input.send_keys(zip_code) + check_button = driver.find_element(By.XPATH, '/html/body/div[4]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[3]/div/button') + check_button.click() + + # Wait for the price to load and extract it + time.sleep(5) + next_data_element = None + try: + next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']") + except: + print('__NEXT_DATA__ element not found') + + if next_data_element is not None: + next_data_text = next_data_element.get_attribute('innerHTML') + next_data_start_index = next_data_text.index('{') + next_data_end_index = next_data_text.rindex('}') + 1 + next_data_json = next_data_text[next_data_start_index:next_data_end_index] + next_data = json.loads(next_data_json) + + # Extract the price from the __NEXT_DATA__ data + price = next_data['offers']['price'] + + # Print the price for the current zip code + print(f"Price for zip code {zip_code}: {price}") + else: + print(f"Price not found for zip code {zip_code}") + +# Close the Selenium driver +driver.quit() diff --git a/fp2_7.py b/fp2_7.py new file mode 100644 index 0000000..83a446e --- /dev/null +++ b/fp2_7.py @@ -0,0 +1,61 @@ +from google.cloud import pubsub_v1 +import csv +import os +import asyncio + +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json" +project_id = "fp-20230425" + +''' +topic_name = "fp-topic-NFL-zipcodes" +subscription_name = "fp-subscription-NFL-zipcodes" +subscriber = pubsub_v1.SubscriberClient() +subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}" +subscriber.delete_subscription(request={"subscription": subscription_path}) +print("Subscription deleted: {}".format(subscription_path)) + +# Delete the topic +publisher = pubsub_v1.PublisherClient() +topic_path = publisher.topic_path(project_id, topic_name) +publisher.delete_topic(request={"topic": topic_path}) +print(f"Topic '{topic_name}' deleted.") + + + +''' +# Create a pubsub topic +publisher = pubsub_v1.PublisherClient() +topic_name = "fp-topic-NFL-zipcodes" +topic_path = publisher.topic_path(project_id, topic_name) +topic = publisher.create_topic(request={"name": topic_path}) + + +# Create a Pub/Sub subscription +subscriber = pubsub_v1.SubscriberClient() +subscription_name = "fp-subscription-NFL-zipcodes" +subscription_path = subscriber.subscription_path(project_id, subscription_name) +subscription = subscriber.create_subscription( + name=subscription_path, topic=topic_path +) + +# Publish a message to the topic +# Read the data from the csv file +file_path = "C:/Users/Karen/Desktop/NFL_Cities.csv" + +with open(file_path, "r") as csv_file: + reader = csv.reader(csv_file) + next(reader) # skip header row + for row in reader: + # Format the data as a string and publish it to the topic + data = ",".join(row).encode("utf-8") + future = publisher.publish(topic_path, data) + print(f"Published message ID {future.result()}") + + +async def receive_messages(): + response = subscriber.pull(subscription=subscription_path, max_messages=1) + for received_message in response.received_messages: + print("Received message: {}".format(received_message.message.data)) + + +asyncio.run(receive_messages()) \ No newline at end of file diff --git a/fp2_8.py b/fp2_8.py new file mode 100644 index 0000000..187de37 --- /dev/null +++ b/fp2_8.py @@ -0,0 +1,55 @@ +import json +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +import time +from google.cloud import pubsub_v1 +import os + +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json" + +options = Options() +options.add_argument('--headless') +options.add_argument('--disable-gpu') +driver = webdriver.Chrome(options=options) + +# Set up Pub/Sub subscription +project_id = "fp-20230425" +subscription_name = "fp-subscription-zipcodes" +subscriber = pubsub_v1.SubscriberClient() +subscription_path = subscriber.subscription_path(project_id, subscription_name) + +def callback(message): + try: + # Get the zip code from the message + zip_code = message.data.decode('utf-8') + print(f"Received zip code: {zip_code}") + + # Load the webpage with the zip code + url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119' + driver.get(url) + driver.find_element(By.ID, 'zipcode-input').send_keys(zip_code) + driver.find_element(By.ID, 'btn-check-zipcode').click() + + # Extract the price from the webpage + next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']") + next_data_text = next_data_element.get_attribute('innerHTML') + next_data_start_index = next_data_text.index('{') + next_data_end_index = next_data_text.rindex('}') + 1 + next_data_json = next_data_text[next_data_start_index:next_data_end_index] + next_data = json.loads(next_data_json) + price = next_data['offers']['price'] + + # Print the result + print(f"Price of rice crackers in zip code {zip_code}: {price}") + except Exception as e: + print(f"Error processing message: {str(e)}") + finally: + message.ack() + +subscriber.subscribe(subscription_path, callback=callback) + +while True: + time.sleep(1) + + diff --git a/fp2_9.py b/fp2_9.py new file mode 100644 index 0000000..6ad2f18 --- /dev/null +++ b/fp2_9.py @@ -0,0 +1,74 @@ +from google.cloud import pubsub_v1 +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +import json +import time +import os + +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json" + +# Configure the Selenium driver options +options = Options() +options.add_argument('--disable-gpu') +driver = webdriver.Chrome(options=options) + +# Set up Pub/Sub subscription +project_id = "fp-20230425" +subscription_name = "fp-subscription-zipcodes" +subscriber = pubsub_v1.SubscriberClient() +subscription_path = subscriber.subscription_path(project_id, subscription_name) + +# Load the product page and extract the __NEXT_DATA__ data +url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119' +driver.get(url) +time.sleep(5) +next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']") +if next_data_element is not None: + next_data_text = next_data_element.get_attribute('innerHTML') + next_data_start_index = next_data_text.index('{') + next_data_end_index = next_data_text.rindex('}') + 1 + next_data_json = next_data_text[next_data_start_index:next_data_end_index] + next_data = json.loads(next_data_json) + price = next_data['offers']['price'] +else: + print('__NEXT_DATA__ element not found') + +# Define callback function for Pub/Sub messages +def callback(message): + # Extract zip code from message data + zip_code = message.data.decode('utf-8').strip() + print(f"Received zip code: {zip_code}") + + # Enter zip code in the input field and click the "Check" button + zip_input = driver.find_element(By.XPATH, '/html/body/div[4]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[2]/input') + zip_input.clear() + zip_input.send_keys(zip_code) + check_button = driver.find_element(By.XPATH, '/html/body/div[4]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[3]/div') + check_button.click() + time.sleep(2) + ''' + # Extract the price of rice crackers for this zip code + try: + price_element = driver.find_element(By.XPATH, '/html/body/div[2]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div') + price = price_element.text.strip()[1:] + except: + price = 'Price not found' + ''' + print(next_data) + price = next_data['offers']['price'] + # Print the price + print(price) + + # Print the zip code and its corresponding price + print(f'{zip_code}: {price}') + + message.ack() # acknowledge receipt of message + +# Start receiving messages +subscriber.subscribe(subscription_path, callback=callback) +print(f'Started receiving messages from subscription {subscription_path}...') + +# Keep the script running indefinitely to continue receiving messages +while True: + time.sleep(10)