update for the NFL zipcodes

2023-05-03 19:04:01 -04:00
parent 7baaac206f
commit 6d88182baa
5 changed files with 303 additions and 0 deletions
@@ -0,0 +1,61 @@
+from google.cloud import pubsub_v1
+import csv
+import os
+import asyncio
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+project_id = "fp-20230425"
+
+'''
+topic_name = "fp-topic-NFL-zipcodes"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
+subscriber.delete_subscription(request={"subscription": subscription_path})
+print("Subscription deleted: {}".format(subscription_path))
+
+# Delete the topic
+publisher = pubsub_v1.PublisherClient()
+topic_path = publisher.topic_path(project_id, topic_name)
+publisher.delete_topic(request={"topic": topic_path})
+print(f"Topic '{topic_name}' deleted.")
+
+
+
+'''
+# pubsub topic
+publisher = pubsub_v1.PublisherClient()
+topic_name = "fp-topic-NFL-zipcodes"
+topic_path = publisher.topic_path(project_id, topic_name)
+
+
+
+# Create a Pub/Sub subscription
+subscriber = pubsub_v1.SubscriberClient()
+subscription_name = "fp-subscription-zipcodes"
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+subscription = subscriber.create_subscription(
+    name=subscription_path, topic=topic_path
+)
+
+# Publish a message to the topic
+# Read the data from the csv file
+file_path = "C:/Users/Karen/Desktop/zipcodes.csv"
+
+with open(file_path, "r") as csv_file:
+    reader = csv.reader(csv_file)
+    next(reader)  # skip header row
+    for row in reader:
+        # Format the data as a string and publish it to the topic
+        data = ",".join(row).encode("utf-8")
+        future = publisher.publish(topic_path, data)
+        print(f"Published message ID {future.result()}")
+
+
+async def receive_messages():
+    response = subscriber.pull(subscription=subscription_path, max_messages=1)
+    for received_message in response.received_messages:
+        print("Received message: {}".format(received_message.message.data))
+       
+    
+asyncio.run(receive_messages())
@@ -0,0 +1,52 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Define the zip codes for which to extract the price
+zip_codes = ['10001', '90001', '90210']
+
+# Load the page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+driver.get(url)
+time.sleep(5)
+
+for zip_code in zip_codes:
+    # Enter the zip code and click the "Check" button
+    zip_code_input = driver.find_element(By.XPATH, '//*[@id="rowToolRef"]/div[2]/div[1]/div/div/div[1]')
+    zip_code_input.clear()
+    zip_code_input.send_keys(zip_code)
+    check_button = driver.find_element(By.XPATH, '/html/body/div[4]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[3]/div/button')
+    check_button.click()
+
+    # Wait for the price to load and extract it
+    time.sleep(5)
+    next_data_element = None
+    try:
+        next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+    except:
+        print('__NEXT_DATA__ element not found')
+
+    if next_data_element is not None:
+        next_data_text = next_data_element.get_attribute('innerHTML')
+        next_data_start_index = next_data_text.index('{')
+        next_data_end_index = next_data_text.rindex('}') + 1
+        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+        next_data = json.loads(next_data_json)
+
+        # Extract the price from the __NEXT_DATA__ data
+        price = next_data['offers']['price']
+
+        # Print the price for the current zip code
+        print(f"Price for zip code {zip_code}: {price}")
+    else:
+        print(f"Price not found for zip code {zip_code}")
+
+# Close the Selenium driver
+driver.quit()
@@ -0,0 +1,61 @@
+from google.cloud import pubsub_v1
+import csv
+import os
+import asyncio
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+project_id = "fp-20230425"
+
+'''
+topic_name = "fp-topic-NFL-zipcodes"
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
+subscriber.delete_subscription(request={"subscription": subscription_path})
+print("Subscription deleted: {}".format(subscription_path))
+
+# Delete the topic
+publisher = pubsub_v1.PublisherClient()
+topic_path = publisher.topic_path(project_id, topic_name)
+publisher.delete_topic(request={"topic": topic_path})
+print(f"Topic '{topic_name}' deleted.")
+
+
+
+'''
+# Create a pubsub topic
+publisher = pubsub_v1.PublisherClient()
+topic_name = "fp-topic-NFL-zipcodes"
+topic_path = publisher.topic_path(project_id, topic_name)
+topic = publisher.create_topic(request={"name": topic_path})
+
+
+# Create a Pub/Sub subscription
+subscriber = pubsub_v1.SubscriberClient()
+subscription_name = "fp-subscription-NFL-zipcodes"
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+subscription = subscriber.create_subscription(
+    name=subscription_path, topic=topic_path
+)
+
+# Publish a message to the topic
+# Read the data from the csv file
+file_path = "C:/Users/Karen/Desktop/NFL_Cities.csv"
+
+with open(file_path, "r") as csv_file:
+    reader = csv.reader(csv_file)
+    next(reader)  # skip header row
+    for row in reader:
+        # Format the data as a string and publish it to the topic
+        data = ",".join(row).encode("utf-8")
+        future = publisher.publish(topic_path, data)
+        print(f"Published message ID {future.result()}")
+
+
+async def receive_messages():
+    response = subscriber.pull(subscription=subscription_path, max_messages=1)
+    for received_message in response.received_messages:
+        print("Received message: {}".format(received_message.message.data))
+       
+    
+asyncio.run(receive_messages())
@@ -0,0 +1,55 @@
+import json
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import time
+from google.cloud import pubsub_v1
+import os
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+
+options = Options()
+options.add_argument('--headless')
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+def callback(message):
+    try:
+        # Get the zip code from the message
+        zip_code = message.data.decode('utf-8')
+        print(f"Received zip code: {zip_code}")
+        
+        # Load the webpage with the zip code
+        url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+        driver.get(url)
+        driver.find_element(By.ID, 'zipcode-input').send_keys(zip_code)
+        driver.find_element(By.ID, 'btn-check-zipcode').click()
+        
+        # Extract the price from the webpage
+        next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+        next_data_text = next_data_element.get_attribute('innerHTML')
+        next_data_start_index = next_data_text.index('{')
+        next_data_end_index = next_data_text.rindex('}') + 1
+        next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+        next_data = json.loads(next_data_json)
+        price = next_data['offers']['price']
+        
+        # Print the result
+        print(f"Price of rice crackers in zip code {zip_code}: {price}")
+    except Exception as e:
+        print(f"Error processing message: {str(e)}")
+    finally:
+        message.ack()
+
+subscriber.subscribe(subscription_path, callback=callback)
+
+while True:
+    time.sleep(1)
+
+
@@ -0,0 +1,74 @@
+from google.cloud import pubsub_v1
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+import json
+import time
+import os
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
+
+# Configure the Selenium driver options
+options = Options()
+options.add_argument('--disable-gpu')
+driver = webdriver.Chrome(options=options)
+
+# Set up Pub/Sub subscription
+project_id = "fp-20230425"
+subscription_name = "fp-subscription-zipcodes"
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+# Load the product page and extract the __NEXT_DATA__ data
+url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
+driver.get(url)
+time.sleep(5)
+next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
+if next_data_element is not None:
+    next_data_text = next_data_element.get_attribute('innerHTML')
+    next_data_start_index = next_data_text.index('{')
+    next_data_end_index = next_data_text.rindex('}') + 1
+    next_data_json = next_data_text[next_data_start_index:next_data_end_index]
+    next_data = json.loads(next_data_json)
+    price = next_data['offers']['price']
+else:
+    print('__NEXT_DATA__ element not found')
+
+# Define callback function for Pub/Sub messages
+def callback(message):
+    # Extract zip code from message data
+    zip_code = message.data.decode('utf-8').strip()
+    print(f"Received zip code: {zip_code}")
+    
+    # Enter zip code in the input field and click the "Check" button
+    zip_input = driver.find_element(By.XPATH, '/html/body/div[4]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[2]/input')
+    zip_input.clear()
+    zip_input.send_keys(zip_code)
+    check_button = driver.find_element(By.XPATH, '/html/body/div[4]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[3]/div')
+    check_button.click()
+    time.sleep(2)
+    '''
+    # Extract the price of rice crackers for this zip code
+    try:
+        price_element = driver.find_element(By.XPATH, '/html/body/div[2]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')
+        price = price_element.text.strip()[1:]
+    except:
+        price = 'Price not found'
+    '''
+    print(next_data)
+    price = next_data['offers']['price']
+    # Print the price
+    print(price)
+
+    # Print the zip code and its corresponding price
+    print(f'{zip_code}: {price}')
+    
+    message.ack()  # acknowledge receipt of message
+
+# Start receiving messages
+subscriber.subscribe(subscription_path, callback=callback)
+print(f'Started receiving messages from subscription {subscription_path}...')
+
+# Keep the script running indefinitely to continue receiving messages
+while True:
+    time.sleep(10)