Initial commit, merge Karen and Sloetjes' branches
This commit is contained in:
cheezedoodle2
@@ -0,0 +1,276 @@
|
||||
import multiprocessing
|
||||
import subprocess
|
||||
import concurrent.futures
|
||||
from google.cloud import pubsub_v1
|
||||
from google.api_core.exceptions import DeadlineExceeded
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium.webdriver.common.by import By
|
||||
import json
|
||||
import time
|
||||
import os
|
||||
from concurrent.futures import TimeoutError
|
||||
import pandas as pd
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
import selenium.common.exceptions
|
||||
import re
|
||||
from fp2_14 import get_product_prices
|
||||
|
||||
# Define the subset of urls
|
||||
url_list = [
|
||||
'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
|
||||
'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
|
||||
'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
|
||||
'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
|
||||
'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
|
||||
'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
|
||||
'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
|
||||
'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
|
||||
'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
|
||||
'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
|
||||
'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
|
||||
]
|
||||
|
||||
|
||||
# Create a pool of worker processes
|
||||
# pool = multiprocessing.Pool(cpu_count)
|
||||
|
||||
# Define the function to run the scraper
|
||||
def run_scraper(url):
|
||||
# scraping logic
|
||||
df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
|
||||
print(f"Created a dataframe: {df}")
|
||||
product_id = url.split("/")[-1][:5]
|
||||
|
||||
options = Options()
|
||||
options.add_argument('--disable-gpu')
|
||||
driver = webdriver.Chrome(options=options)
|
||||
|
||||
project_id = "fp-20230425"
|
||||
subscription_name = "fp-subscription-NFL-zipcodes"
|
||||
subscriber = pubsub_v1.SubscriberClient()
|
||||
subscription_path = subscriber.subscription_path(project_id, subscription_name)
|
||||
|
||||
# product_id = url.split("/")[-1]
|
||||
driver.delete_all_cookies
|
||||
driver.get(url)
|
||||
|
||||
def get_element_text(class_name_prefix):
|
||||
xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against...
|
||||
xpath_str += "concat(' ', " # Step 2: a leading space, followed by...
|
||||
xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces...
|
||||
xpath_str += "' '), " # Step 4: a trailing space...
|
||||
xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
|
||||
try:
|
||||
match = driver.find_element(By.XPATH, xpath_str)
|
||||
return match.get_attribute('textContent')
|
||||
except selenium.common.exceptions.NoSuchElementException as nsee:
|
||||
pass
|
||||
|
||||
xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against...
|
||||
xpath_str += "concat(' ', " # Step 2: a leading space, followed by...
|
||||
xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces...
|
||||
xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
|
||||
try:
|
||||
match = driver.find_element(By.XPATH, xpath_str)
|
||||
return match.get_attribute('textContent')
|
||||
except selenium.common.exceptions.NoSuchElementException as nsee:
|
||||
pass
|
||||
return ''
|
||||
|
||||
# Close the cookies setting window
|
||||
try:
|
||||
cookie_button = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
|
||||
)
|
||||
print(f"cookie_button = {cookie_button}")
|
||||
cookie_button.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Clear any modal dialog
|
||||
try:
|
||||
modal_close = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
|
||||
)
|
||||
print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Define callback function for Pub/Sub messages
|
||||
def callback(message):
|
||||
# Extract zip code from message data
|
||||
zip_code = message.message.data.decode('utf-8')
|
||||
|
||||
print(f"Received zip code: {zip_code}")
|
||||
|
||||
try:
|
||||
try:
|
||||
# Clear any modal dialog
|
||||
modal_close = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
|
||||
)
|
||||
# print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
try:
|
||||
|
||||
# Clear any notice
|
||||
notice_close = WebDriverWait(driver, 5).until(
|
||||
EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
|
||||
# print(f"notice_close = {notice_close}")
|
||||
notice_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Enter zip code in the input field and click the "Check" button
|
||||
zip_button = WebDriverWait(driver, 5).until(
|
||||
EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
|
||||
)
|
||||
# print(f"{zip_button:}")
|
||||
zip_button.click()
|
||||
|
||||
zip_input = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
|
||||
)
|
||||
# print(f"{zip_input:}")
|
||||
zip_input.click()
|
||||
# print("clearing zip_input")
|
||||
zip_input.send_keys("\b\b\b\b\b")
|
||||
# print("sending zip code")
|
||||
zip_input.send_keys(zip_code)
|
||||
|
||||
check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
|
||||
check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
|
||||
# print(f"{check_button:}")
|
||||
check_button.click()
|
||||
# time.sleep(2)
|
||||
|
||||
try:
|
||||
# Clear any notice
|
||||
notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
|
||||
# print(f"notice_close = {notice_close}")
|
||||
notice_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
# Clear any modal dialog
|
||||
modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
|
||||
# print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Record sales count, if any
|
||||
sales_count_element = get_element_text('Header_count')
|
||||
count_start_index = sales_count_element.index('sold ') + len('sold ')
|
||||
count_end_index = sales_count_element.index('+')
|
||||
sales_count = sales_count_element[count_start_index:count_end_index]
|
||||
# Record any sold status notice
|
||||
sold_status = get_element_text('UnavailableCartbtn_container')
|
||||
if sold_status == 'Notify me':
|
||||
sold_status = 'Sold Out'
|
||||
|
||||
# Extract the price of rice crackers for this zip code
|
||||
# Check if there is a minimum order quantity
|
||||
min_quantity: float = 0
|
||||
total_price: float = 0
|
||||
unit_price: float = 0
|
||||
try:
|
||||
minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML')
|
||||
if minimum_quantity_str:
|
||||
regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
|
||||
match = re.search(regexp,minimum_quantity_str)
|
||||
min_quantity, total_price = match.groups()
|
||||
except selenium.common.exceptions.NoSuchElementException as nsee:
|
||||
print("No minimum purhcase string found")
|
||||
except Exception as e:
|
||||
print(f"Exception when finding minimum purchase string: {e}")
|
||||
|
||||
ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
|
||||
ld_json_text = ld_json_element.get_attribute('innerHTML')
|
||||
ld_json_start_index = ld_json_text.index('{')
|
||||
ld_json_end_index = ld_json_text.rindex('}') + 1
|
||||
ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
|
||||
ld_json = json.loads(ld_json_json)
|
||||
ld_json_price = ld_json['offers']['price']
|
||||
# Print the price
|
||||
print(ld_json_price)
|
||||
|
||||
unit_price_element = driver.find_element(By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText')
|
||||
unit_price = unit_price_element.replace("$", "")
|
||||
print(f"unit_price = {unit_price}" )
|
||||
|
||||
subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message
|
||||
# Append the zip code and price to the dataframe
|
||||
#append_to_df(zip_code, price)
|
||||
new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]})
|
||||
return pd.concat([df, new_row], ignore_index=True)
|
||||
#print(f"Recorded the zip code and price: {zip_code},{price}")
|
||||
#print(f"Updated a dataframe: {df}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Caught exception: {e}")
|
||||
print(f"Price not found for zip code {zip_code}")
|
||||
price = 'Price not found'
|
||||
return df
|
||||
|
||||
# Start receiving messages (sync version)
|
||||
while True:
|
||||
pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
|
||||
messages = []
|
||||
try:
|
||||
pull_response = subscriber.pull(request = pull_request)
|
||||
messages = pull_response.received_messages
|
||||
except DeadlineExceeded:
|
||||
pass
|
||||
|
||||
if messages:
|
||||
df = callback(messages[0])
|
||||
print(f"Updated a dataframe: {df}")
|
||||
else:
|
||||
print(f"no messages, trying again at {time.asctime()}")
|
||||
break
|
||||
# subprocess.call(['python', 'fp2_13.py', url])
|
||||
return df
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Determine the number of available CPUs
|
||||
# cpu_count = multiprocessing.cpu_count()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
|
||||
future_to_url = {executor.submit(run_scraper, url): url for url in url_list}
|
||||
results = []
|
||||
for future in concurrent.futures.as_completed(future_to_url):
|
||||
url = future_to_url[future]
|
||||
try:
|
||||
result = future.result()
|
||||
print(f"Scraped data for URL: {url}")
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
print(f"Failed to scrape data for URL: {url}\nError: {e}")
|
||||
|
||||
'''
|
||||
# Run the scraper in parallel for the subset of items
|
||||
results = pool.map(run_scraper, url_list)
|
||||
|
||||
# Close the pool and wait for the processes to finish
|
||||
pool.close()
|
||||
pool.join()
|
||||
'''
|
||||
|
||||
with pd.HDFStore('product_data.h5') as store:
|
||||
for url, result in zip(url_list, results):
|
||||
# Extract the product ID from the URL
|
||||
product_id = url.split("/")[-1][:5]
|
||||
|
||||
# Save the dataframe to the HDF5 file with a unique key
|
||||
key = f'df_{product_id}'
|
||||
store[key] = result
|
||||
# result.to_hdf('product_data.h5', key=key, mode='a')
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[1]:
|
||||
import os
|
||||
from google.cloud import pubsub_v1
|
||||
|
||||
|
||||
# In[2]:
|
||||
# Set the project ID and topic name
|
||||
project_id = 'svig-384801'
|
||||
topic_name = 'nfl-zipcodes'
|
||||
|
||||
|
||||
# In[3]:
|
||||
# Create a publisher client
|
||||
publisher = pubsub_v1.PublisherClient()
|
||||
|
||||
|
||||
# In[4]:
|
||||
# Get the topic path
|
||||
topic_path = publisher.topic_path(project_id, topic_name)
|
||||
|
||||
|
||||
# In[5]:
|
||||
# Define the zip codes for all NFL cities
|
||||
nfl_zipcodes = {
|
||||
'Arizona Cardinals': '85305',
|
||||
'Atlanta Falcons': '30363',
|
||||
'Baltimore Ravens': '21230',
|
||||
'Buffalo Bills': '14127',
|
||||
'Carolina Panthers': '28202',
|
||||
'Chicago Bears': '60605',
|
||||
'Cincinnati Bengals': '45202',
|
||||
'Cleveland Browns': '44115',
|
||||
'Dallas Cowboys': '76011',
|
||||
'Denver Broncos': '80204',
|
||||
'Detroit Lions': '48226',
|
||||
'Green Bay Packers': '54304',
|
||||
'Houston Texans': '77054',
|
||||
'Indianapolis Colts': '46225',
|
||||
'Jacksonville Jaguars': '32202',
|
||||
'Kansas City Chiefs': '64129',
|
||||
'Las Vegas Raiders': '89118',
|
||||
'Los Angeles Chargers': '92626',
|
||||
'Los Angeles Rams': '90305',
|
||||
'Miami Dolphins': '33056',
|
||||
'Minnesota Vikings': '55415',
|
||||
'New England Patriots': '02035',
|
||||
'New Orleans Saints': '70112',
|
||||
'New York Giants': '07073',
|
||||
'New York Jets': '07932',
|
||||
'Philadelphia Eagles': '19145',
|
||||
'Pittsburgh Steelers': '15212',
|
||||
'San Francisco 49ers': '95054',
|
||||
'Seattle Seahawks': '98056',
|
||||
'Tampa Bay Buccaneers': '33607',
|
||||
'Tennessee Titans': '37213',
|
||||
'Washington Football Team': '20785'
|
||||
}
|
||||
|
||||
|
||||
# In[6]:
|
||||
# Publish each zip code to the topic
|
||||
for team, zipcode in nfl_zipcodes.items():
|
||||
data = f'{zipcode}'.encode('utf-8')
|
||||
publisher_future = publisher.publish(topic_path, data=data)
|
||||
message_id = publisher_future.result()
|
||||
print(f"message_id = {message_id}")
|
||||
|
||||
print('All zip codes published.')
|
||||
|
||||
|
||||
# In[ ]:
|
||||
import os
|
||||
import time
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
from google.cloud import pubsub_v1
|
||||
|
||||
# Set the project ID and subscription name
|
||||
project_id = 'svig-384801'
|
||||
topic_name = 'nfl-zipcodes'
|
||||
subscription_name = 'nfl-zipcodes-sub'
|
||||
|
||||
# Set the path to your ChromeDriver executable
|
||||
chrome_driver_path = "/usr/local/bin/chromedriver"
|
||||
|
||||
# Set up a Google Cloud Pub/Sub subscriber
|
||||
subscriber = pubsub_v1.SubscriberClient()
|
||||
subscription_path = subscriber.subscription_path(project_id, subscription_name)
|
||||
|
||||
# Set up a ChromeDriver instance
|
||||
chrome_options = webdriver.ChromeOptions()
|
||||
#chrome_options.add_argument("--headless") # Run Chrome in headless mode
|
||||
chrome_service = Service(chrome_driver_path)
|
||||
chrome_service.start()
|
||||
driver = webdriver.Remote(chrome_service.service_url, options=chrome_options)
|
||||
|
||||
def callback(message):
|
||||
# Get the zip code from the message
|
||||
zip_code = message.data.decode("utf-8")
|
||||
print(f"Received message with zip code: {zip_code}")
|
||||
|
||||
# Open the product page for the rice crackers for the given zip code
|
||||
product_url = f"https://www.sayweee.com/zh/product/Want-Want-Senbei-Rice-Crackers/17119?trace_id=89f7c636-05b8-4f52-abdd-74de3a83f45d&zip={zip_code}"
|
||||
driver.get(product_url)
|
||||
|
||||
# Wait for the price to load
|
||||
price_xpath = "//span[@class='price']"
|
||||
print(EC.presence_of_element_located((By.XPATH, price_xpath)))
|
||||
price_element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, price_xpath)))
|
||||
|
||||
# Extract the price and print it
|
||||
price = price_element.text
|
||||
print(f"Price for zip code {zip_code}: {price}")
|
||||
|
||||
# Acknowledge the message
|
||||
message.ack()
|
||||
|
||||
# Subscribe to the topic
|
||||
streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback)
|
||||
print(f"Listening for messages on {subscription_path}...")
|
||||
streaming_pull_future.result()
|
||||
|
||||
# Keep the script running
|
||||
while True:
|
||||
time.sleep(60)
|
||||
|
||||
|
||||
# %%
|
||||
@@ -0,0 +1,33 @@
|
||||
ID,Team Name,Arena Name,Abbreviation,Conference,Division,Arena Location,ZIP Code
|
||||
1,Arizona Cardinals,State Farm Stadium,ARI,NFC,West,"Glendale, Arizona",85305
|
||||
2,Atlanta Falcons,Mercedes-Benz Stadium,ATL,NFC,South,"Atlanta, Georgia",30313
|
||||
3,Baltimore Ravens,M&T Bank Stadium,BAL,AFC,North,"Baltimore, Maryland",21230
|
||||
4,Buffalo Bills,Bills Stadium,BUF,AFC,East,"Orchard Park, New York",14127
|
||||
5,Carolina Panthers,Bank of America Stadium,CAR,NFC,South,"Charlotte, North Carolina",28202
|
||||
6,Chicago Bears,Soldier Field,CHI,NFC,North,"Chicago, Illinois",60605
|
||||
7,Cincinnati Bengals,Paul Brown Stadium,CIN,AFC,North,"Cincinnati, Ohio",45202
|
||||
8,Cleveland Browns,First Energy Stadium,CLE,AFC,North,"Cleveland, Ohio",44114
|
||||
9,Dallas Cowboys,At&T Stadium,DAL,NFC,East,"Arlington, Texas",76011
|
||||
10,Denver Broncos,Empower Field at Mile High,DEN,AFC,West,"Denver, Colorado",80204
|
||||
11,Detroit Lions,Ford Field,DET,NFC,North,"Detroit, Michigan",48226
|
||||
12,Green Bay Packers,Lambeau Field,GB,NFC,North,"Green Bay, Wisconsin",54304
|
||||
13,Houston Texans,NRG Stadium,HOU,AFC,South,"Houston, Texas",77054
|
||||
14,Indianapolis Colts,Lucas Oil Stadium,IND,AFC,South,"Indianapolis, Indiana",46225
|
||||
15,Jacksonville Jaguars,TIAA Bank Field,JAX,AFC,South,"Jacksonville, Florida",32202
|
||||
16,Kansas City Chiefs,Arrowhead Stadium,KC,AFC,West,"Kansas City, Missouri",64129
|
||||
17,Las Vegas Raiders,Allegiant Stadium,LV,AFC,West,"Paradise, Nevada",89118
|
||||
18,Los Angeles Chargers,SoFi Stadium,LAC,AFC,West,"Inglewood, California",90301
|
||||
19,Los Angeles Rams,SoFi Stadium,LAR,NFC,West,"Inglewood, California",90301
|
||||
20,Miami Dolphins,Hard Rock Stadium,MIA,AFC,East,"Miami Gardens, Florida",33056
|
||||
21,Minnesota Vikings,U.S. Bank Stadium,MIN,NFC,North,"Minneapolis, Minnesota",55415
|
||||
22,New England Patriots,Gillette Stadium,NE,AFC,East,"Foxborough, Massachusetts",02035
|
||||
23,New Orleans Saints,Mercedes-Benz Superdome,NO,NFC,South,"New Orleans, Louisiana",70112
|
||||
24,New York Giants,MetLife Stadium,NYG,NFC,East,"East Rutherford, New Jersey",07073
|
||||
25,New York Jets,MetLife Stadium,NYJ,AFC,East,"East Rutherford, New Jersey",07073
|
||||
26,Philadelphia Eagles,Lincoln Financial Field,PHI,NFC,East,"Philadelphia, Pennsylvania",19148
|
||||
27,Pittsburgh Steelers,Heinz Field,PIT,AFC,North,"Pittsburgh, Pennsylvania",15212
|
||||
28,San Francisco 49ers,Levi's Stadium,SF,NFC,West,"Santa Clara, California",95054
|
||||
29,Seattle Seahawks,CenturyLink Field,SEA,NFC,West,"Seattle, Washington",98134
|
||||
30,Tampa Bay Buccaneers,Raymond James Stadium,TB,NFC,South,"Tampa, Florida",33607
|
||||
31,Tennessee Titans,Nissan Stadium,TEN,AFC,South,"Nashville, Tennessee",37213
|
||||
32,Washington Redskins,FedExField,WAS,NFC,East,"Landover, Maryland",20785
|
||||
|
@@ -0,0 +1,20 @@
|
||||
This branch contains a prototype for a retail price scraper for getweee.com.
|
||||
To run it, install:
|
||||
|
||||
* Python 3.9+,
|
||||
* chrome,
|
||||
* chromedriver,
|
||||
* and the python depedencies:
|
||||
|
||||
```
|
||||
pip install -r /path/to/requirements.txt
|
||||
```
|
||||
|
||||
Once those are installed, install the GCP SDK:
|
||||
|
||||
https://cloud.google.com/sdk/docs/install
|
||||
|
||||
You will also have to log into GCP *twice*, once for the gcloud CLI tools,
|
||||
and again for the GCP SDK libraries:
|
||||
|
||||
https://googleapis.dev/python/google-api-core/latest/auth.html
|
||||
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime, ForeignKey
|
||||
from sqlalchemy.orm import sessionmaker, relationship, declarative_base
|
||||
from flask import Flask, render_template
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
# Create a base class for declarative models
|
||||
Base = declarative_base()
|
||||
|
||||
class Product(Base):
|
||||
__tablename__ = 'product'
|
||||
id = Column(Integer, primary_key=True)
|
||||
name = Column(String)
|
||||
SKU_PLU_UPC_number = Column(String)
|
||||
price_histories = relationship("PriceHistory", back_populates="product")
|
||||
|
||||
class Retailer(Base):
|
||||
__tablename__ = 'retailer'
|
||||
id = Column(Integer, primary_key=True)
|
||||
name = Column(String)
|
||||
products = relationship("Product", secondary="retailer_product")
|
||||
|
||||
class Price(Base):
|
||||
__tablename__ = 'price'
|
||||
id = Column(Integer, primary_key=True)
|
||||
MOQ = Column(Integer)
|
||||
price = Column(Float)
|
||||
|
||||
class ScraperJob(Base):
|
||||
__tablename__ = 'scraper_job'
|
||||
id = Column(Integer, primary_key=True)
|
||||
start_time = Column(DateTime)
|
||||
end_time = Column(DateTime)
|
||||
parameters = Column(String)
|
||||
statistics = Column(String)
|
||||
|
||||
class PriceHistory(Base):
|
||||
__tablename__ = 'price_history'
|
||||
id = Column(Integer, primary_key=True)
|
||||
location = Column(String)
|
||||
zip_code = Column(Integer)
|
||||
city_name = Column(String)
|
||||
price_at_time = Column(DateTime)
|
||||
price = Column(Float)
|
||||
scraper_job_id = Column(Integer, ForeignKey('scraper_job.id'))
|
||||
product_id = Column(Integer, ForeignKey('product.id'))
|
||||
product = relationship("Product", back_populates="price_histories")
|
||||
scraper_job = relationship("ScraperJob")
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = 'user'
|
||||
id = Column(Integer, primary_key=True)
|
||||
username = Column(String)
|
||||
credentials = Column(String)
|
||||
creation_date = Column(DateTime)
|
||||
modification_date = Column(DateTime)
|
||||
|
||||
class RetailerProduct(Base):
|
||||
__tablename__ = 'retailer_product'
|
||||
retailer_id = Column(Integer, ForeignKey('retailer.id'), primary_key=True)
|
||||
product_id = Column(Integer, ForeignKey('product.id'), primary_key=True)
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
# Create the database engine
|
||||
engine = create_engine('sqlite:///user_database.db')
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
# Create a session factory
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
import sys
|
||||
|
||||
# Create a command-line tool that parses sys.argv to allow a user to create/edit/delete user objects.
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
app = Flask(__name__)
|
||||
|
||||
# Create a database connection
|
||||
engine = create_engine('sqlite:///user_database.db')
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
|
||||
@app.route('/')
|
||||
def home():
|
||||
# Get all user objects
|
||||
users = session.query(User).all()
|
||||
return render_template('index.html', users=users)
|
||||
|
||||
app.run()
|
||||
|
||||
|
||||
|
||||
command = sys.argv[1]
|
||||
|
||||
if command == "create":
|
||||
if len(sys.argv) < 4:
|
||||
print("Usage: python script.py create [username] [credentials]")
|
||||
sys.exit(1)
|
||||
|
||||
username = sys.argv[2]
|
||||
credentials = sys.argv[3]
|
||||
|
||||
# Create a new user object
|
||||
new_user = User(username=username, credentials=credentials)
|
||||
session.add(new_user)
|
||||
session.commit()
|
||||
print("User created successfully!")
|
||||
|
||||
elif command == "edit":
|
||||
if len(sys.argv) < 4:
|
||||
print("Usage: python script.py edit [username] [new_credentials]")
|
||||
sys.exit(1)
|
||||
|
||||
username = sys.argv[2]
|
||||
new_credentials = sys.argv[3]
|
||||
|
||||
# Find the user object
|
||||
user = session.query(User).filter_by(username=username).first()
|
||||
|
||||
if user:
|
||||
# Update user credentials
|
||||
user.credentials = new_credentials
|
||||
session.commit()
|
||||
print("User edited successfully!")
|
||||
else:
|
||||
print("User not found!")
|
||||
|
||||
elif command == "delete":
|
||||
if len(sys.argv) < 3:
|
||||
print("Usage: python script.py delete [username]")
|
||||
sys.exit(1)
|
||||
|
||||
username = sys.argv[2]
|
||||
|
||||
# Find the user object
|
||||
user = session.query(User).filter_by(username=username).first()
|
||||
|
||||
if user:
|
||||
# Delete the user object
|
||||
session.delete(user)
|
||||
session.commit()
|
||||
print("User deleted successfully!")
|
||||
else:
|
||||
print("User not found!")
|
||||
|
||||
else:
|
||||
print("Usage: python script.py [create|edit|delete] [username] [credentials]")
|
||||
@@ -0,0 +1,67 @@
|
||||
from google.cloud import pubsub_v1
|
||||
from google.api_core.exceptions import DeadlineExceeded
|
||||
import csv
|
||||
import os
|
||||
import asyncio
|
||||
|
||||
# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
|
||||
project_id = "fp-20230425"
|
||||
# project_id = "svig-384801"
|
||||
|
||||
'''
|
||||
topic_name = "fp-topic-NFL-zipcodes"
|
||||
subscription_name = "fp-subscription-NFL-zipcodes"
|
||||
subscriber = pubsub_v1.SubscriberClient()
|
||||
subscription_path = f"projects/{project_id}/subscriptions/{subscription_name}"
|
||||
subscriber.delete_subscription(request={"subscription": subscription_path})
|
||||
print("Subscription deleted: {}".format(subscription_path))
|
||||
|
||||
# Delete the topic
|
||||
publisher = pubsub_v1.PublisherClient()
|
||||
topic_path = publisher.topic_path(project_id, topic_name)
|
||||
publisher.delete_topic(request={"topic": topic_path})
|
||||
print(f"Topic '{topic_name}' deleted.")
|
||||
|
||||
|
||||
|
||||
'''
|
||||
# pubsub topic
|
||||
publisher = pubsub_v1.PublisherClient()
|
||||
topic_name = "fp-topic-NFL-zipcodes"
|
||||
topic_path = publisher.topic_path(project_id, topic_name)
|
||||
|
||||
# Create a Pub/Sub subscription
|
||||
subscriber = pubsub_v1.SubscriberClient()
|
||||
'''
|
||||
subscription_name = "fp-subscription-zipcodes"
|
||||
subscription_path = subscriber.subscription_path(project_id, subscription_name)
|
||||
subscription = subscriber.create_subscription(
|
||||
name=subscription_path, topic=topic_path
|
||||
)
|
||||
'''
|
||||
# Publish a message to the topic
|
||||
# Read the data from the csv file
|
||||
# file_path = "C:/Users/Karen/Desktop/zipcodes.csv"
|
||||
file_path = "zipcodes.csv"
|
||||
|
||||
with open(file_path, "r") as csv_file:
|
||||
reader = csv.reader(csv_file)
|
||||
next(reader) # skip header row
|
||||
for row in reader:
|
||||
# Format the data as a string and publish it to the topic
|
||||
data = ",".join(row).encode("utf-8")
|
||||
try:
|
||||
future = publisher.publish(topic_path, data)
|
||||
print(f"Published message ID {future.result()}")
|
||||
except DeadlineExceeded as de:
|
||||
print(f"Caught DeadlineExceeded exception: {de}")
|
||||
|
||||
'''
|
||||
async def receive_messages():
|
||||
response = subscriber.pull(subscription=subscription_path, max_messages=1)
|
||||
for received_message in response.received_messages:
|
||||
print("Received message: {}".format(received_message.message.data))
|
||||
|
||||
|
||||
asyncio.run(receive_messages())
|
||||
'''
|
||||
@@ -0,0 +1,372 @@
|
||||
from google.cloud import pubsub_v1
|
||||
from google.api_core.exceptions import DeadlineExceeded
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium.webdriver.support.wait import WebDriverWait
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.common.keys import Keys
|
||||
import selenium.common.exceptions
|
||||
import json
|
||||
import time
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from concurrent.futures import TimeoutError
|
||||
import pandas as pd
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
|
||||
# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
|
||||
|
||||
# Create an empty dataframe to store the prices
|
||||
df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Displayed Price', 'ld+json Price', 'Minimum Quantity', 'Total Price', 'Sales Count', 'Sold Status'])
|
||||
print(f"Created a dataframe: {df}")
|
||||
|
||||
# Configure the Selenium driver options
|
||||
options = Options()
|
||||
options.add_argument('--disable-gpu')
|
||||
driver = webdriver.Chrome(options=options)
|
||||
|
||||
# Set up Pub/Sub subscription
|
||||
project_id = "fp-20230425"
|
||||
subscription_name = "fp-subscription-NFL-zipcodes"
|
||||
subscriber = pubsub_v1.SubscriberClient()
|
||||
subscription_path = subscriber.subscription_path(project_id, subscription_name)
|
||||
|
||||
# Load the product page and extract the __NEXT_DATA__ data
|
||||
url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
|
||||
product_id = url.split("/")[-1]
|
||||
driver.delete_all_cookies
|
||||
driver.get(url)
|
||||
# time.sleep(5)
|
||||
|
||||
# time.sleep(2)
|
||||
# Close the cookies setting window
|
||||
try:
|
||||
# cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")
|
||||
cookie_button = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
|
||||
)
|
||||
print(f"cookie_button = {cookie_button}")
|
||||
cookie_button.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# time.sleep(1)
|
||||
# Clear any modal dialog
|
||||
try:
|
||||
# modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
|
||||
modal_close = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
|
||||
)
|
||||
print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
#time.sleep(2)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def close_modals():
|
||||
try:
|
||||
# time.sleep(2)
|
||||
# Clear any modal dialog
|
||||
modal_close_buttons = driver.find_elements(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
|
||||
print(f"modal_close = {modal_close}")
|
||||
for button in modal_close_buttons:
|
||||
try:
|
||||
button.click()
|
||||
except Exception as e:
|
||||
print(f"Exception when clicking on close button for element {button}: {e}")
|
||||
|
||||
# time.sleep(2)
|
||||
except Exception as e:
|
||||
print(f"Exception when trying to close discount modal dialog: {e}")
|
||||
|
||||
|
||||
try:
|
||||
# time.sleep(1)
|
||||
# Clear any notice
|
||||
notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
|
||||
print(f"notice_close = {notice_close}")
|
||||
notice_close.click()
|
||||
# time.sleep(1)
|
||||
except selenium.common.exceptions.NoSuchElementException as nsee:
|
||||
print("Purchase option change dialog close button not found")
|
||||
except Exception as e:
|
||||
print(f"Exception when trying to close purchase option change dialog: {e}")
|
||||
|
||||
def get_element_text(class_name_prefix):
|
||||
'''
|
||||
Return the text in the first element with a given class name.
|
||||
|
||||
Note that XPath doesn't have a function to check if a given word is present
|
||||
in a list of words, so using contains() will match a class name even if
|
||||
the class name is a substring of the class name being matched against.
|
||||
To prevent this, we use spaces as a delimiter at the beginning and end of
|
||||
the class name, and ensure there's only one space before and after all
|
||||
class names in the class name list.
|
||||
|
||||
Example: consider two class names with cache-busting suffixes:
|
||||
* Header_soldStatusNotice__12345
|
||||
* Header_soldStatusNoticeBackground__ABCDE
|
||||
|
||||
Matching against Header_soldStatusNotice using a naive implementation with
|
||||
`contains()` would match both class names, even though only the first one
|
||||
is the correct match.
|
||||
'''
|
||||
# Construct xpath selector.
|
||||
xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against...
|
||||
xpath_str += "concat(' ', " # Step 2: a leading space, followed by...
|
||||
xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces...
|
||||
xpath_str += "' '), " # Step 4: a trailing space...
|
||||
xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
|
||||
try:
|
||||
match = driver.find_element(By.XPATH, xpath_str)
|
||||
return match.get_attribute('textContent')
|
||||
except selenium.common.exceptions.NoSuchElementException as nsee:
|
||||
pass
|
||||
|
||||
# We didn't find a match, try again by doing a prefix match and assume
|
||||
# the full class name ends with two underscores followed by a cache-
|
||||
# busting string.
|
||||
xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against...
|
||||
xpath_str += "concat(' ', " # Step 2: a leading space, followed by...
|
||||
xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces...
|
||||
xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
|
||||
try:
|
||||
match = driver.find_element(By.XPATH, xpath_str)
|
||||
return match.get_attribute('textContent')
|
||||
except selenium.common.exceptions.NoSuchElementException as nsee:
|
||||
pass
|
||||
|
||||
# No match, return empty string.
|
||||
return ''
|
||||
|
||||
def callback(message):
|
||||
'''
|
||||
Callback function for Pub/Sub messages
|
||||
'''
|
||||
# Extract zip code from message data
|
||||
zip_code = message.message.data.decode('utf-8')
|
||||
|
||||
print(f"Received zip code: {zip_code}")
|
||||
|
||||
while True:
|
||||
try:
|
||||
# Enter zip code in the input field and click the "Check" button
|
||||
WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
|
||||
zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
|
||||
# print(f"{zip_button:}")
|
||||
zip_button.click()
|
||||
# time.sleep(1)
|
||||
|
||||
WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, '//div/label/following-sibling::input'))
|
||||
zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
|
||||
# print(f"{zip_input:}")
|
||||
zip_input.click()
|
||||
# print("clearing zip_input")
|
||||
zip_input.send_keys("\b\b\b\b\b\b\b\b\b\b")
|
||||
for i in range(1,10):
|
||||
zip_input.send_keys(Keys.DELETE)
|
||||
# print("sending zip code")
|
||||
zip_input.send_keys(zip_code)
|
||||
|
||||
WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, "//button[text() = 'Confirm']"))
|
||||
check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
|
||||
print(f"{check_button:}")
|
||||
check_button.click()
|
||||
# time.sleep(2)
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"Exception when changing zip code: {e}, trying to close modal dialogs")
|
||||
close_modals()
|
||||
|
||||
# try:
|
||||
# time.sleep(2)
|
||||
# # Clear any notice
|
||||
# notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
|
||||
# print(f"notice_close = {notice_close}")
|
||||
# notice_close.click()
|
||||
# time.sleep(2)
|
||||
# except:
|
||||
# pass
|
||||
|
||||
try:
|
||||
try:
|
||||
# time.sleep(2)
|
||||
# Clear any modal dialog
|
||||
# modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
|
||||
modal_close = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
|
||||
)
|
||||
print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
# time.sleep(2)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
try:
|
||||
# time.sleep(1)
|
||||
# Clear any notice
|
||||
# notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
|
||||
notice_close = WebDriverWait(driver, 5).until(
|
||||
EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
|
||||
)
|
||||
print(f"notice_close = {notice_close}")
|
||||
notice_close.click()
|
||||
# time.sleep(1)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Enter zip code in the input field and click the "Check" button
|
||||
# zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
|
||||
zip_button = WebDriverWait(driver, 5).until(
|
||||
EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
|
||||
)
|
||||
# print(f"{zip_button:}")
|
||||
zip_button.click()
|
||||
# time.sleep(1)
|
||||
|
||||
# zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
|
||||
zip_input = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
|
||||
)
|
||||
# print(f"{zip_input:}")
|
||||
zip_input.click()
|
||||
# print("clearing zip_input")
|
||||
zip_input.send_keys("\b\b\b\b\b")
|
||||
# print("sending zip code")
|
||||
zip_input.send_keys(zip_code)
|
||||
|
||||
check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
|
||||
check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
|
||||
# print(f"{check_button:}")
|
||||
check_button.click()
|
||||
# time.sleep(2)
|
||||
|
||||
try:
|
||||
# time.sleep(2)
|
||||
# Clear any notice
|
||||
# notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
|
||||
notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
|
||||
# print(f"notice_close = {notice_close}")
|
||||
notice_close.click()
|
||||
#time.sleep(2)
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
# Clear any modal dialog
|
||||
modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
|
||||
# print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Record sales count, if any
|
||||
sales_count = get_element_text('Header_count')
|
||||
# Record any sold status notice
|
||||
sold_status = get_element_text('soldStatusNotice')
|
||||
|
||||
# Check if there's a minimum order quantity
|
||||
min_quantity: float = 0
|
||||
total_price: float = 0
|
||||
try:
|
||||
minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' minQuantity ')]").get_attribute('innerHTML')
|
||||
if minimum_quantity_str:
|
||||
regexp="Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
|
||||
match = re.search(regexp, minimum_quantity_str)
|
||||
min_quantity, total_price = match.groups()
|
||||
|
||||
except selenium.common.exceptions.NoSuchElementException as nsee:
|
||||
print("No minimum purchase string found")
|
||||
except Exception as e:
|
||||
print(f"Exception when finding minimum purchase string: {e}")
|
||||
|
||||
# Get the price as displayed on the web page
|
||||
displayed_price = driver.find_element(By.XPATH, "//div[contains(concat(' ', normalize-space(@class), ' '), ' price_price ')]").get_attribute('innerText').replace('$', '')
|
||||
print(f"displayed_price = {displayed_price}")
|
||||
|
||||
# Extract the price of rice crackers for this zip code
|
||||
'''
|
||||
price_element = driver.find_element(By.XPATH, '/html/body/div[2]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')
|
||||
price = price_element.text.strip()[1:]
|
||||
print(price)
|
||||
'''
|
||||
|
||||
# Get the price from the ld+json element
|
||||
ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
|
||||
ld_json_text = ld_json_element.get_attribute('innerHTML')
|
||||
ld_json_start_index = ld_json_text.index('{')
|
||||
ld_json_end_index = ld_json_text.rindex('}') + 1
|
||||
ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
|
||||
ld_json = json.loads(ld_json_json)
|
||||
ld_json_price = ld_json['offers']['price']
|
||||
# Print the price
|
||||
print(f"ld_json_price = {ld_json_price}")
|
||||
|
||||
subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message
|
||||
# Append the zip code and price to the dataframe
|
||||
#append_to_df(zip_code, price)
|
||||
new_row = pd.DataFrame({'Product ID': [product_id],
|
||||
'Zip Code': [zip_code],
|
||||
'Displayed Price': [displayed_price],
|
||||
'ld+json Price': [ld_json_price],
|
||||
'Minimum Quantity': [min_quantity],
|
||||
'Total Price': [total_price],
|
||||
'Sales Count': [sales_count],
|
||||
'Sold Status': [sold_status]})
|
||||
return pd.concat([df, new_row], ignore_index=True)
|
||||
#print(f"Recorded the zip code and price: {zip_code},{price}")
|
||||
#print(f"Updated a dataframe: {df}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Caught exception {type(e)}: {e}")
|
||||
print(f"Price not found for zip code {zip_code}")
|
||||
price = 'Price not found'
|
||||
return df
|
||||
|
||||
# Start receiving messages (sync version)
|
||||
while True:
|
||||
pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
|
||||
messages = []
|
||||
try:
|
||||
pull_response = subscriber.pull(request = pull_request)
|
||||
messages = pull_response.received_messages
|
||||
except DeadlineExceeded:
|
||||
pass
|
||||
|
||||
if messages:
|
||||
df = callback(messages[0])
|
||||
print(f"Updated a dataframe: {df}")
|
||||
df.to_hdf('product1.h5', key='df',mode='w')
|
||||
else:
|
||||
print(f"No messages in subscription {subscription_path} at {time.asctime()}, trying again...")
|
||||
continue
|
||||
|
||||
# Open the HDF5 file in a separate window
|
||||
if sys.platform.startswith("win32"):
|
||||
os.startfile('product1.h5')
|
||||
|
||||
# Start receiving messages (async version)
|
||||
#flow_control = pubsub_v1.types.FlowControl(max_messages = 1)
|
||||
#streaming_pull_future = subscriber.subscribe(subscription_path,
|
||||
# callback=callback,
|
||||
# flow_control=flow_control)
|
||||
|
||||
# print(f'Started receiving messages from subscription {subscription_path}...')
|
||||
|
||||
# Wrap subscriber in a 'with' block to automatically call close() when done.
|
||||
# with subscriber:
|
||||
# try:
|
||||
# When `timeout` is not set, result() will block indefinitely,
|
||||
# unless an exception is encountered first.
|
||||
# streaming_pull_future.result(timeout=timeout)
|
||||
# except TimeoutError:
|
||||
# streaming_pull_future.cancel() # Trigger the shutdown.
|
||||
# streaming_pull_future.result() # Block until the shutdown is complete.
|
||||
|
||||
# Keep the script running indefinitely to continue receiving messages
|
||||
#while True:
|
||||
# time.sleep(10)
|
||||
@@ -0,0 +1,199 @@
|
||||
from google.cloud import pubsub_v1
|
||||
from google.api_core.exceptions import DeadlineExceeded
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium.webdriver.common.by import By
|
||||
import json
|
||||
import time
|
||||
from concurrent.futures import TimeoutError
|
||||
import pandas as pd
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
|
||||
# Create an empty dataframe to store the prices
|
||||
df = pd.DataFrame(columns=['Zip Code', 'Price'])
|
||||
print(f"Created a dataframe: {df}")
|
||||
|
||||
# Configure the Selenium driver options
|
||||
options = Options()
|
||||
options.add_argument('--disable-gpu')
|
||||
driver = webdriver.Chrome(options=options)
|
||||
|
||||
# Set up Pub/Sub subscription
|
||||
project_id = "fp-20230425"
|
||||
subscription_name = "fp-subscription-NFL-zipcodes"
|
||||
subscriber = pubsub_v1.SubscriberClient()
|
||||
subscription_path = subscriber.subscription_path(project_id, subscription_name)
|
||||
|
||||
# Load the product page and extract the __NEXT_DATA__ data
|
||||
url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
|
||||
driver.delete_all_cookies
|
||||
driver.get(url)
|
||||
# time.sleep(5)
|
||||
|
||||
# time.sleep(2)
|
||||
# Close the cookies setting window
|
||||
try:
|
||||
# cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")
|
||||
cookie_button = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
|
||||
)
|
||||
print(f"cookie_button = {cookie_button}")
|
||||
cookie_button.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# time.sleep(1)
|
||||
# Clear any modal dialog
|
||||
try:
|
||||
# modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
|
||||
modal_close = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
|
||||
)
|
||||
print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
#time.sleep(2)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Define callback function for Pub/Sub messages
|
||||
def callback(message):
|
||||
# Extract zip code from message data
|
||||
zip_code = message.message.data.decode('utf-8')
|
||||
|
||||
print(f"Received zip code: {zip_code}")
|
||||
|
||||
try:
|
||||
try:
|
||||
# time.sleep(2)
|
||||
# Clear any modal dialog
|
||||
# modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
|
||||
modal_close = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
|
||||
)
|
||||
# print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
# time.sleep(2)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
try:
|
||||
# time.sleep(1)
|
||||
# Clear any notice
|
||||
# notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
|
||||
notice_close = WebDriverWait(driver, 5).until(
|
||||
EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
|
||||
)
|
||||
# print(f"notice_close = {notice_close}")
|
||||
notice_close.click()
|
||||
# time.sleep(1)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Enter zip code in the input field and click the "Check" button
|
||||
# zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
|
||||
zip_button = WebDriverWait(driver, 5).until(
|
||||
EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
|
||||
)
|
||||
# print(f"{zip_button:}")
|
||||
zip_button.click()
|
||||
# time.sleep(1)
|
||||
|
||||
# zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
|
||||
zip_input = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
|
||||
)
|
||||
# print(f"{zip_input:}")
|
||||
zip_input.click()
|
||||
# print("clearing zip_input")
|
||||
zip_input.send_keys("\b\b\b\b\b")
|
||||
# print("sending zip code")
|
||||
zip_input.send_keys(zip_code)
|
||||
|
||||
check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
|
||||
check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
|
||||
# print(f"{check_button:}")
|
||||
check_button.click()
|
||||
# time.sleep(2)
|
||||
|
||||
try:
|
||||
# time.sleep(2)
|
||||
# Clear any notice
|
||||
# notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
|
||||
notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
|
||||
# print(f"notice_close = {notice_close}")
|
||||
notice_close.click()
|
||||
#time.sleep(2)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
# Extract the price of rice crackers for this zip code
|
||||
|
||||
# next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
|
||||
next_data_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, '//*[@id="__next"]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')))
|
||||
next_data_text = next_data_element.get_attribute('nodeValue')
|
||||
price = next_data_text.text
|
||||
'''
|
||||
next_data_text = next_data_element.get_attribute('innerHTML')
|
||||
next_data_start_index = next_data_text.index('{')
|
||||
next_data_end_index = next_data_text.rindex('}') + 1
|
||||
next_data_json = next_data_text[next_data_start_index:next_data_end_index]
|
||||
next_data = json.loads(next_data_json)
|
||||
price = next_data['price']
|
||||
'''
|
||||
# Print the price
|
||||
print(price)
|
||||
|
||||
'''
|
||||
try:
|
||||
min_quantity_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, '//script[@id="__NEXT_DATA__"]')))
|
||||
min_quantity_text = min_quantity_element.get_attribute('innerHTML')
|
||||
min_quantity_start_index = min_quantity_text.index('{')
|
||||
min_quantity_end_index = min_quantity_text.rindex('}') + 1
|
||||
min_quantity_json = min_quantity_text[min_quantity_start_index:min_quantity_end_index]
|
||||
min_quantity = json.loads(min_quantity_json)
|
||||
price = min_quantity['product']['price']
|
||||
except:
|
||||
next_data_element = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//script[@type='application/ld+json']")))
|
||||
next_data_text = next_data_element.get_attribute('innerHTML')
|
||||
next_data_start_index = next_data_text.index('{')
|
||||
next_data_end_index = next_data_text.rindex('}') + 1
|
||||
next_data_json = next_data_text[next_data_start_index:next_data_end_index]
|
||||
next_data = json.loads(next_data_json)
|
||||
price = next_data['offers']['price']
|
||||
finally:
|
||||
# Print the price
|
||||
print(price)
|
||||
'''
|
||||
|
||||
subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message
|
||||
# Append the zip code and price to the dataframe
|
||||
#append_to_df(zip_code, price)
|
||||
new_row = pd.DataFrame({'Zip Code': [zip_code], 'Price': [price]})
|
||||
return pd.concat([df, new_row], ignore_index=True)
|
||||
#print(f"Recorded the zip code and price: {zip_code},{price}")
|
||||
#print(f"Updated a dataframe: {df}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Caught exception: {e}")
|
||||
print(f"Price not found for zip code {zip_code}")
|
||||
price = 'Price not found'
|
||||
return df
|
||||
|
||||
# Start receiving messages (sync version)
|
||||
while True:
|
||||
pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
|
||||
messages = []
|
||||
try:
|
||||
pull_response = subscriber.pull(request = pull_request)
|
||||
messages = pull_response.received_messages
|
||||
except DeadlineExceeded:
|
||||
pass
|
||||
|
||||
if messages:
|
||||
df = callback(messages[0])
|
||||
print(f"Updated a dataframe: {df}")
|
||||
else:
|
||||
print(f"no messages, trying again at {time.asctime()}")
|
||||
@@ -0,0 +1,271 @@
|
||||
from google.cloud import pubsub_v1
|
||||
from google.api_core.exceptions import DeadlineExceeded
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium.webdriver.common.by import By
|
||||
import json
|
||||
import time
|
||||
import os
|
||||
from concurrent.futures import TimeoutError
|
||||
import pandas as pd
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
import selenium.common.exceptions
|
||||
import re
|
||||
|
||||
# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
|
||||
|
||||
# Create an empty dataframe to store the prices
|
||||
df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
|
||||
print(f"Created a dataframe: {df}")
|
||||
|
||||
# Configure the Selenium driver options
|
||||
options = Options()
|
||||
options.add_argument('--disable-gpu')
|
||||
driver = webdriver.Chrome(options=options)
|
||||
|
||||
# Set up Pub/Sub subscription
|
||||
project_id = "fp-20230425"
|
||||
subscription_name = "fp-subscription-NFL-zipcodes"
|
||||
subscriber = pubsub_v1.SubscriberClient()
|
||||
subscription_path = subscriber.subscription_path(project_id, subscription_name)
|
||||
|
||||
# Load the product page and extract the __NEXT_DATA__ data
|
||||
url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
|
||||
product_id = url.split("/")[-1][:5]
|
||||
# product_id = url.split("/")[-1]
|
||||
driver.delete_all_cookies
|
||||
driver.get(url)
|
||||
# time.sleep(5)
|
||||
|
||||
def get_element_text(class_name_prefix):
|
||||
'''
|
||||
Return the text in the first element with a given class name.
|
||||
|
||||
Note that XPath doesn't have a function to check if a given word is present
|
||||
in a list of words, so using contains() will match a class name even if
|
||||
the class name is a substring of the class name being matched against.
|
||||
To prevent this, we use spaces as a delimiter at the beginning and end of
|
||||
the class name, and ensure there's only one space before and after all
|
||||
class names in the class name list.
|
||||
|
||||
Example: consider two class names with cache-busting suffixes:
|
||||
* Header_soldStatusNotice__12345
|
||||
* Header_soldStatusNoticeBackground__ABCDE
|
||||
|
||||
Matching against Header_soldStatusNotice using a naive implementation with
|
||||
`contains()` would match both class names, even though only the first one
|
||||
is the correct match.
|
||||
'''
|
||||
# Construct xpath selector.
|
||||
xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against...
|
||||
xpath_str += "concat(' ', " # Step 2: a leading space, followed by...
|
||||
xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces...
|
||||
xpath_str += "' '), " # Step 4: a trailing space...
|
||||
xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
|
||||
try:
|
||||
match = driver.find_element(By.XPATH, xpath_str)
|
||||
return match.get_attribute('textContent')
|
||||
except selenium.common.exceptions.NoSuchElementException as nsee:
|
||||
pass
|
||||
|
||||
# We didn't find a match, try again by doing a prefix match and assume
|
||||
# the full class name ends with two underscores followed by a cache-
|
||||
# busting string.
|
||||
xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against...
|
||||
xpath_str += "concat(' ', " # Step 2: a leading space, followed by...
|
||||
xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces...
|
||||
xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
|
||||
try:
|
||||
match = driver.find_element(By.XPATH, xpath_str)
|
||||
return match.get_attribute('textContent')
|
||||
except selenium.common.exceptions.NoSuchElementException as nsee:
|
||||
pass
|
||||
|
||||
# No match, return empty string.
|
||||
return ''
|
||||
|
||||
|
||||
# time.sleep(2)
|
||||
# Close the cookies setting window
|
||||
try:
|
||||
# cookie_button = driver.find_element(By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")
|
||||
cookie_button = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
|
||||
)
|
||||
print(f"cookie_button = {cookie_button}")
|
||||
cookie_button.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# time.sleep(1)
|
||||
# Clear any modal dialog
|
||||
try:
|
||||
# modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
|
||||
modal_close = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
|
||||
)
|
||||
print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
#time.sleep(2)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Define callback function for Pub/Sub messages
|
||||
def callback(message):
|
||||
# Extract zip code from message data
|
||||
zip_code = message.message.data.decode('utf-8')
|
||||
|
||||
print(f"Received zip code: {zip_code}")
|
||||
|
||||
try:
|
||||
try:
|
||||
# time.sleep(2)
|
||||
# Clear any modal dialog
|
||||
# modal_close = driver.find_element(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
|
||||
modal_close = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
|
||||
)
|
||||
# print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
# time.sleep(2)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
try:
|
||||
# time.sleep(1)
|
||||
# Clear any notice
|
||||
# notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
|
||||
notice_close = WebDriverWait(driver, 5).until(
|
||||
EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
|
||||
)
|
||||
# print(f"notice_close = {notice_close}")
|
||||
notice_close.click()
|
||||
# time.sleep(1)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Enter zip code in the input field and click the "Check" button
|
||||
# zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
|
||||
zip_button = WebDriverWait(driver, 5).until(
|
||||
EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
|
||||
)
|
||||
# print(f"{zip_button:}")
|
||||
zip_button.click()
|
||||
# time.sleep(1)
|
||||
|
||||
# zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
|
||||
zip_input = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
|
||||
)
|
||||
# print(f"{zip_input:}")
|
||||
zip_input.click()
|
||||
# print("clearing zip_input")
|
||||
zip_input.send_keys("\b\b\b\b\b")
|
||||
# print("sending zip code")
|
||||
zip_input.send_keys(zip_code)
|
||||
|
||||
check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
|
||||
check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
|
||||
# print(f"{check_button:}")
|
||||
check_button.click()
|
||||
# time.sleep(2)
|
||||
|
||||
try:
|
||||
# time.sleep(2)
|
||||
# Clear any notice
|
||||
# notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
|
||||
notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
|
||||
# print(f"notice_close = {notice_close}")
|
||||
notice_close.click()
|
||||
#time.sleep(2)
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
# Clear any modal dialog
|
||||
modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
|
||||
# print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Record sales count, if any
|
||||
sales_count_element = get_element_text('Header_count')
|
||||
count_start_index = sales_count_element.index('sold ') + len('sold ')
|
||||
count_end_index = sales_count_element.index('+')
|
||||
sales_count = sales_count_element[count_start_index:count_end_index]
|
||||
# Record any sold status notice
|
||||
sold_status = get_element_text('UnavailableCartbtn_container')
|
||||
if sold_status == 'Notify me':
|
||||
sold_status = 'Sold Out'
|
||||
|
||||
# Extract the price of rice crackers for this zip code
|
||||
# Check if there is a minimum order quantity
|
||||
min_quantity: float = 0
|
||||
total_price: float = 0
|
||||
unit_price: float = 0
|
||||
try:
|
||||
minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML')
|
||||
if minimum_quantity_str:
|
||||
regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
|
||||
match = re.search(regexp,minimum_quantity_str)
|
||||
min_quantity, total_price = match.groups()
|
||||
except selenium.common.exceptions.NoSuchElementException as nsee:
|
||||
print("No minimum purhcase string found")
|
||||
except Exception as e:
|
||||
print(f"Exception when finding minimum purchase string: {e}")
|
||||
|
||||
|
||||
|
||||
ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
|
||||
ld_json_text = ld_json_element.get_attribute('innerHTML')
|
||||
ld_json_start_index = ld_json_text.index('{')
|
||||
ld_json_end_index = ld_json_text.rindex('}') + 1
|
||||
ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
|
||||
ld_json = json.loads(ld_json_json)
|
||||
ld_json_price = ld_json['offers']['price']
|
||||
# Print the price
|
||||
print(ld_json_price)
|
||||
|
||||
unit_price_element = driver.find_element(By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText')
|
||||
unit_price = unit_price_element.replace("$", "")
|
||||
print(f"unit_price = {unit_price}" )
|
||||
|
||||
subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message
|
||||
# Append the zip code and price to the dataframe
|
||||
#append_to_df(zip_code, price)
|
||||
new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]})
|
||||
return pd.concat([df, new_row], ignore_index=True)
|
||||
#print(f"Recorded the zip code and price: {zip_code},{price}")
|
||||
#print(f"Updated a dataframe: {df}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Caught exception: {e}")
|
||||
print(f"Price not found for zip code {zip_code}")
|
||||
price = 'Price not found'
|
||||
return df
|
||||
|
||||
# Start receiving messages (sync version)
|
||||
while True:
|
||||
pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
|
||||
messages = []
|
||||
try:
|
||||
pull_response = subscriber.pull(request = pull_request)
|
||||
messages = pull_response.received_messages
|
||||
except DeadlineExceeded:
|
||||
pass
|
||||
|
||||
if messages:
|
||||
df = callback(messages[0])
|
||||
print(f"Updated a dataframe: {df}")
|
||||
else:
|
||||
print(f"no messages, trying again at {time.asctime()}")
|
||||
break
|
||||
|
||||
# df.to_hdf('product1.h5', key='df',mode='w')
|
||||
# df.to_hdf('product1.h5', key='df',mode='a')
|
||||
|
||||
# Open the HDF5 file in a separate window
|
||||
# os.startfile('product1.h5')
|
||||
@@ -0,0 +1,280 @@
|
||||
from google.cloud import pubsub_v1
|
||||
from google.api_core.exceptions import DeadlineExceeded
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium.webdriver.common.by import By
|
||||
import json
|
||||
import time
|
||||
from concurrent.futures import TimeoutError
|
||||
import pandas as pd
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
import selenium.common.exceptions
|
||||
import re
|
||||
from datetime import datetime
|
||||
import h5py
|
||||
import sys
|
||||
import ntplib
|
||||
import arrow
|
||||
|
||||
# This fp2_14.py is used to load the new price and the timestamp, and append such data in an existing key in the HDF5 file "product1.h5"
|
||||
|
||||
def get_product_prices(project_id, subscription_name, url):
|
||||
# Create an empty dataframe to store the prices
|
||||
df_new = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
|
||||
print(f"Created a dataframe: {df_new}")
|
||||
|
||||
# Configure the Selenium driver options
|
||||
options = Options()
|
||||
options.add_argument('--disable-gpu')
|
||||
driver = webdriver.Chrome(options=options)
|
||||
|
||||
# Set up Pub/Sub subscription
|
||||
'''
|
||||
project_id = "fp-20230425"
|
||||
subscription_name = "fp-subscription-NFL-zipcodes"
|
||||
'''
|
||||
subscriber = pubsub_v1.SubscriberClient()
|
||||
subscription_path = subscriber.subscription_path(project_id, subscription_name)
|
||||
|
||||
|
||||
# Load the product page and extract the __NEXT_DATA__ data
|
||||
# url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
|
||||
product_id = url.split("/")[-1][:5]
|
||||
# product_id = url.split("/")[-1]
|
||||
driver.delete_all_cookies
|
||||
driver.get(url)
|
||||
|
||||
def get_element_text(class_name_prefix):
|
||||
'''
|
||||
Return the text in the first element with a given class name.
|
||||
|
||||
Note that XPath doesn't have a function to check if a given word is present
|
||||
in a list of words, so using contains() will match a class name even if
|
||||
the class name is a substring of the class name being matched against.
|
||||
To prevent this, we use spaces as a delimiter at the beginning and end of
|
||||
the class name, and ensure there's only one space before and after all
|
||||
class names in the class name list.
|
||||
|
||||
Example: consider two class names with cache-busting suffixes:
|
||||
* Header_soldStatusNotice__12345
|
||||
* Header_soldStatusNoticeBackground__ABCDE
|
||||
|
||||
Matching against Header_soldStatusNotice using a naive implementation with
|
||||
`contains()` would match both class names, even though only the first one
|
||||
is the correct match.
|
||||
'''
|
||||
# Construct xpath selector.
|
||||
xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against...
|
||||
xpath_str += "concat(' ', " # Step 2: a leading space, followed by...
|
||||
xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces...
|
||||
xpath_str += "' '), " # Step 4: a trailing space...
|
||||
xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
|
||||
try:
|
||||
match = driver.find_element(By.XPATH, xpath_str)
|
||||
return match.get_attribute('textContent')
|
||||
except selenium.common.exceptions.NoSuchElementException as nsee:
|
||||
pass
|
||||
|
||||
# We didn't find a match, try again by doing a prefix match and assume
|
||||
# the full class name ends with two underscores followed by a cache-
|
||||
# busting string.
|
||||
xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against...
|
||||
xpath_str += "concat(' ', " # Step 2: a leading space, followed by...
|
||||
xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces...
|
||||
xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
|
||||
try:
|
||||
match = driver.find_element(By.XPATH, xpath_str)
|
||||
return match.get_attribute('textContent')
|
||||
except selenium.common.exceptions.NoSuchElementException as nsee:
|
||||
pass
|
||||
|
||||
# No match, return empty string.
|
||||
return ''
|
||||
|
||||
|
||||
def modal_close():
|
||||
try:
|
||||
modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
|
||||
print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
def notice_close():
|
||||
try:
|
||||
# Clear any notice
|
||||
notice_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
|
||||
# print(f"notice_close = {notice_close}")
|
||||
notice_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
def cookie_close():
|
||||
try:
|
||||
cookie_button = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
|
||||
)
|
||||
print(f"cookie_button = {cookie_button}")
|
||||
cookie_button.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
cookie_close()
|
||||
modal_close()
|
||||
# Define callback function for Pub/Sub messages
|
||||
def callback(message):
|
||||
# Extract zip code from message data
|
||||
zip_code = message.message.data.decode('utf-8')
|
||||
|
||||
print(f"Received zip code: {zip_code}")
|
||||
|
||||
try:
|
||||
|
||||
modal_close()
|
||||
notice_close()
|
||||
|
||||
# Enter zip code in the input field and click the "Check" button
|
||||
zip_button = WebDriverWait(driver, 5).until(
|
||||
EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
|
||||
)
|
||||
# print(f"{zip_button:}")
|
||||
zip_button.click()
|
||||
|
||||
# zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
|
||||
zip_input = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
|
||||
)
|
||||
# print(f"{zip_input:}")
|
||||
zip_input.click()
|
||||
# print("clearing zip_input")
|
||||
zip_input.send_keys("\b\b\b\b\b")
|
||||
# print("sending zip code")
|
||||
zip_input.send_keys(zip_code)
|
||||
|
||||
check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
|
||||
check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH,"//button[text() = 'Confirm']")))
|
||||
# print(f"{check_button:}")
|
||||
check_button.click()
|
||||
|
||||
notice_close()
|
||||
modal_close()
|
||||
|
||||
# Record sales count, if any
|
||||
sales_count_element = get_element_text('Header_count')
|
||||
count_start_index = sales_count_element.index('sold ') + len('sold ')
|
||||
count_end_index = sales_count_element.index('+')
|
||||
sales_count = sales_count_element[count_start_index:count_end_index]
|
||||
# Record any sold status notice
|
||||
sold_status = get_element_text('UnavailableCartbtn_container')
|
||||
if sold_status == 'Notify me':
|
||||
sold_status = 'Sold Out'
|
||||
|
||||
# Extract the price of rice crackers for this zip code
|
||||
# Check if there is a minimum order quantity
|
||||
min_quantity: float = 0
|
||||
total_price: float = 0
|
||||
unit_price: float = 0
|
||||
try:
|
||||
minimum_quantity_str = WebDriverWait(driver, 5).until(EC.presence_of_element_located(((By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML'))))
|
||||
if minimum_quantity_str:
|
||||
regexp = "Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
|
||||
match = re.search(regexp,minimum_quantity_str)
|
||||
min_quantity, total_price = match.groups()
|
||||
except selenium.common.exceptions.NoSuchElementException as nsee:
|
||||
print("No minimum purhcase string found")
|
||||
except Exception as e:
|
||||
print(f"Exception when finding minimum purchase string: {e}")
|
||||
|
||||
|
||||
|
||||
ld_json_element = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//script[@type='application/ld+json']")))
|
||||
ld_json_text = ld_json_element.get_attribute('innerHTML')
|
||||
ld_json_start_index = ld_json_text.index('{')
|
||||
ld_json_end_index = ld_json_text.rindex('}') + 1
|
||||
ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
|
||||
ld_json = json.loads(ld_json_json)
|
||||
ld_json_price = ld_json['offers']['price']
|
||||
# Print the price
|
||||
print(ld_json_price)
|
||||
|
||||
unit_price_element = WebDriverWait(driver, 5).until(EC.presence_of_element_located(((By.XPATH,"//div[contains(@class, 'price_price')]").get_attribute('innerText'))))
|
||||
unit_price = unit_price_element.replace("$", "")
|
||||
print(f"unit_price = {unit_price}" )
|
||||
|
||||
subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message
|
||||
# Append the zip code and price to the dataframe
|
||||
#append_to_df(zip_code, price)
|
||||
new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Shown Price': [ld_json_price], 'Minimum Quantity': [min_quantity], 'Total Price': [total_price], 'Unit Price': [unit_price], 'Sales Count': [sales_count], 'Sold Status': [sold_status]})
|
||||
df_new = pd.concat([df_new, new_row], ignore_index=True)
|
||||
return df_new
|
||||
#print(f"Recorded the zip code and price: {zip_code},{price}")
|
||||
#print(f"Updated a dataframe: {df_new}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Caught exception: {e}")
|
||||
print(f"Price not found for zip code {zip_code}")
|
||||
return df_new
|
||||
|
||||
# Start receiving messages (sync version)
|
||||
while True:
|
||||
pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
|
||||
messages = []
|
||||
try:
|
||||
pull_response = subscriber.pull(request = pull_request)
|
||||
messages = pull_response.received_messages
|
||||
except DeadlineExceeded:
|
||||
pass
|
||||
|
||||
if messages:
|
||||
df_new = callback(messages[0])
|
||||
print(f"Updated a dataframe: {df_new}")
|
||||
else:
|
||||
print(f"no messages, trying again at {time.asctime()}")
|
||||
break
|
||||
return df_new
|
||||
|
||||
'''
|
||||
If using sys to get the function, please remove the following part of code.
|
||||
'''
|
||||
if __name__ == '__main__':
|
||||
project_id = input("Enter the project ID: ")
|
||||
subscription_name = input("Enter the subscription name: ")
|
||||
url = input("Enter the URL: ")
|
||||
|
||||
get_product_prices(project_id, subscription_name, url)
|
||||
|
||||
|
||||
df_new = get_product_prices("fp-20230425", "fp-subscription-NFL-zipcodes", "https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119")
|
||||
|
||||
# Now we have the new dataframe "df" recording the new price listed in weee's page
|
||||
# Load such data in the existing HDF5 file
|
||||
file_path = 'product1.h5'
|
||||
with h5py.File(file_path, 'a') as file:
|
||||
# Load the existing dataframe 'df' from the HDF5 file
|
||||
# REPLACE KEY TO GET THE EXISTING DATAFRAME FOR EACH DIFFERENT PRODUCT
|
||||
df = pd.read_hdf(file, key='df')
|
||||
|
||||
# Add timestamp to the new dataframe "df_new"
|
||||
# timestamp = datetime.now()
|
||||
'''
|
||||
To deal with the issue of inconsistent time when running a script on different systems, we can use 'ntplib' that
|
||||
provides time synchronization with a reliable time server.
|
||||
Also, to ensure that the local time is interpreted in the system's local time zone, we use 'arrow' which offers a
|
||||
sensible and human-friendly approach to format and convert the timestamps.
|
||||
'''
|
||||
# Fetch the time from an NTP server
|
||||
ntp_client = ntplib.NTPClient()
|
||||
response = ntp_client.request('pool.ntp.org')
|
||||
ntp_time = datetime.fromtimestamp(response.tx_time)
|
||||
timestamp = arrow.get(ntp_time).to('local')
|
||||
df_new['Timestamp'] = timestamp
|
||||
|
||||
# Append 'df_new' to the existing df stored in the key
|
||||
df_update = df.append(df_new, ignore_index=True)
|
||||
|
||||
# Save 'df2' to the HDF5 file with the key 'df'
|
||||
# REPLACE KEY FOR DIFFERENT PRODUCT
|
||||
df_update.to_hdf(file, key='df', mode='a')
|
||||
|
||||
print("Data appended and saved successfully.")
|
||||
@@ -0,0 +1,126 @@
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium.webdriver.common.by import By
|
||||
import json
|
||||
import time
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
|
||||
# Configure the Selenium driver options
|
||||
options = Options()
|
||||
options.add_argument('--disable-gpu')
|
||||
driver = webdriver.Chrome(options=options)
|
||||
|
||||
# Define the zip codes for which to extract the price
|
||||
zip_codes = ['85305', '30313', '21230', '14127','28202','60605','45202','44114','76011','80204','48226','54304','77054','46225','32202','64129','89118','90301','33056','55415','02035','70112','07073','19148','15212','95054','98134','33607','37213','20785']
|
||||
|
||||
# Load the page and extract the __NEXT_DATA__ data
|
||||
#url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
|
||||
url = 'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261'
|
||||
#url = 'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467'
|
||||
#url = 'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01'
|
||||
#url = 'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224'
|
||||
#url = 'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02'
|
||||
#url = 'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01'
|
||||
#url = 'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f'
|
||||
#url = 'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353?trace_id=7847dd6e-c511-48e2-a76e-a283399967b1'
|
||||
#url = 'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03'
|
||||
|
||||
driver.delete_all_cookies
|
||||
driver.get(url)
|
||||
|
||||
# Close the cookies setting window
|
||||
try:
|
||||
cookie_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button")))
|
||||
print(f"cookie_button = {cookie_button}")
|
||||
cookie_button.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Clear any modal dialog
|
||||
try:
|
||||
modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
|
||||
print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
def callback(zip_code):
|
||||
try:
|
||||
try:
|
||||
# Clear any modal dialog
|
||||
modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
|
||||
# print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
try:
|
||||
# Clear any notice
|
||||
notice_close = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
|
||||
# print(f"notice_close = {notice_close}")
|
||||
notice_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
# Clear any modal dialog
|
||||
modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
|
||||
# print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Enter zip code in the input field and click the "Check" button
|
||||
zip_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')))
|
||||
# print(f"{zip_button:}")
|
||||
zip_button.click()
|
||||
|
||||
zip_input = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input')))
|
||||
# print(f"{zip_input:}")
|
||||
zip_input.click()
|
||||
# print("clearing zip_input")
|
||||
zip_input.send_keys("\b\b\b\b\b")
|
||||
# print("sending zip code")
|
||||
zip_input.send_keys(zip_code)
|
||||
|
||||
try:
|
||||
check_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//button[text() = 'Confirm']")))
|
||||
# print(f"{check_button:}")
|
||||
except:
|
||||
check_button = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[5]/div/div[2]/div/div[2]/div[2]/div/div[1]/form/div/div/div[1]/div/div/div[3]/div/button")))
|
||||
finally:
|
||||
check_button.click()
|
||||
|
||||
try:
|
||||
# Clear any notice
|
||||
notice_close = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")))
|
||||
# print(f"notice_close = {notice_close}")
|
||||
notice_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
# Clear any modal dialog
|
||||
modal_close = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")))
|
||||
# print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Extract the price of rice crackers for this zip code
|
||||
try:
|
||||
sold_out_notice = driver.find_element(By.XPATH, '//*[@id="soldStatusNotice"]')
|
||||
print(f"No such product in this area = {zip_code, sold_out_notice}")
|
||||
except:
|
||||
print(f"The product is available in this area {zip_code}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Caught exception: {e}")
|
||||
print(f"Price not found for zip code {zip_code}")
|
||||
|
||||
for zip_code in zip_codes:
|
||||
callback(zip_code)
|
||||
|
||||
driver.quit()
|
||||
@@ -0,0 +1,166 @@
|
||||
import sqlalchemy
|
||||
from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Float, DateTime
|
||||
from sqlalchemy.orm import declarative_base
|
||||
from sqlalchemy.orm import relationship, sessionmaker
|
||||
import sys
|
||||
from flask import Flask, render_template
|
||||
|
||||
# Create the Flask application
|
||||
app = Flask(__name__)
|
||||
|
||||
# Create the database engine
|
||||
engine = create_engine('sqlite:///scraper_design.db')
|
||||
|
||||
# Create a session factory
|
||||
Session = sessionmaker(bind=engine)
|
||||
|
||||
# Create a base class
|
||||
Base = declarative_base()
|
||||
|
||||
# Define the product class
|
||||
class Product(Base):
|
||||
__tablename__ = 'product'
|
||||
id = Column(Integer, primary_key=True)
|
||||
name = Column(String)
|
||||
SKU_PLU_UPC_number = Column(String)
|
||||
price_histories = relationship("PriceHistory", back_populates="product")
|
||||
|
||||
# Define the retailer class
|
||||
class Retailer(Base):
|
||||
__tablename__ = 'retailer'
|
||||
id = Column(Integer, primary_key=True)
|
||||
name = Column(String)
|
||||
products = relationship("Product", secondary="retailer_product")
|
||||
|
||||
# Define the price class
|
||||
class Price(Base):
|
||||
__tablename__ = 'price'
|
||||
id = Column(Integer, primary_key=True)
|
||||
MOQ = Column(Integer)
|
||||
price = Column(Float)
|
||||
|
||||
# Define the scraperjob class
|
||||
class ScraperJob(Base):
|
||||
__tablename__ = 'scraper_job'
|
||||
id = Column(Integer, primary_key=True)
|
||||
start_time = Column(DateTime)
|
||||
end_time = Column(DateTime)
|
||||
parameters = Column(String)
|
||||
statistics = Column(String)
|
||||
|
||||
# Define the price history class
|
||||
class PriceHistory(Base):
|
||||
__tablename__ = 'price_history'
|
||||
id = Column(Integer, primary_key=True)
|
||||
location = Column(String)
|
||||
zip_code = Column(Integer)
|
||||
city_name = Column(String)
|
||||
price_at_time = Column(DateTime)
|
||||
price = Column(Float)
|
||||
scraper_job_id = Column(Integer, ForeignKey('scraper_job.id'))
|
||||
product_id = Column(Integer, ForeignKey('product.id'))
|
||||
product = relationship("Product", back_populates="price_histories")
|
||||
scraper_job = relationship("ScraperJob")
|
||||
|
||||
# Define the user class
|
||||
class User(Base):
|
||||
__tablename__ = 'user'
|
||||
id = Column(Integer, primary_key=True)
|
||||
username = Column(String)
|
||||
credentials = Column(String)
|
||||
creation_date = Column(DateTime)
|
||||
modification_date = Column(DateTime)
|
||||
def __str__(self):
|
||||
return f"{self.id=} {self.username=} {self.creation_date=} {self.modification_date=}"
|
||||
|
||||
class RetailerProduct(Base):
|
||||
__tablename__ = 'retailer_product'
|
||||
retailer_id = Column(Integer, ForeignKey('retailer.id'), primary_key=True)
|
||||
product_id = Column(Integer, ForeignKey('product.id'), primary_key=True)
|
||||
|
||||
# Create the database tables
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
@app.route('/')
|
||||
def homepage():
|
||||
# Create a session
|
||||
session = Session()
|
||||
|
||||
# Get all the classes from SQLAlchemy
|
||||
classes = Base.__subclasses__()
|
||||
|
||||
statement = sqlalchemy.select(User)
|
||||
users = session.scalars(statement).all()
|
||||
|
||||
# Render the template with the classes
|
||||
return render_template('test3.html', classes=classes, users=users)
|
||||
|
||||
# Create command-line arguments for the user class
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
app.run()
|
||||
|
||||
command = sys.argv[1]
|
||||
|
||||
if command == "create":
|
||||
if len(sys.argv) != 3:
|
||||
print("Usage: python user_tool.py create [username]")
|
||||
return
|
||||
|
||||
username = sys.argv[2]
|
||||
|
||||
# Create a new user object
|
||||
user = User(username=username)
|
||||
|
||||
# Add the user to the database
|
||||
session = Session()
|
||||
session.add(user)
|
||||
session.commit()
|
||||
|
||||
print("User created successfully.")
|
||||
|
||||
elif command == "edit":
|
||||
if len(sys.argv) != 4:
|
||||
print("Usage: python user_tool.py edit [user_id] [new_username]")
|
||||
return
|
||||
|
||||
user_id = int(sys.argv[2])
|
||||
new_username = sys.argv[3]
|
||||
|
||||
# Retrieve the user from the database
|
||||
session = Session()
|
||||
user = session.query(User).get(user_id)
|
||||
|
||||
if user:
|
||||
# Update the user object
|
||||
user.username = new_username
|
||||
session.commit()
|
||||
print("User updated successfully.")
|
||||
else:
|
||||
print("User not found.")
|
||||
|
||||
elif command == "delete":
|
||||
if len(sys.argv) != 3:
|
||||
print("Usage: python user_tool.py delete [user_id]")
|
||||
return
|
||||
|
||||
user_id = int(sys.argv[2])
|
||||
|
||||
# Retrieve the user from the database
|
||||
session = Session()
|
||||
user = session.query(User).get(user_id)
|
||||
|
||||
if user:
|
||||
# Delete the user from the database
|
||||
session.delete(user)
|
||||
session.commit()
|
||||
print("User deleted successfully.")
|
||||
else:
|
||||
print("User not found.")
|
||||
|
||||
else:
|
||||
print("Usage: python user_tool.py [create|edit|delete]")
|
||||
return
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,33 @@
|
||||
ZIP Code
|
||||
85305
|
||||
30313
|
||||
21230
|
||||
14127
|
||||
28202
|
||||
60605
|
||||
45202
|
||||
44114
|
||||
76011
|
||||
80204
|
||||
48226
|
||||
54304
|
||||
77054
|
||||
46225
|
||||
32202
|
||||
64129
|
||||
89118
|
||||
90301
|
||||
90301
|
||||
33056
|
||||
55415
|
||||
02035
|
||||
70112
|
||||
07073
|
||||
07073
|
||||
19148
|
||||
15212
|
||||
95054
|
||||
98134
|
||||
33607
|
||||
37213
|
||||
20785
|
||||
|
@@ -0,0 +1,8 @@
|
||||
google.cloud.pubsub
|
||||
plotly
|
||||
pandas
|
||||
numpy
|
||||
h5py
|
||||
selenium
|
||||
sqlalchemy
|
||||
flask
|
||||
@@ -0,0 +1,33 @@
|
||||
ZIP Code
|
||||
85305
|
||||
30313
|
||||
21230
|
||||
14127
|
||||
28202
|
||||
60605
|
||||
45202
|
||||
44114
|
||||
76011
|
||||
80204
|
||||
48226
|
||||
54304
|
||||
77054
|
||||
46225
|
||||
32202
|
||||
64129
|
||||
89118
|
||||
90301
|
||||
90301
|
||||
33056
|
||||
55415
|
||||
02035
|
||||
70112
|
||||
07073
|
||||
07073
|
||||
19148
|
||||
15212
|
||||
95054
|
||||
98134
|
||||
33607
|
||||
37213
|
||||
20785
|
||||
|
@@ -0,0 +1,30 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Users Table</title>
|
||||
</head>
|
||||
<body>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>ID</th>
|
||||
<th>Username</th>
|
||||
<th>Credentials</th>
|
||||
<th>Creation Date</th>
|
||||
<th>Modification Date</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for user in users %}
|
||||
<tr>
|
||||
<td>{{ user.id }}</td>
|
||||
<td>{{ user.username }}</td>
|
||||
<td>{{ user.credentials }}</td>
|
||||
<td>{{ user.creation_date }}</td>
|
||||
<td>{{ user.modification_date }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,21 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Class Dump</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Class Dump</h1>
|
||||
<ul>
|
||||
{% for class in classes %}
|
||||
<li>{{ class.__name__ }}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
|
||||
<h1>User Dump</h1>
|
||||
<ul>
|
||||
{% for user in users %}
|
||||
<li>{{ user }}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,45 @@
|
||||
# Selected
|
||||
# Want-Want Rice Crackers
|
||||
url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
|
||||
|
||||
# Lay's Potato Chips
|
||||
url = 'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261'
|
||||
|
||||
# Pocky Cookies
|
||||
url = 'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467'
|
||||
|
||||
# Thai Jasmine Rice
|
||||
url = 'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01'
|
||||
|
||||
# Japanese Medium Grain Rice
|
||||
url = 'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224'
|
||||
|
||||
# Millet
|
||||
url = 'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02'
|
||||
|
||||
# Nature's Soy Firm Tofu: https://www.sayweee.com/zh/product/Nature-s-Soy-Firm-Tofu-/19231?trace_id=ee63a29e-a1e4-4065-b213-96aa14a15521
|
||||
|
||||
# Free 2% Reduced Fat Milk: https://www.sayweee.com/zh/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353?trace_id=3030fe4c-04fc-4f8f-9257-7f6395b9e513
|
||||
|
||||
# Washington Fuji Apples: https://www.sayweee.com/zh/product/Washington-Fuji-Apples/94817?trace_id=dc4955fc-bba8-48cf-b1f9-9db3cf077eed
|
||||
|
||||
# Greenlandic Sweet Prawns: https://www.sayweee.com/zh/product/Royal-Greenland-Cooked-Wild-Caught-Greenlandic-Sweet-Prawns-Frozen-1/44656?category=seafood01
|
||||
|
||||
|
||||
|
||||
# Optional
|
||||
# LuoSiFen
|
||||
url = 'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01'
|
||||
|
||||
# Rice Noodles
|
||||
url = 'https://www.sayweee.com/en/product/Sanyangyishi-Fresh-Instant-Rice-Noodles/2156?category=instant01'
|
||||
|
||||
# Cup Noodles
|
||||
url = 'https://www.sayweee.com/en/product/Nissin-Beef-Flavor-Cup-Noodles/102519?category=instant01'
|
||||
|
||||
# Pork Sung
|
||||
url = 'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?category=canned04'
|
||||
|
||||
# Nature's Soy Firm Tofu: https://www.sayweee.com/zh/product/Nature-s-Soy-Firm-Tofu-/19231?trace_id=ee63a29e-a1e4-4065-b213-96aa14a15521
|
||||
# [Not available or sold out in some area] Beef Ribeye Thin Slices: https://www.sayweee.com/zh/product/Beef-Ribeye-Thin-Slices--Frozen-1/4295?category=meat02
|
||||
# Shiitake Mushrooms: https://www.sayweee.com/zh/product/Shiitake-Mushrooms/45782?trace_id=ba388941-f631-4eaa-8570-5b7d4aba64e7
|
||||
Reference in New Issue
Block a user