Compare commits
32
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e72d07462d | ||
|
|
a77fddefc4 | ||
|
|
dd7f954e6c | ||
|
|
b70051c109 | ||
|
|
cb88499ac9 | ||
|
|
f0b8c0fe9d | ||
|
|
caf98f8302 | ||
|
|
33beb1723f | ||
|
|
f5bedf2691 | ||
|
|
74e537e6d1 | ||
|
|
cd52ceb474 | ||
|
|
35ec8040f1 | ||
|
|
245b135a25 | ||
|
|
93817c20d9 | ||
|
|
d5a09ecb17 | ||
|
|
77f2214c3f | ||
|
|
94123d93f0 | ||
|
|
a4d5738583 | ||
|
|
4e3e3fe37e | ||
|
|
9f8c120d8a | ||
|
|
ba730af7cb | ||
|
|
2c113bed43 | ||
|
|
484f1c31e4 | ||
|
|
9445050af1 | ||
|
|
31278dbc6e | ||
|
|
ff270aed61 | ||
|
|
49545ed01b | ||
|
|
d0860db036 | ||
|
|
9d0adc580b | ||
|
|
84f99b0461 | ||
|
|
3eb2fa97a2 | ||
|
|
849f11dfbb |
@@ -1,100 +0,0 @@
|
||||
# A quick brain teaser
|
||||
print('What has a head and a tail but no body?')
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
# A coin
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[1]:
|
||||
|
||||
|
||||
import requests
|
||||
|
||||
url = 'https://www.sayweee.com/zh/product/Vital-Farms-Pasture-Raised-Eggs--Large-18-count/40709'
|
||||
response = requests.get(url)
|
||||
html_content = response.text
|
||||
html_content
|
||||
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
from lxml import html
|
||||
|
||||
tree = html.fromstring(html_content)
|
||||
script = tree.xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
|
||||
next_data = script.split('__NEXT_DATA__ = ')[1].split(';</script>')[0]
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
import json
|
||||
|
||||
data = json.loads(next_data)
|
||||
price = data['props']['initialState']['pdp']['product']['price']
|
||||
print(price)
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[1]:
|
||||
|
||||
|
||||
from google.cloud import pubsub_v1
|
||||
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
# Set the project ID and topic name
|
||||
project_id = "sloetjes-project1"
|
||||
topic_name = "sloetjes-project1"
|
||||
subscription_name = "sloetjes-final-project1"
|
||||
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
# Create a publisher client
|
||||
publisher = pubsub_v1.PublisherClient()
|
||||
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
# Create the fully qualified topic path
|
||||
topic_path = publisher.topic_path(project_id, topic_name)
|
||||
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
# Create the Pub/Sub topic
|
||||
topic = publisher.create_topic(name = topic_path)
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
# Print the topic name
|
||||
print(f"Topic created: {topic.name}")
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
# Create the subscription if it doesn't exist
|
||||
subscription_path = subscriber.subscription_path(project_id, subscription_name)
|
||||
topic_path = publisher.topic_path(project_id, topic_name)
|
||||
|
||||
# Publish a message to the topic
|
||||
message = "Hello, World!"
|
||||
data = message.encode("utf-8")
|
||||
publisher.publish(topic_path, data=data)
|
||||
print(f"Message published: {message}")
|
||||
|
||||
# Pull the message from the subscription
|
||||
response = subscriber.pull(subscription_path, max_messages=1)
|
||||
for received_message in response.received_messages:
|
||||
print(f"Received message: {received_message.message.data}")
|
||||
|
||||
# Acknowledge the message
|
||||
subscriber.acknowledge(subscription_path, [received_message.ack_id])
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[1]:
|
||||
import os
|
||||
from google.cloud import pubsub_v1
|
||||
|
||||
|
||||
# In[2]:
|
||||
# Set the project ID and topic name
|
||||
project_id = 'price-extraction-project'
|
||||
topic_name = 'nfl-zipcodes'
|
||||
|
||||
|
||||
# In[3]:
|
||||
# Create a publisher client
|
||||
publisher = pubsub_v1.PublisherClient()
|
||||
|
||||
|
||||
# In[4]:
|
||||
# Get the topic path
|
||||
topic_path = publisher.topic_path(project_id, topic_name)
|
||||
|
||||
|
||||
# In[5]:
|
||||
# Define the zip codes for all NFL cities
|
||||
nfl_zipcodes = {
|
||||
'Arizona Cardinals': '85305',
|
||||
'Atlanta Falcons': '30363',
|
||||
'Baltimore Ravens': '21230',
|
||||
'Buffalo Bills': '14127',
|
||||
'Carolina Panthers': '28202',
|
||||
'Chicago Bears': '60605',
|
||||
'Cincinnati Bengals': '45202',
|
||||
'Cleveland Browns': '44115',
|
||||
'Dallas Cowboys': '76011',
|
||||
'Denver Broncos': '80204',
|
||||
'Detroit Lions': '48226',
|
||||
'Green Bay Packers': '54304',
|
||||
'Houston Texans': '77054',
|
||||
'Indianapolis Colts': '46225',
|
||||
'Jacksonville Jaguars': '32202',
|
||||
'Kansas City Chiefs': '64129',
|
||||
'Las Vegas Raiders': '89118',
|
||||
'Los Angeles Chargers': '92626',
|
||||
'Los Angeles Rams': '90305',
|
||||
'Miami Dolphins': '33056',
|
||||
'Minnesota Vikings': '55415',
|
||||
'New England Patriots': '02035',
|
||||
'New Orleans Saints': '70112',
|
||||
'New York Giants': '07073',
|
||||
'New York Jets': '07932',
|
||||
'Philadelphia Eagles': '19145',
|
||||
'Pittsburgh Steelers': '15212',
|
||||
'San Francisco 49ers': '95054',
|
||||
'Seattle Seahawks': '98056',
|
||||
'Tampa Bay Buccaneers': '33607',
|
||||
'Tennessee Titans': '37213',
|
||||
'Washington Football Team': '20785'
|
||||
}
|
||||
|
||||
|
||||
# In[6]:
|
||||
# Publish each zip code to the topic
|
||||
for team, zipcode in nfl_zipcodes.items():
|
||||
data = f'{zipcode}'.encode('utf-8')
|
||||
publisher_future = publisher.publish(topic_path, data=data)
|
||||
message_id = publisher_future.result()
|
||||
print(f"message_id = {message_id}")
|
||||
|
||||
print('All zip codes published.')
|
||||
|
||||
|
||||
# In[ ]:
|
||||
import os
|
||||
import time
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
from google.cloud import pubsub_v1
|
||||
|
||||
# Set the project ID and subscription name
|
||||
project_id = 'price-extraction-project'
|
||||
topic_name = 'nfl-zipcodes'
|
||||
subscription_name = 'sloetjes'
|
||||
|
||||
# Set the path to your ChromeDriver executable
|
||||
chrome_driver_path = "/Users/sloetjeswong/Downloads/chromedriver_mac_arm64/chromedriver"
|
||||
|
||||
# Set up a Google Cloud Pub/Sub subscriber
|
||||
subscriber = pubsub_v1.SubscriberClient()
|
||||
subscription_path = subscriber.subscription_path(project_id, subscription_name)
|
||||
|
||||
# Set up a ChromeDriver instance
|
||||
chrome_options = webdriver.ChromeOptions()
|
||||
chrome_options.add_argument("--headless") # Run Chrome in headless mode
|
||||
chrome_service = Service(chrome_driver_path)
|
||||
chrome_service.start()
|
||||
driver = webdriver.Remote(chrome_service.service_url, options=chrome_options)
|
||||
|
||||
def callback(message):
|
||||
# Get the zip code from the message
|
||||
zip_code = message.data.decode("utf-8")
|
||||
print(f"Received message with zip code: {zip_code}")
|
||||
|
||||
# Open the product page for the rice crackers for the given zip code
|
||||
product_url = f"https://www.sayweee.com/zh/product/Want-Want-Senbei-Rice-Crackers/17119?trace_id=89f7c636-05b8-4f52-abdd-74de3a83f45d&zip={zip_code}"
|
||||
driver.get(product_url)
|
||||
|
||||
# Wait for the price to load
|
||||
price_xpath = "//span[@class='price']"
|
||||
price_element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, price_xpath)))
|
||||
|
||||
# Extract the price and print it
|
||||
price = price_element.text
|
||||
print(f"Price for zip code {zip_code}: {price}")
|
||||
|
||||
# Acknowledge the message
|
||||
message.ack()
|
||||
|
||||
# Subscribe to the topic
|
||||
streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback)
|
||||
print(f"Listening for messages on {subscription_path}...")
|
||||
streaming_pull_future.result()
|
||||
|
||||
# Keep the script running
|
||||
while True:
|
||||
time.sleep(60)
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 130 KiB |
+165
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime, ForeignKey
|
||||
from sqlalchemy.orm import sessionmaker, relationship, declarative_base
|
||||
from flask import Flask, render_template
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
# Create a base class for declarative models
|
||||
Base = declarative_base()
|
||||
|
||||
class Product(Base):
|
||||
__tablename__ = 'product'
|
||||
id = Column(Integer, primary_key=True)
|
||||
name = Column(String)
|
||||
SKU_PLU_UPC_number = Column(String)
|
||||
price_histories = relationship("PriceHistory", back_populates="product")
|
||||
|
||||
class Retailer(Base):
|
||||
__tablename__ = 'retailer'
|
||||
id = Column(Integer, primary_key=True)
|
||||
name = Column(String)
|
||||
products = relationship("Product", secondary="retailer_product")
|
||||
|
||||
class Price(Base):
|
||||
__tablename__ = 'price'
|
||||
id = Column(Integer, primary_key=True)
|
||||
MOQ = Column(Integer)
|
||||
price = Column(Float)
|
||||
|
||||
class ScraperJob(Base):
|
||||
__tablename__ = 'scraper_job'
|
||||
id = Column(Integer, primary_key=True)
|
||||
start_time = Column(DateTime)
|
||||
end_time = Column(DateTime)
|
||||
parameters = Column(String)
|
||||
statistics = Column(String)
|
||||
|
||||
class PriceHistory(Base):
|
||||
__tablename__ = 'price_history'
|
||||
id = Column(Integer, primary_key=True)
|
||||
location = Column(String)
|
||||
zip_code = Column(Integer)
|
||||
city_name = Column(String)
|
||||
price_at_time = Column(DateTime)
|
||||
price = Column(Float)
|
||||
scraper_job_id = Column(Integer, ForeignKey('scraper_job.id'))
|
||||
product_id = Column(Integer, ForeignKey('product.id'))
|
||||
product = relationship("Product", back_populates="price_histories")
|
||||
scraper_job = relationship("ScraperJob")
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = 'user'
|
||||
id = Column(Integer, primary_key=True)
|
||||
username = Column(String)
|
||||
credentials = Column(String)
|
||||
creation_date = Column(DateTime)
|
||||
modification_date = Column(DateTime)
|
||||
|
||||
class RetailerProduct(Base):
|
||||
__tablename__ = 'retailer_product'
|
||||
retailer_id = Column(Integer, ForeignKey('retailer.id'), primary_key=True)
|
||||
product_id = Column(Integer, ForeignKey('product.id'), primary_key=True)
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
# Create the database engine
|
||||
engine = create_engine('sqlite:///user_database.db')
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
# Create a session factory
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
import sys
|
||||
|
||||
# Create a command-line tool that parses sys.argv to allow a user to create/edit/delete user objects.
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
app = Flask(__name__)
|
||||
|
||||
# Create a database connection
|
||||
engine = create_engine('sqlite:///user_database.db')
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
|
||||
@app.route('/')
|
||||
def home():
|
||||
# Get all user objects
|
||||
users = session.query(User).all()
|
||||
return render_template('index.html', users=users)
|
||||
|
||||
app.run()
|
||||
|
||||
|
||||
command = sys.argv[1]
|
||||
|
||||
if command == "create":
|
||||
if len(sys.argv) < 4:
|
||||
print("Usage: python script.py create [username] [credentials]")
|
||||
sys.exit(1)
|
||||
|
||||
username = sys.argv[2]
|
||||
credentials = sys.argv[3]
|
||||
|
||||
# Create a new user object
|
||||
new_user = User(username=username, credentials=credentials)
|
||||
session.add(new_user)
|
||||
session.commit()
|
||||
print("User created successfully!")
|
||||
|
||||
elif command == "edit":
|
||||
if len(sys.argv) < 4:
|
||||
print("Usage: python script.py edit [username] [new_credentials]")
|
||||
sys.exit(1)
|
||||
|
||||
username = sys.argv[2]
|
||||
new_credentials = sys.argv[3]
|
||||
|
||||
# Find the user object
|
||||
user = session.query(User).filter_by(username=username).first()
|
||||
|
||||
if user:
|
||||
# Update user credentials
|
||||
user.credentials = new_credentials
|
||||
session.commit()
|
||||
print("User edited successfully!")
|
||||
else:
|
||||
print("User not found!")
|
||||
|
||||
elif command == "delete":
|
||||
if len(sys.argv) < 3:
|
||||
print("Usage: python script.py delete [username]")
|
||||
sys.exit(1)
|
||||
|
||||
username = sys.argv[2]
|
||||
|
||||
# Find the user object
|
||||
user = session.query(User).filter_by(username=username).first()
|
||||
|
||||
if user:
|
||||
# Delete the user object
|
||||
session.delete(user)
|
||||
session.commit()
|
||||
print("User deleted successfully!")
|
||||
else:
|
||||
print("User not found!")
|
||||
|
||||
else:
|
||||
print("Usage: python script.py [create|edit|delete] [username] [credentials]")
|
||||
@@ -0,0 +1,277 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
|
||||
from google.cloud import pubsub_v1
|
||||
from google.api_core.exceptions import DeadlineExceeded
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium.webdriver.common.by import By
|
||||
import json
|
||||
import time
|
||||
from concurrent.futures import TimeoutError
|
||||
import pandas as pd
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
import selenium.common.exceptions
|
||||
import re
|
||||
from datetime import datetime
|
||||
import arrow
|
||||
import subprocess
|
||||
import concurrent.futures
|
||||
|
||||
# This script is used to load the new price and the timestamp and append such data to an existing key in the HDF5 file "product1.h5"
|
||||
|
||||
def get_element_text(driver, class_name_prefix):
|
||||
'''
|
||||
Return the text in the first element with a given class name.
|
||||
|
||||
Note that XPath doesn't have a function to check if a given word is present
|
||||
in a list of words, so using contains() will match a class name even if
|
||||
the class name is a substring of the class name being matched against.
|
||||
To prevent this, we use spaces as a delimiter at the beginning and end of
|
||||
the class name, and ensure there's only one space before and after all
|
||||
class names in the class name list.
|
||||
|
||||
Example: consider two class names with cache-busting suffixes:
|
||||
* Header_soldStatusNotice__12345
|
||||
* Header_soldStatusNoticeBackground__ABCDE
|
||||
|
||||
Matching against Header_soldStatusNotice using a naive implementation with
|
||||
`contains()` would match both class names, even though only the first one
|
||||
is the correct match.
|
||||
'''
|
||||
# Construct xpath selector.
|
||||
xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against...
|
||||
xpath_str += "concat(' ', " # Step 2: a leading space, followed by...
|
||||
xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces...
|
||||
xpath_str += "' '), " # Step 4: a trailing space...
|
||||
xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
|
||||
try:
|
||||
match = driver.find_element(By.XPATH, xpath_str)
|
||||
return match.get_attribute('textContent')
|
||||
except selenium.common.exceptions.NoSuchElementException as nsee:
|
||||
pass
|
||||
|
||||
# We didn't find a match, try again by doing a prefix match and assume
|
||||
# the full class name ends with two underscores followed by a cache-
|
||||
# busting string.
|
||||
xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against...
|
||||
xpath_str += "concat(' ', " # Step 2: a leading space, followed by...
|
||||
xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces...
|
||||
xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
|
||||
try:
|
||||
match = driver.find_element(By.XPATH, xpath_str)
|
||||
return match.get_attribute('textContent')
|
||||
except selenium.common.exceptions.NoSuchElementException as nsee:
|
||||
pass
|
||||
|
||||
# No match, return empty string.
|
||||
return ''
|
||||
|
||||
|
||||
def scrape_product_price(project_id, subscription_name, url):
|
||||
# Create an empty dataframe to store the prices
|
||||
df_new = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
|
||||
print(f"Created a dataframe: {df_new}")
|
||||
|
||||
# Configure the Selenium driver options
|
||||
options = Options()
|
||||
options.add_argument('--disable-gpu')
|
||||
driver = webdriver.Chrome(options=options)
|
||||
|
||||
# Set up Pub/Sub subscription
|
||||
subscriber = pubsub_v1.SubscriberClient()
|
||||
subscription_path = subscriber.subscription_path(project_id, subscription_name)
|
||||
|
||||
# Load the product page and extract the __NEXT_DATA__ data
|
||||
product_id = url.split("/")[-1][:5]
|
||||
driver.delete_all_cookies()
|
||||
driver.get(url)
|
||||
|
||||
# Close the cookies setting window
|
||||
try:
|
||||
cookie_button = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
|
||||
)
|
||||
print(f"cookie_button = {cookie_button}")
|
||||
cookie_button.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Clear any modal dialog
|
||||
try:
|
||||
modal_close = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
|
||||
)
|
||||
print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
def callback(message):
|
||||
# Extract zip code from message data
|
||||
zip_code = message.message.data.decode('utf-8')
|
||||
print(f"Received zip code: {zip_code}")
|
||||
|
||||
try:
|
||||
# Clear any modal dialog
|
||||
modal_close = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
|
||||
)
|
||||
print(f"modal_close = {modal_close}")
|
||||
modal_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
# Clear any notice
|
||||
notice_close = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
|
||||
)
|
||||
print(f"notice_close = {notice_close}")
|
||||
notice_close.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Enter zip code in the input field and click the "Check" button
|
||||
zip_button = WebDriverWait(driver, 5).until(
|
||||
EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
|
||||
)
|
||||
zip_button.click()
|
||||
|
||||
zip_input = WebDriverWait(driver, 2).until(
|
||||
EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
|
||||
)
|
||||
zip_input.click()
|
||||
zip_input.send_keys("\b\b\b\b\b")
|
||||
zip_input.send_keys(zip_code)
|
||||
|
||||
check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
|
||||
check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//button[text() = 'Confirm']")))
|
||||
check_button.click()
|
||||
|
||||
# Record sales count, if any
|
||||
sales_count_element = get_element_text(driver, 'Header_count')
|
||||
count_start_index = sales_count_element.index('sold ') + len('sold ')
|
||||
count_end_index = sales_count_element.index('+')
|
||||
sales_count = sales_count_element[count_start_index:count_end_index]
|
||||
|
||||
# Record any sold status notice
|
||||
sold_status = get_element_text(driver, 'UnavailableCartbtn_container')
|
||||
if sold_status == 'Notify me':
|
||||
sold_status = 'Sold Out'
|
||||
|
||||
# Extract the price of the product for this zip code
|
||||
min_quantity = 0
|
||||
total_price = 0
|
||||
unit_price = 0
|
||||
try:
|
||||
minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]")
|
||||
if minimum_quantity_str:
|
||||
regexp = r"Minimum purchase: (\d+) \(\$([\d.]+)"
|
||||
match = re.search(regexp, minimum_quantity_str.get_attribute('innerHTML'))
|
||||
if match:
|
||||
min_quantity, total_price = match.groups()
|
||||
except selenium.common.exceptions.NoSuchElementException:
|
||||
print("No minimum purchase string found")
|
||||
except Exception as e:
|
||||
print(f"Exception when finding minimum purchase string: {e}")
|
||||
|
||||
ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
|
||||
ld_json_text = ld_json_element.get_attribute('innerHTML')
|
||||
ld_json_start_index = ld_json_text.index('{')
|
||||
ld_json_end_index = ld_json_text.rindex('}') + 1
|
||||
ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
|
||||
ld_json = json.loads(ld_json_json)
|
||||
ld_json_price = ld_json['offers']['price']
|
||||
print(ld_json_price)
|
||||
|
||||
unit_price_element = driver.find_element(By.XPATH, "//div[contains(@class, 'price_price')]")
|
||||
unit_price = unit_price_element.get_attribute('innerText').replace("$", "")
|
||||
print(f"unit_price = {unit_price}")
|
||||
|
||||
subscriber.acknowledge(ack_ids=[message.ack_id], subscription=subscription_path) # acknowledge receipt of message
|
||||
|
||||
new_row = pd.DataFrame({
|
||||
'Product ID': [product_id],
|
||||
'Zip Code': [zip_code],
|
||||
'Shown Price': [ld_json_price],
|
||||
'Minimum Quantity': [min_quantity],
|
||||
'Total Price': [total_price],
|
||||
'Unit Price': [unit_price],
|
||||
'Sales Count': [sales_count],
|
||||
'Sold Status': [sold_status]
|
||||
})
|
||||
return pd.concat([df_new, new_row], ignore_index=True)
|
||||
|
||||
while True:
|
||||
pull_request = pubsub_v1.types.PullRequest(max_messages=1, subscription=subscription_path)
|
||||
messages = []
|
||||
try:
|
||||
pull_response = subscriber.pull(request=pull_request)
|
||||
messages = pull_response.received_messages
|
||||
except DeadlineExceeded:
|
||||
pass
|
||||
|
||||
if messages:
|
||||
df_new = callback(messages[0])
|
||||
print(f"Updated dataframe: {df_new}")
|
||||
else:
|
||||
print(f"No messages, trying again at {time.asctime()}")
|
||||
break
|
||||
|
||||
return df_new
|
||||
|
||||
|
||||
def scrape_multiple_products(project_id, subscription_name, url_list):
|
||||
df_list = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
|
||||
futures = []
|
||||
for url in url_list:
|
||||
future = executor.submit(scrape_product_price, project_id, subscription_name, url)
|
||||
futures.append(future)
|
||||
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
df = future.result()
|
||||
df_list.append(df)
|
||||
print("Scraped data for URL:", url)
|
||||
|
||||
return df_list
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
project_id = input("Enter the project ID: ")
|
||||
subscription_name = input("Enter the subscription name: ")
|
||||
url_list = [
|
||||
'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
|
||||
'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
|
||||
'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
|
||||
'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
|
||||
'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
|
||||
'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
|
||||
'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
|
||||
'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
|
||||
'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
|
||||
'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
|
||||
'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
|
||||
]
|
||||
|
||||
df_list = scrape_multiple_products(project_id, subscription_name, url_list)
|
||||
|
||||
# Concatenate all dataframes in the list
|
||||
df_combined = pd.concat(df_list, ignore_index=True)
|
||||
|
||||
# Save the combined dataframe to HDF5 file
|
||||
with pd.HDFStore('product1.h5') as store:
|
||||
store.append('key', df_combined, format='table')
|
||||
|
||||
# Print the combined dataframe
|
||||
print("Combined DataFrame:")
|
||||
print(df_combined)
|
||||
|
||||
# Save the combined dataframe to a CSV file
|
||||
df_combined.to_csv('product1.csv', index=False)
|
||||
|
||||
print("Data scraping and storage completed.")
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCbUrTPE/8CUByUtG9rBuMP5zeF3x5sZcPmm61bcxBblvHL5Dx1WwIjujhbZ5mhxnmDqxV07MCFZwMzlPjJ7Ow9erZvaXAe5+jS2iTQqu9apfj+U3x2xEZQmoKprXUpWl8G9YDk2wxBqCn//ugkJ5+erIEopNUGPcVhACucpaCT+kld9mHtXBjdUEKUA85HhXiBkuBsv9k/VqL/oXbvHjY8Z6gFP64LcxdNASuf4Qn1wW0w3kXlQRkSoGdSxpGLztXN1v5rvetOzjA8HXixh7TYVWCc+XCa9KsY3QRk7gFyvtIsXz59NMOTgfpCnUQE2Pk9R5wuuA2fXyr+Kg+7NPkd sloetjeswong@Sloetjess-Air
|
||||
@@ -0,0 +1,30 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Users Table</title>
|
||||
</head>
|
||||
<body>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>ID</th>
|
||||
<th>Username</th>
|
||||
<th>Credentials</th>
|
||||
<th>Creation Date</th>
|
||||
<th>Modification Date</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for user in users %}
|
||||
<tr>
|
||||
<td>{{ user.id }}</td>
|
||||
<td>{{ user.username }}</td>
|
||||
<td>{{ user.credentials }}</td>
|
||||
<td>{{ user.creation_date }}</td>
|
||||
<td>{{ user.modification_date }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user