Author SHA1 Message Date
xuanyi.wang e72d07462d Update 'fp2_15_Sloetjes.py' 2023-06-01 22:09:05 +02:00
xuanyi.wang a77fddefc4 Upload files to '' 2023-06-01 22:08:27 +02:00
Sloetjes dd7f954e6c Updated public key 2023-05-31 18:01:11 -04:00
Sloetjes b70051c109 Fixed merge conflicts with sqlalchemy script 2023-05-31 17:55:35 -04:00
Sloetjes cb88499ac9 Fixed merge conflicts with NFL zip code script 2023-05-31 17:54:08 -04:00
xuanyi.wang f0b8c0fe9d Upload files to '' 2023-05-31 20:30:54 +02:00
cheezedoodle2 caf98f8302 Modified logic to start up the web server if no arguments are given. 2023-05-24 16:38:49 -07:00
cheezedoodle2 33beb1723f Added user dump template 2023-05-24 14:53:38 -07:00
cheezedoodle2 f5bedf2691 Import declarative_base from the right module 2023-05-24 14:48:02 -07:00
cheezedoodle2 74e537e6d1 Import declarative_base from the new module 2023-05-24 14:47:33 -07:00
xuanyi.wang cd52ceb474 Upload files to '' 2023-05-24 22:34:11 +02:00
cheezedoodle2 35ec8040f1 Sloetjes said to delete this 2023-05-23 15:05:14 -07:00
xuanyi.wang 245b135a25 Upload files to '' 2023-05-23 19:38:30 +02:00
xuanyi.wang 93817c20d9 Upload files to '' 2023-05-22 21:21:56 +02:00
xuanyi.wang d5a09ecb17 Upload files to '' 2023-05-22 21:11:44 +02:00
cheezedoodle2 77f2214c3f Now it doesn't work in a different way 2023-05-03 17:45:16 -07:00
xuanyi.wang 94123d93f0 Update 'NFL Zip Code.py' 2023-05-04 01:07:45 +02:00
xuanyi.wang a4d5738583 Upload files to '' 2023-05-04 01:05:35 +02:00
xuanyi.wang 4e3e3fe37e Update 'GoogleCloud.py' 2023-04-28 01:37:04 +02:00
Sloetjes 9f8c120d8a GoogleCloud 2023-04-27 19:35:52 -04:00
Sloetjes ba730af7cb Add Egg.py 2023-04-26 19:18:57 -04:00
Sloetjes 2c113bed43 Homework for web server 2023-04-19 20:49:15 -04:00
cheezedoodle2andSloetjes 484f1c31e4 Added loader column 2023-04-19 20:49:15 -04:00
cheezedoodle2andSloetjes 9445050af1 Working version of module inspector. 2023-04-19 20:49:15 -04:00
cheezedoodle2andSloetjes 31278dbc6e Added HTTP performance test 2023-04-19 20:49:15 -04:00
jandSloetjes ff270aed61 Update 'README.md' 2023-04-19 20:49:15 -04:00
cheezedoodle2andSloetjes 49545ed01b Added the ability for verbs to be a dict, for future use by an AI. 2023-04-19 20:49:15 -04:00
cheezedoodle2andSloetjes d0860db036 Added simulation files. 2023-04-19 20:49:15 -04:00
xuanyi.wang 9d0adc580b Upload files to '' 2023-04-19 04:42:57 +02:00
xuanyi.wang 84f99b0461 Upload files to '' 2023-04-14 01:01:24 +02:00
xuanyi.wang 3eb2fa97a2 Upload files to '' 2023-04-14 00:59:10 +02:00
xuanyi.wang 849f11dfbb Upload files to '' 2023-04-13 01:04:03 +02:00
20 changed files with 703 additions and 100 deletions
BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 82 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 75 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 77 KiB

+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env python
# coding: utf-8
# In[1]:
import requests
url = 'https://www.sayweee.com/zh/product/Vital-Farms-Pasture-Raised-Eggs--Large-18-count/40709'
response = requests.get(url)
html_content = response.text
html_content
# In[2]:
from lxml import html
tree = html.fromstring(html_content)
script = tree.xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
next_data = script.split('__NEXT_DATA__ = ')[1].split(';</script>')[0]
# In[ ]:
import json
data = json.loads(next_data)
price = data['props']['initialState']['pdp']['product']['price']
print(price)
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env python
# coding: utf-8
# In[1]:
from google.cloud import pubsub_v1
# In[2]:
# Set the project ID and topic name
project_id = "sloetjes-project1"
topic_name = "sloetjes-project1"
subscription_name = "sloetjes-final-project1"
# In[3]:
# Create a publisher client
publisher = pubsub_v1.PublisherClient()
# In[4]:
# Create the fully qualified topic path
topic_path = publisher.topic_path(project_id, topic_name)
# In[5]:
# Create the Pub/Sub topic
topic = publisher.create_topic(name = topic_path)
# In[ ]:
# Print the topic name
print(f"Topic created: {topic.name}")
# In[ ]:
# Create the subscription if it doesn't exist
subscription_path = subscriber.subscription_path(project_id, subscription_name)
topic_path = publisher.topic_path(project_id, topic_name)
# Publish a message to the topic
message = "Hello, World!"
data = message.encode("utf-8")
publisher.publish(topic_path, data=data)
print(f"Message published: {message}")
# Pull the message from the subscription
response = subscriber.pull(subscription_path, max_messages=1)
for received_message in response.received_messages:
print(f"Received message: {received_message.message.data}")
# Acknowledge the message
subscriber.acknowledge(subscription_path, [received_message.ack_id])
+131
View File
@@ -0,0 +1,131 @@
#!/usr/bin/env python
# coding: utf-8
# In[1]:
import os
from google.cloud import pubsub_v1
# In[2]:
# Set the project ID and topic name
project_id = 'price-extraction-project'
topic_name = 'nfl-zipcodes'
# In[3]:
# Create a publisher client
publisher = pubsub_v1.PublisherClient()
# In[4]:
# Get the topic path
topic_path = publisher.topic_path(project_id, topic_name)
# In[5]:
# Define the zip codes for all NFL cities
nfl_zipcodes = {
'Arizona Cardinals': '85305',
'Atlanta Falcons': '30363',
'Baltimore Ravens': '21230',
'Buffalo Bills': '14127',
'Carolina Panthers': '28202',
'Chicago Bears': '60605',
'Cincinnati Bengals': '45202',
'Cleveland Browns': '44115',
'Dallas Cowboys': '76011',
'Denver Broncos': '80204',
'Detroit Lions': '48226',
'Green Bay Packers': '54304',
'Houston Texans': '77054',
'Indianapolis Colts': '46225',
'Jacksonville Jaguars': '32202',
'Kansas City Chiefs': '64129',
'Las Vegas Raiders': '89118',
'Los Angeles Chargers': '92626',
'Los Angeles Rams': '90305',
'Miami Dolphins': '33056',
'Minnesota Vikings': '55415',
'New England Patriots': '02035',
'New Orleans Saints': '70112',
'New York Giants': '07073',
'New York Jets': '07932',
'Philadelphia Eagles': '19145',
'Pittsburgh Steelers': '15212',
'San Francisco 49ers': '95054',
'Seattle Seahawks': '98056',
'Tampa Bay Buccaneers': '33607',
'Tennessee Titans': '37213',
'Washington Football Team': '20785'
}
# In[6]:
# Publish each zip code to the topic
for team, zipcode in nfl_zipcodes.items():
data = f'{zipcode}'.encode('utf-8')
publisher_future = publisher.publish(topic_path, data=data)
message_id = publisher_future.result()
print(f"message_id = {message_id}")
print('All zip codes published.')
# In[ ]:
import os
import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from google.cloud import pubsub_v1
# Set the project ID and subscription name
project_id = 'price-extraction-project'
topic_name = 'nfl-zipcodes'
subscription_name = 'sloetjes'
# Set the path to your ChromeDriver executable
chrome_driver_path = "/Users/sloetjeswong/Downloads/chromedriver_mac_arm64/chromedriver"
# Set up a Google Cloud Pub/Sub subscriber
subscriber = pubsub_v1.SubscriberClient()
subscription_path = subscriber.subscription_path(project_id, subscription_name)
# Set up a ChromeDriver instance
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--headless") # Run Chrome in headless mode
chrome_service = Service(chrome_driver_path)
chrome_service.start()
driver = webdriver.Remote(chrome_service.service_url, options=chrome_options)
def callback(message):
# Get the zip code from the message
zip_code = message.data.decode("utf-8")
print(f"Received message with zip code: {zip_code}")
# Open the product page for the rice crackers for the given zip code
product_url = f"https://www.sayweee.com/zh/product/Want-Want-Senbei-Rice-Crackers/17119?trace_id=89f7c636-05b8-4f52-abdd-74de3a83f45d&zip={zip_code}"
driver.get(product_url)
# Wait for the price to load
price_xpath = "//span[@class='price']"
price_element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, price_xpath)))
# Extract the price and print it
price = price_element.text
print(f"Price for zip code {zip_code}: {price}")
# Acknowledge the message
message.ack()
# Subscribe to the topic
streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback)
print(f"Listening for messages on {subscription_path}...")
streaming_pull_future.result()
# Keep the script running
while True:
time.sleep(60)
BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 130 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/env python
# coding: utf-8
# In[ ]:
from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime, ForeignKey
from sqlalchemy.orm import sessionmaker, relationship, declarative_base
from flask import Flask, render_template
# In[ ]:
# Create a base class for declarative models
Base = declarative_base()
class Product(Base):
__tablename__ = 'product'
id = Column(Integer, primary_key=True)
name = Column(String)
SKU_PLU_UPC_number = Column(String)
price_histories = relationship("PriceHistory", back_populates="product")
class Retailer(Base):
__tablename__ = 'retailer'
id = Column(Integer, primary_key=True)
name = Column(String)
products = relationship("Product", secondary="retailer_product")
class Price(Base):
__tablename__ = 'price'
id = Column(Integer, primary_key=True)
MOQ = Column(Integer)
price = Column(Float)
class ScraperJob(Base):
__tablename__ = 'scraper_job'
id = Column(Integer, primary_key=True)
start_time = Column(DateTime)
end_time = Column(DateTime)
parameters = Column(String)
statistics = Column(String)
class PriceHistory(Base):
__tablename__ = 'price_history'
id = Column(Integer, primary_key=True)
location = Column(String)
zip_code = Column(Integer)
city_name = Column(String)
price_at_time = Column(DateTime)
price = Column(Float)
scraper_job_id = Column(Integer, ForeignKey('scraper_job.id'))
product_id = Column(Integer, ForeignKey('product.id'))
product = relationship("Product", back_populates="price_histories")
scraper_job = relationship("ScraperJob")
class User(Base):
__tablename__ = 'user'
id = Column(Integer, primary_key=True)
username = Column(String)
credentials = Column(String)
creation_date = Column(DateTime)
modification_date = Column(DateTime)
class RetailerProduct(Base):
__tablename__ = 'retailer_product'
retailer_id = Column(Integer, ForeignKey('retailer.id'), primary_key=True)
product_id = Column(Integer, ForeignKey('product.id'), primary_key=True)
# In[ ]:
# Create the database engine
engine = create_engine('sqlite:///user_database.db')
Base.metadata.create_all(engine)
# In[ ]:
# Create a session factory
Session = sessionmaker(bind=engine)
session = Session()
# In[ ]:
import sys
# Create a command-line tool that parses sys.argv to allow a user to create/edit/delete user objects.
if len(sys.argv) < 2:
app = Flask(__name__)
# Create a database connection
engine = create_engine('sqlite:///user_database.db')
Session = sessionmaker(bind=engine)
session = Session()
@app.route('/')
def home():
# Get all user objects
users = session.query(User).all()
return render_template('index.html', users=users)
app.run()
command = sys.argv[1]
if command == "create":
if len(sys.argv) < 4:
print("Usage: python script.py create [username] [credentials]")
sys.exit(1)
username = sys.argv[2]
credentials = sys.argv[3]
# Create a new user object
new_user = User(username=username, credentials=credentials)
session.add(new_user)
session.commit()
print("User created successfully!")
elif command == "edit":
if len(sys.argv) < 4:
print("Usage: python script.py edit [username] [new_credentials]")
sys.exit(1)
username = sys.argv[2]
new_credentials = sys.argv[3]
# Find the user object
user = session.query(User).filter_by(username=username).first()
if user:
# Update user credentials
user.credentials = new_credentials
session.commit()
print("User edited successfully!")
else:
print("User not found!")
elif command == "delete":
if len(sys.argv) < 3:
print("Usage: python script.py delete [username]")
sys.exit(1)
username = sys.argv[2]
# Find the user object
user = session.query(User).filter_by(username=username).first()
if user:
# Delete the user object
session.delete(user)
session.commit()
print("User deleted successfully!")
else:
print("User not found!")
else:
print("Usage: python script.py [create|edit|delete] [username] [credentials]")
-100
View File
@@ -1,100 +0,0 @@
# A quick brain teaser
print('What has a head and a tail but no body?')
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
# A coin
+277
View File
@@ -0,0 +1,277 @@
#!/usr/bin/env python
# coding: utf-8
from google.cloud import pubsub_v1
from google.api_core.exceptions import DeadlineExceeded
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
import json
import time
from concurrent.futures import TimeoutError
import pandas as pd
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import selenium.common.exceptions
import re
from datetime import datetime
import arrow
import subprocess
import concurrent.futures
# This script is used to load the new price and the timestamp and append such data to an existing key in the HDF5 file "product1.h5"
def get_element_text(driver, class_name_prefix):
'''
Return the text in the first element with a given class name.
Note that XPath doesn't have a function to check if a given word is present
in a list of words, so using contains() will match a class name even if
the class name is a substring of the class name being matched against.
To prevent this, we use spaces as a delimiter at the beginning and end of
the class name, and ensure there's only one space before and after all
class names in the class name list.
Example: consider two class names with cache-busting suffixes:
* Header_soldStatusNotice__12345
* Header_soldStatusNoticeBackground__ABCDE
Matching against Header_soldStatusNotice using a naive implementation with
`contains()` would match both class names, even though only the first one
is the correct match.
'''
# Construct xpath selector.
xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against...
xpath_str += "concat(' ', " # Step 2: a leading space, followed by...
xpath_str += "normalize-space(@class), " # Step 3: the list of class names, separated by single spaces...
xpath_str += "' '), " # Step 4: a trailing space...
xpath_str += f"' {class_name_prefix} ')]" # Step 5: compared against the class name with a single space on either side.
try:
match = driver.find_element(By.XPATH, xpath_str)
return match.get_attribute('textContent')
except selenium.common.exceptions.NoSuchElementException as nsee:
pass
# We didn't find a match, try again by doing a prefix match and assume
# the full class name ends with two underscores followed by a cache-
# busting string.
xpath_str = "//*[contains(" # Step 1: search for a class name by matching the beginning of the class name against...
xpath_str += "concat(' ', " # Step 2: a leading space, followed by...
xpath_str += "normalize-space(@class)), " # Step 3: the list of class names, separated by single spaces...
xpath_str += f"' {class_name_prefix}__')]" # Step 4: compared against the class name prefixed with a single space and ending with two underscores.
try:
match = driver.find_element(By.XPATH, xpath_str)
return match.get_attribute('textContent')
except selenium.common.exceptions.NoSuchElementException as nsee:
pass
# No match, return empty string.
return ''
def scrape_product_price(project_id, subscription_name, url):
# Create an empty dataframe to store the prices
df_new = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Shown Price', 'Minimum Quantity', 'Total Price', 'Unit Price','Sales Count','Sold Status'])
print(f"Created a dataframe: {df_new}")
# Configure the Selenium driver options
options = Options()
options.add_argument('--disable-gpu')
driver = webdriver.Chrome(options=options)
# Set up Pub/Sub subscription
subscriber = pubsub_v1.SubscriberClient()
subscription_path = subscriber.subscription_path(project_id, subscription_name)
# Load the product page and extract the __NEXT_DATA__ data
product_id = url.split("/")[-1][:5]
driver.delete_all_cookies()
driver.get(url)
# Close the cookies setting window
try:
cookie_button = WebDriverWait(driver, 2).until(
EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div[2]/div/div/div[3]/button"))
)
print(f"cookie_button = {cookie_button}")
cookie_button.click()
except:
pass
# Clear any modal dialog
try:
modal_close = WebDriverWait(driver, 2).until(
EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
)
print(f"modal_close = {modal_close}")
modal_close.click()
except:
pass
def callback(message):
# Extract zip code from message data
zip_code = message.message.data.decode('utf-8')
print(f"Received zip code: {zip_code}")
try:
# Clear any modal dialog
modal_close = WebDriverWait(driver, 2).until(
EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
)
print(f"modal_close = {modal_close}")
modal_close.click()
except:
pass
try:
# Clear any notice
notice_close = WebDriverWait(driver, 2).until(
EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
)
print(f"notice_close = {notice_close}")
notice_close.click()
except:
pass
# Enter zip code in the input field and click the "Check" button
zip_button = WebDriverWait(driver, 5).until(
EC.presence_of_element_located((By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]'))
)
zip_button.click()
zip_input = WebDriverWait(driver, 2).until(
EC.presence_of_element_located((By.XPATH, '//div/label/following-sibling::input'))
)
zip_input.click()
zip_input.send_keys("\b\b\b\b\b")
zip_input.send_keys(zip_code)
check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
check_button = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.XPATH, "//button[text() = 'Confirm']")))
check_button.click()
# Record sales count, if any
sales_count_element = get_element_text(driver, 'Header_count')
count_start_index = sales_count_element.index('sold ') + len('sold ')
count_end_index = sales_count_element.index('+')
sales_count = sales_count_element[count_start_index:count_end_index]
# Record any sold status notice
sold_status = get_element_text(driver, 'UnavailableCartbtn_container')
if sold_status == 'Notify me':
sold_status = 'Sold Out'
# Extract the price of the product for this zip code
min_quantity = 0
total_price = 0
unit_price = 0
try:
minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]")
if minimum_quantity_str:
regexp = r"Minimum purchase: (\d+) \(\$([\d.]+)"
match = re.search(regexp, minimum_quantity_str.get_attribute('innerHTML'))
if match:
min_quantity, total_price = match.groups()
except selenium.common.exceptions.NoSuchElementException:
print("No minimum purchase string found")
except Exception as e:
print(f"Exception when finding minimum purchase string: {e}")
ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
ld_json_text = ld_json_element.get_attribute('innerHTML')
ld_json_start_index = ld_json_text.index('{')
ld_json_end_index = ld_json_text.rindex('}') + 1
ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
ld_json = json.loads(ld_json_json)
ld_json_price = ld_json['offers']['price']
print(ld_json_price)
unit_price_element = driver.find_element(By.XPATH, "//div[contains(@class, 'price_price')]")
unit_price = unit_price_element.get_attribute('innerText').replace("$", "")
print(f"unit_price = {unit_price}")
subscriber.acknowledge(ack_ids=[message.ack_id], subscription=subscription_path) # acknowledge receipt of message
new_row = pd.DataFrame({
'Product ID': [product_id],
'Zip Code': [zip_code],
'Shown Price': [ld_json_price],
'Minimum Quantity': [min_quantity],
'Total Price': [total_price],
'Unit Price': [unit_price],
'Sales Count': [sales_count],
'Sold Status': [sold_status]
})
return pd.concat([df_new, new_row], ignore_index=True)
while True:
pull_request = pubsub_v1.types.PullRequest(max_messages=1, subscription=subscription_path)
messages = []
try:
pull_response = subscriber.pull(request=pull_request)
messages = pull_response.received_messages
except DeadlineExceeded:
pass
if messages:
df_new = callback(messages[0])
print(f"Updated dataframe: {df_new}")
else:
print(f"No messages, trying again at {time.asctime()}")
break
return df_new
def scrape_multiple_products(project_id, subscription_name, url_list):
df_list = []
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
futures = []
for url in url_list:
future = executor.submit(scrape_product_price, project_id, subscription_name, url)
futures.append(future)
for future in concurrent.futures.as_completed(futures):
df = future.result()
df_list.append(df)
print("Scraped data for URL:", url)
return df_list
if __name__ == '__main__':
project_id = input("Enter the project ID: ")
subscription_name = input("Enter the subscription name: ")
url_list = [
'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119',
'https://www.sayweee.com/en/product/Lays-Potato-Chips-Cucumber-Flavor/51261',
'https://www.sayweee.com/en/product/Glico-Pocky-Cookies-and-Cream-Family-Pack/79467',
'https://www.sayweee.com/en/product/Xiang-Na-Lan-Thai-Jasmine-Rice/19566?category=dried01',
'https://www.sayweee.com/en/product/Sekka-Premium-Japanese-Medium-Grain-Rice/81224',
'https://www.sayweee.com/en/product/Big-Green-Spring-Farm-Organic-Millet/90991?category=dried02',
'https://www.sayweee.com/en/product/Liuzhou-Guangxi-Specialty-LuoSiFen-Hot-Spicy-Noodles/95442?category=instant01',
'https://www.sayweee.com/en/product/House-Foods-Firm-Tofu-/85175?trace_id=c93f3d1a-80a4-4e1a-8d41-a402ae43186f',
'https://www.sayweee.com/en/product/Lactaid-Lactose-Free-2--Reduced-Fat-Milk/95353',
'https://www.sayweee.com/en/product/Washington-Fuji-Apples/94817?category=fruits03',
'https://www.sayweee.com/en/product/Formosa-Pork-Sung-/23528?trace_id=3c143dd4-4a35-4e8d-93ee-d91f209717e7'
]
df_list = scrape_multiple_products(project_id, subscription_name, url_list)
# Concatenate all dataframes in the list
df_combined = pd.concat(df_list, ignore_index=True)
# Save the combined dataframe to HDF5 file
with pd.HDFStore('product1.h5') as store:
store.append('key', df_combined, format='table')
# Print the combined dataframe
print("Combined DataFrame:")
print(df_combined)
# Save the combined dataframe to a CSV file
df_combined.to_csv('product1.csv', index=False)
print("Data scraping and storage completed.")
BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 10 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

+1
View File
@@ -0,0 +1 @@
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCbUrTPE/8CUByUtG9rBuMP5zeF3x5sZcPmm61bcxBblvHL5Dx1WwIjujhbZ5mhxnmDqxV07MCFZwMzlPjJ7Ow9erZvaXAe5+jS2iTQqu9apfj+U3x2xEZQmoKprXUpWl8G9YDk2wxBqCn//ugkJ5+erIEopNUGPcVhACucpaCT+kld9mHtXBjdUEKUA85HhXiBkuBsv9k/VqL/oXbvHjY8Z6gFP64LcxdNASuf4Qn1wW0w3kXlQRkSoGdSxpGLztXN1v5rvetOzjA8HXixh7TYVWCc+XCa9KsY3QRk7gFyvtIsXz59NMOTgfpCnUQE2Pk9R5wuuA2fXyr+Kg+7NPkd sloetjeswong@Sloetjess-Air
+30
View File
@@ -0,0 +1,30 @@
<!DOCTYPE html>
<html>
<head>
<title>Users Table</title>
</head>
<body>
<table>
<thead>
<tr>
<th>ID</th>
<th>Username</th>
<th>Credentials</th>
<th>Creation Date</th>
<th>Modification Date</th>
</tr>
</thead>
<tbody>
{% for user in users %}
<tr>
<td>{{ user.id }}</td>
<td>{{ user.username }}</td>
<td>{{ user.credentials }}</td>
<td>{{ user.creation_date }}</td>
<td>{{ user.modification_date }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</body>
</html>
BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 11 KiB