Fixed merge conflicts with sqlalchemy script

.vscode/launch.json

@@ -0,0 +1,12 @@
+{
+    "configurations": [
+        {
+            "name": "Python: HotdogSimulation",
+            "type": "python",
+            "request": "launch",
+            "program": "simulation/SimulationRunner.py",
+            "justMyCode": true,
+            "args": "./HotdogSimulation.py"
+        }
+    ]
+}

Egg.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+import requests
+
+url = 'https://www.sayweee.com/zh/product/Vital-Farms-Pasture-Raised-Eggs--Large-18-count/40709'
+response = requests.get(url)
+html_content = response.text
+html_content
+
+
+# In[2]:
+
+
+from lxml import html
+
+tree = html.fromstring(html_content)
+script = tree.xpath('//script[@id="__NEXT_DATA__"]/text()')[0]
+next_data = script.split('__NEXT_DATA__ = ')[1].split(';</script>')[0]
+
+
+# In[ ]:
+
+
+import json
+
+data = json.loads(next_data)
+price = data['props']['initialState']['pdp']['product']['price']
+print(price)
+

GoogleCloud.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+from google.cloud import pubsub_v1
+
+
+# In[2]:
+
+
+# Set the project ID and topic name
+project_id = "sloetjes-project1"
+topic_name = "sloetjes-project1"
+subscription_name = "sloetjes-final-project1"
+
+
+# In[3]:
+
+
+# Create a publisher client
+publisher = pubsub_v1.PublisherClient()
+
+
+# In[4]:
+
+
+# Create the fully qualified topic path
+topic_path = publisher.topic_path(project_id, topic_name)
+
+
+# In[5]:
+
+
+# Create the Pub/Sub topic
+topic = publisher.create_topic(name = topic_path)
+
+
+# In[ ]:
+
+
+# Print the topic name
+print(f"Topic created: {topic.name}")
+
+
+# In[ ]:
+
+
+# Create the subscription if it doesn't exist
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+topic_path = publisher.topic_path(project_id, topic_name)
+
+# Publish a message to the topic
+message = "Hello, World!"
+data = message.encode("utf-8")
+publisher.publish(topic_path, data=data)
+print(f"Message published: {message}")
+
+# Pull the message from the subscription
+response = subscriber.pull(subscription_path, max_messages=1)
+for received_message in response.received_messages:
+    print(f"Received message: {received_message.message.data}")
+
+    # Acknowledge the message
+    subscriber.acknowledge(subscription_path, [received_message.ack_id])

NFL Zip Code.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+import os
+from google.cloud import pubsub_v1
+
+
+# In[2]:
+# Set the project ID and topic name
+project_id = 'price-extraction-project'
+topic_name = 'nfl-zipcodes'
+
+
+# In[3]:
+# Create a publisher client
+publisher = pubsub_v1.PublisherClient()
+
+
+# In[4]:
+# Get the topic path
+topic_path = publisher.topic_path(project_id, topic_name)
+
+
+# In[5]:
+# Define the zip codes for all NFL cities
+nfl_zipcodes = {
+    'Arizona Cardinals': '85305',
+    'Atlanta Falcons': '30363',
+    'Baltimore Ravens': '21230',
+    'Buffalo Bills': '14127',
+    'Carolina Panthers': '28202',
+    'Chicago Bears': '60605',
+    'Cincinnati Bengals': '45202',
+    'Cleveland Browns': '44115',
+    'Dallas Cowboys': '76011',
+    'Denver Broncos': '80204',
+    'Detroit Lions': '48226',
+    'Green Bay Packers': '54304',
+    'Houston Texans': '77054',
+    'Indianapolis Colts': '46225',
+    'Jacksonville Jaguars': '32202',
+    'Kansas City Chiefs': '64129',
+    'Las Vegas Raiders': '89118',
+    'Los Angeles Chargers': '92626',
+    'Los Angeles Rams': '90305',
+    'Miami Dolphins': '33056',
+    'Minnesota Vikings': '55415',
+    'New England Patriots': '02035',
+    'New Orleans Saints': '70112',
+    'New York Giants': '07073',
+    'New York Jets': '07932',
+    'Philadelphia Eagles': '19145',
+    'Pittsburgh Steelers': '15212',
+    'San Francisco 49ers': '95054',
+    'Seattle Seahawks': '98056',
+    'Tampa Bay Buccaneers': '33607',
+    'Tennessee Titans': '37213',
+    'Washington Football Team': '20785'
+}
+
+
+# In[6]:
+# Publish each zip code to the topic
+for team, zipcode in nfl_zipcodes.items():
+    data = f'{zipcode}'.encode('utf-8')
+    publisher_future = publisher.publish(topic_path, data=data)
+    message_id = publisher_future.result()
+    print(f"message_id = {message_id}")
+
+print('All zip codes published.')
+
+
+# In[ ]:
+import os
+import time
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from google.cloud import pubsub_v1
+
+# Set the project ID and subscription name
+project_id = 'price-extraction-project'
+topic_name = 'nfl-zipcodes'
+subscription_name = 'sloetjes'
+
+# Set the path to your ChromeDriver executable
+chrome_driver_path = "/Users/sloetjeswong/Downloads/chromedriver_mac_arm64/chromedriver"
+
+# Set up a Google Cloud Pub/Sub subscriber
+subscriber = pubsub_v1.SubscriberClient()
+subscription_path = subscriber.subscription_path(project_id, subscription_name)
+
+# Set up a ChromeDriver instance
+chrome_options = webdriver.ChromeOptions()
+chrome_options.add_argument("--headless")  # Run Chrome in headless mode
+chrome_service = Service(chrome_driver_path)
+chrome_service.start()
+driver = webdriver.Remote(chrome_service.service_url, options=chrome_options)
+
+def callback(message):
+    # Get the zip code from the message
+    zip_code = message.data.decode("utf-8")
+    print(f"Received message with zip code: {zip_code}")
+
+    # Open the product page for the rice crackers for the given zip code
+    product_url = f"https://www.sayweee.com/zh/product/Want-Want-Senbei-Rice-Crackers/17119?trace_id=89f7c636-05b8-4f52-abdd-74de3a83f45d&zip={zip_code}"
+    driver.get(product_url)
+
+    # Wait for the price to load
+    price_xpath = "//span[@class='price']"
+    price_element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, price_xpath)))
+
+    # Extract the price and print it
+    price = price_element.text
+    print(f"Price for zip code {zip_code}: {price}")
+
+    # Acknowledge the message
+    message.ack()
+
+# Subscribe to the topic
+streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback)
+print(f"Listening for messages on {subscription_path}...")
+streaming_pull_future.result()
+
+# Keep the script running
+while True:
+    time.sleep(60)
+

README.md

@@ -1,6 +1,6 @@
 GTSP Python Bootcamp
 
-This bootcamp is intended for OTP interns with a math and applied analysis
+This bootcamp is intended for OPT interns with a math and applied analysis
 background to get up to speed quickly on how to use Python for practical
 commercial work.  It is not intended for large classes above 2-3 study groups
 of a maximum of 4 students each.

SQLAlchemy.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[ ]:
+
+
+from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime, ForeignKey
+from sqlalchemy.orm import sessionmaker, relationship, declarative_base
+from flask import Flask, render_template
+
+# In[ ]:
+
+
+# Create a base class for declarative models
+Base = declarative_base()
+
+class Product(Base):
+    __tablename__ = 'product'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    SKU_PLU_UPC_number = Column(String)
+    price_histories = relationship("PriceHistory", back_populates="product")
+
+class Retailer(Base):
+    __tablename__ = 'retailer'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    products = relationship("Product", secondary="retailer_product")
+
+class Price(Base):
+    __tablename__ = 'price'
+    id = Column(Integer, primary_key=True)
+    MOQ = Column(Integer)
+    price = Column(Float)
+
+class ScraperJob(Base):
+    __tablename__ = 'scraper_job'
+    id = Column(Integer, primary_key=True)
+    start_time = Column(DateTime)
+    end_time = Column(DateTime)
+    parameters = Column(String)
+    statistics = Column(String)
+
+class PriceHistory(Base):
+    __tablename__ = 'price_history'
+    id = Column(Integer, primary_key=True)
+    location = Column(String)
+    zip_code = Column(Integer)
+    city_name = Column(String)
+    price_at_time = Column(DateTime)
+    price = Column(Float)
+    scraper_job_id = Column(Integer, ForeignKey('scraper_job.id'))
+    product_id = Column(Integer, ForeignKey('product.id'))
+    product = relationship("Product", back_populates="price_histories")
+    scraper_job = relationship("ScraperJob")
+
+class User(Base):
+    __tablename__ = 'user'
+    id = Column(Integer, primary_key=True)
+    username = Column(String)
+    credentials = Column(String)
+    creation_date = Column(DateTime)
+    modification_date = Column(DateTime)
+
+class RetailerProduct(Base):
+    __tablename__ = 'retailer_product'
+    retailer_id = Column(Integer, ForeignKey('retailer.id'), primary_key=True)
+    product_id = Column(Integer, ForeignKey('product.id'), primary_key=True)
+
+
+# In[ ]:
+
+
+# Create the database engine
+engine = create_engine('sqlite:///user_database.db')
+Base.metadata.create_all(engine)
+
+
+# In[ ]:
+
+
+# Create a session factory
+Session = sessionmaker(bind=engine)
+session = Session()
+
+
+# In[ ]:
+
+
+import sys
+
+# Create a command-line tool that parses sys.argv to allow a user to create/edit/delete user objects.
+
+if len(sys.argv) < 2:
+    app = Flask(__name__)
+
+    # Create a database connection
+    engine = create_engine('sqlite:///user_database.db')
+    Session = sessionmaker(bind=engine)
+    session = Session()
+
+    @app.route('/')
+    def home():
+        # Get all user objects
+        users = session.query(User).all()
+        return render_template('index.html', users=users)
+
+    app.run()
+
+
+command = sys.argv[1]
+
+if command == "create":
+    if len(sys.argv) < 4:
+        print("Usage: python script.py create [username] [credentials]")
+        sys.exit(1)
+    
+    username = sys.argv[2]
+    credentials = sys.argv[3]
+
+    # Create a new user object
+    new_user = User(username=username, credentials=credentials)
+    session.add(new_user)
+    session.commit()
+    print("User created successfully!")
+
+elif command == "edit":
+    if len(sys.argv) < 4:
+        print("Usage: python script.py edit [username] [new_credentials]")
+        sys.exit(1)
+
+    username = sys.argv[2]
+    new_credentials = sys.argv[3]
+
+    # Find the user object
+    user = session.query(User).filter_by(username=username).first()
+
+    if user:
+        # Update user credentials
+        user.credentials = new_credentials
+        session.commit()
+        print("User edited successfully!")
+    else:
+        print("User not found!")
+
+elif command == "delete":
+    if len(sys.argv) < 3:
+        print("Usage: python script.py delete [username]")
+        sys.exit(1)
+
+    username = sys.argv[2]
+
+    # Find the user object
+    user = session.query(User).filter_by(username=username).first()
+
+    if user:
+        # Delete the user object
+        session.delete(user)
+        session.commit()
+        print("User deleted successfully!")
+    else:
+        print("User not found!")
+
+else:
+    print("Usage: python script.py [create|edit|delete] [username] [credentials]")

http-performance/Webpage.ipynb

@@ -0,0 +1,78 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0593e529",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from urllib.parse import urlparse\n",
+    "\n",
+    "class JPEGRequestHandler:\n",
+    "    def __init__(self, app):\n",
+    "        self.app = app\n",
+    "\n",
+    "    def __call__(self, environ, start_response):\n",
+    "        url = urlparse(environ['REQUEST_URI'])\n",
+    "        path = url.path\n",
+    "\n",
+    "        if path.endswith('.jpg'):\n",
+    "            file_path = path[1:]\n",
+    "\n",
+    "            if os.path.isfile(file_path):\n",
+    "                with open(file_path, 'rb') as file:\n",
+    "                    response_body = file.read()\n",
+    "                    status = '200 OK'\n",
+    "                    headers = [('Content-type', 'image/jpeg'),\n",
+    "                               ('Content-Length', str(len(response_body)))]\n",
+    "\n",
+    "                    start_response(status, headers)\n",
+    "                    return [response_body]\n",
+    "            else:\n",
+    "                status = '404 Not Found'\n",
+    "                headers = [('Content-type', 'text/plain'),\n",
+    "                           ('Content-Length', '0')]\n",
+    "\n",
+    "                start_response(status, headers)\n",
+    "                return []\n",
+    "        else:\n",
+    "            status = '403 Forbidden'\n",
+    "            headers = [('Content-type', 'text/plain'),\n",
+    "                       ('Content-Length', '0')]\n",
+    "\n",
+    "            start_response(status, headers)\n",
+    "            return []\n",
+    "\n",
+    "if __name__ == '__main__':\n",
+    "    from werkzeug.serving import run_simple\n",
+    "    from werkzeug.middleware.dispatcher import DispatcherMiddleware\n",
+    "\n",
+    "    app = DispatcherMiddleware(JPEGRequestHandler(app), {'/': app})\n",
+    "    run_simple('0.0.0.0', 8000, app, use_reloader=True)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

http-performance/app.py

@@ -0,0 +1,44 @@
+import os
+from urllib.parse import urlparse
+
+class JPEGRequestHandler:
+    def __init__(self, app):
+        self.app = app
+
+    def __call__(self, environ, start_response):
+        url = urlparse(environ['REQUEST_URI'])
+        path = url.path
+
+        if path.endswith('.jpg'):
+            file_path = path[1:]
+
+            if os.path.isfile(file_path):
+                with open(file_path, 'rb') as file:
+                    response_body = file.read()
+                    status = '200 OK'
+                    headers = [('Content-type', 'image/jpeg'),
+                               ('Content-Length', str(len(response_body)))]
+
+                    start_response(status, headers)
+                    return [response_body]
+            else:
+                status = '404 Not Found'
+                headers = [('Content-type', 'text/plain'),
+                           ('Content-Length', '0')]
+
+                start_response(status, headers)
+                return []
+        else:
+            status = '403 Forbidden'
+            headers = [('Content-type', 'text/plain'),
+                       ('Content-Length', '0')]
+
+            start_response(status, headers)
+            return []
+
+if __name__ == '__main__':
+    from werkzeug.serving import run_simple
+    from werkzeug.middleware.dispatcher import DispatcherMiddleware
+
+    app = DispatcherMiddleware(JPEGRequestHandler(app), {'/': app})
+    run_simple('0.0.0.0', 8000, app, use_reloader=True)

http-performance/httpserver.py

@@ -0,0 +1,31 @@
+import os
+from http.server import SimpleHTTPRequestHandler, HTTPServer
+from urllib.parse import urlparse
+
+class JPEGRequestHandler(SimpleHTTPRequestHandler):
+    def do_GET(self):
+        url = urlparse(self.path)
+        path = url.path
+        if path.endswith('.jpg'):
+            file_path = path[1:]
+
+            if os.path.isfile(file_path):
+                self.send_response(200)
+                self.send_header('Content-Type', 'image/jpeg')
+                self.end_headers()
+
+                with open(file_path, 'rb') as file:
+                    self.wfile.write(file.read())
+            else:
+                self.send_error(404, 'File not found')
+        else:
+            self.send_error(403, 'Only .jpg files are allowed')
+
+def run(server_class=HTTPServer, handler_class=JPEGRequestHandler, port=8000):
+    server_address = ('', port)
+    httpd = server_class(server_address, handler_class)
+    print(f'Serving JPEGs on port {port}...')
+    httpd.serve_forever()
+
+if __name__ == '__main__':
+    run()

http-performance/index.html

@@ -0,0 +1,146 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <title>D3.js Image Loading Time Graph</title>
+        <script src="https://d3js.org/d3.v5.min.js"></script>
+        <style>
+            /* Add some basic styling for the chart */
+            .bar {
+                fill: steelblue;
+            }
+    
+            .bar:hover {
+                fill: brown;
+            }
+    
+            .axis {
+                font: 10px sans-serif;
+            }
+    
+            .axis path,
+            .axis line {
+                fill: none;
+                stroke: #000;
+                shape-rendering: crispEdges;
+            }
+        </style>
+    </head>
+    <body>
+        <div id="chart"></div>
+        <div id="stats"></div>
+        <script>
+        // Generate the list of image URLs named kublai0.jpg to kublai118.jpg
+        const numberOfImages = 119;
+        const imagePrefix = 'http://localhost:8000/kublai';
+        const imageExtension = '.jpg';
+        const images = [];
+
+        for (let i = 0; i < numberOfImages; i++) {
+            const imageName = `${imagePrefix}${i}${imageExtension}`;
+            images.push(imageName);
+        }
+
+        // Disable caching using cache-buster technique
+        const cacheBuster = (url) => `${url}?${new Date().getTime()}`;
+
+        // Load images and measure their loading times
+        const loadImages = async () => {
+            const loadingTimes = [];
+
+            for (const url of images) {
+                const startTime = performance.now();
+                const img = new Image();
+                img.src = cacheBuster(url);
+                img.style.display = 'none';
+                document.body.appendChild(img);
+
+                await new Promise((resolve) => {
+                    img.onload = () => {
+                        const endTime = performance.now();
+                        loadingTimes.push(endTime - startTime);
+                        resolve();
+                    };
+                });
+            }
+
+            return loadingTimes;
+        };
+
+        // Create a D3.js bar chart to visualize loading times
+        const createChart = (data) => {
+            const margin = { top: 20, right: 20, bottom: 30, left: 40 },
+                width = 1280 - margin.left - margin.right,
+                height = 500 - margin.top - margin.bottom;
+
+            const x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
+            const y = d3.scaleLinear().rangeRound([height, 0]);
+
+            const svg = d3.select('#chart')
+                .append('svg')
+                .attr('width', width + margin.left + margin.right)
+                .attr('height', height + margin.top + margin.bottom)
+                .append('g')
+                .attr('transform', `translate(${margin.left},${margin.top})`);
+
+            x.domain(data.map((d, i) => i));
+            y.domain([0, d3.max(data)]);
+
+            svg.append('g')
+                .attr('class', 'axis axis--x')
+                .attr('transform', `translate(0,${height})`)
+                .call(d3.axisBottom(x));
+
+            svg.append('g')
+                .attr('class', 'axis axis--y')
+                .call(d3.axisLeft(y).ticks(10))
+                .append('text')
+                .attr('transform', 'rotate(-90)')
+                .attr('y', 6)
+                .attr('dy', '0.71em')
+                .attr('text-anchor', 'end')
+                .text('Loading Time (ms)');
+
+            svg.selectAll('.bar')
+                .data(data)
+                .enter().append('rect')
+                .attr('class', 'bar')
+                .attr('x', (d, i) => x(i))
+                .attr('y', (d) => y(d))
+                .attr('width', x.bandwidth())
+                .attr('height', (d) => height - y(d));
+
+            const stats = document.getElementById('stats');
+            let textContent = '';
+            let average = 0;
+            let sum = 0;
+            let min = parseFloat(data[0]);
+            let max = parseFloat(data[0]);
+            for(datum of data) {
+                loadingTime = parseFloat(datum)
+                sum = sum + loadingTime
+                if(loadingTime > max) {
+                    max = loadingTime;
+                }
+                if(loadingTime < min) {
+                    min = loadingTime;
+                }
+            }
+            average = sum / data.length
+            textContent = 
+                'Total: ' + sum + 
+                ' Average: ' + average + 
+                ' Min: ' + min + 
+                ' Max: ' + max;
+            stats.textContent = textContent;
+        };
+
+        // Load images and create the chart
+        (async () => {
+            const loadingTimes = await loadImages();
+            createChart(loadingTimes);
+        })();
+    </script>
+</body>
+</html>