SVIG/python-bootcamp
7
0
Fork 1
Code Issues Pull Requests 1 Releases Wiki Activity

This fixes issue #1 #2

Open
j wants to merge 123 commits from Branch_Karen into main
pull from: Branch_Karen
merge into: :main
Conversation 0 Commits 123 Files Changed 65
+49 -23
1 changed files with 49 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 722d116e75 - Show all commits
fp2_11.py
+47 -21
View File
@@ -9,6 +9,8 @@ import selenium.common.exceptions
import json import json
import time import time
import os import os
import re
import sys
from concurrent.futures import TimeoutError from concurrent.futures import TimeoutError
import pandas as pd import pandas as pd
from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support.ui import WebDriverWait
@@ -17,7 +19,7 @@ from selenium.webdriver.support import expected_conditions as EC
# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json" # os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "C:/Users/Karen/AppData/Roaming/gcloud/application_default_credentials.json"
# Create an empty dataframe to store the prices # Create an empty dataframe to store the prices
df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Price']) df = pd.DataFrame(columns=['Product ID', 'Zip Code', 'Displayed Price', 'ld+json Price', 'Minimum Quantity', 'Total Price'])
print(f"Created a dataframe: {df}") print(f"Created a dataframe: {df}")
# Configure the Selenium driver options # Configure the Selenium driver options
@@ -69,7 +71,7 @@ def close_modals():
# time.sleep(2) # time.sleep(2)
# Clear any modal dialog # Clear any modal dialog
modal_close_buttons = driver.find_elements(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]") modal_close_buttons = driver.find_elements(By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")
# print(f"modal_close = {modal_close}") print(f"modal_close = {modal_close}")
for button in modal_close_buttons: for button in modal_close_buttons:
try: try:
button.click() button.click()
@@ -85,7 +87,7 @@ def close_modals():
# time.sleep(1) # time.sleep(1)
# Clear any notice # Clear any notice
notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button") notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
# print(f"notice_close = {notice_close}") print(f"notice_close = {notice_close}")
notice_close.click() notice_close.click()
# time.sleep(1) # time.sleep(1)
except selenium.common.exceptions.NoSuchElementException as nsee: except selenium.common.exceptions.NoSuchElementException as nsee:
@@ -122,7 +124,7 @@ def callback(message):
WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, "//button[text() = 'Confirm']")) WebDriverWait(driver, 2).until(lambda d: d.find_element(By.XPATH, "//button[text() = 'Confirm']"))
check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']") check_button = driver.find_element(By.XPATH, "//button[text() = 'Confirm']")
# print(f"{check_button:}") print(f"{check_button:}")
check_button.click() check_button.click()
# time.sleep(2) # time.sleep(2)
break break
@@ -148,7 +150,7 @@ def callback(message):
modal_close = WebDriverWait(driver, 2).until( modal_close = WebDriverWait(driver, 2).until(
EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]")) EC.presence_of_element_located((By.XPATH, "//*[contains(concat(' ', normalize-space(@class), ' '), ' icon-icon-close-s ')]"))
) )
# print(f"modal_close = {modal_close}") print(f"modal_close = {modal_close}")
modal_close.click() modal_close.click()
# time.sleep(2) # time.sleep(2)
except: except:
@@ -162,7 +164,7 @@ def callback(message):
notice_close = WebDriverWait(driver, 5).until( notice_close = WebDriverWait(driver, 5).until(
EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")) EC.presence_of_element_located((By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button"))
) )
# print(f"notice_close = {notice_close}") print(f"notice_close = {notice_close}")
notice_close.click() notice_close.click()
# time.sleep(1) # time.sleep(1)
except: except:
@@ -213,34 +215,58 @@ def callback(message):
except: except:
pass pass
# Check if there's a minimum order quantity
min_quantity: float = 0
total_price: float = 0
try:
minimum_quantity_str = driver.find_element(By.XPATH, "//div[contains(@class, 'minQuantity')]").get_attribute('innerHTML')
if minimum_quantity_str:
regexp="Minimum purchase: (\d+) \(\$(\d+\.\d{2}?)"
match = re.search(regexp, minimum_quantity_str)
min_quantity, total_price = match.groups()
except selenium.common.exceptions.NoSuchElementException as nsee:
print("No minimum purchase string found")
except Exception as e:
print(f"Exception when finding minimum purchase string: {e}")
# Get the price as displayed on the web page
displayed_price = driver.find_element(By.XPATH, "//div[contains(@class, 'price_price')]").get_attribute('innerText').replace('$', '')
print(f"displayed_price = {displayed_price}")
# Extract the price of rice crackers for this zip code # Extract the price of rice crackers for this zip code
''' '''
price_element = driver.find_element(By.XPATH, '/html/body/div[2]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div') price_element = driver.find_element(By.XPATH, '/html/body/div[2]/div/div[1]/div[1]/div/div/div[2]/div[2]/div[1]/div[4]/div[1]/div')
price = price_element.text.strip()[1:] price = price_element.text.strip()[1:]
print(price) print(price)
''' '''
next_data_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
next_data_text = next_data_element.get_attribute('innerHTML') # Get the price from the ld+json element
next_data_start_index = next_data_text.index('{') ld_json_element = driver.find_element(By.XPATH, "//script[@type='application/ld+json']")
next_data_end_index = next_data_text.rindex('}') + 1 ld_json_text = ld_json_element.get_attribute('innerHTML')
next_data_json = next_data_text[next_data_start_index:next_data_end_index] ld_json_start_index = ld_json_text.index('{')
next_data = json.loads(next_data_json) ld_json_end_index = ld_json_text.rindex('}') + 1
price = next_data['offers']['price'] ld_json_json = ld_json_text[ld_json_start_index:ld_json_end_index]
ld_json = json.loads(ld_json_json)
ld_json_price = ld_json['offers']['price']
# Print the price # Print the price
print(price) print(f"ld_json_price = {ld_json_price}")
subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path) # acknowledge receipt of message
# Append the zip code and price to the dataframe # Append the zip code and price to the dataframe
#append_to_df(zip_code, price) #append_to_df(zip_code, price)
new_row = pd.DataFrame({'Product ID': [product_id], 'Zip Code': [zip_code], 'Price': [price]}) new_row = pd.DataFrame({'Product ID': [product_id],
'Zip Code': [zip_code],
'Displayed Price': [displayed_price],
'ld+json Price': [ld_json_price],
'Minimum Quantity': [min_quantity],
'Total Price': [total_price]})
return pd.concat([df, new_row], ignore_index=True) return pd.concat([df, new_row], ignore_index=True)
#print(f"Recorded the zip code and price: {zip_code},{price}") #print(f"Recorded the zip code and price: {zip_code},{price}")
#print(f"Updated a dataframe: {df}") #print(f"Updated a dataframe: {df}")
except Exception as e: except Exception as e:
print(f"Caught exception: {e}") print(f"Caught exception {type(e)}: {e}")
print(f"Price not found for zip code {zip_code}") print(f"Price not found for zip code {zip_code}")
price = 'Price not found' price = 'Price not found'
return df return df
@@ -258,13 +284,13 @@ while True:
if messages: if messages:
df = callback(messages[0]) df = callback(messages[0])
print(f"Updated a dataframe: {df}") print(f"Updated a dataframe: {df}")
else:
print(f"no messages, trying again at {time.asctime()}")
break
df.to_hdf('product1.h5', key='df',mode='w') df.to_hdf('product1.h5', key='df',mode='w')
else:
print(f"No messages in subscription {subscription_path} at {time.asctime()}, trying again...")
continue
# Open the HDF5 file in a separate window # Open the HDF5 file in a separate window
if sys.platform.startswith("win32"):
os.startfile('product1.h5') os.startfile('product1.h5')
# Start receiving messages (async version) # Start receiving messages (async version)