fp2_11.py

@@ -1,4 +1,5 @@
 from google.cloud import pubsub_v1
+from google.api_core.exceptions import DeadlineExceeded
 from selenium import webdriver
 from selenium.webdriver.chrome.options import Options
 from selenium.webdriver.common.by import By
@@ -24,7 +25,6 @@ project_id = "fp-20230425"
 subscription_name = "fp-subscription-NFL-zipcodes"
 subscriber = pubsub_v1.SubscriberClient()
 subscription_path = subscriber.subscription_path(project_id, subscription_name)
-timeout = 5
 
 # Load the product page and extract the __NEXT_DATA__ data
 url = 'https://www.sayweee.com/en/product/Want-Want-Senbei-Rice-Crackers/17119'
@@ -71,12 +71,12 @@ def callback(message):
 
 
         try:
-            time.sleep(5)
+            time.sleep(1)
             # Clear any notice
             notice_close = driver.find_element(By.XPATH, "/html/body/div[3]/div/div[2]/div/div[2]/div/div/div[1]/div[2]/button")
             # print(f"notice_close = {notice_close}")
             notice_close.click()
-            time.sleep(2)
+            time.sleep(1)
         except:
             pass
         
@@ -84,7 +84,7 @@ def callback(message):
         zip_button = driver.find_element(By.XPATH, '/html/body/div[2]/div/header/div[3]/div/div/div[2]/div[1]')
         # print(f"{zip_button:}")
         zip_button.click()
-        time.sleep(2)
+        time.sleep(1)
 
         zip_input = driver.find_element(By.XPATH, '//div/label/following-sibling::input')
         # print(f"{zip_input:}")
@@ -127,6 +127,7 @@ def callback(message):
         # Print the price
         print(price)
 
+        subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
         # Append the zip code and price to the dataframe
         #append_to_df(zip_code, price)
         new_row = pd.DataFrame({'Zip Code': [zip_code], 'Price': [price]})
@@ -134,19 +135,29 @@ def callback(message):
         #print(f"Recorded the zip code and price: {zip_code},{price}")
         #print(f"Updated a dataframe: {df}")
 
-        subscriber.acknowledge(ack_ids = [message.ack_id], subscription = subscription_path)  # acknowledge receipt of message
     except Exception as e:
         print(f"Caught exception: {e}")
         print(f"Price not found for zip code {zip_code}")
         price = 'Price not found'
         return df
 
+import time
+
 # Start receiving messages (sync version)
 while True: 
     pull_request = pubsub_v1.types.PullRequest(max_messages = 1, subscription = subscription_path)
-    pull_response = subscriber.pull(request = pull_request)
+    messages = []
-    df = callback(pull_response.received_messages[0])
+    try:
-    print(f"Updated a dataframe: {df}")
+        pull_response = subscriber.pull(request = pull_request)
+        messages = pull_response.received_messages
+    except DeadlineExceeded:
+        pass
+
+    if messages:
+        df = callback(messages[0])
+        print(f"Updated a dataframe: {df}")
+    else:
+        print(f"no messages, trying again at {time.asctime()}")
 
 # Start receiving messages (async version)
 #flow_control = pubsub_v1.types.FlowControl(max_messages = 1)