Checked the port and set the chrome_options

testing issue linking in body #3

.gitignore

@@ -0,0 +1,3 @@
+bin/**
+lib/**
+share/**

basic_checks.yml

@@ -1,39 +1,203 @@
+- name: Prepare wsl distribution to run snapd
+  hosts: commander
+  tasks:
+  # TODO the conditional should be given only once in a block
+    - name: Install Docker
+      become: true
+      apt:
+        name: docker.io
+        state: present
+      when: "ansible_os_family == 'Debian'"
+
+    - name: Start Docker service
+      become: true
+      service:
+        name: docker
+        state: started
+      when: "ansible_os_family == 'Debian'"
+
+- name: Start Docker for Desktop
+  hosts: commander
+  gather_facts: false
+  tasks:
+    - name: Start Docker for Desktop on MacOS
+      command: open /Applications/Docker.app
+      when: ansible_os_family == "Darwin"
+
+    - name: Install requests module
+      pip:
+        name: requests
+        state: present
+
+    - name: Start the pricer container normally
+      docker_container:
+        name: loving_benz
+        image: dev
+        state: started
+        restart_policy: always
+        ports:
+          - "2222:22"
+      register: container
+
+    - name: Debug container Config.Hostname
+      debug:
+        var: container.Config.Hostname
+
 - name: Basic system checks
-  hosts: all
+  hosts: docker_container
   tasks:
     - name: Check to make sure the package manager is apt
       fail: 
         msg: "Package manager is not apt"
       when: "ansible_pkg_mgr != 'apt'"
       
-- name: Check out the pricer repository
+    - name: Install git
-  hosts: all
+      apt:
-  tasks:
+        name: git
-    - name: Checkout pricer repository
+        state: present
+
+    - name: Create /opt/pricer directory
+      file:
+        path: /opt/pricer
+        state: directory
+        mode: '0755'
+
+    - name: Check out the pricer repository
       git:
         repo: https://gogs.gtsplab.com:5432/SVIG/pricer.git
         dest: /opt/pricer
         version: main
-      become: yes
-      become_user: root
 
-- name: Install Python requirements
+    - name: Install Python requirements setuptools + distutils + pkg-resources + pip
-  hosts: all
+      apt:
-  tasks:
+        name:
-    - name: Install Python requirements
+          - python3-setuptools
+          - python3-distutils
+          - python3-pkg-resources
+          - python3-pip
+        state: present
+
+    - name: Install the pricer Python package dependencies
       ansible.builtin.pip:
         requirements: /opt/pricer/requirements.txt
-      become: yes
 
-    - name: Install missing packages with apt
+    - name: Install Selenium
-      apt:
+      ansible.builtin.pip:
-        name: "{{ item }}"
+        name: selenium
         state: present
-      become: yes
-      with_lines:
-        cmd: "pip show {{ item }}"
-        register: pip_packages
-        changed_when: pip_packages.rc != 0
-      loop_control:
-        loop_var: item
 
+# Install snapd here
+
+# Start the container in privileged mode with:
+# docker run --privileged --cgroupns=host -d -v /sys/fs/cgroup:/sys/fs/cgroup:rw dev /lib/systemd/systemd
+# systemctl start snapd
+# systemctl start udev
+# snap install chromium
+# You can use ansible.community.docker to run the container, then do all the above stuff
+# docker_container_exec might be useful
+#
+# Translate the docker run command into the docker_container ansible module with these args:
+# -v | volumes:
+# --cgroupns | cgroupns_mode:
+# -d | detach: true
+#
+# 1) Start the container with systemd, so we can use snap to install packages
+# 2) Install chromium using snap
+# 3) Stop the container
+# 4) Re-start the container without systemd, using the default RUN command "/usr/sbin/sshd -D"
+# 5) Run the scraper
+
+
+- name: Start pricer container with systemd
+  hosts: commander
+  tasks:
+    - name: Run pricer container
+      docker_container:
+      # TODO: make sure to use the container id from above so you don't start another container
+        name: loving_benz
+        image: dev
+        state: started
+        restart_policy: always
+        volumes:
+          - /sys/fs/cgroup:/sys/fs/cgroup:ro
+        privileged: yes
+        ports:
+          - "2222:22"
+          - "9515:9515"
+        command: /lib/systemd/systemd
+
+- name: Continue installing chromium using snap
+  hosts: docker_container
+  become: true
+  tasks:
+    - name: Install requests module
+      pip:
+        name: requests
+        state: present
+    - name: Install snapd package
+      apt:
+        name: snapd
+        state: present
+    - name: Start udev
+      systemd:
+        name: udev
+        state: started
+    - name: Start snapd
+      systemd:
+        name: snapd.service
+        # Verify the exact name of the systemd service for snapd using `systemctl list-unit-files | grep snapd` on the host
+        state: started
+    - name: Install chromium
+      snap:
+        name: chromium
+        state: present
+
+- name: Install Chromium and Chromedriver
+  hosts: docker_container
+  become: true
+  tasks:
+    - name: Install chromium-chromedriver package
+      apt:
+        name: chromium-chromedriver
+        state: present
+    - name: Install chromium-browser package
+      apt:
+        name: chromium-browser
+        state: present
+
+- name: Verify the chromium browser is running at the specified port
+  hosts: docker_container
+  become: true
+  tasks:
+    - name: install the net-tools
+      apt:
+        name: net-tools
+        state: present
+    - name: Get Chromium browser port
+      command: netstat -tlnp | grep chromium | awk '{split($4,a,":"); print a[length(a)]}'
+      register: chromium_port
+    - name: Display Chromium browser port
+      debug:
+        var: chromium_port.stdout
+
+- name: Run latest version of scraper
+  hosts: docker_container
+  become: true
+  tasks:
+    - name: Set up Chrome options
+      set_fact:
+        chrome_options:
+          - "--headless"
+          - "--no-sandbox"
+          - "--disable-dev-shm-usage"
+    - name: Debug chrome_options variable
+      debug:
+        var: chrome_options
+
+    - name: Run scraper
+      ansible.builtin.script:
+        script: /opt/pricer/fp2_13.py
+        executable: /usr/bin/python3
+        environment:
+          PATH: "/usr/bin/chromedriver/:{{ ansible_env.PATH }}"
+          CHROME_OPTIONS: "{{ chrome_options | join(' ') }}"

basic_checks_Sloetjes.yml

@@ -1,38 +0,0 @@
-- name: Basic system checks
-  hosts: all
-  tasks:
-    - name: Check to make sure the package manager is apt
-      fail: 
-        msg: "Package manager is not apt"
-      when: "ansible_pkg_mgr != 'apt'"
-      
-- name: Clone pricer repository
-  hosts: target_hosts
-  become: yes
-  tasks:
-    - name: Check out pricer repository
-      git:
-        repo: https://github.com/your-username/pricer.git
-        dest: /path/to/pricer
-
-- name: Install Python requirements
-  hosts: target_hosts
-  become: yes
-  tasks:
-    - name: Install Python requirements
-      pip:
-        requirements: /Users/sloetjeswong/pricer/requirements.txt
-
-- name: Install Chromium and Selenium
-  hosts: target_hosts
-  become: yes
-  tasks:
-    - name: Install Chromium
-      apt:
-        name: chromium-browser
-        state: present
-
-    - name: Install Selenium
-      pip:
-        name: selenium
-        state: present

docker/dev/Dockerfile

@@ -8,5 +8,8 @@ RUN sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/s
 RUN sed -i 's/#KbdInteractiveAuthentication yes/KbdInteractiveAuthentication no/' /etc/ssh/sshd_config
 ADD authorized_keys /root/.ssh/authorized_keys
 RUN chmod 600 /root/.ssh/authorized_keys
+# Ansible
 EXPOSE 22
+# Chromedriver
+EXPOSE 9515
 CMD ["/usr/sbin/sshd", "-D"]

docker/dev/authorized_keys

@@ -1,3 +1,3 @@
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpXelnFwuVHcdbHjrvcChhxiBzXmAOPb43Yhb5W+ix93YS/TaWuDSLUxf1l1XAy45ESgMzI0GxMXakVoT5Ns0yrKgpk4ZJRlVGV03SJEwHL80JaQzjXXuo5fFE/CpaNJJHmtzobE1uzs8yz0qkd12iY4yCZvzelazarJNhy/jSFrXJZuVh2jvxkr4SpHQg9VBOJMaDH2cF9s6a45wtjXJz2sUsaLyRIhWGA78xpS6u9gxWX1IZXCcl6TMEktSv503QKQI8YitWOuGr4CkhBlrg9aMhQu6bxDzep0UQVfn2sIwj1m8+bO+mzYI6hHo2odeQtTVcoby0/duYNiWugHO6nkUvvlmYUHonUFs+v68ZO8VJvvQwTMnfnPouY/46WNimQBVTdxU8rSgeHWkvZ1jYE5P2++Ba8zSAf4F8Gk8EEwq+OcFv2IP1Diyn0K21qJG3IllYS863uAPvQBHnKh3rDF6r5rvGN6tJMK7l0R6DvCF1s0g/Thq3GWBp7FlRXL8= 2021-1
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCnA5mrWbhY/8F4T2u4pLsHmfRVdRkQTNTusTFT5chcHddEx+iQcDUjpP2Is5LIxeGi08X+URnQ1raVrcD73US+uueBUvDz/pcfDmS1cH7j98ySDJrdyQnSGNCSIIfV/fhEW+wGn1qTv1WsT6oUt5gxPsTQUP43xuXNXnuk1KLhYP2UjkieQ5TothV4Usml1VtHaTDc8LhoF4h6bJnWuDpaOvdXvVOia/wyfUH0A0rPfinCzJrggSKMNejo9KZMLNyzvEDccg/hv0wKJlgdEFILrk7TpiF9Wv6SRsnLD9qLh2TAzm+FbrysZZEDIe/8v0p6yNP4MoFkPvT4UQAm/+SxgSyvn0eR8oGYQ5RMoTMturzE4CtvHrWd1VRCa/w0xUBwAuXaOEjy8FE3btNrBGOiz3Q7iD1sFxA7FRIaxqmc9U7DWitrcrHv8+Mr7GlzY7ZCL7oW3LdK5UPGJeOdhRMVIPdPh9yaKhDQCHC1ivCFTN3cKuMP9GuaN7Hww7RDQWpbs+uLuuQa7mykkmjmDoM25Bup9oVzj7PpsMMVhEa8Gd+5G1HNpucsxgvwT+bIEpoyKq377pNNecwjL6JzooicVPmDcRvrgSTPpfjA6y4YcdSGvGeTyMBgoU/jPwoMqMqmbYu5qt3qduiKHUPPstPXJSaYNB7AD2sXBkBNJrVZNw== karen_zeng@Karen-hhh
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCovKrQNfHq/rGndAkzJVmGoo5za19ayPIAsJKb5q4rl3J4aRE6VweRlJbOY26cARpTgoO9yEPqYpi5FhRf9YDynZ0ZCUfwW8l/Bc1SEk2W4+cD+aQb9rWlQJBMW85B+/Wy4V8npVwOU0LiE7mZpzLUBErdtFl0r0bxxCfeWwfHqPpt5a3pJFXts7PryEl6V1RE8rYwiUr7exIeoqwcFK3yaLRiy5EqDBcx9pxQxdVMIB2H7U3j/ESqSpOu/vvRI7/1v39o8Ty3JPGTxCvzDljwmWvoIaeot/ThoTdc2O/1psrOic6mUa982VWevA8icOXnzcVaVJKVokEP6bW3WUZwc4NoqUqUjHPJLV3qPO5YzLZWyIW7Z+IQ7Q3907jABf/4ADM0WuicIBn+zX+3DOwXLaf+wPsCdoGRqu+9J2g+BCr23oxsMy2NsXmHmroCG+td1M1S/TGR174Zee8AjqmjSUEyKfvOV+EG1jZiNS7GwwKr+2brAcoCt9xiH2m6af0= sloetjeswong@Sloetjess-Air
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCkEPDdXig0Sk9GEr/PjThYaMN5QF5V5T5jTwZ9UFPn8cNiw2VkXgjCPBmPizenmPfkVCW6xVR1GqXZAJvft9+eJzcJF2cBqkUjZlhFiUXR0ZiR0rA5GPMDRvyZnrqVjALs1eAj5WqlqubKr5aPhVcZ46q38wbR4z6DONR8JLoHGwNjEQrDUWvnHGNeyrWxZsJB5NgByfoINMFCDf3i/XguTs0yo/6IXkuEqG2QBhio/93wA/+5PZNG//WpctGrbUU5fArfXYQ1EI+TSut0UebD/vIbe0AEn8kIRp4u1APU3rhzdkxslKogKow6PLVidRL3bnBfygr5xJTji+3JwI1vWE/u6xGLrBwPUJxAbBnFbORl+ioUfq6hWb4072VP7mOafUquavPWesa0nHJxIpdq5bI8s5y6QeFIgO2d+GRK6CwRFuosnOQJoPIYDCc6+TzwcZWSJkdc8XsTgUoh7GPOHsLDQlZ4R0o2dgKu37X6/aHfMfGKTfHjv5sX3c7UDziBifqWNvDExnF2e6h8eYRzXg2eR6aw7LBuKfe/ZdJovQhdrE4YxrFfOwiCWdy8Eoq3ulY7Qkk6yShyq3E9zcOsVt4WZ4J7OPrnDc4E8VOTvdcftORUuYK3c5hB3VPqCInPqDW8YrEomSanBUeZXQVdsxFxgEEelEDWKDKoHonHkQ== sloetjeswong@Sloetjess-Air

docker/dev/run.sh

@@ -1,3 +1,6 @@
 #!/bin/bash
 # Run the docker container
-docker run -p 2222:22 -d dev
+# This will get systemd up enough to run snapd
+# See: https://docs.docker.com/desktop/release-notes/#bug-fixes-and-minor-changes
+# docker run --privileged --cgroupns=host -d -v /sys/fs/cgroup:/sys/fs/cgroup:rw dev /lib/systemd/systemd
+docker run -p 2222:22 -p 9515:9515 -d dev

fp2_10.py

@@ -42,7 +42,7 @@ subscription = subscriber.create_subscription(
 # Publish a message to the topic
 # Read the data from the csv file
 # file_path = "C:/Users/Karen/Desktop/zipcodes.csv"
-file_path = "zipcodes.csv"
+file_path = "karen-zipcodes.csv"
 
 with open(file_path, "r") as csv_file:
     reader = csv.reader(csv_file)

hosts

@@ -1,5 +1,8 @@
-oci:
+dev:
   hosts:
-    localhost:
+    docker_container:
+      ansible_host: localhost
       ansible_user: root
       ansible_port: 2222
+    commander:
+      ansible_connection: local

requirements.txt

@@ -6,3 +6,8 @@ h5py
 selenium
 sqlalchemy
 flask
+ntplib
+arrow
+h5py
+ansible
+ansible-lint