Feature/decouple aws from playbooks (#104)

* version 4.3.0 initial commit

* Updates to accept aws region and profile args

* inventory file generator update

infra/configure/inventory/group_vars/all.yaml

@@ -6,10 +6,10 @@ service_user: "{{ ansible_facts['env']['LOGNAME'] }}"
 
 veriscope_nm_services: [ nethermind ]
 veriscope_web_prereq_services: [ nginx, redis-server ]
-veriscope_web_services: [ ta, ta-queue, ta-wss, ta-schedule, horizon, ta-node-1 ]
+veriscope_web_services: [ ta, ta-wss, ta-schedule, horizon, ta-node-1 ]
 
 # Location to install veriscope to. Default is /opt/veriscope which is the ONLY possible
-# place to install for now as it is hardcoded everywhere in php and nodejs code. 
+# place to install for now as it is hardcoded everywhere in php and nodejs code.
 # Subsequent paths are derived from install_root
 install_root: /opt/veriscope
 api_root: "{{ install_root }}/veriscope_ta_node"
@@ -18,19 +18,21 @@ dashboard_root: "{{ install_root }}/veriscope_ta_dashboard"
 dashboard_config_path: "{{ dashboard_root }}/.env"
 addressproofs_root: "{{ install_root }}/veriscope_addressproof"
 
-aws_region: us-east-1
-aws_profile: default
+# ssh_priv_key_secret_name: /{{ owner | lower }}/{{ env | lower }}/instances/{{ inventory_hostname }}
 
-ta_dashboard_db_cluster_secret_name: "/{{ env | lower }}/common/ta-db-cluster"
-# The details of the PostgreSQL cluster where the TrustAnchor database is to be created
-ta_dashboard_db_cluster: "{{ lookup('amazon.aws.aws_secret', ta_dashboard_db_cluster_secret_name, bypath=true,
-                          region=aws_region, aws_profile=aws_profile)[ta_dashboard_db_cluster_secret_name] }}"
-pgsql_cltr_host: "{{ ta_dashboard_db_cluster['cluster_endpoint'] }}"
-pgsql_cltr_port: "{{ ta_dashboard_db_cluster['cluster_port'] }}"
-ta_db_name: "trustanchor-{{ inventory_hostname_short }}"
-ta_db_user: "trustanchor-{{ inventory_hostname_short }}"
-ssh_priv_key_secret_name: /{{ owner | lower }}/{{ env | lower }}/instances/{{ inventory_hostname }}
-ta_db_user_pwd_secret_name: /{{ owner | lower }}/{{ env | lower }}/instances/{{ ta_db_user }}
+
+# aws_region: us-east-1
+# aws_profile: default
+
+# ta_dashboard_db_cluster_secret_name: "/{{ env | lower }}/common/ta-db-cluster"
+# # The details of the PostgreSQL cluster where the TrustAnchor database is to be created
+# ta_dashboard_db_cluster: "{{ lookup('amazon.aws.aws_secret', ta_dashboard_db_cluster_secret_name, bypath=true,
+#                           region=aws_region, aws_profile=aws_profile)[ta_dashboard_db_cluster_secret_name] }}"
+# pgsql_cltr_host: "{{ ta_dashboard_db_cluster['cluster_endpoint'] }}"
+# pgsql_cltr_port: "{{ ta_dashboard_db_cluster['cluster_port'] }}"
+# ta_db_name: "trustanchor-{{ inventory_hostname_short }}"
+# ta_db_user: "trustanchor-{{ inventory_hostname_short }}"
+# ta_db_user_pwd_secret_name: /{{ owner | lower }}/{{ env | lower }}/instances/{{ ta_db_user }}
 
 ansible_host_key_checking: false
 ansible_ssh_pipelining: true
@@ -42,10 +44,7 @@ ansible_ssh_private_key_file: ~/.ssh/{{ inventory_hostname }}.pem
 
 nm_root: /opt/nm
 nm_config_dest_path: "{{ nm_root }}/config.cfg"
-# target_nm_version: 1.15.0
-# nethermind_tarball: "https://github.com/NethermindEth/nethermind/releases/download/{{ target_nm_version }}/nethermind-{{ target_nm_version }}.tar.gz"
-nethermind_tarball: https://github.com/NethermindEth/nethermind/releases/download/1.15.0/nethermind-linux-amd64-1.15.0-2b70876-20221228.zip
-# nethermind_tarball: https://github.com/NethermindEth/nethermind/releases/download/1.12.4/nethermind-linux-amd64-1.12.4-1c8b669-20220113.zip
+target_nethermind_version: 1.15.0
 nethermind_rpc: http://localhost:8545
 network_facts:
   veriscope_testnet:

infra/configure/inventory/sample-veriscope-nodes.yaml

@@ -30,6 +30,13 @@ all:
           private_ip: 10.10.10.13
           ssh_priv_key_secret_name: /owner/env/instances/bar-002-nm.my-custom-domain.com
   vars:
+    # Mandatory. The source where TA DB secrets like TA DB user password and cluster info are stored.
+    # Valid values are aws_secretsmanager, env_vars, azure_keyvault, gcp_secretsmanager, hashicorp_vault and ansible_vault.
+    ta_db_secrets:
+      source: aws_secretsmanager
+      aws_region: us-east-1
+      aws_profile: default
+
     # Whether to print debug messages to the screen while running the playbooks. NOTE: It may print secret information too. So, please use with caution.
     debug: false
 
@@ -52,16 +59,3 @@ all:
       email: [email protected]
       # optional password
       password: mysupersecretpassword*!1
-
-    # Use this list to specify which apps to perform update (re-install) on beyond the copying 
-    # the new version of the code during a node update.
-    apps_to_update:
-      - api
-      - dashboard
-
-    # Use this dict to specify the keys and values for changes to TA API config changes
-    ta_api_config_changes:
-      HTTP: http://{{ nm_host }}:8545
-      WS: ws://{{ nm_host }}:8545
-      # HTTP: http://tx.veriscope.network:9400/
-      # WS: ws://tx.veriscope.network:9400/

infra/configure/playbooks/install-nethermind.yaml

@@ -7,11 +7,9 @@
     ethstats_host: "{{ network_facts[veriscope_target]['ethstats_host'] }}"
     ethstats_get_enodes: "{{ network_facts[veriscope_target]['ethstats_get_enodes'] }}"
     ethstats_enabled: true
-
-    nm_tarball_split: "{{ nethermind_tarball | split('/') }}"
-    target_nm_version: "v{{ nm_tarball_split[7] }}"
-    nm_config_template_path: "templates/nm-config/{{ target_nm_version }}/config.cfg.j2"
-    nm_db_backup_archive_url: "{{ network_facts[veriscope_target]['nm_db_backup_bucket_url'] }}/{{ target_nm_version }}/{{ veriscope_target }}.tar.gz"
+    nm_config_template_path: "templates/nm-config/v{{ target_nethermind_version }}/config.cfg.j2"
+    nm_db_backup_archive_s3_key: "v{{ target_nethermind_version }}/{{ veriscope_target }}.tar.gz"
+    nm_db_backup_archive_url: "{{ network_facts[veriscope_target]['nm_db_backup_bucket_url'] }}/v{{ target_nethermind_version }}/{{ veriscope_target }}.tar.gz"
   tasks:
 
     - name: Check inputs - Install nethermind
@@ -20,7 +18,7 @@
           - veriscope_service_hostname is defined and veriscope_service_hostname is truthy
           - veriscope_target is defined and veriscope_target is truthy
           - service_user is defined and service_user is truthy
-          - target_nm_version is defined and target_nm_version is truthy
+          - target_nethermind_version is defined and target_nethermind_version is truthy
         success_msg: Required inputs provided. Proceeding to install nethermind.
         fail_msg: Required inputs not provided. Please provide required values and try again. Please note that values cannot be empty strings.
 
@@ -35,6 +33,7 @@
           - libsnappy-dev
           - libc6-dev
           - libc6
+          - node-ws
           - vim
           - unzip
           - jq
@@ -45,38 +44,41 @@
         state: present
         update_cache: true
 
-    - name: Ensure python requirements are installed for AWS services
+    - name: Ensure python requirements are installed for AWS services - Install Nethermind
       ansible.builtin.import_tasks: ../tasks/ensure-boto3.yaml
 
-    - name: Get deployed Nethermind version
+    - name: Get deployed Nethermind version - Install Nethermind
       ansible.builtin.import_tasks: ../tasks/get-deployed-nm-version.yaml
       ignore_errors: true
 
-    - name: Check if already installed
+    - name: Check if already installed - Install Nethermind
       ansible.builtin.assert:
         that: deployed_nm_version is not defined and deployed_nm_version is not truthy
         success_msg: "Nethermind is not installed on this host. Continuing to install and configure Nethermind..."
         fail_msg: "Nethermind {{ deployed_nm_version }} is already installed on this host. Exiting..."
       ignore_errors: true
 
-    - name: Check if installed version is different
+    - name: Check if installed version is different - Install Nethermind
       ansible.builtin.assert:
-        that: deployed_nm_version != target_nm_version
-        success_msg: "Nethermind {{ deployed_nm_version }} is installed on this host. But requested version was {{ target_nm_version }}. Please run 'upgrade-nethermind.yaml' playbook to upgrade Nethermind."
-        fail_msg: "Nethermind {{ deployed_nm_version }} is already installed on this host which is the same as requested version ({{ target_nm_version }}). Exiting..."
+        that: deployed_nm_version != target_nethermind_version
+        success_msg: "Nethermind {{ deployed_nm_version }} is installed on this host. But requested version was v{{ target_nethermind_version }}. Please run 'upgrade-nethermind.yaml' playbook to upgrade Nethermind."
+        fail_msg: "Nethermind {{ deployed_nm_version }} is already installed on this host which is the same as requested version ({{ target_nethermind_version }}). Exiting..."
       ignore_errors: true
       when: deployed_nm_version is defined and deployed_nm_version is truthy
 
-    - name: Exit if Nethermind is already installed
+    - name: Exit if Nethermind is already installed - Install Nethermind
       ansible.builtin.meta: end_host
       when: deployed_nm_version is defined and deployed_nm_version is truthy
 
-    - name: Print Nethermind installation details
+    - name: Get nethermind binaries archive url - Install Nethermind
+      ansible.builtin.import_tasks: ../tasks/get-nethermind-binaries-archive-url.yaml
+
+    - name: Print Nethermind installation details - Install Nethermind
       ansible.builtin.debug:
-        msg: "Installing Nethermind {{ target_nm_version }} from {{ nethermind_tarball }} at {{ nm_root }} with config from {{ nm_config_template_path }}"
+        msg: "Installing Nethermind v{{ target_nethermind_version }} from {{ nethermind_download_url }} at {{ nm_root }} with config from {{ nm_config_template_path }}"
 
-    - name: Install Nethermind DB server
+    - name: Install Nethermind DB server - Install Nethermind
       ansible.builtin.import_tasks: ../tasks/install-nm-db-server.yaml
 
-    - name: Download and extract Nethermind db from archive
+    - name: Download and extract Nethermind db from archive - Install Nethermind
       ansible.builtin.import_tasks: ../tasks/unarchive-nm-db-from-s3.yaml

infra/configure/playbooks/install-ta-dashboard-webapp.yaml

@@ -6,17 +6,24 @@
     app_url: "https://{{ inventory_hostname }}"
     shyft_onboarding_url: "https://{{ inventory_hostname }}"
     webhook_client_secret: "{{ lookup('password', 'credentials/webhook-client-secret/' + inventory_hostname + ' length=20 chars=ascii_letters,digits', seed=inventory_hostname) }}"
-    ta_db_userpwd: "{{ lookup('amazon.aws.aws_secret', ta_db_user_pwd_secret_name, bypath=true)[ta_db_user_pwd_secret_name] }}"
+
   tasks:
+    - name: Fetch TA DB Cluster secrets
+      ansible.builtin.include_role:
+        name: fetch_ta_db_secrets
+      vars:
+        secret_manager_type: "{{ ta_db_secrets }}"
+      when: operation == "install"
+
     - name: Print TA DB Cluster var - TA Dashboard
       ansible.builtin.debug:
         var: ta_dashboard_db_cluster
-      when: debug is true
+      when: debug is true and operation == "install"
 
     - name: Print TA DB cluster info - TA Dashboard
       ansible.builtin.debug:
         msg: "{{ pgsql_cltr_host }}:{{ pgsql_cltr_port }} ; {{ ta_db_name }} ; {{ ta_db_user }} : {{ ta_db_userpwd }}"
-      when: debug is true
+      when: debug is true and operation == "install"
 
     - name: Check inputs for install and update - TA Dashboard app
       ansible.builtin.assert:
@@ -123,7 +130,7 @@
     # su $SERVICE_USER -c "composer install"
     # su $SERVICE_USER -c "php artisan migrate"
     - name: Install PHP deps of TA Dashboard app (composer install and php artisan migrate)
-      shell: | 
+      shell: |
         composer install
         php artisan migrate
       args:
@@ -139,7 +146,7 @@
     # su $SERVICE_USER -c "php artisan passportenv:link"
     # # ONLY ONCE. SHOULD not run on update
     - name: Deploy Trust Anchor dashboard app
-      shell: | 
+      shell: |
         php artisan db:seed
         php artisan key:generate
         php artisan passport:install
@@ -181,7 +188,7 @@
     # echo "Restarting PHP-based services..."
     # systemctl enable ta-schedule
     # systemctl enable ta-wss
-    # systemctl enable ta 
+    # systemctl enable ta
     # systemctl restart ta-schedule
     # systemctl restart ta-wss
     # systemctl restart ta
@@ -205,7 +212,7 @@
     # su $SERVICE_USER -c "php artisan migrate"
     # popd >/dev/null
     - name: Publish horizon - Install TA Dashboard app
-      shell: | 
+      shell: |
         composer update
         php artisan horizon:publish
         php artisan migrate
@@ -219,18 +226,13 @@
       #   cp scripts/horizon.service /etc/systemd/system/
       #   sed -i "s/User=.*/User=$SERVICE_USER/g" /etc/systemd/system/horizon.service
       # fi
-      # if ! test -s "/etc/systemd/system/ta-queue.service"; then
-      #   cp scripts/ta-queue.service /etc/systemd/system/
-      #   sed -i "s/User=.*/User=$SERVICE_USER/g" /etc/systemd/system/ta-queue.service
-      # fi
-    - name: Create ta-queue and horizon services configs
+    - name: Create horizon services configs
       ansible.builtin.template:
         src: templates/{{ item }}.service.j2
         dest: /etc/systemd/system/{{ item }}.service
       become: true
       with_items:
         - horizon
-        - ta-queue
       when: operation == 'install'
 
       # systemctl daemon-reload
@@ -241,16 +243,13 @@
       when: operation == 'install'
 
     # systemctl enable horizon
-    # systemctl enable ta-queue
     # systemctl restart horizon
-    # systemctl restart ta-queue
-    - name: Enable and restart ta-queue and horizon services
+    - name: Enable and restart horizon services
       ansible.builtin.service:
         name: "{{ item }}"
         enabled: true
         state: restarted
       become: true
       with_items:
         - horizon
-        - ta-queue
       when: operation == 'install'

infra/configure/playbooks/nm-maintenance/restore-nethermind-db.yaml

@@ -36,5 +36,5 @@
 
     - name: Download and extract Nethermind db from archive
       vars:
-        target_nm_version: "{{ deployed_nm_version }}"
+        target_nethermind_version: "{{ deployed_nm_version }}"
       ansible.builtin.import_tasks: ../../tasks/unarchive-nm-db-from-s3.yaml

infra/configure/playbooks/roles/fetch_ta_db_secrets/.travis.yml

@@ -0,0 +1,29 @@
+---
+language: python
+python: "3.10"
+
+# Use the new container infrastructure
+sudo: false
+
+# Install ansible
+addons:
+  apt:
+    packages:
+    - python-pip
+
+install:
+  # Install ansible
+  - pip install ansible
+
+  # Check ansible version
+  - ansible --version
+
+  # Create ansible.cfg with correct roles_path
+  - printf '[defaults]\nroles_path=../' >ansible.cfg
+
+script:
+  # Basic role syntax check
+  - ansible-playbook tests/test.yml -i tests/inventory --syntax-check
+
+notifications:
+  webhooks: https://galaxy.ansible.com/api/v1/notifications/

infra/configure/playbooks/roles/fetch_ta_db_secrets/README.md

@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).

infra/configure/playbooks/roles/fetch_ta_db_secrets/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+# defaults file for fetch_ta_db_secrets
+ta_dashboard_db_cluster_secret_name: "/{{ env | lower }}/common/ta-db-cluster"
+ta_db_user: "trustanchor-{{ inventory_hostname_short }}"
+ta_db_user_pwd_secret_name: /{{ owner | lower }}/{{ env | lower }}/instances/{{ ta_db_user }}

infra/configure/playbooks/roles/fetch_ta_db_secrets/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for fetch_ta_db_secrets