feat: upload built artifacts to the target machine (#170)

.github/workflows/infrstructure.yml

@@ -132,8 +132,7 @@ jobs:
 
       - name: Configure Control Machine
         run: |
-          sudo useradd --create-home ubuntu && sudo usermod --append --groups sudo ubuntu
-
+          ANSIBLE_USER=$([ "${IS_MANUAL_DEPLOYMENT}" == "true" ] && echo "ubuntu" || echo "${USER}")
           ANSIBLE_CONNECTION="ansible_connection=local"
           if [ "${IS_MANUAL_DEPLOYMENT}" == "true" ]; then
             ANSIBLE_CONNECTION=""
@@ -153,9 +152,9 @@ jobs:
 
           cat <<-EOH > infrastructure/nomad/hosts.ini
           [nomad_servers]
-          ${TARGET_MACHINE_IP} ${ANSIBLE_CONNECTION} ansible_user=ubuntu
+          ${TARGET_MACHINE_IP} ${ANSIBLE_CONNECTION} ansible_user=${ANSIBLE_USER}
           [nomad_clients]
-          ${TARGET_MACHINE_IP} ${ANSIBLE_CONNECTION} ansible_user=ubuntu
+          ${TARGET_MACHINE_IP} ${ANSIBLE_CONNECTION} ansible_user=${ANSIBLE_USER}
           EOH
 
           ansible all --inventory infrastructure/nomad/hosts.ini --module-name ping

infrastructure/nomad/playbooks/deploy.yml

@@ -50,7 +50,7 @@
         src: "{{ vault_init_file }}"
       register: vault_init
       become: true
-      become_user: "{{ hostvars[inventory_hostname].ansible_user }}"
+      become_user: "{{ ansible_user }}"
       no_log: true
 
     - name: Parse Vault Initialization File
@@ -135,12 +135,18 @@
             else profiles[profile].jobs
            }}
 
-    - name: Ensure "{{ ansible_env.HOME }}/{{ env }}" Directory Exists
-      ansible.builtin.file:
+    - name: Determine "{{ ansible_env.HOME }}/{{ env }}" Status
+      ansible.builtin.stat:
         path: "{{ ansible_env.HOME }}/{{ env }}"
-        state: directory
-        mode: "0744"
-        recurse: yes
+      register: env_dir
+
+    - name: Check "{{ ansible_env.HOME }}/{{ env }}" Directory Exists
+      assert:
+        that:
+          - env_dir.stat.isdir is defined
+          - env_dir.stat.isdir
+        fail_msg: "The directory {{ ansible_env.HOME }}/{{ env }} does not exist."
+        success_msg: "The directory {{ ansible_env.HOME }}/{{ env }} exists."
 
     - name: Read Existing "meta.json"
       ansible.builtin.shell: |
@@ -228,7 +234,7 @@
           {% if artifact.keystore is defined %}
             case "{{ environments[env].secrets }}" in
               "generate")
-                PASSPHRASE="{{ lookup('password', '/dev/null', length=1024) }}"
+                PASSPHRASE="{{ lookup('password', '/dev/null', length=1024, chars=["ascii_letters", "digits", ".,:_"]) }}"
 
                 RESULT=$(
                   {{ keystore_generator.stdout }} generate \
@@ -462,13 +468,12 @@
       run_once: true
       when: build_artifacts
 
-    - name: Upload Artifacts Async
+    - name: Upload Artifacts Async to AWS S3
       amazon.aws.aws_s3:
         bucket: "{{ aws_s3_bucket }}"
         object: "{{ item.path | basename }}"
         src: "{{ item.path }}"
         mode: put
-        tags: "{{ {'AutoDelete': 'true'} if env == 'devenv' else {} }}"
       loop: "{{ upload_artifacts.files }}"
       loop_control:
         label: "{{ item.path | basename }}"
@@ -477,7 +482,7 @@
       delegate_to: localhost
       run_once: true
       register: upload_artifacts_async
-      when: build_artifacts and (upload_artifacts.files | default([])) | length > 0
+      when: build_artifacts and upload_artifacts.matched > 0 and env != 'devenv'
 
     - name: Wait for Upload Artifacts Async to Complete
       ansible.builtin.async_status:
@@ -491,7 +496,23 @@
         label: "{{ item.item.path | basename }}"
       delegate_to: localhost
       run_once: true
-      when: build_artifacts and (upload_artifacts.files | default([])) | length > 0
+      when: build_artifacts and upload_artifacts.matched > 0 and env != 'devenv'
+
+    - name: Ensure Target Directory Exists and is Empty
+      ansible.builtin.shell: |
+        rm -rf {{ ansible_env.HOME }}/{{ env }}/artifacts/ && mkdir {{ ansible_env.HOME }}/{{ env }}/artifacts/
+      args:
+        executable: bash
+      when: build_artifacts and upload_artifacts.matched > 0 and env == 'devenv'
+
+    - name: Copy Artifacts to Target Machine
+      ansible.builtin.copy:
+        src: "{{ item.path }}"
+        dest: "{{ ansible_env.HOME }}/{{ env }}/artifacts/{{ item.path | basename }}"
+      loop: "{{ upload_artifacts.files }}"
+      loop_control:
+        label: "{{ item.path | basename }}"
+      when: build_artifacts and upload_artifacts.matched > 0 and env == 'devenv'
 
     - name: Push Generated Secrets to Vault
       ansible.builtin.uri:
@@ -508,13 +529,13 @@
       delegate_to: localhost
       no_log: true
 
-    - name: Cleanup Artifacts
+    - name: Cleanup Built Artifacts
       ansible.builtin.file:
         path: "{{ dist_dir }}"
         state: absent
       delegate_to: localhost
       run_once: true
-      when: build_artifacts and (upload_artifacts.files | default([])) | length > 0
+      when: build_artifacts and upload_artifacts.matched > 0
 
     - name: Delete "meta.json"
       file:
@@ -550,16 +571,21 @@
       ansible.builtin.shell: |
         RESULT="$(nomad run {{ ansible_env.HOME }}/{{ env }}/{{ job.name }}.nomad 2>&1)"
         if [ $? -ne 0 ]; then
-          echo "Failed to deploy {{ job.name }}: ${RESULT}"
+          echo "Failed to deploy {{ job.name }}: ${RESULT}."
           exit 1
         fi
 
-        TIMEOUT=300
+        TIMEOUT={% if profile == 'ci' %}600{% else %}300{% endif %}
         START_TIME=$(date +%s)
-        JOB_TYPE=$(nomad job status -json "{{ job.name }}" | jq -r '.[0].Allocations[0].JobType')
+        RESULT=$(nomad job status -json "{{ job.name }}")
+        if [ $? -ne 0 ]; then
+          echo "Failed to get job status for {{ job.name }}: ${RESULT}."
+          exit 1
+        fi
+        JOB_TYPE=$(echo "${RESULT}" | jq -r '.[0].Allocations[0].JobType')
 
         while true; do
-          STATUS=$(nomad job status -json "{{ job.name }}" | jq -r '.[0].Allocations[0].ClientStatus')
+          STATUS=$(echo "${RESULT}"  | jq -r '.[0].Allocations[0].ClientStatus')
 
           case "${JOB_TYPE}" in
             service)
@@ -573,7 +599,12 @@
               fi
               ;;
             *)
+              {% if env != 'devenv' %}
               break
+              {% else %}
+              echo "Unknown job type: ${JOB_TYPE}"
+              exit 1
+              {% endif %}
               ;;
           esac
 
@@ -585,6 +616,11 @@
           fi
 
           sleep 1
+          RESULT=$(nomad job status -json "{{ job.name }}")
+          if [ $? -ne 0 ]; then
+            echo "Failed to get job status for {{ job.name }}: ${RESULT}."
+            exit 1
+          fi
         done
       args:
         executable: bash

infrastructure/nomad/playbooks/init.yml

@@ -1,11 +1,11 @@
 - name: Initialize and Configure Cluster
   hosts: all
   become: yes
-  remote_user: "{{ hostvars[inventory_hostname].ansible_user }}"
   gather_facts: yes
 
   vars:
     version: "unknown"
+    ansible_user_home: "/home/{{ hostvars[inventory_hostname].ansible_user }}"
     nomad_server_ip: "{{ (hostvars[groups['nomad_servers'][0]]['ansible_default_ipv4']['address'] if groups['nomad_servers'] | default([]) | length > 0 else '127.0.0.1') }}"
     nomad_clients_defined: "{{ groups['nomad_clients'] | length > 0 }}"
     nomad_servers_defined: "{{ groups['nomad_servers'] | length > 0 }}"
@@ -113,6 +113,15 @@
         fail_msg: "The profile name is not set correctly."
         success_msg: "The profile name is set to: {{ profile }}."
 
+    - name: Ensure "{{ ansible_user_home }}/{{ env }}" Directory Exists
+      ansible.builtin.file:
+        path: "{{ ansible_user_home }}/{{ env }}/artifacts"
+        state: directory
+        mode: "0744"
+        recurse: yes
+      become: true
+      become_user: "{{ ansible_user }}"
+
   tasks:
     - name: Add DataDog Repository Key
       ansible.builtin.apt_key:
@@ -141,7 +150,7 @@
 
     - name: Add PostgreSQL Repository
       ansible.builtin.apt_repository:
-        repo: "deb http://apt.postgresql.org/pub/repos/apt {{ ansible_distribution_release }}-pgdg main"
+        repo: "deb https://apt.postgresql.org/pub/repos/apt {{ ansible_distribution_release }}-pgdg main"
         state: present
         filename: pgdg.list
 
@@ -231,7 +240,7 @@
         mode: "0400"
       when: vault_status.json.initialized == false
       become: true
-      become_user: "{{ hostvars[inventory_hostname].ansible_user }}"
+      become_user: "{{ ansible_user }}"
       no_log: true
 
     - name: Determine Vault Seal Status
@@ -250,7 +259,7 @@
         - vault_status.json.initialized == true
         - vault_seal_status.json.sealed == true
       become: true
-      become_user: "{{ hostvars[inventory_hostname].ansible_user }}"
+      become_user: "{{ ansible_user }}"
       no_log: true
 
     - name: Parse Vault Initialization File

infrastructure/nomad/playbooks/templates/jobs/version.nomad.j2 → infrastructure/nomad/playbooks/templates/jobs/artifacts.nomad.j2

@@ -1,8 +1,8 @@
 #jinja2: trim_blocks:True, lstrip_blocks:True
-
-# This job exists only to provide deployment information to the Nomad UI.
-job "{{ environments[env].version }}" {
+job "{% if env != 'devenv' %}{{ environments[env].version }}{% else %}artifacts-{{ environments[env].version }}{% endif %}" {
   datacenters = ["{{ datacenter }}"]
+
+  {% if env != 'devenv' %}
   type = "batch"
   priority = 1
 
@@ -11,6 +11,7 @@ job "{{ environments[env].version }}" {
   periodic {
     cron = "0 0 1 1 6"
   }
+  {% endif %}
 
   meta {
     CHAIN_ID = "{{ environments[env].chain_id }}"
@@ -22,7 +23,10 @@ job "{{ environments[env].version }}" {
     TIMESTAMP = "{{ now(utc=true, fmt='%a %Y-%m-%d %H:%M:%S UTC') }}"
   }
 
+  {% if env != 'devenv' %}
   group "info" {
+    count = 1
+
     task "dummy" {
       driver = "exec"
 
@@ -37,4 +41,46 @@ job "{{ environments[env].version }}" {
       }
     }
   }
+  {% else %}
+  group "artifacts-group" {
+    count = 1
+
+    network {
+      mode = "bridge"
+
+      port "http" {
+        static = 1111
+        to = 1111
+      }
+    }
+
+    volume "artifacts-volume" {
+      type = "host"
+      source = "artifacts-volume"
+      read_only = true
+    }
+
+    task "artifacts" {
+      driver = "exec"
+
+      service {
+        name = "artifacts"
+        port = "http"
+        tags = ["http"]
+        provider = "nomad"
+      }
+
+      volume_mount {
+        volume = "artifacts-volume"
+        destination = "/local/artifacts"
+        read_only = true
+      }
+
+      config {
+        command = "python3"
+        args = ["-m", "http.server", "1111", "--directory", "/local/artifacts"]
+      }
+    }
+  }
+  {% endif %}
 }

infrastructure/nomad/playbooks/templates/jobs/contracts-deployer.nomad.j2

@@ -46,9 +46,15 @@ job "{{ job.name }}" {
         destination = "local/foundry.sh"
       }
 
+      {% if env != 'devenv' %}
       artifact {
         source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/contracts_{{ version }}.tar.gz"
       }
+      {% else %}
+      artifact {
+        source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/contracts_{{ version }}.tar.gz"
+      }
+      {% endif %}
 
       template {
         data = <<-EOH

infrastructure/nomad/playbooks/templates/jobs/mev-commit-bridge.nomad.j2

@@ -41,13 +41,21 @@ job "{{ job.name }}" {
         destination = "local/foundry.sh"
       }
 
+      {% if env != 'devenv' %}
       artifact {
         source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/contracts_{{ version }}.tar.gz"
       }
-
       artifact {
         source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/mev-commit-bridge-relayer_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz"
       }
+      {% else %}
+      artifact {
+        source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/contracts_{{ version }}.tar.gz"
+      }
+      artifact {
+        source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/mev-commit-bridge-relayer_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz"
+      }
+      {% endif %}
 
       template {
         data = <<-EOH

infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2

@@ -36,9 +36,15 @@ job "{{ job.name }}" {
       }
       {% endfor %}
 
+      {% if env != 'devenv' %}
       artifact {
         source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/{{ job.target_type }}-emulator_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz"
       }
+      {% else %}
+      artifact {
+        source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/{{ job.target_type }}-emulator_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz"
+      }
+      {% endif %}
 
       template {
         data = <<-EOH

infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2

@@ -47,13 +47,21 @@ job "{{ job.name }}" {
       }
       {% endfor %}
 
+      {% if env != 'devenv' %}
       artifact {
         source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/genesis_{{ version }}.json"
       }
-
       artifact {
         source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/mev-commit-geth_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz"
       }
+      {% else %}
+      artifact {
+        source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/genesis_{{ version }}.json"
+      }
+      artifact {
+        source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/mev-commit-geth_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz"
+      }
+      {% endif %}
 
       template {
         data = <<-EOH