diff --git a/.github/workflows/infrstructure.yml b/.github/workflows/infrstructure.yml index 00ea09fba..f58c0fa9f 100644 --- a/.github/workflows/infrstructure.yml +++ b/.github/workflows/infrstructure.yml @@ -132,8 +132,7 @@ jobs: - name: Configure Control Machine run: | - sudo useradd --create-home ubuntu && sudo usermod --append --groups sudo ubuntu - + ANSIBLE_USER=$([ "${IS_MANUAL_DEPLOYMENT}" == "true" ] && echo "ubuntu" || echo "${USER}") ANSIBLE_CONNECTION="ansible_connection=local" if [ "${IS_MANUAL_DEPLOYMENT}" == "true" ]; then ANSIBLE_CONNECTION="" @@ -153,9 +152,9 @@ jobs: cat <<-EOH > infrastructure/nomad/hosts.ini [nomad_servers] - ${TARGET_MACHINE_IP} ${ANSIBLE_CONNECTION} ansible_user=ubuntu + ${TARGET_MACHINE_IP} ${ANSIBLE_CONNECTION} ansible_user=${ANSIBLE_USER} [nomad_clients] - ${TARGET_MACHINE_IP} ${ANSIBLE_CONNECTION} ansible_user=ubuntu + ${TARGET_MACHINE_IP} ${ANSIBLE_CONNECTION} ansible_user=${ANSIBLE_USER} EOH ansible all --inventory infrastructure/nomad/hosts.ini --module-name ping diff --git a/infrastructure/nomad/playbooks/deploy.yml b/infrastructure/nomad/playbooks/deploy.yml index 3fbc0981a..e2f662305 100644 --- a/infrastructure/nomad/playbooks/deploy.yml +++ b/infrastructure/nomad/playbooks/deploy.yml @@ -50,7 +50,7 @@ src: "{{ vault_init_file }}" register: vault_init become: true - become_user: "{{ hostvars[inventory_hostname].ansible_user }}" + become_user: "{{ ansible_user }}" no_log: true - name: Parse Vault Initialization File @@ -135,12 +135,18 @@ else profiles[profile].jobs }} - - name: Ensure "{{ ansible_env.HOME }}/{{ env }}" Directory Exists - ansible.builtin.file: + - name: Determine "{{ ansible_env.HOME }}/{{ env }}" Status + ansible.builtin.stat: path: "{{ ansible_env.HOME }}/{{ env }}" - state: directory - mode: "0744" - recurse: yes + register: env_dir + + - name: Check "{{ ansible_env.HOME }}/{{ env }}" Directory Exists + assert: + that: + - env_dir.stat.isdir is defined + - env_dir.stat.isdir + fail_msg: "The directory {{ ansible_env.HOME }}/{{ env }} does not exist." + success_msg: "The directory {{ ansible_env.HOME }}/{{ env }} exists." - name: Read Existing "meta.json" ansible.builtin.shell: | @@ -228,7 +234,7 @@ {% if artifact.keystore is defined %} case "{{ environments[env].secrets }}" in "generate") - PASSPHRASE="{{ lookup('password', '/dev/null', length=1024) }}" + PASSPHRASE="{{ lookup('password', '/dev/null', length=1024, chars=["ascii_letters", "digits", ".,:_"]) }}" RESULT=$( {{ keystore_generator.stdout }} generate \ @@ -462,13 +468,12 @@ run_once: true when: build_artifacts - - name: Upload Artifacts Async + - name: Upload Artifacts Async to AWS S3 amazon.aws.aws_s3: bucket: "{{ aws_s3_bucket }}" object: "{{ item.path | basename }}" src: "{{ item.path }}" mode: put - tags: "{{ {'AutoDelete': 'true'} if env == 'devenv' else {} }}" loop: "{{ upload_artifacts.files }}" loop_control: label: "{{ item.path | basename }}" @@ -477,7 +482,7 @@ delegate_to: localhost run_once: true register: upload_artifacts_async - when: build_artifacts and (upload_artifacts.files | default([])) | length > 0 + when: build_artifacts and upload_artifacts.matched > 0 and env != 'devenv' - name: Wait for Upload Artifacts Async to Complete ansible.builtin.async_status: @@ -491,7 +496,23 @@ label: "{{ item.item.path | basename }}" delegate_to: localhost run_once: true - when: build_artifacts and (upload_artifacts.files | default([])) | length > 0 + when: build_artifacts and upload_artifacts.matched > 0 and env != 'devenv' + + - name: Ensure Target Directory Exists and is Empty + ansible.builtin.shell: | + rm -rf {{ ansible_env.HOME }}/{{ env }}/artifacts/ && mkdir {{ ansible_env.HOME }}/{{ env }}/artifacts/ + args: + executable: bash + when: build_artifacts and upload_artifacts.matched > 0 and env == 'devenv' + + - name: Copy Artifacts to Target Machine + ansible.builtin.copy: + src: "{{ item.path }}" + dest: "{{ ansible_env.HOME }}/{{ env }}/artifacts/{{ item.path | basename }}" + loop: "{{ upload_artifacts.files }}" + loop_control: + label: "{{ item.path | basename }}" + when: build_artifacts and upload_artifacts.matched > 0 and env == 'devenv' - name: Push Generated Secrets to Vault ansible.builtin.uri: @@ -508,13 +529,13 @@ delegate_to: localhost no_log: true - - name: Cleanup Artifacts + - name: Cleanup Built Artifacts ansible.builtin.file: path: "{{ dist_dir }}" state: absent delegate_to: localhost run_once: true - when: build_artifacts and (upload_artifacts.files | default([])) | length > 0 + when: build_artifacts and upload_artifacts.matched > 0 - name: Delete "meta.json" file: @@ -550,16 +571,21 @@ ansible.builtin.shell: | RESULT="$(nomad run {{ ansible_env.HOME }}/{{ env }}/{{ job.name }}.nomad 2>&1)" if [ $? -ne 0 ]; then - echo "Failed to deploy {{ job.name }}: ${RESULT}" + echo "Failed to deploy {{ job.name }}: ${RESULT}." exit 1 fi - TIMEOUT=300 + TIMEOUT={% if profile == 'ci' %}600{% else %}300{% endif %} START_TIME=$(date +%s) - JOB_TYPE=$(nomad job status -json "{{ job.name }}" | jq -r '.[0].Allocations[0].JobType') + RESULT=$(nomad job status -json "{{ job.name }}") + if [ $? -ne 0 ]; then + echo "Failed to get job status for {{ job.name }}: ${RESULT}." + exit 1 + fi + JOB_TYPE=$(echo "${RESULT}" | jq -r '.[0].Allocations[0].JobType') while true; do - STATUS=$(nomad job status -json "{{ job.name }}" | jq -r '.[0].Allocations[0].ClientStatus') + STATUS=$(echo "${RESULT}" | jq -r '.[0].Allocations[0].ClientStatus') case "${JOB_TYPE}" in service) @@ -573,7 +599,12 @@ fi ;; *) + {% if env != 'devenv' %} break + {% else %} + echo "Unknown job type: ${JOB_TYPE}" + exit 1 + {% endif %} ;; esac @@ -585,6 +616,11 @@ fi sleep 1 + RESULT=$(nomad job status -json "{{ job.name }}") + if [ $? -ne 0 ]; then + echo "Failed to get job status for {{ job.name }}: ${RESULT}." + exit 1 + fi done args: executable: bash diff --git a/infrastructure/nomad/playbooks/init.yml b/infrastructure/nomad/playbooks/init.yml index e0a87883b..1ae80bd24 100644 --- a/infrastructure/nomad/playbooks/init.yml +++ b/infrastructure/nomad/playbooks/init.yml @@ -1,11 +1,11 @@ - name: Initialize and Configure Cluster hosts: all become: yes - remote_user: "{{ hostvars[inventory_hostname].ansible_user }}" gather_facts: yes vars: version: "unknown" + ansible_user_home: "/home/{{ hostvars[inventory_hostname].ansible_user }}" nomad_server_ip: "{{ (hostvars[groups['nomad_servers'][0]]['ansible_default_ipv4']['address'] if groups['nomad_servers'] | default([]) | length > 0 else '127.0.0.1') }}" nomad_clients_defined: "{{ groups['nomad_clients'] | length > 0 }}" nomad_servers_defined: "{{ groups['nomad_servers'] | length > 0 }}" @@ -113,6 +113,15 @@ fail_msg: "The profile name is not set correctly." success_msg: "The profile name is set to: {{ profile }}." + - name: Ensure "{{ ansible_user_home }}/{{ env }}" Directory Exists + ansible.builtin.file: + path: "{{ ansible_user_home }}/{{ env }}/artifacts" + state: directory + mode: "0744" + recurse: yes + become: true + become_user: "{{ ansible_user }}" + tasks: - name: Add DataDog Repository Key ansible.builtin.apt_key: @@ -141,7 +150,7 @@ - name: Add PostgreSQL Repository ansible.builtin.apt_repository: - repo: "deb http://apt.postgresql.org/pub/repos/apt {{ ansible_distribution_release }}-pgdg main" + repo: "deb https://apt.postgresql.org/pub/repos/apt {{ ansible_distribution_release }}-pgdg main" state: present filename: pgdg.list @@ -231,7 +240,7 @@ mode: "0400" when: vault_status.json.initialized == false become: true - become_user: "{{ hostvars[inventory_hostname].ansible_user }}" + become_user: "{{ ansible_user }}" no_log: true - name: Determine Vault Seal Status @@ -250,7 +259,7 @@ - vault_status.json.initialized == true - vault_seal_status.json.sealed == true become: true - become_user: "{{ hostvars[inventory_hostname].ansible_user }}" + become_user: "{{ ansible_user }}" no_log: true - name: Parse Vault Initialization File diff --git a/infrastructure/nomad/playbooks/templates/jobs/version.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/artifacts.nomad.j2 similarity index 50% rename from infrastructure/nomad/playbooks/templates/jobs/version.nomad.j2 rename to infrastructure/nomad/playbooks/templates/jobs/artifacts.nomad.j2 index df35695e1..eaf875527 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/version.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/artifacts.nomad.j2 @@ -1,8 +1,8 @@ #jinja2: trim_blocks:True, lstrip_blocks:True - -# This job exists only to provide deployment information to the Nomad UI. -job "{{ environments[env].version }}" { +job "{% if env != 'devenv' %}{{ environments[env].version }}{% else %}artifacts-{{ environments[env].version }}{% endif %}" { datacenters = ["{{ datacenter }}"] + + {% if env != 'devenv' %} type = "batch" priority = 1 @@ -11,6 +11,7 @@ job "{{ environments[env].version }}" { periodic { cron = "0 0 1 1 6" } + {% endif %} meta { CHAIN_ID = "{{ environments[env].chain_id }}" @@ -22,7 +23,10 @@ job "{{ environments[env].version }}" { TIMESTAMP = "{{ now(utc=true, fmt='%a %Y-%m-%d %H:%M:%S UTC') }}" } + {% if env != 'devenv' %} group "info" { + count = 1 + task "dummy" { driver = "exec" @@ -37,4 +41,46 @@ job "{{ environments[env].version }}" { } } } + {% else %} + group "artifacts-group" { + count = 1 + + network { + mode = "bridge" + + port "http" { + static = 1111 + to = 1111 + } + } + + volume "artifacts-volume" { + type = "host" + source = "artifacts-volume" + read_only = true + } + + task "artifacts" { + driver = "exec" + + service { + name = "artifacts" + port = "http" + tags = ["http"] + provider = "nomad" + } + + volume_mount { + volume = "artifacts-volume" + destination = "/local/artifacts" + read_only = true + } + + config { + command = "python3" + args = ["-m", "http.server", "1111", "--directory", "/local/artifacts"] + } + } + } + {% endif %} } diff --git a/infrastructure/nomad/playbooks/templates/jobs/contracts-deployer.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/contracts-deployer.nomad.j2 index e0a8c4144..6d722e239 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/contracts-deployer.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/contracts-deployer.nomad.j2 @@ -46,9 +46,15 @@ job "{{ job.name }}" { destination = "local/foundry.sh" } + {% if env != 'devenv' %} artifact { source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/contracts_{{ version }}.tar.gz" } + {% else %} + artifact { + source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/contracts_{{ version }}.tar.gz" + } + {% endif %} template { data = <<-EOH diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-bridge.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-bridge.nomad.j2 index 5f846afdd..d6ebbbe4d 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-bridge.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-bridge.nomad.j2 @@ -41,13 +41,21 @@ job "{{ job.name }}" { destination = "local/foundry.sh" } + {% if env != 'devenv' %} artifact { source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/contracts_{{ version }}.tar.gz" } - artifact { source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/mev-commit-bridge-relayer_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz" } + {% else %} + artifact { + source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/contracts_{{ version }}.tar.gz" + } + artifact { + source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/mev-commit-bridge-relayer_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz" + } + {% endif %} template { data = <<-EOH diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 index 0861a27eb..1a387c401 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 @@ -36,9 +36,15 @@ job "{{ job.name }}" { } {% endfor %} + {% if env != 'devenv' %} artifact { source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/{{ job.target_type }}-emulator_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz" } + {% else %} + artifact { + source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/{{ job.target_type }}-emulator_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz" + } + {% endif %} template { data = <<-EOH diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2 index ac85cf3b9..2f04b0399 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2 @@ -47,13 +47,21 @@ job "{{ job.name }}" { } {% endfor %} + {% if env != 'devenv' %} artifact { source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/genesis_{{ version }}.json" } - artifact { source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/mev-commit-geth_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz" } + {% else %} + artifact { + source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/genesis_{{ version }}.json" + } + artifact { + source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/mev-commit-geth_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz" + } + {% endif %} template { data = <<-EOH diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 index 99dd9aa27..e96b2c7bc 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 @@ -136,13 +136,21 @@ job "{{ job.name }}" { } {% endif %} + {% if env != 'devenv' %} artifact { source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/mev-commit-oracle_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz" } - artifact { source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/contracts_{{ version }}.tar.gz" } + {% else %} + artifact { + source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/mev-commit-oracle_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz" + } + artifact { + source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/contracts_{{ version }}.tar.gz" + } + {% endif %} template { data = <<-EOH diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit.nomad.j2 index 699b30bb3..c10eeefc9 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit.nomad.j2 @@ -47,9 +47,15 @@ job "{{ job.name }}" { } {% endif %} + {% if env != 'devenv' %} artifact { source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/mev-commit_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz" } + {% else %} + artifact { + source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/mev-commit_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz" + } + {% endif %} template { data = <<-EOH diff --git a/infrastructure/nomad/playbooks/templates/services/mev-commit.xyz.hcl.j2 b/infrastructure/nomad/playbooks/templates/services/mev-commit.xyz.hcl.j2 index ce9afb905..9a46761a9 100644 --- a/infrastructure/nomad/playbooks/templates/services/mev-commit.xyz.hcl.j2 +++ b/infrastructure/nomad/playbooks/templates/services/mev-commit.xyz.hcl.j2 @@ -1,3 +1,4 @@ +#jinja2: trim_blocks:True, lstrip_blocks:True server { listen 443 ssl http2; diff --git a/infrastructure/nomad/playbooks/templates/services/nomad.hcl.j2 b/infrastructure/nomad/playbooks/templates/services/nomad.hcl.j2 index adfc3cb08..648d7292c 100644 --- a/infrastructure/nomad/playbooks/templates/services/nomad.hcl.j2 +++ b/infrastructure/nomad/playbooks/templates/services/nomad.hcl.j2 @@ -1,15 +1,16 @@ +#jinja2: trim_blocks:True, lstrip_blocks:True data_dir = "/opt/nomad/data" bind_addr = "0.0.0.0" -{% if nomad_servers_defined -%} +{% if nomad_servers_defined %} server { - enabled = true + enabled = true bootstrap_expect = 1 - raft_protocol = 3 + raft_protocol = 3 } {% endif %} -{% if nomad_clients_defined -%} +{% if nomad_clients_defined %} client { enabled = true servers = ["{{ nomad_server_ip }}:4647"] @@ -24,6 +25,11 @@ client { "/usr" = "/usr" "/opt" = "/opt" } + {% if env == "devenv" %} + host_volume "artifacts-volume" { + path = "{{ ansible_user_home }}/{{ env }}/artifacts" + } + {% endif %} } {% endif %} @@ -37,7 +43,7 @@ log_level = "DEBUG" enable_syslog = true syslog_facility = "LOCAL0" -{% if nomad_servers_defined -%} +{% if nomad_servers_defined %} advertise { http = "{{ ansible_host }}:4646" rpc = "{{ ansible_host }}:4647" diff --git a/infrastructure/nomad/playbooks/templates/services/vault.hcl.j2 b/infrastructure/nomad/playbooks/templates/services/vault.hcl.j2 index 234bfe20e..a7ce994d1 100644 --- a/infrastructure/nomad/playbooks/templates/services/vault.hcl.j2 +++ b/infrastructure/nomad/playbooks/templates/services/vault.hcl.j2 @@ -1,3 +1,4 @@ +#jinja2: trim_blocks:True, lstrip_blocks:True ui = true api_addr = "{{ vault_address }}" diff --git a/infrastructure/nomad/playbooks/variables/profiles.yml b/infrastructure/nomad/playbooks/variables/profiles.yml index 744a1447e..a0085605f 100644 --- a/infrastructure/nomad/playbooks/variables/profiles.yml +++ b/infrastructure/nomad/playbooks/variables/profiles.yml @@ -26,9 +26,9 @@ artifacts: path: p2p jobs: - version: &version_job - name: version - template: version.nomad.j2 + artifacts: &artifacts_job + name: artifacts + template: artifacts.nomad.j2 datadog_agent_logs_collector: &datadog_agent_logs_collector_job name: datadog-agent-logs-collector @@ -504,6 +504,7 @@ jobs: profiles: ci: jobs: + - *artifacts_job - *mev_commit_geth_bootnode1_job - *mev_commit_geth_signer_node1_job - *mev_commit_geth_member_node_job @@ -520,7 +521,7 @@ profiles: devnet: jobs: - - *version_job + - *artifacts_job - *datadog_agent_logs_collector_job - *mev_commit_geth_bootnode1_job - *mev_commit_geth_signer_node1_job @@ -539,6 +540,7 @@ profiles: testnet: jobs: + - *artifacts_job - *datadog_agent_logs_collector_job - *mev_commit_geth_bootnode1_job - *mev_commit_geth_signer_node1_job @@ -554,7 +556,7 @@ profiles: stressnet: jobs: - - *version_job + - *artifacts_job - *datadog_agent_logs_collector_job - *mev_commit_geth_bootnode1_job - *mev_commit_geth_signer_node1_job