diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index fcd760e18..ff25492b8 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -1,4 +1,4 @@ -name: mev-commit-artifacts +name: artifacts on: workflow_dispatch diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cf32fafb..431b0dc8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: mev-commit-ci +name: ci on: push: diff --git a/.github/workflows/infrstructure.yml b/.github/workflows/infrstructure.yml index 79c51adbf..35402f85d 100644 --- a/.github/workflows/infrstructure.yml +++ b/.github/workflows/infrstructure.yml @@ -1,15 +1,21 @@ -name: mev-commit-infrastructure +name: infrastructure on: workflow_run: - workflows: ["mev-commit-ci"] + workflows: + - ci types: - completed +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + jobs: - deploy_and_test: - name: Deploy and Test mev-commit Minimal Infrastructure - runs-on: ubuntu-latest + cluster: + name: Setup and Test Nomad devnet Cluster + runs-on: ubuntu-22.04 + timeout-minutes: 120 if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - name: Checkout code @@ -18,52 +24,52 @@ jobs: submodules: recursive ref: ${{ github.event.workflow_run.head_branch }} - - name: Install jq + - name: Install Dependencies run: | + echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | sudo tee /etc/apt/sources.list.d/goreleaser.list sudo apt-get update - sudo apt-get install -y jq - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + sudo apt-get install --yes goreleaser + + pip install boto3 botocore + pipx inject ansible-core botocore boto3 - - name: Setup Docker Compose + - name: Configure Machine run: | - sudo rm $(which docker-compose) - curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose + aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} + aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws configure set default.region us-west-2 - - name: Build and Start Services - run: ./mev-commit-cli.sh start minimal + cat <<-EOH > infrastructure/nomad/hosts.ini + [nomad_servers] + 127.0.0.1 ansible_connection=local ansible_user=${USER} - - name: Check Service Endpoints - run: | - ips=("172.29.18.2" "172.29.18.3" "172.29.18.4") + [nomad_clients] + 127.0.0.1 ansible_connection=local ansible_user=${USER} + EOH + + ansible-inventory --inventory infrastructure/nomad/hosts.ini hosts.ini --list --yaml + ansible all --inventory infrastructure/nomad/hosts.ini --module-name ping - for ip in "${ips[@]}"; do - echo "Checking service at $ip" + - name: Initialize and Configure Cluster + run: | + ./cluster.sh init --profile ci --debug + working-directory: infrastructure/nomad - attempts=0 - success=false - while [ $attempts -lt 6 ]; do - response=$(curl -k -s "https://$ip:13523/topology") - bidders=$(echo "$response" | jq '.connected_peers.bidders | length') - providers=$(echo "$response" | jq '.connected_peers.providers | length') - if [[ -n "$response" && "$bidders" -gt 0 || "$providers" -gt 0 ]]; then - echo "Service at $ip is OK" - success=true - break - else - echo "Service check failed for $ip. Retrying in 30 seconds..." - attempts=$(( $attempts + 1 )) - sleep 30 - fi - done + - name: Build Artifacts and Deploy Jobs + run: | + ./cluster.sh deploy --profile ci --debug + working-directory: infrastructure/nomad - if [ "$success" == "false" ]; then - echo "Service check failed for $ip after 3 attempts, failing" - exit 1 - fi - done + - name: Destroy Cluster + run: | + ./cluster.sh destroy --profile ci --debug + working-directory: infrastructure/nomad - - name: Stop Services - run: ./mev-commit-cli.sh stop minimal + - name: Initialize Debug Shell + if: failure() + run: | + KEYS=$(curl -sSf -X POST https://eu.relay.tunshell.com/api/sessions) + echo "::add-mask::${KEYS}" + echo "Debug Shell:" + echo "sh <(curl -sSf https://lets.tunshell.com/init.sh) L $(echo ${KEYS} | jq -r .peer2_key) \${TUNSHELL_SECRET} eu.relay.tunshell.com" + curl -sSf https://lets.tunshell.com/init.sh | sh -s -- T $(echo ${KEYS} | jq -r .peer1_key) ${{ secrets.TUNSHELL_SECRET }} eu.relay.tunshell.com diff --git a/.github/workflows/releaser.yml b/.github/workflows/releaser.yml index b2747568d..cb28e41fe 100644 --- a/.github/workflows/releaser.yml +++ b/.github/workflows/releaser.yml @@ -1,4 +1,4 @@ -name: mev-commit-releaser +name: releaser on: push: diff --git a/infrastructure/nomad/README.md b/infrastructure/nomad/README.md index 55d3ab821..cfd51a238 100644 --- a/infrastructure/nomad/README.md +++ b/infrastructure/nomad/README.md @@ -64,12 +64,7 @@ Prepare `hosts.ini` File: This file contains the IP addresses of your Nomad serv ``` > Replace the 192.0.2.X and 198.51.100.X with the IP addresses of your Nomad server and client machines, respectively. > Ensure the ansible_user matches the username on your target machines that has SSH access. - -If your host machine is the same as your control machine add the following to your `hosts.ini` file: -```ini -[local] -127.0.0.1 ansible_connection=local -``` +> If your host machine is the same as your control machine replace the IP addresses with `127.0.0.1 ansible_connection=local`. If you do not want to use the SSH agent, another option is to add the following configuration to every `nomad_server` or `nomad_client` record in the `host.ini` file: `ansible_ssh_private_key_file=/path/to/your/private_key`. For example: @@ -82,6 +77,12 @@ If you do not want to use the SSH agent, another option is to add the following 198.51.100.2 ansible_user=ubuntu ansible_ssh_private_key_file=/path/to/your/private_key ``` +Or if the private key is the same for all machines, you can set the `ansible_ssh_private_key_file` in the `[all:vars]` section: +```ini +[all:vars] +ansible_ssh_private_key_file=/path/to/your/private_key +``` + ## Cluster Management To manage the Nomad cluster, use the `cluster.sh` script. This script allows you to initialize, deploy, and destroy the Nomad cluster. diff --git a/infrastructure/nomad/cluster.sh b/infrastructure/nomad/cluster.sh index 67c402da9..8bbaaaada 100755 --- a/infrastructure/nomad/cluster.sh +++ b/infrastructure/nomad/cluster.sh @@ -13,7 +13,11 @@ deploy_version="HEAD" profile_name="devnet" help() { - echo "Usage: $0 [init [--profile ] [--skip-certificates-setup] [--debug]] [deploy [version=HEAD] [--profile ] [--force-build-templates] [--no-logs-collection] [--debug]] [destroy [--debug]] [--help]" + echo "Usage:" + echo "$0 [init [--profile ] [--skip-certificates-setup] [--debug]]" + echo "$0 [deploy [version=HEAD] [--profile ] [--force-build-templates] [--no-logs-collection] [--debug]]" + echo "$0 [destroy [--debug]] [--help]" + echo "$0 --help" echo echo "Parameters:" echo " init Initialize the environment." @@ -66,7 +70,11 @@ help() { } usage() { - echo "Usage: $0 [init [--profile ] [--skip-certificates-setup] [--debug]] [deploy [version=HEAD] [--profile ] [--force-build-templates] [--no-logs-collection] [--debug]] [destroy [--debug]] [--help]" + echo "Usage:" + echo "$0 [init [--profile ] [--skip-certificates-setup] [--debug]]" + echo "$0 [deploy [version=HEAD] [--profile ] [--force-build-templates] [--no-logs-collection] [--debug]]" + echo "$0 [destroy [--debug]] [--help]" + echo "$0 --help" exit 1 } @@ -236,7 +244,7 @@ main() { ;; esac - ansible-playbook -i hosts.ini "${playbook}" "${flags[@]}" + ansible-playbook --inventory hosts.ini "${playbook}" "${flags[@]}" } main "$@" diff --git a/infrastructure/nomad/playbooks/deploy.yml b/infrastructure/nomad/playbooks/deploy.yml index 91d43864d..d239a8d2c 100644 --- a/infrastructure/nomad/playbooks/deploy.yml +++ b/infrastructure/nomad/playbooks/deploy.yml @@ -13,10 +13,6 @@ ansible.builtin.set_fact: system_arch: "{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}" - - name: Include Variables - include_vars: - file: vars.yml - - name: Check Operating System of the Target System assert: that: @@ -24,6 +20,10 @@ fail_msg: "This playbook only supports Debian systems." success_msg: "Operating system is supported." + - name: Include Variables + include_vars: + file: vars.yml + - name: Load AWS Caller Information amazon.aws.aws_caller_info: register: aws_caller_info diff --git a/infrastructure/nomad/playbooks/init.yml b/infrastructure/nomad/playbooks/init.yml index 49200a015..b73d874d8 100644 --- a/infrastructure/nomad/playbooks/init.yml +++ b/infrastructure/nomad/playbooks/init.yml @@ -18,10 +18,6 @@ ansible.builtin.set_fact: system_arch: "{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}" - - name: Include Variables - include_vars: - file: vars.yml - - name: Check Operating System of the Target System assert: that: @@ -29,31 +25,6 @@ fail_msg: "This playbook only supports Debian systems." success_msg: "Operating system is supported." - - name: Load AWS Caller Information - amazon.aws.aws_caller_info: - register: aws_caller_info - delegate_to: localhost - become: true - become_user: "{{ lookup('env', 'USER') }}" - - - name: Check AWS Caller Information - ansible.builtin.assert: - that: - - aws_caller_info is defined - - aws_caller_info.user_id is defined - - aws_caller_info.user_id | length > 0 - fail_msg: "AWS caller information is invalid or empty." - success_msg: "AWS caller information is valid." - - - name: Check Profile - ansible.builtin.assert: - that: - - profile is defined - - profile != '' - - profile in profiles - fail_msg: "The profile variable is not set correctly." - success_msg: "The profile variable is set to: {{ profile }}." - - name: Install Common Utility Packages and Dependencies ansible.builtin.apt: name: @@ -93,7 +64,35 @@ - cpuid - msr-tools state: present - ignore_errors: yes + + - name: Include Variables + include_vars: + file: vars.yml + + - name: Load AWS Caller Information + amazon.aws.aws_caller_info: + register: aws_caller_info + delegate_to: localhost + become: true + become_user: "{{ lookup('env', 'USER') }}" + + - name: Check AWS Caller Information + ansible.builtin.assert: + that: + - aws_caller_info is defined + - aws_caller_info.user_id is defined + - aws_caller_info.user_id | length > 0 + fail_msg: "AWS caller information is invalid or empty." + success_msg: "AWS caller information is valid." + + - name: Check Profile + ansible.builtin.assert: + that: + - profile is defined + - profile != '' + - profile in profiles + fail_msg: "The profile variable is not set correctly." + success_msg: "The profile variable is set to: {{ profile }}." tasks: - name: Add DataDog Repository Key @@ -204,6 +203,7 @@ validate_certs: no register: vault_init when: vault_status.json.initialized == false + no_log: true - name: Save Vault Unseal Key and Root Token ansible.builtin.copy: @@ -213,6 +213,7 @@ when: vault_status.json.initialized == false become: true become_user: "{{ hostvars[inventory_hostname].ansible_user }}" + no_log: true - name: Determine Vault Seal Status ansible.builtin.uri: @@ -231,6 +232,7 @@ - vault_seal_status.json.sealed == true become: true become_user: "{{ hostvars[inventory_hostname].ansible_user }}" + no_log: true - name: Parse Vault Initialization File ansible.builtin.set_fact: @@ -239,6 +241,7 @@ - vault_init_content is defined - vault_status.json.initialized == true - vault_seal_status.json.sealed == true + no_log: true - name: Unseal Vault ansible.builtin.uri: @@ -260,12 +263,14 @@ retries: 5 delay: 10 when: vault_seal_status.json.sealed == true + no_log: true - name: Fetch Secrets ansible.builtin.set_fact: mev_commit_secrets: "{{ lookup('amazon.aws.aws_secret', profile ~ '/' ~ vault_secret_path) }}" when: unseal_result.json.sealed == false delegate_to: localhost + no_log: true - name: Enable KV Secrets Engine ansible.builtin.uri: @@ -286,6 +291,7 @@ when: - vault_status.json.initialized == false - unseal_result.json.sealed == false + no_log: true - name: Write Retrieved Secret into Vault ansible.builtin.uri: @@ -299,6 +305,7 @@ status_code: [200, 204] validate_certs: no when: unseal_result.json.sealed == false + no_log: true - name: Install Nomad ansible.builtin.apt: @@ -350,6 +357,7 @@ mode: "0644" vars: vault_token: "{{ vault_init.json.root_token }}" + no_log: true - name: Restart and Enable Nomad Service ansible.builtin.systemd: diff --git a/infrastructure/nomad/playbooks/vars.yml b/infrastructure/nomad/playbooks/vars.yml index a207b2e1f..6e89d1aea 100644 --- a/infrastructure/nomad/playbooks/vars.yml +++ b/infrastructure/nomad/playbooks/vars.yml @@ -8,6 +8,20 @@ tls_key_file: "{{ private_keys_dir }}/{{ server_common_name }}.key" tls_ca_crt_file: "{{ ca_certificates_dir }}/ca-{{ server_common_name }}.crt" profiles: + ci: + job_names: + - "mev-commit-geth-bootnode1" + - "mev-commit-geth-signer-node1" + - "mev-commit-geth-member-node" + - "deploy-contracts" + - "mev-commit-bootnode1" + - "mev-commit-provider-node1" + - "mev-commit-provider-node1-funder" + - "mev-commit-oracle" + - "mev-commit-bidder-node1" + - "mev-commit-bidder-node1-funder" + - "mev-commit-provider-emulator-node1" + - "mev-commit-bidder-emulator-node1" devnet: job_names: - "datadog-agent-logs-collector"