infrastructure #1689
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: infrastructure | |
| on: | |
| workflow_run: | |
| workflows: | |
| - ci | |
| types: | |
| - completed | |
| workflow_dispatch: | |
| inputs: | |
| profile: | |
| description: 'Profile' | |
| type: choice | |
| options: | |
| - devnet | |
| - testnet | |
| - stressnet | |
| default: 'devnet' | |
| logs: | |
| description: 'Collect Logs' | |
| type: boolean | |
| default: false | |
| debug: | |
| description: 'Debug Deployment' | |
| type: boolean | |
| default: false | |
| target: | |
| description: 'Target Machine' | |
| type: choice | |
| options: | |
| - lax1 | |
| - nyc1 | |
| - nyc2 | |
| - mia2 | |
| default: 'lax' | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs.target }} | |
| cancel-in-progress: true | |
| jobs: | |
| cluster: | |
| name: Setup and Test Nomad Cluster | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 180 | |
| if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} | |
| steps: | |
| - name: Setup Environment | |
| run: | | |
| echo "${{ secrets.INFRASTRUCTURE_DNS_RECORDS }}" | sudo tee -a /etc/hosts | |
| sudo systemctl restart systemd-resolved | |
| IS_MANUAL_DEPLOYMENT=$([ "${{ github.event_name }}" == "workflow_dispatch" ] && echo true || echo false) | |
| CLUSTER_ENVIRONMENT_FLAG="--environment devenv" | |
| CLUSTER_PROFILE_FLAG=$([ "${IS_MANUAL_DEPLOYMENT}" == "true" ] && echo "--profile ${{ github.event.inputs.profile }}" || echo "--profile ci") | |
| CLUSTER_LOGS_FLAG=$([ "${{ github.event.inputs.logs }}" == "false" ] && echo "--no-logs-collection" || echo "") | |
| CLUSTER_DATADOG_KEY_FLAG=$([ "${IS_MANUAL_DEPLOYMENT}" == "true" ] && echo "--datadog-key ${{ secrets.DATADOG_API_KEY }}" || echo "") | |
| CLUSTER_DEBUG_FLAG=$([ "${{ github.event.inputs.debug }}" == "true" ] && echo "--debug" || echo "") | |
| TARGET_MACHINE_IP=$([ "${IS_MANUAL_DEPLOYMENT}" == "true" ] && echo "$(dig +short ${{ github.event.inputs.target }})" || echo "127.0.0.1") | |
| echo "IS_MANUAL_DEPLOYMENT=${IS_MANUAL_DEPLOYMENT}" >> ${GITHUB_ENV} | |
| echo "CLUSTER_ENVIRONMENT_FLAG=${CLUSTER_ENVIRONMENT_FLAG}" >> ${GITHUB_ENV} | |
| echo "CLUSTER_PROFILE_FLAG=${CLUSTER_PROFILE_FLAG}" >> ${GITHUB_ENV} | |
| echo "CLUSTER_LOGS_FLAG=${CLUSTER_LOGS_FLAG}" >> ${GITHUB_ENV} | |
| echo "CLUSTER_DATADOG_KEY_FLAG=${CLUSTER_DATADOG_KEY_FLAG}" >> ${GITHUB_ENV} | |
| echo "CLUSTER_DEBUG_FLAG=${CLUSTER_DEBUG_FLAG}" >> ${GITHUB_ENV} | |
| echo "TARGET_MACHINE_IP=${TARGET_MACHINE_IP}" >> ${GITHUB_ENV} | |
| - name: Notify - Deployment Initialized | |
| if: ${{ env.IS_MANUAL_DEPLOYMENT == 'true' }} | |
| run: | | |
| WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| PAYLOAD=$(cat <<-EOH | |
| { | |
| "text": "<@${{ github.actor }}> - deployment to <http://${TARGET_MACHINE_IP}:4646/ui|*${{ github.event.inputs.target }}*> has been initialized", | |
| "attachments": [ | |
| { | |
| "color": "#0000FF", | |
| "fields": [ | |
| { | |
| "title": "Workflow", | |
| "value": "<${WORKFLOW_URL}|View Workflow Run>", | |
| "short": false | |
| }, | |
| ] | |
| } | |
| ] | |
| } | |
| EOH | |
| ) | |
| curl -X POST -H 'Content-type: application/json' --data "${PAYLOAD}" "${{ secrets.SLACK_CI_CHANNEL_WEBHOOK_URL }}" | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| ref: ${{ github.event.workflow_run.head_branch }} | |
| - name: Setup Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/go/pkg/mod | |
| ~/.cache/go-build | |
| ~/.local/pipx | |
| /usr/local/bin/goreleaser | |
| key: ${{ runner.os }}-go-${{ hashFiles('**/go.work.sum') }} | |
| restore-keys: ${{ runner.os }}-go- | |
| - name: Setup Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: 1.22 | |
| check-latest: true | |
| cache-dependency-path: go.work.sum | |
| - name: Install Required Dependencies | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| run: | | |
| echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | sudo tee /etc/apt/sources.list.d/goreleaser.list | |
| sudo add-apt-repository --yes ppa:ethereum/ethereum | |
| sudo apt-get update | |
| sudo apt-get install --yes goreleaser ethereum | |
| pip install boto3 botocore | |
| pipx inject ansible-core botocore boto3 | |
| - name: Configure Control Machine | |
| run: | | |
| sudo useradd --create-home ubuntu && sudo usermod --append --groups sudo ubuntu | |
| ANSIBLE_CONNECTION="ansible_connection=local" | |
| if [ "${IS_MANUAL_DEPLOYMENT}" == "true" ]; then | |
| ANSIBLE_CONNECTION="" | |
| export ANSIBLE_HOST_KEY_CHECKING=false | |
| mkdir -p ~/.ssh && \ | |
| chmod 700 ~/.ssh && \ | |
| echo "${{ secrets.INFRASTRUCTURE_DEPLOYMENT_KEY }}" > ~/.ssh/id_ed25519 && \ | |
| chmod 600 ~/.ssh/id_ed25519 | |
| fi | |
| aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws configure set default.region us-west-2 | |
| cp infrastructure/nomad/ansible.cfg.example infrastructure/nomad/ansible.cfg | |
| cat <<-EOH > infrastructure/nomad/hosts.ini | |
| [nomad_servers] | |
| ${TARGET_MACHINE_IP} ${ANSIBLE_CONNECTION} ansible_user=ubuntu | |
| [nomad_clients] | |
| ${TARGET_MACHINE_IP} ${ANSIBLE_CONNECTION} ansible_user=ubuntu | |
| EOH | |
| ansible all --inventory infrastructure/nomad/hosts.ini --module-name ping | |
| - name: Destroy Existing Cluster | |
| if: ${{ env.IS_MANUAL_DEPLOYMENT == 'true' }} | |
| run: | | |
| ./cluster.sh destroy ${CLUSTER_DEBUG_FLAG} | |
| working-directory: infrastructure/nomad | |
| - name: Initialize Cluster | |
| if: ${{ env.IS_MANUAL_DEPLOYMENT == 'false' }} | |
| run: | | |
| START_TIME="$(date +%s)" | |
| ./cluster.sh init ${CLUSTER_ENVIRONMENT_FLAG} ${CLUSTER_PROFILE_FLAG} ${CLUSTER_DEBUG_FLAG} | |
| END_TIME="$(date +%s)" | |
| echo "INIT_DURATION=$(date -ud "@$((END_TIME - START_TIME))" +'%H:%M:%S')" >> ${GITHUB_ENV} | |
| working-directory: infrastructure/nomad | |
| - name: Deploy Cluster | |
| run: | | |
| START_TIME="$(date +%s)" | |
| ./cluster.sh deploy ${CLUSTER_ENVIRONMENT_FLAG} ${CLUSTER_PROFILE_FLAG} ${CLUSTER_LOGS_FLAG} ${CLUSTER_DATADOG_KEY_FLAG} ${CLUSTER_DEBUG_FLAG} | |
| END_TIME="$(date +%s)" | |
| echo "DEPLOY_DURATION=$(date -ud "@$((END_TIME - START_TIME))" +'%H:%M:%S')" >> ${GITHUB_ENV} | |
| working-directory: infrastructure/nomad | |
| - name: Notify - Deployment Successful | |
| if: ${{ env.IS_MANUAL_DEPLOYMENT == 'true' && success() }} | |
| run: | | |
| DEPLOY_DURATION=${DEPLOY_DURATION:-N/A} | |
| RUNNER_DURATION=$(date -ud "@$(( $(date +%s) - $(date +%s -r /proc/$(pgrep -o -x "runner")/stat) ))" +'%H:%M:%S') | |
| WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| PAYLOAD=$(cat <<-EOH | |
| { | |
| "text": "<@${{ github.actor }}> - deployment to <http://${TARGET_MACHINE_IP}:4646/ui|*${{ github.event.inputs.target }}*> was successful", | |
| "attachments": [ | |
| { | |
| "color": "#00FF00", | |
| "fields": [ | |
| { | |
| "title": "Workflow", | |
| "value": "<${WORKFLOW_URL}|View Workflow Run>", | |
| "short": false | |
| }, | |
| { | |
| "title": "Deployment Duration", | |
| "value": "Deploy: ${DEPLOY_DURATION}\nRunner: ${RUNNER_DURATION}", | |
| "short": false | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| EOH | |
| ) | |
| curl -X POST -H 'Content-type: application/json' --data "${PAYLOAD}" "${{ secrets.SLACK_CI_CHANNEL_WEBHOOK_URL }}" | |
| - name: Notify - Deployment Failed | |
| if: ${{ env.IS_MANUAL_DEPLOYMENT == 'true' && failure() }} | |
| run: | | |
| WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| PAYLOAD=$(cat <<-EOH | |
| { | |
| "text": "<@${{ github.actor }}> - deployment to <http://${TARGET_MACHINE_IP}:4646/ui|*${{ github.event.inputs.target }}*> has failed", | |
| "attachments": [ | |
| { | |
| "color": "#FF0000", | |
| "fields": [ | |
| { | |
| "title": "Workflow", | |
| "value": "<${WORKFLOW_URL}|View Workflow Run>", | |
| "short": false | |
| }, | |
| ] | |
| } | |
| ] | |
| } | |
| EOH | |
| ) | |
| curl -X POST -H 'Content-type: application/json' --data "${PAYLOAD}" "${{ secrets.SLACK_CI_CHANNEL_WEBHOOK_URL }}" | |
| - name: Initialize Debug Shell | |
| if: ${{ env.IS_MANUAL_DEPLOYMENT == 'false' && failure() }} | |
| run: | | |
| TUNSHELL_KEYS=$(curl -sSf -X POST https://eu.relay.tunshell.com/api/sessions) | |
| DEBUG_SHELL="sh <(curl -sSf https://lets.tunshell.com/init.sh) L $(echo ${TUNSHELL_KEYS} | jq -r .peer2_key) \${TUNSHELL_SECRET} eu.relay.tunshell.com" | |
| WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| PR_TITLE="$(jq -r .head_commit.message <<< '${{ toJson(github.event.workflow_run) }}')" | |
| PR_URL="$(jq -r '.head_commit.url // ""' <<< '${{ toJson(github.event.workflow_run) }}')" | |
| PAYLOAD=$(cat <<-EOH | |
| { | |
| "text": "<@${{ github.actor }}> infrastructure workflow has failed:", | |
| "attachments": [ | |
| { | |
| "color": "#FF0000", | |
| "fields": [ | |
| { | |
| "title": "Workflow", | |
| "value": "<${WORKFLOW_URL}|View Workflow Run>", | |
| "short": false | |
| }, | |
| $( [ -n "$PR_URL" ] && cat <<-PULL_REQUEST | |
| { | |
| "title": "Pull Request", | |
| "value": "<${PR_URL}|${PR_TITLE}>", | |
| "short": false | |
| }, | |
| PULL_REQUEST | |
| ) | |
| { | |
| "title": "Debug Shell", | |
| "value": "\`\`\`${DEBUG_SHELL}\`\`\`", | |
| "short": false | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| EOH | |
| ) | |
| echo "Debug Shell: ${DEBUG_SHELL}" | |
| curl -X POST -H 'Content-type: application/json' --data "${PAYLOAD}" "${{ secrets.SLACK_CI_CHANNEL_WEBHOOK_URL }}" | |
| curl -sSf https://lets.tunshell.com/init.sh | sh -s -- T $(echo ${TUNSHELL_KEYS} | jq -r .peer1_key) ${{ secrets.TUNSHELL_SECRET }} eu.relay.tunshell.com |