From ce89b4ab1ec230482b67363658a3af18abd48513 Mon Sep 17 00:00:00 2001 From: Mikhail Wall Date: Thu, 12 Dec 2024 18:50:51 +0100 Subject: [PATCH] feat: added geth pos --- .github/workflows/ci.yml | 7 +- .github/workflows/releaser.yml | 2 +- external/geth-pos | 2 +- infrastructure/nomad/playbooks/deploy.yml | 73 +++++++ .../jobs/mev-commit-geth-pos.nomad.j2 | 200 ++++++++++++++++++ 5 files changed, 279 insertions(+), 5 deletions(-) create mode 100644 infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth-pos.nomad.j2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff9260a1f..2720d2109 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,8 @@ jobs: WORKSPACE_MODULES=$(go list -f '{{.Dir}}' -m) ADDITIONAL_MODULES=( - "${GITHUB_WORKSPACE}/external/geth" + "${GITHUB_WORKSPACE}/external/geth", + "${GITHUB_WORKSPACE}/external/geth-pos", ) ALL_MODULES=$(printf "%s\n" "${WORKSPACE_MODULES}" "${ADDITIONAL_MODULES[@]}") @@ -62,7 +63,7 @@ jobs: echo "GO_MODULES<> ${GITHUB_ENV} echo "${ALL_MODULES}" >> ${GITHUB_ENV} echo "EOF" >> ${GITHUB_ENV} - echo "GO_LINT_MODULES=$(printf "%s " $(echo "${ALL_MODULES}" | sed 's|$|/...|' | grep -v '/external/geth'))" >> ${GITHUB_ENV} + echo "GO_LINT_MODULES=$(printf "%s " $(echo "${ALL_MODULES}" | sed 's|$|/...|' | grep -v '/external/geth' | grep -v '/external/geth-pos' ))" >> ${GITHUB_ENV} - name: Run Gofmt run: | @@ -93,7 +94,7 @@ jobs: GOWORK: off - name: Run Test - run: echo ${GO_MODULES} | tr ' ' '\n' | grep -v '/external/geth' | xargs -I {} sh -c 'go test -short -race {}/...' + run: echo ${GO_MODULES} | tr ' ' '\n' | grep -v '/external/geth' | grep -v '/external/geth-pos' | xargs -I {} sh -c 'go test -short -race {}/...' - name: Setup Protobuf uses: bufbuild/buf-setup-action@v1.31.0 diff --git a/.github/workflows/releaser.yml b/.github/workflows/releaser.yml index f097a498d..f49d49938 100644 --- a/.github/workflows/releaser.yml +++ b/.github/workflows/releaser.yml @@ -15,7 +15,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - module: [ bridge/standard, external/geth, oracle, p2p, tools/bidder-cli ] + module: [ bridge/standard, external/geth, external/geth-pos, oracle, p2p, tools/bidder-cli ] steps: - if: ${{ !startsWith(github.ref, 'refs/tags/v') }} diff --git a/external/geth-pos b/external/geth-pos index ae42ab4cf..15cef9171 160000 --- a/external/geth-pos +++ b/external/geth-pos @@ -1 +1 @@ -Subproject commit ae42ab4cf5cc6af23749c6cc53161aa065778710 +Subproject commit 15cef9171c7def0c58e39458095c5d0374c036c4 diff --git a/infrastructure/nomad/playbooks/deploy.yml b/infrastructure/nomad/playbooks/deploy.yml index 3240c727f..e67644792 100644 --- a/infrastructure/nomad/playbooks/deploy.yml +++ b/infrastructure/nomad/playbooks/deploy.yml @@ -9,6 +9,7 @@ build_artifacts: false build_templates: false genesis_file_url: "" + genesis_pos_file_url: "" otel_collector_endpoint_url: "" aws_s3_bucket: "primev-infrastructure-artifacts" @@ -550,6 +551,78 @@ run_once: true when: build_artifacts and genesis_file_url | trim | length > 0 + - name: Assemble Genesis File For Pos Geth + ansible.builtin.shell: | + ALLOCATIONS="$(cat {{ dist_dir }}/allocations.txt 2>/dev/null)" + + ALLOC=$(echo '{}' | jq '.') + for ADDRESS in ${ALLOCATIONS}; do + ALLOC=$( + echo "${ALLOC}" | jq --arg address "0x${ADDRESS}" ' + . + { + ($address): { + "balance": "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD4A51000FDA0FFFF" + } + } + ' + ) + done + + ADDRESS_COUNT=0 + + echo $(cat <<-EOH + { + "config": { + "chainId": {{ environments[env].chain_id }}, + "homesteadBlock": 0, + "daoForkSupport": true, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "muirGlacierBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "arrowGlacierBlock": 0, + "grayGlacierBlock": 0, + "mergeNetsplitBlock": 0,4 + "shanghaiTime": 0, + "cancunTime": 0, + "terminalTotalDifficulty": 0, + "terminalTotalDifficultyPassed": true + }, + "nonce": "0x0", + {% if environments[env].genesis_timestamp is defined %} + "timestamp": "{{ environments[env].genesis_timestamp }}", + {% else %} + "timestamp": "0x$(printf "%x" "$(date +%s)")", + {% endif %} + "difficulty": "0x1", + "gasLimit": "0x1c9c380", + "coinbase": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "alloc": ${ALLOC}, + } + EOH + ) | jq -c '.' > "{{ dist_dir }}/genesis_pos_{{ environments[env].version }}.json" + args: + executable: bash + delegate_to: localhost + run_once: true + when: build_artifacts and genesis_pos_file_url | trim | length == 0 + + - name: Fetch Genesis File for PoS Geth + ansible.builtin.uri: + url: "{{ genesis_pos_file_url }}" + dest: "{{ dist_dir }}/genesis_pos_{{ environments[env].version }}.json" + return_content: no + delegate_to: localhost + run_once: true + when: build_artifacts and genesis_pos_file_url | trim | length > 0 + - name: Prepare launchmevcommit Script ansible.builtin.shell: | SCRIPT="{{ dist_dir }}/launchmevcommit-{{ environments[env].version }}.sh" diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth-pos.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth-pos.nomad.j2 new file mode 100644 index 000000000..069c428ac --- /dev/null +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth-pos.nomad.j2 @@ -0,0 +1,200 @@ +#jinja2: trim_blocks:True, lstrip_blocks:True +job "{{ job.name }}" { + datacenters = ["{{ datacenter }}"] + + group "{{ job.name }}-group" { + count = {{ job.count }} + + {% if env == 'devenv' %} + restart { + attempts = 0 + mode = "fail" + } + + reschedule { + attempts = 0 + unlimited = false + } + {% endif %} + + network { + mode = "bridge" + + dns { + servers = {{ (ansible_facts['dns']['nameservers'] + ['1.1.1.1']) | tojson }} + } + + {% for port_name, port_details in job.ports[0].items() %} + port "{{ port_name }}" { + {% if port_details.get('static') %} + static = {{ port_details['static'] }} + {% endif %} + {% if port_details.get('to') %} + to = {{ port_details['to'] }} + {% endif %} + } + {% endfor %} + } + + volume "data-volume" { + type = "host" + source = "data-volume" + read_only = false + } + + task "node" { + driver = "exec" + kill_timeout = "25s" + + {% if profile == 'testnet' or profile == 'archive' %} + resources { + cores = 6 + memory = 32768 + } + {% elif profile == 'stressnet' or profile == 'stressnet-wl1' %} + resources { + cpu = 2000 + memory = 16384 + } + {% endif %} + + volume_mount { + volume = "data-volume" + destination = "/local/data" + read_only = false + } + + {% for port_name in job.ports[0] %} + service { + name = "{{ job.name }}" + port = "{{ port_name }}" + tags = ["{{ port_name }}"] + provider = "nomad" + } + {% endfor %} + + {% if env != 'devenv' %} + artifact { + source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/config_{{ version }}.toml" + } + artifact { + source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/genesis_pos_{{ version }}.json" + } + artifact { + source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/mev-commit-geth_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz" + } + {% else %} + artifact { + source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/config_{{ version }}.toml" + } + artifact { + source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/genesis_{{ version }}.json" + } + artifact { + source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/mev-commit-geth_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz" + } + {% endif %} + + template { + data = <<-EOH + {%- raw %} + GETH_DATA_DIR="/local/data/{% endraw %}{{ job.name }}{% raw %}/node-{{ env "NOMAD_ALLOC_INDEX" }}" + {% endraw %} + GETH_CONFIG="local/config_{{ version }}.toml" + GENESIS_L1_PATH="local/genesis_{{ version }}.json" + GETH_BIN_PATH="local/mev-commit-geth" + NODE_IP="{{ job.env['ip'] }}" + {% if job.env['type'] != 'signer' %} + PUBLIC_NODE_IP="{{ job.env['public_ip'] }}" + {% endif %} + NET_RESTRICT="{{ job.env['net_restrict'] }}" + GETH_NODE_TYPE="{{ job.env['type'] }}" + GETH_VERBOSITY={{ job.env.get('log-verbosity', '3') }} + GETH_LOG_FORMAT="{{ job.env.get('log-format', 'json') }}" + GETH_LOG_TAGS="{{ 'service.name:' + job.name + '-{{ env "NOMAD_ALLOC_INDEX" }}' + ',service.version:' + version }}" + {% if job.env['type'] != 'archive' %} + {%- raw %} + NODE_KEY="{{ with secret "secret/data/mev-commit" }}{{ .Data.data.{% endraw %}{{ job.artifacts | selectattr('nodekey', 'defined') | map(attribute='nodekey.name') | first }}{% raw %} }}{{ end }}" + {% endraw %} + {% endif %} + {% if job.env['type'] == 'signer' %} + {%- raw %} + GETH_KEYSTORE_DIR="/local/data/{% endraw %}{{ job.name }}{% raw %}/node-{{ env "NOMAD_ALLOC_INDEX" }}/keystore" + GETH_KEYSTORE_FILENAME="{{ with secret "secret/data/mev-commit" }}{{ .Data.data.{% endraw %}{{ job.artifacts | selectattr('keystores', 'defined') | map(attribute='keystores') | first | list | first }}{% raw %}_filename }}{{ end }}" + GETH_KEYSTORE_PASSWORD="{{ with secret "secret/data/mev-commit" }}{{ .Data.data.{% endraw %}{{ job.artifacts | selectattr('keystores', 'defined') | map(attribute='keystores') | first | list | first }}{% raw %}_password }}{{ end }}" + {% endraw %} + {% endif %} + GETH_SYNC_MODE="{{ job.env['sync_mode'] }}" + GETH_STATE_SCHEME="{% if profile == 'archive' %}hash{% else %}path{% endif %}" + {% if job.env['type'] != 'bootnode' %} + {% if geth_bootnode_url is defined %} + BOOTNODE_ENDPOINT="{{ geth_bootnode_url }}" + {% else %} + {%- raw %} + {{- range nomadService "mev-commit-geth-bootnode1" }} + {{- if contains "p2p" .Tags }} + BOOTNODE_ENDPOINT="enode://{{ with secret "secret/data/mev-commit" }}{{ .Data.data.geth_bootnode1_nodekey_address }}{{ end }}@{{ .Address }}:{{ .Port }}" + {{- end }} + {{- end }} + {% endraw %} + {% endif %} + {% endif %} + EOH + destination = "secrets/.env" + env = true + } + + template { + data = <<-EOH + #!/usr/bin/env bash + + {%- raw %} + {{- range nomadService "datadog-agent-logs-collector" }} + {{ if contains "tcp" .Tags }} + exec > >(nc {{ .Address }} {{ .Port }}) 2>&1 + {{ end }} + {{- end }} + {% endraw %} + + mkdir -p "${GETH_DATA_DIR}" > /dev/null 2>&1 + + {% if job.artifacts | selectattr('keystores', 'defined') | list | length > 0 %} + mkdir -p "${GETH_KEYSTORE_DIR}" > /dev/null 2>&1 + {%- raw %} + {{- with secret "secret/data/mev-commit" }} + echo '{{ .Data.data.{% endraw %}{{ job.artifacts | selectattr('keystores', 'defined') | map(attribute='keystores') | first | list | first }}{% raw %} }}' > "${GETH_KEYSTORE_DIR}/${GETH_KEYSTORE_FILENAME}" + {{ end }} + {% endraw %} + {% if job.env['type'] == 'signer' %} + export BLOCK_SIGNER_ADDRESS="$(cat ${GETH_KEYSTORE_DIR}/${GETH_KEYSTORE_FILENAME} | jq -r '.address')" + {% endif %} + {% endif %} + + {%- raw %} + {{ with secret "secret/data/mev-commit" }} + GETH_ZERO_FEE_ADDRESSES="$(echo '{{ .Data.data.contract_deployer_keystore_filename }}' | awk -F'--' '{print "0x"$3}')" + {{ end }} + {{ with secret "secret/data/mev-commit" }} + GETH_ZERO_FEE_ADDRESSES+=",$(echo '{{ .Data.data.oracle_keystore_filename }}' | awk -F'--' '{print "0x"$3}')" + {{ end }} + {{ with secret "secret/data/mev-commit" }} + GETH_ZERO_FEE_ADDRESSES+=",$(echo '{{ .Data.data.bridge_relayer_keystore_filename }}' | awk -F'--' '{print "0x"$3}')" + {{ end }} + {% endraw %} + + export GETH_ZERO_FEE_ADDRESSES + chmod +x local/mev-commit-geth local/entrypoint.sh + exec local/entrypoint.sh + EOH + destination = "local/run.sh" + change_mode = "noop" + perms = "0755" + } + + config { + command = "bash" + args = ["-c", "exec local/run.sh"] + } + } + } +}