refactor: link all geth nodes using static node configuration (#408)

primev · Sep 24, 2024 · a1d9f63 · a1d9f63
1 parent 3aa76a7
commit a1d9f63
Show file tree

Hide file tree

Showing 6 changed files with 59 additions and 14 deletions.
diff --git a/.github/workflows/infrastructure.yml b/.github/workflows/infrastructure.yml
@@ -143,6 +143,7 @@ jobs:
           sudo add-apt-repository --yes ppa:ethereum/ethereum
           sudo apt-get update
           sudo apt-get install --yes goreleaser ethereum
+          sudo snap install remarshal
           python3 -m venv primevenv
           source primevenv/bin/activate
           pip install boto3 botocore

diff --git a/external/geth b/external/geth
diff --git a/infrastructure/nomad/cluster.sh b/infrastructure/nomad/cluster.sh
@@ -100,6 +100,7 @@ check_deps() {
         flock
         ansible
         bootnode
+        remarshal
         goreleaser
     )
     for util in "${required_utilities[@]}"; do

diff --git a/infrastructure/nomad/playbooks/deploy.yml b/infrastructure/nomad/playbooks/deploy.yml
@@ -217,12 +217,14 @@
         SIGNERS="{{ dist_dir }}/signers.txt"
         INVENTORY="{{ dist_dir }}/artifacts.txt"
         ALLOCATIONS="{{ dist_dir }}/allocations.txt"
+        STATIC_NODES="{{ dist_dir }}/geth_static_nodes.json"
         DESTINATION_DIR="{{ dist_dir }}/{{ item.name }}" && mkdir -p ${DESTINATION_DIR}
 
         [ ! -f "${SECRETS}" ] && echo '{}' > "${SECRETS}"
         [ ! -f "${SIGNERS}" ] && touch "${SIGNERS}"
         [ ! -f "${INVENTORY}" ] && touch "${INVENTORY}"
         [ ! -f "${ALLOCATIONS}" ] && touch "${ALLOCATIONS}"
+        [ ! -f "${STATIC_NODES}" ] && echo '{"Node":{"P2P":{"StaticNodes":[]}}}' > "${STATIC_NODES}"
 
         exec 192>>"${SIGNERS}"
         exec 193<>"${INVENTORY}"
@@ -302,21 +304,34 @@
               {% endif %}
             {% endif %}
 
-            {% if artifact.boot_key | default(false) %}
-              bootnode -genkey "${DESTINATION_DIR}/boot.key"
+            {% if artifact.nodekey is defined %}
+              bootnode -genkey "${DESTINATION_DIR}/node.key"
+              if [ $? -ne 0 ]; then exit 1; fi
+
+              nodekey_address=$(bootnode -nodekey "${DESTINATION_DIR}/node.key" -writeaddress)
               if [ $? -ne 0 ]; then exit 1; fi
 
               cat "${SECRETS}" | jq \
-                --arg item_name "{{ (item.name | regex_replace('^mev-commit-', '') | replace('-', '_')) }}" \
-                --arg boot_key "$(cat ${DESTINATION_DIR}/boot.key)" \
-                --arg boot_key_address "$(bootnode -nodekey ${DESTINATION_DIR}/boot.key -writeaddress)" \
+                --arg nodekey "$(cat ${DESTINATION_DIR}/node.key)" \
+                --arg nodekey_address "${nodekey_address}" \
                 '. + {
-                  ($item_name + "_boot_key"): $boot_key,
-                  ($item_name + "_boot_key_address"): $boot_key_address
+                  "{{ artifact.nodekey.name }}": $nodekey,
+                  "{{ artifact.nodekey.name }}_address": $nodekey_address
                 }' \
               > "${SECRETS}.tmp" && mv "${SECRETS}.tmp" "${SECRETS}"
               if [ $? -ne 0 ]; then
-                echo "Error: Failed to write boot key to secrets for {{ item.name }}."
+                echo "Error: Failed to write node key to secrets for {{ item.name }}."
+                exit 1;
+              fi
+
+              cat "${STATIC_NODES}" | jq \
+                --arg nodekey_address "${nodekey_address}" \
+                --arg target_machine_ip "{{ ansible_facts['default_ipv4']['address'] }}" \
+                --arg target_machine_port "{{ item.ports[0].p2p.static }}" \
+                '.Node.P2P.StaticNodes += ["enode://\($nodekey_address)@\($target_machine_ip):\($target_machine_port)"]' \
+              > "${STATIC_NODES}.tmp" && mv "${STATIC_NODES}.tmp" "${STATIC_NODES}"
+              if [ $? -ne 0 ]; then
+                echo "Error: Failed to write enode to static nodes for {{ item.name }}."
                 exit 1; 
               fi
             {% endif %}
@@ -419,6 +434,20 @@
       run_once: true
       when: build_artifacts
 
+    - name: Assemble config.toml for Geth Static Nodes
+      ansible.builtin.shell: |
+        STATIC_NODES="{{ dist_dir }}/geth_static_nodes.json"
+        if [ ! -f "${STATIC_NODES}" ]; then
+          echo "Error: No static nodes found."
+          exit 1
+        fi
+        cat "${STATIC_NODES}" | remarshal --if json --of toml - > "{{ dist_dir }}/config_{{ environments[env].version }}.toml"
+      args:
+        executable: bash
+      delegate_to: localhost
+      run_once: true
+      when: build_artifacts
+
     - name: Fetch Secrets
       ansible.builtin.set_fact:
         mev_commit_secrets: "{{ lookup('amazon.aws.aws_secret', env ~ '/' ~ profile ~ '/' ~ vault_secret_path) }}"

diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2
@@ -198,13 +198,19 @@ job "{{ job.name }}" {
       {% endfor %}
 
       {% if env != 'devenv' %}
+      artifact {
+        source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/config_{{ version }}.toml"
+      }
       artifact {
         source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/genesis_{{ version }}.json"
       }
       artifact {
         source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/mev-commit-geth_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz"
       }
       {% else %}
+      artifact {
+        source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/config_{{ version }}.toml"
+      }
       artifact {
         source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/genesis_{{ version }}.json"
       }
@@ -218,6 +224,7 @@ job "{{ job.name }}" {
           {%- raw %}
           GETH_DATA_DIR="/alloc/data/node-{{ env "NOMAD_ALLOC_INDEX" }}"
           {% endraw %}
+          GETH_CONFIG="local/config_{{ version }}.toml"
           GENESIS_L1_PATH="local/genesis_{{ version }}.json"
           GETH_BIN_PATH="local/mev-commit-geth"
           NODE_IP="{{ job.env['ip'] }}"
@@ -229,11 +236,10 @@ job "{{ job.name }}" {
           GETH_VERBOSITY={{ job.env.get('log-verbosity', '3') }}
           GETH_LOG_FORMAT="{{ job.env.get('log-format', 'json') }}"
           GETH_LOG_TAGS="{{ 'service.name:' + job.name + '-{{ env "NOMAD_ALLOC_INDEX" }}' + ',service.version:' + version }}"
-          {% if job.env['type'] == 'bootnode' %}
           {%- raw %}
-          BOOT_KEY="{{ with secret "secret/data/mev-commit" }}{{ .Data.data.geth_bootnode1_boot_key }}{{ end }}"
+          NODE_KEY="{{ with secret "secret/data/mev-commit" }}{{ .Data.data.{% endraw %}{{ job.artifacts | selectattr('nodekey', 'defined') | map(attribute='nodekey.name') | first }}{% raw %} }}{{ end }}"
           {% endraw %}
-          {% elif job.env['type'] == 'signer' %}
+          {% if job.env['type'] == 'signer' %}
           {%- raw %}
           GETH_KEYSTORE_DIR="/alloc/data/node-{{ env "NOMAD_ALLOC_INDEX" }}/keystore"
           GETH_KEYSTORE_FILENAME="{{ with secret "secret/data/mev-commit" }}{{ .Data.data.{% endraw %}{{ job.artifacts | selectattr('keystore', 'defined') | map(attribute='keystore.name') | first }}{% raw %}_filename }}{{ end }}"
@@ -248,7 +254,7 @@ job "{{ job.name }}" {
           {%- raw %}
           {{- range nomadService "mev-commit-geth-bootnode1" }}
             {{- if contains "p2p" .Tags }}
-          BOOTNODE_ENDPOINT="enode://{{ with secret "secret/data/mev-commit" }}{{ .Data.data.geth_bootnode1_boot_key_address }}{{ end }}@{{ .Address }}:{{ .Port }}"
+          BOOTNODE_ENDPOINT="enode://{{ with secret "secret/data/mev-commit" }}{{ .Data.data.{% endraw %}{{ job.artifacts | selectattr('nodekey', 'defined') | map(attribute='nodekey.name') | first }}_address{% raw %} }}{{ end }}@{{ .Address }}:{{ .Port }}"
             {{- end }}
           {{- end }}
           {% endraw %}

diff --git a/infrastructure/nomad/playbooks/variables/profiles.yml b/infrastructure/nomad/playbooks/variables/profiles.yml
@@ -44,7 +44,8 @@ jobs:
     count: 1
     artifacts:
       - *geth_artifact
-      - boot_key: true
+      - nodekey:
+          name: geth_bootnode1_nodekey
     ports:
       - metrics:
           to: 6060
@@ -55,6 +56,7 @@ jobs:
           static: 8546
           to: 8546
         p2p:
+          static: 30301
           to: 30301
     env:
       ip: 0.0.0.0
@@ -68,6 +70,8 @@ jobs:
     template: mev-commit-geth.nomad.j2
     artifacts:
       - *geth_artifact
+      - nodekey:
+          name: geth_signer_node1_nodekey
       - keystore:
           name: geth_signer1_keystore
           allocation: true
@@ -76,6 +80,7 @@ jobs:
       - metrics:
           to: 6060
         p2p:
+          static: 30311
           to: 30311
     env:
       ip: 0.0.0.0
@@ -88,6 +93,8 @@ jobs:
     template: mev-commit-geth.nomad.j2
     artifacts:
       - *geth_artifact
+      - nodekey:
+          name: geth_member_node_nodekey
     count: 1
     ports:
       - metrics:
@@ -99,6 +106,7 @@ jobs:
           static: 8556
           to: 8546
         p2p:
+          static: 30321
           to: 30311
     env:
       ip: 0.0.0.0