feat: add stressnet profile with more nodes

.github/workflows/infrstructure.yml

@@ -14,6 +14,7 @@ on:
         options:
           - devnet
           - testnet
+          - stressnet
         default: 'devnet'
       init:
         description: 'Initialize'

infrastructure/nomad/playbooks/deploy.yml

@@ -229,6 +229,7 @@
         label: "{{ item.name }}"
       vars:
         job: "{{ item }}"
+        profile: "{{ profile }}"
       when: build_templates is not defined or build_templates
 
     - name: Create "version.txt"

infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2

@@ -94,7 +94,27 @@ job "{{ job.name }}" {
           {{- range nomadService "mev-commit-provider-node1" }}
             {{- if contains "metrics" .Tags }}
             - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics
-              service: "mev-commit-provider-node1"
+              service: {{ .Name }}
+              metrics:
+                - mev_commit*
+                - go*
+                - libp2p*
+            {{- end }}
+          {{- end }}
+          {{- range nomadService "mev-commit-provider-node2" }}
+            {{- if contains "metrics" .Tags }}
+            - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics
+              service: {{ .Name }}
+              metrics:
+                - mev_commit*
+                - go*
+                - libp2p*
+            {{- end }}
+          {{- end }}
+          {{- range nomadService "mev-commit-provider-node3" }}
+            {{- if contains "metrics" .Tags }}
+            - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics
+              service: {{ .Name }}
               metrics:
                 - mev_commit*
                 - go*
@@ -105,7 +125,47 @@ job "{{ job.name }}" {
           {{- range nomadService "mev-commit-bidder-node1" }}
             {{- if contains "metrics" .Tags }}
             - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics
-              service: "mev-commit-bidder-node1"
+              service: {{ .Name }}
+              metrics:
+                - mev_commit*
+                - go*
+                - libp2p*
+            {{- end }}
+          {{- end }}
+          {{- range nomadService "mev-commit-bidder-node2" }}
+            {{- if contains "metrics" .Tags }}
+            - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics
+              service: {{ .Name }}
+              metrics:
+                - mev_commit*
+                - go*
+                - libp2p*
+            {{- end }}
+          {{- end }}
+          {{- range nomadService "mev-commit-bidder-node3" }}
+            {{- if contains "metrics" .Tags }}
+            - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics
+              service: {{ .Name }}
+              metrics:
+                - mev_commit*
+                - go*
+                - libp2p*
+            {{- end }}
+          {{- end }}
+          {{- range nomadService "mev-commit-bidder-node4" }}
+            {{- if contains "metrics" .Tags }}
+            - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics
+              service: {{ .Name }}
+              metrics:
+                - mev_commit*
+                - go*
+                - libp2p*
+            {{- end }}
+          {{- end }}
+          {{- range nomadService "mev-commit-bidder-node5" }}
+            {{- if contains "metrics" .Tags }}
+            - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics
+              service: {{ .Name }}
               metrics:
                 - mev_commit*
                 - go*
@@ -196,7 +256,7 @@ job "{{ job.name }}" {
           {{- range nomadService "mev-commit-provider-emulator-node1" }}
             {{- if contains "metrics" .Tags }}
             - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics
-              service: "mev-commit-provider-emulator-node1"
+              service: {{ .Name }}
               metrics:
                 - mev_commit*
             {{- end }}
@@ -205,7 +265,24 @@ job "{{ job.name }}" {
           {{- range nomadService "mev-commit-bidder-emulator-node1" }}
             {{- if contains "metrics" .Tags }}
             - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics
-              service: "mev-commit-bidder-emulator-node1"
+              service: {{ .Name }}
+              metrics:
+                - mev_commit*
+            {{- end }}
+          {{- end }}
+          {{- range nomadService "mev-commit-provider-emulator-nodes" }}
+            {{- if contains "metrics" .Tags }}
+            - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics
+              service: {{ .Name }}
+              metrics:
+                - mev_commit*
+            {{- end }}
+          {{- end }}
+
+          {{- range nomadService "mev-commit-bidder-emulator-nodes" }}
+            {{- if contains "metrics" .Tags }}
+            - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics
+              service: {{ .Name }}
               metrics:
                 - mev_commit*
             {{- end }}

infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2

@@ -49,8 +49,10 @@ job "{{ job.name }}" {
             if job.get('env') and job.env.get('log-tags')
             else 'service:' + job.name + '-{{ env "NOMAD_ALLOC_INDEX" }}'
           }}"
+
           {%- raw %}
-          {{- range nomadService "{% endraw %}{{ job.target_name }}{% raw %}" }}
+          {{- $idx := add (env "NOMAD_ALLOC_INDEX" | parseInt) 1 }}
+          {{- range nomadService (printf "%s%d" "{% endraw %}{{ job.target_name }}{% raw %}" $idx) }}
             {{- if contains "rpc" .Tags }}
           EMULATOR_IP_PORT="{{ .Address }}:{{ .Port }}"
             {{- end }}

infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2

@@ -43,7 +43,8 @@ job "{{ job.name }}" {
             {{ end }}
           {{- end }}
 
-          {{- range nomadService "{% endraw %}{{ job.target_name }}{% raw %}" }}
+          {{- $idx := add (env "NOMAD_ALLOC_INDEX" | parseInt) 1 }}
+          {{- range nomadService (printf "%s%d" "{% endraw %}{{ job.target_name }}{% raw %}" $idx) }}
             {{- if contains "http" .Tags }}
           START_TIME=$(date +%s)
           TIMEOUT=60

infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2

@@ -36,6 +36,12 @@ job "{{ job.name }}" {
       }
       {% endfor %}
 
+      {% if profile == 'stressnet' %}
+      resources {
+        memory = 16384
+      }
+      {% endif %}
+
       artifact {
         source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/mev-commit-geth_{{ version }}_Linux_x86_64.tar.gz"
       }

infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2

@@ -47,6 +47,12 @@ job "{{ job.name }}" {
         sidecar = true
       }
 
+      {% if profile == 'stressnet' %}
+      resources {
+        memory = 4096
+      }
+      {% endif %}
+
       template {
         data = <<-EOH
           {%- raw %}
@@ -119,6 +125,12 @@ job "{{ job.name }}" {
         source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/contracts_{{ version }}.tar.gz"
       }
 
+      {% if profile == 'stressnet' %}
+      resources {
+        memory = 1024
+      }
+      {% endif %}
+
       template {
         data = <<-EOH
           KEYSTOREGEN_LOG_FMT="{{ job.env.get('log-format', 'json') }}"
@@ -144,7 +156,7 @@ job "{{ job.name }}" {
           MEV_ORACLE_HTTP_PORT="{{ .Port }}"
             {{- end }}
           {{- end }}
-          {{- range nomadService "mev-commit-geth-bootnode1" }}
+          {{- range nomadService "mev-commit-geth-member-node" }}
             {{- if contains "http" .Tags }}
           MEV_ORACLE_SETTLEMENT_RPC_URL_HTTP="http://{{ .Address }}:{{ .Port }}"
             {{- end }}
@@ -171,23 +183,53 @@ job "{{ job.name }}" {
         data = <<-EOH
           #!/usr/bin/env bash
           {%- raw %}
+          ADDRESSES = ""
           {{- range nomadService "mev-commit-provider-node1" }}
             {{ if contains "http" .Tags }}
           TOPOLOGY=$(curl https://{{ .Address}}:{{ .Port }}/v1/debug/topology)
           PROVIDER_ETHEREUM_ADDRESS=$(echo ${TOPOLOGY} | jq -r '.topology.self["Ethereum Address"]')
-          export MEV_ORACLE_OVERRIDE_WINNERS=${PROVIDER_ETHEREUM_ADDRESS}
+          if [ -z "${ADDRESSES}" ]; then
+            ADDRESSES="${PROVIDER_ETHEREUM_ADDRESS}"
+          else
+            ADDRESSES="${ADDRESSES},${PROVIDER_ETHEREUM_ADDRESS}"
+          fi
             {{ end }}
           {{- end }}
-          {% endraw %}
+          {{- range nomadService "mev-commit-provider-node2" }}
+            {{ if contains "http" .Tags }}
+          TOPOLOGY=$(curl https://{{ .Address}}:{{ .Port }}/v1/debug/topology)
+          PROVIDER_ETHEREUM_ADDRESS=$(echo ${TOPOLOGY} | jq -r '.topology.self["Ethereum Address"]')
+          if [ -z "${ADDRESSES}" ]; then
+            ADDRESSES="${PROVIDER_ETHEREUM_ADDRESS}"
+          else
+            ADDRESSES="${ADDRESSES},${PROVIDER_ETHEREUM_ADDRESS}"
+          fi
+            {{ end }}
+          {{- end }}
+          {{- range nomadService "mev-commit-provider-node3" }}
+            {{ if contains "http" .Tags }}
+          TOPOLOGY=$(curl https://{{ .Address}}:{{ .Port }}/v1/debug/topology)
+          PROVIDER_ETHEREUM_ADDRESS=$(echo ${TOPOLOGY} | jq -r '.topology.self["Ethereum Address"]')
+          if [ -z "${ADDRESSES}" ]; then
+            ADDRESSES="${PROVIDER_ETHEREUM_ADDRESS}"
+          else
+            ADDRESSES="${ADDRESSES},${PROVIDER_ETHEREUM_ADDRESS}"
+          fi
+            {{ end }}
+          {{- end }}  
 
-          {%- raw %}
           {{- range nomadService "datadog-agent-logs-collector" }}
             {{ if contains "tcp" .Tags }}
           exec > >(nc {{ .Address }} {{ .Port }}) 2>&1
             {{ end }}
           {{- end }}
           {% endraw %}
 
+          if [ ! -z "${ADDRESSES}" ]; then
+            echo "starting mev-oracle with override winners ${ADDRESSES}"
+            export MEV_ORACLE_OVERRIDE_WINNERS=${ADDRESSES}
+          fi
+
           if [ ! -d "${MEV_ORACLE_KEYSTORE_PATH}" ]; then
             mkdir -p "${MEV_ORACLE_KEYSTORE_PATH}" > /dev/null 2>&1
             mv local/contracts/deployer_keystore/* "${MEV_ORACLE_KEYSTORE_PATH}/"

infrastructure/nomad/playbooks/templates/jobs/mev-commit.nomad.j2

@@ -44,6 +44,12 @@ job "{{ job.name }}" {
         source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/keystore-generator_{{ version }}_Linux_x86_64.tar.gz"
       }
 
+      {% if profile == 'stressnet' %}
+      resources {
+        memory = 1024
+      }
+      {% endif %}
+
       template {
         data = <<-EOH
           KEYSTOREGEN_LOG_FMT="{{ job.env.get('log-format', 'json') }}"