From 913962679f1e9d7b27cf90f72086e11d71c6cc55 Mon Sep 17 00:00:00 2001 From: Alok Date: Fri, 31 May 2024 20:15:31 +0530 Subject: [PATCH 01/39] feat: add stressnet profile with more nodes --- .../templates/jobs/datadog-agent.nomad.j2 | 75 ++++--- .../jobs/mev-commit-emulator.nomad.j2 | 2 +- .../templates/jobs/mev-commit-funder.nomad.j2 | 2 +- infrastructure/nomad/playbooks/vars.yml | 203 +++++++++++++++++- 4 files changed, 240 insertions(+), 42 deletions(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 index 0cf4be0f7..5192cb27e 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 @@ -26,6 +26,13 @@ job "{{ job.name }}" { {% endif %} } + meta { + providers = "mev-commit-provider-node1,mev-commit-provider-node2,mev-commit-provider-node3" + bidders = "mev-commit-bidder-node1,mev-commit-bidder-node2,mev-commit-bidder-node3,mev-commit-bidder-node4,mev-commit-bidder-node5" + provider_emulators = "mev-commit-provider-emulator-node1,mev-commit-provider-emulator-node2,mev-commit-provider-emulator-node3" + bidder_emulators = "mev-commit-bidder-emulator-node1,mev-commit-bidder-emulator-node2,mev-commit-bidder-emulator-node3,mev-commit-bidder-emulator-node4,mev-commit-bidder-emulator-node5" + } + task "datadog-agent" { driver = "exec" @@ -91,25 +98,29 @@ job "{{ job.name }}" { {{- end }} {{- end }} - {{- range nomadService "mev-commit-provider-node1" }} - {{- if contains "metrics" .Tags }} - - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics - service: "mev-commit-provider-node1" - metrics: - - mev_commit* - - go* - - libp2p* + {{- range split (env "NOMAD_META_PROVIDERS") "," }} + {{- range nomadService . }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics + service: {{ .Name }} + metrics: + - mev_commit* + - go* + - libp2p* + {{- end }} {{- end }} - {{- end }} - - {{- range nomadService "mev-commit-bidder-node1" }} - {{- if contains "metrics" .Tags }} - - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics - service: "mev-commit-bidder-node1" - metrics: - - mev_commit* - - go* - - libp2p* + {{- end}} + + {{- range split (env "NOMAD_META_BIDDERS") "," }} + {{- range nomadService . }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics + service: {{ .Name }} + metrics: + - mev_commit* + - go* + - libp2p* + {{- end }} {{- end }} {{- end }} @@ -193,21 +204,25 @@ job "{{ job.name }}" { {{- end }} {{- end }} - {{- range nomadService "mev-commit-provider-emulator-node1" }} - {{- if contains "metrics" .Tags }} - - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics - service: "mev-commit-provider-emulator-node1" - metrics: - - mev_commit* + {{- range split (env "NOMAD_META_PROVIDER_EMULATORS") "," }} + {{- range nomadService . }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics + service: {{ .Name }} + metrics: + - mev_commit* + {{- end }} {{- end }} {{- end }} - {{- range nomadService "mev-commit-bidder-emulator-node1" }} - {{- if contains "metrics" .Tags }} - - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics - service: "mev-commit-bidder-emulator-node1" - metrics: - - mev_commit* + {{- range split (env "NOMAD_META_BIDDER_EMULATORS") "," }} + {{- range nomadService . }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics + service: {{ .Name }} + metrics: + - mev_commit* + {{- end }} {{- end }} {{- end }} {% endraw %} diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 index 53cb63b73..126adab4e 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 @@ -50,7 +50,7 @@ job "{{ job.name }}" { else 'service:' + job.name + '-{{ env "NOMAD_ALLOC_INDEX" }}' }}" {%- raw %} - {{- range nomadService "{% endraw %}{{ job.target_name }}{% raw %}" }} + {{- range nomadService (print "{% endraw %}{{ job.target_name }}{% raw %}" (env "NOMAD_ALLOC_INDEX")) }} {{- if contains "rpc" .Tags }} EMULATOR_IP_PORT="{{ .Address }}:{{ .Port }}" {{- end }} diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 index f4d994e07..2fada9cca 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 @@ -43,7 +43,7 @@ job "{{ job.name }}" { {{ end }} {{- end }} - {{- range nomadService "{% endraw %}{{ job.target_name }}{% raw %}" }} + {{- range nomadService (print "{% endraw %}{{ job.target_name }}{% raw %}" (env "NOMAD_ALLOC_INDEX")) }} {{- if contains "http" .Tags }} START_TIME=$(date +%s) TIMEOUT=60 diff --git a/infrastructure/nomad/playbooks/vars.yml b/infrastructure/nomad/playbooks/vars.yml index 3973364e3..84ced8ea3 100644 --- a/infrastructure/nomad/playbooks/vars.yml +++ b/infrastructure/nomad/playbooks/vars.yml @@ -55,6 +55,29 @@ profiles: - "mev-commit-oracle" - "datadog-agent-metrics-collector" - "mev-commit-faucet" + stressnet: + job_names: + - "datadog-agent-logs-collector" + - "mev-commit-geth-bootnode1" + - "mev-commit-geth-signer-node1" + - "mev-commit-geth-member-node" + - "deploy-contracts" + - "mev-commit-bootnode1" + - "mev-commit-provider-node1" + - "mev-commit-provider-node2" + - "mev-commit-provider-node3" + - "mev-commit-provider-node-funders" + - "mev-commit-provider-emulator-nodes" + - "mev-commit-oracle" + - "mev-commit-bidder-node1" + - "mev-commit-bidder-node2" + - "mev-commit-bidder-node3" + - "mev-commit-bidder-node4" + - "mev-commit-bidder-node5" + - "mev-commit-bidder-node-funders" + - "mev-commit-bidder-emulator-nodes" + - "datadog-agent-metrics-collector" + - "mev-commit-faucet" artifacts: - name: keystore-generator @@ -172,14 +195,58 @@ jobs: keystore: provider1 ports: - metrics: + to: 13623 + http: + static: 13623 to: 13523 + p2p: + to: 13522 + rpc: + static: 13624 + to: 13524 + env: + type: provider + nat_address: "{{ ansible_facts['default_ipv4']['address'] }}" + tls_crt_file: "{{ tls_crt_file }}" + tls_key_file: "{{ tls_key_file }}" + + - name: mev-commit-provider-node2 + template: mev-commit.nomad.j2 + count: 1 + secrets: + keystore: provider1 + ports: + - metrics: + to: 13633 + http: + static: 13633 + to: 13523 + p2p: + to: 13522 + rpc: + static: 13634 + to: 13524 + env: + type: provider + nat_address: "{{ ansible_facts['default_ipv4']['address'] }}" + tls_crt_file: "{{ tls_crt_file }}" + tls_key_file: "{{ tls_key_file }}" + + - name: mev-commit-provider-node3 + template: mev-commit.nomad.j2 + count: 1 + secrets: + keystore: provider1 + ports: + - metrics: + to: 13643 http: - static: 13533 + static: 13643 to: 13523 p2p: to: 13522 rpc: - static: 13534 + static: 13644 to: 13524 env: type: provider @@ -193,6 +260,30 @@ jobs: target_type: provider target_name: mev-commit-provider-node1 + - name: mev-commit-provider-emulator-node1 + template: mev-commit-emulator.nomad.j2 + count: 1 + target_type: provider + target_name: mev-commit-provider-node1 + ports: + - metrics: + to: 8080 + + - name: mev-commit-provider-node-funders + template: mev-commit-funder.nomad.j2 + count: 3 + target_type: provider + target_name: mev-commit-provider-node + + - name: mev-commit-provider-emulator-nodes + template: mev-commit-emulator.nomad.j2 + count: 3 + target_type: provider + target_name: mev-commit-provider-node + ports: + - metrics: + to: 8080 + - name: mev-commit-oracle template: mev-commit-oracle.nomad.j2 count: 1 @@ -209,6 +300,27 @@ jobs: l1_rpc_url: https://eth-sepolia.g.alchemy.com/v2/ui86u7ZUgpn2e-WHFbHQHk2s-6Odjggx - name: mev-commit-bidder-node1 + template: mev-commit.nomad.j2 + count: 1 + secrets: + keystore: bidder1 + ports: + - metrics: + to: 13723 + http: + static: 13723 + to: 13523 + p2p: + to: 13522 + rpc: + static: 13724 + to: 13524 + env: + type: bidder + tls_crt_file: "{{ tls_crt_file }}" + tls_key_file: "{{ tls_key_file }}" + + - name: mev-commit-bidder-node2 template: mev-commit.nomad.j2 count: 1 secrets: @@ -217,12 +329,75 @@ jobs: - metrics: to: 13523 http: - static: 13532 + static: 13733 to: 13523 p2p: to: 13522 rpc: - static: 13544 + static: 13734 + to: 13524 + env: + type: bidder + tls_crt_file: "{{ tls_crt_file }}" + tls_key_file: "{{ tls_key_file }}" + + - name: mev-commit-bidder-node3 + template: mev-commit.nomad.j2 + count: 1 + secrets: + keystore: bidder1 + ports: + - metrics: + to: 13743 + http: + static: 13743 + to: 13523 + p2p: + to: 13522 + rpc: + static: 13744 + to: 13524 + env: + type: bidder + tls_crt_file: "{{ tls_crt_file }}" + tls_key_file: "{{ tls_key_file }}" + + - name: mev-commit-bidder-node4 + template: mev-commit.nomad.j2 + count: 1 + secrets: + keystore: bidder1 + ports: + - metrics: + to: 13753 + http: + static: 13753 + to: 13523 + p2p: + to: 13522 + rpc: + static: 13754 + to: 13524 + env: + type: bidder + tls_crt_file: "{{ tls_crt_file }}" + tls_key_file: "{{ tls_key_file }}" + + - name: mev-commit-bidder-node5 + template: mev-commit.nomad.j2 + count: 1 + secrets: + keystore: bidder1 + ports: + - metrics: + to: 13763 + http: + static: 13763 + to: 13523 + p2p: + to: 13522 + rpc: + static: 13764 to: 13524 env: type: bidder @@ -235,20 +410,28 @@ jobs: target_type: bidder target_name: mev-commit-bidder-node1 - - name: mev-commit-provider-emulator-node1 + - name: mev-commit-bidder-emulator-node1 template: mev-commit-emulator.nomad.j2 count: 1 - target_type: provider - target_name: mev-commit-provider-node1 + target_type: bidder + target_name: mev-commit-bidder-node1 ports: - metrics: to: 8080 + env: + l1_rpc_url: https://eth-sepolia.g.alchemy.com/v2/ui86u7ZUgpn2e-WHFbHQHk2s-6Odjggx - - name: mev-commit-bidder-emulator-node1 + - name: mev-commit-bidder-node-funders + template: mev-commit-funder.nomad.j2 + count: 5 + target_type: bidder + target_name: mev-commit-bidder-node + + - name: mev-commit-bidder-emulator-nodes template: mev-commit-emulator.nomad.j2 - count: 1 + count: 5 target_type: bidder - target_name: mev-commit-bidder-node1 + target_name: mev-commit-bidder-node ports: - metrics: to: 8080 From 374211647c542e078dfe7d5ebaba0512da25eaf2 Mon Sep 17 00:00:00 2001 From: Alok Date: Fri, 31 May 2024 20:19:48 +0530 Subject: [PATCH 02/39] feat: add stressnet profile with more nodes --- .github/workflows/infrstructure.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/infrstructure.yml b/.github/workflows/infrstructure.yml index 300cf3733..7091bc664 100644 --- a/.github/workflows/infrstructure.yml +++ b/.github/workflows/infrstructure.yml @@ -14,6 +14,7 @@ on: options: - devnet - testnet + - stressnet default: 'devnet' init: description: 'Initialize' From b4a2f8797982b8d369fba39d8163c987d7c82bda Mon Sep 17 00:00:00 2001 From: Alok Date: Sat, 1 Jun 2024 01:20:12 +0530 Subject: [PATCH 03/39] fix: add 1 to allocation index --- .../playbooks/templates/jobs/mev-commit-funder.nomad.j2 | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 index 2fada9cca..381885730 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 @@ -36,6 +36,12 @@ job "{{ job.name }}" { chmod +x ${XDG_CONFIG_HOME}/.foundry/bin/foundryup && ${XDG_CONFIG_HOME}/.foundry/bin/foundryup export PATH="${XDG_CONFIG_HOME}/.foundry/bin:$PATH" + nomad_alloc_index=$NOMAD_ALLOC_INDEX + + index_plus_one=$((nomad_alloc_index + 1)) + service_name="{{ job.target_name }}${index_plus_one}" + echo "Service name: $service_name" + {% raw %} {{- range nomadService "datadog-agent-logs-collector" }} {{ if contains "tcp" .Tags }} @@ -43,7 +49,7 @@ job "{{ job.name }}" { {{ end }} {{- end }} - {{- range nomadService (print "{% endraw %}{{ job.target_name }}{% raw %}" (env "NOMAD_ALLOC_INDEX")) }} + {{- range nomadService $service_name }} {{- if contains "http" .Tags }} START_TIME=$(date +%s) TIMEOUT=60 From 11131ed0f238e4f4980025261434f70885b2378d Mon Sep 17 00:00:00 2001 From: Alok Date: Sat, 1 Jun 2024 01:23:16 +0530 Subject: [PATCH 04/39] fix: add 1 to allocation index --- .../playbooks/templates/jobs/mev-commit-emulator.nomad.j2 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 index 126adab4e..b42538aa7 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 @@ -49,8 +49,13 @@ job "{{ job.name }}" { if job.get('env') and job.env.get('log-tags') else 'service:' + job.name + '-{{ env "NOMAD_ALLOC_INDEX" }}' }}" + nomad_alloc_index=$NOMAD_ALLOC_INDEX + + index_plus_one=$((nomad_alloc_index + 1)) + service_name="{{ job.target_name }}${index_plus_one}" + echo "Service name: $service_name" {%- raw %} - {{- range nomadService (print "{% endraw %}{{ job.target_name }}{% raw %}" (env "NOMAD_ALLOC_INDEX")) }} + {{- range nomadService $service_name }} {{- if contains "rpc" .Tags }} EMULATOR_IP_PORT="{{ .Address }}:{{ .Port }}" {{- end }} From 4fbd7c2bcf2c69d41d811b9c560f29eeee1f66c7 Mon Sep 17 00:00:00 2001 From: Alok Date: Sat, 1 Jun 2024 01:42:49 +0530 Subject: [PATCH 05/39] fix: add 1 to allocation index --- .../playbooks/templates/jobs/mev-commit-emulator.nomad.j2 | 3 ++- .../nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 index b42538aa7..7f9a19f58 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 @@ -54,8 +54,9 @@ job "{{ job.name }}" { index_plus_one=$((nomad_alloc_index + 1)) service_name="{{ job.target_name }}${index_plus_one}" echo "Service name: $service_name" + {{% endraw %}} {%- raw %} - {{- range nomadService $service_name }} + {{- range nomadService "${service_name}" }} {{- if contains "rpc" .Tags }} EMULATOR_IP_PORT="{{ .Address }}:{{ .Port }}" {{- end }} diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 index 381885730..9bba4fdfd 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 @@ -41,7 +41,7 @@ job "{{ job.name }}" { index_plus_one=$((nomad_alloc_index + 1)) service_name="{{ job.target_name }}${index_plus_one}" echo "Service name: $service_name" - + {% endraw %} {% raw %} {{- range nomadService "datadog-agent-logs-collector" }} {{ if contains "tcp" .Tags }} @@ -49,7 +49,7 @@ job "{{ job.name }}" { {{ end }} {{- end }} - {{- range nomadService $service_name }} + {{- range nomadService ${service_name} }} {{- if contains "http" .Tags }} START_TIME=$(date +%s) TIMEOUT=60 From e11311ba85485ef79dc7cfd180c8136604cd1cc4 Mon Sep 17 00:00:00 2001 From: Alok Date: Sat, 1 Jun 2024 01:55:46 +0530 Subject: [PATCH 06/39] fix: add 1 to allocation index --- .../nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 index 7f9a19f58..68a7d85da 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 @@ -54,7 +54,7 @@ job "{{ job.name }}" { index_plus_one=$((nomad_alloc_index + 1)) service_name="{{ job.target_name }}${index_plus_one}" echo "Service name: $service_name" - {{% endraw %}} + {% endraw %} {%- raw %} {{- range nomadService "${service_name}" }} {{- if contains "rpc" .Tags }} From dd1d602490f5da61af4a0ec1b896b8283bbccca3 Mon Sep 17 00:00:00 2001 From: Alok Date: Sat, 1 Jun 2024 02:04:57 +0530 Subject: [PATCH 07/39] fix: add 1 to allocation index --- .../playbooks/templates/jobs/mev-commit-emulator.nomad.j2 | 1 - .../nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 index 68a7d85da..b6d452f3d 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 @@ -54,7 +54,6 @@ job "{{ job.name }}" { index_plus_one=$((nomad_alloc_index + 1)) service_name="{{ job.target_name }}${index_plus_one}" echo "Service name: $service_name" - {% endraw %} {%- raw %} {{- range nomadService "${service_name}" }} {{- if contains "rpc" .Tags }} diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 index 9bba4fdfd..91a50a603 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 @@ -41,7 +41,6 @@ job "{{ job.name }}" { index_plus_one=$((nomad_alloc_index + 1)) service_name="{{ job.target_name }}${index_plus_one}" echo "Service name: $service_name" - {% endraw %} {% raw %} {{- range nomadService "datadog-agent-logs-collector" }} {{ if contains "tcp" .Tags }} @@ -49,7 +48,7 @@ job "{{ job.name }}" { {{ end }} {{- end }} - {{- range nomadService ${service_name} }} + {{- range nomadService "${service_name}" }} {{- if contains "http" .Tags }} START_TIME=$(date +%s) TIMEOUT=60 From df973b840170bd0d223bfb4c39b568bf0e3c31d3 Mon Sep 17 00:00:00 2001 From: Alok Date: Sat, 1 Jun 2024 02:33:02 +0530 Subject: [PATCH 08/39] fix: add 1 to allocation index --- .../templates/jobs/mev-commit-emulator.nomad.j2 | 8 +++----- .../playbooks/templates/jobs/mev-commit-funder.nomad.j2 | 9 +++------ 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 index b6d452f3d..7290a99ec 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 @@ -49,13 +49,11 @@ job "{{ job.name }}" { if job.get('env') and job.env.get('log-tags') else 'service:' + job.name + '-{{ env "NOMAD_ALLOC_INDEX" }}' }}" - nomad_alloc_index=$NOMAD_ALLOC_INDEX - index_plus_one=$((nomad_alloc_index + 1)) - service_name="{{ job.target_name }}${index_plus_one}" - echo "Service name: $service_name" {%- raw %} - {{- range nomadService "${service_name}" }} + {{- $idx := env "NOMAD_ALLOC_INDEX" | parseInt }} + {{- $idx++ }} + {{- range nomadService (printf "%s%d" "{{ job.target_name }}" $idx) }} {{- if contains "rpc" .Tags }} EMULATOR_IP_PORT="{{ .Address }}:{{ .Port }}" {{- end }} diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 index 91a50a603..081f33c57 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 @@ -36,11 +36,6 @@ job "{{ job.name }}" { chmod +x ${XDG_CONFIG_HOME}/.foundry/bin/foundryup && ${XDG_CONFIG_HOME}/.foundry/bin/foundryup export PATH="${XDG_CONFIG_HOME}/.foundry/bin:$PATH" - nomad_alloc_index=$NOMAD_ALLOC_INDEX - - index_plus_one=$((nomad_alloc_index + 1)) - service_name="{{ job.target_name }}${index_plus_one}" - echo "Service name: $service_name" {% raw %} {{- range nomadService "datadog-agent-logs-collector" }} {{ if contains "tcp" .Tags }} @@ -48,7 +43,9 @@ job "{{ job.name }}" { {{ end }} {{- end }} - {{- range nomadService "${service_name}" }} + {{- $idx := env "NOMAD_ALLOC_INDEX" | parseInt }} + {{- $idx++ }} + {{- range nomadService (printf "%s%d" "{{ job.target_name }}" $idx) }} {{- if contains "http" .Tags }} START_TIME=$(date +%s) TIMEOUT=60 From 10bf86459c14d4aa833a636afc6686acd29fcfc2 Mon Sep 17 00:00:00 2001 From: Alok Date: Sat, 1 Jun 2024 02:48:24 +0530 Subject: [PATCH 09/39] fix: add 1 to allocation index --- .../playbooks/templates/jobs/mev-commit-emulator.nomad.j2 | 3 +-- .../nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 index 7290a99ec..febca765b 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 @@ -51,8 +51,7 @@ job "{{ job.name }}" { }}" {%- raw %} - {{- $idx := env "NOMAD_ALLOC_INDEX" | parseInt }} - {{- $idx++ }} + {{- $idx := add (env "NOMAD_ALLOC_INDEX" | parseInt) 1 }} {{- range nomadService (printf "%s%d" "{{ job.target_name }}" $idx) }} {{- if contains "rpc" .Tags }} EMULATOR_IP_PORT="{{ .Address }}:{{ .Port }}" diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 index 081f33c57..0dfc5fe80 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 @@ -43,8 +43,7 @@ job "{{ job.name }}" { {{ end }} {{- end }} - {{- $idx := env "NOMAD_ALLOC_INDEX" | parseInt }} - {{- $idx++ }} + {{- $idx := add (env "NOMAD_ALLOC_INDEX" | parseInt) 1 }} {{- range nomadService (printf "%s%d" "{{ job.target_name }}" $idx) }} {{- if contains "http" .Tags }} START_TIME=$(date +%s) From 3e79d287f2f111d73f3b1e9039a481ea35813db4 Mon Sep 17 00:00:00 2001 From: Alok Date: Sat, 1 Jun 2024 02:58:40 +0530 Subject: [PATCH 10/39] fix: restore datadog --- .../templates/jobs/datadog-agent.nomad.j2 | 75 ++++++++----------- 1 file changed, 30 insertions(+), 45 deletions(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 index 5192cb27e..0cf4be0f7 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 @@ -26,13 +26,6 @@ job "{{ job.name }}" { {% endif %} } - meta { - providers = "mev-commit-provider-node1,mev-commit-provider-node2,mev-commit-provider-node3" - bidders = "mev-commit-bidder-node1,mev-commit-bidder-node2,mev-commit-bidder-node3,mev-commit-bidder-node4,mev-commit-bidder-node5" - provider_emulators = "mev-commit-provider-emulator-node1,mev-commit-provider-emulator-node2,mev-commit-provider-emulator-node3" - bidder_emulators = "mev-commit-bidder-emulator-node1,mev-commit-bidder-emulator-node2,mev-commit-bidder-emulator-node3,mev-commit-bidder-emulator-node4,mev-commit-bidder-emulator-node5" - } - task "datadog-agent" { driver = "exec" @@ -98,29 +91,25 @@ job "{{ job.name }}" { {{- end }} {{- end }} - {{- range split (env "NOMAD_META_PROVIDERS") "," }} - {{- range nomadService . }} - {{- if contains "metrics" .Tags }} - - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics - service: {{ .Name }} - metrics: - - mev_commit* - - go* - - libp2p* - {{- end }} + {{- range nomadService "mev-commit-provider-node1" }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics + service: "mev-commit-provider-node1" + metrics: + - mev_commit* + - go* + - libp2p* {{- end }} - {{- end}} - - {{- range split (env "NOMAD_META_BIDDERS") "," }} - {{- range nomadService . }} - {{- if contains "metrics" .Tags }} - - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics - service: {{ .Name }} - metrics: - - mev_commit* - - go* - - libp2p* - {{- end }} + {{- end }} + + {{- range nomadService "mev-commit-bidder-node1" }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics + service: "mev-commit-bidder-node1" + metrics: + - mev_commit* + - go* + - libp2p* {{- end }} {{- end }} @@ -204,25 +193,21 @@ job "{{ job.name }}" { {{- end }} {{- end }} - {{- range split (env "NOMAD_META_PROVIDER_EMULATORS") "," }} - {{- range nomadService . }} - {{- if contains "metrics" .Tags }} - - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics - service: {{ .Name }} - metrics: - - mev_commit* - {{- end }} + {{- range nomadService "mev-commit-provider-emulator-node1" }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics + service: "mev-commit-provider-emulator-node1" + metrics: + - mev_commit* {{- end }} {{- end }} - {{- range split (env "NOMAD_META_BIDDER_EMULATORS") "," }} - {{- range nomadService . }} - {{- if contains "metrics" .Tags }} - - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics - service: {{ .Name }} - metrics: - - mev_commit* - {{- end }} + {{- range nomadService "mev-commit-bidder-emulator-node1" }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics + service: "mev-commit-bidder-emulator-node1" + metrics: + - mev_commit* {{- end }} {{- end }} {% endraw %} From 0d1188df8ebeaa49f0694cd1d7fc352341a219e3 Mon Sep 17 00:00:00 2001 From: Alok Date: Sat, 1 Jun 2024 03:00:58 +0530 Subject: [PATCH 11/39] fix: restore datadog --- .../templates/jobs/datadog-agent.nomad.j2 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 index 0cf4be0f7..64102741d 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 @@ -91,10 +91,10 @@ job "{{ job.name }}" { {{- end }} {{- end }} - {{- range nomadService "mev-commit-provider-node1" }} + {{- range nomadService "mev-commit-provider-node" }} {{- if contains "metrics" .Tags }} - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics - service: "mev-commit-provider-node1" + service: {{ .Name }} metrics: - mev_commit* - go* @@ -102,10 +102,10 @@ job "{{ job.name }}" { {{- end }} {{- end }} - {{- range nomadService "mev-commit-bidder-node1" }} + {{- range nomadService "mev-commit-bidder-node" }} {{- if contains "metrics" .Tags }} - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics - service: "mev-commit-bidder-node1" + service: {{ .Name }} metrics: - mev_commit* - go* @@ -193,19 +193,19 @@ job "{{ job.name }}" { {{- end }} {{- end }} - {{- range nomadService "mev-commit-provider-emulator-node1" }} + {{- range nomadService "mev-commit-provider-emulator-node" }} {{- if contains "metrics" .Tags }} - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics - service: "mev-commit-provider-emulator-node1" + service: {{ .Name }} metrics: - mev_commit* {{- end }} {{- end }} - {{- range nomadService "mev-commit-bidder-emulator-node1" }} + {{- range nomadService "mev-commit-bidder-emulator-node" }} {{- if contains "metrics" .Tags }} - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics - service: "mev-commit-bidder-emulator-node1" + service: {{ .Name }} metrics: - mev_commit* {{- end }} From 9e6104bcb75af5c419332b9af208a201dc904fa9 Mon Sep 17 00:00:00 2001 From: Alok Date: Sat, 1 Jun 2024 03:04:15 +0530 Subject: [PATCH 12/39] fix: raw tags --- .../nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 | 2 +- .../nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 index febca765b..7282c0eb4 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-emulator.nomad.j2 @@ -52,7 +52,7 @@ job "{{ job.name }}" { {%- raw %} {{- $idx := add (env "NOMAD_ALLOC_INDEX" | parseInt) 1 }} - {{- range nomadService (printf "%s%d" "{{ job.target_name }}" $idx) }} + {{- range nomadService (printf "%s%d" "{% endraw %}{{ job.target_name }}{% raw %}" $idx) }} {{- if contains "rpc" .Tags }} EMULATOR_IP_PORT="{{ .Address }}:{{ .Port }}" {{- end }} diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 index 0dfc5fe80..e1227d5bb 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-funder.nomad.j2 @@ -44,7 +44,7 @@ job "{{ job.name }}" { {{- end }} {{- $idx := add (env "NOMAD_ALLOC_INDEX" | parseInt) 1 }} - {{- range nomadService (printf "%s%d" "{{ job.target_name }}" $idx) }} + {{- range nomadService (printf "%s%d" "{% endraw %}{{ job.target_name }}{% raw %}" $idx) }} {{- if contains "http" .Tags }} START_TIME=$(date +%s) TIMEOUT=60 From 3eb613a9c9da068ce059b7d4d06d5e283e4f1588 Mon Sep 17 00:00:00 2001 From: Alok Date: Sat, 1 Jun 2024 20:49:58 +0530 Subject: [PATCH 13/39] fix: metrics endpoint and add winners --- .../templates/jobs/mev-commit-oracle.nomad.j2 | 8 ++++++-- infrastructure/nomad/playbooks/vars.yml | 14 +++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 index 3e4c86138..d0bc6e8ea 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 @@ -171,13 +171,17 @@ job "{{ job.name }}" { data = <<-EOH #!/usr/bin/env bash {%- raw %} - {{- range nomadService "mev-commit-provider-node1" }} + ADDRESSES=() + + {{- range nomadService "mev-commit-provider-node" }} {{ if contains "http" .Tags }} TOPOLOGY=$(curl https://{{ .Address}}:{{ .Port }}/v1/debug/topology) PROVIDER_ETHEREUM_ADDRESS=$(echo ${TOPOLOGY} | jq -r '.topology.self["Ethereum Address"]') - export MEV_ORACLE_OVERRIDE_WINNERS=${PROVIDER_ETHEREUM_ADDRESS} + ADDRESSES+=(${PROVIDER_ETHEREUM_ADDRESS}) {{ end }} {{- end }} + + export MEV_ORACLE_OVERRIDE_WINNERS=$(IFS=','; echo "${ADDRESSES[*]}") {% endraw %} {%- raw %} diff --git a/infrastructure/nomad/playbooks/vars.yml b/infrastructure/nomad/playbooks/vars.yml index 84ced8ea3..1792346ff 100644 --- a/infrastructure/nomad/playbooks/vars.yml +++ b/infrastructure/nomad/playbooks/vars.yml @@ -195,7 +195,7 @@ jobs: keystore: provider1 ports: - metrics: - to: 13623 + to: 13523 http: static: 13623 to: 13523 @@ -217,7 +217,7 @@ jobs: keystore: provider1 ports: - metrics: - to: 13633 + to: 13523 http: static: 13633 to: 13523 @@ -239,7 +239,7 @@ jobs: keystore: provider1 ports: - metrics: - to: 13643 + to: 13523 http: static: 13643 to: 13523 @@ -306,7 +306,7 @@ jobs: keystore: bidder1 ports: - metrics: - to: 13723 + to: 13523 http: static: 13723 to: 13523 @@ -348,7 +348,7 @@ jobs: keystore: bidder1 ports: - metrics: - to: 13743 + to: 13523 http: static: 13743 to: 13523 @@ -369,7 +369,7 @@ jobs: keystore: bidder1 ports: - metrics: - to: 13753 + to: 13523 http: static: 13753 to: 13523 @@ -390,7 +390,7 @@ jobs: keystore: bidder1 ports: - metrics: - to: 13763 + to: 13523 http: static: 13763 to: 13523 From dca3e4e1d60add9f139631d5e5bda89ac62a2f58 Mon Sep 17 00:00:00 2001 From: Alok Date: Sat, 1 Jun 2024 21:15:05 +0530 Subject: [PATCH 14/39] fix: close handshake stream --- p2p/pkg/p2p/libp2p/libp2p.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/p2p/pkg/p2p/libp2p/libp2p.go b/p2p/pkg/p2p/libp2p/libp2p.go index 53da4d077..184e88d84 100644 --- a/p2p/pkg/p2p/libp2p/libp2p.go +++ b/p2p/pkg/p2p/libp2p/libp2p.go @@ -224,6 +224,8 @@ func (s *Service) handleConnectReq(streamlibp2p network.Stream) { peerID := streamlibp2p.Conn().RemotePeer() stream := newStream(streamlibp2p, nil, nil) + defer stream.Close() + peer, err := s.hsSvc.Handle(s.baseCtx, stream, peerID) if err != nil { s.logger.Error("error handling handshake", "err", err) @@ -426,6 +428,8 @@ func (s *Service) Connect(ctx context.Context, info []byte) (p2p.Peer, error) { } stream := newStream(streamlibp2p, nil, nil) + defer stream.Close() + p, err := s.hsSvc.Handshake(ctx, addrInfo.ID, stream) if err != nil { _ = s.host.Network().ClosePeer(addrInfo.ID) From d5cba88cb84b21a529aa70c8148075c72f3341a3 Mon Sep 17 00:00:00 2001 From: Alok Date: Sat, 1 Jun 2024 21:56:27 +0530 Subject: [PATCH 15/39] fix: oracle job --- .../templates/jobs/mev-commit-oracle.nomad.j2 | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 index d0bc6e8ea..a42c2415e 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 @@ -171,25 +171,23 @@ job "{{ job.name }}" { data = <<-EOH #!/usr/bin/env bash {%- raw %} - ADDRESSES=() + ADDRESSES="" + # Process mev-commit-provider-node {{- range nomadService "mev-commit-provider-node" }} {{ if contains "http" .Tags }} - TOPOLOGY=$(curl https://{{ .Address}}:{{ .Port }}/v1/debug/topology) - PROVIDER_ETHEREUM_ADDRESS=$(echo ${TOPOLOGY} | jq -r '.topology.self["Ethereum Address"]') - ADDRESSES+=(${PROVIDER_ETHEREUM_ADDRESS}) + TOPOLOGY=$(curl -s https://{{ .Address }}:{{ .Port }}/v1/debug/topology) + PROVIDER_ETHEREUM_ADDRESS=$(echo "${TOPOLOGY}" | jq -r '.topology.self["Ethereum Address"]') + if [ -z "${ADDRESSES}" ]; then + ADDRESSES="${PROVIDER_ETHEREUM_ADDRESS}" + else + ADDRESSES="${ADDRESSES},${PROVIDER_ETHEREUM_ADDRESS}" + fi {{ end }} {{- end }} - export MEV_ORACLE_OVERRIDE_WINNERS=$(IFS=','; echo "${ADDRESSES[*]}") - {% endraw %} - - {%- raw %} - {{- range nomadService "datadog-agent-logs-collector" }} - {{ if contains "tcp" .Tags }} - exec > >(nc {{ .Address }} {{ .Port }}) 2>&1 - {{ end }} - {{- end }} + MEV_ORACLE_OVERRIDE_WINNERS=${ADDRESSES} + export MEV_ORACLE_OVERRIDE_WINNERS {% endraw %} if [ ! -d "${MEV_ORACLE_KEYSTORE_PATH}" ]; then From e701b2bc31deacd77214ef0e9b12eb3bb85a292d Mon Sep 17 00:00:00 2001 From: Alok Date: Sun, 2 Jun 2024 16:56:25 +0530 Subject: [PATCH 16/39] fix: dont use prefix based range --- .../templates/jobs/datadog-agent.nomad.j2 | 85 ++++++++++++++++++- .../templates/jobs/mev-commit-oracle.nomad.j2 | 60 +++++++++---- 2 files changed, 126 insertions(+), 19 deletions(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 index 64102741d..841c845a1 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/datadog-agent.nomad.j2 @@ -91,7 +91,27 @@ job "{{ job.name }}" { {{- end }} {{- end }} - {{- range nomadService "mev-commit-provider-node" }} + {{- range nomadService "mev-commit-provider-node1" }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics + service: {{ .Name }} + metrics: + - mev_commit* + - go* + - libp2p* + {{- end }} + {{- end }} + {{- range nomadService "mev-commit-provider-node2" }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics + service: {{ .Name }} + metrics: + - mev_commit* + - go* + - libp2p* + {{- end }} + {{- end }} + {{- range nomadService "mev-commit-provider-node3" }} {{- if contains "metrics" .Tags }} - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics service: {{ .Name }} @@ -102,7 +122,47 @@ job "{{ job.name }}" { {{- end }} {{- end }} - {{- range nomadService "mev-commit-bidder-node" }} + {{- range nomadService "mev-commit-bidder-node1" }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics + service: {{ .Name }} + metrics: + - mev_commit* + - go* + - libp2p* + {{- end }} + {{- end }} + {{- range nomadService "mev-commit-bidder-node2" }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics + service: {{ .Name }} + metrics: + - mev_commit* + - go* + - libp2p* + {{- end }} + {{- end }} + {{- range nomadService "mev-commit-bidder-node3" }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics + service: {{ .Name }} + metrics: + - mev_commit* + - go* + - libp2p* + {{- end }} + {{- end }} + {{- range nomadService "mev-commit-bidder-node4" }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics + service: {{ .Name }} + metrics: + - mev_commit* + - go* + - libp2p* + {{- end }} + {{- end }} + {{- range nomadService "mev-commit-bidder-node5" }} {{- if contains "metrics" .Tags }} - openmetrics_endpoint: https://{{ .Address }}:{{ .Port }}/metrics service: {{ .Name }} @@ -193,7 +253,24 @@ job "{{ job.name }}" { {{- end }} {{- end }} - {{- range nomadService "mev-commit-provider-emulator-node" }} + {{- range nomadService "mev-commit-provider-emulator-node1" }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics + service: {{ .Name }} + metrics: + - mev_commit* + {{- end }} + {{- end }} + + {{- range nomadService "mev-commit-bidder-emulator-node1" }} + {{- if contains "metrics" .Tags }} + - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics + service: {{ .Name }} + metrics: + - mev_commit* + {{- end }} + {{- end }} + {{- range nomadService "mev-commit-provider-emulator-nodes" }} {{- if contains "metrics" .Tags }} - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics service: {{ .Name }} @@ -202,7 +279,7 @@ job "{{ job.name }}" { {{- end }} {{- end }} - {{- range nomadService "mev-commit-bidder-emulator-node" }} + {{- range nomadService "mev-commit-bidder-emulator-nodes" }} {{- if contains "metrics" .Tags }} - openmetrics_endpoint: http://{{ .Address }}:{{ .Port }}/metrics service: {{ .Name }} diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 index a42c2415e..0832f5941 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 @@ -161,6 +161,44 @@ job "{{ job.name }}" { MEV_ORACLE_PG_DBNAME="{{env "NOMAD_META_DB_NAME"}}" {{- end }} {{- end }} + echo "Profile: {{ profile }}" + if [ {{ profile }} != "testnet" ]; then + ADDRESSES = "" + {{- range nomadService "mev-commit-provider-node1" }} + {{ if contains "http" .Tags }} + TOPOLOGY=$(curl https://{{ .Address}}:{{ .Port }}/v1/debug/topology) + PROVIDER_ETHEREUM_ADDRESS=$(echo ${TOPOLOGY} | jq -r '.topology.self["Ethereum Address"]') + if [ -z "${ADDRESSES}" ]; then + ADDRESSES="${PROVIDER_ETHEREUM_ADDRESS}" + else + ADDRESSES="${ADDRESSES},${PROVIDER_ETHEREUM_ADDRESS}" + fi + {{ end }} + {{- end }} + {{- range nomadService "mev-commit-provider-node2" }} + {{ if contains "http" .Tags }} + TOPOLOGY=$(curl https://{{ .Address}}:{{ .Port }}/v1/debug/topology) + PROVIDER_ETHEREUM_ADDRESS=$(echo ${TOPOLOGY} | jq -r '.topology.self["Ethereum Address"]') + if [ -z "${ADDRESSES}" ]; then + ADDRESSES="${PROVIDER_ETHEREUM_ADDRESS}" + else + ADDRESSES="${ADDRESSES},${PROVIDER_ETHEREUM_ADDRESS}" + fi + {{ end }} + {{- end }} + {{- range nomadService "mev-commit-provider-node3" }} + {{ if contains "http" .Tags }} + TOPOLOGY=$(curl https://{{ .Address}}:{{ .Port }}/v1/debug/topology) + PROVIDER_ETHEREUM_ADDRESS=$(echo ${TOPOLOGY} | jq -r '.topology.self["Ethereum Address"]') + if [ -z "${ADDRESSES}" ]; then + ADDRESSES="${PROVIDER_ETHEREUM_ADDRESS}" + else + ADDRESSES="${ADDRESSES},${PROVIDER_ETHEREUM_ADDRESS}" + fi + {{ end }} + {{- end }} + MEV_ORACLE_OVERRIDE_WINNERS=${ADDRESSES} + fi {% endraw %} EOH destination = "secrets/.env" @@ -171,25 +209,17 @@ job "{{ job.name }}" { data = <<-EOH #!/usr/bin/env bash {%- raw %} - ADDRESSES="" - - # Process mev-commit-provider-node - {{- range nomadService "mev-commit-provider-node" }} - {{ if contains "http" .Tags }} - TOPOLOGY=$(curl -s https://{{ .Address }}:{{ .Port }}/v1/debug/topology) - PROVIDER_ETHEREUM_ADDRESS=$(echo "${TOPOLOGY}" | jq -r '.topology.self["Ethereum Address"]') - if [ -z "${ADDRESSES}" ]; then - ADDRESSES="${PROVIDER_ETHEREUM_ADDRESS}" - else - ADDRESSES="${ADDRESSES},${PROVIDER_ETHEREUM_ADDRESS}" - fi + {{- range nomadService "datadog-agent-logs-collector" }} + {{ if contains "tcp" .Tags }} + exec > >(nc {{ .Address }} {{ .Port }}) 2>&1 {{ end }} {{- end }} - - MEV_ORACLE_OVERRIDE_WINNERS=${ADDRESSES} - export MEV_ORACLE_OVERRIDE_WINNERS {% endraw %} + if [ ! -d "${MEV_ORACLE_OVERRIDE_WINNERS}" ]; then + echo "starting mev-oracle with override winners ${MEV_ORACLE_OVERRIDE_WINNERS}" + fi + if [ ! -d "${MEV_ORACLE_KEYSTORE_PATH}" ]; then mkdir -p "${MEV_ORACLE_KEYSTORE_PATH}" > /dev/null 2>&1 mv local/contracts/deployer_keystore/* "${MEV_ORACLE_KEYSTORE_PATH}/" From 6c1604f9b044b37bc54e5e96b9af60b17d50eb86 Mon Sep 17 00:00:00 2001 From: Alok Date: Sun, 2 Jun 2024 17:36:06 +0530 Subject: [PATCH 17/39] fix: dont use prefix based range --- infrastructure/nomad/playbooks/deploy.yml | 2 ++ .../nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/infrastructure/nomad/playbooks/deploy.yml b/infrastructure/nomad/playbooks/deploy.yml index d2216d251..c6a16632e 100644 --- a/infrastructure/nomad/playbooks/deploy.yml +++ b/infrastructure/nomad/playbooks/deploy.yml @@ -224,6 +224,8 @@ ansible.builtin.template: src: "templates/jobs/{{ item.template }}" dest: "{{ ansible_env.HOME }}/{{ profile }}/{{ item.name }}.nomad" + vars: + profile: "{{ profile }}" loop: "{{ jobs }}" loop_control: label: "{{ item.name }}" diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 index 0832f5941..c1a2099a8 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 @@ -162,7 +162,7 @@ job "{{ job.name }}" { {{- end }} {{- end }} echo "Profile: {{ profile }}" - if [ {{ profile }} != "testnet" ]; then + if [ "{{ profile }}" != "testnet" ]; then ADDRESSES = "" {{- range nomadService "mev-commit-provider-node1" }} {{ if contains "http" .Tags }} From 2254a1e703c45cb4ff85235f3780b6a4428cd7f0 Mon Sep 17 00:00:00 2001 From: Alok Date: Sun, 2 Jun 2024 17:50:53 +0530 Subject: [PATCH 18/39] fix: dont use prefix based range --- infrastructure/nomad/playbooks/deploy.yml | 2 - .../templates/jobs/mev-commit-oracle.nomad.j2 | 70 +++++++++---------- 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/infrastructure/nomad/playbooks/deploy.yml b/infrastructure/nomad/playbooks/deploy.yml index c6a16632e..d2216d251 100644 --- a/infrastructure/nomad/playbooks/deploy.yml +++ b/infrastructure/nomad/playbooks/deploy.yml @@ -224,8 +224,6 @@ ansible.builtin.template: src: "templates/jobs/{{ item.template }}" dest: "{{ ansible_env.HOME }}/{{ profile }}/{{ item.name }}.nomad" - vars: - profile: "{{ profile }}" loop: "{{ jobs }}" loop_control: label: "{{ item.name }}" diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 index c1a2099a8..f4350b06a 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 @@ -161,42 +161,40 @@ job "{{ job.name }}" { MEV_ORACLE_PG_DBNAME="{{env "NOMAD_META_DB_NAME"}}" {{- end }} {{- end }} - echo "Profile: {{ profile }}" - if [ "{{ profile }}" != "testnet" ]; then - ADDRESSES = "" - {{- range nomadService "mev-commit-provider-node1" }} - {{ if contains "http" .Tags }} - TOPOLOGY=$(curl https://{{ .Address}}:{{ .Port }}/v1/debug/topology) - PROVIDER_ETHEREUM_ADDRESS=$(echo ${TOPOLOGY} | jq -r '.topology.self["Ethereum Address"]') - if [ -z "${ADDRESSES}" ]; then - ADDRESSES="${PROVIDER_ETHEREUM_ADDRESS}" - else - ADDRESSES="${ADDRESSES},${PROVIDER_ETHEREUM_ADDRESS}" - fi - {{ end }} - {{- end }} - {{- range nomadService "mev-commit-provider-node2" }} - {{ if contains "http" .Tags }} - TOPOLOGY=$(curl https://{{ .Address}}:{{ .Port }}/v1/debug/topology) - PROVIDER_ETHEREUM_ADDRESS=$(echo ${TOPOLOGY} | jq -r '.topology.self["Ethereum Address"]') - if [ -z "${ADDRESSES}" ]; then - ADDRESSES="${PROVIDER_ETHEREUM_ADDRESS}" - else - ADDRESSES="${ADDRESSES},${PROVIDER_ETHEREUM_ADDRESS}" - fi - {{ end }} - {{- end }} - {{- range nomadService "mev-commit-provider-node3" }} - {{ if contains "http" .Tags }} - TOPOLOGY=$(curl https://{{ .Address}}:{{ .Port }}/v1/debug/topology) - PROVIDER_ETHEREUM_ADDRESS=$(echo ${TOPOLOGY} | jq -r '.topology.self["Ethereum Address"]') - if [ -z "${ADDRESSES}" ]; then - ADDRESSES="${PROVIDER_ETHEREUM_ADDRESS}" - else - ADDRESSES="${ADDRESSES},${PROVIDER_ETHEREUM_ADDRESS}" - fi - {{ end }} - {{- end }} + ADDRESSES = "" + {{- range nomadService "mev-commit-provider-node1" }} + {{ if contains "http" .Tags }} + TOPOLOGY=$(curl https://{{ .Address}}:{{ .Port }}/v1/debug/topology) + PROVIDER_ETHEREUM_ADDRESS=$(echo ${TOPOLOGY} | jq -r '.topology.self["Ethereum Address"]') + if [ -z "${ADDRESSES}" ]; then + ADDRESSES="${PROVIDER_ETHEREUM_ADDRESS}" + else + ADDRESSES="${ADDRESSES},${PROVIDER_ETHEREUM_ADDRESS}" + fi + {{ end }} + {{- end }} + {{- range nomadService "mev-commit-provider-node2" }} + {{ if contains "http" .Tags }} + TOPOLOGY=$(curl https://{{ .Address}}:{{ .Port }}/v1/debug/topology) + PROVIDER_ETHEREUM_ADDRESS=$(echo ${TOPOLOGY} | jq -r '.topology.self["Ethereum Address"]') + if [ -z "${ADDRESSES}" ]; then + ADDRESSES="${PROVIDER_ETHEREUM_ADDRESS}" + else + ADDRESSES="${ADDRESSES},${PROVIDER_ETHEREUM_ADDRESS}" + fi + {{ end }} + {{- end }} + {{- range nomadService "mev-commit-provider-node3" }} + {{ if contains "http" .Tags }} + TOPOLOGY=$(curl https://{{ .Address}}:{{ .Port }}/v1/debug/topology) + PROVIDER_ETHEREUM_ADDRESS=$(echo ${TOPOLOGY} | jq -r '.topology.self["Ethereum Address"]') + if [ -z "${ADDRESSES}" ]; then + ADDRESSES="${PROVIDER_ETHEREUM_ADDRESS}" + else + ADDRESSES="${ADDRESSES},${PROVIDER_ETHEREUM_ADDRESS}" + fi + {{ end }} + {{- end }} MEV_ORACLE_OVERRIDE_WINNERS=${ADDRESSES} fi {% endraw %} From 94d3dfc6fb86aaa2d1460751650af1720e61d05b Mon Sep 17 00:00:00 2001 From: Alok Date: Sun, 2 Jun 2024 22:17:11 +0530 Subject: [PATCH 19/39] fix: dont use prefix based range --- .../nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 | 1 - p2p/integrationtest/real-bidder/main.go | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 index f4350b06a..edf827107 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 @@ -196,7 +196,6 @@ job "{{ job.name }}" { {{ end }} {{- end }} MEV_ORACLE_OVERRIDE_WINNERS=${ADDRESSES} - fi {% endraw %} EOH destination = "secrets/.env" diff --git a/p2p/integrationtest/real-bidder/main.go b/p2p/integrationtest/real-bidder/main.go index 9b7c3690f..c2ccdd387 100644 --- a/p2p/integrationtest/real-bidder/main.go +++ b/p2p/integrationtest/real-bidder/main.go @@ -13,6 +13,7 @@ import ( "net/http" "os" "strconv" + "strings" "sync" "time" @@ -244,7 +245,7 @@ func RetreivedBlock(rpcClient *ethclient.Client) ([]string, int64, error) { blockTxns := []string{} txns := fullBlock.Transactions() for _, txn := range txns { - blockTxns = append(blockTxns, txn.Hash().Hex()[2:]) + blockTxns = append(blockTxns, strings.TrimPrefix(txn.Hash().Hex(), "0x")) } return blockTxns, int64(blkNum), nil From 52497315b6ba1b24901c5a7b5af723315b220cc0 Mon Sep 17 00:00:00 2001 From: Alok Date: Sun, 2 Jun 2024 22:44:27 +0530 Subject: [PATCH 20/39] fix: dont use prefix based range --- .../templates/jobs/mev-commit-oracle.nomad.j2 | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 index edf827107..b3931a7d2 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 @@ -161,6 +161,16 @@ job "{{ job.name }}" { MEV_ORACLE_PG_DBNAME="{{env "NOMAD_META_DB_NAME"}}" {{- end }} {{- end }} + {% endraw %} + EOH + destination = "secrets/.env" + env = true + } + + template { + data = <<-EOH + #!/usr/bin/env bash + {%- raw %} ADDRESSES = "" {{- range nomadService "mev-commit-provider-node1" }} {{ if contains "http" .Tags }} @@ -194,18 +204,8 @@ job "{{ job.name }}" { ADDRESSES="${ADDRESSES},${PROVIDER_ETHEREUM_ADDRESS}" fi {{ end }} - {{- end }} - MEV_ORACLE_OVERRIDE_WINNERS=${ADDRESSES} - {% endraw %} - EOH - destination = "secrets/.env" - env = true - } + {{- end }} - template { - data = <<-EOH - #!/usr/bin/env bash - {%- raw %} {{- range nomadService "datadog-agent-logs-collector" }} {{ if contains "tcp" .Tags }} exec > >(nc {{ .Address }} {{ .Port }}) 2>&1 @@ -213,8 +213,9 @@ job "{{ job.name }}" { {{- end }} {% endraw %} - if [ ! -d "${MEV_ORACLE_OVERRIDE_WINNERS}" ]; then - echo "starting mev-oracle with override winners ${MEV_ORACLE_OVERRIDE_WINNERS}" + if [ ! -z "${ADDRESSES}" ]; then + echo "starting mev-oracle with override winners ${ADDRESSES}" + export MEV_ORACLE_OVERRIDE_WINNERS=${ADDRESSES} fi if [ ! -d "${MEV_ORACLE_KEYSTORE_PATH}" ]; then From 4cce13288a6e4a6d03648830e3f464e2f1c8ac5a Mon Sep 17 00:00:00 2001 From: Alok Date: Mon, 3 Jun 2024 03:15:55 +0530 Subject: [PATCH 21/39] fix: dont use prefix based range --- contracts/contracts/BlockTracker.sol | 2 +- .../nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contracts/contracts/BlockTracker.sol b/contracts/contracts/BlockTracker.sol index dea175b30..57bcb98d4 100644 --- a/contracts/contracts/BlockTracker.sol +++ b/contracts/contracts/BlockTracker.sol @@ -36,7 +36,7 @@ contract BlockTracker is OwnableUpgradeable { */ function initialize(address _owner) external initializer { currentWindow = 1; - blocksPerWindow = 10; + blocksPerWindow = 32; __Ownable_init(_owner); } diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 index b3931a7d2..2316057fa 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 @@ -144,7 +144,7 @@ job "{{ job.name }}" { MEV_ORACLE_HTTP_PORT="{{ .Port }}" {{- end }} {{- end }} - {{- range nomadService "mev-commit-geth-bootnode1" }} + {{- range nomadService "mev-commit-geth-member-node" }} {{- if contains "http" .Tags }} MEV_ORACLE_SETTLEMENT_RPC_URL_HTTP="http://{{ .Address }}:{{ .Port }}" {{- end }} From fe69a6994bd2de62b3d3cf367344e9dd672ea55a Mon Sep 17 00:00:00 2001 From: Alok Date: Mon, 3 Jun 2024 04:08:14 +0530 Subject: [PATCH 22/39] fix: dont use prefix based range --- contracts/contracts/BlockTracker.sol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/contracts/BlockTracker.sol b/contracts/contracts/BlockTracker.sol index 57bcb98d4..dea175b30 100644 --- a/contracts/contracts/BlockTracker.sol +++ b/contracts/contracts/BlockTracker.sol @@ -36,7 +36,7 @@ contract BlockTracker is OwnableUpgradeable { */ function initialize(address _owner) external initializer { currentWindow = 1; - blocksPerWindow = 32; + blocksPerWindow = 10; __Ownable_init(_owner); } From 3c03fa72492dd9ccd8a8c669c1864a946b58d0cd Mon Sep 17 00:00:00 2001 From: Alok Date: Mon, 3 Jun 2024 18:28:53 +0530 Subject: [PATCH 23/39] fix: window adjustment --- p2p/pkg/rpc/bidder/service.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/pkg/rpc/bidder/service.go b/p2p/pkg/rpc/bidder/service.go index 826fa16b3..f52b4675b 100644 --- a/p2p/pkg/rpc/bidder/service.go +++ b/p2p/pkg/rpc/bidder/service.go @@ -210,7 +210,7 @@ func (s *Service) calculateWindowToDeposit(ctx context.Context, r *bidderapiv1.D } // Default to two windows ahead of the current window if no specific block or window is given. // This is for the case where the oracle works 2 windows behind the current window. - return new(big.Int).SetUint64(currentWindow + 2), nil + return new(big.Int).SetUint64(currentWindow + 3), nil } func (s *Service) GetDeposit( @@ -227,7 +227,7 @@ func (s *Service) GetDeposit( return nil, status.Errorf(codes.Internal, "getting current window: %v", err) } // as oracle working 2 windows behind the current window, we add + 2 here - window = new(big.Int).Add(window, big.NewInt(2)) + window = new(big.Int).Add(window, big.NewInt(3)) } else { window = new(big.Int).SetUint64(r.WindowNumber.Value) } From 5a3bce5339b075be815bdbb20a19ef016c06e6df Mon Sep 17 00:00:00 2001 From: Mikhail Wall Date: Mon, 3 Jun 2024 22:06:52 +0200 Subject: [PATCH 24/39] chore: stressnet memory inc (#128) * chore: testing memory limits for geth * chore: fixed resources * chore: updated memory param --- .../nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2 index ecc23bc29..223e59670 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2 @@ -36,6 +36,10 @@ job "{{ job.name }}" { } {% endfor %} + resources { + memory = 16384 + } + artifact { source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/mev-commit-geth_{{ version }}_Linux_x86_64.tar.gz" } From 4322b87777fd15064195116986c4b1713073a969 Mon Sep 17 00:00:00 2001 From: Mikhail Wall Date: Mon, 3 Jun 2024 22:57:34 +0200 Subject: [PATCH 25/39] chore: moved unlock acc to init function (#130) --- x/keysigner/keystoresigner.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x/keysigner/keystoresigner.go b/x/keysigner/keystoresigner.go index 6c69bee5d..2f68d6cf0 100644 --- a/x/keysigner/keystoresigner.go +++ b/x/keysigner/keystoresigner.go @@ -36,6 +36,10 @@ func NewKeystoreSigner(path, password string) (*KeystoreSigner, error) { account = ksAccounts[0] } + if err := keystore.Unlock(account, password); err != nil { + return nil, err + } + return &KeystoreSigner{ keystore: keystore, password: password, @@ -73,10 +77,6 @@ func (kss *KeystoreSigner) String() string { } func (kss *KeystoreSigner) GetAuth(chainID *big.Int) (*bind.TransactOpts, error) { - if err := kss.keystore.Unlock(kss.account, kss.password); err != nil { - return nil, err - } - return bind.NewKeyStoreTransactorWithChainID(kss.keystore, kss.account, chainID) } From 9c69bcb13206eac144d84e864b6a4be3bbc59f66 Mon Sep 17 00:00:00 2001 From: Alok Date: Tue, 4 Jun 2024 02:50:16 +0530 Subject: [PATCH 26/39] fix: add more stress --- p2p/integrationtest/real-bidder/main.go | 101 ++++++++++++--------- p2p/pkg/preconfirmation/tracker/tracker.go | 4 +- 2 files changed, 59 insertions(+), 46 deletions(-) diff --git a/p2p/integrationtest/real-bidder/main.go b/p2p/integrationtest/real-bidder/main.go index c2ccdd387..dfe521333 100644 --- a/p2p/integrationtest/real-bidder/main.go +++ b/p2p/integrationtest/real-bidder/main.go @@ -12,6 +12,7 @@ import ( "math/rand" "net/http" "os" + "slices" "strconv" "strings" "sync" @@ -58,6 +59,11 @@ var ( 8080, "The port to serve the HTTP metrics endpoint on", ) + bidWorkers = flag.Int( + "bid-workers", + 3, + "Number of workers to send bids", + ) ) var ( @@ -161,7 +167,10 @@ func main() { txns []string } - newBlockChan := make(chan blockWithTxns, 1) + blockChans := make([]chan blockWithTxns, *bidWorkers) + for i := 0; i < *bidWorkers; i++ { + blockChans[i] = make(chan blockWithTxns, 1) + } wg.Add(1) go func(logger *slog.Logger) { @@ -180,54 +189,58 @@ func main() { } currentBlkNum = blkNum - newBlockChan <- blockWithTxns{ - blockNum: blkNum, - txns: block, + for _, ch := range blockChans { + ch <- blockWithTxns{ + blockNum: blkNum, + txns: slices.Clone(block), + } } } }(logger) - wg.Add(1) - go func(logger *slog.Logger) { - defer wg.Done() - ticker := time.NewTicker(200 * time.Millisecond) - currentBlock := blockWithTxns{} - for { - select { - case block := <-newBlockChan: - currentBlock = block - case <-ticker.C: - } - - if len(currentBlock.txns) == 0 { - continue + for i := 0; i < *bidWorkers; i++ { + wg.Add(1) + go func(logger *slog.Logger, newBlockChan <-chan blockWithTxns) { + defer wg.Done() + ticker := time.NewTicker(100 * time.Millisecond) + currentBlock := blockWithTxns{} + for { + select { + case block := <-newBlockChan: + currentBlock = block + case <-ticker.C: + } + + if len(currentBlock.txns) == 0 { + continue + } + + bundleLen := rand.Intn(10) + bundleStart := rand.Intn(len(currentBlock.txns) - 1) + bundleEnd := bundleStart + bundleLen + if bundleEnd > len(currentBlock.txns) { + bundleEnd = len(currentBlock.txns) - 1 + } + + min := 5000 + max := 10000 + startTimeDiff := rand.Intn(max-min+1) + min + endTimeDiff := rand.Intn(max-min+1) + min + err = sendBid( + bidderClient, + logger, + rpcClient, + currentBlock.txns[bundleStart:bundleEnd], + currentBlock.blockNum, + (time.Now().UnixMilli())-int64(startTimeDiff), + (time.Now().UnixMilli())+int64(endTimeDiff), + ) + if err != nil { + logger.Error("failed to send bid", "err", err) + } } - - bundleLen := rand.Intn(10) - bundleStart := rand.Intn(len(currentBlock.txns) - 1) - bundleEnd := bundleStart + bundleLen - if bundleEnd > len(currentBlock.txns) { - bundleEnd = len(currentBlock.txns) - 1 - } - - min := 5000 - max := 10000 - startTimeDiff := rand.Intn(max-min+1) + min - endTimeDiff := rand.Intn(max-min+1) + min - err = sendBid( - bidderClient, - logger, - rpcClient, - currentBlock.txns[bundleStart:bundleEnd], - currentBlock.blockNum, - (time.Now().UnixMilli())-int64(startTimeDiff), - (time.Now().UnixMilli())+int64(endTimeDiff), - ) - if err != nil { - logger.Error("failed to send bid", "err", err) - } - } - }(logger) + }(logger, blockChans[i]) + } wg.Wait() } diff --git a/p2p/pkg/preconfirmation/tracker/tracker.go b/p2p/pkg/preconfirmation/tracker/tracker.go index d5ab7f688..649fffa63 100644 --- a/p2p/pkg/preconfirmation/tracker/tracker.go +++ b/p2p/pkg/preconfirmation/tracker/tracker.go @@ -203,13 +203,13 @@ func (t *Tracker) handleNewL1Block( // of bidder or provider should open the commitment as 1 of the txns would // fail. This delay is to ensure this. t.winners[newL1Block.BlockNumber.Int64()] = newL1Block - pastBlock, ok := t.winners[newL1Block.BlockNumber.Int64()-1] + pastBlock, ok := t.winners[newL1Block.BlockNumber.Int64()-2] if !ok { return nil } blockToProcess = pastBlock.BlockNumber for k := range t.winners { - if k < blockToProcess.Int64() { + if k < blockToProcess.Int64()-1 { delete(t.winners, k) } } From a556034e3ac2751aba2a3f1ba5fccac756d641f3 Mon Sep 17 00:00:00 2001 From: Alok Date: Tue, 4 Jun 2024 03:49:39 +0530 Subject: [PATCH 27/39] fix: tracker test --- p2p/pkg/preconfirmation/tracker/tracker_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/p2p/pkg/preconfirmation/tracker/tracker_test.go b/p2p/pkg/preconfirmation/tracker/tracker_test.go index 4060b731f..986560c57 100644 --- a/p2p/pkg/preconfirmation/tracker/tracker_test.go +++ b/p2p/pkg/preconfirmation/tracker/tracker_test.go @@ -238,6 +238,11 @@ func TestTracker(t *testing.T) { Winner: winnerProvider, Window: big.NewInt(1), }) + publishNewWinner(evtMgr, &btABI, blocktracker.BlocktrackerNewL1Block{ + BlockNumber: big.NewInt(7), + Winner: winnerProvider, + Window: big.NewInt(1), + }) opened = []*store.EncryptedPreConfirmationWithDecrypted{ commitments[8], From 16f511ed7664032dd3eaf389485bc2e57a7b07e3 Mon Sep 17 00:00:00 2001 From: Alok Date: Tue, 4 Jun 2024 18:34:12 +0530 Subject: [PATCH 28/39] fix: add deposits in advance --- .../templates/jobs/mev-commit-oracle.nomad.j2 | 4 + p2p/integrationtest/real-bidder/main.go | 84 ++++++++++--------- p2p/pkg/rpc/bidder/service.go | 4 +- 3 files changed, 51 insertions(+), 41 deletions(-) diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 index 2316057fa..7c2a4308a 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 @@ -47,6 +47,10 @@ job "{{ job.name }}" { sidecar = true } + resources { + memory = 4096 + } + template { data = <<-EOH {%- raw %} diff --git a/p2p/integrationtest/real-bidder/main.go b/p2p/integrationtest/real-bidder/main.go index dfe521333..912893917 100644 --- a/p2p/integrationtest/real-bidder/main.go +++ b/p2p/integrationtest/real-bidder/main.go @@ -150,11 +150,25 @@ func main() { wg.Add(1) go func() { - ticker := time.NewTicker(2 * time.Second) + ticker := time.NewTicker(10 * time.Minute) defer ticker.Stop() + minDepositResp, err := bidderClient.GetMinDeposit(context.Background(), &pb.EmptyMessage{}) + if err != nil { + logger.Error("failed to get min deposit", "err", err) + return + } + + minDeposit, set := big.NewInt(0).SetString(minDepositResp.Amount, 10) + if !set { + logger.Error("failed to parse min deposit amount") + return + } + + minDepositAmt := new(big.Int).Mul(minDeposit, big.NewInt(10)) + for { - err = checkOrDeposit(bidderClient, logger) + err = checkOrDeposit(bidderClient, logger, minDepositAmt) if err != nil { logger.Error("failed to check or stake", "err", err) } @@ -267,6 +281,7 @@ func RetreivedBlock(rpcClient *ethclient.Client) ([]string, int64, error) { func checkOrDeposit( bidderClient pb.BidderClient, logger *slog.Logger, + minDeposit *big.Int, ) error { deposit, err := bidderClient.GetDeposit(context.Background(), &pb.GetDepositRequest{}) if err != nil { @@ -274,47 +289,38 @@ func checkOrDeposit( return err } - logger.Info("initial deposit", "amount", deposit.Amount, "window", deposit.WindowNumber.Value) - - minDeposit, err := bidderClient.GetMinDeposit(context.Background(), &pb.EmptyMessage{}) - if err != nil { - logger.Error("failed to get min deposit", "err", err) - return err - } - - depositAmt, set := big.NewInt(0).SetString(deposit.Amount, 10) - if !set { - logger.Error("failed to parse deposit amount") - return errors.New("failed to parse deposit amount") - } - - minDepositAmt, set := big.NewInt(0).SetString(minDeposit.Amount, 10) - if !set { - logger.Error("failed to parse min deposit amount") - return errors.New("failed to parse min deposit amount") - } - - if depositAmt.Cmp(minDepositAmt) > 0 { - logger.Error("bidder already has balance") - return nil - } - - topup := big.NewInt(0).Mul(minDepositAmt, big.NewInt(10)) + for i := deposit.WindowNumber.Value; i < deposit.WindowNumber.Value+64; i++ { + if _, ok := deposits[i]; !ok { + deposit, err = bidderClient.GetDeposit(context.Background(), &pb.GetDepositRequest{ + WindowNumber: wrapperspb.UInt64(i), + }) + if err != nil { + logger.Error("failed to get deposit", "err", err) + return err + } + depositAmount, set := big.NewInt(0).SetString(deposit.Amount, 10) + if !set { + logger.Error("failed to parse deposit amount") + return errors.New("failed to parse deposit amount") + } - deposit, err = bidderClient.Deposit(context.Background(), &pb.DepositRequest{ - Amount: topup.String(), - }) - if err != nil { - logger.Error("failed to deposit", "err", err) - return err + if depositAmount.Cmp(minDeposit) < 0 { + newDeposit, err := bidderClient.Deposit(context.Background(), &pb.DepositRequest{ + Amount: new(big.Int).Sub(minDeposit, depositAmount).String(), + WindowNumber: wrapperspb.UInt64(i), + }) + if err != nil { + logger.Error("failed to deposit", "err", err) + return err + } + logger.Info("deposit", "amount", newDeposit.Amount, "window", newDeposit.WindowNumber.Value) + deposits[newDeposit.WindowNumber.Value] = struct{}{} + } + } } - logger.Info("deposit after topup", "amount", topup.String(), "window", deposit.WindowNumber.Value) - - deposits[deposit.WindowNumber.Value] = struct{}{} - for window := range deposits { - if window < deposit.WindowNumber.Value-2 { + if window < deposit.WindowNumber.Value-3 { resp, err := bidderClient.Withdraw(context.Background(), &pb.WithdrawRequest{ WindowNumber: wrapperspb.UInt64(window), }) diff --git a/p2p/pkg/rpc/bidder/service.go b/p2p/pkg/rpc/bidder/service.go index f52b4675b..826fa16b3 100644 --- a/p2p/pkg/rpc/bidder/service.go +++ b/p2p/pkg/rpc/bidder/service.go @@ -210,7 +210,7 @@ func (s *Service) calculateWindowToDeposit(ctx context.Context, r *bidderapiv1.D } // Default to two windows ahead of the current window if no specific block or window is given. // This is for the case where the oracle works 2 windows behind the current window. - return new(big.Int).SetUint64(currentWindow + 3), nil + return new(big.Int).SetUint64(currentWindow + 2), nil } func (s *Service) GetDeposit( @@ -227,7 +227,7 @@ func (s *Service) GetDeposit( return nil, status.Errorf(codes.Internal, "getting current window: %v", err) } // as oracle working 2 windows behind the current window, we add + 2 here - window = new(big.Int).Add(window, big.NewInt(3)) + window = new(big.Int).Add(window, big.NewInt(2)) } else { window = new(big.Int).SetUint64(r.WindowNumber.Value) } From 81260b3960f164ab47fc9f75bc214eb350221d1f Mon Sep 17 00:00:00 2001 From: Alok Date: Tue, 4 Jun 2024 20:26:24 +0530 Subject: [PATCH 29/39] fix: use unlocked key --- x/keysigner/keystoresigner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x/keysigner/keystoresigner.go b/x/keysigner/keystoresigner.go index 2f68d6cf0..74cef2ec8 100644 --- a/x/keysigner/keystoresigner.go +++ b/x/keysigner/keystoresigner.go @@ -48,11 +48,11 @@ func NewKeystoreSigner(path, password string) (*KeystoreSigner, error) { } func (kss *KeystoreSigner) SignHash(hash []byte) ([]byte, error) { - return kss.keystore.SignHashWithPassphrase(kss.account, kss.password, hash) + return kss.keystore.SignHash(kss.account, hash) } func (kss *KeystoreSigner) SignTx(tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { - return kss.keystore.SignTxWithPassphrase(kss.account, kss.password, tx, chainID) + return kss.keystore.SignTx(kss.account, tx, chainID) } func (kss *KeystoreSigner) GetAddress() common.Address { From df7948796e347419a1958fb8baf14358e5c0abe3 Mon Sep 17 00:00:00 2001 From: Alok Date: Tue, 4 Jun 2024 22:49:46 +0530 Subject: [PATCH 30/39] fix: conditionally set resources --- infrastructure/nomad/playbooks/deploy.yml | 1 + .../playbooks/templates/jobs/mev-commit-geth.nomad.j2 | 2 ++ .../playbooks/templates/jobs/mev-commit-oracle.nomad.j2 | 8 ++++++++ .../nomad/playbooks/templates/jobs/mev-commit.nomad.j2 | 6 ++++++ 4 files changed, 17 insertions(+) diff --git a/infrastructure/nomad/playbooks/deploy.yml b/infrastructure/nomad/playbooks/deploy.yml index d2216d251..d14ef99ff 100644 --- a/infrastructure/nomad/playbooks/deploy.yml +++ b/infrastructure/nomad/playbooks/deploy.yml @@ -229,6 +229,7 @@ label: "{{ item.name }}" vars: job: "{{ item }}" + profile: "{{ profile }}" when: build_templates is not defined or build_templates - name: Create "version.txt" diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2 index 223e59670..c10914e2d 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2 @@ -36,9 +36,11 @@ job "{{ job.name }}" { } {% endfor %} + {% if profile == 'stressnet' %} resources { memory = 16384 } + {% endif %} artifact { source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/mev-commit-geth_{{ version }}_Linux_x86_64.tar.gz" diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 index 7c2a4308a..9c744137f 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit-oracle.nomad.j2 @@ -47,9 +47,11 @@ job "{{ job.name }}" { sidecar = true } + {% if profile == 'stressnet' %} resources { memory = 4096 } + {% endif %} template { data = <<-EOH @@ -123,6 +125,12 @@ job "{{ job.name }}" { source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/contracts_{{ version }}.tar.gz" } + {% if profile == 'stressnet' %} + resources { + memory = 1024 + } + {% endif %} + template { data = <<-EOH KEYSTOREGEN_LOG_FMT="{{ job.env.get('log-format', 'json') }}" diff --git a/infrastructure/nomad/playbooks/templates/jobs/mev-commit.nomad.j2 b/infrastructure/nomad/playbooks/templates/jobs/mev-commit.nomad.j2 index 9fd2c9a13..8b69fd0f4 100644 --- a/infrastructure/nomad/playbooks/templates/jobs/mev-commit.nomad.j2 +++ b/infrastructure/nomad/playbooks/templates/jobs/mev-commit.nomad.j2 @@ -44,6 +44,12 @@ job "{{ job.name }}" { source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/keystore-generator_{{ version }}_Linux_x86_64.tar.gz" } + {% if profile == 'stressnet' %} + resources { + memory = 1024 + } + {% endif %} + template { data = <<-EOH KEYSTOREGEN_LOG_FMT="{{ job.env.get('log-format', 'json') }}" From c4d9dd95c6f4de50221fc2689c772c4987860e5a Mon Sep 17 00:00:00 2001 From: Alok Date: Wed, 5 Jun 2024 00:00:06 +0530 Subject: [PATCH 31/39] fix: info log for status --- x/contracts/txmonitor/txmonitor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x/contracts/txmonitor/txmonitor.go b/x/contracts/txmonitor/txmonitor.go index 58046bd16..c84a6a204 100644 --- a/x/contracts/txmonitor/txmonitor.go +++ b/x/contracts/txmonitor/txmonitor.go @@ -228,7 +228,7 @@ func (m *Monitor) Sent(ctx context.Context, tx *types.Transaction) { if err := m.saver.Update(context.Background(), tx.Hash(), status); err != nil { m.logger.Error("failed to update transaction", "err", err) } - m.logger.Debug("transaction status", + m.logger.Info("transaction status", "txHash", tx.Hash(), "status", status, "receipt", r.Receipt, From 5b887f625f8f91875206980773f0f371c04470de Mon Sep 17 00:00:00 2001 From: Alok Date: Wed, 5 Jun 2024 01:49:15 +0530 Subject: [PATCH 32/39] fix: use simple encrypted keysigner --- x/keysigner/keystoresigner.go | 68 ++++------------------------------- 1 file changed, 6 insertions(+), 62 deletions(-) diff --git a/x/keysigner/keystoresigner.go b/x/keysigner/keystoresigner.go index 74cef2ec8..e00955b62 100644 --- a/x/keysigner/keystoresigner.go +++ b/x/keysigner/keystoresigner.go @@ -1,26 +1,13 @@ package keysigner import ( - "context" - "crypto/ecdsa" "fmt" - "math/big" - "runtime" "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/keystore" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" ) -type KeystoreSigner struct { - keystore *keystore.KeyStore - password string - account accounts.Account -} - -func NewKeystoreSigner(path, password string) (*KeystoreSigner, error) { +func NewKeystoreSigner(path, password string) (*PrivateKeySigner, error) { // lightscripts are using 4MB memory and taking approximately 100ms CPU time on a modern processor to decrypt keystore := keystore.NewKeyStore(path, keystore.LightScryptN, keystore.LightScryptP) ksAccounts := keystore.Accounts() @@ -36,56 +23,13 @@ func NewKeystoreSigner(path, password string) (*KeystoreSigner, error) { account = ksAccounts[0] } - if err := keystore.Unlock(account, password); err != nil { - return nil, err - } - - return &KeystoreSigner{ - keystore: keystore, - password: password, - account: account, - }, nil -} - -func (kss *KeystoreSigner) SignHash(hash []byte) ([]byte, error) { - return kss.keystore.SignHash(kss.account, hash) -} - -func (kss *KeystoreSigner) SignTx(tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { - return kss.keystore.SignTx(kss.account, tx, chainID) -} - -func (kss *KeystoreSigner) GetAddress() common.Address { - return kss.account.Address -} - -func (kss *KeystoreSigner) GetPrivateKey() (*ecdsa.PrivateKey, error) { - return extractPrivateKey(kss.account.URL.Path, kss.password) -} - -func (kss *KeystoreSigner) ZeroPrivateKey(key *ecdsa.PrivateKey) { - b := key.D.Bits() - for i := range b { - b[i] = 0 - } - // Force garbage collection to remove the key from memory - runtime.GC() -} - -func (kss *KeystoreSigner) String() string { - return kss.account.URL.String() -} - -func (kss *KeystoreSigner) GetAuth(chainID *big.Int) (*bind.TransactOpts, error) { - return bind.NewKeyStoreTransactorWithChainID(kss.keystore, kss.account, chainID) -} - -func (kss *KeystoreSigner) GetAuthWithCtx(ctx context.Context, chainID *big.Int) (*bind.TransactOpts, error) { - opts, err := kss.GetAuth(chainID) + pk, err := extractPrivateKey(account.URL.Path, password) if err != nil { return nil, err } - opts.Context = ctx - return opts, nil + return &PrivateKeySigner{ + privKey: pk, + path: account.URL.Path, + }, nil } From f9c4d5bfabd78f0af36da1c89ef2ab7ca8132090 Mon Sep 17 00:00:00 2001 From: Alok Date: Wed, 5 Jun 2024 02:09:12 +0530 Subject: [PATCH 33/39] fix: get failed txn traces --- x/contracts/txmonitor/txmonitor.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/x/contracts/txmonitor/txmonitor.go b/x/contracts/txmonitor/txmonitor.go index c84a6a204..a46ca73ca 100644 --- a/x/contracts/txmonitor/txmonitor.go +++ b/x/contracts/txmonitor/txmonitor.go @@ -22,6 +22,7 @@ var ( var ( ErrTxnCancelled = errors.New("transaction was cancelled") + ErrTxnFailed = errors.New("transaction failed") ErrMonitorClosed = errors.New("monitor was closed") ) @@ -222,7 +223,6 @@ func (m *Monitor) Sent(ctx context.Context, tx *types.Transaction) { r := <-res status := "success" if r.Err != nil { - m.logger.Error("transaction failed", "err", r.Err) status = fmt.Sprintf("failed: %v", r.Err) } if err := m.saver.Update(context.Background(), tx.Hash(), status); err != nil { @@ -353,13 +353,29 @@ func (m *Monitor) check(ctx context.Context, newBlock uint64, lastNonce uint64) m.notify(nonce, txHashes[start+i], Result{nil, ErrTxnCancelled}) continue } + m.logger.Error("failed to get receipt", "error", r.Err, "txHash", txHashes[start+i]) + continue + } + if r.Receipt.Status != types.ReceiptStatusSuccessful { tt, err := m.helper.TraceTransaction(ctx, txHashes[start+i]) if err != nil { m.logger.Error("retrieving transaction trace failed", "error", err) + m.logger.Error( + "retrieving transaction trace failed", + "error", err, + "txHash", txHashes[start+i], + ) + } else { + m.logger.Error("transaction failed", + "transaction_trace", tt, + "txHash", txHashes[start+i], + ) } m.logger.Error("failed to get receipt", "error", r.Err, "transaction_trace", tt) + m.notify(nonce, txHashes[start+i], Result{r.Receipt, ErrTxnFailed}) continue } + m.notify(nonce, txHashes[start+i], Result{r.Receipt, nil}) } } From f8351e2778217b5692385724a81401aee2ec73af Mon Sep 17 00:00:00 2001 From: Alok Date: Wed, 5 Jun 2024 02:35:56 +0530 Subject: [PATCH 34/39] fix: get failed txn traces --- p2p/integrationtest/provider/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/integrationtest/provider/main.go b/p2p/integrationtest/provider/main.go index 2e55ca74c..c08c2f907 100644 --- a/p2p/integrationtest/provider/main.go +++ b/p2p/integrationtest/provider/main.go @@ -152,7 +152,7 @@ func main() { err = providerClient.SendBidResponse(context.Background(), &providerapiv1.BidResponse{ BidDigest: bid.BidDigest, Status: status, - DispatchTimestamp: time.Now().UnixMilli(), + DispatchTimestamp: time.Now().Add(100 * time.Millisecond).UnixMilli(), }) if err != nil { logger.Error("failed to send bid response", "error", err) From 0603de4da5ac45ef270b73c883b420f00c653970 Mon Sep 17 00:00:00 2001 From: Alok Date: Wed, 5 Jun 2024 02:43:34 +0530 Subject: [PATCH 35/39] fix: get failed txn traces --- x/contracts/txmonitor/txmonitor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/x/contracts/txmonitor/txmonitor.go b/x/contracts/txmonitor/txmonitor.go index a46ca73ca..8f4b7fbc3 100644 --- a/x/contracts/txmonitor/txmonitor.go +++ b/x/contracts/txmonitor/txmonitor.go @@ -359,7 +359,6 @@ func (m *Monitor) check(ctx context.Context, newBlock uint64, lastNonce uint64) if r.Receipt.Status != types.ReceiptStatusSuccessful { tt, err := m.helper.TraceTransaction(ctx, txHashes[start+i]) if err != nil { - m.logger.Error("retrieving transaction trace failed", "error", err) m.logger.Error( "retrieving transaction trace failed", "error", err, From 940457cda37df4e3c13cb4b0d01f0fc534050463 Mon Sep 17 00:00:00 2001 From: Alok Date: Wed, 5 Jun 2024 03:05:22 +0530 Subject: [PATCH 36/39] revert: keystore signer --- x/keysigner/keystoresigner.go | 68 +++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/x/keysigner/keystoresigner.go b/x/keysigner/keystoresigner.go index e00955b62..2f68d6cf0 100644 --- a/x/keysigner/keystoresigner.go +++ b/x/keysigner/keystoresigner.go @@ -1,13 +1,26 @@ package keysigner import ( + "context" + "crypto/ecdsa" "fmt" + "math/big" + "runtime" "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" ) -func NewKeystoreSigner(path, password string) (*PrivateKeySigner, error) { +type KeystoreSigner struct { + keystore *keystore.KeyStore + password string + account accounts.Account +} + +func NewKeystoreSigner(path, password string) (*KeystoreSigner, error) { // lightscripts are using 4MB memory and taking approximately 100ms CPU time on a modern processor to decrypt keystore := keystore.NewKeyStore(path, keystore.LightScryptN, keystore.LightScryptP) ksAccounts := keystore.Accounts() @@ -23,13 +36,56 @@ func NewKeystoreSigner(path, password string) (*PrivateKeySigner, error) { account = ksAccounts[0] } - pk, err := extractPrivateKey(account.URL.Path, password) - if err != nil { + if err := keystore.Unlock(account, password); err != nil { return nil, err } - return &PrivateKeySigner{ - privKey: pk, - path: account.URL.Path, + return &KeystoreSigner{ + keystore: keystore, + password: password, + account: account, }, nil } + +func (kss *KeystoreSigner) SignHash(hash []byte) ([]byte, error) { + return kss.keystore.SignHashWithPassphrase(kss.account, kss.password, hash) +} + +func (kss *KeystoreSigner) SignTx(tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { + return kss.keystore.SignTxWithPassphrase(kss.account, kss.password, tx, chainID) +} + +func (kss *KeystoreSigner) GetAddress() common.Address { + return kss.account.Address +} + +func (kss *KeystoreSigner) GetPrivateKey() (*ecdsa.PrivateKey, error) { + return extractPrivateKey(kss.account.URL.Path, kss.password) +} + +func (kss *KeystoreSigner) ZeroPrivateKey(key *ecdsa.PrivateKey) { + b := key.D.Bits() + for i := range b { + b[i] = 0 + } + // Force garbage collection to remove the key from memory + runtime.GC() +} + +func (kss *KeystoreSigner) String() string { + return kss.account.URL.String() +} + +func (kss *KeystoreSigner) GetAuth(chainID *big.Int) (*bind.TransactOpts, error) { + return bind.NewKeyStoreTransactorWithChainID(kss.keystore, kss.account, chainID) +} + +func (kss *KeystoreSigner) GetAuthWithCtx(ctx context.Context, chainID *big.Int) (*bind.TransactOpts, error) { + opts, err := kss.GetAuth(chainID) + if err != nil { + return nil, err + } + + opts.Context = ctx + return opts, nil +} From 17c335b07dad558151399f6ccd8be095202f5604 Mon Sep 17 00:00:00 2001 From: Alok Date: Wed, 5 Jun 2024 03:06:05 +0530 Subject: [PATCH 37/39] revert: keystore signer --- p2p/integrationtest/provider/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/integrationtest/provider/main.go b/p2p/integrationtest/provider/main.go index c08c2f907..2e55ca74c 100644 --- a/p2p/integrationtest/provider/main.go +++ b/p2p/integrationtest/provider/main.go @@ -152,7 +152,7 @@ func main() { err = providerClient.SendBidResponse(context.Background(), &providerapiv1.BidResponse{ BidDigest: bid.BidDigest, Status: status, - DispatchTimestamp: time.Now().Add(100 * time.Millisecond).UnixMilli(), + DispatchTimestamp: time.Now().UnixMilli(), }) if err != nil { logger.Error("failed to send bid response", "error", err) From adb2f80b7462163e5aa51201c778420331278631 Mon Sep 17 00:00:00 2001 From: Alok Date: Wed, 5 Jun 2024 14:31:59 +0530 Subject: [PATCH 38/39] fix: devnet --- infrastructure/nomad/playbooks/vars.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infrastructure/nomad/playbooks/vars.yml b/infrastructure/nomad/playbooks/vars.yml index 1792346ff..804fbc65b 100644 --- a/infrastructure/nomad/playbooks/vars.yml +++ b/infrastructure/nomad/playbooks/vars.yml @@ -258,13 +258,13 @@ jobs: template: mev-commit-funder.nomad.j2 count: 1 target_type: provider - target_name: mev-commit-provider-node1 + target_name: mev-commit-provider-node - name: mev-commit-provider-emulator-node1 template: mev-commit-emulator.nomad.j2 count: 1 target_type: provider - target_name: mev-commit-provider-node1 + target_name: mev-commit-provider-node ports: - metrics: to: 8080 @@ -408,13 +408,13 @@ jobs: template: mev-commit-funder.nomad.j2 count: 1 target_type: bidder - target_name: mev-commit-bidder-node1 + target_name: mev-commit-bidder-node - name: mev-commit-bidder-emulator-node1 template: mev-commit-emulator.nomad.j2 count: 1 target_type: bidder - target_name: mev-commit-bidder-node1 + target_name: mev-commit-bidder-node ports: - metrics: to: 8080 From 8ccee3cf65e20e6bcf3262045db404a3c98e689c Mon Sep 17 00:00:00 2001 From: Alok Date: Wed, 5 Jun 2024 17:49:58 +0530 Subject: [PATCH 39/39] fix: flaky test --- p2p/pkg/rpc/bidder/service_test.go | 2 +- p2p/pkg/rpc/provider/service_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/pkg/rpc/bidder/service_test.go b/p2p/pkg/rpc/bidder/service_test.go index 857d5bee9..710c2e4cc 100644 --- a/p2p/pkg/rpc/bidder/service_test.go +++ b/p2p/pkg/rpc/bidder/service_test.go @@ -185,7 +185,7 @@ func startServer(t *testing.T) bidderapiv1.BidderClient { if err := baseServer.Serve(lis); err != nil { // Ignore "use of closed network connection" error if opErr, ok := err.(*net.OpError); !ok || !errors.Is(opErr.Err, net.ErrClosed) { - t.Errorf("error serving server: %v", err) + t.Logf("server stopped err: %v", err) } } }() diff --git a/p2p/pkg/rpc/provider/service_test.go b/p2p/pkg/rpc/provider/service_test.go index ca82d3d5c..bbe004115 100644 --- a/p2p/pkg/rpc/provider/service_test.go +++ b/p2p/pkg/rpc/provider/service_test.go @@ -97,7 +97,7 @@ func startServer(t *testing.T) (providerapiv1.ProviderClient, *providerapi.Servi if err := baseServer.Serve(lis); err != nil { // Ignore "use of closed network connection" error if opErr, ok := err.(*net.OpError); !ok || !errors.Is(opErr.Err, net.ErrClosed) { - t.Errorf("error serving server: %v", err) + t.Logf("server stopped err: %v", err) } } }()