Merge branch 'devel' of https://github.com/ledgerwatch/erigon into devel

.dockerignore

@@ -5,5 +5,5 @@
 **/*.dSYM
 
 build
-tests
+tests/testdata
 cmd/prometheus

.github/workflows/ci.yml

@@ -16,6 +16,10 @@ on:
       - synchronize
       - ready_for_review
 
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   tests:
     if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
@@ -24,7 +28,7 @@ jobs:
         os: [ ubuntu-20.04, macos-11 ] # list of os: https://github.com/actions/virtual-environments
     runs-on: ${{ matrix.os }}
 
-    steps:
+    steps:      
       - uses: actions/checkout@v3
       - run: git submodule update --init --recursive --force
       - uses: actions/setup-go@v3
@@ -60,7 +64,7 @@ jobs:
         if: runner.os == 'Linux'
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.47
+          version: v1.49
 
       - name: Test
         run: make test
@@ -72,7 +76,7 @@ jobs:
         os: [ windows-2022 ]
     runs-on: ${{ matrix.os }}
 
-    steps:
+    steps:      
       - uses: actions/checkout@v3
       - run: git submodule update --init --recursive --force
       - uses: actions/setup-go@v3
@@ -104,64 +108,113 @@ jobs:
       - name: Test
         run: .\wmake.ps1 test
 
-  docker:
-    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
+  docker-build-check:
+    # don't run this on devel - the PR must have run it to be merged and it misleads that this pushes the docker image
+    if: (${{ github.event_name == 'push' || !github.event.pull_request.draft }}) && ${{ github.ref != 'refs/heads/devel' }}
     runs-on: ubuntu-20.04
     steps:
+      - uses: AutoModality/action-clean@v1
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0 # fetch git tags for "git describe"
 
-      - name: make docker
+      - name: make docker (see dockerhub for image builds)
         run: DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
 
       # check with root permissions, should be cached from previous build
       - name: sudo make docker
         run: sudo DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
 
   hive:
-    needs:
-      - tests
-      - tests-windows
-      - docker
-    runs-on: ubuntu-20.04
-    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # fetch git tags for "git describe"
+     if: ${{ github.ref == 'refs/heads/devel' }}
+     needs:
+       - tests
+       - tests-windows
+     runs-on: self-hosted
+     steps:
+       - uses: AutoModality/action-clean@v1
+       - uses: actions/checkout@v3
+         with:
+           fetch-depth: 0 # fetch git tags for "git describe"
+
+       - name: build erigon image
+         run: DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
+
+       # check with root permissions, should be cached from previous build
+       - name: build erigon image (root permissions)
+         run: sudo DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
+
+       - name: run hive and parse output
+         run: |
+          sudo mkdir -p /results-${{ github.run_id }}
+          docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}:/work thorax/hive:latest --sim ethereum/engine --results-root=/work/results-${{ github.run_id }} --client erigon_ci-$GITHUB_SHA --exit.fail=false
+          docker run --rm --pull always -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}:/work --entrypoint /app/hivecioutput thorax/hive:latest --resultsdir=/work/results-${{ github.run_id }} --outdir=/work/results-${{ github.run_id }} --exclusionsfile=/work/hive/exclusions.json
+
+       - name: archive hive results
+         uses: actions/upload-artifact@v3
+         if: always()
+         with:
+           name: test-results
+           path: results-${{ github.run_id }}/*.xml
+
+       - name: clean up containers
+         if: always()
+         run: |
+          ids=$(docker ps -a -q)
+          for id in $ids
+          do
+            echo "stopping/removing container: $id"
+            docker stop $id && docker rm $id
+          done
+
+  hive-results:
+    needs: hive
+    name: Hive results
+    runs-on: self-hosted
+
+    permissions:
+      checks: write
+      pull-requests: write
+      actions: read
 
-      - name: build erigon image
-        run: DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
-
-      # check with root permissions, should be cached from previous build
-      - name: build erigon image (root permissions)
-        run: sudo DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
-
-      - name: run hive
-        run: sudo mkdir /results && docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}:/work gatewayfm/hive:latest --sim ethereum/engine --results-root=/work/results --client erigon_ci-$GITHUB_SHA --docker.output --loglevel 5
-
-      - name: parse hive output
-        run: docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}:/work --entrypoint /app/hivecioutput gatewayfm/hive:latest --resultsdir=/work/results --outdir=/work/results
-
-      - name: archive hive results
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: hive-ci-output
-          path: results/*.xml
-
-  event_file:
-    needs:
-      - tests
-      - tests-windows
-      - docker
-    name: archive event file
-    runs-on: ubuntu-latest
-    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
     steps:
-      - name: upload
-        uses: actions/upload-artifact@v2
-        with:
-          name: event file
-          path: ${{ github.event_path }}
+    - name: parse hive results
+      uses: phoenix-actions/test-reporting@v8
+      with:
+        artifact: test-results
+        name: Tests
+        path: '*.xml'
+        reporter: java-junit
+
+    - name: set badge color
+      shell: bash
+      run: |
+        case ${{ fromJSON( steps.test-results.outputs.json ).conclusion }} in
+          success)
+            echo "BADGE_COLOR=31c653" >> $GITHUB_ENV
+            ;;
+          failure)
+            echo "BADGE_COLOR=800000" >> $GITHUB_ENV
+            ;;
+          neutral)
+            echo "BADGE_COLOR=696969" >> $GITHUB_ENV
+            ;;
+        esac
+
+    - name: create badge
+      uses: emibcn/badge-action@d6f51ff11b5c3382b3b88689ae2d6db22d9737d1
+      with:
+        label: Hive
+        status: '${{ fromJSON( steps.test-results.outputs.json ).formatted.stats.tests }} tests, ${{ fromJSON( steps.test-results.outputs.json ).formatted.stats.runs }} runs: ${{ fromJSON( steps.test-results.outputs.json ).conclusion }}'
+        color: ${{ env.BADGE_COLOR }}
+        path: badge.svg
+
+    - name: upload badge to gist
+      if: >
+        github.event_name == 'workflow_run' && github.event.workflow_run.head_branch == 'devel' ||
+        github.event_name != 'workflow_run' && github.ref == 'refs/heads/devel'
+      uses: andymckay/append-gist-action@1fbfbbce708a39bd45846f0955ed5521f2099c6d
+      with:
+        token: ${{ secrets.GIST_TOKEN }}
+        gistURL: https://gist.githubusercontent.com/revitteth/dc492845ba6eb694e6c7279224634b20
+        file: badge.svg

.github/workflows/coverage.yml

@@ -51,5 +51,5 @@ jobs:
         uses: andymckay/append-gist-action@1fbfbbce708a39bd45846f0955ed5521f2099c6d
         with:
           token: ${{ secrets.GIST_TOKEN }}
-          gistURL: https://gist.githubusercontent.com/revittm/ee38e9beb22353eef6b88f2ad6ed7aa9
+          gistURL: https://gist.githubusercontent.com/revitteth/ee38e9beb22353eef6b88f2ad6ed7aa9
           file: badge.svg

.github/workflows/release.yml

@@ -1,5 +1,10 @@
 name: Release
 
+# Uncomment the following to let goreleaser automatically
+# create a GitHub release when a tag is pushed.
+# permissions:
+#   contents: write
+
 on:
   push:
     branches-ignore:

.github/workflows/stale-issues.yml

@@ -0,0 +1,23 @@
+name: 'Close stale issues and PRs'
+on:
+  schedule:
+    - cron: '30 1 * * *'
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v6
+        with: # core team are exempt
+          exempt-issue-assignees: 'AskAlexSharov,realLedgerwatch,AndreaLanfranchi,yperbasis,vorot93,b00ris,JekaMas,mandrigin,Giulio2002,tjayrush,revitteth,hexoscott' 
+          exempt-pr-assignees: 'AskAlexSharov,realLedgerwatch,AndreaLanfranchi,yperbasis,vorot93,b00ris,JekaMas,mandrigin,Giulio2002,tjayrush,revitteth,hexoscott'
+          stale-issue-message: 'This issue is stale because it has been open for 40 days with no activity. Remove stale label or comment, or this will be closed in 7 days.'
+          stale-pr-message: 'This PR is stale because it has been open for 40 days with no activity.'
+          close-issue-message: 'This issue was closed because it has been stalled for 7 days with no activity.'
+          days-before-stale: 40
+          days-before-close: 7
+          days-before-pr-close: -1 # don't close PRs

.gitignore

@@ -81,3 +81,4 @@ dist
 .env
 coverage.out
 
+dist

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "tests"]
 	path = tests/testdata
 	url = https://github.com/ethereum/tests
-[submodule "libmdbx"]
-	path = libmdbx
-	url = https://github.com/torquem-ch/libmdbx.git

.golangci.yml

@@ -5,19 +5,19 @@ linters:
   disable-all: true
   enable:
     - gofmt
-    - deadcode
     - errcheck
-#    - gosimple # 1.18
+    - gosimple
     - govet
     - ineffassign
-#    - staticcheck # 1.18
+    - staticcheck
 #    - structcheck # 1.18
 #    - unused # 1.18
-    - varcheck
-#    - gocritic
-#    - bodyclose # 1.18
-#    - gosec
+    - gocritic
+    - bodyclose
+    - gosec
 #    - forcetypeassert
+    - prealloc
+    - unconvert
 
 linters-settings:
   gocritic:

Dockerfile

@@ -1,5 +1,5 @@
 # syntax = docker/dockerfile:1.2
-FROM docker.io/library/golang:1.18-alpine3.15 AS builder
+FROM docker.io/library/golang:1.19-alpine3.16 AS builder
 
 RUN apk --no-cache add build-base linux-headers git bash ca-certificates libstdc++
 
@@ -11,9 +11,9 @@ RUN --mount=type=cache,target=/root/.cache \
     --mount=type=cache,target=/go/pkg/mod \
     make all db-tools
 
-FROM docker.io/library/alpine:3.15
+FROM docker.io/library/alpine:3.16
 
-RUN apk add --no-cache ca-certificates libstdc++ tzdata
+RUN apk add --no-cache ca-certificates curl libstdc++ jq tzdata
 # copy compiled artifacts from builder
 COPY --from=builder /app/build/bin/* /usr/local/bin/