Merge remote-tracking branch 'ledgerwatch/devel' into upstream-changes

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: Continuous integration
+name: CI
 on:
   push:
     branches:
@@ -10,8 +10,15 @@ on:
       - devel
       - alpha
       - stable
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+
 jobs:
   tests:
+    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
     strategy:
       matrix:
         os: [ ubuntu-20.04, macos-11 ] # list of os: https://github.com/actions/virtual-environments
@@ -53,14 +60,13 @@ jobs:
         if: runner.os == 'Linux'
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.46
-          skip-pkg-cache: true
-          skip-build-cache: true
+          version: v1.47
 
       - name: Test
         run: make test
 
   tests-windows:
+    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
     strategy:
       matrix:
         os: [ windows-2022 ]
@@ -99,15 +105,63 @@ jobs:
         run: .\wmake.ps1 test
 
   docker:
+    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0 # fetch git tags for "git describe"
 
       - name: make docker
-        run: DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
+        run: DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
 
       # check with root permissions, should be cached from previous build
       - name: sudo make docker
-        run: sudo DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
+        run: sudo DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
+
+  hive:
+    needs:
+      - tests
+      - tests-windows
+      - docker
+    runs-on: ubuntu-20.04
+    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # fetch git tags for "git describe"
+
+      - name: build erigon image
+        run: DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
+
+      # check with root permissions, should be cached from previous build
+      - name: build erigon image (root permissions)
+        run: sudo DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
+
+      - name: run hive
+        run: sudo mkdir /results && docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}:/work gatewayfm/hive:latest --sim ethereum/engine --results-root=/work/results --client erigon_ci-$GITHUB_SHA --docker.output --loglevel 5
+
+      - name: parse hive output
+        run: docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}:/work --entrypoint /app/hivecioutput gatewayfm/hive:latest --resultsdir=/work/results --outdir=/work/results
+
+      - name: archive hive results
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: hive-ci-output
+          path: results/*.xml
+
+  event_file:
+    needs:
+      - tests
+      - tests-windows
+      - docker
+    name: archive event file
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
+    steps:
+      - name: upload
+        uses: actions/upload-artifact@v2
+        with:
+          name: event file
+          path: ${{ github.event_path }}

.github/workflows/coverage.yml

@@ -0,0 +1,55 @@
+name: Coverage
+on:
+  push:
+    branches:
+      - devel
+
+jobs:
+  coverage:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v3
+      - run: git submodule update --init --recursive --force
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18.x
+
+      - name: install dependencies on Linux
+        if: runner.os == 'Linux'
+        run: sudo apt update && sudo apt install build-essential
+
+      - name: run coverage
+        run: echo "COVERAGE=$(make coverage)" >> $GITHUB_ENV
+
+      - name: set badge color
+        shell: bash
+        run: |
+          if [ ${{ env.COVERAGE }} -lt 40 ] 
+          then
+          echo "BADGE_COLOR=800000" >> $GITHUB_ENV
+          elif [ ${{ env.COVERAGE }} -lt 75 ]
+          then
+              echo "BADGE_COLOR=696969" >> $GITHUB_ENV
+          else
+              echo "BADGE_COLOR=31c653" >> $GITHUB_ENV
+          fi
+
+      - name: create badge
+        uses: emibcn/badge-action@d6f51ff11b5c3382b3b88689ae2d6db22d9737d1
+        with:
+          label: Coverage
+          status: ${{ env.COVERAGE }}
+          color: ${{ env.BADGE_COLOR }}
+          path: badge.svg
+
+      - name: upload badge to gist
+        if: >
+          github.event_name == 'workflow_run' && github.event.workflow_run.head_branch == 'devel' ||
+          github.event_name != 'workflow_run' && github.ref == 'refs/heads/devel'
+        uses: andymckay/append-gist-action@1fbfbbce708a39bd45846f0955ed5521f2099c6d
+        with:
+          token: ${{ secrets.GIST_TOKEN }}
+          gistURL: https://gist.githubusercontent.com/revittm/ee38e9beb22353eef6b88f2ad6ed7aa9
+          file: badge.svg

.github/workflows/hive-results.yml

@@ -0,0 +1,75 @@
+name: Hive results
+
+on:
+  workflow_run:
+    workflows: ["CI"]
+    types:
+      - completed
+
+jobs:
+  hive-results:
+    name: Hive results
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion != 'skipped'
+
+    permissions:
+      checks: write
+      pull-requests: write
+      actions: read
+
+    steps:
+      - name: download and extract artifacts
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        run: |
+          mkdir -p artifacts && cd artifacts
+          
+          artifacts_url=${{ github.event.workflow_run.artifacts_url }}
+          
+          gh api "$artifacts_url" -q '.artifacts[] | [.name, .archive_download_url] | @tsv' | while read artifact
+          do
+            IFS=$'\t' read name url <<< "$artifact"
+            gh api $url > "$name.zip"
+            unzip -d "$name" "$name.zip"
+          done
+
+      - name: publish hive test results
+        uses: EnricoMi/publish-unit-test-result-action@v1
+        with:
+          commit: ${{ github.event.workflow_run.head_sha }}
+          event_file: artifacts/Event File/event.json
+          event_name: ${{ github.event.workflow_run.event }}
+          files: "artifacts/**/*.xml"
+
+      - name: set badge color
+        shell: bash
+        run: |
+          case ${{ fromJSON( steps.test-results.outputs.json ).conclusion }} in
+            success)
+              echo "BADGE_COLOR=31c653" >> $GITHUB_ENV
+              ;;
+            failure)
+              echo "BADGE_COLOR=800000" >> $GITHUB_ENV
+              ;;
+            neutral)
+              echo "BADGE_COLOR=696969" >> $GITHUB_ENV
+              ;;
+          esac
+
+      - name: create badge
+        uses: emibcn/badge-action@d6f51ff11b5c3382b3b88689ae2d6db22d9737d1
+        with:
+          label: Hive
+          status: '${{ fromJSON( steps.test-results.outputs.json ).formatted.stats.tests }} tests, ${{ fromJSON( steps.test-results.outputs.json ).formatted.stats.runs }} runs: ${{ fromJSON( steps.test-results.outputs.json ).conclusion }}'
+          color: ${{ env.BADGE_COLOR }}
+          path: badge.svg
+
+      - name: upload badge to gist
+        if: >
+          github.event_name == 'workflow_run' && github.event.workflow_run.head_branch == 'devel' ||
+          github.event_name != 'workflow_run' && github.ref == 'refs/heads/devel'
+        uses: andymckay/append-gist-action@1fbfbbce708a39bd45846f0955ed5521f2099c6d
+        with:
+          token: ${{ secrets.GIST_TOKEN }}
+          gistURL: https://gist.githubusercontent.com/revittm/dc492845ba6eb694e6c7279224634b20
+          file: badge.svg

.gitignore

@@ -78,4 +78,5 @@ docker-compose.*.yml
 
 dist
 .env
+coverage.out
 

.gitmodules

@@ -1,13 +1,6 @@
 [submodule "tests"]
 	path = tests/testdata
 	url = https://github.com/ethereum/tests
-[submodule "turbo/snapshotsync/snapshothashes/erigon-snapshots"]
-	path = turbo/snapshotsync/snapshothashes/erigon-snapshots
-	url = https://github.com/ledgerwatch/erigon-snapshot.git
-[submodule "cmd/downloader/trackers/trackerslist"]
-	path = cmd/downloader/trackers/trackerslist
-	url = https://github.com/ngosang/trackerslist.git
-
 [submodule "libmdbx"]
 	path = libmdbx
 	url = https://github.com/torquem-ch/libmdbx.git