saving/restoring classess is waaaaaaay longer than just recompiling

Signed-off-by: Justin Florentine <[email protected]>
hyperledger · Dec 6, 2023 · 2fc042c · 2fc042c
1 parent bf9789a
commit 2fc042c
Show file tree

Hide file tree

Showing 17 changed files with 797 additions and 80 deletions.
diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml
@@ -0,0 +1,91 @@
+name: acceptance-tests
+on:
+  workflow_call:
+
+
+env:
+  GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
+  total-runners: 6
+
+jobs:
+  runner-indexes:
+    runs-on: ubuntu-22.04
+    name: Generate runner indexes
+    outputs:
+      json: ${{ steps.generate-index-list.outputs.json }}
+    steps:
+      - id: generate-index-list
+        run: |
+          MAX_INDEX=$((${{ env.total-runners }}-1))
+          INDEX_LIST=$(seq 0 ${MAX_INDEX})
+          INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST})
+          echo "::set-output name=json::${INDEX_JSON}"
+  acceptanceTestEthereum:
+    runs-on: [ self-hosted, X64, Linux ]
+    name: "Acceptance Runner #${{ matrix.runner-index }}: Run acceptance tests in parallel"
+    needs:
+      - runner-indexes
+    if: ${{ github.actor != 'dependabot[bot]' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        runner-index: ${{ fromjson(needs.runner-indexes.outputs.json) }}
+    steps:
+      - name: export runner UID
+        run: echo "runner_uid=$UID" >> $GITHUB_ENV
+      - name: install native packages
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libsodium23 libsodium-dev libjemalloc-dev apt-transport-https haveged libnss3-tools
+          sudo service haveged restart
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 17
+          cache: gradle
+      - name: Restore classes
+        uses: actions/cache@v3
+        with:
+          path: "**/*.class"
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
+      - name: Split tests
+        id: split-tests
+        uses: chaosaffe/[email protected]
+        with:
+          glob: 'acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/**/*Test.java'
+          split-total: ${{ env.total-runners }}
+          split-index: ${{ matrix.runner-index }}
+
+      - name: write out test list
+        run: echo "${{ steps.split-tests.outputs.test-suite }}" >> testList.txt
+      - name: format gradle args
+        #regex means: first truncate file paths to align with package name, then swap path delimiter with package delimiter,
+        #then drop file extension, then insert --tests option between each.
+        run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@@g' -e 's@/@.@g' -e 's/\.java//g' -e 's/\ /\ --tests\ /g' >> gradleArgs.txt
+      - name: list keystores
+        run: find . -name "*.jks"
+      - name: run acceptance tests
+        run: ./gradlew acceptanceTest --tests `cat gradleArgs.txt`
+      - name: cleanup tempfiles
+        run: rm testList.txt gradleArgs.txt
+      - name: Correct Ownership in GITHUB_WORKSPACE directory
+        uses: peter-murray/reset-workspace-ownership-action@v1
+        with:
+          user_id: ${{ env.runner_uid }}
+      - name: Upload Test Report
+        uses: actions/upload-artifact@v3
+        if: always() # always run even if the previous step fails
+        with:
+          name: junit-acceptance-${{ matrix.runner-index }}-test-results
+          path: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml'
+          retention-days: 1
+      - name: Acceptance Test Report html
+        uses: actions/upload-artifact@v3
+        if: always() # always run even if the previous step fails
+        with:
+          name: html-acceptance-${{ matrix.runner-index }}-test-results
+          path: 'acceptance-tests/tests/build/reports/tests/acceptanceTest/**'
+          retention-days: 1
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -1,36 +0,0 @@
-name: checks
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-  workflow_dispatch:
-
-jobs:
-  spotless:
-    runs-on: [besu-research-ubuntu-16]
-    if: ${{ github.actor != 'dependabot[bot]' }}
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v4
-      - name: Set up Java
-        uses: actions/setup-java@v3
-        with:
-          distribution: adopt
-          java-version: 17
-          cache: gradle
-      - name: spotless
-        run: ./gradlew --no-daemon --parallel clean spotlessCheck
-  javadoc_17:
-    runs-on: [besu-research-ubuntu-8]
-    if: ${{ github.actor != 'dependabot[bot]' }}
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v4
-      - name: Set up Java 17
-        uses: actions/setup-java@v3
-        with:
-          distribution: adopt
-          java-version: 17
-          cache: gradle
-      - name: javadoc (JDK 17)
-        run: ./gradlew --no-daemon clean javadoc

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,172 @@
+name: ci
+on:
+  workflow_call:
+
+
+env:
+  GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
+  total-runners: 6
+
+jobs:
+  javadoc_17:
+    runs-on: [ self-hosted ]
+    needs: compile
+    if: ${{ github.actor != 'dependabot[bot]' }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Set up Java 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 17
+          cache: gradle
+      - name: Restore classes
+        uses: actions/cache@v3
+        with:
+          path: "**/*.class"
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
+      - name: javadoc (JDK 17)
+        run: ./gradlew --no-daemon javadoc
+  assemble:
+    runs-on: [self-hosted, Linux, X64]
+    needs: compile
+    timeout-minutes: 30
+    if: ${{ github.actor != 'dependabot[bot]' }}
+    steps:
+      - name: export runner UID
+        run: echo "runner_uid=$UID" >> $GITHUB_ENV
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 17
+          cache: gradle
+      - name: Restore classes
+        uses: actions/cache@v3
+        with:
+          path: "**/*.class"
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
+      - name: Assemble
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: assemble -Dorg.gradle.parallel=true
+      - uses: actions/[email protected]
+        with:
+          name: besu-zip
+          path: build/distributions/*.zip
+      - name: Correct Ownership in GITHUB_WORKSPACE directory
+        uses: peter-murray/reset-workspace-ownership-action@v1
+        with:
+          user_id: ${{env.runner_uid}}
+  testWindows:
+    runs-on: windows-2022
+    needs: assemble
+    timeout-minutes: 10
+    if: ${{ github.actor != 'dependabot[bot]' }}
+    steps:
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 17
+      - name: Download zip
+        uses: actions/download-artifact@v3
+        with:
+          name: besu-zip
+      - name: test Besu
+        run: |
+          unzip besu-*.zip -d besu-tmp
+          cd besu-tmp
+          mv besu-* ../besu
+          cd ..
+          besu\bin\besu.bat --help
+          besu\bin\besu.bat --version
+  runner-indexes:
+    runs-on: ubuntu-22.04
+    name: Generate runner indexes
+    outputs:
+      json: ${{ steps.generate-index-list.outputs.json }}
+    steps:
+      - id: generate-index-list
+        run: |
+          MAX_INDEX=$((${{ env.total-runners }}-1))
+          INDEX_LIST=$(seq 0 ${MAX_INDEX})
+          INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST})
+          echo "::set-output name=json::${INDEX_JSON}"
+  run-tests:
+    runs-on: [self-hosted, Linux, X64]
+    timeout-minutes: 60
+    name: "Run test suite"
+    needs:
+      - compile
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 17
+          cache: gradle
+      - name: Restore classes
+        uses: actions/cache@v3
+        with:
+          path: "**/*.class"
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
+      - name: test
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: test
+      - name: Upload Test Report
+        uses: actions/upload-artifact@v3
+        if: always() # always run even if the previous step fails
+        with:
+          name: junit-test-results
+          path: '**/build/test-results/test/TEST-*.xml'
+          retention-days: 1
+      - name: Upload Test Report html
+        uses: actions/upload-artifact@v3
+        if: always() # always run even if the previous step fails
+        with:
+          name: junit-test-results-html
+          path: '**/build/reports/*'
+          retention-days: 1
+  integration-test:
+    runs-on: [self-hosted, Linux, X64]
+    needs: compile
+    timeout-minutes: 30
+    if: ${{ github.actor != 'dependabot[bot]' }}
+    steps:
+      - name: export runner UID
+        run: echo "runner_uid=$UID" >> $GITHUB_ENV
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 17
+          cache: gradle
+      - name: Restore classes
+        uses: actions/cache@v3
+        with:
+          path: "**/*.class"
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }}
+      - name: test
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: integrationTest
+      - name: Correct Ownership in GITHUB_WORKSPACE directory
+        uses: peter-murray/reset-workspace-ownership-action@v1
+        with:
+          user_id: ${{ env.runner_uid }}
+      - name: Upload Test Report
+        uses: actions/upload-artifact@v3
+        if: always() # always run even if the previous step fails
+        with:
+          name: junit-integration-test-results
+          path: '**/build/test-results/integrationTest/TEST-*.xml'
+          retention-days: 1
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -12,15 +12,7 @@
 name: "CodeQL"
 
 on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-    paths-ignore:
-      - '**/*.json'
-      - '**/*.md'
-      - '**/*.properties'
-      - '**/*.txt'
+  workflow_call:
 jobs:
   analyze:
     name: Analyze
@@ -60,7 +52,6 @@ jobs:
     # Autobuild failed (OOM)
     # Hence, supply memory args for gradle build
     - run: |
-        JAVA_OPTS="-Xmx1000M" ./gradlew --no-scan compileJava
-
+        JAVA_OPTS="-Xmx2048M" ./gradlew --no-scan compileJava
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml
@@ -0,0 +1,63 @@
+name: compile
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+
+env:
+  GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false"
+  total-runners: 6
+
+jobs:
+  compile:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    if: ${{ github.actor != 'dependabot[bot]' }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 17
+          cache: gradle
+      - name: Compile
+        uses: gradle/gradle-build-action@v2
+        with:
+          gradle-home-cache-cleanup: true
+          cache-read-only: false
+          arguments: compileJava compileTestJava -Dorg.gradle.parallel=true
+  compileJmh:
+    runs-on: ubuntu-22.04
+    needs: compile
+    timeout-minutes: 30
+    if: ${{ github.actor != 'dependabot[bot]' }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 17
+          cache: gradle
+      - name: compileJmh
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: compileJmh
+
+  javadoc_17:
+    runs-on: ubuntu-22.04
+    if: ${{ github.actor != 'dependabot[bot]' }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Set up Java 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 17
+          cache: gradle
+      - name: javadoc (JDK 17)
+        run: ./gradlew --no-daemon javadoc