Adopt publishing and tagging

whisklabs · Nov 5, 2024 · 88f3d17 · 88f3d17
1 parent 23cf88d
commit 88f3d17
Show file tree

Hide file tree

Showing 16 changed files with 473 additions and 132 deletions.
diff --git a/.envrc b/.envrc
@@ -0,0 +1 @@
+use flake
diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
diff --git a/.github/actions/scala_restore_cache/action.yml b/.github/actions/scala_restore_cache/action.yml
@@ -0,0 +1,50 @@
+name: restore-sbt-cache
+description: 'cache sbt deps and compiled classes'
+author: 'Dmitry Muzyka'
+inputs:
+  clean_build:
+    description: 'build all'
+    required: false
+    default: "false"
+runs:
+  using: 'composite'
+  steps:
+    - name: Make cache key files
+      shell: bash
+      run: |
+        echo "${LAST_COMPLETED_JOB_SHA1}" > last-completed-job.sha1
+        echo "${GITHUB_SHA}" > current.sha1
+        echo "${LATEST_CACHE_SHA1}" > latest-maybe-failed.sha1
+        date +%Y-%m > current_month
+
+    - name: Restore compilation cache
+      if: ${{ inputs.clean_build != 'true' }}
+      uses: actions/cache/restore@v3
+      with:
+        path: |
+          build_cache
+        key: v2-build-cache-${{ github.job }}-${{ hashFiles('current_month') }}-${{ hashFiles('current.sha1') }}
+        restore-keys: |
+          v2-build-cache-${{ github.job }}-${{ hashFiles('current_month') }}-${{ hashFiles('current.sha1') }}
+          v2-build-cache-${{ github.job }}-${{ hashFiles('current_month') }}-${{ hashFiles('latest-maybe-failed.sha1') }}
+          v2-build-cache-${{ github.job }}-${{ hashFiles('current_month') }}-${{ hashFiles('last-completed-job.sha1') }}
+          v2-build-cache-${{ github.job }}-${{ hashFiles('current_month') }}-
+
+    - name: Apply compilation cache
+      if: ${{ inputs.clean_build != 'true' }}
+      shell: bash
+      run: |
+        RUNNER_TRACKING_ID="" && tar -xf build_cache/targets.tar 2>/dev/null &
+
+    - name: cleanup sbt cache once per month
+      shell: bash
+      run: |
+          mkdir -p ~/.cache
+          test -f ~/.cache/.timecreated || touch ~/.cache/.timecreated
+          if test "$(find  ~/.cache/.timecreated -mtime +30)"; then
+            echo "run cleanup"
+            find ~/.cache ~/.ivy2 -type f -mtime +30 -delete
+            find ~/.cache ~/.ivy2 -type d -empty -delete
+            mkdir -p ~/.cache
+            touch ~/.cache/.timecreated
+          fi
diff --git a/.github/actions/scala_save_cache/action.yml b/.github/actions/scala_save_cache/action.yml
@@ -0,0 +1,44 @@
+name: save-sbt-cache
+description: 'cache sbt deps and compiled classes'
+author: 'Dmitry Muzyka'
+inputs:
+  clean_build:
+    description: 'build all'
+    required: false
+    default: "false"
+runs:
+  using: 'composite'
+  steps:
+    - name: Build cache directory
+      shell: bash
+      run: |
+        targets=$(git clean -ndx | cut -c14- | grep /$ | grep -E -v '(build_cache|persist|test_results)')
+        tar -cf build_cache/targets.tar --exclude=*/target/test-reports/*.xml --exclude=*.log --exclude=*/target/scoverage-report --exclude=*/target/coverage-report --exclude=project/target/active.json $targets || :
+
+    - name: Make cache key file
+      shell: bash
+      run: |
+          echo "${GITHUB_SHA}" > current.sha1
+          date +%Y-%m > current_month
+
+    - name: check if compilation cache exists
+      id: check-compilation-cache
+      uses: actions/cache/restore@v3
+      with:
+        path: |
+          build_cache
+        key: v2-build-cache-${{ github.job }}-${{ hashFiles('current_month') }}-${{ hashFiles('current.sha1') }}
+        lookup-only: 'true'
+
+    - name: Save compilation cache
+      if: steps.check-compilation-cache.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v3
+      with:
+        path: |
+          build_cache
+        key: v2-build-cache-${{ github.job }}-${{ hashFiles('current_month') }}-${{ hashFiles('current.sha1') }}
+
+    - name: Clean up
+      shell: bash
+      run: rm -rf build_cache current.sha1
+    # / save sbt cache
diff --git a/.github/actions/set_vars/action.yml b/.github/actions/set_vars/action.yml
@@ -0,0 +1,57 @@
+name: set-ci-vars
+description: 'set ci vars.'
+author: 'Dmitry Muzyka'
+inputs:
+  clean_build:
+    description: 'build all'
+    required: false
+    default: "false"
+runs:
+  using: 'composite'
+  steps:
+    - name: "set git config"
+      shell: bash
+      run: |
+        git config --global --add safe.directory "$(pwd)"
+        git fetch origin master
+        git describe --tags --abbrev=0 --first-parent
+
+    # set vars
+    - name: set branch
+      shell: bash
+      run: |
+        branch="$(echo -n ${{ github.event.ref }} | sed 's#refs/heads/##g; s#/#-#g' | tr '[:upper:]' '[:lower:]')"
+        echo "branch=\"${branch}\"" >> $GITHUB_ENV
+
+    - name: set new version
+      shell: bash
+      run: |
+        if [[ ${{env.branch}} == "master" ]]; then
+            version="$(date +'%Y.%m.%d')-${{github.run_number}}"
+        else
+          version="$(date +'%Y.%m.%d')-${branch}-${{github.run_number}}"
+        fi
+        version=$(echo $version | sed 's/"//g')
+        echo "version=$version"
+        echo "version=$version" >> $GITHUB_ENV
+
+    - name: Set LAST_COMPLETED_JOB_SHA1
+      shell: bash
+      run: |
+          if [[ '${{ inputs.clean_build }}' == 'true' ]]; then
+              echo "clean build, will bundle & test all"
+              echo "LAST_COMPLETED_JOB_SHA1=None" >> $GITHUB_ENV
+          else
+            .github/helpers/set-last-completed-job-sha1.sh
+          fi
+
+    - name: print debug info
+      shell: bash
+      run: |
+          echo "previous successful build sha: ${LAST_COMPLETED_JOB_SHA1}"
+          echo "previous successful version: $(git tag --points-at ${LAST_COMPLETED_JOB_SHA1})"
+          echo "latest cache sha: ${LATEST_CACHE_SHA1}"
+          echo "branch: ${branch}"
+          echo "version: ${version}"
+
+    # /set vars
diff --git a/.github/helpers/set-last-completed-job-sha1.sh b/.github/helpers/set-last-completed-job-sha1.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+set -exo pipefail
+
+### find previous sucessfull tag
+# shellcheck disable=SC2154
+if [[ ${branch} == "master" ]]; then
+    prev_build=$(git describe --tags --abbrev=0 --first-parent --exclude='????.??.??-*-*')
+else
+    prev_build=$(git describe --tags --abbrev=0 --first-parent)
+fi
+# if we have multiple tags on same commit, pick latest
+prev_build=$(git tag --points-at "${prev_build}" | sort | tail -n 1)
+git_prev_build_sha=$(git rev-list -n1 "${prev_build}")
+
+function get_diff_count() {
+  changed=$(git diff "${1}" --shortstat -- | awk '{print $1}')
+  if [[ -z $changed ]]; then
+    changed=0
+  fi
+  echo -n $changed
+}
+LAST_COMPLETED_JOB_SHA1=${git_prev_build_sha}
+
+
+# this needed as we have tag only healthy builds. And probably don't want to compare with unhealthy build
+master_tag=$(git describe --tags --abbrev=0 --first-parent --exclude='????.??.??-*-*' origin/master)
+master_sha=$(git rev-list -n1 "${master_tag}")
+changes_master=$(get_diff_count "${master_sha}")
+
+changes_git_tag=$(get_diff_count "${git_prev_build_sha}")
+
+changes_current_candidate=$(get_diff_count "${LAST_COMPLETED_JOB_SHA1}")
+
+if [[ ${changes_current_candidate} -gt ${changes_master} ]]; then
+  echo "as previuos will choose version from master ${master_tag}"
+  LAST_COMPLETED_JOB_SHA1=${master_sha}
+  changes_current_candidate=$(get_diff_count "${LAST_COMPLETED_JOB_SHA1}")
+elif [[ ${changes_current_candidate} -gt ${changes_git_tag} ]]; then
+  echo "as previuos will choose version from git tag ${prev_build}"
+  LAST_COMPLETED_JOB_SHA1=${git_prev_build_sha}
+  changes_current_candidate=$(get_diff_count "${LAST_COMPLETED_JOB_SHA1}")
+fi
+echo "LAST_COMPLETED_JOB_SHA1=${LAST_COMPLETED_JOB_SHA1}" >> $GITHUB_ENV
+
+
+LATEST_CACHE_SHA1=${LAST_COMPLETED_JOB_SHA1}
+
+
+set +xo pipefail
+retrive_tag_num=30
+last_builds=$(git tag -l --sort='-authordate' | head -n "${retrive_tag_num}" | uniq)
+set -o pipefail
+last_builds_array=($last_builds)
+tmp_dir='/tmp/'
+# changes_current_candidate
+(get_diff_count "${LATEST_CACHE_SHA1}"; echo -n " ${LATEST_CACHE_SHA1}") > "$tmp_dir/current.diff" &
+# git tags
+for i in $(seq 0 "$((retrive_tag_num-1))"); do
+  (get_diff_count "${last_builds_array[$i]}"; echo " ${last_builds_array[$i]}") > "$tmp_dir/$i.diff" &
+done
+wait
+cache_candidate=$(cat "$tmp_dir"/*.diff | sort -n | head -n 1)
+tag_with_min_changes=$(echo "$cache_candidate" | awk '{print $2}')
+min_changes=$(echo "$cache_candidate" | awk '{print $1}')
+
+echo "as previous will choose version from commit  ${tag_with_min_changes} with ${min_changes} changes"
+LATEST_CACHE_SHA1=$(git rev-list -n1 "${tag_with_min_changes}")
+changes_current_candidate=$(get_diff_count "${LATEST_CACHE_SHA1}")
+
+echo "LATEST_CACHE_SHA1=${LATEST_CACHE_SHA1}" >> $GITHUB_ENV
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,78 +1,77 @@
-name: CI
-
-on: [push, pull_request]
+name: ci
+on:
+  push:
+    branches:
+      - '**'
+  workflow_dispatch:
+    inputs:
+      clean_build:
+        description: 'build all'
+        required: false
+        default: false
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref_name != 'master' }}
 
 jobs:
-  build:
-    runs-on: ${{matrix.os}}
-    strategy:
-      matrix:
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-
-    steps:
-    - uses: actions/checkout@v4
-    - uses: olafurpg/setup-scala@v14
-      with:
-        java-version: 8
-    - name: Mount caches
-      uses: actions/cache@v2
-      with:
-        path: |
-          ~/.sbt
-          ~/.ivy2/cache
-          ~/.cache/coursier
-        key: ${{ runner.os }}-sbt-${{ hashFiles('**/*.sbt') }}
-    - name: Remove native tests (Windows only)
-      if: ${{ runner.os == 'Windows' }}
-      run: |
-        rm -rf examples/scalapb-crossproject
-      shell: bash
-    - name: Compile and test
-      run: |
-       sbt test
-       cd examples
-       for d in */ ; do cd "$d" && sbt test && cd ../ ; done
-      shell: bash
-    - name: Format check
-      if: ${{ runner.os == 'Linux' }}
-      run: |
-        sbt scalafmtCheck test:scalafmtCheck scalafmtSbtCheck
-  scripted:
-    runs-on: ${{matrix.os}}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        scripted-sbt: ["1.2.8", "1.3.13", "project"]
-
-    steps:
-    - uses: actions/checkout@v4
-    - uses: olafurpg/setup-scala@v14
-      with:
-        java-version: 11
-    - name: Mount caches
-      uses: actions/cache@v2
-      with:
-        path: |
-          ~/.sbt
-          ~/.ivy2/cache
-          ~/.cache/coursier
-        key: ${{ runner.os }}-sbt-${{ hashFiles('**/*.sbt') }}
-    - name: Compile and run scripted tests with older version
-      if: ${{ matrix.scripted-sbt != 'project' }}
-      env:
-        SCRIPTED_SBT: ${{ matrix.scripted-sbt }}
-      run: |
-       sbt "set scriptedSbt := \"$SCRIPTED_SBT\"" scripted
-      shell: bash
-    - name: Compile and run scripted tests with project version
-      if: ${{ matrix.scripted-sbt == 'project' }}
-      run: |
-       sbt "set scriptedSbt := sbtVersion.value" scripted
-      shell: bash
-  # Single final job for mergify.
-  ci-passed:
+  test:
     runs-on: ubuntu-latest
-    needs: [build, scripted]
+    timeout-minutes: 25
+    env:
+      # define Java options for both official sbt and sbt-extras
+      JAVA_OPTS: -XX:MinRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0 -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
+      JVM_OPTS:  -XX:MinRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0 -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
     steps:
-    - run: ':'
+      - id: checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 100
+          fetch-tags: true
+          sparse-checkout-cone-mode: false
+
+      # set vars
+      - name: set vars
+        uses: ./.github/actions/set_vars
+        with:
+          clean_build: ${{ github.event.inputs.clean_build }}
+
+      - name: Setup Scala
+        uses: actions/setup-java@v4
+        with:
+          java-version: "21"
+          distribution: 'temurin'
+
+      - name: restore cache
+        uses: ./.github/actions/scala_restore_cache
+        with:
+          clean_build: ${{ github.event.inputs.clean_build }}
+
+      - name: Setup M2 Credentials
+        run: mkdir -p ~/.m2 && echo ${{secrets.M2_CREDENTIALS}} | base64 -d > ~/.m2/.credentials
+
+      - name: set branch
+        run: |
+          branch="$(echo -n ${{ github.event.ref }} | sed 's#refs/heads/##g; s#/#-#g' | tr '[:upper:]' '[:lower:]')"
+          echo "branch=\"${branch}\"" >> $GITHUB_ENV
+
+      - name: set new version
+        run: |
+          if [[ ${{env.branch}} == "master" ]]; then
+              version="$(date +'%Y.%m.%d')-${{github.run_number}}"
+          else
+            version="$(date +'%Y.%m.%d')-${branch}-${{github.run_number}}"
+          fi
+          version=$(echo $version | sed 's/"//g')
+          echo "version=$version"
+          echo "version=$version" >> $GITHUB_ENV
+
+      - run: |
+          echo "version in ThisBuild := \"${{env.version}}\"" > version.sbt
+
+      - run: sbt publish
+
+      - name: label vcs
+        run: git tag $version && git push --tag
+      - name: Save Scala Cache
+        uses: ./.github/actions/scala_save_cache
+