From 1cc02136a2ff5c5d06d2fd3fdc1b855cbae8360e Mon Sep 17 00:00:00 2001 From: Aleksandr Mikhailov Date: Tue, 5 Nov 2024 08:58:56 +0200 Subject: [PATCH] Use hashing instead of timestamps (#1) * Use hashing instead of timestamps * change CI routines --- .envrc | 1 + .github/workflows/ci.yml | 80 ++++++++++++++++++ .gitignore | 3 + build.sbt | 15 +++- flake.lock | 60 ++++++++++++++ flake.nix | 91 +++++++++++++++++++++ project/plugins.sbt | 4 + shell.nix | 17 ---- src/main/scala/sbtprotoc/ProtocPlugin.scala | 12 +-- version.sbt | 1 + 10 files changed, 260 insertions(+), 24 deletions(-) create mode 100644 .envrc create mode 100644 .github/workflows/ci.yml create mode 100644 flake.lock create mode 100644 flake.nix delete mode 100644 shell.nix create mode 100644 version.sbt diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..3550a30f --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..99a786ce --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,80 @@ +name: ci +on: + push: + branches: + - '**' + workflow_dispatch: + inputs: + clean_build: + description: 'build all' + required: false + default: false +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref_name != 'master' }} + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + # define Java options for both official sbt and sbt-extras + JAVA_OPTS: -XX:MinRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0 -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + JVM_OPTS: -XX:MinRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0 -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + SBT_NATIVE_CLIENT: "true" + steps: + - id: checkout + uses: actions/checkout@v3 + with: + fetch-depth: 100 + fetch-tags: true + sparse-checkout-cone-mode: false + + # set vars + - name: set vars + uses: whisklabs/gha-workflows/actions/set_vars@master + with: + clean_build: ${{ github.event.inputs.clean_build }} + + - name: Setup Scala + uses: actions/setup-java@v4 + with: + java-version: "21" + distribution: 'temurin' + + - name: restore cache + uses: whisklabs/gha-workflows/actions/scala_restore_cache@master + with: + clean_build: ${{ github.event.inputs.clean_build }} + + - name: Setup M2 Credentials + run: mkdir -p ~/.m2 && echo ${{secrets.M2_CREDENTIALS}} | base64 -d > ~/.m2/.credentials + + - name: set branch + run: | + branch="$(echo -n ${{ github.event.ref }} | sed 's#refs/heads/##g; s#/#-#g' | tr '[:upper:]' '[:lower:]')" + echo "branch=\"${branch}\"" >> $GITHUB_ENV + + - name: set new version + run: | + if [[ ${{env.branch}} == "master" ]]; then + version="$(date +'%Y.%m.%d')-${{github.run_number}}" + else + version="$(date +'%Y.%m.%d')-${branch}-${{github.run_number}}" + fi + version=$(echo $version | sed 's/"//g') + echo "version=$version" + echo "version=$version" >> $GITHUB_ENV + + - run: | + echo "version in ThisBuild := \"${{env.version}}\"" > version.sbt + + - run: sbt +outboxProcessor/docker:publishLocal + + - run: docker login -u ${{secrets.DOCKER_USER}} -p ${{secrets.DOCKER_PASS}} quay.io + - run: docker push quay.io/whisk/outbox-processor:${{env.version}} + - name: label vcs + run: git tag $version && git push --tag + - name: Save Scala Cache + uses: whisklabs/gha-workflows/actions/scala_save_cache@master + diff --git a/.gitignore b/.gitignore index 0021a7c4..ec192b7a 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ project/plugins/project/ .scala_dependencies .worksheet metals.sbt + +.direnv +.share \ No newline at end of file diff --git a/build.sbt b/build.sbt index 00cc1d82..a418cc2e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,9 @@ +import sbt.Keys.publishMavenStyle + name := "sbt-protoc" +resolvers += "internal.repo.read" at "https://nexus.whisk-dev.com/repository/whisk-maven-group/" + description := "SBT plugin for generating code from Protocol Buffer using protoc" scalacOptions := Seq("-deprecation", "-unchecked", "-Xlint", "-Yno-adapted-args") @@ -40,6 +44,15 @@ inThisBuild( "thesamet@gmail.com", url("https://www.thesamet.com") ) - ) + ), + scmInfo := Some( + ScmInfo( + url("https://github.com/whisklabs/sbt-protoc"), + "scm:git:github.com/whisklabs/sbt-protoc.git" + ) + ), + publishMavenStyle := true, + credentials += Credentials(Path.userHome / ".m2" / ".credentials"), + publishTo := Some("internal.repo" at "https://nexus.whisk-dev.com/repository/whisk-maven2/") ) ) diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..3c587ae0 --- /dev/null +++ b/flake.lock @@ -0,0 +1,60 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1724235759, + "narHash": "sha256-EWe9KcHXywxUoMzf6FSyl3RoWQ8qvTFC5CHfu089plc=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "38e287d50728dc05c79fcec33fbc5ad18ea20d60", + "type": "github" + }, + "original": { + "owner": "nixos", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..89d48324 --- /dev/null +++ b/flake.nix @@ -0,0 +1,91 @@ +{ + description = "A flake for getting started with Scala."; + inputs = { + nixpkgs.url = "github:nixos/nixpkgs"; + flake-utils.url = "github:numtide/flake-utils"; + }; + outputs = + { self + , nixpkgs + , flake-utils + , + }: + let + supportedSystems = [ + "aarch64-darwin" + "aarch64-linux" + "x86_64-linux" + "x86_64-darwin" + ]; + in + flake-utils.lib.eachSystem supportedSystems ( + system: + let + buildOverlays = + { nixpkgs, system, ... }: + let + pkgs = import nixpkgs { inherit system; }; + makeOverlays = + let + armOverlay = + _: prev: + let + pkgsForx86 = import nixpkgs { localSystem = "x86_64-darwin"; }; + in + prev.lib.optionalAttrs (prev.stdenv.isDarwin && prev.stdenv.isAarch64) { + inherit (pkgsForx86) bloop; + }; + ammoniteOverlay = final: prev: { ammonite = prev.ammonite.override { jre = pkgs.graalvm-ce; }; }; + bloopOverlay = final: prev: { bloop = prev.bloop.override { jre = final.graalvm-ce; }; }; + scalaCliOverlay = final: prev: { scala-cli = prev.scala-cli.override { jre = final.graalvm-ce; }; }; + javaOverlay = final: _: { + jdk = pkgs.graalvm-ce; + jre = pkgs.graalvm-ce; + }; + in + [ + javaOverlay + armOverlay + bloopOverlay + scalaCliOverlay + ammoniteOverlay + ]; + + makePackages = + let + overlays = makeOverlays; + in + import nixpkgs { inherit system overlays; }; + default = makePackages; + in + { + inherit default; + }; + + pkgs = buildOverlays { inherit nixpkgs system; }; + java = pkgs.default.graalvm-ce; + in + { + devShells.default = pkgs.default.mkShell { + buildInputs = with pkgs.default; [ + ammonite + bloop + coursier + graalvm-ce + sbt + scala-cli + scalafmt + ]; + shellHook = '' + #SHOVE THIS JDK SOMEWHERE TO MAKE IDEA HAPPY + mkdir -p ./.share + if [ -L "./.share/java" ]; then + unlink "./.share/java" + fi + ln -sf ${java} ./.share/java + ''; + }; + formatter = pkgs.default.nixpkgs-fmt; + } + ); +} diff --git a/project/plugins.sbt b/project/plugins.sbt index 16cfaad9..5be00b09 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,3 +3,7 @@ libraryDependencies += "org.scala-sbt" %% "scripted-plugin" % sbtVersion.value addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0") addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.9.0") + +credentials += Credentials(Path.userHome / ".m2" / ".credentials") +resolvers += "internal.repo.read" at "https://nexus.whisk-dev.com/repository/whisk-maven-group/" +addSbtPlugin("com.whisk" % "whisk-sbt-plugin" % "2024.10.29-2357") diff --git a/shell.nix b/shell.nix deleted file mode 100644 index 97f70a84..00000000 --- a/shell.nix +++ /dev/null @@ -1,17 +0,0 @@ -{pkgs ? import { - config = { - packageOverrides = pkgs: { - sbt = pkgs.sbt.override { jre = pkgs.openjdk11; }; - }; - }; -}} : -pkgs.mkShell { - buildInputs = [ - pkgs.sbt - pkgs.openjdk11 - pkgs.nodejs - - # keep this line if you use bash - pkgs.bashInteractive - ]; -} diff --git a/src/main/scala/sbtprotoc/ProtocPlugin.scala b/src/main/scala/sbtprotoc/ProtocPlugin.scala index 8ae10cf3..47d78a43 100644 --- a/src/main/scala/sbtprotoc/ProtocPlugin.scala +++ b/src/main/scala/sbtprotoc/ProtocPlugin.scala @@ -484,7 +484,7 @@ object ProtocPlugin extends AutoPlugin { val cached = FileFunction.cached( streams.cacheDirectory / dep.name, inStyle = FilesInfo.lastModified, - outStyle = FilesInfo.exists + outStyle = FilesInfo.hash ) { deps => IO.createDirectory(extractTarget) deps.flatMap { dep => @@ -539,7 +539,7 @@ object ProtocPlugin extends AutoPlugin { private[this] val classloaderCache = new java.util.concurrent.ConcurrentHashMap[ BridgeArtifact, - (FilesInfo[ModifiedFileInfo], URLClassLoader) + (FilesInfo[HashFileInfo], URLClassLoader) ] private[this] def schemasTask(key: TaskKey[_]): Def.Initialize[Task[Set[File]]] = Def.task { @@ -612,7 +612,7 @@ object ProtocPlugin extends AutoPlugin { // stamped and compared with the previous stamp to reload the classloader upon changes. Artifact resolution is // memoized across invocations for the entire sbt session, unless cacheArtifactResolution is false. val stampedClassLoadersByArtifact - : Map[BridgeArtifact, (FilesInfo[ModifiedFileInfo], ClassLoader)] = + : Map[BridgeArtifact, (FilesInfo[HashFileInfo], ClassLoader)] = targets .collect { case Target(SandboxedJvmGenerator(_, artifact, _, _), _, _) => artifact } .distinct @@ -622,7 +622,7 @@ object ProtocPlugin extends AutoPlugin { { (_, prevValue) => def stampClasspath(files: Seq[File]) = // artifact paths can be JARs or directories, so a recursive stamp is needed - FileInfo.lastModified(files.toSet[File].allPaths.get.toSet) + FileInfo.hash(files.toSet[File].allPaths.get.toSet) if (prevValue == null) { // first time this classpath is requested since the start of sbt @@ -681,7 +681,7 @@ object ProtocPlugin extends AutoPlugin { } import CacheImplicits._ - type Stamp = (Arguments, Seq[FilesInfo[ModifiedFileInfo]]) + type Stamp = (Arguments, Seq[FilesInfo[HashFileInfo]]) val cachedCompile = Tracked.inputChanged[Stamp, Set[File]]( cacheFile / "input" ) { case (inChanged, _) => @@ -711,7 +711,7 @@ object ProtocPlugin extends AutoPlugin { val sandboxedArtifactsStamps = stampedClassLoadersByArtifact.values.map(_._1).toSeq val inputStamp = - FileInfo.lastModified(schemas ++ arguments.includePaths.allPaths.get) + FileInfo.hash(schemas ++ arguments.includePaths.allPaths.get) cachedCompile((arguments, sandboxedArtifactsStamps :+ inputStamp)).toSeq } } diff --git a/version.sbt b/version.sbt new file mode 100644 index 00000000..c85970f7 --- /dev/null +++ b/version.sbt @@ -0,0 +1 @@ +ThisBuild / version := sys.env.getOrElse("BUILD_VERSION", "0.1.0")