From 724c155623d5865660d2a36e10c842af8485fb96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ossowski?= Date: Thu, 31 Aug 2023 14:58:09 +0200 Subject: [PATCH] improve performance of multimodule build by reusing compiler objects --- .../ForkedSbtIncrementalCompilerMain.java | 19 +-- .../InProcessSbtIncrementalCompiler.java | 112 ++++++++++-- .../java/sbt_inc/SbtIncrementalCompiler.java | 5 +- .../java/sbt_inc/SbtIncrementalCompilers.java | 159 ++++++------------ .../scala_maven/ScalaCompilerSupport.java | 10 +- 5 files changed, 167 insertions(+), 138 deletions(-) diff --git a/src/main/java/sbt_inc/ForkedSbtIncrementalCompilerMain.java b/src/main/java/sbt_inc/ForkedSbtIncrementalCompilerMain.java index 9f143be6..ac14c213 100644 --- a/src/main/java/sbt_inc/ForkedSbtIncrementalCompilerMain.java +++ b/src/main/java/sbt_inc/ForkedSbtIncrementalCompilerMain.java @@ -8,7 +8,6 @@ import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; -import sbt.internal.inc.ScalaInstance; import sbt.util.Level; import sbt.util.Logger; import scala.Enumeration; @@ -165,26 +164,22 @@ public void success(Function0 message) { public void trace(Function0 t) {} }; - ScalaInstance scalaInstance = - ScalaInstances.makeScalaInstance( - parsedArgs.scalaVersion, - parsedArgs.compilerAndDependencies, - parsedArgs.libraryAndDependencies); - SbtIncrementalCompiler incrementalCompiler = SbtIncrementalCompilers.makeInProcess( parsedArgs.javaHome, - parsedArgs.cacheFile, - parsedArgs.compileOrder, - scalaInstance, parsedArgs.compilerBridgeJar, - sbtLogger); + sbtLogger, + parsedArgs.scalaVersion, + parsedArgs.compilerAndDependencies, + parsedArgs.libraryAndDependencies); incrementalCompiler.compile( parsedArgs.classpathElements, parsedArgs.sources, parsedArgs.classesDirectory, parsedArgs.scalacOptions, - parsedArgs.javacOptions); + parsedArgs.javacOptions, + parsedArgs.compileOrder, + parsedArgs.cacheFile); } } diff --git a/src/main/java/sbt_inc/InProcessSbtIncrementalCompiler.java b/src/main/java/sbt_inc/InProcessSbtIncrementalCompiler.java index c16c1f28..0a54ebdb 100644 --- a/src/main/java/sbt_inc/InProcessSbtIncrementalCompiler.java +++ b/src/main/java/sbt_inc/InProcessSbtIncrementalCompiler.java @@ -5,34 +5,56 @@ package sbt_inc; import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.*; import sbt.internal.inc.*; +import sbt.internal.inc.FileAnalysisStore; +import sbt.internal.inc.ScalaInstance; +import scala.Option; +import scala.jdk.FunctionWrappers; import xsbti.Logger; +import xsbti.PathBasedFile; +import xsbti.T2; import xsbti.VirtualFile; import xsbti.compile.*; public final class InProcessSbtIncrementalCompiler implements SbtIncrementalCompiler { private final Compilers compilers; - private final AnalysisStore analysisStore; - private final Setup setup; - private final IncrementalCompiler compiler; - private final CompileOrder compileOrder; private final Logger sbtLogger; public InProcessSbtIncrementalCompiler( - Compilers compilers, - AnalysisStore analysisStore, - Setup setup, - IncrementalCompiler compiler, - CompileOrder compileOrder, - Logger sbtLogger) { - this.compilers = compilers; - this.analysisStore = analysisStore; - this.setup = setup; - this.compiler = compiler; - this.compileOrder = compileOrder; + File javaHome, + File compilerBridgeJar, + Logger sbtLogger, + String scalaVersion, + Collection compilerAndDependencies, + Collection libraryAndDependencies) { + this.sbtLogger = sbtLogger; + + ScalaInstance scalaInstance = + ScalaInstances.makeScalaInstance( + scalaVersion, compilerAndDependencies, libraryAndDependencies); + + compilers = makeCompilers(scalaInstance, javaHome, compilerBridgeJar); + compiler = ZincUtil.defaultIncrementalCompiler(); + } + + private static Compilers makeCompilers( + ScalaInstance scalaInstance, File javaHome, File compilerBridgeJar) { + ScalaCompiler scalaCompiler = + new AnalyzingCompiler( + scalaInstance, // scalaInstance + ZincCompilerUtil.constantBridgeProvider(scalaInstance, compilerBridgeJar), // provider + ClasspathOptionsUtil.auto(), // classpathOptions + new FunctionWrappers.FromJavaConsumer<>(noop -> {}), // onArgsHandler + Option.apply(null) // classLoaderCache + ); + + return ZincUtil.compilers( + scalaInstance, ClasspathOptionsUtil.boot(), Option.apply(javaHome.toPath()), scalaCompiler); } @Override @@ -41,7 +63,9 @@ public void compile( Collection sources, File classesDirectory, Collection scalacOptions, - Collection javacOptions) { + Collection javacOptions, + CompileOrder compileOrder, + File cacheFile) { // incremental compiler needs to add the output dir in the classpath for Java + Scala Collection fullClasspathElements = new ArrayList<>(classpathElements); @@ -67,13 +91,65 @@ public void compile( Optional.empty() // _earlyOutput ); - Inputs inputs = Inputs.of(compilers, options, setup, previousResult()); + AnalysisStore analysisStore = AnalysisStore.getCachedStore(FileAnalysisStore.binary(cacheFile)); + Inputs inputs = + Inputs.of( + compilers, options, makeSetup(cacheFile, sbtLogger), previousResult(analysisStore)); CompileResult newResult = compiler.compile(inputs, sbtLogger); + analysisStore.set(AnalysisContents.create(newResult.analysis(), newResult.setup())); } - private PreviousResult previousResult() { + private static Setup makeSetup(File cacheFile, xsbti.Logger sbtLogger) { + PerClasspathEntryLookup lookup = + new PerClasspathEntryLookup() { + @Override + public Optional analysis(VirtualFile classpathEntry) { + Path path = ((PathBasedFile) classpathEntry).toPath(); + + String analysisStoreFileName = null; + if (Files.isDirectory(path)) { + if (path.getFileName().toString().equals("classes")) { + analysisStoreFileName = "compile"; + + } else if (path.getFileName().toString().equals("test-classes")) { + analysisStoreFileName = "test-compile"; + } + } + + if (analysisStoreFileName != null) { + File analysisStoreFile = + path.getParent().resolve("analysis").resolve(analysisStoreFileName).toFile(); + if (analysisStoreFile.exists()) { + return AnalysisStore.getCachedStore(FileAnalysisStore.binary(analysisStoreFile)) + .get() + .map(AnalysisContents::getAnalysis); + } + } + return Optional.empty(); + } + + @Override + public DefinesClass definesClass(VirtualFile classpathEntry) { + return classpathEntry.name().equals("rt.jar") + ? className -> false + : Locate.definesClass(classpathEntry); + } + }; + + return Setup.of( + lookup, // lookup + false, // skip + cacheFile, // cacheFile + CompilerCache.fresh(), // cache + IncOptions.of(), // incOptions + new LoggedReporter(100, sbtLogger, pos -> pos), // reporter + Optional.empty(), // optionProgress + new T2[] {}); + } + + private PreviousResult previousResult(AnalysisStore analysisStore) { Optional analysisContents = analysisStore.get(); if (analysisContents.isPresent()) { AnalysisContents analysisContents0 = analysisContents.get(); diff --git a/src/main/java/sbt_inc/SbtIncrementalCompiler.java b/src/main/java/sbt_inc/SbtIncrementalCompiler.java index 3b9adadd..9a71cc6e 100644 --- a/src/main/java/sbt_inc/SbtIncrementalCompiler.java +++ b/src/main/java/sbt_inc/SbtIncrementalCompiler.java @@ -6,6 +6,7 @@ import java.io.File; import java.util.Collection; +import xsbti.compile.CompileOrder; public interface SbtIncrementalCompiler { @@ -14,5 +15,7 @@ void compile( Collection sources, File classesDirectory, Collection scalacOptions, - Collection javacOptions); + Collection javacOptions, + CompileOrder compileOrder, + File cacheFile); } diff --git a/src/main/java/sbt_inc/SbtIncrementalCompilers.java b/src/main/java/sbt_inc/SbtIncrementalCompilers.java index 4e931838..459c4510 100644 --- a/src/main/java/sbt_inc/SbtIncrementalCompilers.java +++ b/src/main/java/sbt_inc/SbtIncrementalCompilers.java @@ -6,35 +6,29 @@ import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.*; import java.util.stream.Collectors; import org.apache.commons.exec.LogOutputStream; import org.apache.maven.plugin.logging.Log; import sbt.internal.inc.*; -import sbt.internal.inc.FileAnalysisStore; import sbt.internal.inc.ScalaInstance; import sbt.util.Logger; -import scala.Option; -import scala.jdk.FunctionWrappers; import scala_maven.MavenArtifactResolver; import scala_maven.VersionNumber; import scala_maven_executions.Fork; import scala_maven_executions.ForkLogger; -import xsbti.PathBasedFile; -import xsbti.T2; -import xsbti.VirtualFile; import xsbti.compile.*; public final class SbtIncrementalCompilers { + + private static File COMPILER_BRIDGE_JAR = null; + private static SbtIncrementalCompiler SBT_INCREMENTAL_COMPILER = null; + public static SbtIncrementalCompiler make( File javaHome, MavenArtifactResolver resolver, File secondaryCacheDir, Log mavenLogger, - File cacheFile, - CompileOrder compileOrder, VersionNumber scalaVersion, Collection compilerAndDependencies, Collection libraryAndDependencies, @@ -43,29 +37,37 @@ public static SbtIncrementalCompiler make( List forkBootClasspath) throws Exception { - ScalaInstance scalaInstance = - ScalaInstances.makeScalaInstance( - scalaVersion.toString(), compilerAndDependencies, libraryAndDependencies); + if (COMPILER_BRIDGE_JAR == null) { + if (mavenLogger.isInfoEnabled()) { + mavenLogger.info("Building compiler bridge JAR"); + } + + ScalaInstance scalaInstance = + ScalaInstances.makeScalaInstance( + scalaVersion.toString(), compilerAndDependencies, libraryAndDependencies); - File compilerBridgeJar = - CompilerBridgeFactory.getCompiledBridgeJar( - scalaVersion, scalaInstance, secondaryCacheDir, resolver, mavenLogger); + COMPILER_BRIDGE_JAR = + CompilerBridgeFactory.getCompiledBridgeJar( + scalaVersion, scalaInstance, secondaryCacheDir, resolver, mavenLogger); + } else { + if (mavenLogger.isInfoEnabled()) { + mavenLogger.info("Reusing compiler bridge JAR: " + COMPILER_BRIDGE_JAR.getAbsolutePath()); + } + } if (jvmArgs == null || jvmArgs.length == 0) { return makeInProcess( javaHome, - cacheFile, - compileOrder, - scalaInstance, - compilerBridgeJar, - new MavenLoggerSbtAdapter(mavenLogger)); + COMPILER_BRIDGE_JAR, + new MavenLoggerSbtAdapter(mavenLogger), + scalaVersion.toString(), + compilerAndDependencies, + libraryAndDependencies); } else { return makeForkedProcess( javaHome, - cacheFile, - compileOrder, - compilerBridgeJar, - scalaVersion, + COMPILER_BRIDGE_JAR, + scalaVersion.toString(), compilerAndDependencies, libraryAndDependencies, mavenLogger, @@ -77,27 +79,33 @@ public static SbtIncrementalCompiler make( static SbtIncrementalCompiler makeInProcess( File javaHome, - File cacheFile, - CompileOrder compileOrder, - ScalaInstance scalaInstance, File compilerBridgeJar, - Logger sbtLogger) { - - Compilers compilers = makeCompilers(scalaInstance, javaHome, compilerBridgeJar); - AnalysisStore analysisStore = AnalysisStore.getCachedStore(FileAnalysisStore.binary(cacheFile)); - Setup setup = makeSetup(cacheFile, sbtLogger); - IncrementalCompiler compiler = ZincUtil.defaultIncrementalCompiler(); + Logger sbtLogger, + String scalaVersion, + Collection compilerAndDependencies, + Collection libraryAndDependencies) { - return new InProcessSbtIncrementalCompiler( - compilers, analysisStore, setup, compiler, compileOrder, sbtLogger); + if (SBT_INCREMENTAL_COMPILER == null) { + try { + SBT_INCREMENTAL_COMPILER = + new InProcessSbtIncrementalCompiler( + javaHome, + compilerBridgeJar, + sbtLogger, + scalaVersion, + compilerAndDependencies, + libraryAndDependencies); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return SBT_INCREMENTAL_COMPILER; } private static SbtIncrementalCompiler makeForkedProcess( File javaHome, - File cacheFile, - CompileOrder compileOrder, File compilerBridgeJar, - VersionNumber scalaVersion, + String scalaVersion, Collection compilerAndDependencies, Collection libraryAndDependencies, Log mavenLogger, @@ -108,7 +116,13 @@ private static SbtIncrementalCompiler makeForkedProcess( List forkClasspath = pluginArtifacts.stream().map(File::getPath).collect(Collectors.toList()); - return (classpathElements, sources, classesDirectory, scalacOptions, javacOptions) -> { + return (classpathElements, + sources, + classesDirectory, + scalacOptions, + javacOptions, + compileOrder, + cacheFile) -> { try { String[] args = new ForkedSbtIncrementalCompilerMain.Args( @@ -116,7 +130,7 @@ private static SbtIncrementalCompiler makeForkedProcess( cacheFile, compileOrder, compilerBridgeJar, - scalaVersion.toString(), + scalaVersion, compilerAndDependencies, libraryAndDependencies, classpathElements, @@ -180,67 +194,4 @@ public void close() throws IOException { } }; } - - private static Compilers makeCompilers( - ScalaInstance scalaInstance, File javaHome, File compilerBridgeJar) { - ScalaCompiler scalaCompiler = - new AnalyzingCompiler( - scalaInstance, // scalaInstance - ZincCompilerUtil.constantBridgeProvider(scalaInstance, compilerBridgeJar), // provider - ClasspathOptionsUtil.auto(), // classpathOptions - new FunctionWrappers.FromJavaConsumer<>(noop -> {}), // onArgsHandler - Option.apply(null) // classLoaderCache - ); - - return ZincUtil.compilers( - scalaInstance, ClasspathOptionsUtil.boot(), Option.apply(javaHome.toPath()), scalaCompiler); - } - - private static Setup makeSetup(File cacheFile, xsbti.Logger sbtLogger) { - PerClasspathEntryLookup lookup = - new PerClasspathEntryLookup() { - @Override - public Optional analysis(VirtualFile classpathEntry) { - Path path = ((PathBasedFile) classpathEntry).toPath(); - - String analysisStoreFileName = null; - if (Files.isDirectory(path)) { - if (path.getFileName().toString().equals("classes")) { - analysisStoreFileName = "compile"; - - } else if (path.getFileName().toString().equals("test-classes")) { - analysisStoreFileName = "test-compile"; - } - } - - if (analysisStoreFileName != null) { - File analysisStoreFile = - path.getParent().resolve("analysis").resolve(analysisStoreFileName).toFile(); - if (analysisStoreFile.exists()) { - return AnalysisStore.getCachedStore(FileAnalysisStore.binary(analysisStoreFile)) - .get() - .map(AnalysisContents::getAnalysis); - } - } - return Optional.empty(); - } - - @Override - public DefinesClass definesClass(VirtualFile classpathEntry) { - return classpathEntry.name().equals("rt.jar") - ? className -> false - : Locate.definesClass(classpathEntry); - } - }; - - return Setup.of( - lookup, // lookup - false, // skip - cacheFile, // cacheFile - CompilerCache.fresh(), // cache - IncOptions.of(), // incOptions - new LoggedReporter(100, sbtLogger, pos -> pos), // reporter - Optional.empty(), // optionProgress - new T2[] {}); - } } diff --git a/src/main/java/scala_maven/ScalaCompilerSupport.java b/src/main/java/scala_maven/ScalaCompilerSupport.java index 4907ba49..07dcd887 100644 --- a/src/main/java/scala_maven/ScalaCompilerSupport.java +++ b/src/main/java/scala_maven/ScalaCompilerSupport.java @@ -307,8 +307,6 @@ private int incrementalCompile( new MavenArtifactResolver(factory, session), secondaryCacheDir, getLog(), - cacheFile, - compileOrder, sc.version(), sc.findCompilerAndDependencies().stream() .map(Artifact::getFile) @@ -323,7 +321,13 @@ private int incrementalCompile( try { incremental.compile( - classpathElements, sources, outputDir, getScalacOptions(), getJavacOptions()); + classpathElements, + sources, + outputDir, + getScalacOptions(), + getJavacOptions(), + compileOrder, + cacheFile); } catch (xsbti.CompileFailed e) { if (compileInLoop) { compileErrors = true;