Shared Caching for NeoForm Task Outputs

README.md

@@ -8,6 +8,19 @@ our official [Documentation](https://docs.neoforged.net/neogradle/docs/).
 
 To see the latest available version of NeoGradle, visit the [NeoForged project page](https://projects.neoforged.net/neoforged/neogradle).
 
+## Configuring Shared NeoForm Cache
+
+NeoForm is the toolkit used to provide a Minecraft JAR-File suitable for compiling your mod against.
+Since this is a rather resource-intensive task, the intermediary steps and final result of that
+process can be cached outside the project folder.
+
+The settings of this caching subsystem can be changed using [Gradle properties](https://docs.gradle.org/current/userguide/project_properties.html).
+
+| Property                                           | Description                                                                                                                                                                                     |
+|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `neogradle.subsystems.neoFormCache.enabled`        | Can be used to fully disable the caching by setting this to `false`. The default is `true`.                                                                                                     |
+| `neogradle.subsystems.neoFormCache.CacheDirectory` | The path to a directory where the cache is stored. Defaults to `${GRADLE_USER_HOME}/caches/neoForm` (see [Gradle Directories](https://docs.gradle.org/current/userguide/directory_layout.html)) |
+
 ## Override Decompiler Settings
 
 The settings used by the decompiler when preparing Minecraft dependencies can be overridden

build.gradle

@@ -218,5 +218,14 @@ subprojects.forEach { subProject ->
                 project.changelog.publish publication
             }
         }
+
+        evalSubProject.tasks.withType(org.gradle.api.publish.maven.tasks.AbstractPublishToMaven).configureEach { task ->
+            doLast {
+                if (task.state.didWork) {
+                    MavenPublication publication = task.publication
+                    println("Published ${publication.groupId}:${publication.artifactId}:${publication.version}")
+                }
+            }
+        }
     }
 }

common/src/main/java/net/neoforged/gradle/common/CommonProjectPlugin.java

@@ -1,6 +1,7 @@
 package net.neoforged.gradle.common;
 
 import net.neoforged.gradle.common.caching.CentralCacheService;
+import net.neoforged.gradle.common.caching.SharedCacheService;
 import net.neoforged.gradle.common.dependency.ExtraJarDependencyManager;
 import net.neoforged.gradle.common.extensions.*;
 import net.neoforged.gradle.common.extensions.dependency.creation.ProjectBasedDependencyCreator;
@@ -21,6 +22,7 @@
 import net.neoforged.gradle.dsl.common.extensions.*;
 import net.neoforged.gradle.dsl.common.extensions.dependency.replacement.DependencyReplacement;
 import net.neoforged.gradle.dsl.common.extensions.repository.Repository;
+import net.neoforged.gradle.dsl.common.extensions.subsystems.NeoFormCache;
 import net.neoforged.gradle.dsl.common.extensions.subsystems.Subsystems;
 import net.neoforged.gradle.dsl.common.runs.run.Run;
 import net.neoforged.gradle.dsl.common.runs.type.RunType;
@@ -42,10 +44,11 @@
 import java.util.Set;
 
 public class CommonProjectPlugin implements Plugin<Project> {
-    
+
     public static final String ASSETS_SERVICE = "ng_assets";
     public static final String LIBRARIES_SERVICE = "ng_libraries";
-
+    public static final String NEOFORM_CACHE_SERVICE = "ng_neoform_cache";
+
     @Override
     public void apply(Project project) {
         //Apply the evaluation extension to monitor immediate execution of indirect tasks when evaluation already happened.
@@ -57,11 +60,11 @@ public void apply(Project project) {
         project.getPluginManager().apply(IdeaPlugin.class);
         project.getPluginManager().apply(IdeaExtPlugin.class);
         project.getPluginManager().apply(EclipsePlugin.class);
-        
+
         //Register the assets service
         CentralCacheService.register(project, ASSETS_SERVICE, FileCacheUtils.getAssetsCacheDirectory(project));
         CentralCacheService.register(project, LIBRARIES_SERVICE, FileCacheUtils.getLibrariesCacheDirectory(project));
-        
+
         project.getExtensions().create("allRuntimes", RuntimesExtension.class);
         project.getExtensions().create(IdeManagementExtension.class, "ideManager", IdeManagementExtension.class, project);
         project.getExtensions().create(ArtifactDownloader.class, "artifactDownloader", ArtifactDownloaderExtension.class, project);
@@ -78,14 +81,21 @@ public void apply(Project project) {
         extensionManager.registerExtension("mappings", Mappings.class, (p) -> p.getObjects().newInstance(MappingsExtension.class, p));
         extensionManager.registerExtension("subsystems", Subsystems.class, (p) -> p.getObjects().newInstance(SubsystemsExtension.class, p));
 
+        // The shared cache can only be registered after the subsystems extension
+        SharedCacheService.register(project, NEOFORM_CACHE_SERVICE, params -> {
+            NeoFormCache cacheSettings = project.getExtensions().getByType(Subsystems.class).getNeoFormCache();
+            params.getEnabled().set(cacheSettings.getEnabled().orElse(params.getEnabled().get()));
+            params.getCacheDirectory().set(cacheSettings.getCacheDirectory().orElse(params.getCacheDirectory().get()));
+        });
+
         OfficialNamingChannelConfigurator.getInstance().configure(project);
 
         project.getTasks().register("handleNamingLicense", DisplayMappingsLicenseTask.class, task -> {
             task.getLicense().set(project.provider(() -> {
                 final Mappings mappings = project.getExtensions().getByType(Mappings.class);
                 if (mappings.getChannel().get().getHasAcceptedLicense().get())
                     return null;
-                
+
                 return mappings.getChannel().get().getLicenseText().get();
             }));
         });
@@ -107,15 +117,15 @@ public void apply(Project project) {
                 RunsConstants.Extensions.RUN_TYPES,
                 project.getObjects().domainObjectContainer(RunType.class, name -> project.getObjects().newInstance(RunType.class, name))
         );
-        
+
         project.getExtensions().add(
                 RunsConstants.Extensions.RUNS,
                 project.getObjects().domainObjectContainer(Run.class, name -> RunsUtil.create(project, name))
         );
-        
+
         IdeRunIntegrationManager.getInstance().setup(project);
     }
-    
+
     private void applyAfterEvaluate(final Project project) {
         RuntimesExtension runtimesExtension = project.getExtensions().getByType(RuntimesExtension.class);
         runtimesExtension.bakeDefinitions();
@@ -139,9 +149,9 @@ private void applyAfterEvaluate(final Project project) {
 
                 if (run.getConfigureFromDependencies().get()) {
                     final RunImpl runImpl = (RunImpl) run;
-                    
+
                     final Set<CommonRuntimeDefinition<?>> definitionSet = new HashSet<>();
-                    
+
                     runImpl.getModSources().get().forEach(sourceSet -> {
                         try {
                             final Optional<CommonRuntimeDefinition<?>> definition = TaskDependencyUtils.findRuntimeDefinition(project, sourceSet);
@@ -150,14 +160,14 @@ private void applyAfterEvaluate(final Project project) {
                             throw new RuntimeException("Failed to configure run: " + run.getName() + " there are multiple runtime definitions found for the source set: " + sourceSet.getName(), e);
                         }
                     });
-                    
+
                     definitionSet.forEach(definition -> {
-                       definition.configureRun(runImpl);
+                        definition.configureRun(runImpl);
                     });
                 }
             }
         }));
-        
+
         IdeRunIntegrationManager.getInstance().apply(project);
     }
 }

common/src/main/java/net/neoforged/gradle/common/caching/CacheKey.java

@@ -0,0 +1,42 @@
+package net.neoforged.gradle.common.caching;
+
+import org.jetbrains.annotations.Nullable;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public final class CacheKey {
+    private static final String CACHE_DOMAIN_ALL = "all";
+    @Nullable
+    private final String cacheDomain;
+    private final String hashCode;
+    private final String sourceMaterial;
+
+    CacheKey(@Nullable String cacheDomain, String hashCode, String sourceMaterial) {
+        this.cacheDomain = cacheDomain;
+        this.hashCode = hashCode;
+        this.sourceMaterial = sourceMaterial;
+    }
+
+    @Nullable
+    String getCacheDomain() {
+        return cacheDomain;
+    }
+
+    String getHashCode() {
+        return hashCode;
+    }
+
+    String getSourceMaterial() {
+        return sourceMaterial;
+    }
+
+    Path asPath(@Nullable String extension) {
+        String filename = hashCode;
+        if (extension != null) {
+            filename += "." + extension;
+        }
+
+        return Paths.get(cacheDomain != null ? cacheDomain : CACHE_DOMAIN_ALL, filename);
+    }
+}

common/src/main/java/net/neoforged/gradle/common/caching/FileHashing.java

@@ -0,0 +1,65 @@
+package net.neoforged.gradle.common.caching;
+
+import net.neoforged.gradle.util.HashFunction;
+import org.apache.commons.io.output.NullOutputStream;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributeView;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Provides in-memory caching of file-hashes based on last-modification time and size.
+ */
+class FileHashing {
+    private final Map<Path, CachedHash> cachedHashes = new ConcurrentHashMap<>();
+
+    public byte[] getMd5Hash(Path path) {
+        return cachedHashes.compute(path, CachedHash::compute).hashValue;
+    }
+
+    private static final class CachedHash {
+        private final long lastModified;
+        private final long fileSize;
+        private final byte[] hashValue;
+
+        public static CachedHash compute(Path path, @Nullable CachedHash cachedHash) {
+            try {
+                // Instead of reading size + last modified separately, we use this function to make race conditions
+                // less likely. We still don't know if the underlying OS APIs return this information atomically,
+                // but if they do, we at least make use of that fact.
+                BasicFileAttributes attributes = Files.getFileAttributeView(path, BasicFileAttributeView.class)
+                        .readAttributes();
+                long lastModified = attributes.lastModifiedTime().toMillis();
+                long fileSize = attributes.size();
+
+                if (cachedHash != null && cachedHash.lastModified == lastModified && cachedHash.fileSize == fileSize) {
+                    return cachedHash;
+                }
+
+                // Compute the digest in a streaming fashion without reading the full file into memory
+                MessageDigest digest = HashFunction.MD5.get();
+                try (DigestOutputStream out = new DigestOutputStream(NullOutputStream.NULL_OUTPUT_STREAM, digest)) {
+                    Files.copy(path, out);
+                }
+
+                return new CachedHash(digest.digest(), lastModified, fileSize);
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }
+
+        public CachedHash(byte[] hashValue, long lastModified, long fileSize) {
+            this.hashValue = hashValue;
+            this.lastModified = lastModified;
+            this.fileSize = fileSize;
+        }
+    }
+}

common/src/main/java/net/neoforged/gradle/common/caching/HashCodeBuilder.java

@@ -0,0 +1,48 @@
+package net.neoforged.gradle.common.caching;
+
+import org.apache.commons.codec.binary.Hex;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+final class HashCodeBuilder {
+    private final FileHashing fileHashing;
+    private final StringBuilder sourceMaterial = new StringBuilder();
+    private final MessageDigest digest;
+
+    public HashCodeBuilder(FileHashing fileHashing) {
+        this.fileHashing = fileHashing;
+        // Relativize any path to gradle home or project root,
+        // which will work for anything but maven local dependencies
+        try {
+            digest = MessageDigest.getInstance("MD5");
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException("Standard algorithm MD5 is missing.", e);
+        }
+    }
+
+    public void add(Path path) {
+        byte[] fileHash = fileHashing.getMd5Hash(path);
+        String fileHashHex = Hex.encodeHexString(fileHash);
+        add(fileHash, "HASHED-CONTENT(" + path + ") = " + fileHashHex);
+    }
+
+    public void add(String data) {
+        add(data.getBytes(StandardCharsets.UTF_8), "STRING(" + data + ")");
+    }
+
+    public void add(byte[] data, String sourceMaterial) {
+        digest.update(data);
+        this.sourceMaterial.append(sourceMaterial).append('\n');
+    }
+
+    public String buildHashCode() {
+        return Hex.encodeHexString(digest.digest());
+    }
+
+    public String buildSourceMaterial() {
+        return sourceMaterial.toString();
+    }
+}