From 50a2fc456d3993b5d3598a7914b4f204b4864cfc Mon Sep 17 00:00:00 2001 From: Sebastian Hartte Date: Sat, 14 Dec 2024 18:04:56 +0100 Subject: [PATCH 1/4] Implement generation of split-source Manifest entries. --- .../dependency/ExtraJarDependencyManager.java | 6 +- .../runtime/tasks/GenerateExtraJar.java | 113 ++++++++++++++++-- 2 files changed, 105 insertions(+), 14 deletions(-) diff --git a/common/src/main/java/net/neoforged/gradle/common/dependency/ExtraJarDependencyManager.java b/common/src/main/java/net/neoforged/gradle/common/dependency/ExtraJarDependencyManager.java index 463bd8579..36f02d4bb 100644 --- a/common/src/main/java/net/neoforged/gradle/common/dependency/ExtraJarDependencyManager.java +++ b/common/src/main/java/net/neoforged/gradle/common/dependency/ExtraJarDependencyManager.java @@ -90,13 +90,17 @@ private ReplacementResult generateReplacement(final Project project, final Depen return replacements.computeIfAbsent(minecraftVersion, (v) -> { final MinecraftArtifactCache minecraftArtifactCacheExtension = project.getExtensions().getByType(MinecraftArtifactCache.class); - Map> tasks = minecraftArtifactCacheExtension.cacheGameVersionTasks(project, minecraftVersion, DistributionType.CLIENT); + Map> tasks = minecraftArtifactCacheExtension.cacheGameVersionTasks(project, minecraftVersion, DistributionType.JOINED); final TaskProvider extraJarTaskProvider = project.getTasks().register("create" + minecraftVersion + StringUtils.capitalize(dependency.getName()) + "ExtraJar", GenerateExtraJar.class, task -> { task.getOriginalJar().set(tasks.get(GameArtifact.CLIENT_JAR).flatMap(WithOutput::getOutput)); + task.getServerJar().set(tasks.get(GameArtifact.SERVER_JAR).flatMap(WithOutput::getOutput)); + task.getMappings().set(tasks.get(GameArtifact.CLIENT_MAPPINGS).flatMap(WithOutput::getOutput)); task.getOutput().set(project.getLayout().getBuildDirectory().dir("jars/extra/" + dependency.getName()).map(cacheDir -> cacheDir.dir(Objects.requireNonNull(minecraftVersion)).file( dependency.getName() + "-extra.jar"))); task.dependsOn(tasks.get(GameArtifact.CLIENT_JAR)); + task.dependsOn(tasks.get(GameArtifact.SERVER_JAR)); + task.dependsOn(tasks.get(GameArtifact.CLIENT_MAPPINGS)); }); return new ReplacementResult( diff --git a/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java b/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java index d64b73620..f748e1ade 100644 --- a/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java +++ b/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java @@ -1,12 +1,11 @@ package net.neoforged.gradle.common.runtime.tasks; +import net.minecraftforge.srgutils.IMappingFile; import net.neoforged.gradle.common.services.caching.CachedExecutionService; import net.neoforged.gradle.common.services.caching.jobs.ICacheableJob; import net.neoforged.gradle.dsl.common.tasks.NeoGradleBase; import net.neoforged.gradle.dsl.common.tasks.WithOutput; import net.neoforged.gradle.dsl.common.tasks.WithWorkspace; -import net.neoforged.gradle.util.ZipBuildingFileTreeVisitor; -import org.gradle.api.file.FileTree; import org.gradle.api.file.RegularFileProperty; import org.gradle.api.provider.Property; import org.gradle.api.services.ServiceReference; @@ -16,9 +15,18 @@ import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; +import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; -import java.util.zip.ZipOutputStream; +import java.util.HashSet; +import java.util.Set; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; @CacheableTask public abstract class GenerateExtraJar extends NeoGradleBase implements WithOutput, WithWorkspace { @@ -34,28 +42,107 @@ public GenerateExtraJar() { @TaskAction public void run() throws Throwable { getCacheService().get() - .cached( - this, - ICacheableJob.Default.file(getOutput(), this::doRun) - ).execute(); + .cached( + this, + ICacheableJob.Default.file(getOutput(), this::doRun) + ).execute(); } private void doRun() throws Exception { final File originalJar = getOriginalJar().get().getAsFile(); final File outputJar = ensureFileWorkspaceReady(getOutput()); + // Official mappings are Named -> Obf and need to be reversed + var mappings = IMappingFile.load(getMappings().getAsFile().get()).reverse(); + try (var clientZip = new JarFile(getOriginalJar().getAsFile().get()); + var serverZip = new JarFile(getServerJar().getAsFile().get())) { + var clientFiles = getFileIndex(clientZip); + clientFiles.remove(JarFile.MANIFEST_NAME); + var serverFiles = getFileIndex(serverZip); + serverFiles.remove(JarFile.MANIFEST_NAME); - final FileTree inputTree = getArchiveOperations().zipTree(originalJar); - final FileTree filteredInput = inputTree.matching(filter -> { - filter.exclude("**/*.class"); - }); + var manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + manifest.getMainAttributes().putValue("NeoForm-Minecraft-Dists", "server client"); - try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputJar))) { - filteredInput.visit(new ZipBuildingFileTreeVisitor(zos)); + addSourceDistEntries(clientFiles, serverFiles, "client", mappings, manifest); + addSourceDistEntries(serverFiles, clientFiles, "server", mappings, manifest); + + try (var jos = new JarOutputStream(new BufferedOutputStream(new FileOutputStream(outputJar)), manifest)) { + // Generally ignore directories, manifests and class files + var clientEntries = clientZip.entries(); + while (clientEntries.hasMoreElements()) { + var clientEntry = clientEntries.nextElement(); + if (isResourceEntry(clientEntry)) { + jos.putNextEntry(clientEntry); + try (var clientIn = clientZip.getInputStream(clientEntry)) { + clientIn.transferTo(jos); + } + jos.closeEntry(); + } + } + } + } + } + + private static void addSourceDistEntries(Set distFiles, + Set otherDistFiles, + String dist, + IMappingFile mappings, + Manifest manifest) { + for (var file : distFiles) { + if (!otherDistFiles.contains(file)) { + var fileAttr = new Attributes(1); + fileAttr.putValue("NeoForm-Minecraft-Dist", dist); + + if (mappings != null && file.endsWith(".class")) { + file = mappings.remapClass(file.substring(0, file.length() - ".class".length())) + ".class"; + } + manifest.getEntries().put(file, fileAttr); + } } } + private static Set getFileIndex(ZipFile zipFile) { + var result = new HashSet(zipFile.size()); + + var entries = zipFile.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + if (!entry.isDirectory()) { + result.add(entry.getName()); + } + } + + return result; + } + + private static boolean isResourceEntry(JarEntry entry) { + return !entry.getName().endsWith(".class") + && !entry.isDirectory() + && !entry.getName().equals(JarFile.MANIFEST_NAME) + && !isSignatureFile(entry.getName()); + } + + private static boolean isSignatureFile(String name) { + return name.startsWith("META-INF/") + && ( + name.endsWith(".SF") + || name.endsWith(".RSA") + || name.endsWith(".EC") + || name.endsWith(".DSA") + ); + } + @InputFile @PathSensitive(PathSensitivity.NONE) public abstract RegularFileProperty getOriginalJar(); + + @InputFile + @PathSensitive(PathSensitivity.NONE) + public abstract RegularFileProperty getServerJar(); + + @InputFile + @PathSensitive(PathSensitivity.NONE) + public abstract RegularFileProperty getMappings(); } From cc2a49c7d8328711b89bcf792a808fe38f280c62 Mon Sep 17 00:00:00 2001 From: Sebastian Hartte Date: Sat, 14 Dec 2024 18:06:37 +0100 Subject: [PATCH 2/4] Implement generation of split-source Manifest entries. --- .../gradle/common/runtime/tasks/GenerateExtraJar.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java b/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java index f748e1ade..1bc1522c1 100644 --- a/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java +++ b/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java @@ -63,7 +63,7 @@ private void doRun() throws Exception { var manifest = new Manifest(); manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); - manifest.getMainAttributes().putValue("NeoForm-Minecraft-Dists", "server client"); + manifest.getMainAttributes().putValue("Minecraft-Dists", "server client"); addSourceDistEntries(clientFiles, serverFiles, "client", mappings, manifest); addSourceDistEntries(serverFiles, clientFiles, "server", mappings, manifest); @@ -93,7 +93,7 @@ private static void addSourceDistEntries(Set distFiles, for (var file : distFiles) { if (!otherDistFiles.contains(file)) { var fileAttr = new Attributes(1); - fileAttr.putValue("NeoForm-Minecraft-Dist", dist); + fileAttr.putValue("Minecraft-Dist", dist); if (mappings != null && file.endsWith(".class")) { file = mappings.remapClass(file.substring(0, file.length() - ".class".length())) + ".class"; From 3394f98316d098ecc837355ce9d22588f782f673 Mon Sep 17 00:00:00 2001 From: Sebastian Hartte Date: Sat, 14 Dec 2024 18:52:40 +0100 Subject: [PATCH 3/4] Properly handle bundled server.jar --- .../runtime/tasks/GenerateExtraJar.java | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java b/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java index 1bc1522c1..1a6f1399d 100644 --- a/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java +++ b/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java @@ -14,10 +14,14 @@ import org.gradle.api.tasks.PathSensitive; import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; +import org.jetbrains.annotations.Nullable; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; import java.util.HashSet; import java.util.Set; import java.util.jar.Attributes; @@ -103,7 +107,24 @@ private static void addSourceDistEntries(Set distFiles, } } - private static Set getFileIndex(ZipFile zipFile) { + private Set getFileIndex(ZipFile zipFile) throws IOException { + // Support "nested" ZIP-Files as in recent server jars + var embeddedVersionPath = readEmbeddedVersionPath(zipFile); + if (embeddedVersionPath != null) { + // Extract the embedded zip and instead index that + var versionJarEntry = zipFile.getEntry(embeddedVersionPath); + if (versionJarEntry == null) { + throw new IOException("version list in jar file " + zipFile + " refers to missing entry " + embeddedVersionPath); + } + var unbundledFile = new File(getTemporaryDir(), "unpacked.jar"); + try (var in = zipFile.getInputStream(versionJarEntry)) { + Files.copy(in, unbundledFile.toPath()); + } + try (ZipFile zf = new ZipFile(unbundledFile)) { + return getFileIndex(zf); + } + } + var result = new HashSet(zipFile.size()); var entries = zipFile.entries(); @@ -134,6 +155,40 @@ private static boolean isSignatureFile(String name) { ); } + /** + * Server jars support embedding the actual jar file using an indirection via a version listing at + * META-INF/versions.list + * This method will try to read that list and return the path to the actual jar embedded in the bundle jar. + */ + @Nullable + private static String readEmbeddedVersionPath(ZipFile zipFile) throws IOException { + var entry = zipFile.getEntry("META-INF/versions.list"); + if (entry == null) { + return null; + } + + var entries = new ArrayList(); + try (var in = zipFile.getInputStream(entry)) { + for (var line : new String(in.readAllBytes()).split("\n")) { + if (line.isBlank()) { + continue; + } + String[] pts = line.split("\t"); + if (pts.length != 3) + throw new IOException("Invalid file list line: " + line + " in " + zipFile); + entries.add(pts[2]); + } + } + + if (entries.isEmpty()) { + return null; + } else if (entries.size() == 1) { + return "META-INF/versions/" + entries.get(0); + } else { + throw new IOException("Version file list contains more than one entry in " + zipFile); + } + } + @InputFile @PathSensitive(PathSensitivity.NONE) public abstract RegularFileProperty getOriginalJar(); From 4c3509c8405dc729e72f809761c2e1cb90aca7f9 Mon Sep 17 00:00:00 2001 From: Sebastian Hartte Date: Sat, 14 Dec 2024 21:10:09 +0100 Subject: [PATCH 4/4] Optimize --- .../common/runtime/tasks/GenerateExtraJar.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java b/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java index 1a6f1399d..f35f9a250 100644 --- a/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java +++ b/common/src/main/java/net/neoforged/gradle/common/runtime/tasks/GenerateExtraJar.java @@ -6,6 +6,7 @@ import net.neoforged.gradle.dsl.common.tasks.NeoGradleBase; import net.neoforged.gradle.dsl.common.tasks.WithOutput; import net.neoforged.gradle.dsl.common.tasks.WithWorkspace; +import net.neoforged.gradle.util.FileUtils; import org.gradle.api.file.RegularFileProperty; import org.gradle.api.provider.Property; import org.gradle.api.services.ServiceReference; @@ -27,10 +28,10 @@ import java.util.jar.Attributes; import java.util.jar.JarEntry; import java.util.jar.JarFile; -import java.util.jar.JarOutputStream; import java.util.jar.Manifest; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; @CacheableTask public abstract class GenerateExtraJar extends NeoGradleBase implements WithOutput, WithWorkspace { @@ -53,7 +54,6 @@ public void run() throws Throwable { } private void doRun() throws Exception { - final File originalJar = getOriginalJar().get().getAsFile(); final File outputJar = ensureFileWorkspaceReady(getOutput()); // Official mappings are Named -> Obf and need to be reversed @@ -72,17 +72,21 @@ private void doRun() throws Exception { addSourceDistEntries(clientFiles, serverFiles, "client", mappings, manifest); addSourceDistEntries(serverFiles, clientFiles, "server", mappings, manifest); - try (var jos = new JarOutputStream(new BufferedOutputStream(new FileOutputStream(outputJar)), manifest)) { + try (var zos = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(outputJar)))) { + zos.putNextEntry(FileUtils.getStableEntry(JarFile.MANIFEST_NAME)); + manifest.write(zos); + zos.closeEntry(); + // Generally ignore directories, manifests and class files var clientEntries = clientZip.entries(); while (clientEntries.hasMoreElements()) { var clientEntry = clientEntries.nextElement(); if (isResourceEntry(clientEntry)) { - jos.putNextEntry(clientEntry); + zos.putNextEntry(clientEntry); try (var clientIn = clientZip.getInputStream(clientEntry)) { - clientIn.transferTo(jos); + clientIn.transferTo(zos); } - jos.closeEntry(); + zos.closeEntry(); } } }