Skip to content

[POC] Use Apache Commons Compress for faster zip transfers #81

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Technici4n wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

[POC] Use Apache Commons Compress for faster zip transfers #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
706b71a
Test case
Technici4n Jul 19, 2025
a56bf36
Rerun split every time
Technici4n Jul 19, 2025
f77fc67
Proof of concept with Apache Commons Compress
Technici4n Jul 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ dependencies {
implementation 'net.fabricmc:fabric-loom-native:0.2.1'
implementation 'net.neoforged:srgutils:1.0.10'
implementation 'net.neoforged.installertools:problems-api:3.0.3'
implementation 'org.apache.commons:commons-compress:1.27.1'
annotationProcessor 'info.picocli:picocli-codegen:4.7.6'

testImplementation platform('org.junit:junit-bom:5.13.2')
Expand Down Expand Up @@ -234,6 +235,11 @@ idea {
programParameters = "download-assets --neoforge net.neoforged:neoforge:20.6.72-beta:userdev --write-properties build/assets.properties --write-json build/assets.json"
moduleRef(project, sourceSets.main)
}
"NeoForm 1.21 (joined) strip only"(Application) {
mainClass = mainClassName
programParameters = "run --dist joined --neoform net.neoforged:neoform:1.21-20240613.152323@zip --write-result=node.stripClient.output.output:build/stripped-client.jar --write-result=node.stripServer.output.output:build/stripped-server.jar"
moduleRef(project, sourceSets.main)
}
}
}
}
Expand Down
35 changes: 17 additions & 18 deletions src/main/java/net/neoforged/neoform/runtime/actions/SplitResourcesFromClassesAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
import net.neoforged.neoform.runtime.cache.CacheKeyBuilder;
import net.neoforged.neoform.runtime.engine.ProcessingEnvironment;
import net.neoforged.srgutils.IMappingFile;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
import org.apache.commons.compress.archivers.zip.ZipFile;
import org.jetbrains.annotations.Nullable;

import java.io.BufferedOutputStream;
Expand All @@ -18,12 +21,9 @@
import java.util.function.Predicate;
import java.util.jar.Attributes;
import java.util.jar.JarFile;
import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

/**
* Copies a Jar file while applying a filename filter.
Expand Down Expand Up @@ -81,11 +81,12 @@ public void run(ProcessingEnvironment environment) throws IOException, Interrupt
.asMatchPredicate();
}

// TODO: new ZipFile is deprecated
try (var jar = new ZipFile(inputJar.toFile());
var classesFileOut = new BufferedOutputStream(Files.newOutputStream(classesJar));
var resourcesFileOut = new BufferedOutputStream(Files.newOutputStream(resourcesJar));
var classesJarOut = new JarOutputStream(classesFileOut);
var resourcesJarOut = new JarOutputStream(resourcesFileOut);
var classesJarOut = new ZipArchiveOutputStream(classesFileOut);
var resourcesJarOut = new ZipArchiveOutputStream(resourcesFileOut);
) {
if (generateDistManifestSettings != null) {
generateDistSourceManifest(
Expand All @@ -98,7 +99,7 @@ public void run(ProcessingEnvironment environment) throws IOException, Interrupt
);
}

var entries = jar.entries();
var entries = jar.getEntries();
while (entries.hasMoreElements()) {
var entry = entries.nextElement();
if (entry.isDirectory()) {
Expand All @@ -119,11 +120,7 @@ public void run(ProcessingEnvironment environment) throws IOException, Interrupt

var destinationStream = filename.endsWith(".class") ? classesJarOut : resourcesJarOut;

destinationStream.putNextEntry(entry);
try (var is = jar.getInputStream(entry)) {
is.transferTo(destinationStream);
}
destinationStream.closeEntry();
destinationStream.addRawArchiveEntry(entry, jar.getRawInputStream(entry));
}
}
}
Expand All @@ -133,7 +130,7 @@ private static void generateDistSourceManifest(String distId,
String otherDistId,
Path otherDistJarPath,
Path mappingsPath,
JarOutputStream resourcesJarOut) throws IOException {
ZipArchiveOutputStream resourcesJarOut) throws IOException {
var mappings = mappingsPath != null ? IMappingFile.load(mappingsPath.toFile()) : null;

// Use the time-stamp of either of the two input files (whichever is newer)
Expand All @@ -152,11 +149,11 @@ private static void generateDistSourceManifest(String distId,
addSourceDistEntries(ourFiles, theirFiles, distId, mappings, manifest);
addSourceDistEntries(theirFiles, ourFiles, otherDistId, mappings, manifest);

var manifestEntry = new ZipEntry(JarFile.MANIFEST_NAME);
var manifestEntry = new ZipArchiveEntry(JarFile.MANIFEST_NAME);
manifestEntry.setTimeLocal(MANIFEST_TIME);
resourcesJarOut.putNextEntry(manifestEntry);
resourcesJarOut.putArchiveEntry(manifestEntry);
manifest.write(resourcesJarOut);
resourcesJarOut.closeEntry();
resourcesJarOut.closeArchiveEntry();
}

private static void addSourceDistEntries(Set<String> distFiles,
Expand All @@ -178,11 +175,11 @@ private static void addSourceDistEntries(Set<String> distFiles,
}

private static Set<String> getFileIndex(ZipFile zipFile) {
var result = new HashSet<String>(zipFile.size());
var result = new HashSet<String>();

var entries = zipFile.entries();
var entries = zipFile.getEntries();
while (entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
ZipArchiveEntry entry = entries.nextElement();
if (!entry.isDirectory()) {
result.add(entry.getName());
}
Expand Down Expand Up @@ -222,6 +219,8 @@ public void generateSplitManifest(String distId, String otherDistId) {
@Override
public void computeCacheKey(CacheKeyBuilder ck) {
super.computeCacheKey(ck);
// TODO: remove :P
ck.add("force rerun", "" + Math.random());
ck.addStrings("deny patterns", denyListPatterns.stream().map(Pattern::pattern).toList());
if (generateDistManifestSettings != null) {
ck.add("generate dist manifest - our dist", generateDistManifestSettings.distId);
Expand Down