From f909e26fe414f7e029b93ea77c19b64289d73e08 Mon Sep 17 00:00:00 2001 From: shartte Date: Sat, 15 Jun 2024 09:49:03 +0200 Subject: [PATCH] Rework how download locations for libraries are resolved (#206) --- .../platform/model/InstallerProfile.groovy | 129 ++++++++----- .../dsl/platform/util/LibraryCollector.groovy | 171 ++++++++++++++---- .../extensions/DynamicProjectExtension.java | 30 +-- .../platform/tasks/CreateLauncherJson.java | 3 +- .../tasks/CreateLegacyInstallerJson.java | 14 +- 5 files changed, 239 insertions(+), 108 deletions(-) diff --git a/dsl/platform/src/main/groovy/net/neoforged/gradle/dsl/platform/model/InstallerProfile.groovy b/dsl/platform/src/main/groovy/net/neoforged/gradle/dsl/platform/model/InstallerProfile.groovy index ac7dfba4f..827f7fd5f 100644 --- a/dsl/platform/src/main/groovy/net/neoforged/gradle/dsl/platform/model/InstallerProfile.groovy +++ b/dsl/platform/src/main/groovy/net/neoforged/gradle/dsl/platform/model/InstallerProfile.groovy @@ -12,15 +12,17 @@ import groovy.transform.CompileStatic import net.minecraftforge.gdi.ConfigurableDSLElement import net.minecraftforge.gdi.annotations.ClosureEquivalent import net.minecraftforge.gdi.annotations.DSLProperty -import net.neoforged.gradle.dsl.common.util.ConfigurationUtils -import net.neoforged.gradle.dsl.platform.util.CoordinateCollector import net.neoforged.gradle.dsl.platform.util.LibraryCollector import net.neoforged.gradle.util.ModuleDependencyUtils +import org.apache.commons.io.FilenameUtils import org.gradle.api.Action +import org.gradle.api.NamedDomainObjectProvider import org.gradle.api.Project +import org.gradle.api.UnknownDomainObjectException import org.gradle.api.artifacts.Configuration -import org.gradle.api.artifacts.Dependency +import org.gradle.api.artifacts.component.ModuleComponentIdentifier import org.gradle.api.artifacts.repositories.MavenArtifactRepository +import org.gradle.api.artifacts.result.ResolvedArtifactResult import org.gradle.api.model.ObjectFactory import org.gradle.api.provider.ListProperty import org.gradle.api.provider.MapProperty @@ -34,8 +36,6 @@ import org.jetbrains.annotations.Nullable import javax.inject.Inject import java.lang.reflect.Type -import java.util.function.BiConsumer -import java.util.function.BiFunction import java.util.stream.Collectors import static net.neoforged.gradle.dsl.common.util.PropertyUtils.deserializeBool @@ -139,6 +139,11 @@ abstract class InstallerProfile implements ConfigurableDSLElement getData(); + /** + * Track which tools were already added to avoid re-resolving the download URLs for the same tool over and over. + */ + private final Set toolLibrariesAdded = new HashSet<>(); + void data(String key, @Nullable String client, @Nullable String server) { getData().put(key, getObjectFactory().newInstance(DataFile.class).configure { DataFile it -> if (client != null) @@ -162,48 +167,86 @@ abstract class InstallerProfile implements ConfigurableDSLElement getProcessors(); - @ClosureEquivalent - void processor(Project project, Action configurator) { - final Processor processor = getObjectFactory().newInstance(Processor.class).configure(new Action() { - @Override - void execute(Processor processor) { - configurator.execute(processor) - - processor.getClasspath().set(processor.getJar().map(tool -> { - final Configuration detached = ConfigurationUtils.temporaryConfiguration( - project, - "InstallerProfileCoordinateLookup" + ModuleDependencyUtils.toConfigurationName(tool), - project.getDependencies().create(tool) - ) - - final CoordinateCollector filler = new CoordinateCollector(project.getObjects()) - detached.getAsFileTree().visit filler - return filler.getCoordinates() - })) - } - }); + private NamedDomainObjectProvider getOrCreateConfigurationForTool(Project project, String tool) { + var configName = "neoForgeInstallerTool" + ModuleDependencyUtils.toConfigurationName(tool) + try { + return project.configurations.named(configName) + } catch (UnknownDomainObjectException ignored) { + return project.configurations.register(configName, (Configuration spec) -> { + spec.canBeConsumed = false + spec.canBeResolved = true + spec.dependencies.add(project.getDependencies().create(tool)) + }) + } + } - getProcessors().add(processor) + private static Provider> gatherLibrariesFromConfiguration(Project project, Provider configurationProvider) { + var repositoryUrls = project.getRepositories() + .withType(MavenArtifactRepository).stream().map { it.url }.collect(Collectors.toList()) + var logger = project.logger - getLibraries().addAll project.providers.zip( - processor.getClasspath(), processor.getJar(), new BiFunction, String, Iterable>() { - @Override - Iterable apply(Set classPath, String tool) { - final Set dependencyCoordinates = new HashSet<>(classPath) - dependencyCoordinates.add(tool) - - final Dependency[] dependencies = dependencyCoordinates.stream().map { coord -> project.getDependencies().create(coord) }.toArray(Dependency[]::new) - final Configuration configuration = ConfigurationUtils.temporaryConfiguration( - project, - "InstallerProfileLibraryLookup" + ModuleDependencyUtils.toConfigurationName(tool), - dependencies) - - final LibraryCollector collector = new LibraryCollector(project.getObjects(), project.getRepositories() - .withType(MavenArtifactRepository).stream().map { it.url }.collect(Collectors.toList())) - configuration.getAsFileTree().visit collector - return collector.getLibraries() + var objectFactory = project.objects + + // We use a property because it is *not* re-evaluated when queried, while a normal provider is + var property = project.objects.setProperty(Library.class) + property.set(configurationProvider.flatMap { config -> + logger.info("Finding download URLs for configuration ${config.name}") + config.incoming.artifacts.resolvedArtifacts.map { artifacts -> + var libraryCollector = new LibraryCollector(objectFactory, repositoryUrls, logger) + + for (ResolvedArtifactResult resolvedArtifact in artifacts) { + libraryCollector.visit(resolvedArtifact) + } + + libraryCollector.getLibraries() } }) + property.finalizeValueOnRead() + property.disallowChanges() + return property + } + + private static Provider> gatherLibraryIdsFromConfiguration(Project project, Provider configurationProvider) { + // We use a property because it is *not* re-evaluated when queried, while a normal provider is + var property = project.objects.setProperty(String.class) + def logger = project.logger + property.set(configurationProvider.flatMap { config -> + config.incoming.artifacts.resolvedArtifacts.map { artifacts -> + artifacts.collect { + def componentId = it.id.componentIdentifier + if (componentId instanceof ModuleComponentIdentifier) { + var group = componentId.getGroup() + var module = componentId.getModule() + var version = componentId.getVersion() + var classifier = LibraryCollector.guessMavenClassifier(it.file, componentId) + var extension = FilenameUtils.getExtension(it.file.name) + if (classifier != "") { + version += ":" + classifier + } + return "$group:$module:$version@$extension".toString() + } else { + logger.warn("Cannot handle component: " + componentId) + return null + } + } + } + }) + property.finalizeValueOnRead() + property.disallowChanges() + return property + } + + @ClosureEquivalent + void processor(Project project, String tool, Action configurator) { + var processor = getObjectFactory().newInstance(Processor.class).configure(configurator); + processor.jar.set(tool) + getProcessors().add(processor) + + var toolConfiguration = getOrCreateConfigurationForTool(project, tool) + processor.classpath.set(gatherLibraryIdsFromConfiguration(project, toolConfiguration)) + if (toolLibrariesAdded.add(tool)) { + getLibraries().addAll gatherLibrariesFromConfiguration(project, toolConfiguration) + } } @Nested diff --git a/dsl/platform/src/main/groovy/net/neoforged/gradle/dsl/platform/util/LibraryCollector.groovy b/dsl/platform/src/main/groovy/net/neoforged/gradle/dsl/platform/util/LibraryCollector.groovy index 72ac47186..90f42da7d 100644 --- a/dsl/platform/src/main/groovy/net/neoforged/gradle/dsl/platform/util/LibraryCollector.groovy +++ b/dsl/platform/src/main/groovy/net/neoforged/gradle/dsl/platform/util/LibraryCollector.groovy @@ -5,23 +5,103 @@ import net.neoforged.gradle.dsl.platform.model.Artifact import net.neoforged.gradle.dsl.platform.model.Library import net.neoforged.gradle.dsl.platform.model.LibraryDownload import net.neoforged.gradle.util.HashFunction +import org.apache.commons.io.FilenameUtils +import org.gradle.api.artifacts.component.ModuleComponentIdentifier +import org.gradle.api.artifacts.result.ResolvedArtifactResult +import org.gradle.api.logging.Logger import org.gradle.api.model.ObjectFactory import org.jetbrains.annotations.Nullable +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse import java.nio.file.Files +import java.util.concurrent.CompletableFuture +import java.util.concurrent.Future +import java.util.function.Function @CompileStatic class LibraryCollector extends ModuleIdentificationVisitor { + /** + * Hosts from which we allow the installer to download. + * We whitelist here to avoid redirecting player download traffic to anyone not affiliated with Mojang or us. + */ + private static final String HOST_WHITELIST = List.of( + "minecraft.net", + "neoforged.net", + "mojang.com" + ); + + private static final URI MOJANG_MAVEN = URI.create("https://libraries.minecraft.net") + private static final URI NEOFORGED_MAVEN = URI.create("https://maven.neoforged.net/releases") + private final ObjectFactory objectFactory; private final List repositoryUrls - private final List libraries = new ArrayList<>(); + private final List> libraries = new ArrayList<>(); - LibraryCollector(ObjectFactory objectFactory, List repoUrl) { + private final HttpClient httpClient = HttpClient.newBuilder().build(); + private final Logger logger + + LibraryCollector(ObjectFactory objectFactory, List repoUrl, Logger logger) { super(objectFactory); + this.logger = logger this.objectFactory = objectFactory; - this.repositoryUrls = repoUrl + this.repositoryUrls = new ArrayList<>(repoUrl) + + // Only remote repositories make sense (no maven local) + repositoryUrls.removeIf { + var lowercaseScheme = it.scheme.toLowerCase(Locale.ROOT) + lowercaseScheme != "https" && lowercaseScheme != "http" + } + // Allow only URLs from whitelisted hosts + repositoryUrls.removeIf { uri -> + var lowercaseHost = uri.host.toLowerCase(Locale.ROOT) + !HOST_WHITELIST.any { lowercaseHost == it || lowercaseHost.endsWith("." + it) } + } + // Always try Mojang Maven first, then our installer Maven + repositoryUrls.removeIf { it.host == MOJANG_MAVEN.host } + repositoryUrls.removeIf { it.host == NEOFORGED_MAVEN.host && it.path.startsWith(NEOFORGED_MAVEN.path) } + repositoryUrls.add(0, NEOFORGED_MAVEN) + repositoryUrls.add(0, MOJANG_MAVEN) + + logger.info("Collecting libraries from:") + for (var repo in repositoryUrls) { + logger.info(" - $repo") + } + } + + void visit(ResolvedArtifactResult artifactResult) { + def componentId = artifactResult.id.componentIdentifier + if (componentId instanceof ModuleComponentIdentifier) { + visitModule( + artifactResult.file, + componentId.getGroup(), + componentId.getModule(), + componentId.getVersion(), + guessMavenClassifier(artifactResult.file, componentId), + FilenameUtils.getExtension(artifactResult.file.name) + ) + } else { + logger.warn("Cannot handle component: " + componentId) + } + } + + static String guessMavenClassifier(File file, ModuleComponentIdentifier id) { + var artifact = id.module + var version = id.version + var expectedBasename = artifact + "-" + version; + var filename = file.name + var startOfExt = filename.lastIndexOf('.'); + if (startOfExt != -1) { + filename = filename.substring(0, startOfExt); + } + + if (filename.startsWith(expectedBasename + "-")) { + return filename.substring((expectedBasename + "-").length()); + } + return "" } @Override @@ -33,56 +113,75 @@ class LibraryCollector extends ModuleIdentificationVisitor { library.getDownload().set(download); download.getArtifact().set(artifact); - final String path = group.replace(".", "/") + "/" + module + "/" + version + "/" + module + "-" + version + (classifier.isEmpty() ? "" : "-" + classifier) + "." + extension; - String url = getMavenServerFor(path) + "/" + path; - int pos = 0 - while (attemptConnection(url) !== 200 && pos < repositoryUrls.size()) { - url = repositoryUrls.get(pos++).resolve(path).toString() - } - final String name = group + ":" + module + ":" + version + (classifier.isEmpty() ? "" : ":" + classifier) + "@" + extension; + final String path = group.replace(".", "/") + "/" + module + "/" + version + "/" + module + "-" + version + (classifier.isEmpty() ? "" : "-" + classifier) + "." + extension; library.getName().set(name); try { artifact.getPath().set(path); - artifact.getUrl().set(url); artifact.getSha1().set(HashFunction.SHA1.hash(file)); artifact.getSize().set(Files.size(file.toPath())); } catch (IOException e) { throw new UncheckedIOException(e); } - libraries.add(library); - } + // Try each configured repository in-order to find the file + CompletableFuture libraryFuture = null; + for (var repositoryUrl in repositoryUrls) { + def artifactUri = joinUris(repositoryUrl, path) + var request = HttpRequest.newBuilder(artifactUri) + .method("HEAD", HttpRequest.BodyPublishers.noBody()) + .build() - private static int attemptConnection(String url) { - try { - final conn = (HttpURLConnection) url.toURL().openConnection() - conn.setRequestMethod('HEAD') - conn.connect() - int rc = conn.responseCode - conn.disconnect() - return rc - } catch (Exception ignored) { - return 404 + Function> makeRequest = (String previousError) -> { + httpClient.sendAsync(request, HttpResponse.BodyHandlers.discarding()) + .thenApply { response -> + if (response.statusCode() != 200) { + logger.info(" Got ${response.statusCode()} for ${artifactUri}") + String message = "Could not find ${artifactUri}: ${response.statusCode()}" + // Prepend error message from previous repo if they all fail + if (previousError != null) { + message = previousError + "\n" + message + } + throw new RuntimeException(message) + } + logger.info(" Found $name -> $artifactUri") + artifact.getUrl().set(artifactUri.toString()); + library + } + }; + + if (libraryFuture == null) { + libraryFuture = makeRequest(null) + } else { + libraryFuture = libraryFuture.exceptionallyCompose { error -> + makeRequest(error.getMessage()) + } + } } + + libraries.add(libraryFuture); } - private static String getMavenServerFor(String path) { - try { - final URL mojangMavenUrl = new URL("https://libraries.minecraft.net/" + path); - final HttpURLConnection connection = (HttpURLConnection) mojangMavenUrl.openConnection(); - connection.setRequestMethod("HEAD"); - connection.connect(); - return connection.getResponseCode() == 200 ? "https://libraries.minecraft.net" : "https://maven.neoforged.net/releases"; - } catch (MalformedURLException e) { - throw new RuntimeException(e); - } catch (IOException e) { - return "https://maven.neoforged.net/releases"; + private static URI joinUris(URI repositoryUrl, String path) { + var baseUrl = repositoryUrl.toString() + if (baseUrl.endsWith("/") && path.startsWith("/")) { + while (path.startsWith("/")) { + path = path.substring(1) + } + return URI.create(baseUrl + path) + } else if (!baseUrl.endsWith("/") && !path.startsWith("/")) { + return URI.create(baseUrl + "/" + path) + } else { + return URI.create(baseUrl + path) } } - List getLibraries() { - return libraries; + Set getLibraries() { + var result = libraries.collect { + it.get() + } + logger.info("Collected ${result.size()} libraries") + return new HashSet<>(result) } } diff --git a/platform/src/main/java/net/neoforged/gradle/platform/extensions/DynamicProjectExtension.java b/platform/src/main/java/net/neoforged/gradle/platform/extensions/DynamicProjectExtension.java index 41c4f409a..dfeae5e87 100644 --- a/platform/src/main/java/net/neoforged/gradle/platform/extensions/DynamicProjectExtension.java +++ b/platform/src/main/java/net/neoforged/gradle/platform/extensions/DynamicProjectExtension.java @@ -392,9 +392,8 @@ public void runtime(final String neoFormVersion, Directory patches, Directory re profile.data("MC_SRG", String.format("[net.minecraft:client:%s:srg]", neoFormVersion), String.format("[net.minecraft:server:%s:srg]", neoFormVersion)); profile.data("PATCHED", String.format("[%s:%s:%s:client]", "net.neoforged", "neoforge", project.getVersion()), String.format("[%s:%s:%s:server]", "net.neoforged", "neoforge", project.getVersion())); profile.data("MCP_VERSION", String.format("'%s'", neoFormVersion), String.format("'%s'", neoFormVersion)); - profile.processor(project, processor -> { + profile.processor(project, Constants.INSTALLERTOOLS, processor -> { processor.server(); - processor.getJar().set(Constants.INSTALLERTOOLS); processor.getArguments().addAll("--task", "EXTRACT_FILES", "--archive", "{INSTALLER}", "--from", "data/run.sh", "--to", "{ROOT}/run.sh", "--exec", "{ROOT}/run.sh", @@ -407,44 +406,35 @@ public void runtime(final String neoFormVersion, Directory patches, Directory re "--from", "data/unix_args.txt", "--to", String.format("{ROOT}/libraries/%s/%s/%s/unix_args.txt", project.getGroup().toString().replaceAll("\\.", "/"), project.getName(), project.getVersion())); }); - profile.processor(project, processor -> { + profile.processor(project, Constants.INSTALLERTOOLS, processor -> { processor.server(); - processor.getJar().set(Constants.INSTALLERTOOLS); processor.getArguments().addAll("--task", "BUNDLER_EXTRACT", "--input", "{MINECRAFT_JAR}", "--output", "{ROOT}/libraries/", "--libraries"); }); - profile.processor(project, processor -> { + profile.processor(project, Constants.INSTALLERTOOLS, processor -> { processor.server(); - processor.getJar().set(Constants.INSTALLERTOOLS); processor.getArguments().addAll("--task", "BUNDLER_EXTRACT", "--input", "{MINECRAFT_JAR}", "--output", "{MC_UNPACKED}", "--jar-only"); }); - profile.processor(project, processor -> { - processor.getJar().set(Constants.INSTALLERTOOLS); + profile.processor(project, Constants.INSTALLERTOOLS, processor -> { processor.getArguments().addAll("--task", "MCP_DATA", "--input", String.format("[%s]", neoformDependency), "--output", "{MAPPINGS}", "--key", "mappings"); }); - profile.processor(project, processor -> { - processor.getJar().set(Constants.INSTALLERTOOLS); + profile.processor(project, Constants.INSTALLERTOOLS, processor -> { processor.getArguments().addAll("--task", "DOWNLOAD_MOJMAPS", "--version", runtimeDefinition.getSpecification().getMinecraftVersion(), "--side", "{SIDE}", "--output", "{MOJMAPS}"); }); - profile.processor(project, processor -> { - processor.getJar().set(Constants.INSTALLERTOOLS); + profile.processor(project, Constants.INSTALLERTOOLS, processor -> { processor.getArguments().addAll("--task", "MERGE_MAPPING", "--left", "{MAPPINGS}", "--right", "{MOJMAPS}", "--output", "{MERGED_MAPPINGS}", "--classes", "--fields", "--methods", "--reverse-right"); }); - profile.processor(project, processor -> { + profile.processor(project, Constants.JARSPLITTER, processor -> { processor.client(); - processor.getJar().set(Constants.JARSPLITTER); processor.getArguments().addAll("--input", "{MINECRAFT_JAR}", "--slim", "{MC_SLIM}", "--extra", "{MC_EXTRA}", "--srg", "{MERGED_MAPPINGS}"); }); - profile.processor(project, processor -> { + profile.processor(project, Constants.JARSPLITTER, processor -> { processor.server(); - processor.getJar().set(Constants.JARSPLITTER); processor.getArguments().addAll("--input", "{MC_UNPACKED}", "--slim", "{MC_SLIM}", "--extra", "{MC_EXTRA}", "--srg", "{MERGED_MAPPINGS}"); }); - profile.processor(project, processor -> { - processor.getJar().set(Constants.FART); + profile.processor(project, Constants.FART, processor -> { processor.getArguments().addAll("--input", "{MC_SLIM}", "--output", "{MC_SRG}", "--names", "{MERGED_MAPPINGS}", "--ann-fix", "--ids-fix", "--src-fix", "--record-fix"); }); - profile.processor(project, processor -> { - processor.getJar().set(Constants.BINARYPATCHER); + profile.processor(project, Constants.BINARYPATCHER, processor -> { processor.getArguments().addAll("--clean", "{MC_SRG}", "--output", "{PATCHED}", "--apply", "{BINPATCH}"); }); diff --git a/platform/src/main/java/net/neoforged/gradle/platform/tasks/CreateLauncherJson.java b/platform/src/main/java/net/neoforged/gradle/platform/tasks/CreateLauncherJson.java index 30146f45d..79e2f063b 100644 --- a/platform/src/main/java/net/neoforged/gradle/platform/tasks/CreateLauncherJson.java +++ b/platform/src/main/java/net/neoforged/gradle/platform/tasks/CreateLauncherJson.java @@ -35,7 +35,8 @@ public void run() { clone.getLibraries().addAll( getProviderFactory().provider(() -> { - final LibraryCollector profileFiller = new LibraryCollector(getObjectFactory(), getRepositoryURLs().get()); + getLogger().info("Collecting libraries for Launcher Profile"); + final LibraryCollector profileFiller = new LibraryCollector(getObjectFactory(), getRepositoryURLs().get(), getLogger()); getLibraries().getAsFileTree().visit(profileFiller); return profileFiller.getLibraries(); }) diff --git a/platform/src/main/java/net/neoforged/gradle/platform/tasks/CreateLegacyInstallerJson.java b/platform/src/main/java/net/neoforged/gradle/platform/tasks/CreateLegacyInstallerJson.java index ec3bbf9e1..851772f03 100644 --- a/platform/src/main/java/net/neoforged/gradle/platform/tasks/CreateLegacyInstallerJson.java +++ b/platform/src/main/java/net/neoforged/gradle/platform/tasks/CreateLegacyInstallerJson.java @@ -31,14 +31,12 @@ public void run() { final InstallerProfile profile = getProfile().get(); final InstallerProfile copy = gson.fromJson(gson.toJson(profile), InstallerProfile.class); - - copy.getLibraries().addAll( - getProviderFactory().provider(() -> { - final LibraryCollector profileFiller = new LibraryCollector(getObjectFactory(), getRepositoryURLs().get()); - getLibraries().getAsFileTree().visit(profileFiller); - return profileFiller.getLibraries(); - }) - ); + + getLogger().info("Collecting gameplay libraries for installer"); + var profileFiller = new LibraryCollector(getObjectFactory(), getRepositoryURLs().get(), getLogger()); + getLibraries().getAsFileTree().visit(profileFiller); + copy.getLibraries().addAll(profileFiller.getLibraries()); + try { Files.write(output.toPath(), gson.toJson(copy).getBytes()); } catch (IOException e) {