Skip to content

Commit

Permalink
Rework how download locations for libraries are resolved (#206)
Browse files Browse the repository at this point in the history
  • Loading branch information
@shartte
shartte authored Jun 15, 2024
1 parent 48ae9fa commit f909e26
Show file tree
Hide file tree
Showing 5 changed files with 239 additions and 108 deletions.
129 changes: 86 additions & 43 deletions dsl/platform/src/main/groovy/net/neoforged/gradle/dsl/platform/model/InstallerProfile.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,17 @@ import groovy.transform.CompileStatic
import net.minecraftforge.gdi.ConfigurableDSLElement
import net.minecraftforge.gdi.annotations.ClosureEquivalent
import net.minecraftforge.gdi.annotations.DSLProperty
import net.neoforged.gradle.dsl.common.util.ConfigurationUtils
import net.neoforged.gradle.dsl.platform.util.CoordinateCollector
import net.neoforged.gradle.dsl.platform.util.LibraryCollector
import net.neoforged.gradle.util.ModuleDependencyUtils
import org.apache.commons.io.FilenameUtils
import org.gradle.api.Action
import org.gradle.api.NamedDomainObjectProvider
import org.gradle.api.Project
import org.gradle.api.UnknownDomainObjectException
import org.gradle.api.artifacts.Configuration
import org.gradle.api.artifacts.Dependency
import org.gradle.api.artifacts.component.ModuleComponentIdentifier
import org.gradle.api.artifacts.repositories.MavenArtifactRepository
import org.gradle.api.artifacts.result.ResolvedArtifactResult
import org.gradle.api.model.ObjectFactory
import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.MapProperty
Expand All @@ -34,8 +36,6 @@ import org.jetbrains.annotations.Nullable

import javax.inject.Inject
import java.lang.reflect.Type
import java.util.function.BiConsumer
import java.util.function.BiFunction
import java.util.stream.Collectors

import static net.neoforged.gradle.dsl.common.util.PropertyUtils.deserializeBool
Expand Down Expand Up @@ -139,6 +139,11 @@ abstract class InstallerProfile implements ConfigurableDSLElement<InstallerProfi
@Optional
abstract MapProperty<String, DataFile> getData();

/**
* Track which tools were already added to avoid re-resolving the download URLs for the same tool over and over.
*/
private final Set<String> toolLibrariesAdded = new HashSet<>();

void data(String key, @Nullable String client, @Nullable String server) {
getData().put(key, getObjectFactory().newInstance(DataFile.class).configure { DataFile it ->
if (client != null)
Expand All @@ -162,48 +167,86 @@ abstract class InstallerProfile implements ConfigurableDSLElement<InstallerProfi
@Optional
abstract ListProperty<Processor> getProcessors();

@ClosureEquivalent
void processor(Project project, Action<Processor> configurator) {
final Processor processor = getObjectFactory().newInstance(Processor.class).configure(new Action<Processor>() {
@Override
void execute(Processor processor) {
configurator.execute(processor)

processor.getClasspath().set(processor.getJar().map(tool -> {
final Configuration detached = ConfigurationUtils.temporaryConfiguration(
project,
"InstallerProfileCoordinateLookup" + ModuleDependencyUtils.toConfigurationName(tool),
project.getDependencies().create(tool)
)

final CoordinateCollector filler = new CoordinateCollector(project.getObjects())
detached.getAsFileTree().visit filler
return filler.getCoordinates()
}))
}
});
private NamedDomainObjectProvider<Configuration> getOrCreateConfigurationForTool(Project project, String tool) {
var configName = "neoForgeInstallerTool" + ModuleDependencyUtils.toConfigurationName(tool)
try {
return project.configurations.named(configName)
} catch (UnknownDomainObjectException ignored) {
return project.configurations.register(configName, (Configuration spec) -> {
spec.canBeConsumed = false
spec.canBeResolved = true
spec.dependencies.add(project.getDependencies().create(tool))
})
}
}

getProcessors().add(processor)
private static Provider<Set<Library>> gatherLibrariesFromConfiguration(Project project, Provider<Configuration> configurationProvider) {
var repositoryUrls = project.getRepositories()
.withType(MavenArtifactRepository).stream().map { it.url }.collect(Collectors.toList())
var logger = project.logger

getLibraries().addAll project.providers.zip(
processor.getClasspath(), processor.getJar(), new BiFunction<Set<String>, String, Iterable<Library>>() {
@Override
Iterable<Library> apply(Set<String> classPath, String tool) {
final Set<String> dependencyCoordinates = new HashSet<>(classPath)
dependencyCoordinates.add(tool)

final Dependency[] dependencies = dependencyCoordinates.stream().map { coord -> project.getDependencies().create(coord) }.toArray(Dependency[]::new)
final Configuration configuration = ConfigurationUtils.temporaryConfiguration(
project,
"InstallerProfileLibraryLookup" + ModuleDependencyUtils.toConfigurationName(tool),
dependencies)

final LibraryCollector collector = new LibraryCollector(project.getObjects(), project.getRepositories()
.withType(MavenArtifactRepository).stream().map { it.url }.collect(Collectors.toList()))
configuration.getAsFileTree().visit collector
return collector.getLibraries()
var objectFactory = project.objects

// We use a property because it is *not* re-evaluated when queried, while a normal provider is
var property = project.objects.setProperty(Library.class)
property.set(configurationProvider.flatMap { config ->
logger.info("Finding download URLs for configuration ${config.name}")
config.incoming.artifacts.resolvedArtifacts.map { artifacts ->
var libraryCollector = new LibraryCollector(objectFactory, repositoryUrls, logger)

for (ResolvedArtifactResult resolvedArtifact in artifacts) {
libraryCollector.visit(resolvedArtifact)
}

libraryCollector.getLibraries()
}
})
property.finalizeValueOnRead()
property.disallowChanges()
return property
}

private static Provider<Set<String>> gatherLibraryIdsFromConfiguration(Project project, Provider<Configuration> configurationProvider) {
// We use a property because it is *not* re-evaluated when queried, while a normal provider is
var property = project.objects.setProperty(String.class)
def logger = project.logger
property.set(configurationProvider.flatMap { config ->
config.incoming.artifacts.resolvedArtifacts.map { artifacts ->
artifacts.collect {
def componentId = it.id.componentIdentifier
if (componentId instanceof ModuleComponentIdentifier) {
var group = componentId.getGroup()
var module = componentId.getModule()
var version = componentId.getVersion()
var classifier = LibraryCollector.guessMavenClassifier(it.file, componentId)
var extension = FilenameUtils.getExtension(it.file.name)
if (classifier != "") {
version += ":" + classifier
}
return "$group:$module:$version@$extension".toString()
} else {
logger.warn("Cannot handle component: " + componentId)
return null
}
}
}
})
property.finalizeValueOnRead()
property.disallowChanges()
return property
}

@ClosureEquivalent
void processor(Project project, String tool, Action<Processor> configurator) {
var processor = getObjectFactory().newInstance(Processor.class).configure(configurator);
processor.jar.set(tool)
getProcessors().add(processor)

var toolConfiguration = getOrCreateConfigurationForTool(project, tool)
processor.classpath.set(gatherLibraryIdsFromConfiguration(project, toolConfiguration))
if (toolLibrariesAdded.add(tool)) {
getLibraries().addAll gatherLibrariesFromConfiguration(project, toolConfiguration)
}
}

@Nested
Expand Down
171 changes: 135 additions & 36 deletions dsl/platform/src/main/groovy/net/neoforged/gradle/dsl/platform/util/LibraryCollector.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,103 @@ import net.neoforged.gradle.dsl.platform.model.Artifact
import net.neoforged.gradle.dsl.platform.model.Library
import net.neoforged.gradle.dsl.platform.model.LibraryDownload
import net.neoforged.gradle.util.HashFunction
import org.apache.commons.io.FilenameUtils
import org.gradle.api.artifacts.component.ModuleComponentIdentifier
import org.gradle.api.artifacts.result.ResolvedArtifactResult
import org.gradle.api.logging.Logger
import org.gradle.api.model.ObjectFactory
import org.jetbrains.annotations.Nullable

import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import java.nio.file.Files
import java.util.concurrent.CompletableFuture
import java.util.concurrent.Future
import java.util.function.Function

@CompileStatic
class LibraryCollector extends ModuleIdentificationVisitor {

/**
* Hosts from which we allow the installer to download.
* We whitelist here to avoid redirecting player download traffic to anyone not affiliated with Mojang or us.
*/
private static final String HOST_WHITELIST = List.of(
"minecraft.net",
"neoforged.net",
"mojang.com"
);

private static final URI MOJANG_MAVEN = URI.create("https://libraries.minecraft.net")
private static final URI NEOFORGED_MAVEN = URI.create("https://maven.neoforged.net/releases")

private final ObjectFactory objectFactory;
private final List<URI> repositoryUrls

private final List<Library> libraries = new ArrayList<>();
private final List<Future<Library>> libraries = new ArrayList<>();

LibraryCollector(ObjectFactory objectFactory, List<URI> repoUrl) {
private final HttpClient httpClient = HttpClient.newBuilder().build();
private final Logger logger

LibraryCollector(ObjectFactory objectFactory, List<URI> repoUrl, Logger logger) {
super(objectFactory);
this.logger = logger
this.objectFactory = objectFactory;
this.repositoryUrls = repoUrl
this.repositoryUrls = new ArrayList<>(repoUrl)

// Only remote repositories make sense (no maven local)
repositoryUrls.removeIf {
var lowercaseScheme = it.scheme.toLowerCase(Locale.ROOT)
lowercaseScheme != "https" && lowercaseScheme != "http"
}
// Allow only URLs from whitelisted hosts
repositoryUrls.removeIf { uri ->
var lowercaseHost = uri.host.toLowerCase(Locale.ROOT)
!HOST_WHITELIST.any { lowercaseHost == it || lowercaseHost.endsWith("." + it) }
}
// Always try Mojang Maven first, then our installer Maven
repositoryUrls.removeIf { it.host == MOJANG_MAVEN.host }
repositoryUrls.removeIf { it.host == NEOFORGED_MAVEN.host && it.path.startsWith(NEOFORGED_MAVEN.path) }
repositoryUrls.add(0, NEOFORGED_MAVEN)
repositoryUrls.add(0, MOJANG_MAVEN)

logger.info("Collecting libraries from:")
for (var repo in repositoryUrls) {
logger.info(" - $repo")
}
}

void visit(ResolvedArtifactResult artifactResult) {
def componentId = artifactResult.id.componentIdentifier
if (componentId instanceof ModuleComponentIdentifier) {
visitModule(
artifactResult.file,
componentId.getGroup(),
componentId.getModule(),
componentId.getVersion(),
guessMavenClassifier(artifactResult.file, componentId),
FilenameUtils.getExtension(artifactResult.file.name)
)
} else {
logger.warn("Cannot handle component: " + componentId)
}
}

static String guessMavenClassifier(File file, ModuleComponentIdentifier id) {
var artifact = id.module
var version = id.version
var expectedBasename = artifact + "-" + version;
var filename = file.name
var startOfExt = filename.lastIndexOf('.');
if (startOfExt != -1) {
filename = filename.substring(0, startOfExt);
}

if (filename.startsWith(expectedBasename + "-")) {
return filename.substring((expectedBasename + "-").length());
}
return ""
}

@Override
Expand All @@ -33,56 +113,75 @@ class LibraryCollector extends ModuleIdentificationVisitor {
library.getDownload().set(download);
download.getArtifact().set(artifact);

final String path = group.replace(".", "/") + "/" + module + "/" + version + "/" + module + "-" + version + (classifier.isEmpty() ? "" : "-" + classifier) + "." + extension;
String url = getMavenServerFor(path) + "/" + path;
int pos = 0
while (attemptConnection(url) !== 200 && pos < repositoryUrls.size()) {
url = repositoryUrls.get(pos++).resolve(path).toString()
}

final String name = group + ":" + module + ":" + version + (classifier.isEmpty() ? "" : ":" + classifier) + "@" + extension;
final String path = group.replace(".", "/") + "/" + module + "/" + version + "/" + module + "-" + version + (classifier.isEmpty() ? "" : "-" + classifier) + "." + extension;

library.getName().set(name);
try {
artifact.getPath().set(path);
artifact.getUrl().set(url);
artifact.getSha1().set(HashFunction.SHA1.hash(file));
artifact.getSize().set(Files.size(file.toPath()));
} catch (IOException e) {
throw new UncheckedIOException(e);
}

libraries.add(library);
}
// Try each configured repository in-order to find the file
CompletableFuture<Library> libraryFuture = null;
for (var repositoryUrl in repositoryUrls) {
def artifactUri = joinUris(repositoryUrl, path)
var request = HttpRequest.newBuilder(artifactUri)
.method("HEAD", HttpRequest.BodyPublishers.noBody())
.build()

private static int attemptConnection(String url) {
try {
final conn = (HttpURLConnection) url.toURL().openConnection()
conn.setRequestMethod('HEAD')
conn.connect()
int rc = conn.responseCode
conn.disconnect()
return rc
} catch (Exception ignored) {
return 404
Function<String, CompletableFuture<Library>> makeRequest = (String previousError) -> {
httpClient.sendAsync(request, HttpResponse.BodyHandlers.discarding())
.thenApply { response ->
if (response.statusCode() != 200) {
logger.info(" Got ${response.statusCode()} for ${artifactUri}")
String message = "Could not find ${artifactUri}: ${response.statusCode()}"
// Prepend error message from previous repo if they all fail
if (previousError != null) {
message = previousError + "\n" + message
}
throw new RuntimeException(message)
}
logger.info(" Found $name -> $artifactUri")
artifact.getUrl().set(artifactUri.toString());
library
}
};

if (libraryFuture == null) {
libraryFuture = makeRequest(null)
} else {
libraryFuture = libraryFuture.exceptionallyCompose { error ->
makeRequest(error.getMessage())
}
}
}

libraries.add(libraryFuture);
}

private static String getMavenServerFor(String path) {
try {
final URL mojangMavenUrl = new URL("https://libraries.minecraft.net/" + path);
final HttpURLConnection connection = (HttpURLConnection) mojangMavenUrl.openConnection();
connection.setRequestMethod("HEAD");
connection.connect();
return connection.getResponseCode() == 200 ? "https://libraries.minecraft.net" : "https://maven.neoforged.net/releases";
} catch (MalformedURLException e) {
throw new RuntimeException(e);
} catch (IOException e) {
return "https://maven.neoforged.net/releases";
private static URI joinUris(URI repositoryUrl, String path) {
var baseUrl = repositoryUrl.toString()
if (baseUrl.endsWith("/") && path.startsWith("/")) {
while (path.startsWith("/")) {
path = path.substring(1)
}
return URI.create(baseUrl + path)
} else if (!baseUrl.endsWith("/") && !path.startsWith("/")) {
return URI.create(baseUrl + "/" + path)
} else {
return URI.create(baseUrl + path)
}
}

List<Library> getLibraries() {
return libraries;
Set<Library> getLibraries() {
var result = libraries.collect {
it.get()
}
logger.info("Collected ${result.size()} libraries")
return new HashSet<>(result)
}
}
Loading

0 comments on commit f909e26

Please sign in to comment.