From 01a261743847d783127ebc732fe1e0c59a8de513 Mon Sep 17 00:00:00 2001 From: AlexanderBartash Date: Fri, 4 Oct 2024 04:24:19 +0300 Subject: [PATCH] Improved build performance by pre-creating Ivy XML files in the extracted IDE location. --- CHANGELOG.md | 1 + api/IntelliJPlatformGradlePlugin.api | 10 +- .../transform/ExtractorTransformer.kt | 75 +++- .../IntelliJPlatformDependenciesHelper.kt | 420 ++++++++++++------ .../IntelliJPlatformRepositoriesHelper.kt | 114 +++-- .../platform/gradle/models/IvyModule.kt | 80 ++-- 6 files changed, 478 insertions(+), 222 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1eea5fe352..7d46c91a03 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Changed +- Improved build performance by pre-creating Ivy XML files in the extracted IDE location. - Improved build performance by making the local Ivy repository first in the list and prefixing all "fake" artifact groups with "com.jetbrains.localhost-only" and excluding that group from already declared remote repositories. ### Fixed diff --git a/api/IntelliJPlatformGradlePlugin.api b/api/IntelliJPlatformGradlePlugin.api index ac54bbf5bc..3e9ffd5b3a 100644 --- a/api/IntelliJPlatformGradlePlugin.api +++ b/api/IntelliJPlatformGradlePlugin.api @@ -432,7 +432,7 @@ public abstract interface class org/jetbrains/intellij/platform/gradle/artifacts public abstract class org/jetbrains/intellij/platform/gradle/artifacts/transform/ExtractorTransformer : org/gradle/api/artifacts/transform/TransformAction { public static final field Companion Lorg/jetbrains/intellij/platform/gradle/artifacts/transform/ExtractorTransformer$Companion; - public fun (Lorg/gradle/api/file/ArchiveOperations;Lorg/gradle/process/ExecOperations;Lorg/gradle/api/model/ObjectFactory;Lorg/gradle/api/file/FileSystemOperations;)V + public fun (Lorg/gradle/api/file/ArchiveOperations;Lorg/gradle/process/ExecOperations;Lorg/gradle/api/model/ObjectFactory;Lorg/gradle/api/provider/ProviderFactory;Lorg/gradle/api/file/FileSystemOperations;)V public abstract fun getInputArtifact ()Lorg/gradle/api/provider/Provider; public fun transform (Lorg/gradle/api/artifacts/transform/TransformOutputs;)V } @@ -688,9 +688,13 @@ public final class org/jetbrains/intellij/platform/gradle/extensions/IntelliJPla } public final class org/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformDependenciesHelper { + public static final field Companion Lorg/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformDependenciesHelper$Companion; public fun (Lorg/gradle/api/artifacts/dsl/RepositoryHandler;Lorg/gradle/api/artifacts/ConfigurationContainer;Lorg/gradle/api/artifacts/dsl/DependencyHandler;Lorg/gradle/api/file/ProjectLayout;Lorg/gradle/api/model/ObjectFactory;Lorg/gradle/api/provider/ProviderFactory;Lorg/gradle/api/resources/ResourceHandler;Ljava/nio/file/Path;)V } +public final class org/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformDependenciesHelper$Companion { +} + public abstract class org/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformExtension : org/gradle/api/plugins/ExtensionAware { public static final field Companion Lorg/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformExtension$Companion; public fun (Lorg/gradle/api/artifacts/ConfigurationContainer;Lorg/gradle/api/provider/ProviderFactory;Ljava/nio/file/Path;)V @@ -955,9 +959,13 @@ public final class org/jetbrains/intellij/platform/gradle/extensions/IntelliJPla } public final class org/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformRepositoriesHelper { + public static final field Companion Lorg/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformRepositoriesHelper$Companion; public fun (Lorg/gradle/api/artifacts/dsl/RepositoryHandler;Lorg/gradle/api/provider/ProviderFactory;Lorg/gradle/api/model/ObjectFactory;Lorg/gradle/api/flow/FlowScope;Lorg/gradle/api/flow/FlowProviders;Lorg/gradle/api/invocation/Gradle;Ljava/nio/file/Path;)V } +public final class org/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformRepositoriesHelper$Companion { +} + public abstract class org/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformTestingExtension : org/gradle/api/plugins/ExtensionAware { public static final field Companion Lorg/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformTestingExtension$Companion; public fun (Lorg/gradle/api/Project;)V diff --git a/src/main/kotlin/org/jetbrains/intellij/platform/gradle/artifacts/transform/ExtractorTransformer.kt b/src/main/kotlin/org/jetbrains/intellij/platform/gradle/artifacts/transform/ExtractorTransformer.kt index a1cec58fef..7e8c66900c 100644 --- a/src/main/kotlin/org/jetbrains/intellij/platform/gradle/artifacts/transform/ExtractorTransformer.kt +++ b/src/main/kotlin/org/jetbrains/intellij/platform/gradle/artifacts/transform/ExtractorTransformer.kt @@ -2,6 +2,7 @@ package org.jetbrains.intellij.platform.gradle.artifacts.transform +import com.jetbrains.plugin.structure.intellij.plugin.IdePluginManager import org.gradle.api.artifacts.Configuration import org.gradle.api.artifacts.dsl.DependencyHandler import org.gradle.api.artifacts.transform.InputArtifact @@ -14,11 +15,19 @@ import org.gradle.api.file.FileSystemOperations import org.gradle.api.file.FileTree import org.gradle.api.model.ObjectFactory import org.gradle.api.provider.Provider +import org.gradle.api.provider.ProviderFactory import org.gradle.api.tasks.Classpath import org.gradle.kotlin.dsl.registerTransform import org.gradle.process.ExecOperations import org.gradle.work.DisableCachingByDefault +import org.jetbrains.intellij.platform.gradle.Constants import org.jetbrains.intellij.platform.gradle.Constants.Configurations.Attributes +import org.jetbrains.intellij.platform.gradle.extensions.IntelliJPlatformDependenciesHelper +import org.jetbrains.intellij.platform.gradle.extensions.IntelliJPlatformDependenciesHelper.Companion.collectBundledPluginDependencies +import org.jetbrains.intellij.platform.gradle.extensions.IntelliJPlatformDependenciesHelper.Companion.getBundledPlugins +import org.jetbrains.intellij.platform.gradle.models.IvyModule +import org.jetbrains.intellij.platform.gradle.models.productInfo +import org.jetbrains.intellij.platform.gradle.models.toBundledIvyArtifacts import org.jetbrains.intellij.platform.gradle.utils.Logger import org.jetbrains.intellij.platform.gradle.utils.asPath import org.jetbrains.intellij.platform.gradle.utils.resolvePlatformPath @@ -36,6 +45,7 @@ abstract class ExtractorTransformer @Inject constructor( private val archiveOperations: ArchiveOperations, private val execOperations: ExecOperations, private val objectFactory: ObjectFactory, + private val providerFactory: ProviderFactory, private val fileSystemOperations: FileSystemOperations, ) : TransformAction { @@ -105,9 +115,13 @@ abstract class ExtractorTransformer @Inject constructor( execOperations.exec { commandLine("hdiutil", "detach", "-force", "-quiet", tempDirectory) } + + createIvyXmls(tempDirectory) } - else -> {} + else -> { + createIvyXmls(targetDirectory) + } } log.info("Extracting to '$targetDirectory' completed.") @@ -116,6 +130,60 @@ abstract class ExtractorTransformer @Inject constructor( } } + /** + * Pre-create Ivy XML files so that later this directory can be used as an Ivy repository without having to discover + * modules & plugins each time the build is run. + */ + private fun createIvyXmls(platformPath: Path) { + val isIde = platformPath.listDirectoryEntries().map { it.name }.containsAll( + listOf("bin", "lib", "plugins", "product-info.json") + ) + if (!isIde) { + log.info("The directory '$platformPath' is not an IDE, Ivy repository is not needed there.") + return + } + + log.info("Creating an Ivy repository in '$platformPath'.") + + val writtenIvyModules = HashSet() + val productInfo = platformPath.productInfo() + val version = productInfo.buildNumber + + val pluginManager = IdePluginManager.createManager(createTempDirectory()) + val plugins = platformPath.getBundledPlugins(pluginManager) + + for (plugin in plugins.values) { + val pluginId = plugin.pluginId + val pluginVersion = plugin.pluginVersion + val pluginPath = plugin.originalFile + if (null == pluginId || null == pluginVersion || null == pluginPath) { + continue + } + + val group = Constants.Configurations.Dependencies.BUNDLED_PLUGIN_GROUP + + IntelliJPlatformDependenciesHelper.writeIvyModule( + group, pluginId, version, writtenIvyModules, providerFactory, platformPath + ) { + IvyModule( + info = IvyModule.Info(group, pluginId, pluginVersion), + publications = pluginPath.toBundledIvyArtifacts(platformPath), + dependencies = plugin.collectBundledPluginDependencies( + emptyList(), + productInfo, + platformPath, + plugins, + writtenIvyModules, + providerFactory, + platformPath + ), + ) + } + } + + log.info("Creation of an Ivy repository in '$platformPath' finished.") + } + private fun dmgTree(path: Path): FileTree { log.info("Extracting DMG archive '$path' to temporary directory.") @@ -153,7 +221,7 @@ abstract class ExtractorTransformer @Inject constructor( // such as `.background` meta-directory we have to exclude. it.file.run { (name == "Applications" && Files.isSymbolicLink(toPath())) - || it.relativePath.startsWith('.') + || it.relativePath.startsWith('.') } } } @@ -167,8 +235,7 @@ abstract class ExtractorTransformer @Inject constructor( intellijPlatformTestClasspath: Configuration, ) { Attributes.ArtifactType.Archives.forEach { - dependencies.artifactTypes.maybeCreate(it.toString()) - .attributes.attribute(Attributes.extracted, false) + dependencies.artifactTypes.maybeCreate(it.toString()).attributes.attribute(Attributes.extracted, false) } listOf(compileClasspathConfiguration, testCompileClasspathConfiguration, intellijPlatformTestClasspath).forEach { diff --git a/src/main/kotlin/org/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformDependenciesHelper.kt b/src/main/kotlin/org/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformDependenciesHelper.kt index 12a0a8173e..0ec2005a69 100644 --- a/src/main/kotlin/org/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformDependenciesHelper.kt +++ b/src/main/kotlin/org/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformDependenciesHelper.kt @@ -65,7 +65,6 @@ class IntelliJPlatformDependenciesHelper( private val rootProjectDirectory: Path, ) { - private val log = Logger(javaClass) private val pluginManager = IdePluginManager.createManager() private val writtenIvyModules = mutableSetOf() @@ -130,17 +129,7 @@ class IntelliJPlatformDependenciesHelper( get() = platformPath.map { it.productInfo() } private val bundledPlugins by lazy { - platformPath.get() - .resolve("plugins") - .listDirectoryEntries() - .filter { it.isDirectory() } - .mapNotNull { path -> - // TODO: try not to parse all plugins at once - pluginManager.safelyCreatePlugin(path) - .onFailure { log.warn(it.message.orEmpty()) } - .getOrNull() - } - .associateBy { requireNotNull(it.pluginId) } + platformPath.get().getBundledPlugins(pluginManager) } // @@ -287,6 +276,11 @@ class IntelliJPlatformDependenciesHelper( .apply(action) }) + /** Allows to check if local IntelliJ platform installation is being used. */ + internal fun isIntelliJPlatformLocal(): Boolean { + return configurations[Configurations.INTELLIJ_PLATFORM_LOCAL].dependencies.isNotEmpty() + } + /** * A base method for adding a dependency on a plugin for IntelliJ Platform. * @@ -319,10 +313,12 @@ class IntelliJPlatformDependenciesHelper( configurationName: String = Configurations.INTELLIJ_PLATFORM_BUNDLED_PLUGINS, action: DependencyAction = {}, ) = configurations[configurationName].dependencies.addAllLater(cachedListProvider { - val bundledPlugins = bundledPluginsProvider.orNull - requireNotNull(bundledPlugins) { "The `intellijPlatform.bundledPlugins` dependency helper was called with no `bundledPlugins` value provided." } + val bundledPluginsList = bundledPluginsProvider.orNull + requireNotNull(bundledPluginsList) { "The `intellijPlatform.bundledPlugins` dependency helper was called with no `bundledPlugins` value provided." } + + this.registerIntellijPlatformIvyRepo() - bundledPlugins + bundledPluginsList .map(String::trim) .filter(String::isNotEmpty) .map { dependencies.createIntelliJPlatformBundledPlugin(it) } @@ -344,6 +340,8 @@ class IntelliJPlatformDependenciesHelper( val bundledModules = bundledModulesProvider.orNull requireNotNull(bundledModules) { "The `intellijPlatform.bundledModules` dependency helper was called with no `bundledModules` value provided." } + this.registerIntellijPlatformIvyRepo() + bundledModules .map(String::trim) .filter(String::isNotEmpty) @@ -351,6 +349,60 @@ class IntelliJPlatformDependenciesHelper( .onEach(action) }) + /** + * Registers an Ivy repository containing an IDE's bundled plugins & modules. + * It has to be done like this, because if that repository is not local i.e. + * [org.jetbrains.intellij.platform.gradle.extensions.IntelliJPlatformDependenciesHelper.createIntelliJPlatformLocal] + * we do not know its location until + * [org.jetbrains.intellij.platform.gradle.artifacts.transform.ExtractorTransformer] + * has finished. For the local repository we do the same just to keep it simple. + */ + private fun registerIntellijPlatformIvyRepo() { + // We can not use IntelliJPlatformRepositoriesHelper.createExclusiveIvyRepository here because it would try + // to change already declared repositories and if any of them have been already "used" by something, it would + // throw an exception. See: + // org.gradle.api.internal.artifacts.repositories.DefaultRepositoryContentDescriptor.assertMutable + repositories.forEach { + try { + it.content { + // For performance reasons exclude the group from already added repos, since we do not expect it to + // exist in any public repositories. + // The ones declared after, should not matter, as long as the artifact is found in this repo, + // because Gradle checks repos in their declaration order. + // Tests on an env with removed caches show that this is actually necessary to prevent extra requests. + excludeGroupAndSubgroups(Dependencies.BUNDLED_MODULE_GROUP) + excludeGroupAndSubgroups(Dependencies.BUNDLED_PLUGIN_GROUP) + } + } catch (e: Exception) { + // Ignore, do not care. + } + } + + val ivyLocation = if (isIntelliJPlatformLocal()) { + // For local IntelliJ platform we can not use pre-create Ivy XML files (see ExtractorTransformer) because it + // may be located in location like "/opt/ideaIU" were will not have file system permissions to write. + // Because of that we use project's directory. + rootProjectDirectory + } else { + // If not in local mode, then we are using IntelliJ platform processed by the ExtractorTransformer, where + // Ivy XML files already exist, and we should add it as a local repository on the fly. This improves + // performance because we do not have to locate plugins & create those files, since it is done only once by + // the ExtractorTransformer. + platformPath.get() + } + + // For performance reasons make it exclusive. + IntelliJPlatformRepositoriesHelper.createIvyArtifactRepository( + "Ivy IntelliJ Platform Artifacts Repository", + repositories, + providers.localPlatformArtifactsPath(ivyLocation), + platformPath.get() + ).content { + includeGroupAndSubgroups(Dependencies.BUNDLED_MODULE_GROUP) + includeGroupAndSubgroups(Dependencies.BUNDLED_PLUGIN_GROUP) + } + } + /** * A base method for adding a dependency on a local plugin for IntelliJ Platform. * @@ -700,7 +752,7 @@ class IntelliJPlatformDependenciesHelper( val type = localProductInfo.productCode.toIntelliJPlatformType() val version = localProductInfo.buildNumber - writeIvyModule(Dependencies.LOCAL_IDE_GROUP, type.code, version) { + writeIvyModule(Dependencies.LOCAL_IDE_GROUP, type.code, version, writtenIvyModules, providers, rootProjectDirectory) { IvyModule( info = IvyModule.Info( organisation = Dependencies.LOCAL_IDE_GROUP, @@ -742,38 +794,55 @@ class IntelliJPlatformDependenciesHelper( * @param id The ID of the bundled plugin. */ private fun DependencyHandler.createIntelliJPlatformBundledPlugin(id: String): Dependency { - val plugin = bundledPlugins[id] - - requireNotNull(plugin) { - val unresolvedPluginId = when (id) { - "copyright" -> "Use correct plugin ID 'com.intellij.copyright' instead of 'copyright'." - "css", "css-impl" -> "Use correct plugin ID 'com.intellij.css' instead of 'css'/'css-impl'." - "DatabaseTools" -> "Use correct plugin ID 'com.intellij.database' instead of 'DatabaseTools'." - "Groovy" -> "Use correct plugin ID 'org.intellij.groovy' instead of 'Groovy'." - "gradle" -> "Use correct plugin ID 'com.intellij.gradle' instead of 'gradle'." - "java" -> "Use correct plugin ID 'com.intellij.java' instead of 'java'." - "Kotlin" -> "Use correct plugin ID 'org.jetbrains.kotlin' instead of 'Kotlin'." - "markdown" -> "Use correct plugin ID 'org.intellij.plugins.markdown' instead of 'markdown'." - "maven" -> "Use correct plugin ID 'org.jetbrains.idea.maven' instead of 'maven'." - "yaml" -> "Use correct plugin ID 'org.jetbrains.plugins.yaml' instead of 'yaml'." - else -> "Could not find bundled plugin with ID: '$id'." + // We can not use baseVersion for creating dependencies, because it may be something like "242-EAP-SNAPSHOT", + // which is a value provided by the user in the build.gradle file. + // But the IDE is extracted by the Extractor transformer and Ivy XML files are created in there as well, + // and that class does not know if the user is going to use an alias like "242-EAP-SNAPSHOT" or a build number, + // so we always use the build number. + //val version = baseVersion.orElse(productInfo.map { it.version }).get() + val version = productInfo.get().buildNumber + + // For local IntelliJ platform we can not use pre-create Ivy XML files (see ExtractorTransformer) because it + // may be located in location like "/opt/ideaIU" were will not have file system permissions to write. + if (isIntelliJPlatformLocal()) { + val plugin = bundledPlugins[id] + + requireNotNull(plugin) { + val unresolvedPluginId = when (id) { + "copyright" -> "Use correct plugin ID 'com.intellij.copyright' instead of 'copyright'." + "css", "css-impl" -> "Use correct plugin ID 'com.intellij.css' instead of 'css'/'css-impl'." + "DatabaseTools" -> "Use correct plugin ID 'com.intellij.database' instead of 'DatabaseTools'." + "Groovy" -> "Use correct plugin ID 'org.intellij.groovy' instead of 'Groovy'." + "gradle" -> "Use correct plugin ID 'com.intellij.gradle' instead of 'gradle'." + "java" -> "Use correct plugin ID 'com.intellij.java' instead of 'java'." + "Kotlin" -> "Use correct plugin ID 'org.jetbrains.kotlin' instead of 'Kotlin'." + "markdown" -> "Use correct plugin ID 'org.intellij.plugins.markdown' instead of 'markdown'." + "maven" -> "Use correct plugin ID 'org.jetbrains.idea.maven' instead of 'maven'." + "yaml" -> "Use correct plugin ID 'org.jetbrains.plugins.yaml' instead of 'yaml'." + else -> "Could not find bundled plugin with ID: '$id'." + } + "$unresolvedPluginId See https://jb.gg/ij-plugin-dependencies." } - "$unresolvedPluginId See https://jb.gg/ij-plugin-dependencies." - } - - val artifactPath = requireNotNull(plugin.originalFile) - val version = baseVersion.orElse(productInfo.map { it.version }).get() - writeIvyModule(Dependencies.BUNDLED_PLUGIN_GROUP, id, version) { - IvyModule( - info = IvyModule.Info( - organisation = Dependencies.BUNDLED_PLUGIN_GROUP, - module = id, - revision = version, - ), - publications = artifactPath.toBundledPluginIvyArtifacts(), - dependencies = plugin.collectBundledPluginDependencies(), - ) + val artifactPath = requireNotNull(plugin.originalFile) + writeIvyModule( + Dependencies.BUNDLED_PLUGIN_GROUP, id, version, + writtenIvyModules, providers, rootProjectDirectory + ) { + IvyModule( + info = IvyModule.Info( + organisation = Dependencies.BUNDLED_PLUGIN_GROUP, + module = id, + revision = version, + ), + publications = artifactPath.toBundledIvyArtifacts(platformPath.get()), + dependencies = plugin.collectBundledPluginDependencies( + emptyList(), + productInfo.get(), platformPath.get(), bundledPlugins, + writtenIvyModules, providers, rootProjectDirectory, + ), + ) + } } return create( @@ -789,24 +858,37 @@ class IntelliJPlatformDependenciesHelper( * @param id The ID of the bundled module. */ private fun DependencyHandler.createIntelliJPlatformBundledModule(id: String): Dependency { - val bundledModule = productInfo.get().layout - .find { layout -> layout.name == id } - .let { requireNotNull(it) { "Specified bundledModule '$id' doesn't exist." } } - val platformPath = platformPath.get() - val artifactPaths = bundledModule.classPath.flatMap { path -> - platformPath.resolve(path).toBundledModuleIvyArtifacts() - } - val version = baseVersion.orElse(productInfo.map { it.version }).get() + // We can not use baseVersion for creating dependencies, because it may be something like "242-EAP-SNAPSHOT", + // which is a value provided by the user in the build.gradle file. + // But the IDE is extracted by the Extractor transformer and Ivy XML files are created in there as well, + // and that class does not know if the user is going to use an alias like "242-EAP-SNAPSHOT" or a build number, + // so we always use the build number. + //val version = baseVersion.orElse(productInfo.map { it.version }).get() + val version = productInfo.get().buildNumber + + // For local IntelliJ platform we can not use pre-create Ivy XML files (see ExtractorTransformer) because it + // may be located in location like "/opt/ideaIU" were will not have file system permissions to write. + if (isIntelliJPlatformLocal()) { + val bundledModule = productInfo.get().layout + .find { layout -> layout.name == id } + .let { requireNotNull(it) { "Specified bundledModule '$id' doesn't exist." } } + val platformPath = platformPath.get() + val artifactPaths = bundledModule.classPath.flatMap { path -> + platformPath.resolve(path).toBundledIvyArtifacts(platformPath) + } - writeIvyModule(Dependencies.BUNDLED_MODULE_GROUP, id, version) { - IvyModule( - info = IvyModule.Info( - organisation = Dependencies.BUNDLED_MODULE_GROUP, - module = id, - revision = version, - ), - publications = artifactPaths, - ) + writeIvyModule( + Dependencies.BUNDLED_MODULE_GROUP, id, version, writtenIvyModules, providers, rootProjectDirectory + ) { + IvyModule( + info = IvyModule.Info( + organisation = Dependencies.BUNDLED_MODULE_GROUP, + module = id, + revision = version, + ), + publications = artifactPaths, + ) + } } return create( @@ -816,67 +898,140 @@ class IntelliJPlatformDependenciesHelper( ) } - /** - * Collects all dependencies on plugins or modules of the current [IdePlugin]. - * The [path] parameter is a list of already traversed entities, used to avoid circular dependencies when walking recursively. - * - * @param path IDs of already traversed plugins or modules. - */ - private fun IdePlugin.collectBundledPluginDependencies(path: List = emptyList()): List { - val id = requireNotNull(pluginId) - val dependencyIds = (dependencies.map { it.id } + optionalDescriptors.map { it.dependency.id } + modulesDescriptors.map { it.name } - id).toSet() - val buildNumber by lazy { productInfo.get().buildNumber } - val platformPath by lazy { platformPath.get() } - - val plugins = dependencyIds - .mapNotNull { bundledPlugins[it] } - .map { plugin -> - val artifactPath = requireNotNull(plugin.originalFile) - val group = Dependencies.BUNDLED_PLUGIN_GROUP - val name = requireNotNull(plugin.pluginId) - val version = requireNotNull(plugin.pluginVersion) - - writeIvyModule(group, name, version) { - IvyModule( - info = IvyModule.Info(group, name, version), - publications = artifactPath.toBundledPluginIvyArtifacts(), - dependencies = when { - id in path -> emptyList() - else -> plugin.collectBundledPluginDependencies(path + id) - }, - ) + companion object { + private val log = Logger(IntelliJPlatformDependenciesHelper::class.java) + + internal fun Path.getBundledPlugins(pluginManager: IdePluginManager): Map { + return this.resolve("plugins") + .listDirectoryEntries() + .filter { it.isDirectory() } + .mapNotNull { path -> + // TODO: try not to parse all plugins at once + pluginManager.safelyCreatePlugin(path) + .onFailure { log.warn(it.message.orEmpty()) } + .getOrNull() } + .associateBy { requireNotNull(it.pluginId) } + } - IvyModule.Dependency(group, name, version) - } + /** + * Collects all dependencies on plugins or modules of the current [IdePlugin]. + * The [path] parameter is a list of already traversed entities, used to avoid circular dependencies when walking recursively. + * + * @param path IDs of already traversed plugins or modules. + */ + internal fun IdePlugin.collectBundledPluginDependencies( + path: List = emptyList(), + + productInfo: ProductInfo, + platformPath: Path, + bundledPlugins: Map, + + writtenIvyModules: MutableSet, + providers: ProviderFactory, + pathToWriteInto: Path + ): List { + val id = requireNotNull(pluginId) + val dependencyIds = ( + dependencies.map { it.id } + + optionalDescriptors.map { it.dependency.id } + + modulesDescriptors.map { it.name } - id + ).toSet() + val buildNumber= productInfo.buildNumber + + val plugins = dependencyIds + .mapNotNull { bundledPlugins[it] } + .map { plugin -> + val artifactPath = requireNotNull(plugin.originalFile) + val group = Dependencies.BUNDLED_PLUGIN_GROUP + val name = requireNotNull(plugin.pluginId) + val version = requireNotNull(plugin.pluginVersion) + + writeIvyModule(group, name, version, writtenIvyModules, providers, pathToWriteInto) { + IvyModule( + info = IvyModule.Info(group, name, version), + publications = artifactPath.toBundledIvyArtifacts(platformPath), + dependencies = when { + id in path -> emptyList() + else -> plugin.collectBundledPluginDependencies( + path + id, + productInfo, platformPath, bundledPlugins, + writtenIvyModules, providers, pathToWriteInto, + ) + }, + ) + } - val layoutItems = productInfo.get().layout - .filter { layout -> layout.name in dependencyIds } - .filter { layout -> layout.classPath.isNotEmpty() } - - val modules = dependencyIds - .filterNot { bundledPlugins.containsKey(it) } - .mapNotNull { layoutItems.find { layout -> layout.name == it } } - .filterNot { it.classPath.isEmpty() } - .map { - val artifactPaths = it.classPath.flatMap { path -> - platformPath.resolve(path).toBundledModuleIvyArtifacts() + IvyModule.Dependency(group, name, version) } - val group = Dependencies.BUNDLED_MODULE_GROUP - val name = it.name - val version = buildNumber - - writeIvyModule(group, name, version) { - IvyModule( - info = IvyModule.Info(group, name, version), - publications = artifactPaths, - ) + + val layoutItems = productInfo.layout + .filter { layout -> layout.name in dependencyIds } + .filter { layout -> layout.classPath.isNotEmpty() } + + val modules = dependencyIds + .filterNot { bundledPlugins.containsKey(it) } + .mapNotNull { layoutItems.find { layout -> layout.name == it } } + .filterNot { it.classPath.isEmpty() } + .map { + val artifactPaths = it.classPath.flatMap { path -> + platformPath.resolve(path).toBundledIvyArtifacts(platformPath) + } + val group = Dependencies.BUNDLED_MODULE_GROUP + val name = it.name + val version = buildNumber + + writeIvyModule(group, name, version, writtenIvyModules, providers, pathToWriteInto) { + IvyModule( + info = IvyModule.Info(group, name, version), + publications = artifactPaths, + ) + } + + IvyModule.Dependency(group, name, version) } - IvyModule.Dependency(group, name, version) + return plugins + modules + } + + /** + * Creates and writes the Ivy module file for the specified group, artifact, and version, if absent. + * + * @param group The group identifier for the Ivy module. + * @param artifact The artifact name for the Ivy module. + * @param version The version of the Ivy module. + * @param block A lambda that returns an instance of IvyModule to be serialized into the file. + */ + internal fun writeIvyModule( + group: String, + artifact: String, + version: String, + + writtenIvyModules: MutableSet, + providers: ProviderFactory, + pathToWriteInto: Path, + + block: () -> IvyModule + ) = apply { + val fileName = "$group-$artifact-$version.xml" + if (writtenIvyModules.contains(fileName)) { + return@apply } - return plugins + modules + val ivyFile = providers + .localPlatformArtifactsPath(pathToWriteInto) + .resolve(fileName) + + ivyFile + .apply { parent.createDirectories() } + .apply { deleteIfExists() } + .createFile() + .writeText(XML { + indentString = " " + }.encodeToString(block())) + + writtenIvyModules.add(fileName) + } } /** @@ -903,7 +1058,7 @@ class IntelliJPlatformDependenciesHelper( val version = plugin.pluginVersion ?: "0.0.0" val name = plugin.pluginId ?: artifactPath.name - writeIvyModule(Dependencies.LOCAL_PLUGIN_GROUP, name, version) { + writeIvyModule(Dependencies.LOCAL_PLUGIN_GROUP, name, version, writtenIvyModules, providers, rootProjectDirectory) { IvyModule( info = IvyModule.Info( organisation = Dependencies.LOCAL_PLUGIN_GROUP, @@ -944,7 +1099,7 @@ class IntelliJPlatformDependenciesHelper( val name = javaVendorVersion.substringBefore(javaVersion).trim('-') val version = javaVendorVersion.removePrefix(name).trim('-') - writeIvyModule(Dependencies.LOCAL_JETBRAINS_RUNTIME_GROUP, name, version) { + writeIvyModule(Dependencies.LOCAL_JETBRAINS_RUNTIME_GROUP, name, version, writtenIvyModules, providers, rootProjectDirectory) { IvyModule( info = IvyModule.Info( organisation = Dependencies.LOCAL_JETBRAINS_RUNTIME_GROUP, @@ -1027,35 +1182,6 @@ class IntelliJPlatformDependenciesHelper( } } - /** - * Creates and writes the Ivy module file for the specified group, artifact, and version, if absent. - * - * @param group The group identifier for the Ivy module. - * @param artifact The artifact name for the Ivy module. - * @param version The version of the Ivy module. - * @param block A lambda that returns an instance of IvyModule to be serialized into the file. - */ - private fun writeIvyModule(group: String, artifact: String, version: String, block: () -> IvyModule) = apply { - val fileName = "$group-$artifact-$version.xml" - if (writtenIvyModules.contains(fileName)) { - return@apply - } - - val ivyFile = providers - .localPlatformArtifactsPath(rootProjectDirectory) - .resolve(fileName) - - ivyFile - .apply { parent.createDirectories() } - .apply { deleteIfExists() } - .createFile() - .writeText(XML { - indentString = " " - }.encodeToString(block())) - - writtenIvyModules.add(fileName) - } - /** * Creates an IntelliJ Platform dependency and excludes transitive dependencies provided by the current IntelliJ Platform. * diff --git a/src/main/kotlin/org/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformRepositoriesHelper.kt b/src/main/kotlin/org/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformRepositoriesHelper.kt index 0ec800df7c..0564cd41d2 100644 --- a/src/main/kotlin/org/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformRepositoriesHelper.kt +++ b/src/main/kotlin/org/jetbrains/intellij/platform/gradle/extensions/IntelliJPlatformRepositoriesHelper.kt @@ -14,7 +14,7 @@ import org.gradle.api.provider.ProviderFactory import org.gradle.authentication.http.HttpHeaderAuthentication import org.gradle.internal.os.OperatingSystem import org.gradle.kotlin.dsl.* -import org.jetbrains.intellij.platform.gradle.Constants.Configurations +import org.jetbrains.intellij.platform.gradle.Constants.Configurations.Dependencies import org.jetbrains.intellij.platform.gradle.CustomPluginRepositoryType import org.jetbrains.intellij.platform.gradle.GradleProperties import org.jetbrains.intellij.platform.gradle.artifacts.repositories.PluginArtifactRepository @@ -144,36 +144,89 @@ class IntelliJPlatformRepositoriesHelper( } internal fun createLocalIvyRepository(repositoryName: String, action: IvyRepositoryAction = {}): IvyArtifactRepository { - repositories.forEach { - it.content { - // For performance reasons exclude the group from already added repos, since we do not expect it to - // exist in any public repositories. - // The ones declared after, should not matter, as long as the artifact is found in this repo, - // because Gradle checks repos in their declaration order. - // Tests on an env with removed caches show that this is actually necessary to prevent extra requests. - excludeGroupAndSubgroups(Configurations.Dependencies.JB_LOCAL_PREFIX) + val ivyLocationPath = providers.localPlatformArtifactsPath(rootProjectDirectory) + + return createExclusiveIvyRepository( + repositoryName, + setOf( + Dependencies.LOCAL_IDE_GROUP, + Dependencies.LOCAL_PLUGIN_GROUP, + Dependencies.LOCAL_JETBRAINS_RUNTIME_GROUP + ), + repositories, + ivyLocationPath, + // We can not know it, since this repo is used to store all kinds of things located in unrelated paths. + artifactLocationPath = null, + action + ) + } + + companion object { + internal fun createExclusiveIvyRepository( + repositoryName: String, + exclusiveGroups: Set = emptySet(), + repositories: RepositoryHandler, + ivyLocationPath: Path, + artifactLocationPath: Path?, + action: IvyRepositoryAction = {}, + ): IvyArtifactRepository { + val repository = createIvyArtifactRepository( + repositoryName, repositories, ivyLocationPath, artifactLocationPath + ) + + // For performance reasons make it exclusive. + // https://docs.gradle.org/current/userguide/declaring_repositories_adv.html#declaring_content_exclusively_found_in_one_repository + repositories.exclusiveContent { + forRepository { + repository + } + + filter { + for (exclusiveGroup in exclusiveGroups) { + includeGroupAndSubgroups(exclusiveGroup) + } + } } + + repository.apply { + action() + } + + return repository } - return repositories.ivy { + /** + * @see org.jetbrains.intellij.platform.gradle.models.IvyModuleKt.toIvyArtifact + * @see org.jetbrains.intellij.platform.gradle.models.IvyModuleKt.toBundledIvyArtifacts + * @see org.jetbrains.intellij.platform.gradle.models.IvyModuleKt.toLocalPluginIvyArtifacts + * @see org.jetbrains.intellij.platform.gradle.extensions.IntelliJPlatformDependenciesHelper.registerIntellijPlatformIvyRepo + */ + internal fun createIvyArtifactRepository( + repositoryName: String, + repositories: RepositoryHandler, + ivyLocationPath: Path, + artifactLocationPath: Path? + ) = repositories.ivy { name = repositoryName // Location of Ivy files generated for the current project. - val localPlatformArtifactsPath = providers.localPlatformArtifactsPath(rootProjectDirectory) - ivyPattern("${localPlatformArtifactsPath.pathString}/[organization]-[module]-[revision].[ext]") - - // "type" may contain an optional absolute path to the actual location of the artifact, in different situations - // it may point to a completely unrelated locations on the file system. - // Yes, attribute named types does not fit very good for storing a path, but there is no better alternative. - // "artifact" maps to IvyModule#name here we expect to have the second part of the path. - // "ext" is file extension, e.g. "jar" or "directory", not used here. - // - // The type can be empty fur the "artifact" (IvyModule#name) can not. - // - // It has to be prefixed by "/" because: If this pattern is not a fully-qualified URL, it will be interpreted - // as a file relative to the project directory. - val pattern = "/([type])[artifact]" - artifactPattern(pattern) + ivyPattern("${ivyLocationPath.pathString}/[organization]-[module]-[revision].[ext]") + + // "type" may contain an optional absolute or relative path to the actual location of the artifact, + // in different situations it may point to a completely unrelated locations on the file system. + // Yes, attribute named types does not fit very good for storing a path, but there is no better alternative. + // + // "artifact" maps to IvyModule#name here we expect to have the second part of the path, which is file name. + // If "type" is empty, then name may contain both the path and file name. + // + // "ext" is file extension, e.g. "jar" or "directory", not used here. + // + // The "type" can be empty but the "artifact" (IvyModule#name) can not. + // + // It has to be prefixed by "/" because if this pattern is not a fully-qualified URL, it will be interpreted + // as a file relative to the project directory. + val pattern = "${artifactLocationPath ?: ""}/([type])[artifact]" + artifactPattern(pattern) /** * Because artifact paths always start with `/` (see [toPublication] for details), @@ -182,17 +235,8 @@ class IntelliJPlatformRepositoriesHelper( * starting with `c` for the sake of micro-optimization. */ if (OperatingSystem.current().isWindows) { - (('c'..'z') + 'a' + 'b').forEach { artifactPattern("$it:$pattern") } - } - }.apply { - content { - includeGroup(Configurations.Dependencies.BUNDLED_MODULE_GROUP) - includeGroup(Configurations.Dependencies.BUNDLED_PLUGIN_GROUP) - includeGroup(Configurations.Dependencies.LOCAL_IDE_GROUP) - includeGroup(Configurations.Dependencies.LOCAL_PLUGIN_GROUP) - includeGroup(Configurations.Dependencies.LOCAL_JETBRAINS_RUNTIME_GROUP) + (('c'..'z') + 'a' + 'b').forEach { artifactPattern("$it:$pattern") } } - action() } } } diff --git a/src/main/kotlin/org/jetbrains/intellij/platform/gradle/models/IvyModule.kt b/src/main/kotlin/org/jetbrains/intellij/platform/gradle/models/IvyModule.kt index f41a09871e..1ad25c9f51 100644 --- a/src/main/kotlin/org/jetbrains/intellij/platform/gradle/models/IvyModule.kt +++ b/src/main/kotlin/org/jetbrains/intellij/platform/gradle/models/IvyModule.kt @@ -74,7 +74,7 @@ data class IvyModule( * * ... * - * + * * * * ``` @@ -83,7 +83,7 @@ data class IvyModule( * * ... * - * + * * * * ``` @@ -99,10 +99,14 @@ data class IvyModule( * but that's not a huge problem. * * @see IntelliJPlatformRepositoriesExtension.jetbrainsIdeInstallers + * @see org.jetbrains.intellij.platform.gradle.extensions.IntelliJPlatformRepositoriesHelper.createIvyArtifactRepository */ internal fun Path.toIvyArtifact(): IvyModule.Artifact { val alwaysEmpty = "" - val trimmedAbsolutePath = this.removeLeadingPathSeparator() + + // We need to remove the leading separator because the artifact pattern in the Ivy repo already has it. + val trimmedAbsolutePath = this.normalizeWindowsPath().removePrefix("/") + val ext = when { this.isDirectory() -> "directory" else -> this.extension @@ -112,46 +116,51 @@ internal fun Path.toIvyArtifact(): IvyModule.Artifact { // IntelliJPlatformRepositoriesHelper.createLocalIvyRepository the pattern is "/([type])[artifact]" where the type // is marked as optional. // In this case, it should be ok to have an absolute path in the name, because this is a fallback method, - // which called for directories only (at least at the moment of writing this). + // which called for directories only (at least at the moment of writing this), where it is not clear how to split + // it into two pieces: path and artifact. // See comments in explodeIntoIvyJarsArtifacts on why having abs path in name may be bad. return IvyModule.Artifact(type = alwaysEmpty, name = trimmedAbsolutePath, ext = ext) } -internal fun Path.toBundledPluginIvyArtifacts(): List { - return this.explodeIntoIvyJarsArtifacts() -} - -internal fun Path.toBundledModuleIvyArtifacts(): List { - return this.explodeIntoIvyJarsArtifacts() +internal fun Path.toBundledIvyArtifacts(basePath: Path): List { + return this.explodeIntoIvyJarsArtifacts(basePath) } internal fun Path.toLocalPluginIvyArtifacts(): List { - return this.explodeIntoIvyJarsArtifacts() + // For local plugins we do not use relative paths. + return this.explodeIntoIvyJarsArtifacts(null) } -/** This method is a bit too universal. It could be split into 3 separate with unnecessary logic removed. */ -private fun Path.explodeIntoIvyJarsArtifacts(): List { +/** + * @see org.jetbrains.intellij.platform.gradle.extensions.IntelliJPlatformRepositoriesHelper.createIvyArtifactRepository + */ +private fun Path.explodeIntoIvyJarsArtifacts(basePath: Path? = null): List { if (this.isDirectory()) { val jars: List = CollectorTransformer.collectJars(this) return jars.map { it: Path -> - val containingDirTrimmedAbsPath = it.containingDirTrimmedAbsPath() + // If basePath is not null, we will make the paths relative to it. + val artifactPath = it.containingDirTrimmedPath(basePath) + val fileNameWithExt = it.fileName.toString() val ext = it.extension // E.g. - // - // The reason why we put tha absolute path into type is that the name should not have it, because artifact name - // may come up in files like Gradle's verification-metadata.xml - // https://docs.gradle.org/current/userguide/dependency_verification.html - // Which will make them not portable between different environments. - IvyModule.Artifact(type = containingDirTrimmedAbsPath, name = fileNameWithExt, ext = ext) + // + // + // The reason why we put the path into type is that the name should not have it, because: + // - artifact name may come up in files like Gradle's verification-metadata.xml which will make them not + // portable between different environments. + // https://github.com/JetBrains/intellij-platform-gradle-plugin/issues/1778 + // https://github.com/JetBrains/intellij-platform-gradle-plugin/issues/1779 + // https://docs.gradle.org/current/userguide/dependency_verification.html + // - Name may also come up in Gradle errors, if for some reason the artifact is not resolved. In that case + // the artifact coordinates may look very weird like: + // com.jetbrains.localhost-only.bundledPlugin:/some/path/more/path/some.jar:123.456.789 + IvyModule.Artifact(type = artifactPath, name = fileNameWithExt, ext = ext) } } else { - val containingDirTrimmedAbsPath = this.containingDirTrimmedAbsPath() + val containingDirTrimmedAbsPath= this.containingDirTrimmedPath(basePath) + val fileNameWithExt = this.fileName.toString() val ext = this.extension @@ -159,18 +168,19 @@ private fun Path.explodeIntoIvyJarsArtifacts(): List { } } -private fun Path.containingDirTrimmedAbsPath(): String { - return this.removeLeadingPathSeparator().substringBeforeLast("/").suffixIfNot("/") -} - /** - * For explanation on why we need to remove the leading separator, see - * [org.jetbrains.intellij.platform.gradle.extensions.IntelliJPlatformRepositoriesHelper.createLocalIvyRepository] - * - * There the artifact pattern has a slash in the beginning, so we have to remove it here. + * @see org.jetbrains.intellij.platform.gradle.extensions.IntelliJPlatformRepositoriesHelper.createIvyArtifactRepository */ -private fun Path.removeLeadingPathSeparator(): String { - return this.normalizeWindowsPath().removePrefix("/") +private fun Path.containingDirTrimmedPath(basePath: Path? = null): String { + var pathString = this.normalizeWindowsPath() + + if (null != basePath) { + // If base path is given, we make the returned path relative to it. + pathString = pathString.removePrefix(basePath.absolutePathString()) + } + + // We need to remove the leading separator because the artifact pattern in the Ivy repo already has it. + return pathString.removePrefix("/").substringBeforeLast("/").suffixIfNot("/") } private fun Path.normalizeWindowsPath(): String {