From 82b730cca2ce20b1cdfdd88847a6dc9d9db3c227 Mon Sep 17 00:00:00 2001 From: Patrick Ziegler Date: Fri, 1 Dec 2023 23:32:08 +0100 Subject: [PATCH] Add non-IU artifact repositories to the planner resolution #792 Given that units from non-IU locations are considered when resolving IUs, it may happen that units are contained in the resolution path, which are not contained by any repository. This may break previously functional target definitions. To resolve this problem, an in-memory artifact and metadata repository is created, based on the metadata of the non-IU locations. Those repositories are then added to the provisioning context for further processing. Amends dd321b56a92b8e86f98de6f0dae25fbc43e4f825 --- .../core/target/FileArtifactRepository.java | 179 ++++++++++++++++++ .../core/target/FileMetadataRepository.java | 56 ++++++ .../internal/core/target/P2TargetUtils.java | 48 ++++- .../META-INF/MANIFEST.MF | 1 + .../tests/target/IUBundleContainerTests.java | 73 ++++++- 5 files changed, 347 insertions(+), 10 deletions(-) create mode 100644 ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/FileArtifactRepository.java create mode 100644 ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/FileMetadataRepository.java diff --git a/ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/FileArtifactRepository.java b/ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/FileArtifactRepository.java new file mode 100644 index 00000000000..d815700d2f7 --- /dev/null +++ b/ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/FileArtifactRepository.java @@ -0,0 +1,179 @@ +/******************************************************************************* + * Copyright (c) 2023 Patrick Ziegler and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Patrick Ziegler - initial API and implementation + *******************************************************************************/ +package org.eclipse.pde.internal.core.target; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import org.eclipse.core.runtime.Assert; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.IStatus; +import org.eclipse.core.runtime.MultiStatus; +import org.eclipse.core.runtime.Status; +import org.eclipse.core.runtime.SubMonitor; +import org.eclipse.equinox.frameworkadmin.BundleInfo; +import org.eclipse.equinox.internal.p2.metadata.expression.QueryResult; +import org.eclipse.equinox.p2.core.IProvisioningAgent; +import org.eclipse.equinox.p2.core.ProvisionException; +import org.eclipse.equinox.p2.metadata.IArtifactKey; +import org.eclipse.equinox.p2.publisher.eclipse.BundlesAction; +import org.eclipse.equinox.p2.publisher.eclipse.FeaturesAction; +import org.eclipse.equinox.p2.query.IQuery; +import org.eclipse.equinox.p2.query.IQueryResult; +import org.eclipse.equinox.p2.query.IQueryable; +import org.eclipse.equinox.p2.repository.artifact.IArtifactDescriptor; +import org.eclipse.equinox.p2.repository.artifact.IArtifactRequest; +import org.eclipse.equinox.p2.repository.artifact.spi.AbstractArtifactRepository; +import org.eclipse.equinox.p2.repository.artifact.spi.ArtifactDescriptor; +import org.eclipse.pde.core.target.ITargetLocation; +import org.eclipse.pde.core.target.TargetBundle; +import org.eclipse.pde.core.target.TargetFeature; +import org.eclipse.pde.internal.core.PDECore; +import org.eclipse.pde.internal.core.ifeature.IFeatureModel; + +/** + * In-Memory representation of a artifact repository based on a non-IU target + * location. This repository is used during the planner resolution of an IU + * target location to supply artifacts from other (non-IU) locations. + */ +@SuppressWarnings("restriction") +public class FileArtifactRepository extends AbstractArtifactRepository { + private static final String NAME = "Non-IU Artifact Repository"; + private static final String DESCRIPTION = """ + In-Memory repository created for a single Non-IU repository, used + during the planner resolution of a real IU repository, in order to + grant access to external, installable units. + """; + private static final String SCHEME = "memory://"; //$NON-NLS-1$ + private final Map artifacts = new HashMap<>(); + + public FileArtifactRepository(IProvisioningAgent agent, ITargetLocation targetLocation) { + super(agent, NAME, null, null, URI.create(SCHEME + UUID.randomUUID()), DESCRIPTION, null, null); + Assert.isTrue(targetLocation.isResolved()); + for (TargetBundle targetBundle : targetLocation.getBundles()) { + if (!targetBundle.getStatus().isOK()) { + PDECore.log(Status.warning("Target bundle is not resolved: " + targetBundle)); + continue; + } + BundleInfo bundleInfo = targetBundle.getBundleInfo(); + URI bundleLocation = bundleInfo.getLocation(); + if (bundleLocation == null) { + PDECore.log(Status.warning("Bundle location not found for: " + bundleInfo)); + continue; + } + File bundleFile = new File(bundleLocation); + if (!bundleFile.exists()) { + PDECore.log(Status.warning("Bundle file doesn't exist: " + bundleFile)); + continue; + } + IArtifactKey artifactKey = BundlesAction.createBundleArtifactKey(bundleInfo.getSymbolicName(), + bundleInfo.getVersion()); + IArtifactDescriptor artifactDesriptor = new ArtifactDescriptor(artifactKey); + if (artifacts.containsKey(artifactDesriptor)) { + PDECore.log(Status.warning("Artifact already exists: " + artifactDesriptor)); + continue; + } + artifacts.put(artifactDesriptor, bundleFile); + } + for (TargetFeature targetFeature : targetLocation.getFeatures()) { + String featureLocation = ((IFeatureModel) targetFeature.getFeatureModel()).getInstallLocation(); + if (featureLocation == null) { + PDECore.log(Status.warning("Feature location not found for: " + targetFeature)); + continue; + } + File featureFile = new File(featureLocation); + if (!featureFile.exists()) { + PDECore.log(Status.warning("Feature file doesn't exist: " + featureFile)); + continue; + } + IArtifactKey artifactKey = FeaturesAction.createFeatureArtifactKey(targetFeature.getId(), + targetFeature.getVersion()); + IArtifactDescriptor artifactDesriptor = new ArtifactDescriptor(artifactKey); + if (artifacts.containsKey(artifactDesriptor)) { + PDECore.log(Status.warning("Artifact already exists: " + artifactDesriptor)); + continue; + } + artifacts.put(artifactDesriptor, featureFile); + } + } + + @Override + public IStatus getRawArtifact(IArtifactDescriptor descriptor, OutputStream destination, IProgressMonitor monitor) { + File artifactFile = artifacts.get(descriptor); + if (artifactFile == null) { + return Status.error("Artifact not found: " + descriptor); + } + try (FileInputStream is = new FileInputStream(artifactFile)) { + is.transferTo(destination); + return Status.OK_STATUS; + } catch (IOException e) { + return Status.error(e.getLocalizedMessage(), e); + } + } + + @Override + public IQueryable descriptorQueryable() { + return (query, monitor) -> query.perform(artifacts.keySet().iterator()); + } + + @Override + public IQueryResult query(IQuery query, IProgressMonitor monitor) { + return new QueryResult<>(artifacts.keySet().stream().map(IArtifactDescriptor::getArtifactKey).iterator()); + } + + @Override + public boolean contains(IArtifactDescriptor descriptor) { + return artifacts.containsValue(descriptor); + } + + @Override + public boolean contains(IArtifactKey key) { + return artifacts.keySet().stream().anyMatch(descriptor -> key.equals(descriptor.getArtifactKey())); + } + + @Override + public IStatus getArtifact(IArtifactDescriptor descriptor, OutputStream destination, IProgressMonitor monitor) { + return getRawArtifact(descriptor, destination, monitor); + } + + @Override + public IArtifactDescriptor[] getArtifactDescriptors(IArtifactKey key) { + return artifacts.keySet().stream() // + .filter(descriptor -> key.equals(descriptor.getArtifactKey())) // + .toArray(IArtifactDescriptor[]::new); + } + + @Override + public IStatus getArtifacts(IArtifactRequest[] requests, IProgressMonitor monitor) { + MultiStatus multiStatus = new MultiStatus(getClass(), IStatus.INFO, ""); + SubMonitor subMonitor = SubMonitor.convert(monitor, requests.length); + for (IArtifactRequest request : requests) { + request.perform(this, subMonitor.split(1)); + multiStatus.add(request.getResult()); + } + return Status.OK_STATUS; + } + + @Override + public OutputStream getOutputStream(IArtifactDescriptor descriptor) throws ProvisionException { + throw new ProvisionException("Artifact repository must not be modified!"); + } + +} diff --git a/ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/FileMetadataRepository.java b/ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/FileMetadataRepository.java new file mode 100644 index 00000000000..c835c44503b --- /dev/null +++ b/ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/FileMetadataRepository.java @@ -0,0 +1,56 @@ +/******************************************************************************* + * Copyright (c) 2023 Patrick Ziegler and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Patrick Ziegler - initial API and implementation + *******************************************************************************/ +package org.eclipse.pde.internal.core.target; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.equinox.p2.core.IProvisioningAgent; +import org.eclipse.equinox.p2.metadata.IInstallableUnit; +import org.eclipse.equinox.p2.query.IQuery; +import org.eclipse.equinox.p2.query.IQueryResult; +import org.eclipse.equinox.p2.repository.IRepositoryReference; +import org.eclipse.equinox.p2.repository.metadata.spi.AbstractMetadataRepository; + +/** + * In-Memory representation of a metadata repository based on a non-IU target + * location. This repository is used during the planner resolution of an IU + * target location to supply the metadata from other (non-IU) locations. + */ +public class FileMetadataRepository extends AbstractMetadataRepository { + private final List installableUnits; + + public FileMetadataRepository(IProvisioningAgent agent, List installableUnits) { + super(agent); + this.installableUnits = List.copyOf(installableUnits); + } + + + @Override + public Collection getReferences() { + return Collections.emptySet(); + } + + @Override + public IQueryResult query(IQuery query, IProgressMonitor monitor) { + return query.perform(installableUnits.iterator()); + } + + @Override + public void initialize(RepositoryState state) { + // nothing to do + } +} diff --git a/ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/P2TargetUtils.java b/ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/P2TargetUtils.java index 15175d3d8ef..748bcdfabc3 100644 --- a/ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/P2TargetUtils.java +++ b/ui/org.eclipse.pde.core/src/org/eclipse/pde/internal/core/target/P2TargetUtils.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2010, 2018 EclipseSource Inc. and others. + * Copyright (c) 2010, 2023 EclipseSource Inc. and others. * * This program and the accompanying materials * are made available under the terms of the Eclipse Public License 2.0 @@ -17,6 +17,7 @@ package org.eclipse.pde.internal.core.target; import java.io.File; +import java.lang.reflect.Field; import java.net.URI; import java.nio.file.Path; import java.util.ArrayList; @@ -32,6 +33,7 @@ import java.util.Set; import java.util.WeakHashMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; import org.eclipse.core.runtime.CoreException; import org.eclipse.core.runtime.IPath; @@ -69,6 +71,7 @@ import org.eclipse.equinox.p2.metadata.Version; import org.eclipse.equinox.p2.planner.IPlanner; import org.eclipse.equinox.p2.planner.IProfileChangeRequest; +import org.eclipse.equinox.p2.query.CollectionResult; import org.eclipse.equinox.p2.query.IQuery; import org.eclipse.equinox.p2.query.IQueryResult; import org.eclipse.equinox.p2.query.IQueryable; @@ -1078,11 +1081,40 @@ private void resolveWithPlanner(ITargetDefinition target, IProgressMonitor monit request.setInstallableUnitProfileProperty(unit, PROP_INSTALLED_IU, Boolean.toString(true)); } - ProvisioningContext context = new ProvisioningContext(getAgent()); + IProvisioningAgent agent = getAgent(); + List extraInstallableUnits = new ArrayList<>(); + List extraArtifactRepositories = new ArrayList<>(); + List extraMetadataRepositories = new ArrayList<>(); + getAdditionalProvisionIUs(target, container -> { + List installableUnits = InstallableUnitGenerator // + .generateInstallableUnits(container.getBundles(), container.getFeatures()) // + .toList(); + extraArtifactRepositories.add(new FileArtifactRepository(agent, container)); + extraMetadataRepositories.add(new FileMetadataRepository(agent, installableUnits)); + extraInstallableUnits.addAll(installableUnits); + }); + ProvisioningContext context = new ProvisioningContext(agent) { + @Override + public IQueryable getArtifactRepositories(IProgressMonitor monitor) { + return (query, monitor2) -> { + List artifactRepositories = new ArrayList<>(); + try { + IQueryable oldQueryable = super.getArtifactRepositories(monitor2); + Field f = oldQueryable.getClass().getDeclaredField("repositories"); + f.setAccessible(true); + artifactRepositories.addAll((List) f.get(oldQueryable)); + } catch(Exception e) { + e.printStackTrace(); + } + artifactRepositories.addAll(extraArtifactRepositories); + return new CollectionResult(artifactRepositories); + }; + } + }; context.setProperty(ProvisioningContext.FOLLOW_REPOSITORY_REFERENCES, Boolean.toString(true)); context.setMetadataRepositories(getMetadataRepositories(target).toArray(URI[]::new)); context.setArtifactRepositories(getArtifactRepositories(target).toArray(URI[]::new)); - context.setExtraInstallableUnits(getAdditionalProvisionIUs(target)); + context.setExtraInstallableUnits(extraInstallableUnits); IProvisioningPlan plan = planner.getProvisioningPlan(request, context, subMonitor.split(20)); IStatus status = plan.getStatus(); @@ -1553,8 +1585,8 @@ private Collection getMetadataRepositories(ITargetDefinition target) throws return result; } - private List getAdditionalProvisionIUs(ITargetDefinition target) throws CoreException { - List result = new ArrayList<>(); + private void getAdditionalProvisionIUs(ITargetDefinition target, Consumer callback) + throws CoreException { ITargetLocation[] containers = target.getTargetLocations(); if (containers != null) { for (ITargetLocation container : containers) { @@ -1564,17 +1596,15 @@ private List getAdditionalProvisionIUs(ITargetDefinition targe } if (container instanceof TargetReferenceBundleContainer targetRefContainer) { ITargetDefinition referencedTargetDefinition = targetRefContainer.getTargetDefinition(); - result.addAll(getAdditionalProvisionIUs(referencedTargetDefinition)); + getAdditionalProvisionIUs(referencedTargetDefinition, callback); continue; } if (!container.isResolved()) { container.resolve(target, new NullProgressMonitor()); } - InstallableUnitGenerator.generateInstallableUnits(container.getBundles(), container.getFeatures()) - .forEach(result::add); + callback.accept(container); } } - return result; } private static final String NATIVE_ARTIFACTS = "nativeArtifacts"; //$NON-NLS-1$ diff --git a/ui/org.eclipse.pde.ui.tests/META-INF/MANIFEST.MF b/ui/org.eclipse.pde.ui.tests/META-INF/MANIFEST.MF index cdb72b61dfc..9955a310935 100644 --- a/ui/org.eclipse.pde.ui.tests/META-INF/MANIFEST.MF +++ b/ui/org.eclipse.pde.ui.tests/META-INF/MANIFEST.MF @@ -30,6 +30,7 @@ Require-Bundle: org.eclipse.pde.ui, org.eclipse.equinox.p2.repository, org.eclipse.equinox.p2.metadata, org.eclipse.equinox.p2.engine, + org.eclipse.equinox.p2.publisher.eclipse, org.eclipse.ui.forms, org.eclipse.ui.workbench.texteditor, org.eclipse.jface.text, diff --git a/ui/org.eclipse.pde.ui.tests/src/org/eclipse/pde/ui/tests/target/IUBundleContainerTests.java b/ui/org.eclipse.pde.ui.tests/src/org/eclipse/pde/ui/tests/target/IUBundleContainerTests.java index c97b2e16727..6f1274572c7 100644 --- a/ui/org.eclipse.pde.ui.tests/src/org/eclipse/pde/ui/tests/target/IUBundleContainerTests.java +++ b/ui/org.eclipse.pde.ui.tests/src/org/eclipse/pde/ui/tests/target/IUBundleContainerTests.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2009, 2018 IBM Corporation and others. + * Copyright (c) 2009, 2023 IBM Corporation and others. * * This program and the accompanying materials * are made available under the terms of the Eclipse Public License 2.0 @@ -17,16 +17,22 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.jar.JarFile; import javax.xml.parsers.DocumentBuilder; @@ -37,8 +43,11 @@ import org.eclipse.core.runtime.IStatus; import org.eclipse.equinox.frameworkadmin.BundleInfo; import org.eclipse.equinox.p2.metadata.IInstallableUnit; +import org.eclipse.equinox.p2.publisher.eclipse.BundlesAction; +import org.eclipse.equinox.p2.publisher.eclipse.FeaturesAction; import org.eclipse.equinox.p2.query.IQueryResult; import org.eclipse.equinox.p2.query.QueryUtil; +import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository; import org.eclipse.equinox.p2.repository.metadata.IMetadataRepository; import org.eclipse.equinox.p2.repository.metadata.IMetadataRepositoryManager; import org.eclipse.pde.core.plugin.IPluginModelBase; @@ -47,6 +56,8 @@ import org.eclipse.pde.core.target.ITargetLocation; import org.eclipse.pde.core.target.ITargetPlatformService; import org.eclipse.pde.internal.core.PDECore; +import org.eclipse.pde.internal.core.target.DirectoryBundleContainer; +import org.eclipse.pde.internal.core.target.FileArtifactRepository; import org.eclipse.pde.internal.core.target.IUBundleContainer; import org.eclipse.pde.internal.core.target.P2TargetUtils; import org.eclipse.pde.internal.core.target.TargetDefinition; @@ -182,6 +193,66 @@ public void testResolveSingleBundle() throws Exception { doResolutionTest(new String[]{"bundle.a1"}, bundles); } + /** + * Tests whether the in-memory artifact repository is correctly created from + * a non-IU target location. + */ + @Test + public void testResolveFileRepository() throws Exception { + File repoFolder = new File(getURI("/tests/sites/site.a.b")); + // DirectoryBundleContainer expects features to be unpacked + unjarAll(new File(repoFolder, "features")); + ITargetLocation container = new DirectoryBundleContainer(repoFolder.getAbsolutePath()); + + ITargetDefinition definition = getNewTarget(); + definition.setTargetLocations(new ITargetLocation[] { container }); + + container.resolve(definition, null); + IArtifactRepository repo = new FileArtifactRepository(null, container); + + assertTrue(repo.contains(BundlesAction.createBundleArtifactKey("bundle.a1", "1.0.0"))); + assertTrue(repo.contains(BundlesAction.createBundleArtifactKey("bundle.a2", "1.0.0"))); + assertTrue(repo.contains(BundlesAction.createBundleArtifactKey("bundle.a3", "1.0.0"))); + assertTrue(repo.contains(BundlesAction.createBundleArtifactKey("bundle.b1", "1.0.0"))); + assertTrue(repo.contains(BundlesAction.createBundleArtifactKey("bundle.b2", "1.0.0"))); + assertTrue(repo.contains(BundlesAction.createBundleArtifactKey("bundle.b3", "1.0.0"))); + assertTrue(repo.contains(FeaturesAction.createFeatureArtifactKey("feature.a", "1.0.0"))); + assertTrue(repo.contains(FeaturesAction.createFeatureArtifactKey("feature.b", "1.0.0"))); + } + + private void unjarAll(File directory) throws IOException { + for (File jarFile : directory.listFiles()) { + if (!jarFile.isFile() || !jarFile.getName().endsWith(".jar")) { + continue; + } + String jarFileName = jarFile.getName(); + String jarFileBaseName = jarFileName.substring(0, jarFileName.length() - ".jar".length()); + + File outDir = new File(directory, jarFileBaseName); + outDir.mkdir(); + outDir.deleteOnExit(); + + try (JarFile jar = new JarFile(jarFile)) { + jar.stream().forEach(jarEntry -> { + File outFile = new File(outDir, jarEntry.getName()); + outFile.deleteOnExit(); + if (outFile.isDirectory()) { + outFile.mkdir(); + return; + } + try (InputStream is = jar.getInputStream(jarEntry)) { + try (OutputStream os = new FileOutputStream(outFile)) { + is.transferTo(os); + } + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + }); + } + } + } + /** * Tests that contents should be equal. */