From 5d0845e54dd43f6b4bb65c58941b12b634f6bfe1 Mon Sep 17 00:00:00 2001 From: Hannes Wellmann Date: Sun, 27 Aug 2023 15:02:27 +0200 Subject: [PATCH] Add an option to filter repo-references before being added to a p2-repo Add a new parameter named 'repositoryReferenceFilter' to the AssembleRepositoryMojo to better control the automatic addition of repository references with matching locations. This can be used with ANT-style or Java RegEx patterns as follows: https://foo.bar.org/hidden/** https://foo.bar.org/secret/** %regex[http(s)?:\/\/foo\.bar\.org\/.*] The is especially convenient in combination with the automatic addition of IU Target-Repository or POM-Repository references, if some but not all repos should be added. --- RELEASE_NOTES.md | 22 +++++ .../category.xml | 9 ++ .../p2Repository.repositoryRef.filter/pom.xml | 55 +++++++++++ ...efLocationP2RepositoryIntegrationTest.java | 99 +++++++------------ .../p2Repository/RepositoryIncludeTest.java | 2 +- .../p2/repository/AssembleRepositoryMojo.java | 75 +++++++++++++- .../tycho/test/util/P2RepositoryTool.java | 73 +++++--------- 7 files changed, 216 insertions(+), 119 deletions(-) create mode 100644 tycho-its/projects/p2Repository.repositoryRef.filter/category.xml create mode 100644 tycho-its/projects/p2Repository.repositoryRef.filter/pom.xml diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 0778c14791..5a4ef0de22 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -23,6 +23,28 @@ Repositories can contain references to other repositories (e.g. to find addition +``` +### new option to filter added repository-references when assembling a p2-repository + +The repository references automatically added to a assembled p2-repository (via `tycho-p2-repository-plugin`'s `addIUTargetRepositoryReferences` or `addPomRepositoryReferences`) +can now be filtered by their location using exclusion and inclusion patterns and therefore allows more fine-grained control which references are added. +```xml + + org.eclipse.tycho + tycho-p2-repository-plugin + ${tycho-version} + + ... other configuration options ... + + + https://foo.bar.org/hidden/** + https://foo.bar.org/secret/** + + %regex[http(s)?:\/\/foo\.bar\.org\/.*] + + + + ``` ## 4.0.0 diff --git a/tycho-its/projects/p2Repository.repositoryRef.filter/category.xml b/tycho-its/projects/p2Repository.repositoryRef.filter/category.xml new file mode 100644 index 0000000000..6f6665dcc6 --- /dev/null +++ b/tycho-its/projects/p2Repository.repositoryRef.filter/category.xml @@ -0,0 +1,9 @@ + + + + + Test Category Description + + + + diff --git a/tycho-its/projects/p2Repository.repositoryRef.filter/pom.xml b/tycho-its/projects/p2Repository.repositoryRef.filter/pom.xml new file mode 100644 index 0000000000..33764661d8 --- /dev/null +++ b/tycho-its/projects/p2Repository.repositoryRef.filter/pom.xml @@ -0,0 +1,55 @@ + + + 4.0.0 + + 1.0.0 + tycho-its-project.p2Repository.repositoryRef.location + repositoryRef.location + eclipse-repository + + + + repo1 + https://download.eclipse.org/tm4e/releases/0.8.1 + p2 + + + repo2 + https://download.eclipse.org/lsp4e/releases/0.24.1 + p2 + + + repo3 + https://download.eclipse.org/lsp4j/updates/releases/0.21.1 + p2 + + + + + + org.eclipse.tycho + tycho-maven-plugin + ${tycho-version} + true + + + org.eclipse.tycho + tycho-p2-repository-plugin + ${tycho-version} + + false + true + + + https://download.eclipse.org/lsp4e/** + https://download.eclipse.org/lsp4j/** + + %regex[http(s)?:\/\/download\.eclipse\.org\/.*] + + + + + + diff --git a/tycho-its/src/test/java/org/eclipse/tycho/test/p2Repository/RepoRefLocationP2RepositoryIntegrationTest.java b/tycho-its/src/test/java/org/eclipse/tycho/test/p2Repository/RepoRefLocationP2RepositoryIntegrationTest.java index 401858bfb9..802954f6b7 100644 --- a/tycho-its/src/test/java/org/eclipse/tycho/test/p2Repository/RepoRefLocationP2RepositoryIntegrationTest.java +++ b/tycho-its/src/test/java/org/eclipse/tycho/test/p2Repository/RepoRefLocationP2RepositoryIntegrationTest.java @@ -12,92 +12,63 @@ *******************************************************************************/ package org.eclipse.tycho.test.p2Repository; +import static org.eclipse.equinox.p2.repository.IRepository.ENABLED; +import static org.eclipse.equinox.p2.repository.IRepository.NONE; +import static org.eclipse.equinox.p2.repository.IRepository.TYPE_ARTIFACT; +import static org.eclipse.equinox.p2.repository.IRepository.TYPE_METADATA; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import java.io.File; import java.util.List; -import java.util.Objects; import org.apache.maven.it.Verifier; import org.eclipse.tycho.test.AbstractTychoIntegrationTest; +import org.eclipse.tycho.test.util.P2RepositoryTool; +import org.eclipse.tycho.test.util.P2RepositoryTool.RepositoryReference; import org.eclipse.tycho.test.util.ResourceUtil; -import org.junit.BeforeClass; import org.junit.Test; -import de.pdark.decentxml.Document; -import de.pdark.decentxml.Element; -import de.pdark.decentxml.XMLParser; - public class RepoRefLocationP2RepositoryIntegrationTest extends AbstractTychoIntegrationTest { - private static Verifier verifier; - - private static class RepositoryReferenceData { - private String uri; - private String type; - private String enabled; - - public RepositoryReferenceData(String uri, String type, String enabled) { - this.uri = uri; - this.type = type; - this.enabled = enabled; - } + @Test + public void testRefLocation() throws Exception { + Verifier verifier = getVerifier("/p2Repository.repositoryRef.location", false); + verifier.addCliOption("-Dtest-data-repo=" + ResourceUtil.P2Repositories.ECLIPSE_LATEST.toString()); + verifier.executeGoal("package"); + verifier.verifyErrorFreeLog(); - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - RepositoryReferenceData other = (RepositoryReferenceData) obj; - return Objects.equals(enabled, other.enabled) && Objects.equals(type, other.type) - && Objects.equals(uri, other.uri); - } + P2RepositoryTool p2Repo = P2RepositoryTool.forEclipseRepositoryModule(new File(verifier.getBasedir())); + List allRepositoryReferences = p2Repo.getAllRepositoryReferences(); - @Override - public String toString() { - return "[uri=" + uri + ", type=" + type + ", enabled=" + enabled + "]"; - } + assertEquals(4, allRepositoryReferences.size()); + assertThat(allRepositoryReferences, + containsInAnyOrder(new RepositoryReference("http://some.where", TYPE_ARTIFACT, NONE), + new RepositoryReference("http://some.where", TYPE_METADATA, NONE), + new RepositoryReference("http://some.where.else", TYPE_ARTIFACT, ENABLED), + new RepositoryReference("http://some.where.else", TYPE_METADATA, ENABLED))); } - @BeforeClass - public static void executeBuild() throws Exception { - verifier = new RepoRefLocationP2RepositoryIntegrationTest().getVerifier("/p2Repository.repositoryRef.location", - false); - verifier.addCliOption("-Dtest-data-repo=" + ResourceUtil.P2Repositories.ECLIPSE_LATEST.toString()); + @Test + public void testReferenceFiltering() throws Exception { + // Of course it is actually a bit pointless to filter explicitly specified + // references, but it makes the test simple/faster instead of preparing a + // target-definition with IU-location so that it can be added automatically, + // which is the main use-case. + Verifier verifier = getVerifier("/p2Repository.repositoryRef.filter", false); verifier.executeGoal("package"); verifier.verifyErrorFreeLog(); - } - @Test - public void testRefLocation() throws Exception { - File target = new File(verifier.getBasedir(), "target"); - File repository = new File(target, "repository"); - File contentXml = new File(repository, "content.xml"); - assertTrue(contentXml.isFile()); - File artifactXml = new File(repository, "artifacts.xml"); - assertTrue(artifactXml.isFile()); - assertTrue(new File(target, "category.xml").isFile()); + P2RepositoryTool p2Repo = P2RepositoryTool.forEclipseRepositoryModule(new File(verifier.getBasedir())); + List allRepositoryReferences = p2Repo.getAllRepositoryReferences(); - Document artifactsDocument = XMLParser.parse(contentXml); - // See MetadataRepositoryIO.Writer#writeRepositoryReferences - List repositories = artifactsDocument.getChild("repository").getChild("references") - .getChildren("repository"); - assertEquals(4, repositories.size()); - List actual = repositories.stream() - .map(e -> new RepositoryReferenceData(e.getAttributeValue("uri"), e.getAttributeValue("type"), - e.getAttributeValue("options"))) - .toList(); - assertThat(actual, - containsInAnyOrder(new RepositoryReferenceData("http://some.where", "1", "0"), - new RepositoryReferenceData("http://some.where", "0", "0"), - new RepositoryReferenceData("http://some.where.else", "1", "1"), - new RepositoryReferenceData("http://some.where.else", "0", "1"))); + assertEquals(4, allRepositoryReferences.size()); + assertThat(allRepositoryReferences, containsInAnyOrder( // + new RepositoryReference("https://download.eclipse.org/tm4e/releases/0.8.1", TYPE_ARTIFACT, ENABLED), + new RepositoryReference("https://download.eclipse.org/tm4e/releases/0.8.1", TYPE_METADATA, ENABLED), + new RepositoryReference("https://some.where/from/category", TYPE_ARTIFACT, ENABLED), + new RepositoryReference("https://some.where/from/category", TYPE_METADATA, ENABLED))); } } diff --git a/tycho-its/src/test/java/org/eclipse/tycho/test/p2Repository/RepositoryIncludeTest.java b/tycho-its/src/test/java/org/eclipse/tycho/test/p2Repository/RepositoryIncludeTest.java index d19c5035e3..97707c1529 100644 --- a/tycho-its/src/test/java/org/eclipse/tycho/test/p2Repository/RepositoryIncludeTest.java +++ b/tycho-its/src/test/java/org/eclipse/tycho/test/p2Repository/RepositoryIncludeTest.java @@ -31,7 +31,7 @@ public void testFilterProvided() throws Exception { P2RepositoryTool p2Repo = P2RepositoryTool .forEclipseRepositoryModule(new File(verifier.getBasedir(), "repository")); p2Repo.getUniqueIU("bundle"); - p2Repo.assertNumberOfUnits(1, u -> u.id.equals("a.jre.javase") || u.id.endsWith(".test.category")); + p2Repo.assertNumberOfUnits(1, u -> u.id().equals("a.jre.javase") || u.id().endsWith(".test.category")); assertTrue("Bundle artifact not found!", p2Repo.findBundleArtifact("bundle").isPresent()); p2Repo.assertNumberOfBundles(1); p2Repo.assertNumberOfFeatures(0); diff --git a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java index bf4c13b3f3..3a52ed3b0e 100644 --- a/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java +++ b/tycho-p2-repository-plugin/src/main/java/org/eclipse/tycho/plugins/p2/repository/AssembleRepositoryMojo.java @@ -19,6 +19,8 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.function.Predicate; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.maven.model.Repository; @@ -29,6 +31,7 @@ import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; import org.codehaus.plexus.util.FileUtils; +import org.codehaus.plexus.util.MatchPatterns; import org.eclipse.tycho.PackagingType; import org.eclipse.tycho.ReactorProject; import org.eclipse.tycho.TychoConstants; @@ -73,6 +76,12 @@ */ @Mojo(name = "assemble-repository", defaultPhase = LifecyclePhase.PACKAGE, threadSafe = true) public class AssembleRepositoryMojo extends AbstractRepositoryMojo { + + public static class RepositoryReferenceFilter { + List exclude; + List include; + } + private static final Object LOCK = new Object(); /** *

@@ -217,6 +226,40 @@ public class AssembleRepositoryMojo extends AbstractRepositoryMojo { @Parameter() private boolean addIUTargetRepositoryReferences; + /** + * A list of patterns to filter the automatically derived repository references by including or + * excluding their location to control if they are eventually added to the assembled repository. + *

+ * Each pattern is either an inclusion or exclusion and an arbitrary number of + * each can be specified. The location of a repository reference must match at least one + * inclusion-pattern (if any is specified) and must not be match by any + * exclusion-pattern, in order to be eventually added to the assembled repository.
+ * The specified filters are only applied to those repository references derived from the + * target-definition or pom file, when {@link #addIUTargetRepositoryReferences} respectively + * {@link #addPomRepositoryReferences} is set to {@code true}. + *

+ *

+ * Configuration example + * + * 

+     * <repositoryReferenceFilter>
+     *   <exclude>
+     *     <location>https://foo.bar.org/hidden/**</location>
+     *     <location>https://foo.bar.org/secret/**</location>
+     *   </exclude>
+     *   <include>%regex[http(s)?:\/\/foo\.bar\.org\/.*]</include>
+     * </repositoryReferenceFilter>
+     *
+ * + * It contains two exclusion patterns using {@code ANT}-style syntax and one + * inclusion using a {@code Java RegEx} {@link Pattern} (enclosed in + * {@code %regex[]}). The inclusion pattern uses the shorthand + * notation for singleton lists. + *

+ */ + @Parameter + private RepositoryReferenceFilter repositoryReferenceFilter = null; + /** * If enabled, an * OSGi @@ -265,11 +308,14 @@ public void execute() throws MojoExecutionException, MojoFailureException { .flatMap(List::stream)// .map(ref -> new RepositoryReference(ref.getName(), ref.getLocation(), ref.isEnabled()))// .collect(Collectors.toCollection(ArrayList::new)); + Predicate autoReferencesFilter = buildRepositoryReferenceLocationFilter(); if (addPomRepositoryReferences) { for (Repository pomRepo : getProject().getRepositories()) { if ("p2".equals(pomRepo.getLayout())) { - repositoryReferences - .add(new RepositoryReference(pomRepo.getName(), pomRepo.getUrl(), true)); + String locationURL = pomRepo.getUrl(); + if (autoReferencesFilter.test(locationURL)) { + repositoryReferences.add(new RepositoryReference(pomRepo.getName(), locationURL, true)); + } } } } @@ -278,14 +324,17 @@ public void execute() throws MojoExecutionException, MojoFailureException { .getTargetPlatformConfiguration(getProject()).getTargets()) { for (Location location : targetDefinitionFile.getLocations()) { if (location instanceof InstallableUnitLocation iu) { - for (org.eclipse.tycho.targetplatform.TargetDefinition.Repository iuRepo : iu - .getRepositories()) { - repositoryReferences.add(new RepositoryReference(null, iuRepo.getLocation(), true)); + for (var iuRepo : iu.getRepositories()) { + String locationURL = iuRepo.getLocation(); + if (autoReferencesFilter.test(locationURL)) { + repositoryReferences.add(new RepositoryReference(null, locationURL, true)); + } } } } } } + DestinationRepositoryDescriptor destinationRepoDescriptor = new DestinationRepositoryDescriptor( destination, repositoryName, compress, xzCompress, keepNonXzIndexFiles, !createArtifactRepository, true, extraArtifactRepositoryProperties, repositoryReferences); @@ -351,4 +400,20 @@ private List getCategories(final File categoriesDirectory) { return eclipseRepositoryProject.loadCategories(categoriesDirectory); } + private Predicate buildRepositoryReferenceLocationFilter() { + Predicate filter = l -> true; + if (repositoryReferenceFilter != null) { + if (repositoryReferenceFilter.include != null && !repositoryReferenceFilter.include.isEmpty()) { + MatchPatterns inclusionPattern = MatchPatterns.from(repositoryReferenceFilter.include); + filter = l -> inclusionPattern.matches(l, true); + } + if (repositoryReferenceFilter.exclude != null && !repositoryReferenceFilter.exclude.isEmpty()) { + MatchPatterns exclusionPattern = MatchPatterns.from(repositoryReferenceFilter.exclude); + Predicate exclusionFilter = l -> !exclusionPattern.matches(l, true); + filter = filter.and(exclusionFilter); + } + } + return filter; + } + } diff --git a/tycho-testing-harness/src/main/java/org/eclipse/tycho/test/util/P2RepositoryTool.java b/tycho-testing-harness/src/main/java/org/eclipse/tycho/test/util/P2RepositoryTool.java index 3d27c89ee2..d4ecba057d 100644 --- a/tycho-testing-harness/src/main/java/org/eclipse/tycho/test/util/P2RepositoryTool.java +++ b/tycho-testing-harness/src/main/java/org/eclipse/tycho/test/util/P2RepositoryTool.java @@ -7,11 +7,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; import javax.xml.parsers.DocumentBuilderFactory; @@ -28,7 +28,7 @@ public class P2RepositoryTool { - private static final ThreadLocal xPathTool = ThreadLocal + private static final ThreadLocal XPATH_TOOL = ThreadLocal .withInitial(() -> XPathFactory.newInstance().newXPath()); private static final Pattern strictVersionRangePattern = Pattern.compile("\\[([^,]*),\\1\\]"); private final File repoLocation; @@ -213,6 +213,21 @@ public List getAllProvidedPackages() throws Exception { return getValues(contentXml, "/repository/units/unit/provides/provided[@namespace='java.package']/@name"); } + public List getAllRepositoryReferences() throws Exception { + loadMetadata(); + // See MetadataRepositoryIO.Writer#writeRepositoryReferences + List references = getNodes(contentXml, "/repository/references/repository"); + List result = new ArrayList<>(); + for (Node reference : references) { + String uri = getAttribute(reference, "@uri"); + int type = Integer.parseInt(getAttribute(reference, "@type")); + int options = Integer.parseInt(getAttribute(reference, "@options")); + result.add(new RepositoryReference(uri, type, options)); + } + + return result; + } + private void loadMetadata() throws Exception { if (contentXml != null) return; @@ -223,27 +238,17 @@ private void loadMetadata() throws Exception { } private static XPath getXPathTool() { - return xPathTool.get(); + return XPATH_TOOL.get(); } static List getNodes(Object startingPoint, String expression) throws XPathExpressionException { NodeList nodeList = (NodeList) getXPathTool().evaluate(expression, startingPoint, XPathConstants.NODESET); - List result = new ArrayList<>(nodeList.getLength()); - for (int ix = 0; ix < nodeList.getLength(); ++ix) { - result.add(nodeList.item(ix)); - } - return result; + return IntStream.range(0, nodeList.getLength()).mapToObj(nodeList::item).toList(); } static List getValues(Object startingPoint, String expression) throws XPathExpressionException { - NodeList nodeList = (NodeList) getXPathTool().evaluate(expression, startingPoint, XPathConstants.NODESET); - - List result = new ArrayList<>(nodeList.getLength()); - for (int ix = 0; ix < nodeList.getLength(); ++ix) { - result.add(nodeList.item(ix).getNodeValue()); - } - return result; + return getNodes(startingPoint, expression).stream().map(Node::getNodeValue).toList(); } static String getAttribute(Node node, String expression) throws XPathExpressionException { @@ -300,14 +305,7 @@ public List getProperties() throws Exception { } public List getRequiredIds() throws Exception { - List result = new ArrayList<>(); - - List requiredIds = getNodes(unitElement, "requires/required/@name"); - for (Node id : requiredIds) { - result.add(id.getNodeValue()); - } - - return result; + return getValues(unitElement, "requires/required/@name"); } /** @@ -371,33 +369,10 @@ public List getProvidedCapabilities() throws Exception { } } - public static final class IdAndVersion { - public final String id; - public final String version; - - public IdAndVersion(String id, String version) { - this.id = id; - this.version = version; - } - - @Override - public int hashCode() { - return Objects.hash(id, version); - } - - @Override - public boolean equals(Object obj) { - return this == obj || // - (obj instanceof IdAndVersion other && // - Objects.equals(id, other.id) && // - Objects.equals(version, other.version)); - } - - @Override - public String toString() { - return "id=" + id + ", version=" + version; - } + public static final record IdAndVersion(String id, String version) { + } + public static final record RepositoryReference(String uri, int type, int options) { } public static IdAndVersion withIdAndVersion(String id, String version) {