Skip to content

Commit

Permalink
[TP-Editor] Completely parallelize P2 metadata fetching
Browse files Browse the repository at this point in the history
  • Loading branch information
@HannesWell
HannesWell committed Oct 19, 2024
1 parent ad86de6 commit 52e96a6
Show file tree
Hide file tree
Showing 6 changed files with 70 additions and 106 deletions.
30 changes: 9 additions & 21 deletions ...c/org/eclipse/pde/internal/genericeditor/target/extension/command/UpdateUnitVersions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.eclipse.core.commands.AbstractHandler;
import org.eclipse.core.commands.ExecutionEvent;
import org.eclipse.core.commands.ExecutionException;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.equinox.p2.metadata.IVersionedId;
import org.eclipse.jface.dialogs.IPageChangeProvider;
import org.eclipse.jface.dialogs.MessageDialog;
Expand All @@ -33,7 +32,6 @@
import org.eclipse.pde.internal.genericeditor.target.extension.model.RepositoryCache;
import org.eclipse.pde.internal.genericeditor.target.extension.model.UnitNode;
import org.eclipse.pde.internal.genericeditor.target.extension.model.xml.Parser;
import org.eclipse.pde.internal.genericeditor.target.extension.p2.UpdateJob;
import org.eclipse.swt.widgets.Display;
import org.eclipse.ui.IEditorInput;
import org.eclipse.ui.IEditorPart;
Expand Down Expand Up @@ -69,20 +67,19 @@ public Object execute(ExecutionEvent event) throws ExecutionException {

int offsetChange = 0;
String documentText = document.get();
for (Node n1 : locationsNode.get(0).getChildNodesByTag(ITargetConstants.LOCATION_TAG)) {
LocationNode locationNode = (LocationNode) n1;

List<LocationNode> locationNodes = locationsNode.get(0).getChildNodesByTag(ITargetConstants.LOCATION_TAG)
.stream().map(LocationNode.class::cast).toList();

// Fetch all repos at once to fetch pending metadata in parallel
locationNodes.stream().map(LocationNode::getRepositoryLocations).flatMap(List::stream)
.forEach(RepositoryCache::prefetchP2MetadataOfRepository);

for (LocationNode locationNode : locationNodes) {
List<String> repositoryLocations = locationNode.getRepositoryLocations();
if (repositoryLocations.isEmpty()) {
continue;
}
if (!repositoryLocations.stream().allMatch(RepositoryCache::isUpToDate)) {
try {
updateCache(locationNode);
} catch (InterruptedException e) {
e.printStackTrace();
continue;
}
}
Map<String, List<IVersionedId>> repositoryUnits = RepositoryCache
.fetchP2UnitsFromRepos(repositoryLocations);
for (Node n2 : locationNode.getChildNodesByTag(ITargetConstants.UNIT_TAG)) {
Expand Down Expand Up @@ -129,15 +126,6 @@ public Object execute(ExecutionEvent event) throws ExecutionException {
});
}

private void updateCache(LocationNode locationNode) throws InterruptedException {
Job job = new UpdateJob(locationNode);
job.setUser(true);
job.schedule();
while (job.getResult() == null) {
Thread.sleep(50);
}
}

private IDocument getDocument() {
IEditorPart editor = PlatformUI.getWorkbench().getActiveWorkbenchWindow().getActivePage().getActiveEditor();
IDocumentProvider provider = null;
Expand Down
63 changes: 46 additions & 17 deletions ...on/src/org/eclipse/pde/internal/genericeditor/target/extension/model/RepositoryCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,17 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.eclipse.core.runtime.ILog;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.equinox.p2.metadata.IVersionedId;
import org.eclipse.osgi.util.NLS;
import org.eclipse.pde.internal.genericeditor.target.extension.p2.Messages;
import org.eclipse.pde.internal.genericeditor.target.extension.p2.P2Fetcher;

/**
Expand All @@ -42,7 +48,7 @@ private RepositoryCache() {
// avoid instantiation
}

private static final Map<URI, Map<String, List<IVersionedId>>> CACHE = new ConcurrentHashMap<>();
private static final Map<URI, CompletableFuture<Map<String, List<IVersionedId>>>> CACHE = new ConcurrentHashMap<>();

/**
* Fetches information and caches it.
Expand All @@ -58,20 +64,46 @@ private RepositoryCache() {
*/
public static Map<String, List<IVersionedId>> fetchP2UnitsFromRepos(List<String> repositories) {
if (repositories.size() == 1) {
return fetchP2DataOfRepo(repositories.get(0));
return getFutureValue(fetchP2DataOfRepo(repositories.get(0)));
}
var repos = repositories.stream().map(RepositoryCache::fetchP2DataOfRepo).toList();
return toSortedMap(repos.stream().map(Map::values).flatMap(Collection::stream).flatMap(List::stream));
// Fetch all repos at once to await pending metadata in parallel
return toSortedMap(repos.stream().map(RepositoryCache::getFutureValue) //
.map(Map::values).flatMap(Collection::stream).flatMap(List::stream));
}

private static Map<String, List<IVersionedId>> fetchP2DataOfRepo(String repository) {
public static void prefetchP2MetadataOfRepository(String repository) {
fetchP2DataOfRepo(repository);
}

private static Future<Map<String, List<IVersionedId>>> fetchP2DataOfRepo(String repository) {
URI location;
try {
location = new URI(repository);
} catch (URISyntaxException e) {
return Map.of();
return CompletableFuture.failedFuture(e);
}
return CACHE.computeIfAbsent(location, repo -> toSortedMap(P2Fetcher.fetchAvailableUnits(repo)));
return CACHE.compute(location, (repo, f) -> {
if (f != null && (!f.isDone() || !f.isCompletedExceptionally() && !f.isCancelled())) {
return f; // computation is running or has succeeded
}
CompletableFuture<Map<String, List<IVersionedId>>> future = new CompletableFuture<>();
// Fetching P2 repository information is a costly operation
// time-wise. Thus it is done in a job.
Job job = Job.create(NLS.bind(Messages.UpdateJob_P2DataFetch, repo), m -> {
try {
Map<String, List<IVersionedId>> units = toSortedMap(P2Fetcher.fetchAvailableUnits(repo, m));
future.complete(units);
} catch (Throwable e) {
future.completeExceptionally(e);
// Only log the failure, don't open an error-dialog.
ILog.get().warn(e.getMessage(), e);
}
});
job.setUser(true);
job.schedule();
return future;
});
}

private static final Comparator<IVersionedId> BY_ID_FIRST_THEN_DESCENDING_VERSION = Comparator
Expand All @@ -83,6 +115,14 @@ private static Map<String, List<IVersionedId>> toSortedMap(Stream<IVersionedId>
Collectors.groupingBy(IVersionedId::getId, LinkedHashMap::new, Collectors.toUnmodifiableList()));
}

private static Map<String, List<IVersionedId>> getFutureValue(Future<Map<String, List<IVersionedId>>> future) {
try {
return future.get();
} catch (Exception e) { // interrupted, canceled or execution failure
return Map.of();
}
}

/**
*
* Method used to narrow down proposals in case a prefix is provided.
Expand Down Expand Up @@ -129,15 +169,4 @@ public static List<IVersionedId> getUnitsBySearchTerm(String repo, String search
return allUnits.values().stream().flatMap(List::stream) //
.filter(unit -> unit.getId().contains(searchTerm)).toList();
}

/**
* Classic cache up-to-date check.
*
* @param repo
* repository URL
* @return whether the cache is up to date for this repo
*/
public static boolean isUpToDate(String repo) {
return CACHE.get(URI.create(repo)) != null;
}
}
1 change: 0 additions & 1 deletion ...or.extension/src/org/eclipse/pde/internal/genericeditor/target/extension/p2/Messages.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
public class Messages extends NLS {
private static final String BUNDLE_NAME = "org.eclipse.pde.internal.genericeditor.target.extension.p2.messages"; //$NON-NLS-1$
public static String UpdateJob_P2DataFetch;
public static String UpdateJob_ErrorMessage;
static {
// initialize resource bundle
NLS.initializeMessages(BUNDLE_NAME, Messages.class);
Expand Down
33 changes: 13 additions & 20 deletions ...r.extension/src/org/eclipse/pde/internal/genericeditor/target/extension/p2/P2Fetcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,11 @@
import java.net.URI;
import java.util.stream.Stream;

import org.eclipse.core.runtime.ILog;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.SubMonitor;
import org.eclipse.equinox.p2.core.IProvisioningAgent;
import org.eclipse.equinox.p2.core.IProvisioningAgentProvider;
import org.eclipse.equinox.p2.core.ProvisionException;
import org.eclipse.equinox.p2.metadata.IInstallableUnit;
import org.eclipse.equinox.p2.metadata.IVersionedId;
import org.eclipse.equinox.p2.metadata.VersionedId;
Expand All @@ -45,28 +46,20 @@ public class P2Fetcher {
* URL string of a p2 repository
* @return List of available installable unit models. See {@link UnitNode}
*/
public static Stream<IVersionedId> fetchAvailableUnits(URI repositoryLocation) {
public static Stream<IVersionedId> fetchAvailableUnits(URI repositoryLocation, IProgressMonitor monitor)
throws CoreException {
SubMonitor subMonitor = SubMonitor.convert(monitor, 31);
BundleContext context = FrameworkUtil.getBundle(P2Fetcher.class).getBundleContext();
ServiceReference<IProvisioningAgentProvider> sr = context.getServiceReference(IProvisioningAgentProvider.class);
try {
BundleContext context = FrameworkUtil.getBundle(P2Fetcher.class).getBundleContext();
ServiceReference<IProvisioningAgentProvider> sr = context
.getServiceReference(IProvisioningAgentProvider.class);
IProvisioningAgentProvider agentProvider = context.getService(sr);
IProvisioningAgent agent = null;
try {
agent = agentProvider.createAgent(null);
} catch (ProvisionException e) {
ILog.get().error("Failed to create provisioning-agent", e);
} finally {
context.ungetService(sr);
}
IProvisioningAgent agent = agentProvider.createAgent(null);
IMetadataRepositoryManager manager = agent.getService(IMetadataRepositoryManager.class);
IMetadataRepository repository = manager.loadRepository(repositoryLocation, null);
IQueryResult<IInstallableUnit> allUnits = repository.query(QueryUtil.ALL_UNITS, null);

IMetadataRepository repository = manager.loadRepository(repositoryLocation, subMonitor.split(30));
IQueryResult<IInstallableUnit> allUnits = repository.query(QueryUtil.ALL_UNITS, subMonitor.split(1));
return allUnits.stream().map(iu -> new VersionedId(iu.getId(), iu.getVersion()));
} catch (Exception e) {
ILog.get().error("Failed to fetch metadata of repository: " + repositoryLocation, e);
return Stream.empty();
} finally {
context.ungetService(sr);
}
}

Expand Down
44 changes: 0 additions & 44 deletions ...r.extension/src/org/eclipse/pde/internal/genericeditor/target/extension/p2/UpdateJob.java
View file

This file was deleted.

5 changes: 2 additions & 3 deletions ...ension/src/org/eclipse/pde/internal/genericeditor/target/extension/p2/messages.properties
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
###############################################################################
# Copyright (c) 2016 Red Hat Inc. and others
# Copyright (c) 2016, 2024 Red Hat Inc. and others
#
# This program and the accompanying materials
# are made available under the terms of the Eclipse Public License 2.0
Expand All @@ -11,5 +11,4 @@
# Contributors:
# Sopot Cela (Red Hat Inc.) - initial implementation
###############################################################################
UpdateJob_P2DataFetch=Fetching p2 metadata from repository
UpdateJob_ErrorMessage=Issue fetching data from repository. Please check URL or see log for even more details.
UpdateJob_P2DataFetch=Fetching p2 metadata from repository: {0}

0 comments on commit 52e96a6

Please sign in to comment.