From e4a744238a202ed3b0649ed8fe039c5c15d0671b Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Thu, 18 Apr 2024 09:16:59 +0200 Subject: [PATCH 01/31] feat: upgrade to EDC 0.6 and port the service patterns from embedded matchmaking. --- .gitignore | 1 - .../agents/AgreementControllerImpl.java | 17 +- .../eclipse/tractusx/agents/SkillStore.java | 4 +- .../tractusx/agents/http/AgentController.java | 6 +- .../tractusx/agents/http/GraphController.java | 6 +- .../matchmaking/SharedObjectManager.java | 38 +- .../agents/service/DataManagement.java | 509 ++++++++++++- .../agents/service/DataManagementImpl.java | 683 ------------------ .../agents/service/DataspaceSynchronizer.java | 438 +++++++---- .../agents/service/EdcSkillStore.java | 6 +- .../agents/service/InMemorySkillStore.java | 2 +- 11 files changed, 837 insertions(+), 873 deletions(-) delete mode 100644 matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagementImpl.java diff --git a/.gitignore b/.gitignore index 8b951521..40e37e8c 100644 --- a/.gitignore +++ b/.gitignore @@ -35,7 +35,6 @@ __pycache__ .classpath .project .settings/ -*.bak *cs-cleanup.xml *cs-formatter.xml .checkstyle diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java index c432eddc..d54d0926 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java @@ -17,6 +17,7 @@ package org.eclipse.tractusx.agents; import com.nimbusds.jose.JWSObject; +import jakarta.json.Json; import jakarta.json.JsonValue; import jakarta.ws.rs.BadRequestException; import jakarta.ws.rs.ClientErrorException; @@ -38,7 +39,6 @@ import org.eclipse.tractusx.agents.model.TransferProcess; import org.eclipse.tractusx.agents.model.TransferRequest; import org.eclipse.tractusx.agents.service.DataManagement; -import org.eclipse.tractusx.agents.service.DataManagementImpl; import org.eclipse.tractusx.agents.service.DataspaceSynchronizer; import org.eclipse.tractusx.agents.utils.CallbackAddress; import org.eclipse.tractusx.agents.utils.DataAddress; @@ -120,7 +120,7 @@ public String toString() { public void receiveEdcCallback(EndpointDataReference dataReference) { var agreementId = dataReference.getId(); monitor.debug(String.format("An endpoint data reference for agreement %s has been posted.", agreementId)); - synchronized (agreementStore) { + synchronized (processStore) { for (Map.Entry process : processStore.entrySet()) { if (process.getValue().getId().equals(agreementId)) { synchronized (endpointStore) { @@ -283,7 +283,7 @@ public EndpointDataReference createAgreement(String remoteUrl, String asset) thr var contractNegotiationRequest = ContractNegotiationRequest.Builder.newInstance() .offerId(contractOfferDescription) .connectorId("provider") - .connectorAddress(String.format(DataManagementImpl.DSP_PATH, remoteUrl)) + .connectorAddress(String.format(DataManagement.DSP_PATH, remoteUrl)) .protocol("dataspace-protocol-http") .localBusinessPartnerNumber(config.getBusinessPartnerNumber()) .remoteBusinessPartnerNumber(contractOffers.getParticipantId()) @@ -359,7 +359,7 @@ public EndpointDataReference createAgreement(String remoteUrl, String asset) thr .assetId(asset) .contractId(agreement.getId()) .connectorId(config.getBusinessPartnerNumber()) - .connectorAddress(String.format(DataManagementImpl.DSP_PATH, remoteUrl)) + .connectorAddress(String.format(DataManagement.DSP_PATH, remoteUrl)) .protocol("dataspace-protocol-http") .dataDestination(dataDestination) .managedResources(false) @@ -369,9 +369,14 @@ public EndpointDataReference createAgreement(String remoteUrl, String asset) thr monitor.debug(String.format("About to initiate transfer for agreement %s (for asset %s at connector %s)", negotiation.getContractAgreementId(), asset, remoteUrl)); String transferId; + TransferProcess process; try { - transferId = dataManagement.initiateHttpProxyTransferProcess(transferRequest); + synchronized (processStore) { + transferId = dataManagement.initiateHttpProxyTransferProcess(transferRequest); + process = new TransferProcess(Json.createObjectBuilder().add("@id", transferId).add("https://w3id.org/edc/v0.0.1/ns/state", "UNINITIALIZED").build()); + registerProcess(asset, process); + } } catch (IOException ioe) { deactivate(asset); throw new InternalServerErrorException(String.format("HttpProxy transfer for agreement %s could not be initiated.", agreement.getId()), ioe); @@ -380,8 +385,6 @@ public EndpointDataReference createAgreement(String remoteUrl, String asset) thr monitor.debug(String.format("About to check transfer %s (for asset %s at connector %s)", transferId, asset, remoteUrl)); // Check negotiation state - TransferProcess process = null; - startTime = System.currentTimeMillis(); // EDC 0.5.1 has a problem with the checker configuration and wont process to COMPLETED diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/SkillStore.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/SkillStore.java index df355d79..495e87a8 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/SkillStore.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/SkillStore.java @@ -56,10 +56,12 @@ static Matcher matchSkill(String key) { * @param contract of skill optional * @param dist of skill required * @param isFederated whether skill maybe synchronized in catalogue + * @param allowServicePattern regex for service to call in skill + * @param denyServicePattern regex for services denied in skill * @param ontologies a set of ontologies * @return skill id */ - String put(String key, String skill, String name, String description, String version, String contract, SkillDistribution dist, boolean isFederated, String... ontologies); + String put(String key, String skill, String name, String description, String version, String contract, SkillDistribution dist, boolean isFederated, String allowServicePattern, String denyServicePattern, String... ontologies); /** * return the skill distribution diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/AgentController.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/AgentController.java index 12c0d603..b3353093 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/AgentController.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/AgentController.java @@ -454,11 +454,13 @@ public Response postSkill(String query, @QueryParam("contract") String contract, @QueryParam("distributionMode") SkillDistribution mode, @QueryParam("isFederated") boolean isFederated, + @QueryParam("allowServicesPattern") String allowServicePattern, + @QueryParam("denyServicesPattern") String denyServicePattern, @QueryParam("ontology") String[] ontologies ) { - monitor.debug(String.format("Received a POST skill request %s %s %s %s %s %b %s ", asset, name, description, version, contract, mode.getMode(), isFederated, query)); + monitor.debug(String.format("Received a POST skill request %s %s %s %s %s %b %s %s %s ", asset, name, description, version, contract, mode.getMode(), isFederated, allowServicePattern, denyServicePattern, query)); Response.ResponseBuilder rb; - if (skillStore.put(asset, query, name, description, version, contract, mode, isFederated, ontologies) != null) { + if (skillStore.put(asset, query, name, description, version, contract, mode, isFederated, allowServicePattern, denyServicePattern, ontologies) != null) { rb = Response.ok(); } else { rb = Response.status(HttpStatus.SC_CREATED); diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/GraphController.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/GraphController.java index cd90ae4c..062986b7 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/GraphController.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/GraphController.java @@ -97,10 +97,12 @@ public Response postAsset(String content, @QueryParam("shape") String shape, @QueryParam("isFederated") boolean isFederated, @QueryParam("ontology") String[] ontologies, + @QueryParam("allowServicesPattern") String allow, + @QueryParam("denyServicesPattern") String deny, @Context HttpServletRequest request ) { ExternalFormat format = ExternalFormat.valueOfFormat(request.getContentType()); - monitor.debug(String.format("Received a POST asset request %s %s %s %s %s %b in format %s", asset, name, description, version, contract, shape, isFederated, format)); + monitor.debug(String.format("Received a POST asset request %s %s %s %s %s %b %s %s in format %s", asset, name, description, version, contract, shape, isFederated, allow, deny, format)); if (format == null) { return Response.status(Response.Status.BAD_REQUEST).build(); } @@ -121,7 +123,7 @@ public Response postAsset(String content, if (shape == null) { shape = String.format("@prefix : <%s#> .\\n", asset); } - management.createOrUpdateGraph(asset, name, description, version, contract, ontologiesString, shape, isFederated); + management.createOrUpdateGraph(asset, name, description, version, contract, ontologiesString, shape, isFederated, allow, deny); long reg = store.registerAsset(asset, content, format); MediaType med = MediaType.APPLICATION_JSON_TYPE; ResponseBuilder resBuild = Response.ok(reg, med); diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/SharedObjectManager.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/SharedObjectManager.java index 2b6932df..a6f1289b 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/SharedObjectManager.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/SharedObjectManager.java @@ -30,7 +30,7 @@ import org.eclipse.tractusx.agents.http.DelegationServiceImpl; import org.eclipse.tractusx.agents.http.GraphController; import org.eclipse.tractusx.agents.rdf.RdfStore; -import org.eclipse.tractusx.agents.service.DataManagementImpl; +import org.eclipse.tractusx.agents.service.DataManagement; import org.eclipse.tractusx.agents.service.DataspaceSynchronizer; import org.eclipse.tractusx.agents.service.EdcSkillStore; import org.eclipse.tractusx.agents.sparql.DataspaceServiceExecutor; @@ -50,20 +50,20 @@ public class SharedObjectManager { private static final SharedObjectManager INSTANCE = new SharedObjectManager(); - private Monitor monitor; - private TypeManager typeManager; - private AgentConfig agentConfig; - private RdfStore rdfStore; - private ServiceExecutorRegistry reg; - private SparqlQueryProcessor processor; - private DataManagementImpl catalogService; - private SkillStore skillStore; - private AgreementControllerImpl agreementController; - private AgentController agentController; - private GraphController graphController; - private DelegationServiceImpl delegationService; - private DataspaceSynchronizer synchronizer; - private OkHttpClient httpClient; + private final Monitor monitor; + private final TypeManager typeManager; + private final AgentConfig agentConfig; + private final RdfStore rdfStore; + private final ServiceExecutorRegistry reg; + private final SparqlQueryProcessor processor; + private final DataManagement catalogService; + private final SkillStore skillStore; + private final AgreementControllerImpl agreementController; + private final AgentController agentController; + private final GraphController graphController; + private final DelegationServiceImpl delegationService; + private final DataspaceSynchronizer synchronizer; + private final OkHttpClient httpClient; private SharedObjectManager() { @@ -93,7 +93,7 @@ private SharedObjectManager() { Config emptyConfig = ConfigFactory.fromProperties(props); this.agentConfig = new AgentConfig(monitor, emptyConfig); this.httpClient = new OkHttpClient(); - this.catalogService = new DataManagementImpl(monitor, typeManager, httpClient, agentConfig); + this.catalogService = new DataManagement(monitor, typeManager, httpClient, agentConfig); agreementController = new AgreementControllerImpl(monitor, agentConfig, catalogService); this.rdfStore = new RdfStore(agentConfig, monitor); ScheduledExecutorService executorService = Executors.newScheduledThreadPool(agentConfig.getThreadPoolSize()); @@ -119,9 +119,7 @@ public static String convertToCurl(Request request) { curlCommand.append(" -X ").append(request.method()); // Add headers - request.headers().toMultimap().forEach((name, values) -> { - values.forEach(value -> curlCommand.append(" -H '").append(name).append(": ").append(value).append("'")); - }); + request.headers().toMultimap().forEach((name, values) -> values.forEach(value -> curlCommand.append(" -H '").append(name).append(": ").append(value).append("'"))); // Add request body if present if (request.body() != null) { @@ -164,7 +162,7 @@ public SparqlQueryProcessor getProcessor() { return processor; } - public DataManagementImpl getCatalogService() { + public DataManagement getCatalogService() { return catalogService; } diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java index 0bd9d0d6..4b031fb2 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java @@ -16,6 +16,17 @@ // SPDX-License-Identifier: Apache-2.0 package org.eclipse.tractusx.agents.service; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import jakarta.ws.rs.InternalServerErrorException; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.ResponseBody; +import org.eclipse.tractusx.agents.AgentConfig; +import org.eclipse.tractusx.agents.jsonld.JsonLd; +import org.eclipse.tractusx.agents.matchmaking.SharedObjectManager; import org.eclipse.tractusx.agents.model.Asset; import org.eclipse.tractusx.agents.model.ContractAgreement; import org.eclipse.tractusx.agents.model.ContractNegotiation; @@ -24,38 +35,237 @@ import org.eclipse.tractusx.agents.model.IdResponse; import org.eclipse.tractusx.agents.model.TransferProcess; import org.eclipse.tractusx.agents.model.TransferRequest; +import org.eclipse.tractusx.agents.utils.Criterion; +import org.eclipse.tractusx.agents.utils.Monitor; import org.eclipse.tractusx.agents.utils.QuerySpec; +import org.eclipse.tractusx.agents.utils.TypeManager; import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.List; +import static java.lang.String.format; /** * DataManagement * is a service wrapper around the management endpoint * of the EDC control plane */ -public interface DataManagement { +public class DataManagement { + /** + * some constants when interacting with control plane + */ + public static final String DSP_PATH = "%1$s/api/v1/dsp"; + public static final String CATALOG_CALL = "%1$s/v2/catalog/request"; + // catalog request 0.5.>=1 + public static final String CATALOG_REQUEST_BODY = "{" + + "\"@context\": {}," + + "\"protocol\": \"dataspace-protocol-http\"," + + "\"counterPartyAddress\": \"%1$s\", " + + "\"querySpec\": %2$s }"; + + public static final String ASSET_CREATE_CALL = "%1$s%2$s/assets"; + public static final String ASSET_UPDATE_CALL = "%1$s%2$s/assets/%3$s"; + + /** + * template for skill asset creation + */ + public static final String SKILL_ASSET_CREATE_BODY = "{\n" + + " \"@context\": {\n" + + " \"rdf\": \"http://www.w3.org/1999/02/22-rdf-syntax-ns#\",\n" + + " \"rdfs\": \"http://www.w3.org/2000/01/rdf-schema#\",\n" + + " \"cx-common\": \"https://w3id.org/catenax/ontology/common#\",\n" + + " \"xsd\": \"http://www.w3.org/2001/XMLSchema#\",\n" + + " \"sh\": \"http://www.w3.org/ns/shacl#\"\n" + + " },\n" + + " \"asset\": {\n" + + " \"@type\": \"Asset\",\n" + + " \"@id\": \"%1$s\", \n" + + " \"properties\": {\n" + + " \"name\": \"%2$s\",\n" + + " \"description\": \"%3$s\",\n" + + " \"version\": \"%4$s\",\n" + + " \"contenttype\": \"application/json, application/xml\",\n" + + "%5$s" + + " \"rdf:type\": \"cx-common:SkillAsset\",\n" + + " \"rdfs:isDefinedBy\": \"%6$s\",\n" + + " \"cx-common:implementsProtocol\": \"cx-common:Protocol?w3c:http:SKILL\",\n" + + " \"cx-common:distributionMode\": \"%7$s\",\n" + + " \"cx-common:isFederated\": \"%8$b^^xsd:boolean\"\n" + + " },\n" + + " \"privateProperties\": {\n" + + " \"cx-common:query\":%9$s\n" + + " }\n" + + " },\n" + + " \"dataAddress\": {\n" + + " \"id\":\"%1$s\",\n" + + " \"@type\": \"DataAddress\",\n" + + " \"type\": \"cx-common:Protocol?w3c:http:SKILL\",\n" + + " \"baseUrl\":\"https://w3id.org/catenax\",\n" + + " \"proxyPath\": \"false\",\n" + + " \"proxyMethod\": \"true\",\n" + + " \"proxyQueryParams\": \"true\",\n" + + " \"proxyBody\": \"true\",\n" + + " \"cx-common:allowServicePattern\": \"%10$s\",\n" + + " \"cx-common:denyServicePattern\": \"%11$s\"\n" + + " }\n" + + "}\n"; + + /** + * template for graph asset creation + */ + public static final String GRAPH_ASSET_CREATE_BODY = "{\n" + + " \"@context\": {\n" + + " \"rdf\": \"http://www.w3.org/1999/02/22-rdf-syntax-ns#\",\n" + + " \"rdfs\": \"http://www.w3.org/2000/01/rdf-schema#\",\n" + + " \"cx-common\": \"https://w3id.org/catenax/ontology/common#\",\n" + + " \"xsd\": \"http://www.w3.org/2001/XMLSchema#\",\n" + + " \"sh\": \"http://www.w3.org/ns/shacl#\"\n" + + " },\n" + + " \"asset\": {\n" + + " \"@type\": \"Asset\",\n" + + " \"@id\": \"%1$s\", \n" + + " \"properties\": {\n" + + " \"name\": \"%2$s\",\n" + + " \"description\": \"%3$s\",\n" + + " \"version\": \"%4$s\",\n" + + " \"contenttype\": \"application/json, application/xml\",\n" + + "%5$s" + + " \"rdf:type\": \"cx-common:GraphAsset\",\n" + + " \"rdfs:isDefinedBy\": \"%6$s\",\n" + + " \"cx-common:implementsProtocol\": \"cx-common:Protocol?w3c:http:SPARQL\",\n" + + " \"sh:shapesGraph\": \"%7$s\",\n" + + " \"cx-common:isFederated\": \"%8$b^^xsd:boolean\"\n" + + " }\n" + + " },\n" + + " \"dataAddress\": {\n" + + " \"id\": \"%1$s\", \n" + + " \"@type\": \"DataAddress\",\n" + + " \"type\": \"cx-common:Protocol?w3c:http:SPARQL\",\n" + + " \"baseUrl\":\"https://w3id.org/catenax\",\n" + + " \"proxyPath\": \"false\",\n" + + " \"proxyMethod\": \"true\",\n" + + " \"proxyQueryParams\": \"true\",\n" + + " \"proxyBody\": \"true\",\n" + + " \"cx-common:allowServicePattern\": \"%10$s\",\n" + + " \"cx-common:denyServicePattern\": \"%11$s\"\n" + + " }\n" + + "}\n"; + + public static final String ASSET_CALL = "%s%s/assets/request"; + + // negotiation request 0.5.>=1 + public static final String NEGOTIATION_REQUEST_BODY = "{\n" + + "\"@context\": { \"odrl\": \"http://www.w3.org/ns/odrl/2/\"},\n" + + "\"@type\": \"NegotiationInitiateRequestDto\",\n" + + "\"connectorAddress\": \"%1$s\",\n" + + "\"protocol\": \"dataspace-protocol-http\",\n" + + "\"providerId\": \"%3$s\",\n" + + "\"connectorId\": \"%2$s\",\n" + + "\"offer\": {\n" + + " \"offerId\": \"%4$s\",\n" + + " \"assetId\": \"%5$s\",\n" + + " \"policy\": %6$s\n" + + "}\n" + + "}"; + + public static final String NEGOTIATION_INITIATE_CALL = "%s/v2/contractnegotiations"; + public static final String NEGOTIATION_CHECK_CALL = "%s/v2/contractnegotiations/%s"; + public static final String TRANSFER_INITIATE_CALL = "%s/v2/transferprocesses"; + + public static final String TRANSFER_REQUEST_BODY = "{\n" + + " \"@context\": {\n" + + " \"odrl\": \"http://www.w3.org/ns/odrl/2/\"\n" + + " },\n" + + " \"assetId\": \"%1$s\",\n" + + " \"connectorAddress\": \"%2$s\",\n" + + " \"connectorId\": \"%5$s\",\n" + + " \"contractId\": \"%3$s\",\n" + + " \"dataDestination\": {\n" + + " \"type\": \"HttpProxy\"\n" + + " },\n" + + " \"managedResources\": false,\n" + + " \"privateProperties\": {\n" + + " \"receiverHttpEndpoint\": \"%4$s\"\n" + + " },\n" + + " \"protocol\": \"dataspace-protocol-http\",\n" + + " \"transferType\": {\n" + + " \"contentType\": \"application/octet-stream\",\n" + + " \"isFinite\": true\n" + + " }\n" + + "}"; + public static final String TRANSFER_CHECK_CALL = "%s/v2/transferprocesses/%s"; + public static final String AGREEMENT_CHECK_CALL = "%s/v2/contractagreements/%s"; + + /** + * references to EDC services + */ + private final Monitor monitor; + private final ObjectMapper objectMapper; + private final OkHttpClient httpClient; + private final AgentConfig config; + + /** + * creates a service wrapper + * + * @param monitor logger + * @param typeManager serialization + * @param httpClient remoting + * @param config typed config + */ + public DataManagement(Monitor monitor, TypeManager typeManager, OkHttpClient httpClient, AgentConfig config) { + this.monitor = monitor; + this.objectMapper = typeManager.getMapper(); + this.httpClient = httpClient; + this.config = config; + } /** * Search for a dedicated asset + * TODO imperformant + * TODO replace by accessing the federated data catalogue * * @param remoteControlPlaneIdsUrl url of the remote control plane ids endpoint - * @param assetId (connector-unique) identifier of the asset + * @param assetId (connector-unique) identifier of the asset * @return a collection of contract options to access the given asset * @throws IOException in case that the remote call did not succeed */ - DcatCatalog findContractOffers(String remoteControlPlaneIdsUrl, String assetId) throws IOException; + public DcatCatalog findContractOffers(String remoteControlPlaneIdsUrl, String assetId) throws IOException { + QuerySpec findAsset = QuerySpec.Builder.newInstance().filter( + List.of(new Criterion("https://w3id.org/edc/v0.0.1/ns/id", "=", assetId)) + ).build(); + return getCatalog(remoteControlPlaneIdsUrl, findAsset); + } /** * Access the catalogue * * @param remoteControlPlaneIdsUrl url of the remote control plane ids endpoint - * @param spec query specification + * @param spec query specification * @return catalog object * @throws IOException in case something went wrong */ - DcatCatalog getCatalog(String remoteControlPlaneIdsUrl, QuerySpec spec) throws IOException; + public DcatCatalog getCatalog(String remoteControlPlaneIdsUrl, QuerySpec spec) throws IOException { + var url = String.format(CATALOG_CALL, config.getControlPlaneManagementUrl()); + var catalogSpec = String.format(CATALOG_REQUEST_BODY, String.format(DSP_PATH, remoteControlPlaneIdsUrl), objectMapper.writeValueAsString(spec)); + + var request = new Request.Builder().url(url).post(RequestBody.create(catalogSpec, MediaType.parse("application/json"))); + config.getControlPlaneManagementHeaders().forEach(request::addHeader); + + try (var response = httpClient.newCall(request.build()).execute()) { + var body = response.body(); + + if (!response.isSuccessful() || body == null) { + throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); + } + + return JsonLd.processCatalog(body.string()); + } catch (Exception e) { + monitor.severe(format("Error in calling the control plane at %s", url), e); + throw e; + } + } /** * Access the (provider control plane) catalogue @@ -64,24 +274,111 @@ public interface DataManagement { * @return catalog object * @throws IOException in case something went wrong */ - List listAssets(QuerySpec spec) throws IOException; + public List listAssets(QuerySpec spec) throws IOException { + + String version = "/v3"; + var url = String.format(ASSET_CALL, config.getControlPlaneManagementProviderUrl(), version); + var assetObject = (ObjectNode) objectMapper.readTree(objectMapper.writeValueAsString(spec)); + assetObject.set("@context", objectMapper.createObjectNode()); + var assetSpec = objectMapper.writeValueAsString(assetObject); + + var request = new Request.Builder().url(url).post(RequestBody.create(assetSpec, MediaType.parse("application/json"))); + config.getControlPlaneManagementHeaders().forEach(request::addHeader); + + try (var response = httpClient.newCall(request.build()).execute()) { + var body = response.body(); + + if (!response.isSuccessful() || body == null) { + throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); + } + + return JsonLd.processAssetList(body.string()); + } catch (Exception e) { + monitor.severe(format("Error in calling the control plane at %s", url), e); + throw e; + } + } + + /** + * helper to create or update assets + * + * @param assetSpec json text of the asset description + * @return a response listing the id of the created/updated asset + * @throws IOException in case something goes wrong + */ + protected IdResponse createOrUpdateAsset(String assetId, String assetSpec) throws IOException { + String version = "/v3"; + var url = String.format(ASSET_CREATE_CALL, config.getControlPlaneManagementProviderUrl(), version); + var request = new Request.Builder().url(url).post(RequestBody.create(assetSpec, MediaType.parse("application/json"))); + config.getControlPlaneManagementHeaders().forEach(request::addHeader); + + try (var response = httpClient.newCall(request.build()).execute()) { + ResponseBody body = response.body(); + + if (!response.isSuccessful() || body == null) { + + if (response.code() != 409 || body == null) { + throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); + } + + url = String.format(ASSET_UPDATE_CALL, config.getControlPlaneManagementProviderUrl(), version, assetId); + var patchRequest = new Request.Builder().url(url).patch(RequestBody.create(assetSpec, MediaType.parse("application/json"))); + config.getControlPlaneManagementHeaders().forEach(patchRequest::addHeader); + + try (var patchResponse = httpClient.newCall(patchRequest.build()).execute()) { + body = patchResponse.body(); + if (!patchResponse.isSuccessful() || body == null) { + monitor.warning(format("Failure in updating the resource at %s. Ignoring", url)); + return null; + } + } + } + return JsonLd.processIdResponse(body.string()); + } catch (Exception e) { + monitor.severe(format("Error in calling the control plane at %s", url), e); + throw e; + } + } /** * creates or updates a given skill asset * - * @param assetId key - * @param name of skill - * @param description of skill - * @param version of skill - * @param contract of skill - * @param ontologies of skill - * @param distributionMode of skill - * @param isFederated whether it should be distributed - * @param query of skill + * @param assetId key + * @param name of skill + * @param description of skill + * @param version of skill + * @param contract of skill + * @param ontologies of skill + * @param distributionMode of skill + * @param isFederated whether it should be distributed + * @param query of skill + * @param allowServicePattern option allow service pattern + * @param denyServicePattern optional deny service pattern * @return idresponse * @throws IOException in case interaction with EDC went wrong */ - IdResponse createOrUpdateSkill(String assetId, String name, String description, String version, String contract, String ontologies, String distributionMode, boolean isFederated, String query) throws IOException; + public IdResponse createOrUpdateSkill(String assetId, String name, String description, String version, String contract, + String ontologies, String distributionMode, boolean isFederated, String query, String allowServicePattern, + String denyServicePattern) throws IOException { + if (contract != null) { + contract = String.format(" \"cx-common:publishedUnderContract\": \"%1$s\",\n", contract); + } else { + contract = ""; + } + String body = SKILL_ASSET_CREATE_BODY; + + if (allowServicePattern == null) { + allowServicePattern = config.getServiceAllowPattern().pattern(); + } + if (denyServicePattern == null) { + denyServicePattern = config.getServiceDenyPattern().pattern(); + } + + var assetSpec = String.format(body, assetId, name, description, version, contract, ontologies, distributionMode, + isFederated, query, allowServicePattern, denyServicePattern); + + return createOrUpdateAsset(assetId, assetSpec); + } /** * creates or updates a given graph asset @@ -94,10 +391,21 @@ public interface DataManagement { * @param ontologies of graph * @param shape of graph * @param isFederated whether it should be distributed + * @param allowServicePattern option allow service pattern + * @param denyServicePattern optional deny service pattern * @return idresponse * @throws IOException in case interaction with EDC went wrong */ - IdResponse createOrUpdateGraph(String assetId, String name, String description, String version, String contract, String ontologies, String shape, boolean isFederated) throws IOException; + public IdResponse createOrUpdateGraph(String assetId, String name, String description, String version, String contract, String ontologies, String shape, boolean isFederated, String allowServicePattern, String denyServicePattern) throws IOException { + if (contract != null) { + contract = String.format(" \"cx-common:publishedUnderContract\": \"%1$s\",\n", contract); + } else { + contract = ""; + } + String body = GRAPH_ASSET_CREATE_BODY; + var assetSpec = String.format(body, assetId, name, description, version, contract, ontologies, shape, isFederated, allowServicePattern, denyServicePattern); + return createOrUpdateAsset(assetId, assetSpec); + } /** * deletes an existing aseet @@ -106,7 +414,24 @@ public interface DataManagement { * @return idresponse */ - IdResponse deleteAsset(String assetId) throws IOException; + public IdResponse deleteAsset(String assetId) throws IOException { + String version = "/v3"; + var url = String.format(ASSET_UPDATE_CALL, config.getControlPlaneManagementProviderUrl(), version, assetId); + var request = new Request.Builder().url(url).delete(); + config.getControlPlaneManagementHeaders().forEach(request::addHeader); + try (var response = httpClient.newCall(request.build()).execute()) { + ResponseBody body = response.body(); + if (response.isSuccessful() && body != null) { + return JsonLd.processIdResponse(body.string()); + } else { + monitor.warning(format("Failure in calling the control plane at %s. Ignoring", url)); + return null; + } + } catch (Exception e) { + monitor.severe(format("Error in calling the control plane at %s", url), e); + throw e; + } + } /** * initiates negotation @@ -115,7 +440,45 @@ public interface DataManagement { * @return negotiation id * @throws IOException in case something went wronf */ - String initiateNegotiation(ContractNegotiationRequest negotiationRequest) throws IOException; + public String initiateNegotiation(ContractNegotiationRequest negotiationRequest) throws IOException { + var url = String.format(NEGOTIATION_INITIATE_CALL, config.getControlPlaneManagementUrl()); + + // use a version specific call + String template = NEGOTIATION_REQUEST_BODY; + + var negotiateSpec = String.format(template, + negotiationRequest.getConnectorAddress(), + negotiationRequest.getLocalBusinessPartnerNumber(), + negotiationRequest.getRemoteBusinessPartnerNumber(), + negotiationRequest.getOffer().getOfferId(), + negotiationRequest.getOffer().getAssetId(), + negotiationRequest.getOffer().getPolicy().asString()); + + var requestBody = RequestBody.create(negotiateSpec, MediaType.parse("application/json")); + + var request = new Request.Builder() + .url(url) + .post(requestBody); + config.getControlPlaneManagementHeaders().forEach(request::addHeader); + + try (var response = httpClient.newCall(request.build()).execute()) { + var body = response.body(); + + if (!response.isSuccessful() || body == null) { + throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); + } + + + var negotiationId = JsonLd.processIdResponse(body.string()).getId(); + + monitor.debug("Started negotiation with ID: " + negotiationId); + + return negotiationId; + } catch (Exception e) { + monitor.severe(format("Error in calling the control plane at %s", url), e); + throw e; + } + } /** * return state of contract negotiation @@ -124,16 +487,58 @@ public interface DataManagement { * @return status of the negotiation * @throws IOException in case something went wrong */ - ContractNegotiation getNegotiation(String negotiationId) throws IOException; + public ContractNegotiation getNegotiation(String negotiationId) throws IOException { + var url = String.format(NEGOTIATION_CHECK_CALL, config.getControlPlaneManagementUrl(), negotiationId); + var request = new Request.Builder() + .url(url); + config.getControlPlaneManagementHeaders().forEach(request::addHeader); + + try (var response = httpClient.newCall(request.build()).execute()) { + var body = response.body(); + + if (!response.isSuccessful() || body == null) { + throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); + } + + var negotiation = JsonLd.processContractNegotiation(body.string()); + monitor.debug(format("Negotiation %s is in state '%s' (agreementId: %s)", negotiationId, negotiation.getState(), negotiation.getContractAgreementId())); + + return negotiation; + } catch (Exception e) { + monitor.severe(format("Error in calling the Control plane at %s", url), e); + throw e; + } + } /** - * return the contract agreement + * get a contract agreement by its id * * @param agreementId id of the agreement * @return contract agreement * @throws IOException something wild happens */ - ContractAgreement getAgreement(String agreementId) throws IOException; + public ContractAgreement getAgreement(String agreementId) throws IOException { + var url = String.format(AGREEMENT_CHECK_CALL, config.getControlPlaneManagementUrl(), URLEncoder.encode(agreementId, StandardCharsets.UTF_8)); + var request = new Request.Builder() + .url(url); + config.getControlPlaneManagementHeaders().forEach(request::addHeader); + + try (var response = httpClient.newCall(request.build()).execute()) { + var body = response.body(); + + if (!response.isSuccessful() || body == null) { + throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); + } + + var agreement = JsonLd.processContractAgreement(body.string()); + monitor.debug(format("Agreement %s found for asset %s", agreementId, agreement.getAssetId())); + + return agreement; + } catch (Exception e) { + monitor.severe(format("Error in calling the Control plane at %s", url), e); + throw e; + } + } /** * Initiates a transfer @@ -142,7 +547,42 @@ public interface DataManagement { * @return transfer id * @throws IOException in case something went wrong */ - String initiateHttpProxyTransferProcess(TransferRequest transferRequest) throws IOException; + public String initiateHttpProxyTransferProcess(TransferRequest transferRequest) throws IOException { + var url = String.format(TRANSFER_INITIATE_CALL, config.getControlPlaneManagementUrl()); + + var transferSpec = String.format(TRANSFER_REQUEST_BODY, + transferRequest.getAssetId(), + transferRequest.getConnectorAddress(), + transferRequest.getContractId(), + transferRequest.getCallbackAddresses().get(0).getUri(), + transferRequest.getConnectorAddress()); + + var requestBody = RequestBody.create(transferSpec, MediaType.parse("application/json")); + + var request = new Request.Builder() + .url(url) + .post(requestBody); + config.getControlPlaneManagementHeaders().forEach(request::addHeader); + + try (var response = httpClient.newCall(request.build()).execute()) { + var body = response.body(); + + if (!response.isSuccessful() || body == null) { + throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); + } + + // For debugging purposes: + // var transferProcessId = TransferId.Builder.newInstance().id(body.string()).build(); + var transferProcessId = JsonLd.processIdResponse(body.string()).getId(); + + monitor.debug(format("Transfer process (%s) initiated", transferProcessId)); + + return transferProcessId; + } catch (Exception e) { + monitor.severe(format("Error in calling the control plane at %s", url), e); + throw e; + } + } /** * return state of transfer process @@ -151,6 +591,27 @@ public interface DataManagement { * @return state of the transfer process * @throws IOException in case something went wrong */ - TransferProcess getTransfer(String transferProcessId) throws IOException; + public TransferProcess getTransfer(String transferProcessId) throws IOException { + var url = String.format(TRANSFER_CHECK_CALL, config.getControlPlaneManagementUrl(), transferProcessId); + var request = new Request.Builder() + .url(url); + config.getControlPlaneManagementHeaders().forEach(request::addHeader); + + try (var response = httpClient.newCall(request.build()).execute()) { + var body = response.body(); + + if (!response.isSuccessful() || body == null) { + throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); + } + + var process = JsonLd.processTransferProcess(body.string()); + monitor.info(format("Transfer %s is in state '%s'", transferProcessId, process.getState())); + + return process; + } catch (Exception e) { + monitor.severe(format("Error in calling the Control plane at %s", url), e); + throw e; + } + } } \ No newline at end of file diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagementImpl.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagementImpl.java deleted file mode 100644 index 9b0b4547..00000000 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagementImpl.java +++ /dev/null @@ -1,683 +0,0 @@ -// Copyright (c) 2022,2024 Contributors to the Eclipse Foundation -// -// See the NOTICE file(s) distributed with this work for additional -// information regarding copyright ownership. -// -// This program and the accompanying materials are made available under the -// terms of the Apache License, Version 2.0 which is available at -// https://www.apache.org/licenses/LICENSE-2.0. -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations -// under the License. -// -// SPDX-License-Identifier: Apache-2.0 -package org.eclipse.tractusx.agents.service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import jakarta.ws.rs.InternalServerErrorException; -import okhttp3.MediaType; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.ResponseBody; -import org.eclipse.tractusx.agents.AgentConfig; -import org.eclipse.tractusx.agents.jsonld.JsonLd; -import org.eclipse.tractusx.agents.matchmaking.SharedObjectManager; -import org.eclipse.tractusx.agents.model.Asset; -import org.eclipse.tractusx.agents.model.ContractAgreement; -import org.eclipse.tractusx.agents.model.ContractNegotiation; -import org.eclipse.tractusx.agents.model.ContractNegotiationRequest; -import org.eclipse.tractusx.agents.model.DcatCatalog; -import org.eclipse.tractusx.agents.model.IdResponse; -import org.eclipse.tractusx.agents.model.TransferProcess; -import org.eclipse.tractusx.agents.model.TransferRequest; -import org.eclipse.tractusx.agents.utils.Criterion; -import org.eclipse.tractusx.agents.utils.Monitor; -import org.eclipse.tractusx.agents.utils.QuerySpec; -import org.eclipse.tractusx.agents.utils.TypeManager; - -import java.io.IOException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.List; - -import static java.lang.String.format; - -/** - * DataManagement - * is a service wrapper around the management endpoint - * of the EDC control plane - */ -public class DataManagementImpl implements DataManagement { - /** - * some constants when interacting with control plane - */ - public static final String DSP_PATH = "%s/api/v1/dsp"; - public static final String CATALOG_CALL = "%s/v2/catalog/request"; - public static final String CATALOG_REQUEST_BODY = "{" + - "\"@context\": {}," + - "\"protocol\": \"dataspace-protocol-http\"," + - "\"providerUrl\": \"%1$s\", " + - "\"counterPartyAddress\": \"%1$s\", " + - "\"querySpec\": %2$s }"; - - public static final String ASSET_CREATE_CALL = "%s%s/assets"; - public static final String ASSET_UPDATE_CALL = "%s%s/assets/%s"; - - /** - * template for skill asset creation - */ - public static final String SKILL_ASSET_CREATE_BODY = "{\n" + - " \"@context\": {\n" + - " \"rdf\": \"http://www.w3.org/1999/02/22-rdf-syntax-ns#\",\n" + - " \"rdfs\": \"http://www.w3.org/2000/01/rdf-schema#\",\n" + - " \"cx-common\": \"https://w3id.org/catenax/ontology/common#\",\n" + - " \"xsd\": \"http://www.w3.org/2001/XMLSchema#\",\n" + - " \"sh\": \"http://www.w3.org/ns/shacl#\"\n" + - " },\n" + - " \"asset\": {\n" + - " \"@type\": \"Asset\",\n" + - " \"@id\": \"%1$s\", \n" + - " \"properties\": {\n" + - " \"name\": \"%2$s\",\n" + - " \"description\": \"%3$s\",\n" + - " \"version\": \"%4$s\",\n" + - " \"contenttype\": \"application/json, application/xml\",\n" + - "%5$s" + - " \"rdf:type\": \"cx-common:SkillAsset\",\n" + - " \"rdfs:isDefinedBy\": \"%6$s\",\n" + - " \"cx-common:implementsProtocol\": \"cx-common:Protocol?w3c:http:SKILL\",\n" + - " \"cx-common:distributionMode\": \"%7$s\",\n" + - " \"cx-common:isFederated\": \"%8$b^^xsd:boolean\"\n" + - " },\n" + - " \"privateProperties\": {\n" + - " \"cx-common:query\":%9$s\n" + - " }\n" + - " },\n" + - " \"dataAddress\": {\n" + - " \"id\":\"%1$s\",\n" + - " \"@type\": \"DataAddress\",\n" + - " \"type\": \"cx-common:Protocol?w3c:http:SKILL\",\n" + - " \"baseUrl\":\"https://w3id.org/catenax\",\n" + - " \"proxyPath\": \"false\",\n" + - " \"proxyMethod\": \"true\",\n" + - " \"proxyQueryParams\": \"true\",\n" + - " \"proxyBody\": \"true\"\n" + - " }\n" + - "}\n"; - - public static final String SKILL_ASSET_CREATE_BODY_V3 = "{\n" + - " \"@context\": {\n" + - " \"rdf\": \"http://www.w3.org/1999/02/22-rdf-syntax-ns#\",\n" + - " \"rdfs\": \"http://www.w3.org/2000/01/rdf-schema#\",\n" + - " \"cx-common\": \"https://w3id.org/catenax/ontology/common#\",\n" + - " \"sh\": \"http://www.w3.org/ns/shacl#\"\n" + - " },\n" + - " \"@id\": \"%1$s\", \n" + - " \"properties\": {\n" + - " \"name\": \"%2$s\",\n" + - " \"description\": \"%3$s\",\n" + - " \"version\": \"%4$s\",\n" + - " \"contenttype\": \"application/json, application/xml\",\n" + - "%5$s" + - " \"rdf:type\": \"cx-common:SkillAsset\",\n" + - " \"rdfs:isDefinedBy\": \"%6$s\",\n" + - " \"cx-common:implementsProtocol\": \"cx-common:Protocol?w3c:http:SKILL\",\n" + - " \"cx-common:distributionMode\": \"%7$s\",\n" + - " \"cx-common:isFederated\": \"%8$b^^xsd:boolean\"\n" + - " },\n" + - " \"privateProperties\": {\n" + - " \"cx-common:query\":%9$s\n" + - " },\n" + - " \"dataAddress\": {\n" + - " \"id\":\"%1$s\",\n" + - " \"@type\": \"DataAddress\",\n" + - " \"type\": \"cx-common:Protocol?w3c:http:SKILL\",\n" + - " \"baseUrl\":\"https://w3id.org/catenax\",\n" + - " \"proxyPath\": \"false\",\n" + - " \"proxyMethod\": \"true\",\n" + - " \"proxyQueryParams\": \"true\",\n" + - " \"proxyBody\": \"true\"\n" + - " }\n" + - "}\n"; - - - /** - * template for graph asset creation - */ - public static final String GRAPH_ASSET_CREATE_BODY = "{\n" + - " \"@context\": {\n" + - " \"rdf\": \"http://www.w3.org/1999/02/22-rdf-syntax-ns#\",\n" + - " \"rdfs\": \"http://www.w3.org/2000/01/rdf-schema#\",\n" + - " \"cx-common\": \"https://w3id.org/catenax/ontology/common#\",\n" + - " \"xsd\": \"http://www.w3.org/2001/XMLSchema#\",\n" + - " \"sh\": \"http://www.w3.org/ns/shacl#\"\n" + - " },\n" + - " \"asset\": {\n" + - " \"@type\": \"Asset\",\n" + - " \"@id\": \"%1$s\", \n" + - " \"properties\": {\n" + - " \"name\": \"%2$s\",\n" + - " \"description\": \"%3$s\",\n" + - " \"version\": \"%4$s\",\n" + - " \"contenttype\": \"application/json, application/xml\",\n" + - "%5$s" + - " \"rdf:type\": \"cx-common:GraphAsset\",\n" + - " \"rdfs:isDefinedBy\": \"%6$s\",\n" + - " \"cx-common:implementsProtocol\": \"cx-common:Protocol?w3c:http:SPARQL\",\n" + - " \"sh:shapesGraph\": \"%7$s\",\n" + - " \"cx-common:isFederated\": \"%8$b^^xsd:boolean\"\n" + - " }\n" + - " },\n" + - " \"dataAddress\": {\n" + - " \"id\": \"%1$s\", \n" + - " \"@type\": \"DataAddress\",\n" + - " \"type\": \"cx-common:Protocol?w3c:http:SPARQL\",\n" + - " \"baseUrl\":\"https://w3id.org/catenax\",\n" + - " \"proxyPath\": \"false\",\n" + - " \"proxyMethod\": \"true\",\n" + - " \"proxyQueryParams\": \"true\",\n" + - " \"proxyBody\": \"true\"\n" + - " }\n" + - "}\n"; - - public static final String GRAPH_ASSET_CREATE_BODY_V3 = "{\n" + - " \"@context\": {\n" + - " \"rdf\": \"http://www.w3.org/1999/02/22-rdf-syntax-ns#\",\n" + - " \"rdfs\": \"http://www.w3.org/2000/01/rdf-schema#\",\n" + - " \"cx-common\": \"https://w3id.org/catenax/ontology/common#\",\n" + - " \"xsd\": \"http://www.w3.org/2001/XMLSchema#\",\n" + - " \"sh\": \"http://www.w3.org/ns/shacl#\"\n" + - " },\n" + - " \"@id\": \"%1$s\", \n" + - " \"properties\": {\n" + - " \"name\": \"%2$s\",\n" + - " \"description\": \"%3$s\",\n" + - " \"version\": \"%4$s\",\n" + - " \"contenttype\": \"application/json, application/xml\",\n" + - "%5$s" + - " \"rdf:type\": \"cx-common:GraphAsset\",\n" + - " \"rdfs:isDefinedBy\": \"%6$s\",\n" + - " \"cx-common:implementsProtocol\": \"cx-common:Protocol?w3c:http:SPARQL\",\n" + - " \"sh:shapesGraph\": \"%7$s\",\n" + - " \"cx-common:isFederated\": \"%8$b^^xsd:boolean\"\n" + - " },\n" + - " \"dataAddress\": {\n" + - " \"id\": \"%1$s\", \n" + - " \"@type\": \"DataAddress\",\n" + - " \"type\": \"cx-common:Protocol?w3c:http:SPARQL\",\n" + - " \"baseUrl\":\"https://w3id.org/catenax\",\n" + - " \"proxyPath\": \"false\",\n" + - " \"proxyMethod\": \"true\",\n" + - " \"proxyQueryParams\": \"true\",\n" + - " \"proxyBody\": \"true\"\n" + - " }\n" + - "}\n"; - - public static final String ASSET_CALL = "%s%s/assets/request"; - - // negotiation request 0.5.>=1 - public static final String NEGOTIATION_REQUEST_BODY = "{\n" + - "\"@context\": { \"odrl\": \"http://www.w3.org/ns/odrl/2/\"},\n" + - "\"@type\": \"NegotiationInitiateRequestDto\",\n" + - "\"connectorAddress\": \"%1$s\",\n" + - "\"protocol\": \"dataspace-protocol-http\",\n" + - "\"providerId\": \"%3$s\",\n" + - "\"connectorId\": \"%2$s\",\n" + - "\"offer\": {\n" + - " \"offerId\": \"%4$s\",\n" + - " \"assetId\": \"%5$s\",\n" + - " \"policy\": %6$s\n" + - "}\n" + - "}"; - - // negotiation request 0.5.0 - roles of provider and connector are wrong - public static final String NEGOTIATION_REQUEST_BODY_PRERELEASE = "{\n" + - "\"@context\": { \"odrl\": \"http://www.w3.org/ns/odrl/2/\"},\n" + - "\"@type\": \"NegotiationInitiateRequestDto\",\n" + - "\"connectorAddress\": \"%1$s\",\n" + - "\"protocol\": \"dataspace-protocol-http\",\n" + - "\"providerId\": \"%2$s\",\n" + - "\"connectorId\": \"%3$s\",\n" + - "\"offer\": {\n" + - " \"offerId\": \"%4$s\",\n" + - " \"assetId\": \"%5$s\",\n" + - " \"policy\": %6$s\n" + - "}\n" + - "}"; - - - public static final String NEGOTIATION_INITIATE_CALL = "%s/v2/contractnegotiations"; - public static final String NEGOTIATION_CHECK_CALL = "%s/v2/contractnegotiations/%s"; - public static final String TRANSFER_INITIATE_CALL = "%s/v2/transferprocesses"; - - public static final String TRANSFER_REQUEST_BODY = "{\n" + - " \"@context\": {\n" + - " \"odrl\": \"http://www.w3.org/ns/odrl/2/\"\n" + - " },\n" + - " \"assetId\": \"%1$s\",\n" + - " \"connectorAddress\": \"%2$s\",\n" + - " \"connectorId\": \"%5$s\",\n" + - " \"contractId\": \"%3$s\",\n" + - " \"dataDestination\": {\n" + - " \"type\": \"HttpProxy\"\n" + - " },\n" + - " \"managedResources\": false,\n" + - " \"privateProperties\": {\n" + - " \"receiverHttpEndpoint\": \"%4$s\"\n" + - " },\n" + - " \"protocol\": \"dataspace-protocol-http\",\n" + - " \"transferType\": {\n" + - " \"contentType\": \"application/octet-stream\",\n" + - " \"isFinite\": true\n" + - " }\n" + - "}"; - public static final String TRANSFER_CHECK_CALL = "%s/v2/transferprocesses/%s"; - public static final String AGREEMENT_CHECK_CALL = "%s/v2/contractagreements/%s"; - - /** - * references to EDC services - */ - private final Monitor monitor; - private final ObjectMapper objectMapper; - private final OkHttpClient httpClient; - private final AgentConfig config; - - /** - * creates a service wrapper - * - * @param monitor logger - * @param typeManager serialization - * @param httpClient remoting - * @param config typed config - */ - public DataManagementImpl(Monitor monitor, TypeManager typeManager, OkHttpClient httpClient, AgentConfig config) { - this.monitor = monitor; - this.objectMapper = typeManager.getMapper(); - this.httpClient = httpClient; - this.config = config; - } - - /** - * Search for a dedicated asset - * TODO imperformant - * TODO replace by accessing the federated data catalogue - * - * @param remoteControlPlaneIdsUrl url of the remote control plane ids endpoint - * @param assetId (connector-unique) identifier of the asset - * @return a collection of contract options to access the given asset - * @throws IOException in case that the remote call did not succeed - */ - public DcatCatalog findContractOffers(String remoteControlPlaneIdsUrl, String assetId) throws IOException { - QuerySpec findAsset = QuerySpec.Builder.newInstance().filter( - List.of(new Criterion("https://w3id.org/edc/v0.0.1/ns/id", "=", assetId)) - ).build(); - return getCatalog(remoteControlPlaneIdsUrl, findAsset); - } - - /** - * Access the catalogue - * - * @param remoteControlPlaneIdsUrl url of the remote control plane ids endpoint - * @param spec query specification - * @return catalog object - * @throws IOException in case something went wrong - */ - public DcatCatalog getCatalog(String remoteControlPlaneIdsUrl, QuerySpec spec) throws IOException { - - var url = String.format(CATALOG_CALL, config.getControlPlaneManagementUrl()); - var catalogSpec = String.format(CATALOG_REQUEST_BODY, String.format(DSP_PATH, remoteControlPlaneIdsUrl), objectMapper.writeValueAsString(spec)); - - var request = new Request.Builder().url(url).post(RequestBody.create(catalogSpec, MediaType.parse("application/json"))); - config.getControlPlaneManagementHeaders().forEach(request::addHeader); - //request.addHeader("x-api-key", "foo"); - monitor.debug("sending " + "http://oem-control-plane2:8181", null); - monitor.debug(SharedObjectManager.getInstance().convertToCurl(request.build())); - try (var response = httpClient.newCall(request.build()).execute()) { - var body = response.body(); - - if (!response.isSuccessful() || body == null) { - throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); - } - - return JsonLd.processCatalog(body.string()); - } catch (Exception e) { - monitor.severe(format("Error in calling the control plane at %s", url), e); - throw e; - } - } - - /** - * Access the (provider control plane) catalogue - * - * @param spec query specification - * @return catalog object - * @throws IOException in case something went wrong - */ - public List listAssets(QuerySpec spec) throws IOException { - - String version = config.isPrerelease() ? "/v2" : "/v3"; - var url = String.format(ASSET_CALL, config.getControlPlaneManagementProviderUrl(), version); - var assetObject = (ObjectNode) objectMapper.readTree(objectMapper.writeValueAsString(spec)); - assetObject.set("@context", objectMapper.createObjectNode()); - var assetSpec = objectMapper.writeValueAsString(assetObject); - - var request = new Request.Builder().url(url).post(RequestBody.create(assetSpec, MediaType.parse("application/json"))); - config.getControlPlaneManagementHeaders().forEach(request::addHeader); - - try (var response = httpClient.newCall(request.build()).execute()) { - var body = response.body(); - - if (!response.isSuccessful() || body == null) { - throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); - } - - return JsonLd.processAssetList(body.string()); - } catch (Exception e) { - monitor.severe(format("Error in calling the control plane at %s", url), e); - throw e; - } - } - - /** - * helper to create or update assets - * - * @param assetSpec json text of the asset description - * @return a response listing the id of the created/updated asset - * @throws IOException in case something goes wrong - */ - protected IdResponse createOrUpdateAsset(String assetId, String assetSpec) throws IOException { - String version = config.isPrerelease() ? "/v2" : "/v3"; - var url = String.format(ASSET_CREATE_CALL, config.getControlPlaneManagementProviderUrl(), version); - var request = new Request.Builder().url(url).post(RequestBody.create(assetSpec, MediaType.parse("application/json"))); - config.getControlPlaneManagementHeaders().forEach(request::addHeader); - - try (var response = httpClient.newCall(request.build()).execute()) { - ResponseBody body = response.body(); - - if (!response.isSuccessful() || body == null) { - - if (response.code() != 409 || body == null) { - throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); - } - - url = String.format(ASSET_UPDATE_CALL, config.getControlPlaneManagementProviderUrl(), version, assetId); - var patchRequest = new Request.Builder().url(url).patch(RequestBody.create(assetSpec, MediaType.parse("application/json"))); - config.getControlPlaneManagementHeaders().forEach(patchRequest::addHeader); - - try (var patchResponse = httpClient.newCall(patchRequest.build()).execute()) { - body = patchResponse.body(); - if (!patchResponse.isSuccessful() || body == null) { - monitor.warning(format("Failure in updating the resource at %s. Ignoring", url)); - return null; - } - } - } - return JsonLd.processIdResponse(body.string()); - } catch (Exception e) { - monitor.severe(format("Error in calling the control plane at %s", url), e); - throw e; - } - } - - /** - * creates or updates a given skill asset - * - * @param assetId key - * @param name of skill - * @param description of skill - * @param version of skill - * @param contract of skill - * @param ontologies of skill - * @param distributionMode of skill - * @param isFederated whether it should be distributed - * @param query of skill - * @return idresponse - * @throws IOException in case interaction with EDC went wrong - */ - public IdResponse createOrUpdateSkill(String assetId, String name, String description, String version, String contract, String ontologies, String distributionMode, boolean isFederated, String query) throws IOException { - if (contract != null) { - contract = String.format(" \"cx-common:publishedUnderContract\": \"%1$s\",\n", contract); - } else { - contract = ""; - } - String body = config.isPrerelease() ? SKILL_ASSET_CREATE_BODY : SKILL_ASSET_CREATE_BODY_V3; - var assetSpec = String.format(body, assetId, name, description, version, contract, ontologies, distributionMode, isFederated, query); - return createOrUpdateAsset(assetId, assetSpec); - } - - /** - * creates or updates a given graph asset - * - * @param assetId key - * @param name of graph - * @param description of graph - * @param version of graph - * @param contract of graph - * @param ontologies of graph - * @param shape of graph - * @param isFederated whether it should be distributed - * @return idresponse - * @throws IOException in case interaction with EDC went wrong - */ - public IdResponse createOrUpdateGraph(String assetId, String name, String description, String version, String contract, String ontologies, String shape, boolean isFederated) throws IOException { - if (contract != null) { - contract = String.format(" \"cx-common:publishedUnderContract\": \"%1$s\",\n", contract); - } else { - contract = ""; - } - String body = config.isPrerelease() ? GRAPH_ASSET_CREATE_BODY : GRAPH_ASSET_CREATE_BODY_V3; - var assetSpec = String.format(body, assetId, name, description, version, contract, ontologies, shape, isFederated); - return createOrUpdateAsset(assetId, assetSpec); - } - - /** - * deletes an existing aseet - * - * @param assetId key of the asset - * @return idresponse - */ - - public IdResponse deleteAsset(String assetId) throws IOException { - String version = config.isPrerelease() ? "/v2" : "/v3"; - var url = String.format(ASSET_UPDATE_CALL, config.getControlPlaneManagementProviderUrl(), version, assetId); - var request = new Request.Builder().url(url).delete(); - config.getControlPlaneManagementHeaders().forEach(request::addHeader); - try (var response = httpClient.newCall(request.build()).execute()) { - ResponseBody body = response.body(); - if (response.isSuccessful() && body != null) { - return JsonLd.processIdResponse(body.string()); - } else { - monitor.warning(format("Failure in calling the control plane at %s. Ignoring", url)); - return null; - } - } catch (Exception e) { - monitor.severe(format("Error in calling the control plane at %s", url), e); - throw e; - } - } - - /** - * initiates negotation - * - * @param negotiationRequest outgoing request - * @return negotiation id - * @throws IOException in case something went wronf - */ - public String initiateNegotiation(ContractNegotiationRequest negotiationRequest) throws IOException { - var url = String.format(NEGOTIATION_INITIATE_CALL, config.getControlPlaneManagementUrl()); - - String template = config.isPrerelease() ? NEGOTIATION_REQUEST_BODY_PRERELEASE : NEGOTIATION_REQUEST_BODY; - var negotiateSpec = String.format(template, - negotiationRequest.getConnectorAddress(), - negotiationRequest.getLocalBusinessPartnerNumber(), - negotiationRequest.getRemoteBusinessPartnerNumber(), - negotiationRequest.getOffer().getOfferId(), - negotiationRequest.getOffer().getAssetId(), - negotiationRequest.getOffer().getPolicy().asString()); - - var requestBody = RequestBody.create(negotiateSpec, MediaType.parse("application/json")); - - var request = new Request.Builder() - .url(url) - .post(requestBody); - config.getControlPlaneManagementHeaders().forEach(request::addHeader); - - try (var response = httpClient.newCall(request.build()).execute()) { - var body = response.body(); - - if (!response.isSuccessful() || body == null) { - throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); - } - - - var negotiationId = JsonLd.processIdResponse(body.string()).getId(); - - monitor.debug("Started negotiation with ID: " + negotiationId); - - return negotiationId; - } catch (Exception e) { - monitor.severe(format("Error in calling the control plane at %s", url), e); - throw e; - } - } - - /** - * return state of contract negotiation - * - * @param negotiationId id of the negotation to inbestigate - * @return status of the negotiation - * @throws IOException in case something went wrong - */ - public ContractNegotiation getNegotiation(String negotiationId) throws IOException { - var url = String.format(NEGOTIATION_CHECK_CALL, config.getControlPlaneManagementUrl(), negotiationId); - var request = new Request.Builder() - .url(url); - config.getControlPlaneManagementHeaders().forEach(request::addHeader); - - try (var response = httpClient.newCall(request.build()).execute()) { - var body = response.body(); - - if (!response.isSuccessful() || body == null) { - throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); - } - - var negotiation = JsonLd.processContractNegotiation(body.string()); - monitor.debug(format("Negotiation %s is in state '%s' (agreementId: %s)", negotiationId, negotiation.getState(), negotiation.getContractAgreementId())); - - return negotiation; - } catch (Exception e) { - monitor.severe(format("Error in calling the Control plane at %s", url), e); - throw e; - } - } - - /** - * get a contract agreement by its id - * - * @param agreementId id of the agreement - * @return contract agreement - * @throws IOException something wild happens - */ - public ContractAgreement getAgreement(String agreementId) throws IOException { - var url = String.format(AGREEMENT_CHECK_CALL, config.getControlPlaneManagementUrl(), URLEncoder.encode(agreementId, StandardCharsets.UTF_8)); - var request = new Request.Builder() - .url(url); - config.getControlPlaneManagementHeaders().forEach(request::addHeader); - - try (var response = httpClient.newCall(request.build()).execute()) { - var body = response.body(); - - if (!response.isSuccessful() || body == null) { - throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); - } - - var agreement = JsonLd.processContractAgreement(body.string()); - monitor.debug(format("Agreement %s found for asset %s", agreementId, agreement.getAssetId())); - - return agreement; - } catch (Exception e) { - monitor.severe(format("Error in calling the Control plane at %s", url), e); - throw e; - } - } - - /** - * Initiates a transfer - * - * @param transferRequest request - * @return transfer id - * @throws IOException in case something went wrong - */ - public String initiateHttpProxyTransferProcess(TransferRequest transferRequest) throws IOException { - var url = String.format(TRANSFER_INITIATE_CALL, config.getControlPlaneManagementUrl()); - - var transferSpec = String.format(TRANSFER_REQUEST_BODY, - transferRequest.getAssetId(), - transferRequest.getConnectorAddress(), - transferRequest.getContractId(), - transferRequest.getCallbackAddresses().get(0).getUri(), - transferRequest.getConnectorAddress()); - - var requestBody = RequestBody.create(transferSpec, MediaType.parse("application/json")); - - var request = new Request.Builder() - .url(url) - .post(requestBody); - config.getControlPlaneManagementHeaders().forEach(request::addHeader); - - try (var response = httpClient.newCall(request.build()).execute()) { - var body = response.body(); - - if (!response.isSuccessful() || body == null) { - throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); - } - - // For debugging purposes: - // var transferProcessId = TransferId.Builder.newInstance().id(body.string()).build(); - var transferProcessId = JsonLd.processIdResponse(body.string()).getId(); - - monitor.debug(format("Transfer process (%s) initiated", transferProcessId)); - - return transferProcessId; - } catch (Exception e) { - monitor.severe(format("Error in calling the control plane at %s", url), e); - throw e; - } - } - - /** - * return state of transfer process - * - * @param transferProcessId id of the transfer process - * @return state of the transfer process - * @throws IOException in case something went wrong - */ - public TransferProcess getTransfer(String transferProcessId) throws IOException { - var url = String.format(TRANSFER_CHECK_CALL, config.getControlPlaneManagementUrl(), transferProcessId); - var request = new Request.Builder() - .url(url); - config.getControlPlaneManagementHeaders().forEach(request::addHeader); - - try (var response = httpClient.newCall(request.build()).execute()) { - var body = response.body(); - - if (!response.isSuccessful() || body == null) { - throw new InternalServerErrorException(format("Control plane responded with: %s %s", response.code(), body != null ? body.string() : "")); - } - - var process = JsonLd.processTransferProcess(body.string()); - monitor.info(format("Transfer %s is in state '%s'", transferProcessId, process.getState())); - - return process; - } catch (Exception e) { - monitor.severe(format("Error in calling the Control plane at %s", url), e); - throw e; - } - } - -} \ No newline at end of file diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataspaceSynchronizer.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataspaceSynchronizer.java index 6b832af8..ef3d437c 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataspaceSynchronizer.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataspaceSynchronizer.java @@ -43,42 +43,104 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; /** * A service which keeps a triple store and - * the associated dataspace in sync + * the associated dataspace (partner catalogues) in sync */ public class DataspaceSynchronizer implements Runnable { /** * constants */ - protected static final Node CX_ASSET = NodeFactory.createURI("https://w3id.org/catenax/ontology/common#offers"); - protected static final Map ASSET_PROPERTY_MAP = new HashMap<>(); + + public static final String COMMON_NAMESPACE = "https://w3id.org/catenax/ontology/common#"; + public static final String TAXO_NAMESPACE = "https://w3id.org/catenax/taxonomy#"; + public static final String EDC_NAMESPACE = "https://w3id.org/edc/v0.0.1/ns/"; + public static final String RDF_NAMESPACE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"; + public static final Node RDF_TYPE = createUri(RDF_NAMESPACE, "type"); + public static final String RDF_SCHEMA_NAMESPACE = "http://www.w3.org/2000/01/rdf-schema#"; + public static final String DC_NAMESPACE = "https://purl.org/dc/terms/"; + public static final String DC_TYPE = DC_NAMESPACE + "type"; + public static final String SHACL_NAMESPACE = "http://www.w3.org/ns/shacl#"; + public static final String SHAPES_GRAPH = SHACL_NAMESPACE + "shapesGraph"; + public static final String CX_SCHEMA_NAMESPACE = "https://w3id.org/catenax/ontology/schema#"; + public static final String XML_SCHEMA_NAMESPACE = "http://www.w3.org/2001/XMLSchema#"; + + public static final Node SHAPE_OBJECT = createUri(CX_SCHEMA_NAMESPACE, "shapeObject"); + protected static final Node CX_ASSET = createUri(COMMON_NAMESPACE, "offers"); + protected static final Set EXPECTED_COMPLEX_OBJECTS = new HashSet<>(); protected static final QuerySpec FEDERATED_ASSET_QUERY = QuerySpec.Builder.newInstance() - .filter(List.of(new Criterion("https://w3id.org/catenax/ontology/common#isFederated", "=", "true^^xsd:boolean"))).build(); + .filter(List.of(new Criterion(COMMON_NAMESPACE + "isFederated", "=", "true^^xsd:boolean"))).build(); - protected MonitorWrapper monitorWrapper; + protected static final Map ASSET_PROPERTY_MAP = new HashMap<>(); static { - ASSET_PROPERTY_MAP.put("https://w3id.org/edc/v0.0.1/ns/id", NodeFactory.createURI("https://w3id.org/catenax/ontology/common#id")); - ASSET_PROPERTY_MAP.put("https://w3id.org/edc/v0.0.1/ns/name", NodeFactory.createURI("https://w3id.org/catenax/ontology/common#name")); - ASSET_PROPERTY_MAP.put("https://w3id.org/edc/v0.0.1/ns/description", NodeFactory.createURI("https://w3id.org/catenax/ontology/common#description")); - ASSET_PROPERTY_MAP.put("https://w3id.org/edc/v0.0.1/ns/version", NodeFactory.createURI("https://w3id.org/catenax/ontology/common#version")); - ASSET_PROPERTY_MAP.put("https://w3id.org/edc/v0.0.1/ns/contenttype", NodeFactory.createURI("https://w3id.org/catenax/ontology/common#contentType")); - ASSET_PROPERTY_MAP.put("http://www.w3.org/1999/02/22-rdf-syntax-ns#type", NodeFactory.createURI("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")); - ASSET_PROPERTY_MAP.put("http://www.w3.org/2000/01/rdf-schema#isDefinedBy", NodeFactory.createURI("http://www.w3.org/2000/01/rdf-schema#isDefinedBy")); - ASSET_PROPERTY_MAP.put("https://w3id.org/catenax/ontology/common#implementsProtocol", NodeFactory.createURI("https://w3id.org/catenax/ontology/common#implementsProtocol")); - ASSET_PROPERTY_MAP.put("http://www.w3.org/ns/shacl#shapesGraph", NodeFactory.createURI("http://www.w3.org/ns/shacl#shapesGraph")); - ASSET_PROPERTY_MAP.put("https://w3id.org/catenax/ontology/common#isFederated", NodeFactory.createURI("https://w3id.org/catenax/ontology/common#isFederated")); - ASSET_PROPERTY_MAP.put("https://w3id.org/catenax/ontology/common#publishedUnderContract", NodeFactory.createURI("https://w3id.org/catenax/ontology/common#publishedUnderContract")); - ASSET_PROPERTY_MAP.put("https://w3id.org/catenax/ontology/common#satisfiesRole", NodeFactory.createURI("https://w3id.org/catenax/ontology/common#satisfiesRole")); + registerPredicate(COMMON_NAMESPACE, "id", false); + registerPredicate(COMMON_NAMESPACE, "name", false); + registerPredicate(COMMON_NAMESPACE, "description", false); + registerPredicate(COMMON_NAMESPACE, "version", false); + registerPredicate(COMMON_NAMESPACE, "contenttype", false); + registerPredicate(DC_NAMESPACE, "type", true); + // map the rdf definition to dublin core + ASSET_PROPERTY_MAP.put(RDF_NAMESPACE + "type", createUri(DC_NAMESPACE, "type")); + registerPredicate(RDF_SCHEMA_NAMESPACE, "isDefinedBy", true); + registerPredicate(COMMON_NAMESPACE, "implementsProtocol", true); + registerPredicate(SHACL_NAMESPACE, "shapesGraph", false); + registerPredicate(COMMON_NAMESPACE, "isFederated", true); + registerPredicate(COMMON_NAMESPACE, "publishedUnderContract", true); + registerPredicate(COMMON_NAMESPACE, "satisfiesRole", true); + } + + protected static final Map PREDEFINED_NS = new HashMap<>( + ); + + static { + PREDEFINED_NS.put("cx-common:", COMMON_NAMESPACE); + PREDEFINED_NS.put("cx-taxo:", TAXO_NAMESPACE); + PREDEFINED_NS.put("edc:", EDC_NAMESPACE); + PREDEFINED_NS.put("rdf:", RDF_NAMESPACE); + PREDEFINED_NS.put("rdfs:", RDF_SCHEMA_NAMESPACE); + PREDEFINED_NS.put("sh:", SHACL_NAMESPACE); + PREDEFINED_NS.put("cx-sh:", CX_SCHEMA_NAMESPACE); + PREDEFINED_NS.put("dct:", DC_NAMESPACE); + PREDEFINED_NS.put("xsd:", XML_SCHEMA_NAMESPACE); + } + + /** + * static helper to create an uri node from a prefix and name + * + * @param prefix namespace + * @param name entity name + * @return uri node + */ + protected static Node createUri(String prefix, String name) { + return NodeFactory.createURI(prefix + name); + } + + /** + * make sure these asset properties can be defined with or without prefix + * + * @param prefix namespace + * @param name id + */ + protected static void registerPredicate(String prefix, String name, boolean isUri) { + String key = prefix + name; + if (isUri) { + EXPECTED_COMPLEX_OBJECTS.add(key); + } + Node target = createUri(prefix, name); + ASSET_PROPERTY_MAP.put(key, target); + key = EDC_NAMESPACE + name; + ASSET_PROPERTY_MAP.put(key, target); } /** @@ -89,6 +151,7 @@ public class DataspaceSynchronizer implements Runnable { protected final DataManagement dataManagement; protected final RdfStore rdfStore; protected final Monitor monitor; + protected final MonitorWrapper monitorWrapper; /** * internal state @@ -98,11 +161,11 @@ public class DataspaceSynchronizer implements Runnable { /** * creates the synchronizer * - * @param service scheduler - * @param config edc config + * @param service scheduler + * @param config edc config * @param dataManagement data management service remoting - * @param rdfStore a triple store for persistance - * @param monitor logging subsystem + * @param rdfStore a triple store for persistance + * @param monitor logging subsystem */ public DataspaceSynchronizer(ScheduledExecutorService service, AgentConfig config, DataManagement dataManagement, RdfStore rdfStore, Monitor monitor) { this.service = service; @@ -154,44 +217,8 @@ public void run() { DcatCatalog catalog = dataManagement.getCatalog(remote, FEDERATED_ASSET_QUERY); Node graph = rdfStore.getDefaultGraph(); Node connector = NodeFactory.createURI(remote.replace("https", "edcs").replace("http", "edc")); - Quad findAssets = Quad.create(graph, connector, CX_ASSET, Node.ANY); - Iterator assetQuads = rdfStore.getDataSet().find(findAssets); - int tupleCount = 0; - while (assetQuads.hasNext()) { - Quad quadAsset = assetQuads.next(); - Quad findAssetProps = Quad.create(graph, quadAsset.getObject(), Node.ANY, Node.ANY); - Iterator propQuads = rdfStore.getDataSet().find(findAssetProps); - while (propQuads.hasNext()) { - Quad quadProp = propQuads.next(); - if (quadProp.getPredicate().isURI() && quadProp.getPredicate().getURI().equals("http://www.w3.org/ns/shacl#shapesGraph") && quadProp.getObject().isURI()) { - Quad findSubGraphsProps = Quad.create(quadProp.getObject(), Node.ANY, Node.ANY, Node.ANY); - Iterator subGraphQuads = rdfStore.getDataSet().find(findSubGraphsProps); - while (subGraphQuads.hasNext()) { - rdfStore.getDataSet().delete(subGraphQuads.next()); - tupleCount++; - } - } - rdfStore.getDataSet().delete(quadProp); - tupleCount++; - } - rdfStore.getDataSet().delete(quadAsset); - tupleCount++; - } - monitor.debug(String.format("About to delete %d old tuples.", tupleCount)); - List offers = catalog.getDatasets(); - tupleCount = 0; - if (offers != null) { - monitor.debug(String.format("Found a catalog with %d entries for remote connector %s", offers.size(), remote)); - for (DcatDataset offer : catalog.getDatasets()) { - for (Quad quad : convertToQuads(graph, connector, offer)) { - tupleCount++; - rdfStore.getDataSet().add(quad); - } - } - } else { - monitor.warning(String.format("Found an empty catalog for remote connector %s", remote)); - } - monitor.debug(String.format("About to add %d new tuples.", tupleCount)); + deleteConnectorFacts(graph, connector); + addConnectorFacts(remote, catalog, graph, connector); rdfStore.commit(); } catch (Throwable io) { monitor.warning(String.format("Could not synchronize remote connector %s because of %s. Going ahead.", remote, io)); @@ -211,6 +238,120 @@ public void run() { } } + /** + * adds new facts about the catalog thats been collected from the given connector + * + * @param remote url of the remote connector + * @param catalog retrieved catalogue + * @param graph to store the facts + * @param connector uri node representing the connector + * @return number of fact triples/quads added + */ + public int addConnectorFacts(String remote, DcatCatalog catalog, Node graph, Node connector) { + int tupleCount; + List offers = catalog.getDatasets(); + tupleCount = 0; + if (offers != null) { + monitor.debug(String.format("Found a catalog with %d entries for remote connector %s", offers.size(), remote)); + for (DcatDataset offer : catalog.getDatasets()) { + tupleCount += addOfferFacts(graph, connector, offer); + } + } else { + monitor.warning(String.format("Found an empty catalog for remote connector %s", remote)); + } + monitor.debug(String.format("About to add %d new tuples.", tupleCount)); + return tupleCount; + } + + /** + * adds new facts about a concrete data offer collected from a given connector + * + * @param graph to store the facts + * @param connector uri node representing the connector + * @param offer to convert into facts + * @return number of facts triples/quads added + */ + public int addOfferFacts(Node graph, Node connector, DcatDataset offer) { + int tupleCount = 0; + var quads = convertToQuads(graph, connector, offer); + for (Quad quad : quads) { + tupleCount++; + rdfStore.getDataSet().add(quad); + } + return tupleCount; + } + + /** + * deletes existing facts about a connector + * + * @param graph to delete the facts from + * @param connector to clean up the facts for + * @return number of fact triples/quads deleted + */ + public int deleteConnectorFacts(Node graph, Node connector) { + // find all offers attached to the connector + Quad findAssets = Quad.create(graph, connector, CX_ASSET, Node.ANY); + Iterator assetQuads = rdfStore.getDataSet().find(findAssets); + int tupleCount = 0; + while (assetQuads.hasNext()) { + Quad quadAsset = assetQuads.next(); + Node assetNode = quadAsset.getObject(); + tupleCount += deleteShaclShapes(graph, assetNode); + tupleCount += deleteAssetProperties(graph, assetNode); + rdfStore.getDataSet().delete(quadAsset); + tupleCount++; + } + monitor.debug(String.format("About to delete %d old tuples.", tupleCount)); + return tupleCount; + } + + /** + * deletes properties of a given offer/asset + * + * @param graph where facts are stored + * @param assetNode to delete properties from + * @return number of property facts deleted + */ + public int deleteAssetProperties(Node graph, Node assetNode) { + int tupleCount = 0; + // remove all remaining properties associated to the offer/asset + Quad findAssetProps = Quad.create(graph, assetNode, Node.ANY, Node.ANY); + Iterator propQuads = rdfStore.getDataSet().find(findAssetProps); + while (propQuads.hasNext()) { + Quad quadProp = propQuads.next(); + rdfStore.getDataSet().delete(quadProp); + tupleCount++; + } + return tupleCount; + } + + /** + * delete all shacl shapes associated to an asset node + * + * @param graph where the shapes are stored + * @param assetNode to delete the shapes from + * @return number of associated shapes (and their properties) + */ + public int deleteShaclShapes(Node graph, Node assetNode) { + int tupleCount = 0; + // remove all shacl shapes associated to the offer/asset + Quad findAssetShapes = Quad.create(graph, assetNode, SHAPE_OBJECT, Node.ANY); + Iterator shapesQuad = rdfStore.getDataSet().find(findAssetShapes); + while (shapesQuad.hasNext()) { + Quad shapeQuad = shapesQuad.next(); + Node shapesObject = shapeQuad.getObject(); + Quad findShapesPredicates = Quad.create(graph, shapesObject, Node.ANY, Node.ANY); + Iterator shapesFacts = rdfStore.getDataSet().find(findShapesPredicates); + while (shapesFacts.hasNext()) { + rdfStore.getDataSet().delete(shapesFacts.next()); + tupleCount++; + } + rdfStore.getDataSet().delete(shapeQuad); + tupleCount++; + } + return tupleCount; + } + /** * Workaround the castration of the IDS catalogue * @@ -219,13 +360,13 @@ public void run() { */ public static Map getProperties(DcatDataset offer) { Map assetProperties = new HashMap<>(offer.getProperties()); - if (!assetProperties.containsKey("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")) { + if (!assetProperties.containsKey(DC_TYPE) && !assetProperties.containsKey(RDF_TYPE.getURI()) && !assetProperties.containsKey(EDC_NAMESPACE + "type")) { String assetType = JsonLd.asString(assetProperties.getOrDefault("@id", Json.createValue("cx-common:Asset"))); int indexOfQuestion = assetType.indexOf("?"); if (indexOfQuestion > 0) { assetType = assetType.substring(0, indexOfQuestion - 1); } - assetProperties.put("http://www.w3.org/1999/02/22-rdf-syntax-ns#type", Json.createValue(assetType)); + assetProperties.put(DC_TYPE, Json.createValue(assetType)); } if (!assetProperties.containsKey("@id")) { assetProperties.put("@id", Json.createValue(UUID.randomUUID().toString())); @@ -236,16 +377,15 @@ public static Map getProperties(DcatDataset offer) { /** * convert a given contract offer into quads * - * @param graph default graph + * @param graph default graph * @param connector parent connector hosting the offer - * @param offer the contract offer + * @param offer the contract offer * @return a collection of quads */ public Collection convertToQuads(Node graph, Node connector, DcatDataset offer) { Map assetProperties = getProperties(offer); - List quads = new ArrayList<>(); - String offerId = assetProperties.get("@id").toString(); + String offerId = JsonLd.asString(assetProperties.get("@id")); Node assetNode = NodeFactory.createURI(offerId); quads.add(Quad.create(graph, connector, @@ -254,77 +394,27 @@ public Collection convertToQuads(Node graph, Node connector, DcatDataset o for (Map.Entry assetProp : assetProperties.entrySet()) { String key = assetProp.getKey(); Node node = ASSET_PROPERTY_MAP.get(key); - while (node == null && key.indexOf('.') >= 0) { - key = key.substring(key.lastIndexOf(".") + 1); + // strip off language modifiers + if (node == null && key.indexOf("@") >= 0) { + String langSuffix = key.substring(key.lastIndexOf("@")); + key = key.substring(0, key.lastIndexOf("@")); node = ASSET_PROPERTY_MAP.get(key); + if (node != null) { + node = createUri(node.getURI(), "@" + langSuffix); + } } + // did we find some result if (node != null) { String pureProperty = JsonLd.asString(assetProp.getValue()); if (pureProperty != null) { try { - switch (key) { - case "https://w3id.org/catenax/ontology/common#publishedUnderContract": - case "http://www.w3.org/2000/01/rdf-schema#isDefinedBy": - case "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": - case "https://w3id.org/catenax/ontology/common#isFederated": - case "https://w3id.org/catenax/ontology/common#satisfiesRole": - case "https://w3id.org/catenax/ontology/common#implementsProtocol": - String[] urls = pureProperty.split(","); - for (String url : urls) { - Node o; - url = url.trim(); - if (url.startsWith("<") && url.endsWith(">")) { - url = url.substring(1, url.length() - 1); - o = NodeFactory.createURI(url); - } else if (url.startsWith("\"") && url.endsWith("\"")) { - url = url.substring(1, url.length() - 1); - o = NodeFactory.createLiteral(url); - } else if (url.startsWith("cx-common:")) { - url = url.substring(10); - o = NodeFactory.createURI("https://w3id.org/catenax/ontology/common#" + url); - } else if (url.contains("^^")) { - int typeAnnotation = url.indexOf("^^"); - String type = url.substring(typeAnnotation + 2); - url = url.substring(0, typeAnnotation); - o = NodeFactory.createLiteral(url, NodeFactory.getType(type)); - } else { - o = NodeFactory.createLiteral(url); - } - quads.add(Quad.create(graph, assetNode, node, o)); - } - break; - case "http://www.w3.org/ns/shacl#shapesGraph": - Node newGraph = NodeFactory.createURI("https://w3id.org/catenax/ontology/common#GraphShape?id=" + offerId.hashCode()); - quads.add(Quad.create(graph, assetNode, node, newGraph)); - Sink quadSink = new Sink<>() { - - @Override - public void close() { - } - - @Override - public void send(Quad quad) { - quads.add(quad); - } - - @Override - public void flush() { - } - }; - StreamRDF dest = StreamRDFLib.sinkQuads(quadSink); - StreamRDF graphDest = StreamRDFLib.extendTriplesToQuads(newGraph, dest); - StreamRDFCounting countingDest = StreamRDFLib.count(graphDest); - ErrorHandler errorHandler = ErrorHandlerFactory.errorHandlerStd(monitorWrapper); - RDFParser.create() - .errorHandler(errorHandler) - .source(new StringReader(pureProperty)) - .lang(Lang.TTL) - .parse(countingDest); - monitor.debug(String.format("Added shapes subgraph %s with %d triples", newGraph, countingDest.countTriples())); - break; - default: - quads.add(Quad.create(graph, assetNode, node, NodeFactory.createLiteral(pureProperty))); - } //switch + if (SHAPES_GRAPH.equals(key)) { + addShapesFacts(graph, quads, assetNode, pureProperty); + } else if (EXPECTED_COMPLEX_OBJECTS.contains(node.getURI())) { + addComplexFacts(graph, quads, assetNode, node, pureProperty); + } else { + quads.add(Quad.create(graph, assetNode, node, NodeFactory.createLiteral(pureProperty))); + } } catch (Throwable t) { monitor.debug(String.format("Could not correctly add asset triples for predicate %s with original value %s because of %s", node, pureProperty, t.getMessage())); } @@ -334,4 +424,92 @@ public void flush() { return quads; } + /** + * converts the given object string into complex facts + * + * @param graph to store the facts + * @param quads list to add the facts to + * @param assetNode subject to attach the facts to + * @param node the predicated under which the facts are holding + * @param objectString the actual value(s) to attach the predicate to + */ + private static void addComplexFacts(Node graph, List quads, Node assetNode, Node node, String objectString) { + // do we expect our predicate to hit a uri + String[] urls = objectString.split(","); + for (String url : urls) { + Node o = null; + url = url.trim(); + if (url.startsWith("<") && url.endsWith(">")) { + url = url.substring(1, url.length() - 1); + o = NodeFactory.createURI(url); + } else if (url.startsWith("\"") && url.endsWith("\"")) { + url = url.substring(1, url.length() - 1); + o = NodeFactory.createLiteral(url); + } else if (url.contains("^^")) { + int typeAnnotation = url.indexOf("^^"); + String type = url.substring(typeAnnotation + 2); + url = url.substring(0, typeAnnotation); + o = NodeFactory.createLiteral(url, NodeFactory.getType(type)); + } else { + for (var entry : PREDEFINED_NS.entrySet()) { + if (url.startsWith(entry.getKey())) { + url = url.substring(entry.getKey().length()); + o = NodeFactory.createURI(entry.getValue() + url); + break; + } + } + if (o == null) { + o = NodeFactory.createLiteral(url); + } + } + quads.add(Quad.create(graph, assetNode, node, o)); + // make sure both rdf:type and dc:type are set + if (node.getURI().equals(DC_TYPE)) { + quads.add(Quad.create(graph, assetNode, RDF_TYPE, o)); + } + } + } + + /** + * converts the given object string into a shapes graph + * + * @param graph to store the facts in + * @param quads to add the facts to + * @param assetNode asset to attach the shapes to + * @param shapesDescription a hopefully valid shacl shape turtle + */ + private void addShapesFacts(Node graph, List quads, Node assetNode, String shapesDescription) { + StreamRDF dest = StreamRDFLib.sinkQuads(new Sink<>() { + + protected Set connectedSubjects = new HashSet<>(); + + @Override + public void close() { + } + + @Override + public void send(Quad quad) { + String subject = quad.getSubject().toString(false); + if (!connectedSubjects.contains(subject)) { + quads.add(new Quad(graph, assetNode, SHAPE_OBJECT, quad.getSubject())); + connectedSubjects.add(subject); + } + quads.add(quad); + } + + @Override + public void flush() { + } + }); + StreamRDF graphDest = StreamRDFLib.extendTriplesToQuads(graph, dest); + StreamRDFCounting countingDest = StreamRDFLib.count(graphDest); + ErrorHandler errorHandler = ErrorHandlerFactory.errorHandlerStd(monitorWrapper); + RDFParser.create() + .errorHandler(errorHandler) + .source(new StringReader(shapesDescription)) + .lang(Lang.TTL) + .parse(countingDest); + monitor.debug(String.format("Added shapes subgraph to asset %s with %d triples", assetNode, countingDest.countTriples())); + } + } diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/EdcSkillStore.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/EdcSkillStore.java index 74a672d4..9b814585 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/EdcSkillStore.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/EdcSkillStore.java @@ -51,7 +51,7 @@ public boolean isSkill(String key) { } @Override - public String put(String key, String skill, String name, String description, String version, String contract, SkillDistribution dist, boolean isFederated, String... ontologies) { + public String put(String key, String skill, String name, String description, String version, String contract, SkillDistribution dist, boolean isFederated, String allowServicePatern, String denyServicePattern, String... ontologies) { if (name == null) { name = "No name given"; } @@ -75,7 +75,9 @@ public String put(String key, String skill, String name, String description, Str ontologiesString, dist.getDistributionMode(), isFederated, - typeManager.getMapper().writeValueAsString(TextNode.valueOf(skill)) + typeManager.getMapper().writeValueAsString(TextNode.valueOf(skill)), + allowServicePatern, + denyServicePattern ).getId(); } catch (IOException e) { return null; diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/InMemorySkillStore.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/InMemorySkillStore.java index 853ef591..24568cf5 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/InMemorySkillStore.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/InMemorySkillStore.java @@ -43,7 +43,7 @@ public boolean isSkill(String key) { } @Override - public String put(String key, String skill, String name, String description, String version, String contract, SkillDistribution dist, boolean isFederated, String... ontologies) { + public String put(String key, String skill, String name, String description, String version, String contract, SkillDistribution dist, boolean isFederated, String allowServicePattern, String denyServicePattern, String... ontologies) { skills.put(key, skill); return key; } From 8320f7b1090347be543cbc7301aaf15bb91745fa Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Thu, 18 Apr 2024 09:21:11 +0200 Subject: [PATCH 02/31] chore: upgrade version with EDC 0.6 support. --- .github/workflows/build.yml | 8 ++++---- README.md | 8 ++++---- charts/conforming-agent/Chart.yaml | 4 ++-- charts/conforming-agent/README.md | 4 ++-- charts/matchmaking-agent/Chart.yaml | 4 ++-- charts/provisioning-agent/Chart.yaml | 4 ++-- charts/provisioning-agent/README.md | 4 ++-- charts/remoting-agent/Chart.yaml | 4 ++-- charts/remoting-agent/README.md | 4 ++-- conforming/README.md | 14 +++++++------- conforming/pom.xml | 2 +- matchmaking/README.md | 6 +++--- matchmaking/pom.xml | 2 +- pom.xml | 2 +- provisioning/README.md | 10 +++++----- provisioning/pom.xml | 2 +- remoting/README.md | 14 +++++++------- remoting/pom.xml | 2 +- upgrade_version.sh | 2 +- 19 files changed, 50 insertions(+), 50 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c4bda6e8..8d36a2b5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -142,7 +142,7 @@ jobs: type=semver,pattern={{version}} type=semver,pattern={{major}} type=semver,pattern={{major}}.{{minor}} - type=raw,value=1.12.18-SNAPSHOT,enable=${{ github.event.inputs.deploy_docker == 'true' || github.ref == format('refs/heads/{0}', 'main') }} + type=raw,value=1.12.19-SNAPSHOT,enable=${{ github.event.inputs.deploy_docker == 'true' || github.ref == format('refs/heads/{0}', 'main') }} type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} # build in any case, but push only main and version tag settings @@ -181,7 +181,7 @@ jobs: type=semver,pattern={{version}} type=semver,pattern={{major}} type=semver,pattern={{major}}.{{minor}} - type=raw,value=1.12.18-SNAPSHOT,enable=${{ github.event.inputs.deploy_docker == 'true' || github.ref == format('refs/heads/{0}', 'main') }} + type=raw,value=1.12.19-SNAPSHOT,enable=${{ github.event.inputs.deploy_docker == 'true' || github.ref == format('refs/heads/{0}', 'main') }} type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} # build in any case, but push only main and version tag settings @@ -220,7 +220,7 @@ jobs: type=semver,pattern={{version}} type=semver,pattern={{major}} type=semver,pattern={{major}}.{{minor}} - type=raw,value=1.12.18-SNAPSHOT,enable=${{ github.event.inputs.deploy_docker == 'true' || github.ref == format('refs/heads/{0}', 'main') }} + type=raw,value=1.12.19-SNAPSHOT,enable=${{ github.event.inputs.deploy_docker == 'true' || github.ref == format('refs/heads/{0}', 'main') }} type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} # build in any case, but push only main and version tag settings @@ -280,6 +280,6 @@ jobs: type=semver,pattern={{version}} type=semver,pattern={{major}} type=semver,pattern={{major}}.{{minor}} - type=raw,value=1.12.18-SNAPSHOT,enable=${{ github.event.inputs.deploy_docker == 'true' || github.ref == format('refs/heads/{0}', 'main') }} + type=raw,value=1.12.19-SNAPSHOT,enable=${{ github.event.inputs.deploy_docker == 'true' || github.ref == format('refs/heads/{0}', 'main') }} type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} diff --git a/README.md b/README.md index ae65905f..1277910c 100644 --- a/README.md +++ b/README.md @@ -114,10 +114,10 @@ kubectl wait --namespace ingress-nginx \ --selector=app.kubernetes.io/component=controller \ --timeout=90s # transfer images -kind load docker-image docker.io/tractusx/conforming-agent:1.12.18-SNAPSHOT --name ka -kind load docker-image docker.io/tractusx/matchmaking-agent:1.12.18-SNAPSHOT --name ka -kind load docker-image docker.io/tractusx/provisioning-agent:1.12.18-SNAPSHOT --name ka -kind load docker-image docker.io/tractusx/remoting-agent:1.12.18-SNAPSHOT --name ka +kind load docker-image docker.io/tractusx/conforming-agent:1.12.19-SNAPSHOT --name ka +kind load docker-image docker.io/tractusx/matchmaking-agent:1.12.19-SNAPSHOT --name ka +kind load docker-image docker.io/tractusx/provisioning-agent:1.12.19-SNAPSHOT --name ka +kind load docker-image docker.io/tractusx/remoting-agent:1.12.19-SNAPSHOT --name ka # run container test ct install --charts charts/conforming-agent ct install --charts charts/matchmaking-agent diff --git a/charts/conforming-agent/Chart.yaml b/charts/conforming-agent/Chart.yaml index 3eaeef11..c8395c6d 100644 --- a/charts/conforming-agent/Chart.yaml +++ b/charts/conforming-agent/Chart.yaml @@ -28,7 +28,7 @@ home: https://github.com/eclipse-tractusx/knowledge-agents/ sources: - https://github.com/eclipse-tractusx/knowledge-agents/tree/main/conforming type: application -appVersion: "1.12.18-SNAPSHOT" -version: 1.12.18-SNAPSHOT +appVersion: "1.12.19-SNAPSHOT" +version: 1.12.19-SNAPSHOT maintainers: - name: 'Tractus-X Knowledge Agents Team' diff --git a/charts/conforming-agent/README.md b/charts/conforming-agent/README.md index 2ce475a0..15b03048 100644 --- a/charts/conforming-agent/README.md +++ b/charts/conforming-agent/README.md @@ -20,7 +20,7 @@ # conforming-agent -![Version: 1.12.18-SNAPSHOT](https://img.shields.io/badge/Version-1.12.17--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.12.18-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.12.17--SNAPSHOT-informational?style=flat-square) +![Version: 1.12.19-SNAPSHOT](https://img.shields.io/badge/Version-1.12.17--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.12.19-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.12.17--SNAPSHOT-informational?style=flat-square) A Helm chart for the Tractus-X Conforming Agent which is a container to assess the conformity of all other parts of the Agent-Enabled Dataspace. @@ -31,7 +31,7 @@ This chart has no prerequisites. ## TL;DR ```shell $ helm repo add eclipse-tractusx https://eclipse-tractusx.github.io/charts/dev -$ helm install my-release eclipse-tractusx/conforming-agent --version 1.12.18-SNAPSHOT +$ helm install my-release eclipse-tractusx/conforming-agent --version 1.12.19-SNAPSHOT ``` ## Maintainers diff --git a/charts/matchmaking-agent/Chart.yaml b/charts/matchmaking-agent/Chart.yaml index 77f8a74e..1c45e99f 100644 --- a/charts/matchmaking-agent/Chart.yaml +++ b/charts/matchmaking-agent/Chart.yaml @@ -28,7 +28,7 @@ home: https://github.com/eclipse-tractusx/knowledge-agents/ sources: - https://github.com/eclipse-tractusx/knowledge-agents/tree/main/matchmaking type: application -appVersion: "1.12.18-SNAPSHOT" -version: 1.12.18-SNAPSHOT +appVersion: "1.12.19-SNAPSHOT" +version: 1.12.19-SNAPSHOT maintainers: - name: 'Tractus-X Knowledge Agents Team' diff --git a/charts/provisioning-agent/Chart.yaml b/charts/provisioning-agent/Chart.yaml index b8c54bac..7a750611 100644 --- a/charts/provisioning-agent/Chart.yaml +++ b/charts/provisioning-agent/Chart.yaml @@ -28,7 +28,7 @@ home: https://github.com/eclipse-tractusx/knowledge-agents/ sources: - https://github.com/eclipse-tractusx/knowledge-agents/tree/main/provisioning type: application -appVersion: "1.12.18-SNAPSHOT" -version: 1.12.18-SNAPSHOT +appVersion: "1.12.19-SNAPSHOT" +version: 1.12.19-SNAPSHOT maintainers: - name: 'Tractus-X Knowledge Agents Team' diff --git a/charts/provisioning-agent/README.md b/charts/provisioning-agent/README.md index 0669f57e..b91d1917 100644 --- a/charts/provisioning-agent/README.md +++ b/charts/provisioning-agent/README.md @@ -20,7 +20,7 @@ # provisioning-agent -![Version: 1.12.18-SNAPSHOT](https://img.shields.io/badge/Version-1.12.17--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.12.18-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.12.17--SNAPSHOT-informational?style=flat-square) +![Version: 1.12.19-SNAPSHOT](https://img.shields.io/badge/Version-1.12.17--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.12.19-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.12.17--SNAPSHOT-informational?style=flat-square) A Helm chart for the Tractus-X Provisioning Agent which is a container to Bridge Agent-Enabled Connector and Relational Data Sources. @@ -31,7 +31,7 @@ This chart has no prerequisites. ## TL;DR ```shell $ helm repo add eclipse-tractusx https://eclipse-tractusx.github.io/charts/dev -$ helm install my-release eclipse-tractusx/provisioning-agent --version 1.12.18-SNAPSHOT +$ helm install my-release eclipse-tractusx/provisioning-agent --version 1.12.19-SNAPSHOT ``` ## Maintainers diff --git a/charts/remoting-agent/Chart.yaml b/charts/remoting-agent/Chart.yaml index 8881bae3..be261f5f 100644 --- a/charts/remoting-agent/Chart.yaml +++ b/charts/remoting-agent/Chart.yaml @@ -28,7 +28,7 @@ home: https://github.com/eclipse-tractusx/knowledge-agents/ sources: - https://github.com/eclipse-tractusx/knowledge-agents/tree/main/remoting type: application -appVersion: "1.12.18-SNAPSHOT" -version: 1.12.18-SNAPSHOT +appVersion: "1.12.19-SNAPSHOT" +version: 1.12.19-SNAPSHOT maintainers: - name: 'Tractus-X Knowledge Agents Team' diff --git a/charts/remoting-agent/README.md b/charts/remoting-agent/README.md index ce533ed9..b10fbb23 100644 --- a/charts/remoting-agent/README.md +++ b/charts/remoting-agent/README.md @@ -19,7 +19,7 @@ --> # remoting-agent -![Version: 1.12.18-SNAPSHOT](https://img.shields.io/badge/Version-1.12.17--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.12.18-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.12.17--SNAPSHOT-informational?style=flat-square) +![Version: 1.12.19-SNAPSHOT](https://img.shields.io/badge/Version-1.12.17--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.12.19-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.12.17--SNAPSHOT-informational?style=flat-square) A Helm chart for the Tractus-X Remoting Agent which is a container to Bridge Agent-Enabled Connector and REST APIs. @@ -30,7 +30,7 @@ This chart has no prerequisites. ## TL;DR ```shell $ helm repo add eclipse-tractusx https://eclipse-tractusx.github.io/charts/dev -$ helm install my-release eclipse-tractusx/remoting-agent --version 1.12.18-SNAPSHOT +$ helm install my-release eclipse-tractusx/remoting-agent --version 1.12.19-SNAPSHOT ``` ## Maintainers diff --git a/conforming/README.md b/conforming/README.md index 46805873..ce4e3e80 100644 --- a/conforming/README.md +++ b/conforming/README.md @@ -54,15 +54,15 @@ mvn package This will generate -- a [plugin jar](target/original-conforming-agent-1.12.18-SNAPSHOT.jar) containing all necessary components to be dropped into a Jakarta-Compatible Web Server. -- a [standalone jar](target/conforming-agent-1.12.18-SNAPSHOT.jar) including the Jakarta-Reference Implementation (Glassfish). +- a [plugin jar](target/original-conforming-agent-1.12.19-SNAPSHOT.jar) containing all necessary components to be dropped into a Jakarta-Compatible Web Server. +- a [standalone jar](target/conforming-agent-1.12.19-SNAPSHOT.jar) including the Jakarta-Reference Implementation (Glassfish). ### Run Locally -The [standalone jar](target/conforming-agent-1.12.18-SNAPSHOT.jar) may be started as follows +The [standalone jar](target/conforming-agent-1.12.19-SNAPSHOT.jar) may be started as follows ```console -java -cp target/conforming-agent-1.12.18-SNAPSHOT.jar org.eclipse.tractusx.agents.conforming.Bootstrap" +java -cp target/conforming-agent-1.12.19-SNAPSHOT.jar org.eclipse.tractusx.agents.conforming.Bootstrap" ``` ### Containerizing @@ -76,7 +76,7 @@ mvn install -Pwith-docker-image or invoke the following docker command after a successful package run ```console -docker build -t tractusx/conforming-agent:1.12.18-SNAPSHOT -f src/main/docker/Dockerfile . +docker build -t tractusx/conforming-agent:1.12.19-SNAPSHOT -f src/main/docker/Dockerfile . ``` This will create a docker image based on a minimal java environment for running the Glassfish-based standalone jar. @@ -85,7 +85,7 @@ To run the docker image, you could invoke this command ```console docker run -p 8080:8080 \ - tractusx/conforming-agent:1.12.18-SNAPSHOT + tractusx/conforming-agent:1.12.19-SNAPSHOT ```` Afterwards, you should be able to access the [local SparQL endpoint](http://localhost:8080/) via @@ -143,7 +143,7 @@ It can be added to your umbrella chart.yaml by the following snippet dependencies: - name: conforming-agent repository: https://eclipse-tractusx.github.io/charts/dev - version: 1.12.18-SNAPSHOT + version: 1.12.19-SNAPSHOT alias: my-conforming-agent ``` diff --git a/conforming/pom.xml b/conforming/pom.xml index 310eab51..e606ad83 100644 --- a/conforming/pom.xml +++ b/conforming/pom.xml @@ -27,7 +27,7 @@ org.eclipse.tractusx agents - 1.12.18-SNAPSHOT + 1.12.19-SNAPSHOT ../pom.xml diff --git a/matchmaking/README.md b/matchmaking/README.md index 11f15707..656891fe 100644 --- a/matchmaking/README.md +++ b/matchmaking/README.md @@ -66,10 +66,10 @@ This will generate ### Run Locally -The [standalone jar](target/matchmaking-agent-1.12.18-SNAPSHOT.jar) may be started as follows +The [standalone jar](target/matchmaking-agent-1.12.19-SNAPSHOT.jar) may be started as follows ```console -java -Dproperty.file.location="dataplane.properties" -cp ../matchmaking-agent-1.12.18-SNAPSHOT.jar org.eclipse.tractusx.agents.conforming.Bootstrap +java -Dproperty.file.location="dataplane.properties" -cp ../matchmaking-agent-1.12.19-SNAPSHOT.jar org.eclipse.tractusx.agents.conforming.Bootstrap ``` Make sure that jar file, properties file and dataspace.ttl are in the same directory Then you should be able to reach the /graph endpoint @@ -126,7 +126,7 @@ It can be added to your umbrella chart.yaml by the following snippet dependencies: - name: matchmaking-agent repository: https://eclipse-tractusx.github.io/charts/dev - version: 1.12.18-SNAPSHOT + version: 1.12.19-SNAPSHOT alias: my-matchmmaking-agent ``` diff --git a/matchmaking/pom.xml b/matchmaking/pom.xml index 6509d5b3..3bc2242c 100644 --- a/matchmaking/pom.xml +++ b/matchmaking/pom.xml @@ -27,7 +27,7 @@ org.eclipse.tractusx agents - 1.12.18-SNAPSHOT + 1.12.19-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 8a8c2a61..3c812737 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 4.0.0 org.eclipse.tractusx agents - 1.12.18-SNAPSHOT + 1.12.19-SNAPSHOT pom Tractus-X Knowledge Agents Reference Implementations Provides Reference Implementations and Artifacts to Realize Semantic Dataspace Backends diff --git a/provisioning/README.md b/provisioning/README.md index 0625dc26..d1d5f23a 100644 --- a/provisioning/README.md +++ b/provisioning/README.md @@ -111,7 +111,7 @@ mvn package ``` This will generate -- a [pluging jar](target/provisioning-agent-1.12.18-SNAPSHOT.jar) which maybe dropped into an Ontop server (into the lib folder) +- a [pluging jar](target/provisioning-agent-1.12.19-SNAPSHOT.jar) which maybe dropped into an Ontop server (into the lib folder) ### Containerizing (Provisioning Agent) @@ -124,7 +124,7 @@ mvn install -Pwith-docker-image or invoke the following docker command after a successful package run ```console -docker build -t tractusx/provisioning-agent:1.12.18-SNAPSHOT -f src/main/docker/Dockerfile . +docker build -t tractusx/provisioning-agent:1.12.19-SNAPSHOT -f src/main/docker/Dockerfile . ``` The image contains @@ -144,7 +144,7 @@ docker run -p 8080:8080 \ -v $(pwd)/resources/university-role1.obda:/input/mapping.obda \ -v $(pwd)/resources/university-role1.properties:/input/settings.properties \ -v $(pwd)/resources/university.sql:/tmp/university.sql \ - tractusx/provisioning-agent:1.12.18-SNAPSHOT + tractusx/provisioning-agent:1.12.19-SNAPSHOT ```` Afterwards, you should be able to access the [local SparQL endpoint](http://localhost:8080/) via @@ -192,7 +192,7 @@ docker run -p 8080:8080 -p 8082:8082 \ -e ONTOP_MAPPING_FILE="/input/role1.obda /input/role2.obda" \ -e ONTOP_PROPERTIES_FILE="/input/role1.properties /input/role2.properties" \ -e ONTOP_DEV_MODE="false false" \ - tractusx/provisioning-agent:1.12.18-SNAPSHOT + tractusx/provisioning-agent:1.12.19-SNAPSHOT ```` Accessing entities spanning two schemas using the first role/endpoint delivers a greater count @@ -297,7 +297,7 @@ It can be added to your umbrella chart.yaml by the following snippet dependencies: - name: provisioning-agent repository: https://eclipse-tractusx.github.io/charts/dev - version: 1.12.18-SNAPSHOT + version: 1.12.19-SNAPSHOT alias: my-provider-agent ``` diff --git a/provisioning/pom.xml b/provisioning/pom.xml index 2c1d3bdd..bd759fca 100644 --- a/provisioning/pom.xml +++ b/provisioning/pom.xml @@ -27,7 +27,7 @@ org.eclipse.tractusx agents - 1.12.18-SNAPSHOT + 1.12.19-SNAPSHOT ../pom.xml diff --git a/remoting/README.md b/remoting/README.md index 2319d882..cb54413f 100644 --- a/remoting/README.md +++ b/remoting/README.md @@ -134,15 +134,15 @@ mvn package ``` This will generate -- a [standalone jar](target/remoting-agent-1.12.18-SNAPSHOT.jar) containing all necessary rdf4j components to build your own repository server. -- a [pluging jar](target/original-remoting-agent-1.12.18-SNAPSHOT.jar) which maybe dropped into an rdf4j server for remoting support. +- a [standalone jar](target/remoting-agent-1.12.19-SNAPSHOT.jar) containing all necessary rdf4j components to build your own repository server. +- a [pluging jar](target/original-remoting-agent-1.12.19-SNAPSHOT.jar) which maybe dropped into an rdf4j server for remoting support. ### Run Locally -The standalone jar](target/remoting-agent-1.12.18-SNAPSHOT.jar) contains an example application that runs a sample repository against a sample source +The standalone jar](target/remoting-agent-1.12.19-SNAPSHOT.jar) contains an example application that runs a sample repository against a sample source ```console -java -jar target/remoting-agent-1.12.18-SNAPSHOT.jar -Dorg.slf4j.simpleLogger.defaultLogLevel=DEBUG +java -jar target/remoting-agent-1.12.19-SNAPSHOT.jar -Dorg.slf4j.simpleLogger.defaultLogLevel=DEBUG ``` ### Containerizing @@ -156,7 +156,7 @@ mvn install -Pwith-docker-image or invoke the following docker command after a successful package run ```console -docker build -t tractusx/remoting-agent:1.12.18-SNAPSHOT -f src/main/docker/Dockerfile . +docker build -t tractusx/remoting-agent:1.12.19-SNAPSHOT -f src/main/docker/Dockerfile . ``` This will create a docker image including an extended rdf4j-server as well as an interactive rdf4j-workbench. @@ -166,7 +166,7 @@ To run the docker image, you could invoke this command ```console docker run -p 8081:8081 \ -v $(pwd)/src/test:/var/rdf4j/config \ - tractusx/remoting-agent:1.12.18-SNAPSHOT + tractusx/remoting-agent:1.12.19-SNAPSHOT ```` Afterwards, you should be able to access the [local SparQL endpoint](http://localhost:8081/) via @@ -224,7 +224,7 @@ It can be added to your umbrella chart.yaml by the following snippet dependencies: - name: remoting-agent repository: https://eclipse-tractusx.github.io/charts/dev - version: 1.12.18-SNAPSHOT + version: 1.12.19-SNAPSHOT alias: my-remoting-agent ``` diff --git a/remoting/pom.xml b/remoting/pom.xml index 276eca51..b4a8b183 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -26,7 +26,7 @@ org.eclipse.tractusx agents - 1.12.18-SNAPSHOT + 1.12.19-SNAPSHOT ../pom.xml diff --git a/upgrade_version.sh b/upgrade_version.sh index d4df3391..a432f540 100755 --- a/upgrade_version.sh +++ b/upgrade_version.sh @@ -16,7 +16,7 @@ # # SPDX-License-Identifier: Apache-2.0 -OLD_VERSION=1.12.18-SNAPSHOT +OLD_VERSION=1.12.19-SNAPSHOT echo Upgrading from $OLD_VERSION to $1 PATTERN=s/$OLD_VERSION/$1/g LC_ALL=C From aa2ad833a8cd56881e9704a1cc0bf283eb1b6bd1 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Fri, 19 Apr 2024 07:38:19 +0200 Subject: [PATCH 03/31] fix: unused import and stale reference to renamed implementation class. --- .../agents/http/transfer/AgentSourceController.java | 4 ++-- .../eclipse/tractusx/agents/service/DataManagement.java | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/transfer/AgentSourceController.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/transfer/AgentSourceController.java index a58ac4e6..7aad3f28 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/transfer/AgentSourceController.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/transfer/AgentSourceController.java @@ -38,7 +38,7 @@ import org.eclipse.tractusx.agents.SkillStore; import org.eclipse.tractusx.agents.http.HttpUtils; import org.eclipse.tractusx.agents.rdf.RdfStore; -import org.eclipse.tractusx.agents.service.DataManagementImpl; +import org.eclipse.tractusx.agents.service.DataManagement; import org.eclipse.tractusx.agents.service.EdcSkillStore; import org.eclipse.tractusx.agents.sparql.SparqlQueryProcessor; import org.eclipse.tractusx.agents.sparql.SparqlQuerySerializerFactory; @@ -81,7 +81,7 @@ public AgentSourceController() { SerializerRegistry.get().addQuerySerializer(Syntax.syntaxSPARQL_11, arqQuerySerializerFactory); this.processor = new SparqlQueryProcessor(reg, monitor, config, rdfStore, typeManager); OkHttpClient httpClient = new OkHttpClient(); - DataManagementImpl catalogService = new DataManagementImpl(monitor, typeManager, httpClient, config); + DataManagement catalogService = new DataManagement(monitor, typeManager, httpClient, config); skillStore = new EdcSkillStore(catalogService, typeManager, config); } diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java index 4b031fb2..2a8bb317 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java @@ -26,7 +26,6 @@ import okhttp3.ResponseBody; import org.eclipse.tractusx.agents.AgentConfig; import org.eclipse.tractusx.agents.jsonld.JsonLd; -import org.eclipse.tractusx.agents.matchmaking.SharedObjectManager; import org.eclipse.tractusx.agents.model.Asset; import org.eclipse.tractusx.agents.model.ContractAgreement; import org.eclipse.tractusx.agents.model.ContractNegotiation; @@ -377,7 +376,7 @@ public IdResponse createOrUpdateSkill(String assetId, String name, String descri var assetSpec = String.format(body, assetId, name, description, version, contract, ontologies, distributionMode, isFederated, query, allowServicePattern, denyServicePattern); - return createOrUpdateAsset(assetId, assetSpec); + return createOrUpdateAsset(assetId, assetSpec); } /** @@ -396,7 +395,9 @@ public IdResponse createOrUpdateSkill(String assetId, String name, String descri * @return idresponse * @throws IOException in case interaction with EDC went wrong */ - public IdResponse createOrUpdateGraph(String assetId, String name, String description, String version, String contract, String ontologies, String shape, boolean isFederated, String allowServicePattern, String denyServicePattern) throws IOException { + public IdResponse createOrUpdateGraph(String assetId, String name, String description, String version, String contract, + String ontologies, String shape, boolean isFederated, String allowServicePattern, + String denyServicePattern) throws IOException { if (contract != null) { contract = String.format(" \"cx-common:publishedUnderContract\": \"%1$s\",\n", contract); } else { From 57087faea87d97e0bbebe654607861e3c3591540 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Tue, 23 Apr 2024 15:41:58 +0200 Subject: [PATCH 04/31] feat: upgrade to jena 4.9 and align with agent plane versions. --- DEPENDENCIES | 92 ++++++++++--------- conforming/src/main/docker/Dockerfile | 7 +- matchmaking/pom.xml | 11 +-- matchmaking/src/main/docker/Dockerfile | 7 +- .../org/eclipse/tractusx/agents/TupleSet.java | 2 +- .../eclipse/tractusx/agents/rdf/RdfStore.java | 1 - pom.xml | 24 ++--- remoting/src/main/docker/Dockerfile | 4 +- 8 files changed, 79 insertions(+), 69 deletions(-) diff --git a/DEPENDENCIES b/DEPENDENCIES index 938e4dab..5e66a2ae 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -18,6 +18,7 @@ maven/mavencentral/com.fasterxml.jackson.module/jackson-module-jaxb-annotations/ maven/mavencentral/com.fasterxml.jackson.module/jackson-module-parameter-names/2.15.2, Apache-2.0, approved, #8803 maven/mavencentral/com.fasterxml.woodstox/woodstox-core/6.5.1, Apache-2.0, approved, #7950 maven/mavencentral/com.github.andrewoma.dexx/collection/0.7, MIT, approved, CQ22160 +maven/mavencentral/com.github.ben-manes.caffeine/caffeine/3.1.6, Apache-2.0, approved, clearlydefined maven/mavencentral/com.github.jsonld-java/jsonld-java/0.13.4, BSD-3-Clause, approved, CQ22136 maven/mavencentral/com.github.jsqlparser/jsqlparser/4.4, LGPL-2.1 OR Apache-2.0, approved, clearlydefined maven/mavencentral/com.github.stephenc.jcip/jcip-annotations/1.0-1, Apache-2.0, approved, CQ21949 @@ -30,13 +31,13 @@ maven/mavencentral/com.google.guava/listenablefuture/9999.0-empty-to-avoid-confl maven/mavencentral/com.google.inject.extensions/guice-assistedinject/5.0.1, Apache-2.0, approved, clearlydefined maven/mavencentral/com.google.inject/guice/5.0.1, Apache-2.0, approved, clearlydefined maven/mavencentral/com.google.j2objc/j2objc-annotations/2.8, Apache-2.0, approved, clearlydefined -maven/mavencentral/com.google.protobuf/protobuf-java/3.22.2, BSD-3-Clause, approved, #8370 +maven/mavencentral/com.google.protobuf/protobuf-java/3.23.3, BSD-3-Clause, approved, #8634 maven/mavencentral/com.jayway.jsonpath/json-path/2.7.0, Apache-2.0, approved, clearlydefined -maven/mavencentral/com.nimbusds/nimbus-jose-jwt/9.31, Apache-2.0, approved, clearlydefined +maven/mavencentral/com.nimbusds/nimbus-jose-jwt/9.37.3, Apache-2.0, approved, #11701 maven/mavencentral/com.opencsv/opencsv/5.3, Apache-2.0, approved, clearlydefined -maven/mavencentral/com.squareup.okhttp3/okhttp/4.9.3, Apache-2.0 AND MPL-2.0, approved, #3225 -maven/mavencentral/com.squareup.okio/okio-jvm/3.4.0, Apache-2.0, approved, clearlydefined -maven/mavencentral/com.squareup.okio/okio/3.4.0, Apache-2.0, approved, clearlydefined +maven/mavencentral/com.squareup.okhttp3/okhttp/4.12.0, Apache-2.0, approved, #11156 +maven/mavencentral/com.squareup.okio/okio-jvm/3.6.0, Apache-2.0, approved, #11158 +maven/mavencentral/com.squareup.okio/okio/3.6.0, Apache-2.0, approved, #11155 maven/mavencentral/com.vaadin.external.google/android-json/0.0.20131108.vaadin1, Apache-2.0, approved, CQ21310 maven/mavencentral/commons-beanutils/commons-beanutils/1.9.4, Apache-2.0, approved, CQ12654 maven/mavencentral/commons-cli/commons-cli/1.5.0, Apache-2.0, approved, clearlydefined @@ -48,21 +49,21 @@ maven/mavencentral/commons-lang/commons-lang/2.6, Apache-2.0, approved, CQ6183 maven/mavencentral/commons-logging/commons-logging/1.2, Apache-2.0, approved, CQ10162 maven/mavencentral/io.github.classgraph/classgraph/4.8.154, MIT, approved, CQ22530 maven/mavencentral/io.github.solf/nullanno/3.0.0, EPL-1.0, approved, #11625 -maven/mavencentral/io.micrometer/micrometer-commons/1.10.5, Apache-2.0 AND (Apache-2.0 AND MIT), approved, #7333 -maven/mavencentral/io.micrometer/micrometer-core/1.10.5, Apache-2.0 AND (Apache-2.0 AND MIT), approved, #6977 +maven/mavencentral/io.micrometer/micrometer-commons/1.11.1, Apache-2.0 AND (Apache-2.0 AND MIT), approved, #9243 +maven/mavencentral/io.micrometer/micrometer-core/1.11.1, Apache-2.0 AND (Apache-2.0 AND MIT), approved, #9238 maven/mavencentral/io.micrometer/micrometer-core/1.9.17, Apache-2.0 AND (Apache-2.0 AND MIT), approved, #7711 -maven/mavencentral/io.micrometer/micrometer-observation/1.10.5, Apache-2.0, approved, #7331 -maven/mavencentral/io.micrometer/micrometer-registry-prometheus/1.10.5, Apache-2.0, approved, #4721 +maven/mavencentral/io.micrometer/micrometer-observation/1.11.1, Apache-2.0, approved, #9242 +maven/mavencentral/io.micrometer/micrometer-registry-prometheus/1.11.1, Apache-2.0, approved, #9805 maven/mavencentral/io.mikael/urlbuilder/2.0.9, Apache-2.0, approved, #9815 -maven/mavencentral/io.netty/netty-buffer/4.1.94.Final, Apache-2.0, approved, CQ21842 -maven/mavencentral/io.netty/netty-codec/4.1.94.Final, Apache-2.0 AND BSD-3-Clause AND MIT, approved, CQ20926 -maven/mavencentral/io.netty/netty-common/4.1.94.Final, Apache-2.0 AND MIT AND CC0-1.0, approved, CQ21843 -maven/mavencentral/io.netty/netty-handler/4.1.94.Final, Apache-2.0 AND BSD-3-Clause AND MIT, approved, CQ20926 -maven/mavencentral/io.netty/netty-resolver/4.1.94.Final, Apache-2.0 AND BSD-3-Clause AND MIT, approved, CQ20926 -maven/mavencentral/io.netty/netty-transport-classes-epoll/4.1.94.Final, Apache-2.0, approved, #6366 -maven/mavencentral/io.netty/netty-transport-native-epoll/4.1.94.Final, Apache-2.0 AND BSD-3-Clause AND MIT, approved, CQ20926 -maven/mavencentral/io.netty/netty-transport-native-unix-common/4.1.94.Final, Apache-2.0 AND BSD-3-Clause AND MIT, approved, CQ20926 -maven/mavencentral/io.netty/netty-transport/4.1.94.Final, Apache-2.0 AND BSD-3-Clause AND MIT, approved, CQ20926 +maven/mavencentral/io.netty/netty-buffer/4.1.101.Final, Apache-2.0, approved, CQ21842 +maven/mavencentral/io.netty/netty-codec/4.1.101.Final, Apache-2.0 AND BSD-3-Clause AND MIT, approved, CQ20926 +maven/mavencentral/io.netty/netty-common/4.1.101.Final, Apache-2.0 AND MIT AND CC0-1.0, approved, CQ21843 +maven/mavencentral/io.netty/netty-handler/4.1.101.Final, Apache-2.0 AND BSD-3-Clause AND MIT, approved, CQ20926 +maven/mavencentral/io.netty/netty-resolver/4.1.101.Final, Apache-2.0 AND BSD-3-Clause AND MIT, approved, CQ20926 +maven/mavencentral/io.netty/netty-transport-classes-epoll/4.1.101.Final, Apache-2.0, approved, #6366 +maven/mavencentral/io.netty/netty-transport-native-epoll/4.1.101.Final, Apache-2.0 AND BSD-3-Clause AND MIT, approved, CQ20926 +maven/mavencentral/io.netty/netty-transport-native-unix-common/4.1.101.Final, Apache-2.0 AND BSD-3-Clause AND MIT, approved, CQ20926 +maven/mavencentral/io.netty/netty-transport/4.1.101.Final, Apache-2.0 AND BSD-3-Clause AND MIT, approved, CQ20926 maven/mavencentral/io.prometheus/simpleclient/0.16.0, Apache-2.0, approved, clearlydefined maven/mavencentral/io.prometheus/simpleclient_common/0.16.0, Apache-2.0, approved, clearlydefined maven/mavencentral/io.prometheus/simpleclient_tracer_common/0.16.0, Apache-2.0, approved, clearlydefined @@ -122,23 +123,22 @@ maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.14, Apache-2.0, approv maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16, Apache-2.0, approved, CQ23528 maven/mavencentral/org.apache.httpcomponents/httpmime/4.5.13, Apache-2.0, approved, CQ11718 maven/mavencentral/org.apache.httpcomponents/httpmime/4.5.14, Apache-2.0, approved, CQ11718 -maven/mavencentral/org.apache.jena/jena-arq/4.8.0, Apache-2.0 AND (Apache-2.0 AND EPL-2.0) AND (Apache-2.0 AND EPL-1.0), approved, #8883 -maven/mavencentral/org.apache.jena/jena-base/4.8.0, Apache-2.0, approved, #8887 -maven/mavencentral/org.apache.jena/jena-core/4.8.0, Apache-2.0 AND (Apache-2.0 AND BSD-3-Clause), approved, #8873 -maven/mavencentral/org.apache.jena/jena-dboe-base/4.8.0, Apache-2.0, approved, #8884 -maven/mavencentral/org.apache.jena/jena-dboe-index/4.8.0, Apache-2.0, approved, #8879 -maven/mavencentral/org.apache.jena/jena-dboe-storage/4.8.0, Apache-2.0, approved, #8907 -maven/mavencentral/org.apache.jena/jena-dboe-trans-data/4.8.0, Apache-2.0, approved, #8874 -maven/mavencentral/org.apache.jena/jena-dboe-transaction/4.8.0, Apache-2.0, approved, #8892 -maven/mavencentral/org.apache.jena/jena-fuseki-core/4.8.0, Apache-2.0 AND (EPL-2.0 OR Apache-2.0), approved, #9841 -maven/mavencentral/org.apache.jena/jena-iri/4.8.0, Apache-2.0, approved, clearlydefined -maven/mavencentral/org.apache.jena/jena-rdfconnection/4.8.0, Apache-2.0, approved, #8897 -maven/mavencentral/org.apache.jena/jena-rdfpatch/4.8.0, Apache-2.0, approved, #8903 -maven/mavencentral/org.apache.jena/jena-shacl/4.8.0, Apache-2.0 AND W3C-20150513, approved, #8905 -maven/mavencentral/org.apache.jena/jena-shaded-guava/4.8.0, Apache-2.0 AND CC0-1.0 AND LicenseRef-Public-Domain, approved, #8877 -maven/mavencentral/org.apache.jena/jena-shex/4.8.0, Apache-2.0, approved, #8908 -maven/mavencentral/org.apache.jena/jena-tdb/4.8.0, Apache-2.0, approved, #8909 -maven/mavencentral/org.apache.jena/jena-tdb2/4.8.0, Apache-2.0, approved, #8881 +maven/mavencentral/org.apache.jena/jena-arq/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-base/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-core/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-dboe-base/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-dboe-index/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-dboe-storage/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-dboe-trans-data/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-dboe-transaction/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-fuseki-core/4.9.0, Apache-2.0, restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-iri/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-rdfconnection/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-rdfpatch/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-shacl/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-shex/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-tdb/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-tdb2/4.9.0, , restricted, clearlydefined maven/mavencentral/org.apache.logging.log4j/log4j-api/2.17.2, Apache-2.0, approved, clearlydefined maven/mavencentral/org.apache.logging.log4j/log4j-to-slf4j/2.17.2, Apache-2.0, approved, #2163 maven/mavencentral/org.apache.lucene/lucene-analyzers-common/8.9.0, Apache-2.0 AND BSD-3-Clause AND LicenseRef-scancode-iso-8879, approved, #6152 @@ -182,7 +182,7 @@ maven/mavencentral/org.eclipse.jetty/jetty-io/9.4.53.v20231009, EPL-2.0 OR Apach maven/mavencentral/org.eclipse.jetty/jetty-security/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty/jetty-server/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty/jetty-servlet/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-servlets/10.0.14, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-servlets/10.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty/jetty-util/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty/jetty-util/9.4.53.v20231009, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.rdf4j/rdf4j-client/4.3.8, BSD-3-Clause, approved, technology.rdf4j @@ -299,16 +299,22 @@ maven/mavencentral/org.hamcrest/hamcrest/2.2, BSD-3-Clause, approved, clearlydef maven/mavencentral/org.hdrhistogram/HdrHistogram/2.1.12, BSD-2-Clause OR LicenseRef-Public-Domain, approved, CQ13192 maven/mavencentral/org.javabits.jgrapht/jgrapht-core/0.9.3, EPL-2.0 OR LGPL-2.1-or-later, approved, #9816 maven/mavencentral/org.javassist/javassist/3.29.2-GA, Apache-2.0 AND LGPL-2.1-or-later AND MPL-1.1, approved, #6023 -maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib-common/1.8.0, Apache-2.0, approved, #8910 -maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib-jdk7/1.8.0, Apache-2.0, approved, #8807 -maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib-jdk8/1.8.0, Apache-2.0, approved, #8919 -maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib/1.4.10, Apache-2.0, approved, clearlydefined +maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib-common/1.9.10, Apache-2.0, approved, #14186 +maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib-jdk7/1.8.21, Apache-2.0, approved, #8807 +maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib-jdk8/1.8.21, Apache-2.0, approved, #8919 +maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib/1.8.21, Apache-2.0, approved, #8865 maven/mavencentral/org.jetbrains/annotations/13.0, Apache-2.0, approved, clearlydefined +maven/mavencentral/org.junit.jupiter/junit-jupiter-api/5.10.2, EPL-2.0, approved, #9714 maven/mavencentral/org.junit.jupiter/junit-jupiter-api/5.9.2, EPL-2.0, approved, #3133 +maven/mavencentral/org.junit.jupiter/junit-jupiter-engine/5.10.2, EPL-2.0, approved, #9711 maven/mavencentral/org.junit.jupiter/junit-jupiter-engine/5.9.2, EPL-2.0, approved, #3125 +maven/mavencentral/org.junit.jupiter/junit-jupiter-params/5.10.2, EPL-2.0, approved, #9708 maven/mavencentral/org.junit.jupiter/junit-jupiter-params/5.9.2, EPL-2.0, approved, #3134 +maven/mavencentral/org.junit.jupiter/junit-jupiter/5.10.2, EPL-2.0, approved, #13393 maven/mavencentral/org.junit.jupiter/junit-jupiter/5.9.2, EPL-2.0, approved, #6972 +maven/mavencentral/org.junit.platform/junit-platform-commons/1.10.2, EPL-2.0, approved, #9715 maven/mavencentral/org.junit.platform/junit-platform-commons/1.9.2, EPL-2.0, approved, #3130 +maven/mavencentral/org.junit.platform/junit-platform-engine/1.10.2, EPL-2.0, approved, #9709 maven/mavencentral/org.junit.platform/junit-platform-engine/1.9.2, EPL-2.0, approved, #3128 maven/mavencentral/org.jvnet.mimepull/mimepull/1.9.15, CDDL-1.1 OR GPL-2.0-only WITH Classpath-exception-2.0, approved, CQ21484 maven/mavencentral/org.latencyutils/LatencyUtils/2.0.3, BSD-2-Clause, approved, CQ17408 @@ -322,8 +328,11 @@ maven/mavencentral/org.mockito/mockito-core/4.11.0, MIT AND (Apache-2.0 AND MIT) maven/mavencentral/org.mockito/mockito-junit-jupiter/4.11.0, MIT, approved, clearlydefined maven/mavencentral/org.objenesis/objenesis/3.3, Apache-2.0, approved, clearlydefined maven/mavencentral/org.opentest4j/opentest4j/1.2.0, Apache-2.0, approved, clearlydefined +maven/mavencentral/org.opentest4j/opentest4j/1.3.0, Apache-2.0, approved, #9713 maven/mavencentral/org.ow2.asm/asm/9.1, BSD-3-Clause, approved, CQ23029 maven/mavencentral/org.postgresql/postgresql/42.6.0, BSD-2-Clause AND Apache-2.0, approved, #9159 +maven/mavencentral/org.roaringbitmap/RoaringBitmap/0.9.45, Apache-2.0, approved, clearlydefined +maven/mavencentral/org.roaringbitmap/shims/0.9.45, Apache-2.0, approved, clearlydefined maven/mavencentral/org.simpleframework/simple-common/6.0.1, Apache-2.0, approved, CQ16868 maven/mavencentral/org.simpleframework/simple-http/6.0.1, Apache-2.0, approved, CQ16869 maven/mavencentral/org.simpleframework/simple-transport/6.0.1, Apache-2.0, approved, CQ16870 @@ -331,7 +340,7 @@ maven/mavencentral/org.skyscreamer/jsonassert/1.5.1, Apache-2.0, approved, clear maven/mavencentral/org.slf4j/jcl-over-slf4j/1.7.32, Apache-2.0, approved, CQ12843 maven/mavencentral/org.slf4j/jcl-over-slf4j/1.7.36, Apache-2.0, approved, CQ12843 maven/mavencentral/org.slf4j/jul-to-slf4j/1.7.36, MIT, approved, CQ12842 -maven/mavencentral/org.slf4j/slf4j-api/2.0.7, MIT, approved, #5915 +maven/mavencentral/org.slf4j/slf4j-api/2.0.12, MIT, approved, #5915 maven/mavencentral/org.springframework.boot/spring-boot-actuator-autoconfigure/2.7.18, Apache-2.0, approved, #3273 maven/mavencentral/org.springframework.boot/spring-boot-actuator/2.7.18, Apache-2.0, approved, #4316 maven/mavencentral/org.springframework.boot/spring-boot-autoconfigure/2.7.18, Apache-2.0, approved, #4314 @@ -362,3 +371,4 @@ maven/mavencentral/org.unbescape/unbescape/1.1.6.RELEASE, Apache-2.0, approved, maven/mavencentral/org.xerial.snappy/snappy-java/1.1.10.5, Apache-2.0 AND (Apache-2.0 AND BSD-3-Clause), approved, #9098 maven/mavencentral/org.xmlunit/xmlunit-core/2.9.1, Apache-2.0, approved, #6272 maven/mavencentral/org.yaml/snakeyaml/2.0, Apache-2.0 AND (Apache-2.0 OR BSD-3-Clause OR EPL-1.0 OR GPL-2.0-or-later OR LGPL-2.1-or-later), approved, #7275 +maven/mavencentral/org.yaml/snakeyaml/2.2, Apache-2.0 AND (Apache-2.0 OR BSD-3-Clause OR EPL-1.0 OR GPL-2.0-or-later OR LGPL-2.1-or-later), approved, #10232 diff --git a/conforming/src/main/docker/Dockerfile b/conforming/src/main/docker/Dockerfile index 6ccd2747..4745853c 100644 --- a/conforming/src/main/docker/Dockerfile +++ b/conforming/src/main/docker/Dockerfile @@ -15,7 +15,7 @@ # # SPDX-License-Identifier: Apache-2.0 -FROM eclipse-temurin:21-jre-alpine +FROM eclipse-temurin:22_36-jre-alpine ARG JAR ARG LIB @@ -45,4 +45,7 @@ HEALTHCHECK NONE EXPOSE 8080 -ENTRYPOINT ["java","-cp","/app/lib/*", "org.eclipse.tractusx.agents.conforming.Bootstrap"] \ No newline at end of file +ENTRYPOINT ["java", \ + "-cp", \ + "/app/lib/*", \ + "org.eclipse.tractusx.agents.conforming.Bootstrap"] \ No newline at end of file diff --git a/matchmaking/pom.xml b/matchmaking/pom.xml index 3bc2242c..f58b93d4 100644 --- a/matchmaking/pom.xml +++ b/matchmaking/pom.xml @@ -257,17 +257,12 @@ com.squareup.okhttp3 okhttp ${okhttp.version} - + diff --git a/matchmaking/src/main/docker/Dockerfile b/matchmaking/src/main/docker/Dockerfile index 625b0346..022c0b86 100644 --- a/matchmaking/src/main/docker/Dockerfile +++ b/matchmaking/src/main/docker/Dockerfile @@ -15,7 +15,7 @@ # # SPDX-License-Identifier: Apache-2.0 -FROM eclipse-temurin:21-jre-alpine +FROM eclipse-temurin:22_36-jre-alpine ARG JAR ARG LIB @@ -47,4 +47,7 @@ HEALTHCHECK NONE EXPOSE 8080 -ENTRYPOINT ["java","-cp","/app/lib/*", "org.eclipse.tractusx.agents.matchmaking.Bootstrap"] \ No newline at end of file +ENTRYPOINT ["java", \ + "-cp", \ + "/app/lib/*", \ + "org.eclipse.tractusx.agents.matchmaking.Bootstrap"] \ No newline at end of file diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/TupleSet.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/TupleSet.java index c1805544..aea210ad 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/TupleSet.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/TupleSet.java @@ -16,7 +16,7 @@ // SPDX-License-Identifier: Apache-2.0 package org.eclipse.tractusx.agents; -import org.apache.jena.ext.com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ArrayListMultimap; import java.util.ArrayList; import java.util.Arrays; diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/rdf/RdfStore.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/rdf/RdfStore.java index 1157e57f..be47c584 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/rdf/RdfStore.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/rdf/RdfStore.java @@ -36,7 +36,6 @@ import org.eclipse.tractusx.agents.MonitorWrapper; import org.eclipse.tractusx.agents.utils.Monitor; -//import java.io.*; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.IOException; diff --git a/pom.xml b/pom.xml index 3c812737..d0d6fe60 100644 --- a/pom.xml +++ b/pom.xml @@ -37,28 +37,29 @@ 11 false - 5.9.2 - 4.6.1 - 3.2.4 - 4.9.3 + 5.10.2 + 5.2.0 + 3.3.2 + 4.12.0 4.3.8 4.5.14 1.2 2.15.2 3.1.0 - 9.31 - 5.0.2 + 9.37.3 4.0.1 + 5.0.2 2.3.1 5.1.2 + 2.5 3.1.0 5.3.31 2.7.18 32.1.2-jre 9.0.83 - 4.1.94.Final + 4.1.101.Final 9.4.53.v20231009 - 2.0 + 2.2 1.1.10.5 42.6.0 3.7.2 @@ -66,13 +67,12 @@ 2.40 3.1.3 11.0.15 - 4.8.0 - 3.4.0 - 2.5 + 4.9.0 + 3.6.0 2.0.1.Final 2.10.1 1.2.13 - 2.0.7 + 2.0.12 UTF-8 diff --git a/remoting/src/main/docker/Dockerfile b/remoting/src/main/docker/Dockerfile index 1fd1eca0..b3d32d74 100644 --- a/remoting/src/main/docker/Dockerfile +++ b/remoting/src/main/docker/Dockerfile @@ -19,7 +19,7 @@ # Build Container: Fixes diverse vulnerabilities in guava <32, netty, jetty, spring-framework <5.3.28 and spring-web (all 5 versions - need to exclude a deprecated package from the jar) ## -FROM eclipse-temurin:21-jdk-alpine AS build +FROM eclipse-temurin:22_36-jdk-alpine AS build COPY target/original-remoting-agent-*.jar /opt/lib/ #COPY target/lib/guava-*.jar /opt/lib/ @@ -66,7 +66,7 @@ COPY resources/web/callback.xml /tmp/rdf4j-server/WEB-INF/callback.xml # Target Container: Use a valid base image ## -FROM eclipse-temurin:21-jre-alpine +FROM eclipse-temurin:22_36-jre-alpine ARG APP_USER=tomcat ARG APP_UID=10001 From 0dc166e60724ecbc0af0251db75350d801ba0ae0 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Mon, 29 Apr 2024 14:42:58 +0200 Subject: [PATCH 05/31] feature: switch to EDC 0.7.0 management and transfer callback. --- .../agents/AgreementControllerImpl.java | 29 +++--- .../agents/utils/EndpointDataReference.java | 85 ++++++++++------ .../eclipse/tractusx/agents/utils/Event.java | 33 +++++++ .../tractusx/agents/utils/EventEnvelope.java | 81 ++++++++++++++++ .../agents/utils/TransferProcessEvent.java | 96 +++++++++++++++++++ .../agents/utils/TransferProcessStarted.java | 63 ++++++++++++ 6 files changed, 342 insertions(+), 45 deletions(-) create mode 100644 matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/Event.java create mode 100644 matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/EventEnvelope.java create mode 100644 matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/TransferProcessEvent.java create mode 100644 matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/TransferProcessStarted.java diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java index d54d0926..9906d579 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java @@ -43,7 +43,9 @@ import org.eclipse.tractusx.agents.utils.CallbackAddress; import org.eclipse.tractusx.agents.utils.DataAddress; import org.eclipse.tractusx.agents.utils.EndpointDataReference; +import org.eclipse.tractusx.agents.utils.EventEnvelope; import org.eclipse.tractusx.agents.utils.Monitor; +import org.eclipse.tractusx.agents.utils.TransferProcessStarted; import java.io.IOException; import java.text.ParseException; @@ -117,21 +119,20 @@ public String toString() { * @param dataReference contains the actual call token */ @POST - public void receiveEdcCallback(EndpointDataReference dataReference) { - var agreementId = dataReference.getId(); - monitor.debug(String.format("An endpoint data reference for agreement %s has been posted.", agreementId)); - synchronized (processStore) { - for (Map.Entry process : processStore.entrySet()) { - if (process.getValue().getId().equals(agreementId)) { - synchronized (endpointStore) { - monitor.debug(String.format("Agreement %s belongs to asset %s.", agreementId, process.getKey())); - endpointStore.put(process.getKey(), dataReference); - return; - } - } - } + public void receiveEdcCallback(EventEnvelope dataReference) { + var processId = dataReference.getPayload().getTransferProcessId(); + var assetId = dataReference.getPayload().getAssetId(); + monitor.debug(String.format("A transfer process %s for asset %s has been started.", processId, assetId)); + synchronized (endpointStore) { + EndpointDataReference newRef = EndpointDataReference.Builder.newInstance() + .id(dataReference.getId()) + .contractId(dataReference.getPayload().getContractId()) + .endpoint(dataReference.getPayload().getDataAddress().getStringProperty("https://w3id.org/edc/v0.0.1/ns/endpoint", null)) + .authCode("Authorization") + .authKey(dataReference.getPayload().getDataAddress().getStringProperty("https://w3id.org/edc/v0.0.1/ns/authorization", null)) + .build(); + endpointStore.put(assetId, newRef); } - monitor.debug(String.format("Agreement %s has no active asset. Guess that came for another plane. Ignoring.", agreementId)); } /** diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/EndpointDataReference.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/EndpointDataReference.java index c71fb0fb..d885c4b1 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/EndpointDataReference.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/EndpointDataReference.java @@ -24,34 +24,32 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; -import java.util.UUID; import static org.eclipse.tractusx.agents.utils.CoreConstants.EDC_NAMESPACE; /** * Describes an endpoint serving data. */ + @JsonDeserialize(builder = EndpointDataReference.Builder.class) @JsonIgnoreProperties(ignoreUnknown = true) public class EndpointDataReference { + public static final String EDR_SIMPLE_TYPE = "EDR"; public static final String ID = EDC_NAMESPACE + "id"; + public static final String CONTRACT_ID = EDC_NAMESPACE + "contractId"; public static final String AUTH_CODE = EDC_NAMESPACE + "authCode"; public static final String AUTH_KEY = EDC_NAMESPACE + "authKey"; public static final String ENDPOINT = EDC_NAMESPACE + "endpoint"; - private final String id; - private final String endpoint; - private final String authKey; - private final String authCode; - private final Map properties; - - private EndpointDataReference(String id, String endpoint, String authKey, String authCode, Map properties) { - this.id = id; - this.endpoint = endpoint; - this.authKey = authKey; - this.authCode = authCode; - this.properties = properties; + private final Map properties = new HashMap<>(); + private String id; + private String contractId; + private String endpoint; + private String authKey; + private String authCode; + + private EndpointDataReference() { } @NotNull @@ -59,6 +57,11 @@ public String getId() { return id; } + @NotNull + public String getContractId() { + return contractId; + } + @NotNull public String getEndpoint() { return endpoint; @@ -79,15 +82,24 @@ public Map getProperties() { return properties; } + @Override + public String toString() { + return "EndpointDataReference{" + + "properties=" + properties + + ", id='" + id + '\'' + + ", contractId='" + contractId + '\'' + + ", endpoint='" + endpoint + '\'' + + ", authKey='" + authKey + '\'' + + ", authCode='" + authCode + '\'' + + '}'; + } + @JsonPOJOBuilder(withPrefix = "") public static class Builder { - private final Map properties = new HashMap<>(); - private String id = UUID.randomUUID().toString(); - private String endpoint; - private String authKey; - private String authCode; + private final EndpointDataReference edr; private Builder() { + edr = new EndpointDataReference(); } @JsonCreator @@ -96,40 +108,51 @@ public static EndpointDataReference.Builder newInstance() { } public EndpointDataReference.Builder id(String id) { - this.id = id; + edr.id = id; + return this; + } + + public EndpointDataReference.Builder contractId(String contractId) { + edr.contractId = contractId; return this; } public EndpointDataReference.Builder endpoint(String address) { - this.endpoint = address; + edr.endpoint = address; return this; } public EndpointDataReference.Builder authKey(String authKey) { - this.authKey = authKey; + edr.authKey = authKey; return this; } public EndpointDataReference.Builder authCode(String authCode) { - this.authCode = authCode; + edr.authCode = authCode; + return this; + } + + public EndpointDataReference.Builder property(String key, Object value) { + edr.properties.put(key, value); return this; } public EndpointDataReference.Builder properties(Map properties) { - this.properties.putAll(properties); + edr.properties.putAll(properties); return this; } public EndpointDataReference build() { - Objects.requireNonNull(endpoint, "endpoint"); - if (authKey != null) { - Objects.requireNonNull(authCode, "authCode"); + Objects.requireNonNull(edr.id, "id"); + Objects.requireNonNull(edr.contractId, "contractId"); + Objects.requireNonNull(edr.endpoint, "endpoint"); + if (edr.authKey != null) { + Objects.requireNonNull(edr.authCode, "authCode"); } - if (authCode != null) { - Objects.requireNonNull(authKey, "authKey"); + if (edr.authCode != null) { + Objects.requireNonNull(edr.authKey, "authKey"); } - return new EndpointDataReference(id, endpoint, authKey, authCode, properties); + return edr; } } -} - +} \ No newline at end of file diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/Event.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/Event.java new file mode 100644 index 00000000..391df97b --- /dev/null +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/Event.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024 T-Systems International GmbH + * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ +package org.eclipse.tractusx.agents.utils; + +import java.util.ArrayList; +import java.util.List; + +public abstract class Event { + + public List getCallbackAddresses() { + return new ArrayList<>(); + } + + + /** + * The name of the event in dot notation. + * + * @return the event name. + */ + public abstract String name(); +} \ No newline at end of file diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/EventEnvelope.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/EventEnvelope.java new file mode 100644 index 00000000..344e2430 --- /dev/null +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/EventEnvelope.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2024 T-Systems International GmbH + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ +package org.eclipse.tractusx.agents.utils; + +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.util.UUID; + +public class EventEnvelope { + + protected String id; + + protected long at; + + protected E payload; + + public String getId() { + return id; + } + + public long getAt() { + return at; + } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", include = JsonTypeInfo.As.EXTERNAL_PROPERTY) + public E getPayload() { + return payload; + } + + public static class Builder { + + protected final EventEnvelope envelope; + + protected Builder() { + envelope = new EventEnvelope<>(); + } + + public static Builder newInstance() { + return new Builder<>(); + } + + public Builder id(String id) { + envelope.id = id; + return this; + } + + public Builder at(long at) { + envelope.at = at; + return this; + } + + public Builder payload(E payload) { + envelope.payload = payload; + return this; + } + + public EventEnvelope build() { + if (envelope.id == null) { + envelope.id = UUID.randomUUID().toString(); + } + if (envelope.at == 0) { + throw new IllegalStateException("Event 'at' field must be set"); + } + return envelope; + } + + } + +} diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/TransferProcessEvent.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/TransferProcessEvent.java new file mode 100644 index 00000000..4ab685f9 --- /dev/null +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/TransferProcessEvent.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2024 T-Systems International GmbH + * Copyright (c) 2022 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ +package org.eclipse.tractusx.agents.utils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Class as organizational between level to catch events of type TransferProcess to catch them together in an Event Subscriber + * Contains data related to transfer processes + */ +public abstract class TransferProcessEvent extends Event { + + protected String transferProcessId; + protected List callbackAddresses = new ArrayList<>(); + protected String assetId; + protected String type; + protected String contractId; + + public String getTransferProcessId() { + return transferProcessId; + } + + public String getAssetId() { + return assetId; + } + + public String getType() { + return type; + } + + public String getContractId() { + return contractId; + } + + @Override + public List getCallbackAddresses() { + return callbackAddresses; + } + + public abstract static class Builder> { + + protected final T event; + + protected Builder(T event) { + this.event = event; + } + + public B transferProcessId(String transferProcessId) { + event.transferProcessId = transferProcessId; + return (B) this; + } + + public B assetId(String assetId) { + event.assetId = assetId; + return (B) this; + } + + public B callbackAddresses(List callbackAddresses) { + event.callbackAddresses = callbackAddresses; + return self(); + } + + public B type(String type) { + event.type = type; + return self(); + } + + public B contractId(String contractId) { + event.contractId = contractId; + return self(); + } + + public abstract B self(); + + public T build() { + Objects.requireNonNull(event.transferProcessId, "transferProcess id can't be null"); + return event; + } + + } + +} diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/TransferProcessStarted.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/TransferProcessStarted.java new file mode 100644 index 00000000..ac99da64 --- /dev/null +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/utils/TransferProcessStarted.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024 T-Systems International GmbH + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ +package org.eclipse.tractusx.agents.utils; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; + +/** + * This event is raised when the TransferProcess has been started. + */ +@JsonDeserialize(builder = TransferProcessStarted.Builder.class) +public class TransferProcessStarted extends TransferProcessEvent { + private DataAddress dataAddress; + + private TransferProcessStarted() { + } + + public DataAddress getDataAddress() { + return dataAddress; + } + + @Override + public String name() { + return "transfer.process.started"; + } + + @JsonPOJOBuilder(withPrefix = "") + public static class Builder extends TransferProcessEvent.Builder { + + private Builder() { + super(new TransferProcessStarted()); + } + + @JsonCreator + public static Builder newInstance() { + return new Builder(); + } + + public Builder dataAddress(DataAddress dataAddress) { + event.dataAddress = dataAddress; + return this; + } + + @Override + public Builder self() { + return this; + } + } + +} \ No newline at end of file From 8383b88da1142c53443bf09f542e76f2507da37c Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Tue, 30 Apr 2024 14:09:14 +0200 Subject: [PATCH 06/31] feat: use the EDC 0.7 management peculiarities including the transfer process callback. --- .../agents/service/DataManagement.java | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java index 2a8bb317..371e6a93 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java @@ -72,16 +72,15 @@ public class DataManagement { */ public static final String SKILL_ASSET_CREATE_BODY = "{\n" + " \"@context\": {\n" + + " \"@vocab\": \"https://w3id.org/edc/v0.0.1/ns/\"," + " \"rdf\": \"http://www.w3.org/1999/02/22-rdf-syntax-ns#\",\n" + " \"rdfs\": \"http://www.w3.org/2000/01/rdf-schema#\",\n" + " \"cx-common\": \"https://w3id.org/catenax/ontology/common#\",\n" + " \"xsd\": \"http://www.w3.org/2001/XMLSchema#\",\n" + " \"sh\": \"http://www.w3.org/ns/shacl#\"\n" + " },\n" + - " \"asset\": {\n" + - " \"@type\": \"Asset\",\n" + - " \"@id\": \"%1$s\", \n" + - " \"properties\": {\n" + + " \"@id\": \"%1$s\", \n" + + " \"properties\": {\n" + " \"name\": \"%2$s\",\n" + " \"description\": \"%3$s\",\n" + " \"version\": \"%4$s\",\n" + @@ -92,10 +91,9 @@ public class DataManagement { " \"cx-common:implementsProtocol\": \"cx-common:Protocol?w3c:http:SKILL\",\n" + " \"cx-common:distributionMode\": \"%7$s\",\n" + " \"cx-common:isFederated\": \"%8$b^^xsd:boolean\"\n" + - " },\n" + - " \"privateProperties\": {\n" + + " },\n" + + " \"privateProperties\": {\n" + " \"cx-common:query\":%9$s\n" + - " }\n" + " },\n" + " \"dataAddress\": {\n" + " \"id\":\"%1$s\",\n" + @@ -116,16 +114,15 @@ public class DataManagement { */ public static final String GRAPH_ASSET_CREATE_BODY = "{\n" + " \"@context\": {\n" + + " \"@vocab\": \"https://w3id.org/edc/v0.0.1/ns/\"," + " \"rdf\": \"http://www.w3.org/1999/02/22-rdf-syntax-ns#\",\n" + " \"rdfs\": \"http://www.w3.org/2000/01/rdf-schema#\",\n" + " \"cx-common\": \"https://w3id.org/catenax/ontology/common#\",\n" + " \"xsd\": \"http://www.w3.org/2001/XMLSchema#\",\n" + " \"sh\": \"http://www.w3.org/ns/shacl#\"\n" + " },\n" + - " \"asset\": {\n" + - " \"@type\": \"Asset\",\n" + - " \"@id\": \"%1$s\", \n" + - " \"properties\": {\n" + + " \"@id\": \"%1$s\", \n" + + " \"properties\": {\n" + " \"name\": \"%2$s\",\n" + " \"description\": \"%3$s\",\n" + " \"version\": \"%4$s\",\n" + @@ -136,7 +133,8 @@ public class DataManagement { " \"cx-common:implementsProtocol\": \"cx-common:Protocol?w3c:http:SPARQL\",\n" + " \"sh:shapesGraph\": \"%7$s\",\n" + " \"cx-common:isFederated\": \"%8$b^^xsd:boolean\"\n" + - " }\n" + + " },\n" + + " \"privateProperties\": {\n" + " },\n" + " \"dataAddress\": {\n" + " \"id\": \"%1$s\", \n" + @@ -185,14 +183,20 @@ public class DataManagement { " \"type\": \"HttpProxy\"\n" + " },\n" + " \"managedResources\": false,\n" + - " \"privateProperties\": {\n" + - " \"receiverHttpEndpoint\": \"%4$s\"\n" + - " },\n" + " \"protocol\": \"dataspace-protocol-http\",\n" + " \"transferType\": {\n" + " \"contentType\": \"application/octet-stream\",\n" + " \"isFinite\": true\n" + " }\n" + + " \"callbackAddresses\": [\n" + + " {\n" + + " \"transactional\": false,\n" + + " \"uri\": \"%4$s\",\n" + + " \"events\": [\n" + + " \"transfer.process.started\"\n" + + " ],\n" + + " }\n" + + " ] \n" + "}"; public static final String TRANSFER_CHECK_CALL = "%s/v2/transferprocesses/%s"; public static final String AGREEMENT_CHECK_CALL = "%s/v2/contractagreements/%s"; From 8d8ad838fe0e638b9833e5e04939601e93885416 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Mon, 6 May 2024 08:39:08 +0200 Subject: [PATCH 07/31] fix: debug the transfer process/callback mechanism. --- .../eclipse/tractusx/agents/AgentConfig.java | 53 ++++++++++---- .../agents/AgreementControllerImpl.java | 6 +- .../tractusx/agents/jsonld/JsonLd.java | 4 ++ .../tractusx/agents/model/DcatCatalog.java | 7 +- .../tractusx/agents/model/OdrlPolicy.java | 13 ++++ .../agents/service/DataManagement.java | 71 +++++++++++-------- .../agents/service/DataspaceSynchronizer.java | 14 ++-- 7 files changed, 112 insertions(+), 56 deletions(-) diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java index 43f62468..986b7226 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java @@ -19,7 +19,9 @@ import org.eclipse.tractusx.agents.utils.Config; import org.eclipse.tractusx.agents.utils.Monitor; +import java.util.HashMap; import java.util.Map; +import java.util.UUID; import java.util.regex.Pattern; /** @@ -49,22 +51,22 @@ public class AgentConfig { public static final String NEGOTIATION_TIMEOUT_PROPERTY = "cx.agent.negotiation.timeout"; public static final long DEFAULT_NEGOTIATION_TIMEOUT = 30000; - + public static final String NEGOTIATION_POLLINTERVAL_PROPERTY = "cx.agent.negotiation.poll"; public static final long DEFAULT_NEGOTIATION_POLLINTERVAL = 1000; - + public static final String DATASPACE_SYNCINTERVAL_PROPERTY = "cx.agent.dataspace.synchronization"; public static final long DEFAULT_DATASPACE_SYNCINTERVAL = -1; - + public static final String DATASPACE_SYNCCONNECTORS_PROPERTY = "cx.agent.dataspace.remotes"; public static final String RDF_STORE = "cx.agent.rdf.store"; public static final String VALIDATION_ENDPOINTS = "edc.dataplane.token.validation.endpoints"; - + public static final String FEDERATION_SERVICE_BATCH_SIZE = "cx.agent.federation.batch.max"; public static final long DEFAULT_FEDERATION_SERVICE_BATCH_SIZE = Long.MAX_VALUE; - + public static final String THREAD_POOL_SIZE = "cx.agent.threadpool.size"; public static final int DEFAULT_THREAD_POOL_SIZE = 4; @@ -110,7 +112,7 @@ public class AgentConfig { protected final Pattern serviceDenyPattern; protected final Pattern serviceAssetAllowPattern; protected final Pattern serviceAssetDenyPattern; - + /** * references to EDC services */ @@ -121,7 +123,7 @@ public class AgentConfig { * creates the typed config * * @param monitor logger - * @param config untyped config + * @param config untyped config */ public AgentConfig(Monitor monitor, Config config) { this.monitor = monitor; @@ -253,19 +255,42 @@ public long getDataspaceSynchronizationInterval() { return config.getLong(DATASPACE_SYNCINTERVAL_PROPERTY, DEFAULT_DATASPACE_SYNCINTERVAL); } + protected volatile Map knownConnectors; + /** * access * - * @return array of connector urls to synchronize, null if no sync + * @return map of business partner ids to connector urls to synchronize with, null if no sync */ - public String[] getDataspaceSynchronizationConnectors() { - String[] connectors = config.getString(DATASPACE_SYNCCONNECTORS_PROPERTY, "").split(","); - if (connectors.length == 1 && (connectors[0] == null || connectors[0].length() == 0)) { - return null; + public Map getDataspaceSynchronizationConnectors() { + if (knownConnectors == null) { + synchronized (config) { + if (knownConnectors == null) { + knownConnectors = new HashMap<>(); + String[] connectors = config.getString(DATASPACE_SYNCCONNECTORS_PROPERTY, "").split(","); + for (String connector : connectors) { + String[] entry = connector.split("="); + if (entry.length > 0) { + String key = UUID.randomUUID().toString(); + String value = entry[0]; + if (entry.length > 1) { + key = entry[0]; + value = entry[1]; + } + knownConnectors.put(key, value); + } + } + } + } } - return connectors; + return knownConnectors; } - + + /** + * access + * + * @return the location of the rdf store + */ public String getRdfStore() { return config.getString(RDF_STORE, null); } diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java index 9906d579..6ac46774 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java @@ -61,7 +61,7 @@ * An endpoint/service that receives information from the control plane */ @Consumes({MediaType.APPLICATION_JSON}) -@Path("/endpoint-data-reference") +@Path("/transfer-process-started") public class AgreementControllerImpl implements AgreementController { /** @@ -128,8 +128,8 @@ public void receiveEdcCallback(EventEnvelope dataReferen .id(dataReference.getId()) .contractId(dataReference.getPayload().getContractId()) .endpoint(dataReference.getPayload().getDataAddress().getStringProperty("https://w3id.org/edc/v0.0.1/ns/endpoint", null)) - .authCode("Authorization") - .authKey(dataReference.getPayload().getDataAddress().getStringProperty("https://w3id.org/edc/v0.0.1/ns/authorization", null)) + .authKey("Authorization") + .authCode(dataReference.getPayload().getDataAddress().getStringProperty("https://w3id.org/edc/v0.0.1/ns/authorization", null)) .build(); endpointStore.put(assetId, newRef); } diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/jsonld/JsonLd.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/jsonld/JsonLd.java index 07f8a82a..8c5f5a9b 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/jsonld/JsonLd.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/jsonld/JsonLd.java @@ -135,6 +135,10 @@ public static JSONTYPE processJsonLd(JSONTYPE sourc prefix = prefix + ":"; } prop = prefix + prop.substring(colonIndex + 1); + } else { + if (!prop.startsWith("@") && namespaces.containsKey("@vocab")) { + prop = namespaces.get("@vocab") + prop; + } } object.add(prop, processJsonLd(value, namespaces)); }); diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/model/DcatCatalog.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/model/DcatCatalog.java index 42c13972..38326ecc 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/model/DcatCatalog.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/model/DcatCatalog.java @@ -32,7 +32,10 @@ public class DcatCatalog extends JsonLdObject { public DcatCatalog(JsonObject node) { super(node); - JsonValue dataset = node.get("https://www.w3.org/ns/dcat/dataset"); + JsonValue dataset = node.getOrDefault( + "https://www.w3.org/ns/dcat/dataset", + node.get("http://www.w3.org/ns/dcat#dataset") + ); if (dataset != null) { if (dataset.getValueType() == JsonValue.ValueType.ARRAY) { for (JsonValue ds : dataset.asJsonArray()) { @@ -45,7 +48,7 @@ public DcatCatalog(JsonObject node) { } public String getParticipantId() { - return object.getString("https://w3id.org/edc/v0.0.1/ns/participantId", "anonymous"); + return object.getString("https://w3id.org/dspace/v0.8/participantId", "anonymous"); } public List getDatasets() { diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/model/OdrlPolicy.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/model/OdrlPolicy.java index 369f29c4..a73b45c0 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/model/OdrlPolicy.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/model/OdrlPolicy.java @@ -17,6 +17,7 @@ package org.eclipse.tractusx.agents.model; import jakarta.json.JsonObject; +import org.eclipse.tractusx.agents.jsonld.JsonLd; import org.eclipse.tractusx.agents.jsonld.JsonLdObject; /** @@ -28,4 +29,16 @@ public OdrlPolicy(JsonObject node) { super(node); } + public String getPermissionAsString() { + return JsonLd.asString(object.get("http://www.w3.org/ns/odrl/2/permission")); + } + + public String getObligationAsString() { + return JsonLd.asString(object.get("http://www.w3.org/ns/odrl/2/obligation")); + } + + public String getProhibitionAsString() { + return JsonLd.asString(object.get("http://www.w3.org/ns/odrl/2/prohibition")); + } + } diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java index 371e6a93..bf3db82a 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java @@ -43,6 +43,7 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.UUID; import static java.lang.String.format; @@ -61,8 +62,9 @@ public class DataManagement { public static final String CATALOG_REQUEST_BODY = "{" + "\"@context\": {}," + "\"protocol\": \"dataspace-protocol-http\"," + - "\"counterPartyAddress\": \"%1$s\", " + - "\"querySpec\": %2$s }"; + "\"counterPartyAddress\": \"%2$s\", " + + "\"counterPartyId\": \"%1$s\", " + + "\"querySpec\": %3$s }"; public static final String ASSET_CREATE_CALL = "%1$s%2$s/assets"; public static final String ASSET_UPDATE_CALL = "%1$s%2$s/assets/%3$s"; @@ -153,19 +155,25 @@ public class DataManagement { public static final String ASSET_CALL = "%s%s/assets/request"; // negotiation request 0.5.>=1 - public static final String NEGOTIATION_REQUEST_BODY = "{\n" + - "\"@context\": { \"odrl\": \"http://www.w3.org/ns/odrl/2/\"},\n" + - "\"@type\": \"NegotiationInitiateRequestDto\",\n" + - "\"connectorAddress\": \"%1$s\",\n" + - "\"protocol\": \"dataspace-protocol-http\",\n" + - "\"providerId\": \"%3$s\",\n" + - "\"connectorId\": \"%2$s\",\n" + - "\"offer\": {\n" + - " \"offerId\": \"%4$s\",\n" + - " \"assetId\": \"%5$s\",\n" + - " \"policy\": %6$s\n" + - "}\n" + - "}"; + public static final String NEGOTIATION_REQUEST_BODY = + "{\n" + + " \"@context\": {\n" + + " \"@vocab\": \"https://w3id.org/edc/v0.0.1/ns/\"\n" + + " },\n" + + " \"@type\": \"https://w3id.org/edc/v0.0.1/ns/ContractRequest\",\n" + + " \"counterPartyAddress\": \"%1$s\",\n" + + " \"protocol\": \"dataspace-protocol-http\",\n" + + " \"policy\": {\n" + + " \"@context\": \"http://www.w3.org/ns/odrl.jsonld\",\n" + + " \"@type\": \"odrl:Offer\",\n" + + " \"@id\": \"%4$s\",\n" + + " \"target\": \"%5$s\",\n" + + " \"assigner\": \"%3$s\",\n" + + " \"odrl:permission\": %6$s,\n" + + " \"odrl:prohibition\": %7$s,\n" + + " \"odrl:obligation\": %8$s\n" + + " }\n" + + "}"; public static final String NEGOTIATION_INITIATE_CALL = "%s/v2/contractnegotiations"; public static final String NEGOTIATION_CHECK_CALL = "%s/v2/contractnegotiations/%s"; @@ -173,28 +181,25 @@ public class DataManagement { public static final String TRANSFER_REQUEST_BODY = "{\n" + " \"@context\": {\n" + - " \"odrl\": \"http://www.w3.org/ns/odrl/2/\"\n" + + " \"@vocab\": \"https://w3id.org/edc/v0.0.1/ns/\"\n" + " },\n" + + " \"@type\": \"TransferRequest\",\n" + " \"assetId\": \"%1$s\",\n" + - " \"connectorAddress\": \"%2$s\",\n" + - " \"connectorId\": \"%5$s\",\n" + " \"contractId\": \"%3$s\",\n" + + " \"counterPartyAddress\": \"%2$s\",\n" + " \"dataDestination\": {\n" + " \"type\": \"HttpProxy\"\n" + " },\n" + - " \"managedResources\": false,\n" + " \"protocol\": \"dataspace-protocol-http\",\n" + - " \"transferType\": {\n" + - " \"contentType\": \"application/octet-stream\",\n" + - " \"isFinite\": true\n" + - " }\n" + + " \"transferType\": \"HttpData-PULL\",\n" + + " \"privateProperties\": {},\n" + " \"callbackAddresses\": [\n" + " {\n" + " \"transactional\": false,\n" + " \"uri\": \"%4$s\",\n" + " \"events\": [\n" + " \"transfer.process.started\"\n" + - " ],\n" + + " ]\n" + " }\n" + " ] \n" + "}"; @@ -238,20 +243,24 @@ public DcatCatalog findContractOffers(String remoteControlPlaneIdsUrl, String as QuerySpec findAsset = QuerySpec.Builder.newInstance().filter( List.of(new Criterion("https://w3id.org/edc/v0.0.1/ns/id", "=", assetId)) ).build(); - return getCatalog(remoteControlPlaneIdsUrl, findAsset); + String partnerId = config.getDataspaceSynchronizationConnectors().entrySet() + .stream().filter(entry -> entry.getValue().equals(remoteControlPlaneIdsUrl)) + .findFirst().map(entry -> entry.getKey()).orElse(UUID.randomUUID().toString()); + return getCatalog(partnerId, remoteControlPlaneIdsUrl, findAsset); } /** * Access the catalogue * + * @param partnerId business partner id * @param remoteControlPlaneIdsUrl url of the remote control plane ids endpoint * @param spec query specification * @return catalog object * @throws IOException in case something went wrong */ - public DcatCatalog getCatalog(String remoteControlPlaneIdsUrl, QuerySpec spec) throws IOException { + public DcatCatalog getCatalog(String partnerId, String remoteControlPlaneIdsUrl, QuerySpec spec) throws IOException { var url = String.format(CATALOG_CALL, config.getControlPlaneManagementUrl()); - var catalogSpec = String.format(CATALOG_REQUEST_BODY, String.format(DSP_PATH, remoteControlPlaneIdsUrl), objectMapper.writeValueAsString(spec)); + var catalogSpec = String.format(CATALOG_REQUEST_BODY, partnerId, String.format(DSP_PATH, remoteControlPlaneIdsUrl), objectMapper.writeValueAsString(spec)); var request = new Request.Builder().url(url).post(RequestBody.create(catalogSpec, MediaType.parse("application/json"))); config.getControlPlaneManagementHeaders().forEach(request::addHeader); @@ -457,7 +466,10 @@ public String initiateNegotiation(ContractNegotiationRequest negotiationRequest) negotiationRequest.getRemoteBusinessPartnerNumber(), negotiationRequest.getOffer().getOfferId(), negotiationRequest.getOffer().getAssetId(), - negotiationRequest.getOffer().getPolicy().asString()); + negotiationRequest.getOffer().getPolicy().getPermissionAsString(), + negotiationRequest.getOffer().getPolicy().getObligationAsString(), + negotiationRequest.getOffer().getPolicy().getProhibitionAsString() + ); var requestBody = RequestBody.create(negotiateSpec, MediaType.parse("application/json")); @@ -559,8 +571,7 @@ public String initiateHttpProxyTransferProcess(TransferRequest transferRequest) transferRequest.getAssetId(), transferRequest.getConnectorAddress(), transferRequest.getContractId(), - transferRequest.getCallbackAddresses().get(0).getUri(), - transferRequest.getConnectorAddress()); + transferRequest.getCallbackAddresses().get(0).getUri()); var requestBody = RequestBody.create(transferSpec, MediaType.parse("application/json")); diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataspaceSynchronizer.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataspaceSynchronizer.java index ef3d437c..d84e6c97 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataspaceSynchronizer.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataspaceSynchronizer.java @@ -183,9 +183,9 @@ public synchronized void start() { if (!isStarted) { isStarted = true; long interval = config.getDataspaceSynchronizationInterval(); - String[] connectors = config.getDataspaceSynchronizationConnectors(); - if (interval > 0 && connectors != null && connectors.length > 0) { - monitor.info(String.format("Starting dataspace synchronization on %d connectors with interval %d milliseconds", connectors.length, interval)); + Map connectors = config.getDataspaceSynchronizationConnectors(); + if (interval > 0 && connectors != null && connectors.size() > 0) { + monitor.info(String.format("Starting dataspace synchronization on %d connectors with interval %d milliseconds", connectors.size(), interval)); service.schedule(this, interval, TimeUnit.MILLISECONDS); } } @@ -209,16 +209,16 @@ public synchronized void shutdown() { public void run() { monitor.debug("Synchronization run has been started"); if (isStarted) { - for (String remote : config.getDataspaceSynchronizationConnectors()) { + for (Map.Entry remote : config.getDataspaceSynchronizationConnectors().entrySet()) { if (isStarted) { monitor.debug(String.format("About to synchronize remote connector %s", remote)); rdfStore.startTx(); try { - DcatCatalog catalog = dataManagement.getCatalog(remote, FEDERATED_ASSET_QUERY); + DcatCatalog catalog = dataManagement.getCatalog(remote.getKey(), remote.getValue(), FEDERATED_ASSET_QUERY); Node graph = rdfStore.getDefaultGraph(); - Node connector = NodeFactory.createURI(remote.replace("https", "edcs").replace("http", "edc")); + Node connector = NodeFactory.createURI(remote.getValue().replace("https", "edcs").replace("http", "edc")); deleteConnectorFacts(graph, connector); - addConnectorFacts(remote, catalog, graph, connector); + addConnectorFacts(remote.getValue(), catalog, graph, connector); rdfStore.commit(); } catch (Throwable io) { monitor.warning(String.format("Could not synchronize remote connector %s because of %s. Going ahead.", remote, io)); From b910cd51f2e401b6318e9c3d039f232eab317dcd Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Mon, 6 May 2024 08:44:05 +0200 Subject: [PATCH 08/31] style: javadoc problem --- .../src/main/java/org/eclipse/tractusx/agents/AgentConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java index 986b7226..6b035cd8 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java @@ -288,7 +288,7 @@ public Map getDataspaceSynchronizationConnectors() { /** * access - * + * * @return the location of the rdf store */ public String getRdfStore() { From d4d2d1d618e2a6f4f7f0600f5d0b684f44e32e11 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Mon, 6 May 2024 16:34:25 +0200 Subject: [PATCH 09/31] fix: correct logging and shutting down of callbacks. --- .../tractusx/agents/AgreementControllerImpl.java | 2 +- .../tractusx/agents/matchmaking/Bootstrap.java | 16 ++++++++-------- .../agents/matchmaking/SharedObjectManager.java | 10 +++++++++- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java index 6ac46774..e3aa1a66 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java @@ -110,7 +110,7 @@ public AgreementControllerImpl(Monitor monitor, AgentConfig config, DataManageme */ @Override public String toString() { - return super.toString() + "/endpoint-data-reference"; + return super.toString() + "/transfer-process-started"; } /** diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/Bootstrap.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/Bootstrap.java index 548a0131..57740da3 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/Bootstrap.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/Bootstrap.java @@ -51,19 +51,19 @@ public class Bootstrap { private Monitor monitor; - Server externalServer; - Server internalServer; + Server server; + SharedObjectManager sharedObjectManager; /** * Starts Grizzly HTTP server exposing JAX-RS resources defined in this application. */ public Bootstrap() { handleSignal("TERM"); - SharedObjectManager sharedObjectManager = SharedObjectManager.getInstance(); + this.sharedObjectManager = SharedObjectManager.getInstance(); this.monitor = sharedObjectManager.getMonitor(); // Create Jetty server - Server server = new Server(); + this.server = new Server(); HandlerCollection handlerList = new HandlerCollection(); server.setHandler(handlerList); @@ -122,13 +122,13 @@ public Bootstrap() { monitor.debug("Trying to start the server"); try { server.start(); + sharedObjectManager.start(); server.join(); } catch (Exception e) { + sharedObjectManager.shutdown(); e.printStackTrace(); } - - } /** @@ -151,8 +151,8 @@ public static void main(String[] args) throws IOException { private void shutdown() { System.out.println("Shutdown initiated"); - internalServer.destroy(); - externalServer.destroy(); + server.destroy(); + SharedObjectManager.getInstance().shutdown(); System.exit(0); } diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/SharedObjectManager.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/SharedObjectManager.java index a6f1289b..92ad8ffa 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/SharedObjectManager.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/SharedObjectManager.java @@ -111,7 +111,15 @@ private SharedObjectManager() { this.graphController = new GraphController(monitor, rdfStore, catalogService, agentConfig); } - + + public void start() { + synchronizer.start(); + } + + public void shutdown() { + synchronizer.shutdown(); + } + public static String convertToCurl(Request request) { StringBuilder curlCommand = new StringBuilder("curl"); From 9c79adb7112ea3a092add6a3e5e52000e088cbb8 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Mon, 6 May 2024 17:05:17 +0200 Subject: [PATCH 10/31] fix: correct graph asset registration template. --- .../org/eclipse/tractusx/agents/service/DataManagement.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java index bf3db82a..1c30ebda 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java @@ -147,8 +147,8 @@ public class DataManagement { " \"proxyMethod\": \"true\",\n" + " \"proxyQueryParams\": \"true\",\n" + " \"proxyBody\": \"true\",\n" + - " \"cx-common:allowServicePattern\": \"%10$s\",\n" + - " \"cx-common:denyServicePattern\": \"%11$s\"\n" + + " \"cx-common:allowServicePattern\": \"%9$s\",\n" + + " \"cx-common:denyServicePattern\": \"%10$s\"\n" + " }\n" + "}\n"; From 971c58cd73ffe7978403ebb5a4f5965aa0daf8c4 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Mon, 6 May 2024 18:16:25 +0200 Subject: [PATCH 11/31] fix: skill update/creation needs to escape regexes. --- .../tractusx/agents/service/DataManagement.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java index 1c30ebda..e2db94b0 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/DataManagement.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import jakarta.json.Json; import jakarta.ws.rs.InternalServerErrorException; import okhttp3.MediaType; import okhttp3.OkHttpClient; @@ -67,7 +68,7 @@ public class DataManagement { "\"querySpec\": %3$s }"; public static final String ASSET_CREATE_CALL = "%1$s%2$s/assets"; - public static final String ASSET_UPDATE_CALL = "%1$s%2$s/assets/%3$s"; + public static final String ASSET_UPDATE_CALL = "%1$s%2$s/assets"; /** * template for skill asset creation @@ -106,8 +107,8 @@ public class DataManagement { " \"proxyMethod\": \"true\",\n" + " \"proxyQueryParams\": \"true\",\n" + " \"proxyBody\": \"true\",\n" + - " \"cx-common:allowServicePattern\": \"%10$s\",\n" + - " \"cx-common:denyServicePattern\": \"%11$s\"\n" + + " \"cx-common:allowServicePattern\": %10$s,\n" + + " \"cx-common:denyServicePattern\": %11$s\n" + " }\n" + "}\n"; @@ -147,8 +148,8 @@ public class DataManagement { " \"proxyMethod\": \"true\",\n" + " \"proxyQueryParams\": \"true\",\n" + " \"proxyBody\": \"true\",\n" + - " \"cx-common:allowServicePattern\": \"%9$s\",\n" + - " \"cx-common:denyServicePattern\": \"%10$s\"\n" + + " \"cx-common:allowServicePattern\": %9$s,\n" + + " \"cx-common:denyServicePattern\": %10$s\n" + " }\n" + "}\n"; @@ -334,7 +335,7 @@ protected IdResponse createOrUpdateAsset(String assetId, String assetSpec) throw } url = String.format(ASSET_UPDATE_CALL, config.getControlPlaneManagementProviderUrl(), version, assetId); - var patchRequest = new Request.Builder().url(url).patch(RequestBody.create(assetSpec, MediaType.parse("application/json"))); + var patchRequest = new Request.Builder().url(url).put(RequestBody.create(assetSpec, MediaType.parse("application/json"))); config.getControlPlaneManagementHeaders().forEach(patchRequest::addHeader); try (var patchResponse = httpClient.newCall(patchRequest.build()).execute()) { @@ -386,6 +387,9 @@ public IdResponse createOrUpdateSkill(String assetId, String name, String descri denyServicePattern = config.getServiceDenyPattern().pattern(); } + allowServicePattern = Json.createValue(allowServicePattern).toString(); + denyServicePattern = Json.createValue(denyServicePattern).toString(); + var assetSpec = String.format(body, assetId, name, description, version, contract, ontologies, distributionMode, isFederated, query, allowServicePattern, denyServicePattern); From 8648d0ad43e108f1c59723c0b6120d8a40ba8dad Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Tue, 7 May 2024 12:38:24 +0200 Subject: [PATCH 12/31] fix: add default runMode=ALL for skill registration. remove redundant edc version property. --- .../eclipse/tractusx/agents/AgentConfig.java | 51 +++++++++++-------- .../agents/AgreementControllerImpl.java | 2 +- .../tractusx/agents/http/AgentController.java | 2 +- .../agents/matchmaking/Bootstrap.java | 2 +- .../agents/service/EdcSkillStore.java | 3 ++ 5 files changed, 36 insertions(+), 24 deletions(-) diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java index 6b035cd8..5c7c2da4 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java @@ -93,9 +93,6 @@ public class AgentConfig { public static final String SERVICE_DENY_ASSET_PROPERTY = "cx.agent.service.asset.deny"; public static final String DEFAULT_SERVICE_DENY_ASSET_PATTERN = "^$"; - - public static final String TX_EDC_VERSION_PROPERTY = "cx.agent.edc.version"; - public static final String MATCHMAKING_PORT = "web.http.internal.port"; public static final String MATCHMAKING_PATH = "web.http.internal.path"; @@ -415,42 +412,54 @@ public Pattern getServiceAssetDenyPattern() { /** * access * - * @return tx edc version as a string + * @return port for exposing the matchmaking agent */ - public String getEdcVersion() { - return config.getString(TX_EDC_VERSION_PROPERTY, "0.5.0"); + public int getMatchmakingPort() { + return config.getInteger(MATCHMAKING_PORT); } /** - * check + * access * - * @return whether the edc version is less than 23.09 + * @return path for exposing the matchmaking agent */ - public boolean isPrerelease() { - return getEdcVersion().compareTo("0.5.0") <= 0; - } - - public int getMatchmakingPort() { - return config.getInteger(MATCHMAKING_PORT); - } - public String getMatchmakingPath() { return config.getString(MATCHMAKING_PATH); } - + + /** + * access + * + * @return port for exposing the standard services + */ public int getDefaultPort() { return config.getInteger(DEFAULT_PORT); } - + + /** + * access + * + * @return path for exposing the standard services + */ public String getDefaultPath() { return config.getString(DEFAULT_PATH); } - + + /** + * access + * + * @return port for exposing the edc transfer callback + */ public int getCallbackPort() { return config.getInteger(CALLBACK_PORT); } - - public String getCallbacktPath() { + + /** + * access + * + * @return path for exposing the edc transfer callback + */ + public String getCallbackPath() { return config.getString(CALLBACK_PATH); } diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java index e3aa1a66..4e6d8f89 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgreementControllerImpl.java @@ -389,7 +389,7 @@ public EndpointDataReference createAgreement(String remoteUrl, String asset) thr startTime = System.currentTimeMillis(); // EDC 0.5.1 has a problem with the checker configuration and wont process to COMPLETED - String expectedTransferState = config.isPrerelease() ? "COMPLETED" : "STARTED"; + String expectedTransferState = "STARTED"; try { while ((System.currentTimeMillis() - startTime < config.getNegotiationTimeout()) && (process == null || !process.getState().equals(expectedTransferState))) { diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/AgentController.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/AgentController.java index b3353093..f4cd5eb8 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/AgentController.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/AgentController.java @@ -458,7 +458,7 @@ public Response postSkill(String query, @QueryParam("denyServicesPattern") String denyServicePattern, @QueryParam("ontology") String[] ontologies ) { - monitor.debug(String.format("Received a POST skill request %s %s %s %s %s %b %s %s %s ", asset, name, description, version, contract, mode.getMode(), isFederated, allowServicePattern, denyServicePattern, query)); + monitor.debug(String.format("Received a POST skill request %s %s %s %s %s %b %s %s %s ", asset, name, description, version, contract, mode, isFederated, allowServicePattern, denyServicePattern, query)); Response.ResponseBuilder rb; if (skillStore.put(asset, query, name, description, version, contract, mode, isFederated, allowServicePattern, denyServicePattern, ontologies) != null) { rb = Response.ok(); diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/Bootstrap.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/Bootstrap.java index 57740da3..c21a00e5 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/Bootstrap.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/matchmaking/Bootstrap.java @@ -106,7 +106,7 @@ public Bootstrap() { int port3 = conf.getCallbackPort(); monitor.debug(String.format("Configuring server on port %s", port3)); ServletContextHandler handler3 = new ServletContextHandler(ServletContextHandler.SESSIONS); - handler3.setContextPath(conf.getCallbacktPath()); + handler3.setContextPath(conf.getCallbackPath()); ResourceConfig resourceConfig3 = new ResourceConfig(); monitor.debug(String.format("Registering agreement controller %s", sharedObjectManager.getAgreementController())); resourceConfig3.register(sharedObjectManager.getAgreementController()); diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/EdcSkillStore.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/EdcSkillStore.java index 9b814585..7c1e5bb3 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/EdcSkillStore.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/EdcSkillStore.java @@ -64,6 +64,9 @@ public String put(String key, String skill, String name, String description, Str if (contract == null) { contract = config.getDefaultSkillContract(); } + if (dist == null) { + dist = SkillDistribution.ALL; + } String ontologiesString = String.join(",", ontologies); try { return management.createOrUpdateSkill( From f0613f9e5dd9f9b39c7c84b636b1ae0a03abb724 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Tue, 7 May 2024 18:09:40 +0200 Subject: [PATCH 13/31] feat: upgrade chart to the EDC 0.7 interaction pattern- --- .../ci/integration-values.yaml | 40 ++++++++++ .../matchmaking-agent/templates/_helpers.tpl | 18 +++++ .../templates/configmap.yaml | 80 +++++++++++++++++++ .../templates/deployment.yaml | 29 ++++--- charts/matchmaking-agent/values.yaml | 76 ++++++++++++++++-- 5 files changed, 224 insertions(+), 19 deletions(-) create mode 100644 charts/matchmaking-agent/ci/integration-values.yaml diff --git a/charts/matchmaking-agent/ci/integration-values.yaml b/charts/matchmaking-agent/ci/integration-values.yaml new file mode 100644 index 00000000..ba663502 --- /dev/null +++ b/charts/matchmaking-agent/ci/integration-values.yaml @@ -0,0 +1,40 @@ +# +# Copyright (c) 2023, 2024 T-Systems International GmbH +# Copyright (c) 2023 ZF Friedrichshafen AG +# Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH +# Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) +# Copyright (c) 2021, 2023 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +image: + registry: ghcr.io/catenax-ng/tx-knowledge-agents/ + tag: 1.12.19-SNAPSHOT + +participant: + id: "BPNL0000000DUMMY" + +controlplane: + endpoints: + management: + authKey: "bla" + +agent: + default: + - dataspace.ttl + + diff --git a/charts/matchmaking-agent/templates/_helpers.tpl b/charts/matchmaking-agent/templates/_helpers.tpl index 14407a0c..c4d24372 100644 --- a/charts/matchmaking-agent/templates/_helpers.tpl +++ b/charts/matchmaking-agent/templates/_helpers.tpl @@ -86,3 +86,21 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{/* +Validation URL +*/}} +{{- define "agent.controlplane.url.validation" -}} +{{- printf "http://%s-controlplane:%v%s/token" ( include "agent.fullname" $ ) $.Values.controlplane.endpoints.control.port $.Values.controlplane.endpoints.control.path -}} +{{- end }} + +{{/* +join a map +*/}} +{{- define "agent.remotes" -}} +{{- $res := dict "servers" (list) -}} +{{- range $bpn, $connector := .Values.agent.connectors -}} +{{- $noop := printf "$s=%s" $bpn $connector | append $res.servers | set $res "servers" -}} +{{- end -}} +{{- join "," $res.servers -}} +{{- end -}} diff --git a/charts/matchmaking-agent/templates/configmap.yaml b/charts/matchmaking-agent/templates/configmap.yaml index cfb5cf22..b38317f9 100644 --- a/charts/matchmaking-agent/templates/configmap.yaml +++ b/charts/matchmaking-agent/templates/configmap.yaml @@ -25,4 +25,84 @@ metadata: data: logback.xml: |- {{- .Values.logging.configuration | nindent 4 }} + configuration.properties: |- + ################################################ + # MATCHMAKING AGENT CONFIG + ############################################### + ### + # Chapter General Settings + ### + + edc.participant.id={{ .Values.participant.id | required ".Values.participant.id is required"}} + + ### + # Chapter Web Server / Ports + ### + + edc.api.auth.key={{ .Values.controlplane.endpoints.management.authKey | required ".Values.controlplane.endpoints.mangement.authKey is required" }} + web.http.default.port={{ .Values.agent.endpoints.default.port }} + web.http.default.path={{ .Values.agent.endpoints.default.path }} + web.http.callback.port={{ .Values.agent.endpoints.callback.port }} + web.http.callback.path={{ .Values.agent.endpoints.callback.path }} + web.http.internal.port={{ .Values.agent.endpoints.internal.port }} + web.http.internal.path={{ .Values.agent.endpoints.internal.path }} + + cx.agent.matchmaking.internal.api={{ .Values.agent.endpoints.internal.port }} + cx.agent.matchmaking.external.api={{ .Values.agent.endpoints.internal.port }} + + # Single Control Plane setup + edc.dataplane.token.validation.endpoint={{ include "agent.controlplane.url.validation" .}} + + ### + # Chapter Dataspace + ### + + cx.agent.controlplane.protocol={{ printf "http://%s-controlplane:%v" ( include "agent.fullname" . ) .Values.controlplane.endpoints.protocol.port }} + cx.agent.controlplane.management={{ printf "http://%s-controlplane:%v%s" ( include "agent.fullname" . ) .Values.controlplane.endpoints.management.port .Values.controlplane.endpoints.management.path }} + cx.agent.callback={{ printf "http://%s:%v%s/transfer-process-started" ( include "agent.fullname" . ) .Values.agent.endpoints.callback.port .Values.agent.endpoints.callback.path }} + {{- if .Values.agent.skillcontract }} + cx.agent.skill.contract.default={{ .Values.agent.skillcontract }} + {{- end }} + {{- if .Values.agent.graphcontract }} + cx.agent.graph.contract.default={{ .Values.agent.graphcontract }} + {{- end }} + + {{- if .Values.agent.services }} + ################### + # AGENT SERVICES # + ################### + {{- if .Values.agent.services.allow }} + cx.agent.service.allow={{ .Values.agent.services.allow }} + {{- end }} + {{- if .Values.agent.services.deny }} + cx.agent.service.deny={{ .Values.agent.services.deny }} + {{- end }} + {{- if .Values.agent.services.asset }} + {{- if .Values.agent.services.asset.allow }} + cx.agent.service.asset.allow={{ .Values.agent.services.asset.allow }} + {{- end }} + {{- if .Values.agent.services.asset.deny }} + cx.agent.service.asset.deny={{ .Values.agent.services.asset.deny }} + {{- end }} + {{- end }} + {{- end }} + + ################### + # AGENT SYNC # + ################### + cx.agent.dataspace.remotes={{- include "agent.remotes" . }} + cx.agent.dataspace.synchronization={{ .Values.agent.synchronization | required ".Values.agent.synchronization should be set if connectors are specified" }} + cx.agent.federation.batch.max={{ .Values.agent.maxbatchsize }} + + {{- if .Values.agent.default }} + ############### + # AGENT INIT # + ############### + cx.agent.asset.file={{ join "," .Values.agent.default }} + {{- end }} + {{- range $config_name, $config_value := .Values.configs }} + {{ $config_name }}: |- + {{ $config_value | nindent 4 }} + {{- end }} + \ No newline at end of file diff --git a/charts/matchmaking-agent/templates/deployment.yaml b/charts/matchmaking-agent/templates/deployment.yaml index a78a7c14..bb351af1 100644 --- a/charts/matchmaking-agent/templates/deployment.yaml +++ b/charts/matchmaking-agent/templates/deployment.yaml @@ -97,9 +97,18 @@ spec: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: - name: configuration - mountPath: /var/rdf4j/server/conf/logback.xml + mountPath: /app/logback.xml subPath: logback.xml readOnly: true + - name: configuration + mountPath: /app/configuration.properties + subPath: configuration.properties + readOnly: true + {{- range $config_name, $config_value := .Values.configs }} + - name: "configuration" + mountPath: {{ printf "/app/%s" $config_name | quote }} + subPath: {{ printf "%s" $config_name | quote }} + {{- end }} volumes: - name: configuration configMap: @@ -107,20 +116,14 @@ spec: items: - key: logback.xml path: logback.xml + - key: configuration.properties + path: configuration.properties + {{- range $config_name, $config_value := .Values.configs }} + - key: {{ printf "%s" $config_name | quote }} + path: {{ printf "%s" $config_name | quote }} + {{- end }} - name: tmp emptyDir: {} - - name: logs - emptyDir: {} - - name: work - emptyDir: {} - - name: conf - emptyDir: {} - - name: rdf4jconf - emptyDir: {} - - name: rdf4jlogs - emptyDir: {} - - name: rdf4jrepositories - emptyDir: {} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/matchmaking-agent/values.yaml b/charts/matchmaking-agent/values.yaml index fb1cca12..cd7040cf 100644 --- a/charts/matchmaking-agent/values.yaml +++ b/charts/matchmaking-agent/values.yaml @@ -26,9 +26,9 @@ replicaCount: 1 image: # -- target registry - registry: docker.io/ + registry: docker.io/tractusx/ # -- Which derivate of agent to use - repository: tractusx/matchmaking-agent + repository: matchmaking-agent # -- Overrides the image tag whose default is the chart appVersion tag: "" # -- Overrides the image digest @@ -138,6 +138,44 @@ startupProbe: # -- number of seconds until a timeout is assumed timeoutSeconds: 5 +participant: + # -- BPN Number + id: "" + +# -- References to the control plane deployment +controlplane: + endpoints: + # -- data management api, used by internal users, can be added to an ingress and must not be internet facing + management: + # -- port for incoming api calls + port: 8081 + # -- path for incoming api calls + path: /management + # -- authentication key, must be attached to each 'X-Api-Key' request header + authKey: "" + # -- control api, used for internal control calls. can be added to the internal ingress, but should probably not + control: + # -- port for incoming api calls + port: 8083 + # -- path for incoming api calls + path: /control + # -- dsp api, used for inter connector communication and must be internet facing + protocol: + # -- port for incoming api calls + port: 8084 + # -- path for incoming api calls + path: /api/v1/dsp + +# -- A set of additional configuration files +configs: + # -- An example of an empty graph in ttl syntax + dataspace.ttl: | + ################################################################# + # Catena-X Agent Bootstrap Graph in TTL/RDF/OWL FORMAT + ################################################################# + @prefix : . + @base . + ## Endpoints exposed by the matchmaking agent agent: endpoints: @@ -148,17 +186,17 @@ agent: # -- An auth object for default security auth: {} # -- The path mapping the "default" api is going to be exposed by - path: "api" + path: "/api" # -- An optional regex path match (whose match groups could be used in an nginx-annotation of the ingress) regex: /(.*) ## Source api exposing EDC inject - source: + internal: # -- The network port, which the "source" api is going to be exposed by the container, pod and service port: "8080" # -- An auth object for default security auth: {} # -- The path mapping the "source" api is going to be exposed by - path: "agentsource" + path: "/agentsource" # -- An optional regex path match (whose match groups could be used in an nginx-annotation of the ingress) regex: /(.*) ## Source api exposing EDC callback @@ -168,9 +206,35 @@ agent: # -- An auth object for default security auth: {} # -- The path mapping the "callback" api is going to be exposed by - path: "callback" + path: "/callback" # -- An optional regex path match (whose match groups could be used in an nginx-annotation of the ingress) regex: /(.*) + # -- A list of local or remote graph descriptions to build the default meta-graph/federated data catalogue + default: + - dataspace.ttl + - https://w3id.org/catenax/ontology.ttl + # -- The synchronization interval in ms to update the federated data catalogue + synchronization: -1 + # -- A map of partner ids to remote connector IDS URLs to synchronize with + connectors: {} + # -- Names the visible contract under which new skills are published (if not otherwise specified) + skillcontract: Contract?partner=Skill + # -- Names the visible contract under which new graphs are published (if not otherwise specified) + graphcontract: Contract?partner=Graph + # -- Sets the maximal batch size when delegating to agents and services + maxbatchsize: '9223372036854775807' + # -- A set of configs for regulating outgoing service calls + services: + # -- A regular expression which outgoing service URLs must match (unless overwritten by a specific asset property) + allow: '(edcs?://.*)|(https://query\\.wikidata\\.org/sparql)' + # -- A regular expression which outgoing service URLs must not match (unless overwritten by a specific asset property) + deny: 'http://.*' + # -- A set of configs for regulating outgoing service calls when providing an asset (when no specific asset property is given) + asset: + # -- A regular expression which outgoing service URLs must match (unless overwritten by a specific asset property) + allow: '(edcs?://.*)' + # -- A regular expression which outgoing service URLs must not match (unless overwritten by a specific asset property) + deny: 'https?://.*' service: # -- [Service type](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) to expose the running application on a set of Pods as a network service. From 204e9d15103eddace7ac7bcfc4e93dfd87167336 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Tue, 7 May 2024 18:10:12 +0200 Subject: [PATCH 14/31] fix: do not use temporary images in chart testing workflow. --- charts/matchmaking-agent/ci/integration-values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/matchmaking-agent/ci/integration-values.yaml b/charts/matchmaking-agent/ci/integration-values.yaml index ba663502..8e1a2bcc 100644 --- a/charts/matchmaking-agent/ci/integration-values.yaml +++ b/charts/matchmaking-agent/ci/integration-values.yaml @@ -22,8 +22,8 @@ # image: - registry: ghcr.io/catenax-ng/tx-knowledge-agents/ - tag: 1.12.19-SNAPSHOT +# registry: ghcr.io/catenax-ng/tx-knowledge-agents/ +# tag: 1.12.19-SNAPSHOT participant: id: "BPNL0000000DUMMY" From 0ee7e649a8d3e4695ad0e6d22d44c59dd1cd98c9 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Wed, 8 May 2024 18:35:13 +0200 Subject: [PATCH 15/31] chore|fix: correct meta-data for matchmaking agent before push. --- .github/workflows/build.yml | 40 ++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8d36a2b5..baa532be 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -244,6 +244,24 @@ jobs: password: ${{ secrets.DOCKER_HUB_TOKEN || secrets.GITHUB_TOKEN }} repository: ${{ steps.set-docker-repo.outputs.REPO }}/provisioning-agent + # Create SemVer or ref tags dependent of trigger event + - name: Docker Meta Matchmaking + id: meta-match + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: | + ${{ steps.set-docker-repo.outputs.REPO }}/provisioning-agent + # Automatically prepare image tags; See action docs for more examples. + # semver patter will generate tags like these for example :1 :1.2 :1.2.3 + tags: | + type=sha,event=branch + type=sha,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=1.12.19-SNAPSHOT,enable=${{ github.event.inputs.deploy_docker == 'true' || github.ref == format('refs/heads/{0}', 'main') }} + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} + # build in any case, but push only main and version tag settings - name: Matchmaking Container Build and Push uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 @@ -252,8 +270,8 @@ jobs: file: matchmaking/src/main/docker/Dockerfile # Build image for verification purposes on every trigger event. Only push if event is not a PR push: ${{ ( github.event.inputs.deploy_docker == 'true' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') ) }} - tags: ${{ steps.meta-remote.outputs.tags }} - labels: ${{ steps.meta-remote.outputs.labels }} + tags: ${{ steps.meta-match.outputs.tags }} + labels: ${{ steps.meta-match.outputs.labels }} # Important step to push image description to DockerHub - since this is version independent, we always take it from main - name: Update Docker Hub description for Matchmaking Agent @@ -265,21 +283,3 @@ jobs: password: ${{ secrets.DOCKER_HUB_TOKEN || secrets.GITHUB_TOKEN }} repository: ${{ steps.set-docker-repo.outputs.REPO }}/matchmaking-agent - # Create SemVer or ref tags dependent of trigger event - - name: Docker Meta Matchmaking - id: meta-match - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 - with: - images: | - ${{ steps.set-docker-repo.outputs.REPO }}/provisioning-agent - # Automatically prepare image tags; See action docs for more examples. - # semver patter will generate tags like these for example :1 :1.2 :1.2.3 - tags: | - type=sha,event=branch - type=sha,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}} - type=semver,pattern={{major}}.{{minor}} - type=raw,value=1.12.19-SNAPSHOT,enable=${{ github.event.inputs.deploy_docker == 'true' || github.ref == format('refs/heads/{0}', 'main') }} - type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} - From f991056abfd1419c554b28c890a78c1e8af165cf Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Mon, 13 May 2024 08:50:08 +0200 Subject: [PATCH 16/31] feat: align with the agent plane charts. --- .../ci/integration-values.yaml | 2 +- .../matchmaking-agent/templates/_helpers.tpl | 56 +++++++++++++++++-- .../templates/configmap.yaml | 10 ++-- charts/matchmaking-agent/values.yaml | 5 ++ 4 files changed, 63 insertions(+), 10 deletions(-) diff --git a/charts/matchmaking-agent/ci/integration-values.yaml b/charts/matchmaking-agent/ci/integration-values.yaml index 8e1a2bcc..9209e3a9 100644 --- a/charts/matchmaking-agent/ci/integration-values.yaml +++ b/charts/matchmaking-agent/ci/integration-values.yaml @@ -21,7 +21,7 @@ # SPDX-License-Identifier: Apache-2.0 # -image: +#image: # registry: ghcr.io/catenax-ng/tx-knowledge-agents/ # tag: 1.12.19-SNAPSHOT diff --git a/charts/matchmaking-agent/templates/_helpers.tpl b/charts/matchmaking-agent/templates/_helpers.tpl index c4d24372..160fb65e 100644 --- a/charts/matchmaking-agent/templates/_helpers.tpl +++ b/charts/matchmaking-agent/templates/_helpers.tpl @@ -39,6 +39,19 @@ If release name contains chart name it will be used as a full name. {{- end }} {{- end }} +{{/* +Create a default fully qualified app name for the connector. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "agent.connector.fullname" -}} +{{- if .Values.connector }} +{{- printf "%s-%s" .Release.Name .Values.connector | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s" .Release.Name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} + {{/* Create chart name and version as used by the chart label. */}} @@ -87,11 +100,46 @@ Create the name of the service account to use {{- end }} {{- end }} +{{/* +Control DSP URL +*/}} +{{- define "agent.controlplane.url.protocol" -}} +{{- if (and .Values.controlplane.url .Values.controlplane.url.protocol) }}{{/* if dsp api url has been specified explicitly */}} +{{- .Values.controlplane.url.protocol }} +{{- else }}{{/* else when dsp api url has not been specified explicitly */}} +{{- with (index .Values.controlplane.ingresses 0) }} +{{- if .enabled }}{{/* if ingress enabled */}} +{{- if .tls.enabled }}{{/* if TLS enabled */}} +{{- printf "https://%s" .hostname -}} +{{- else }}{{/* else when TLS not enabled */}} +{{- printf "http://%s" .hostname -}} +{{- end }}{{/* end if tls */}} +{{- else }}{{/* else when ingress not enabled */}} +{{- printf "http://%s-controlplane:%v" ( include "agent.connector.fullname" $ ) $.Values.controlplane.endpoints.protocol.port -}} +{{- end }}{{/* end if ingress */}} +{{- end }}{{/* end with ingress */}} +{{- end }}{{/* end if .Values.controlplane.url.protocol */}} +{{- end }} + +{{/* +Validation URL +*/}} +{{- define "agent.controlplane.url.control" -}} +{{- printf "http://%s-controlplane:%v%s" ( include "agent.connector.fullname" $ ) .Values.controlplane.endpoints.control.port .Values.controlplane.endpoints.control.path -}} +{{- end }} + {{/* Validation URL */}} -{{- define "agent.controlplane.url.validation" -}} -{{- printf "http://%s-controlplane:%v%s/token" ( include "agent.fullname" $ ) $.Values.controlplane.endpoints.control.port $.Values.controlplane.endpoints.control.path -}} +{{- define "agent.controlplane.url.management" -}} +{{- printf "http://%s-controlplane:%v%s" ( include "agent.connector.fullname" $ ) .Values.controlplane.endpoints.management.port .Values.controlplane.endpoints.management.path -}} +{{- end }} + +{{/* +Data Control URL (Expects the Chart Root to be accessible via .root, the current dataplane via .dataplane) +*/}} +{{- define "agent.url.callback" -}} +{{- printf "http://%s:%v%s" (include "agent.fullname" . ) .Values.agent.endpoints.callback.port .Values.agent.endpoints.callback.path -}} {{- end }} {{/* @@ -100,7 +148,7 @@ join a map {{- define "agent.remotes" -}} {{- $res := dict "servers" (list) -}} {{- range $bpn, $connector := .Values.agent.connectors -}} -{{- $noop := printf "$s=%s" $bpn $connector | append $res.servers | set $res "servers" -}} +{{- $noop := printf "%s=%s" $bpn $connector | append $res.servers | set $res "servers" -}} {{- end -}} -{{- join "," $res.servers -}} +{{- join "," $res.servers }} {{- end -}} diff --git a/charts/matchmaking-agent/templates/configmap.yaml b/charts/matchmaking-agent/templates/configmap.yaml index b38317f9..e30fab8d 100644 --- a/charts/matchmaking-agent/templates/configmap.yaml +++ b/charts/matchmaking-agent/templates/configmap.yaml @@ -52,15 +52,15 @@ data: cx.agent.matchmaking.external.api={{ .Values.agent.endpoints.internal.port }} # Single Control Plane setup - edc.dataplane.token.validation.endpoint={{ include "agent.controlplane.url.validation" .}} + edc.dataplane.token.validation.endpoint={{ include "agent.controlplane.url.control" .}}/token ### # Chapter Dataspace ### - cx.agent.controlplane.protocol={{ printf "http://%s-controlplane:%v" ( include "agent.fullname" . ) .Values.controlplane.endpoints.protocol.port }} - cx.agent.controlplane.management={{ printf "http://%s-controlplane:%v%s" ( include "agent.fullname" . ) .Values.controlplane.endpoints.management.port .Values.controlplane.endpoints.management.path }} - cx.agent.callback={{ printf "http://%s:%v%s/transfer-process-started" ( include "agent.fullname" . ) .Values.agent.endpoints.callback.port .Values.agent.endpoints.callback.path }} + cx.agent.controlplane.protocol={{ include "agent.controlplane.url.protocol" . }} + cx.agent.controlplane.management={{ include "agent.controlplane.url.management" . }} + cx.agent.callback={{ include "agent.url.callback" . }}/transfer-process-started {{- if .Values.agent.skillcontract }} cx.agent.skill.contract.default={{ .Values.agent.skillcontract }} {{- end }} @@ -91,7 +91,7 @@ data: ################### # AGENT SYNC # ################### - cx.agent.dataspace.remotes={{- include "agent.remotes" . }} + cx.agent.dataspace.remotes={{ include "agent.remotes" . }} cx.agent.dataspace.synchronization={{ .Values.agent.synchronization | required ".Values.agent.synchronization should be set if connectors are specified" }} cx.agent.federation.batch.max={{ .Values.agent.maxbatchsize }} diff --git a/charts/matchmaking-agent/values.yaml b/charts/matchmaking-agent/values.yaml index cd7040cf..1f9cc9e4 100644 --- a/charts/matchmaking-agent/values.yaml +++ b/charts/matchmaking-agent/values.yaml @@ -142,6 +142,9 @@ participant: # -- BPN Number id: "" +# -- Name of the connector deployment +connector: "" + # -- References to the control plane deployment controlplane: endpoints: @@ -165,6 +168,8 @@ controlplane: port: 8084 # -- path for incoming api calls path: /api/v1/dsp + ingresses: + - enabled: false # -- A set of additional configuration files configs: From 1cbf7548d3338cd13cabd8f27044d52863cc7bae Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Mon, 13 May 2024 09:00:43 +0200 Subject: [PATCH 17/31] chore: update DEPENDENCIES. --- DEPENDENCIES | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/DEPENDENCIES b/DEPENDENCIES index 5e66a2ae..3ebc5347 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -123,22 +123,22 @@ maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.14, Apache-2.0, approv maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16, Apache-2.0, approved, CQ23528 maven/mavencentral/org.apache.httpcomponents/httpmime/4.5.13, Apache-2.0, approved, CQ11718 maven/mavencentral/org.apache.httpcomponents/httpmime/4.5.14, Apache-2.0, approved, CQ11718 -maven/mavencentral/org.apache.jena/jena-arq/4.9.0, , restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-base/4.9.0, , restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-core/4.9.0, , restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-dboe-base/4.9.0, , restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-dboe-index/4.9.0, , restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-dboe-storage/4.9.0, , restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-dboe-trans-data/4.9.0, , restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-dboe-transaction/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-arq/4.9.0, Apache-2.0, restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-base/4.9.0, Apache-2.0, restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-core/4.9.0, Apache-2.0, restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-dboe-base/4.9.0, Apache-2.0, restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-dboe-index/4.9.0, Apache-2.0, restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-dboe-storage/4.9.0, Apache-2.0, restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-dboe-trans-data/4.9.0, Apache-2.0, restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-dboe-transaction/4.9.0, Apache-2.0, restricted, clearlydefined maven/mavencentral/org.apache.jena/jena-fuseki-core/4.9.0, Apache-2.0, restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-iri/4.9.0, , restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-rdfconnection/4.9.0, , restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-rdfpatch/4.9.0, , restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-shacl/4.9.0, , restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-shex/4.9.0, , restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-tdb/4.9.0, , restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-tdb2/4.9.0, , restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-iri/4.9.0, Apache-2.0, approved, clearlydefined +maven/mavencentral/org.apache.jena/jena-rdfconnection/4.9.0, Apache-2.0, restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-rdfpatch/4.9.0, Apache-2.0, restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-shacl/4.9.0, Apache-2.0, restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-shex/4.9.0, Apache-2.0, restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-tdb/4.9.0, Apache-2.0, restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-tdb2/4.9.0, Apache-2.0, restricted, clearlydefined maven/mavencentral/org.apache.logging.log4j/log4j-api/2.17.2, Apache-2.0, approved, clearlydefined maven/mavencentral/org.apache.logging.log4j/log4j-to-slf4j/2.17.2, Apache-2.0, approved, #2163 maven/mavencentral/org.apache.lucene/lucene-analyzers-common/8.9.0, Apache-2.0 AND BSD-3-Clause AND LicenseRef-scancode-iso-8879, approved, #6152 @@ -300,9 +300,9 @@ maven/mavencentral/org.hdrhistogram/HdrHistogram/2.1.12, BSD-2-Clause OR License maven/mavencentral/org.javabits.jgrapht/jgrapht-core/0.9.3, EPL-2.0 OR LGPL-2.1-or-later, approved, #9816 maven/mavencentral/org.javassist/javassist/3.29.2-GA, Apache-2.0 AND LGPL-2.1-or-later AND MPL-1.1, approved, #6023 maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib-common/1.9.10, Apache-2.0, approved, #14186 -maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib-jdk7/1.8.21, Apache-2.0, approved, #8807 -maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib-jdk8/1.8.21, Apache-2.0, approved, #8919 -maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib/1.8.21, Apache-2.0, approved, #8865 +maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib-jdk7/1.9.10, Apache-2.0, approved, #14193 +maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib-jdk8/1.9.10, Apache-2.0, approved, #14191 +maven/mavencentral/org.jetbrains.kotlin/kotlin-stdlib/1.9.10, Apache-2.0, approved, #11827 maven/mavencentral/org.jetbrains/annotations/13.0, Apache-2.0, approved, clearlydefined maven/mavencentral/org.junit.jupiter/junit-jupiter-api/5.10.2, EPL-2.0, approved, #9714 maven/mavencentral/org.junit.jupiter/junit-jupiter-api/5.9.2, EPL-2.0, approved, #3133 From 4e0356dc3f87ec125e6a38d2f0fdfd6d8f6410ed Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Mon, 13 May 2024 09:06:59 +0200 Subject: [PATCH 18/31] fix: chart linting in integration values. --- charts/matchmaking-agent/ci/integration-values.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/charts/matchmaking-agent/ci/integration-values.yaml b/charts/matchmaking-agent/ci/integration-values.yaml index 9209e3a9..cc8c0acd 100644 --- a/charts/matchmaking-agent/ci/integration-values.yaml +++ b/charts/matchmaking-agent/ci/integration-values.yaml @@ -21,7 +21,7 @@ # SPDX-License-Identifier: Apache-2.0 # -#image: +# image: # registry: ghcr.io/catenax-ng/tx-knowledge-agents/ # tag: 1.12.19-SNAPSHOT @@ -36,5 +36,3 @@ controlplane: agent: default: - dataspace.ttl - - From ff5cfbb80000611f324278138443ce7ec0f56b0d Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Mon, 13 May 2024 12:46:34 +0200 Subject: [PATCH 19/31] fix: chart testing with kind requires a different registry/repository mapping. --- charts/matchmaking-agent/ci/integration-values.yaml | 3 ++- charts/matchmaking-agent/values.yaml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/charts/matchmaking-agent/ci/integration-values.yaml b/charts/matchmaking-agent/ci/integration-values.yaml index cc8c0acd..a5a3977c 100644 --- a/charts/matchmaking-agent/ci/integration-values.yaml +++ b/charts/matchmaking-agent/ci/integration-values.yaml @@ -22,7 +22,8 @@ # # image: -# registry: ghcr.io/catenax-ng/tx-knowledge-agents/ +# registry: ghcr.io/catenax-ng/ +# repository: tx-knowledge-agents/matchmaking-agent # tag: 1.12.19-SNAPSHOT participant: diff --git a/charts/matchmaking-agent/values.yaml b/charts/matchmaking-agent/values.yaml index 1f9cc9e4..74613089 100644 --- a/charts/matchmaking-agent/values.yaml +++ b/charts/matchmaking-agent/values.yaml @@ -26,9 +26,9 @@ replicaCount: 1 image: # -- target registry - registry: docker.io/tractusx/ + registry: docker.io/ # -- Which derivate of agent to use - repository: matchmaking-agent + repository: tractusx/matchmaking-agent # -- Overrides the image tag whose default is the chart appVersion tag: "" # -- Overrides the image digest From 49aeb813d2f31ada085c6bfa6384d18156301bfc Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Wed, 15 May 2024 09:19:40 +0200 Subject: [PATCH 20/31] chore: update DEPENDENCIES after successful IP checks. --- DEPENDENCIES | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/DEPENDENCIES b/DEPENDENCIES index 3ebc5347..63b25f20 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -123,22 +123,22 @@ maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.14, Apache-2.0, approv maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16, Apache-2.0, approved, CQ23528 maven/mavencentral/org.apache.httpcomponents/httpmime/4.5.13, Apache-2.0, approved, CQ11718 maven/mavencentral/org.apache.httpcomponents/httpmime/4.5.14, Apache-2.0, approved, CQ11718 -maven/mavencentral/org.apache.jena/jena-arq/4.9.0, Apache-2.0, restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-base/4.9.0, Apache-2.0, restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-core/4.9.0, Apache-2.0, restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-dboe-base/4.9.0, Apache-2.0, restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-dboe-index/4.9.0, Apache-2.0, restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-dboe-storage/4.9.0, Apache-2.0, restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-dboe-trans-data/4.9.0, Apache-2.0, restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-dboe-transaction/4.9.0, Apache-2.0, restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-fuseki-core/4.9.0, Apache-2.0, restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-arq/4.9.0, Apache-2.0 AND (Apache-2.0 AND EPL-2.0) AND (Apache-2.0 AND EPL-1.0), approved, #14711 +maven/mavencentral/org.apache.jena/jena-base/4.9.0, Apache-2.0, approved, #14713 +maven/mavencentral/org.apache.jena/jena-core/4.9.0, Apache-2.0 AND (Apache-2.0 AND BSD-3-Clause), approved, #14700 +maven/mavencentral/org.apache.jena/jena-dboe-base/4.9.0, Apache-2.0, approved, #14705 +maven/mavencentral/org.apache.jena/jena-dboe-index/4.9.0, Apache-2.0, approved, #14710 +maven/mavencentral/org.apache.jena/jena-dboe-storage/4.9.0, Apache-2.0, approved, #14704 +maven/mavencentral/org.apache.jena/jena-dboe-trans-data/4.9.0, Apache-2.0, approved, #14707 +maven/mavencentral/org.apache.jena/jena-dboe-transaction/4.9.0, Apache-2.0, approved, #14712 +maven/mavencentral/org.apache.jena/jena-fuseki-core/4.9.0, Apache-2.0 AND (EPL-2.0 OR Apache-2.0), approved, #14702 maven/mavencentral/org.apache.jena/jena-iri/4.9.0, Apache-2.0, approved, clearlydefined -maven/mavencentral/org.apache.jena/jena-rdfconnection/4.9.0, Apache-2.0, restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-rdfpatch/4.9.0, Apache-2.0, restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-shacl/4.9.0, Apache-2.0, restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-shex/4.9.0, Apache-2.0, restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-tdb/4.9.0, Apache-2.0, restricted, clearlydefined -maven/mavencentral/org.apache.jena/jena-tdb2/4.9.0, Apache-2.0, restricted, clearlydefined +maven/mavencentral/org.apache.jena/jena-rdfconnection/4.9.0, Apache-2.0, approved, #14703 +maven/mavencentral/org.apache.jena/jena-rdfpatch/4.9.0, Apache-2.0, approved, #14714 +maven/mavencentral/org.apache.jena/jena-shacl/4.9.0, Apache-2.0 AND W3C-20150513, approved, #14706 +maven/mavencentral/org.apache.jena/jena-shex/4.9.0, Apache-2.0, approved, #14709 +maven/mavencentral/org.apache.jena/jena-tdb/4.9.0, Apache-2.0, approved, #14708 +maven/mavencentral/org.apache.jena/jena-tdb2/4.9.0, Apache-2.0, approved, #14701 maven/mavencentral/org.apache.logging.log4j/log4j-api/2.17.2, Apache-2.0, approved, clearlydefined maven/mavencentral/org.apache.logging.log4j/log4j-to-slf4j/2.17.2, Apache-2.0, approved, #2163 maven/mavencentral/org.apache.lucene/lucene-analyzers-common/8.9.0, Apache-2.0 AND BSD-3-Clause AND LicenseRef-scancode-iso-8879, approved, #6152 From aec847077645e0b6f8ec5d575a6c9c1127cf838f Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Wed, 15 May 2024 10:15:07 +0200 Subject: [PATCH 21/31] fix: version incompatibilities between logback (given) and slf4j (needed) in the more recent versions. --- DEPENDENCIES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPENDENCIES b/DEPENDENCIES index 63b25f20..f47e9b82 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -340,7 +340,7 @@ maven/mavencentral/org.skyscreamer/jsonassert/1.5.1, Apache-2.0, approved, clear maven/mavencentral/org.slf4j/jcl-over-slf4j/1.7.32, Apache-2.0, approved, CQ12843 maven/mavencentral/org.slf4j/jcl-over-slf4j/1.7.36, Apache-2.0, approved, CQ12843 maven/mavencentral/org.slf4j/jul-to-slf4j/1.7.36, MIT, approved, CQ12842 -maven/mavencentral/org.slf4j/slf4j-api/2.0.12, MIT, approved, #5915 +maven/mavencentral/org.slf4j/slf4j-api/2.0.7, MIT, approved, #5915 maven/mavencentral/org.springframework.boot/spring-boot-actuator-autoconfigure/2.7.18, Apache-2.0, approved, #3273 maven/mavencentral/org.springframework.boot/spring-boot-actuator/2.7.18, Apache-2.0, approved, #4316 maven/mavencentral/org.springframework.boot/spring-boot-autoconfigure/2.7.18, Apache-2.0, approved, #4314 From 33ddaee0e3c780049af35520ff4fc9a2dbb40ce5 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Wed, 15 May 2024 10:15:39 +0200 Subject: [PATCH 22/31] fix: version incompatibilities between logback (given) and slf4j (needed) in the more recent versions. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d0d6fe60..9f7f91ab 100644 --- a/pom.xml +++ b/pom.xml @@ -72,7 +72,7 @@ 2.0.1.Final 2.10.1 1.2.13 - 2.0.12 + 2.0.7 UTF-8 From 3178cbb2996101d533d3c8a164202d9262282258 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Wed, 15 May 2024 18:17:15 +0200 Subject: [PATCH 23/31] fix: correct image name in docker meta-data of matchmaking agent --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index baa532be..bd8684bd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -250,7 +250,7 @@ jobs: uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 with: images: | - ${{ steps.set-docker-repo.outputs.REPO }}/provisioning-agent + ${{ steps.set-docker-repo.outputs.REPO }}/matchmaking-agent # Automatically prepare image tags; See action docs for more examples. # semver patter will generate tags like these for example :1 :1.2 :1.2.3 tags: | From 35e6287a79c8d201545cb3096b33d8a318e25605 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Thu, 16 May 2024 12:03:10 +0200 Subject: [PATCH 24/31] style: copyright license headers should use fixed period formatting. --- .github/workflows/dependencies.yml | 2 +- charts/matchmaking-agent/ci/integration-values.yaml | 2 +- .../org/eclipse/tractusx/agents/http/DelegationResponse.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 3964bc82..dc436091 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -1,5 +1,5 @@ ############################################################### -# Copyright (c) 2021, 2024 Contributors to the Eclipse Foundation +# Copyright (c) 2021,2024 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. diff --git a/charts/matchmaking-agent/ci/integration-values.yaml b/charts/matchmaking-agent/ci/integration-values.yaml index a5a3977c..ee5c9556 100644 --- a/charts/matchmaking-agent/ci/integration-values.yaml +++ b/charts/matchmaking-agent/ci/integration-values.yaml @@ -1,5 +1,5 @@ # -# Copyright (c) 2023, 2024 T-Systems International GmbH +# Copyright (c) 2023,2024 T-Systems International GmbH # Copyright (c) 2023 ZF Friedrichshafen AG # Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH # Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/DelegationResponse.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/DelegationResponse.java index a269481f..39f95d92 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/DelegationResponse.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/DelegationResponse.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023, 2024 T-Systems International GmbH -// Copyright (c) 2023 ,2024 Contributors to the Eclipse Foundation +// Copyright (c) 2023,2024 T-Systems International GmbH +// Copyright (c) 2023,2024 Contributors to the Eclipse Foundation // // See the NOTICE file(s) distributed with this work for additional // information regarding copyright ownership. From 62b66e3b021cf84e02ad11f8908f9580acaf54a5 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Thu, 16 May 2024 12:13:25 +0200 Subject: [PATCH 25/31] fix: vulnerable dependencies as of trivy results 2024-05-16 --- matchmaking/pom.xml | 6 ++++++ pom.xml | 3 ++- remoting/pom.xml | 6 ++++++ remoting/src/main/docker/Dockerfile | 2 ++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/matchmaking/pom.xml b/matchmaking/pom.xml index f58b93d4..12214e0d 100644 --- a/matchmaking/pom.xml +++ b/matchmaking/pom.xml @@ -285,6 +285,12 @@ + + org.apache.commons + commons-compress + ${commons.compress.version} + + org.junit.jupiter junit-jupiter-engine diff --git a/pom.xml b/pom.xml index 9f7f91ab..0e83dfe3 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,7 @@ 4.3.8 4.5.14 1.2 + 1.26.0 2.15.2 3.1.0 9.37.3 @@ -61,7 +62,7 @@ 9.4.53.v20231009 2.2 1.1.10.5 - 42.6.0 + 42.7.2 3.7.2 2.2.12 2.40 diff --git a/remoting/pom.xml b/remoting/pom.xml index b4a8b183..f73cdb2b 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -306,6 +306,12 @@ ${jetty.version} test + + org.eclipse.jetty.http2 + http2-common + ${jetty.version} + test + org.apache.zookeeper zookeeper diff --git a/remoting/src/main/docker/Dockerfile b/remoting/src/main/docker/Dockerfile index b3d32d74..567a5a83 100644 --- a/remoting/src/main/docker/Dockerfile +++ b/remoting/src/main/docker/Dockerfile @@ -25,6 +25,7 @@ COPY target/original-remoting-agent-*.jar /opt/lib/ #COPY target/lib/guava-*.jar /opt/lib/ COPY target/lib/netty-*.jar /opt/lib/ COPY target/lib/http2-hpack-*.jar /opt/lib/ +COPY target/lib/http2-common-*.jar /opt/lib/ COPY target/lib/jetty-*.jar /opt/lib/ COPY target/lib/spring-*.jar /opt/lib/ COPY target/lib/logback-*.jar /opt/lib/ @@ -52,6 +53,7 @@ RUN wget -q -O /tmp/rdf4j.zip "https://rhlx01.hs-esslingen.de/pub/Mirrors/eclips # rm /tmp/rdf4j-server/WEB-INF/lib/guava-*.jar && \ rm /tmp/rdf4j-server/WEB-INF/lib/netty-*.jar && \ rm /tmp/rdf4j-server/WEB-INF/lib/http2-hpack-*.jar && \ + rm /tmp/rdf4j-server/WEB-INF/lib/http2-common-*.jar && \ rm /tmp/rdf4j-server/WEB-INF/lib/jetty-*.jar && \ rm /tmp/rdf4j-server/WEB-INF/lib/spring-*.jar && \ rm /tmp/rdf4j-server/WEB-INF/lib/logback-*.jar && \ From 828523cc0ea0cec6f8354d51803e71eb43c43c42 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Thu, 16 May 2024 12:14:04 +0200 Subject: [PATCH 26/31] style: copyright license headers should use fixed period formatting. --- .github/dependabot.yml | 2 +- .github/workflows/helm-chart-release.yml | 2 +- .github/workflows/kics.yml | 2 +- .github/workflows/trivy.yml | 2 +- .tractusx | 2 +- charts/matchmaking-agent/ci/integration-values.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5851b3b9..45f6fca1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,5 @@ # -# Copyright (c) 2021, 2023 Contributors to the Eclipse Foundation +# Copyright (c) 2021,2023 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. diff --git a/.github/workflows/helm-chart-release.yml b/.github/workflows/helm-chart-release.yml index f01e9f76..8ab748dc 100644 --- a/.github/workflows/helm-chart-release.yml +++ b/.github/workflows/helm-chart-release.yml @@ -1,6 +1,6 @@ --- # -# Copyright (c) 2022, 2023 Contributors to the Eclipse Foundation +# Copyright (c) 2022,2023 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. diff --git a/.github/workflows/kics.yml b/.github/workflows/kics.yml index 7b42889e..a13833a3 100644 --- a/.github/workflows/kics.yml +++ b/.github/workflows/kics.yml @@ -1,6 +1,6 @@ --- # -# Copyright (c) 2021, 2023 Contributors to the Eclipse Foundation +# Copyright (c) 2021,2023 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index aadb2187..4c36609c 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -1,6 +1,6 @@ --- # -# Copyright (c) 2021, 2023 Contributors to the Eclipse Foundation +# Copyright (c) 2021,2023 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. diff --git a/.tractusx b/.tractusx index b0c559de..4509c16c 100644 --- a/.tractusx +++ b/.tractusx @@ -1,5 +1,5 @@ ############################################################### -# Copyright (c) 2022, 2023 Contributors to the Eclipse Foundation +# Copyright (c) 2022,2023 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. diff --git a/charts/matchmaking-agent/ci/integration-values.yaml b/charts/matchmaking-agent/ci/integration-values.yaml index ee5c9556..e457eae1 100644 --- a/charts/matchmaking-agent/ci/integration-values.yaml +++ b/charts/matchmaking-agent/ci/integration-values.yaml @@ -3,7 +3,7 @@ # Copyright (c) 2023 ZF Friedrichshafen AG # Copyright (c) 2023 Mercedes-Benz Tech Innovation GmbH # Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) -# Copyright (c) 2021, 2023 Contributors to the Eclipse Foundation +# Copyright (c) 2021,2023 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. From 6c10395128e6cf71dc83bb27ac1d92918fc68c8e Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Thu, 16 May 2024 12:14:56 +0200 Subject: [PATCH 27/31] fix: important code smells/forgery threats as of codeql results 2024-05-16 --- .../eclipse/tractusx/agents/AgentConfig.java | 21 ++++-- .../eclipse/tractusx/agents/SkillStore.java | 34 +++------- .../tractusx/agents/http/AgentController.java | 65 +++++++++---------- .../agents/http/DelegationServiceImpl.java | 42 ++++++------ .../tractusx/agents/http/GraphController.java | 36 +++++----- .../agents/service/EdcSkillStore.java | 4 +- .../agents/service/InMemorySkillStore.java | 11 +++- 7 files changed, 112 insertions(+), 101 deletions(-) diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java index 5c7c2da4..f58ecf3e 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/AgentConfig.java @@ -61,7 +61,7 @@ public class AgentConfig { public static final String DATASPACE_SYNCCONNECTORS_PROPERTY = "cx.agent.dataspace.remotes"; public static final String RDF_STORE = "cx.agent.rdf.store"; - + public static final String VALIDATION_ENDPOINTS = "edc.dataplane.token.validation.endpoints"; public static final String FEDERATION_SERVICE_BATCH_SIZE = "cx.agent.federation.batch.max"; @@ -95,10 +95,10 @@ public class AgentConfig { public static final String DEFAULT_SERVICE_DENY_ASSET_PATTERN = "^$"; public static final String MATCHMAKING_PORT = "web.http.internal.port"; public static final String MATCHMAKING_PATH = "web.http.internal.path"; - + public static final String DEFAULT_PORT = "web.http.default.port"; public static final String DEFAULT_PATH = "web.http.default.path"; - + public static final String CALLBACK_PORT = "web.http.callback.port"; public static final String CALLBACK_PATH = "web.http.callback.path"; @@ -109,6 +109,7 @@ public class AgentConfig { protected final Pattern serviceDenyPattern; protected final Pattern serviceAssetAllowPattern; protected final Pattern serviceAssetDenyPattern; + protected final Pattern assetReferencePattern; /** * references to EDC services @@ -129,6 +130,7 @@ public AgentConfig(Monitor monitor, Config config) { serviceDenyPattern = Pattern.compile(config.getString(SERVICE_DENY_PROPERTY, DEFAULT_SERVICE_DENY_PATTERN)); serviceAssetAllowPattern = Pattern.compile(config.getString(SERVICE_ALLOW_ASSET_PROPERTY, DEFAULT_SERVICE_ALLOW_ASSET_PATTERN)); serviceAssetDenyPattern = Pattern.compile(config.getString(SERVICE_DENY_ASSET_PROPERTY, DEFAULT_SERVICE_DENY_ASSET_PATTERN)); + assetReferencePattern = Pattern.compile("((?[^#]+)#)?(?.+)"); } /** @@ -374,7 +376,7 @@ public String getDefaultGraphContract() { } /** - * * access + * * access * * @return regular expression for allowed service URLs */ @@ -409,6 +411,15 @@ public Pattern getServiceAssetDenyPattern() { return serviceAssetDenyPattern; } + /** + * access + * + * @return regular expression for asset references + */ + public Pattern getAssetReferencePattern() { + return assetReferencePattern; + } + /** * access * @@ -462,5 +473,5 @@ public int getCallbackPort() { public String getCallbackPath() { return config.getString(CALLBACK_PATH); } - + } diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/SkillStore.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/SkillStore.java index 495e87a8..3170a346 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/SkillStore.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/SkillStore.java @@ -17,25 +17,11 @@ package org.eclipse.tractusx.agents; import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** * interface to a skill store */ public interface SkillStore { - - Pattern SKILL_PATTERN = Pattern.compile("((?[^#]+)#)?(?.*Skill(Asset)?.*)"); - - /** - * match a given asset - * - * @param key asset name - * @return matcher - */ - static Matcher matchSkill(String key) { - return SKILL_PATTERN.matcher(key); - } /** * check a given asset for being a skill @@ -48,17 +34,17 @@ static Matcher matchSkill(String key) { /** * register a skill * - * @param key asset name required - * @param skill query text required - * @param name of skill optional - * @param description of skill optional - * @param version of skill optional - * @param contract of skill optional - * @param dist of skill required - * @param isFederated whether skill maybe synchronized in catalogue + * @param key asset name required + * @param skill query text required + * @param name of skill optional + * @param description of skill optional + * @param version of skill optional + * @param contract of skill optional + * @param dist of skill required + * @param isFederated whether skill maybe synchronized in catalogue * @param allowServicePattern regex for service to call in skill - * @param denyServicePattern regex for services denied in skill - * @param ontologies a set of ontologies + * @param denyServicePattern regex for services denied in skill + * @param ontologies a set of ontologies * @return skill id */ String put(String key, String skill, String name, String description, String version, String contract, SkillDistribution dist, boolean isFederated, String allowServicePattern, String denyServicePattern, String... ontologies); diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/AgentController.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/AgentController.java index f4cd5eb8..8cc49b7b 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/AgentController.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/AgentController.java @@ -39,7 +39,6 @@ import java.util.Optional; import java.util.regex.Matcher; -import java.util.regex.Pattern; /** * The Agent Controller exposes a REST API endpoint @@ -62,16 +61,15 @@ public class AgentController { // the actual Matchmaking Agent is a Fuseki engine protected final SparqlQueryProcessor processor; protected final DelegationService delegationService; - - public static final Pattern GRAPH_PATTERN = Pattern.compile("((?[^#]+)#)?(?.*Graph(Asset)?.*)"); - /** + + /** * creates a new agent controller * - * @param monitor logging subsystem + * @param monitor logging subsystem * @param agreementController agreement controller for remote skill/queries - * @param config configuration - * @param processor sparql processor + * @param config configuration + * @param processor sparql processor */ public AgentController(Monitor monitor, AgreementController agreementController, AgentConfig config, SparqlQueryProcessor processor, SkillStore skillStore, DelegationService delegationService) { this.monitor = monitor; @@ -98,7 +96,7 @@ public String toString() { * @return response */ @POST - @Consumes({"application/sparql-query", "application/sparql-results+json"}) + @Consumes({ "application/sparql-query", "application/sparql-results+json" }) public Response postSparqlQuery(@QueryParam("asset") String asset, @Context HttpHeaders headers, @Context HttpServletRequest request, @@ -121,7 +119,7 @@ public Response postSparqlQuery(@QueryParam("asset") String asset, * @return response compatible with graphdb convention */ @POST - @Consumes({"application/x-www-form-urlencoded"}) + @Consumes({ "application/x-www-form-urlencoded" }) public Response postFormQuery(@QueryParam("asset") String asset, @Context HttpHeaders headers, @Context HttpServletRequest request, @@ -146,7 +144,7 @@ public Response postFormQuery(@QueryParam("asset") String asset, */ @POST @Path("/repositories/AGENT") - @Consumes({"application/x-www-form-urlencoded"}) + @Consumes({ "application/x-www-form-urlencoded" }) public Response postFormRepositoryQuery(@QueryParam("asset") String asset, @Context HttpHeaders headers, @Context HttpServletRequest request, @@ -381,17 +379,15 @@ public Response executeQuery(String asset, HttpHeaders headers, HttpServletReque String remoteUrl = null; if (asset != null) { - Matcher matcher = GRAPH_PATTERN.matcher(asset); + Matcher matcher = config.getAssetReferencePattern().matcher(asset); if (matcher.matches()) { remoteUrl = matcher.group("url"); - graph = matcher.group("graph"); - } else { - matcher = SkillStore.matchSkill(asset); - if (!matcher.matches()) { - return Response.status(Response.Status.BAD_REQUEST).build(); + asset = matcher.group("asset"); + if (asset.contains("Graph")) { + graph = asset; + } else if (asset.contains("Skill")) { + skill = asset; } - remoteUrl = matcher.group("url"); - skill = matcher.group("skill"); } } @@ -409,15 +405,14 @@ public Response executeQuery(String asset, HttpHeaders headers, HttpServletReque } try { - // exchange skill against text - if (asset != null) { - if (skillStore.isSkill(asset)) { - Optional skillOption = skillStore.get(asset); - if (skillOption.isPresent()) { - skill = skillOption.get(); - } else { - return HttpUtils.respond(monitor, headers, HttpStatus.SC_NOT_FOUND, "The requested skill is not registered.", null); - } + // exchange skill against text locally + if (asset != null && skill != null) { + Optional skillOption = skillStore.get(skill); + if (skillOption.isPresent()) { + skill = skillOption.get(); + } else { + skill = null; + return HttpUtils.respond(monitor, headers, HttpStatus.SC_NOT_FOUND, "The requested skill is not registered.", null); } } @@ -432,15 +427,15 @@ public Response executeQuery(String asset, HttpHeaders headers, HttpServletReque /** * endpoint for posting a skill * - * @param query mandatory query - * @param asset asset key - * @param name asset name + * @param query mandatory query + * @param asset asset key + * @param name asset name * @param description asset description - * @param version asset version - * @param contract asset contract - * @param mode asset mode + * @param version asset version + * @param contract asset contract + * @param mode asset mode * @param isFederated whether it appears in fed catalogue - * @param ontologies list of ontologies + * @param ontologies list of ontologies * @return only status */ @POST @@ -476,7 +471,7 @@ public Response postSkill(String query, */ @GET @Path("/skill") - @Produces({"application/sparql-query"}) + @Produces({ "application/sparql-query" }) public Response getSkill(@QueryParam("asset") String asset) { monitor.debug(String.format("Received a GET skill request %s", asset)); Response.ResponseBuilder rb; diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/DelegationServiceImpl.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/DelegationServiceImpl.java index 4daff70f..acf4f1b3 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/DelegationServiceImpl.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/DelegationServiceImpl.java @@ -46,6 +46,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -53,7 +54,7 @@ * A service that may delegate an incoming * agent http request ot another agent in the * dataspace - * deals with the special case of remote/provided (textual) + * deals with the special case of remote/provided (textual) * skills which should be executed locally nevertheless */ public class DelegationServiceImpl implements DelegationService { @@ -61,7 +62,8 @@ public class DelegationServiceImpl implements DelegationService { protected final AgreementController agreementController; protected final Monitor monitor; protected final OkHttpClient client; - public static final TypeReference> WARNING_TYPE_REFERENCE = new TypeReference<>(){}; + public static final TypeReference> WARNING_TYPE_REFERENCE = new TypeReference<>() { + }; protected final TypeManager typeManager; protected final AgentConfig config; @@ -69,8 +71,8 @@ public class DelegationServiceImpl implements DelegationService { * creates a new delegation service * * @param agreementController EDC agreement helper - * @param monitor logging facility - * @param client outgoing http infrastructure + * @param monitor logging facility + * @param client outgoing http infrastructure */ public DelegationServiceImpl(AgreementController agreementController, Monitor monitor, OkHttpClient client, TypeManager typeManager, AgentConfig config) { this.agreementController = agreementController; @@ -84,11 +86,11 @@ public DelegationServiceImpl(AgreementController agreementController, Monitor mo * the actual execution is done by delegating to the Dataspace * * @param remoteUrl remote connector - * @param skill target skill - * @param graph target graph + * @param skill target skill + * @param graph target graph * @return a wrapped response which indicates the runMode that the execution should be done */ - public DelegationResponse executeQueryRemote(String remoteUrl, String skill, String graph, HttpHeaders headers, HttpServletRequest request, HttpServletResponse response, UriInfo uri) { + public DelegationResponse executeQueryRemote(String remoteUrl, String skill, String graph, HttpHeaders headers, HttpServletRequest request, HttpServletResponse response, UriInfo uri) { Pattern serviceAllowPattern = config.getServiceAllowPattern(); if (!serviceAllowPattern.matcher(remoteUrl).matches()) { return new DelegationResponse(HttpUtils.respond(monitor, headers, HttpStatus.SC_FORBIDDEN, String.format("Service %s does not match the allowed service pattern %s", remoteUrl, serviceAllowPattern.pattern()), null)); @@ -131,7 +133,7 @@ public DelegationResponse executeQueryRemote(String remoteUrl, String skill, Str * route a get request * * @param dataReference the encoded call embedding - * @param subUrl protocol-specific part + * @param subUrl protocol-specific part * @return a wrapped response which indicates the runMode that the execution should be done * @throws IOException in case something strange happens */ @@ -156,7 +158,7 @@ public DelegationResponse sendGetRequest(EndpointDataReference dataReference, St * route a post request * * @param dataReference the encoded call embedding - * @param subUrl protocol-specific part + * @param subUrl protocol-specific part * @return a wrapped response which indicates the runMode that the execution should be done * @throws IOException in case something strange happens */ @@ -179,7 +181,7 @@ public DelegationResponse sendPostRequest(EndpointDataReference dataReference, S requestBuilder.post(okhttp3.RequestBody.create(request.getInputStream().readAllBytes(), parsedContentType)); var newRequest = requestBuilder.build(); - + return new DelegationResponse(sendRequest(newRequest, response), Response.status(response.getStatus()).build()); } @@ -190,11 +192,11 @@ public DelegationResponse sendPostRequest(EndpointDataReference dataReference, S * computes the url to target the given data plane * * @param connectorUrl data plane url - * @param subUrl sub-path to use - * @param headers containing additional info that we need to wrap into a transfer request + * @param subUrl sub-path to use + * @param headers containing additional info that we need to wrap into a transfer request * @return typed url */ - protected HttpUrl getUrl(String connectorUrl, String subUrl, HttpHeaders headers, UriInfo uri) { + protected HttpUrl getUrl(String connectorUrl, String subUrl, HttpHeaders headers, UriInfo uri) { var url = connectorUrl; // EDC public api slash problem @@ -209,11 +211,13 @@ protected HttpUrl getUrl(String connectorUrl, String subUrl, HttpHeaders headers HttpUrl.Builder httpBuilder = Objects.requireNonNull(okhttp3.HttpUrl.parse(url)).newBuilder(); for (Map.Entry> param : uri.getQueryParameters().entrySet()) { String key = param.getKey(); - if (PARAMETER_KEY_ALLOW.matcher(key).matches()) { + Matcher keyMatcher = PARAMETER_KEY_ALLOW.matcher(key); + if (keyMatcher.matches()) { for (String value : param.getValue()) { - if (PARAMETER_VALUE_ALLOW.matcher(value).matches()) { - String recodeKey = HttpUtils.urlEncodeParameter(key); - String recodeValue = HttpUtils.urlEncodeParameter(value); + Matcher valueMatcher = PARAMETER_VALUE_ALLOW.matcher(value); + if (valueMatcher.matches()) { + String recodeKey = HttpUtils.urlEncodeParameter(keyMatcher.group()); + String recodeValue = HttpUtils.urlEncodeParameter(valueMatcher.group()); httpBuilder = httpBuilder.addQueryParameter(recodeKey, recodeValue); } } @@ -234,7 +238,7 @@ protected HttpUrl getUrl(String connectorUrl, String subUrl, HttpHeaders headers /** * generic sendRequest method which extracts the result string of textual responses * - * @param request predefined request + * @param request predefined request * @param response the final response * @return the text of a downloaded skill if runMode = consumer, null otherwise * @throws IOException in case something goes wrong @@ -245,7 +249,7 @@ protected String sendRequest(okhttp3.Request request, HttpServletResponse respon if (!myResponse.isSuccessful()) { monitor.warning(String.format("Data plane call was not successful: %s", myResponse.code())); } - + Optional> warnings = Optional.empty(); var body = myResponse.body(); diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/GraphController.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/GraphController.java index 062986b7..d07a09df 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/GraphController.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/GraphController.java @@ -36,6 +36,7 @@ import org.eclipse.tractusx.agents.utils.Monitor; import java.io.IOException; +import java.util.regex.Matcher; /** * The Graph Controller exposes a REST API endpoint @@ -55,7 +56,7 @@ public class GraphController { * creates a new agent controller * * @param monitor logging subsystem - * @param store the rdf store to extend + * @param store the rdf store to extend */ public GraphController(Monitor monitor, RdfStore store, DataManagement management, AgentConfig config) { this.monitor = monitor; @@ -75,19 +76,19 @@ public String toString() { /** * endpoint for posting a ttl into a local graph asset * - * @param content mandatory content - * @param asset asset key - * @param name asset name + * @param content mandatory content + * @param asset asset key + * @param name asset name * @param description asset description - * @param version asset version - * @param contract asset contract - * @param shape asset shape + * @param version asset version + * @param contract asset contract + * @param shape asset shape * @param isFederated whether it appears in fed catalogue - * @param ontologies list of ontologies + * @param ontologies list of ontologies * @return response indicating the number of triples updated */ @POST - @Consumes({"text/turtle", "text/csv"}) + @Consumes({ "text/turtle", "text/csv" }) public Response postAsset(String content, @QueryParam("asset") String asset, @QueryParam("assetName") String name, @@ -143,15 +144,20 @@ public Response postAsset(String content, */ @DELETE public Response deleteAsset(@QueryParam("asset") String asset, - @Context HttpHeaders headers, - @Context HttpServletRequest request, - @Context HttpServletResponse response, - @Context UriInfo uri + @Context HttpHeaders headers, + @Context HttpServletRequest request, + @Context HttpServletResponse response, + @Context UriInfo uri ) { monitor.debug(String.format("Received a DELETE request %s for asset %s", request, asset)); try { - management.deleteAsset(asset); - return Response.ok(store.deleteAsset(asset), MediaType.APPLICATION_JSON_TYPE).build(); + Matcher assetMatcher = config.getAssetReferencePattern().matcher(asset); + if (assetMatcher.matches()) { + management.deleteAsset(assetMatcher.group()); + return Response.ok(store.deleteAsset(asset), MediaType.APPLICATION_JSON_TYPE).build(); + } else { + return Response.status(Response.Status.NOT_ACCEPTABLE).build(); + } } catch (IOException e) { return Response.status(Response.Status.BAD_REQUEST).build(); } diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/EdcSkillStore.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/EdcSkillStore.java index 7c1e5bb3..788073a3 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/EdcSkillStore.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/EdcSkillStore.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.util.List; import java.util.Optional; +import java.util.regex.Matcher; /** * Implements a skill store based on EDC assets @@ -47,7 +48,8 @@ public EdcSkillStore(DataManagement management, TypeManager typeManager, AgentCo @Override public boolean isSkill(String key) { - return SkillStore.matchSkill(key).matches(); + Matcher matcher = config.getAssetReferencePattern().matcher(key); + return matcher.matches() && matcher.group("asset").contains("Skill"); } @Override diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/InMemorySkillStore.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/InMemorySkillStore.java index 24568cf5..04036e10 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/InMemorySkillStore.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/service/InMemorySkillStore.java @@ -16,12 +16,14 @@ // SPDX-License-Identifier: Apache-2.0 package org.eclipse.tractusx.agents.service; +import org.eclipse.tractusx.agents.AgentConfig; import org.eclipse.tractusx.agents.SkillDistribution; import org.eclipse.tractusx.agents.SkillStore; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.regex.Matcher; /** * An in-memory store for local skills @@ -31,15 +33,20 @@ public class InMemorySkillStore implements SkillStore { // temporary local skill store protected final Map skills = new HashMap<>(); + protected AgentConfig config; + /** * create the store */ - public InMemorySkillStore() { + public InMemorySkillStore(AgentConfig config) { + this.config = config; } + @Override public boolean isSkill(String key) { - return SkillStore.matchSkill(key).matches(); + Matcher matcher = config.getAssetReferencePattern().matcher(key); + return matcher.matches() && matcher.group("asset").contains("Skill"); } @Override From ac9f68ea52d25c9a5dd237078c64284539cef599 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Thu, 16 May 2024 13:20:08 +0200 Subject: [PATCH 28/31] fix: important code smells/forgery threats as of codeql results 2024-05-16 --- .../tractusx/agents/http/DelegationServiceImpl.java | 8 ++++---- .../org/eclipse/tractusx/agents/http/GraphController.java | 2 +- pom.xml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/DelegationServiceImpl.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/DelegationServiceImpl.java index acf4f1b3..b02fb2e0 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/DelegationServiceImpl.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/DelegationServiceImpl.java @@ -185,8 +185,8 @@ public DelegationResponse sendPostRequest(EndpointDataReference dataReference, S return new DelegationResponse(sendRequest(newRequest, response), Response.status(response.getStatus()).build()); } - protected static final Pattern PARAMETER_KEY_ALLOW = Pattern.compile("^(?!asset$)[^&?=]+$"); - protected static final Pattern PARAMETER_VALUE_ALLOW = Pattern.compile("^.+$"); + protected static final Pattern PARAMETER_KEY_ALLOW = Pattern.compile("^(?(?!asset$)[^&?=]+)$"); + protected static final Pattern PARAMETER_VALUE_ALLOW = Pattern.compile("^(?[^&]+)$"); /** * computes the url to target the given data plane @@ -213,11 +213,11 @@ protected HttpUrl getUrl(String connectorUrl, String subUrl, HttpHeaders headers String key = param.getKey(); Matcher keyMatcher = PARAMETER_KEY_ALLOW.matcher(key); if (keyMatcher.matches()) { + String recodeKey = HttpUtils.urlEncodeParameter(keyMatcher.group("param")); for (String value : param.getValue()) { Matcher valueMatcher = PARAMETER_VALUE_ALLOW.matcher(value); if (valueMatcher.matches()) { - String recodeKey = HttpUtils.urlEncodeParameter(keyMatcher.group()); - String recodeValue = HttpUtils.urlEncodeParameter(valueMatcher.group()); + String recodeValue = HttpUtils.urlEncodeParameter(valueMatcher.group("value")); httpBuilder = httpBuilder.addQueryParameter(recodeKey, recodeValue); } } diff --git a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/GraphController.java b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/GraphController.java index d07a09df..452f04f9 100644 --- a/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/GraphController.java +++ b/matchmaking/src/main/java/org/eclipse/tractusx/agents/http/GraphController.java @@ -153,7 +153,7 @@ public Response deleteAsset(@QueryParam("asset") String asset, try { Matcher assetMatcher = config.getAssetReferencePattern().matcher(asset); if (assetMatcher.matches()) { - management.deleteAsset(assetMatcher.group()); + management.deleteAsset(assetMatcher.group("asset")); return Response.ok(store.deleteAsset(asset), MediaType.APPLICATION_JSON_TYPE).build(); } else { return Response.status(Response.Status.NOT_ACCEPTABLE).build(); diff --git a/pom.xml b/pom.xml index 0e83dfe3..1f107c40 100644 --- a/pom.xml +++ b/pom.xml @@ -59,7 +59,7 @@ 32.1.2-jre 9.0.83 4.1.101.Final - 9.4.53.v20231009 + 9.4.54.v20240208 2.2 1.1.10.5 42.7.2 From ee749dab13bc5676eafde15d182fbc29bc21ff0d Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Thu, 16 May 2024 13:23:10 +0200 Subject: [PATCH 29/31] chore: update DEPENDENCIES after vulnerability fixing. --- DEPENDENCIES | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/DEPENDENCIES b/DEPENDENCIES index f47e9b82..7ec842cb 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -45,6 +45,7 @@ maven/mavencentral/commons-codec/commons-codec/1.14, Apache-2.0, approved, clear maven/mavencentral/commons-codec/commons-codec/1.15, Apache-2.0 AND BSD-3-Clause AND LicenseRef-Public-Domain, approved, CQ22641 maven/mavencentral/commons-collections/commons-collections/3.2.2, Apache-2.0, approved, CQ10385 maven/mavencentral/commons-io/commons-io/2.11.0, Apache-2.0, approved, CQ23745 +maven/mavencentral/commons-io/commons-io/2.15.1, Apache-2.0, approved, #11244 maven/mavencentral/commons-lang/commons-lang/2.6, Apache-2.0, approved, CQ6183 maven/mavencentral/commons-logging/commons-logging/1.2, Apache-2.0, approved, CQ10162 maven/mavencentral/io.github.classgraph/classgraph/4.8.154, MIT, approved, CQ22530 @@ -104,9 +105,10 @@ maven/mavencentral/net.bytebuddy/byte-buddy/1.12.21, Apache-2.0 AND BSD-3-Clause maven/mavencentral/net.minidev/accessors-smart/2.4.7, Apache-2.0, approved, #7515 maven/mavencentral/net.minidev/json-smart/2.4.7, Apache-2.0, approved, #3288 maven/mavencentral/org.apache.commons/commons-collections4/4.4, Apache-2.0, approved, clearlydefined -maven/mavencentral/org.apache.commons/commons-compress/1.23.0, Apache-2.0 AND BSD-3-Clause, approved, #7506 +maven/mavencentral/org.apache.commons/commons-compress/1.26.0, Apache-2.0 AND (Apache-2.0 AND BSD-3-Clause), approved, #13288 maven/mavencentral/org.apache.commons/commons-csv/1.10.0, Apache-2.0, approved, clearlydefined maven/mavencentral/org.apache.commons/commons-lang3/3.12.0, Apache-2.0, approved, clearlydefined +maven/mavencentral/org.apache.commons/commons-lang3/3.14.0, Apache-2.0, approved, #11677 maven/mavencentral/org.apache.commons/commons-math3/3.6.1, Apache-2.0 AND BSD-3-Clause AND BSD-2-Clause, approved, CQ22025 maven/mavencentral/org.apache.commons/commons-rdf-api/0.5.0, Apache-2.0, approved, clearlydefined maven/mavencentral/org.apache.commons/commons-rdf-simple/0.5.0, Apache-2.0, approved, #9811 @@ -168,23 +170,23 @@ maven/mavencentral/org.codehaus.woodstox/stax2-api/3.1.4, BSD-2-Clause, approved maven/mavencentral/org.codehaus.woodstox/stax2-api/4.2.1, BSD-2-Clause, approved, #2670 maven/mavencentral/org.codehaus.woodstox/woodstox-core-asl/4.4.1, Apache-2.0, approved, clearlydefined maven/mavencentral/org.eclipse.jetty.http2/http2-client/9.4.52.v20230823, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty.http2/http2-common/9.4.52.v20230823, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty.http2/http2-hpack/9.4.53.v20231009, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty.http2/http2-common/9.4.54.v20240208, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty.http2/http2-hpack/9.4.54.v20240208, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty.http2/http2-http-client-transport/9.4.52.v20230823, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty.toolchain/jetty-jakarta-servlet-api/5.0.2, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-alpn-client/9.4.53.v20231009, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-alpn-java-client/9.4.53.v20231009, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-client/9.4.53.v20231009, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-alpn-client/9.4.54.v20240208, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-alpn-java-client/9.4.54.v20240208, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-client/9.4.54.v20240208, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty/jetty-http/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-http/9.4.53.v20231009, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-http/9.4.54.v20240208, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty/jetty-io/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-io/9.4.53.v20231009, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-io/9.4.54.v20240208, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty/jetty-security/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty/jetty-server/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty/jetty-servlet/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty/jetty-servlets/10.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty/jetty-util/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-util/9.4.53.v20231009, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-util/9.4.54.v20240208, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.rdf4j/rdf4j-client/4.3.8, BSD-3-Clause, approved, technology.rdf4j maven/mavencentral/org.eclipse.rdf4j/rdf4j-collection-factory-api/4.3.8, BSD-3-Clause, approved, technology.rdf4j maven/mavencentral/org.eclipse.rdf4j/rdf4j-collection-factory-mapdb/4.3.8, BSD-3-Clause, approved, technology.rdf4j @@ -330,7 +332,7 @@ maven/mavencentral/org.objenesis/objenesis/3.3, Apache-2.0, approved, clearlydef maven/mavencentral/org.opentest4j/opentest4j/1.2.0, Apache-2.0, approved, clearlydefined maven/mavencentral/org.opentest4j/opentest4j/1.3.0, Apache-2.0, approved, #9713 maven/mavencentral/org.ow2.asm/asm/9.1, BSD-3-Clause, approved, CQ23029 -maven/mavencentral/org.postgresql/postgresql/42.6.0, BSD-2-Clause AND Apache-2.0, approved, #9159 +maven/mavencentral/org.postgresql/postgresql/42.7.2, BSD-2-Clause AND Apache-2.0, approved, #11681 maven/mavencentral/org.roaringbitmap/RoaringBitmap/0.9.45, Apache-2.0, approved, clearlydefined maven/mavencentral/org.roaringbitmap/shims/0.9.45, Apache-2.0, approved, clearlydefined maven/mavencentral/org.simpleframework/simple-common/6.0.1, Apache-2.0, approved, CQ16868 From e9da5f59fd1df6b8e2703f170b2fe7acd33d6cad Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Fri, 17 May 2024 18:02:57 +0200 Subject: [PATCH 30/31] docs: changelog for upcoming release. --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a4025c6..ab768399 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,32 @@ All notable changes to this product will be documented in this file. # Released +## [1.12.19] - 2024-05-17 + +### Added + +- Matchmaking Agent: Carve-Out of the SPARQL-Processor from [Tractus-X EDC Agent Plane (KA-EDC)](http://github.com/eclipse-tractusx/knowledge-agents-edc) in order to hide graph data in the internal network. +- Matchmaking Agent: Implements a new API extension to upload/delete graph assets by ttl and csv. + +### Changed + +- Remoting Agent: Deal with mapping of taxonomical constants to API elements. Support for JSON-serialization of partial results. +- Upgraded to the latest possible version of dependent libraries + +### Removed + +## [1.11.16] - 2024-02-20 + +### Added + +### Changed + +- Remoting Agent upgraded to latest RDF4J distribution +- Provisioning Agent upgrade to latest Ontop release +- Upgraded to the latest possible version of dependent libraries + +### Removed + ## [1.10.15] - 2023-11-28 ### Added From e2ba8877594be2af1ca4955d027ac379f6382c16 Mon Sep 17 00:00:00 2001 From: "Dr. Christoph \"Schorsch\" Jung" Date: Sat, 18 May 2024 08:29:51 +0200 Subject: [PATCH 31/31] docs: document the charts. --- charts/conforming-agent/README.md | 2 +- charts/matchmaking-agent/README.md | 51 ++++++++++++++++++++++++----- charts/provisioning-agent/README.md | 2 +- charts/remoting-agent/README.md | 2 +- 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/charts/conforming-agent/README.md b/charts/conforming-agent/README.md index 15b03048..3635bf73 100644 --- a/charts/conforming-agent/README.md +++ b/charts/conforming-agent/README.md @@ -20,7 +20,7 @@ # conforming-agent -![Version: 1.12.19-SNAPSHOT](https://img.shields.io/badge/Version-1.12.17--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.12.19-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.12.17--SNAPSHOT-informational?style=flat-square) +![Version: 1.12.19-SNAPSHOT](https://img.shields.io/badge/Version-1.12.19--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.12.19-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.12.19--SNAPSHOT-informational?style=flat-square) A Helm chart for the Tractus-X Conforming Agent which is a container to assess the conformity of all other parts of the Agent-Enabled Dataspace. diff --git a/charts/matchmaking-agent/README.md b/charts/matchmaking-agent/README.md index 86d547e7..08f85dd3 100644 --- a/charts/matchmaking-agent/README.md +++ b/charts/matchmaking-agent/README.md @@ -19,9 +19,9 @@ --> # matchmaking-agent -![Version: 1.10.15-SNAPSHOT](https://img.shields.io/badge/Version-1.10.2--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.10.15-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.10.2--SNAPSHOT-informational?style=flat-square) +![Version: 1.12.19-SNAPSHOT](https://img.shields.io/badge/Version-1.12.19--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.12.19-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.12.19--SNAPSHOT-informational?style=flat-square) -A Helm chart for the Tractus-X Matchmaking Agent which is a container to Bridge Agent-Enabled Connector and REST APIs. +A Helm chart for the Tractus-X Matchmaking Agent which is a container encompassing data storage capabilities accessible from the dataplane by a REST API This chart has no prerequisites. @@ -30,7 +30,7 @@ This chart has no prerequisites. ## TL;DR ```shell $ helm repo add eclipse-tractusx https://eclipse-tractusx.github.io/charts/dev -$ helm install my-release eclipse-tractusx/matchmaking-agent --version 1.10.15-SNAPSHOT +$ helm install my-release eclipse-tractusx/matchmaking-agent --version 1.12.19-SNAPSHOT ``` ## Maintainers @@ -48,16 +48,51 @@ $ helm install my-release eclipse-tractusx/matchmaking-agent --version 1.10.15-S | Key | Type | Default | Description | |-----|------|---------|-------------| | affinity | object | `{}` | [Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) constrains which nodes the Pod can be scheduled on based on node labels. | +| agent.connectors | object | `{}` | A map of partner ids to remote connector IDS URLs to synchronize with | +| agent.default[0] | string | `"dataspace.ttl"` | | +| agent.default[1] | string | `"https://w3id.org/catenax/ontology.ttl"` | | +| agent.endpoints.callback.auth | object | `{}` | An auth object for default security | +| agent.endpoints.callback.path | string | `"/callback"` | The path mapping the "callback" api is going to be exposed by | +| agent.endpoints.callback.port | string | `"8087"` | The network port, which the "callback" api is going to be exposed by the container, pod and service | +| agent.endpoints.callback.regex | string | `"/(.*)"` | An optional regex path match (whose match groups could be used in an nginx-annotation of the ingress) | | agent.endpoints.default.auth | object | `{}` | An auth object for default security | -| agent.endpoints.default.path | string | `""` | The path mapping the "default" api is going to be exposed by | -| agent.endpoints.default.port | string | `"8081"` | The network port, which the "default" api is going to be exposed by the container, pod and service | +| agent.endpoints.default.path | string | `"/api"` | The path mapping the "default" api is going to be exposed by | +| agent.endpoints.default.port | string | `"8082"` | The network port, which the "default" api is going to be exposed by the container, pod and service | | agent.endpoints.default.regex | string | `"/(.*)"` | An optional regex path match (whose match groups could be used in an nginx-annotation of the ingress) | +| agent.endpoints.internal.auth | object | `{}` | An auth object for default security | +| agent.endpoints.internal.path | string | `"/agentsource"` | The path mapping the "source" api is going to be exposed by | +| agent.endpoints.internal.port | string | `"8080"` | The network port, which the "source" api is going to be exposed by the container, pod and service | +| agent.endpoints.internal.regex | string | `"/(.*)"` | An optional regex path match (whose match groups could be used in an nginx-annotation of the ingress) | +| agent.graphcontract | string | `"Contract?partner=Graph"` | Names the visible contract under which new graphs are published (if not otherwise specified) | +| agent.maxbatchsize | string | `"9223372036854775807"` | Sets the maximal batch size when delegating to agents and services | +| agent.services | object | `{"allow":"(edcs?://.*)|(https://query\\\\.wikidata\\\\.org/sparql)","asset":{"allow":"(edcs?://.*)","deny":"https?://.*"},"deny":"http://.*"}` | A set of configs for regulating outgoing service calls | +| agent.services.allow | string | `"(edcs?://.*)|(https://query\\\\.wikidata\\\\.org/sparql)"` | A regular expression which outgoing service URLs must match (unless overwritten by a specific asset property) | +| agent.services.asset | object | `{"allow":"(edcs?://.*)","deny":"https?://.*"}` | A set of configs for regulating outgoing service calls when providing an asset (when no specific asset property is given) | +| agent.services.asset.allow | string | `"(edcs?://.*)"` | A regular expression which outgoing service URLs must match (unless overwritten by a specific asset property) | +| agent.services.asset.deny | string | `"https?://.*"` | A regular expression which outgoing service URLs must not match (unless overwritten by a specific asset property) | +| agent.services.deny | string | `"http://.*"` | A regular expression which outgoing service URLs must not match (unless overwritten by a specific asset property) | +| agent.skillcontract | string | `"Contract?partner=Skill"` | Names the visible contract under which new skills are published (if not otherwise specified) | +| agent.synchronization | int | `-1` | The synchronization interval in ms to update the federated data catalogue | | automountServiceAccountToken | bool | `false` | Whether to [automount kubernetes API credentials](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#use-the-default-service-account-to-access-the-api-server) into the pod | | autoscaling.enabled | bool | `false` | Enables [horizontal pod autoscaling](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) | | autoscaling.maxReplicas | int | `100` | Maximum replicas if resource consumption exceeds resource threshholds | | autoscaling.minReplicas | int | `1` | Minimal replicas if resource consumption falls below resource threshholds | | autoscaling.targetCPUUtilizationPercentage | int | `80` | targetAverageUtilization of cpu provided to a pod | | autoscaling.targetMemoryUtilizationPercentage | int | `80` | targetAverageUtilization of memory provided to a pod | +| configs | object | `{"dataspace.ttl":"#################################################################\n# Catena-X Agent Bootstrap Graph in TTL/RDF/OWL FORMAT\n#################################################################\n@prefix : .\n@base .\n"}` | A set of additional configuration files | +| configs."dataspace.ttl" | string | `"#################################################################\n# Catena-X Agent Bootstrap Graph in TTL/RDF/OWL FORMAT\n#################################################################\n@prefix : .\n@base .\n"` | An example of an empty graph in ttl syntax | +| connector | string | `""` | Name of the connector deployment | +| controlplane | object | `{"endpoints":{"control":{"path":"/control","port":8083},"management":{"authKey":"","path":"/management","port":8081},"protocol":{"path":"/api/v1/dsp","port":8084}},"ingresses":[{"enabled":false}]}` | References to the control plane deployment | +| controlplane.endpoints.control | object | `{"path":"/control","port":8083}` | control api, used for internal control calls. can be added to the internal ingress, but should probably not | +| controlplane.endpoints.control.path | string | `"/control"` | path for incoming api calls | +| controlplane.endpoints.control.port | int | `8083` | port for incoming api calls | +| controlplane.endpoints.management | object | `{"authKey":"","path":"/management","port":8081}` | data management api, used by internal users, can be added to an ingress and must not be internet facing | +| controlplane.endpoints.management.authKey | string | `""` | authentication key, must be attached to each 'X-Api-Key' request header | +| controlplane.endpoints.management.path | string | `"/management"` | path for incoming api calls | +| controlplane.endpoints.management.port | int | `8081` | port for incoming api calls | +| controlplane.endpoints.protocol | object | `{"path":"/api/v1/dsp","port":8084}` | dsp api, used for inter connector communication and must be internet facing | +| controlplane.endpoints.protocol.path | string | `"/api/v1/dsp"` | path for incoming api calls | +| controlplane.endpoints.protocol.port | int | `8084` | port for incoming api calls | | customLabels | object | `{}` | Additional custom Labels to add | | env | object | `{}` | Container environment variables e.g. for configuring [JAVA_TOOL_OPTIONS](https://docs.oracle.com/javase/8/docs/technotes/guides/troubleshoot/envvars002.html) Ex.: JAVA_TOOL_OPTIONS: > -Dhttp.proxyHost=proxy -Dhttp.proxyPort=80 -Dhttp.nonProxyHosts="localhost|127.*|[::1]" -Dhttps.proxyHost=proxy -Dhttps.proxyPort=443 | | envSecretName | string | `nil` | [Kubernetes Secret Resource](https://kubernetes.io/docs/concepts/configuration/secret/) name to load environment variables from | @@ -73,7 +108,7 @@ $ helm install my-release eclipse-tractusx/matchmaking-agent --version 1.10.15-S | ingresses[0].certManager.issuer | string | `""` | If preset enables certificate generation via cert-manager namespace scoped issuer | | ingresses[0].className | string | `""` | Defines the [ingress class](https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class) to use | | ingresses[0].enabled | bool | `false` | | -| ingresses[0].endpoints | list | `["default"]` | Agent endpoints exposed by this ingress resource | +| ingresses[0].endpoints | list | `["default","source","callback"]` | Agent endpoints exposed by this ingress resource | | ingresses[0].hostname | string | `"matchmaking-agent.local"` | The hostname to be used to precisely map incoming traffic onto the underlying network service | | ingresses[0].prefix | string | `""` | Optional prefix that will be prepended to the paths of the endpoints | | ingresses[0].tls | object | `{"enabled":false,"secretName":""}` | TLS [tls class](https://kubernetes.io/docs/concepts/services-networking/ingress/#tls) applied to the ingress resource | @@ -86,6 +121,7 @@ $ helm install my-release eclipse-tractusx/matchmaking-agent --version 1.10.15-S | logging.configuration | string | `"\n\n \n \n %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n\n \n \n \n \n \n \n"` | Logback Xml | | nameOverride | string | `""` | Overrides the charts name | | nodeSelector | object | `{}` | [Node-Selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) to constrain the Pod to nodes with specific labels. | +| participant.id | string | `""` | BPN Number | | podAnnotations | object | `{}` | [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) added to deployed [pods](https://kubernetes.io/docs/concepts/workloads/pods/) | | podSecurityContext.fsGroup | int | `30000` | The owner for volumes and any files created within volumes will belong to this guid | | podSecurityContext.runAsGroup | int | `30000` | Processes within a pod will belong to this guid | @@ -96,7 +132,6 @@ $ helm install my-release eclipse-tractusx/matchmaking-agent --version 1.10.15-S | readinessProbe.periodSeconds | int | `300` | Number of seconds each period lasts. | | readinessProbe.timeoutSeconds | int | `5` | number of seconds until a timeout is assumed | | replicaCount | int | `1` | Specifies how many replicas of a deployed pod shall be created during the deployment Note: If horizontal pod autoscaling is enabled this setting has no effect | -| repositories | object | `{}` | A map of repository names to configuration ttl files | | resources | object | `{"limits":{"cpu":"250m","memory":"768Mi"},"requests":{"cpu":"250m","memory":"768Mi"}}` | [Resource management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) applied to the deployed pod We recommend 25% of a cpu, 512MB per server and 256MB per endpoint | | securityContext.allowPrivilegeEscalation | bool | `false` | Controls [Privilege Escalation](https://kubernetes.io/docs/concepts/security/pod-security-policy/#privilege-escalation) enabling setuid binaries changing the effective user ID | | securityContext.capabilities.add | list | `["NET_BIND_SERVICE"]` | Specifies which capabilities to add to issue specialized syscalls | @@ -117,4 +152,4 @@ $ helm install my-release eclipse-tractusx/matchmaking-agent --version 1.10.15-S | tolerations | list | `[]` | [Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) are applied to Pods to schedule onto nodes with matching taints. | ---------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0) +Autogenerated from chart metadata using [helm-docs v1.11.2](https://github.com/norwoodj/helm-docs/releases/v1.11.2) diff --git a/charts/provisioning-agent/README.md b/charts/provisioning-agent/README.md index b91d1917..2b7fb466 100644 --- a/charts/provisioning-agent/README.md +++ b/charts/provisioning-agent/README.md @@ -20,7 +20,7 @@ # provisioning-agent -![Version: 1.12.19-SNAPSHOT](https://img.shields.io/badge/Version-1.12.17--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.12.19-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.12.17--SNAPSHOT-informational?style=flat-square) +![Version: 1.12.19-SNAPSHOT](https://img.shields.io/badge/Version-1.12.19--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.12.19-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.12.19--SNAPSHOT-informational?style=flat-square) A Helm chart for the Tractus-X Provisioning Agent which is a container to Bridge Agent-Enabled Connector and Relational Data Sources. diff --git a/charts/remoting-agent/README.md b/charts/remoting-agent/README.md index b10fbb23..c0472772 100644 --- a/charts/remoting-agent/README.md +++ b/charts/remoting-agent/README.md @@ -19,7 +19,7 @@ --> # remoting-agent -![Version: 1.12.19-SNAPSHOT](https://img.shields.io/badge/Version-1.12.17--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.12.19-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.12.17--SNAPSHOT-informational?style=flat-square) +![Version: 1.12.19-SNAPSHOT](https://img.shields.io/badge/Version-1.12.19--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.12.19-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.12.19--SNAPSHOT-informational?style=flat-square) A Helm chart for the Tractus-X Remoting Agent which is a container to Bridge Agent-Enabled Connector and REST APIs.