diff --git a/src/main/java/iudx/catalogue/server/apiserver/MlayerApis.java b/src/main/java/iudx/catalogue/server/apiserver/MlayerApis.java index 1a8c364c..5fb374c8 100644 --- a/src/main/java/iudx/catalogue/server/apiserver/MlayerApis.java +++ b/src/main/java/iudx/catalogue/server/apiserver/MlayerApis.java @@ -574,9 +574,14 @@ public void getMlayerDatasetHandler(RoutingContext routingContext) { */ public void getMlayerPopularDatasetsHandler(RoutingContext routingContext) { LOGGER.debug("Info : fetching the data for the landing Page"); + String instance = ""; + if (routingContext.request().getParam(INSTANCE) != null) { + instance = routingContext.request().getParam(INSTANCE); + } + LOGGER.debug("Instance {}", instance); HttpServerResponse response = routingContext.response(); response.putHeader(HEADER_CONTENT_TYPE, MIME_APPLICATION_JSON); - mlayerService.getMlayerPopularDatasets( + mlayerService.getMlayerPopularDatasets(instance, handler -> { if (handler.succeeded()) { response.setStatusCode(200).end(handler.result().toString()); diff --git a/src/main/java/iudx/catalogue/server/database/Constants.java b/src/main/java/iudx/catalogue/server/database/Constants.java index 124eb7b5..c924dd03 100644 --- a/src/main/java/iudx/catalogue/server/database/Constants.java +++ b/src/main/java/iudx/catalogue/server/database/Constants.java @@ -207,6 +207,16 @@ public class Constants { + " \"iudx:Provider\"}}]}}]}},\"_source\":{\"includes\": [\"id\",\"description\"," + "\"type\",\"resourceGroup\",\"accessPolicy\",\"provider\",\"itemCreatedAt\"," + "\"instance\",\"label\"]},\"size\":10000}"; + + public static final String GET_DATASET_BY_INSTANCE = + "{\"query\":{\"bool\":{\"should\":[{\"bool\":{\"must\":[{\"match\":{\"type.keyword\":" + + "\"iudx:ResourceGroup\"}},{\"match\": {\"instance.keyword\": \"$1\"}}]}}," + + "{\"bool\":" + + "{\"must\":[{\"match\":{\"type.keyword\":\"iudx:Resource\"}},{\"match\":" + + " {\"instance.keyword\": \"$1\"}}]}},{\"bool\":{\"must\":[{\"match\":" + + "{\"type.keyword\": \"iudx:Provider\"}}]}}]}},\"_source\":{\"includes\": " + + "[\"id\",\"description\",\"type\",\"resourceGroup\",\"accessPolicy\"," + + "\"provider\",\"itemCreatedAt\",\"instance\",\"label\"]},\"size\":10000}\n"; public static final String GET_SORTED_MLAYER_INSTANCES = "{\"query\": {\"match_all\":{}},\"sort\":[{\"name\":\"asc\"}],\"_source\": " + "{\"includes\": [\"name\",\"cover\",\"icon\"]},\"size\":10000}"; diff --git a/src/main/java/iudx/catalogue/server/database/DatabaseService.java b/src/main/java/iudx/catalogue/server/database/DatabaseService.java index cded70b2..592778f5 100644 --- a/src/main/java/iudx/catalogue/server/database/DatabaseService.java +++ b/src/main/java/iudx/catalogue/server/database/DatabaseService.java @@ -297,7 +297,7 @@ DatabaseService updateMlayerInstance( DatabaseService getMlayerDataset(String datasetId, Handler> handler); @Fluent - DatabaseService getMlayerPopularDatasets(JsonArray highestCountResource, + DatabaseService getMlayerPopularDatasets(String instance, JsonArray highestCountResource, Handler> handler); /* create db service with nlp and geocoding */ diff --git a/src/main/java/iudx/catalogue/server/database/DatabaseServiceImpl.java b/src/main/java/iudx/catalogue/server/database/DatabaseServiceImpl.java index 128e3226..e5355d01 100644 --- a/src/main/java/iudx/catalogue/server/database/DatabaseServiceImpl.java +++ b/src/main/java/iudx/catalogue/server/database/DatabaseServiceImpl.java @@ -1925,7 +1925,7 @@ public DatabaseService getMlayerDataset( } @Override - public DatabaseService getMlayerPopularDatasets( + public DatabaseService getMlayerPopularDatasets(String instance, JsonArray highestCountResource, Handler> handler) { Promise instanceResult = Promise.promise(); @@ -1935,7 +1935,7 @@ public DatabaseService getMlayerPopularDatasets( searchSortedMlayerInstances(instanceResult); allMlayerDomains(domainResult); - datasets(datasetResult, highestCountResource); + datasets(instance, datasetResult, highestCountResource); CompositeFuture.all(instanceResult.future(), domainResult.future(), datasetResult.future()) .onComplete( ar -> { @@ -2081,9 +2081,16 @@ public int compare(JsonObject record1, JsonObject record2) { return jsonComparator; } - private void datasets(Promise datasetResult, JsonArray highestCountResource) { + private void datasets(String instance, Promise datasetResult, + JsonArray highestCountResource) { + String providerAndResources = ""; + if (instance.isBlank()) { + providerAndResources = GET_PROVIDER_AND_RESOURCES; + } else { + providerAndResources = GET_DATASET_BY_INSTANCE.replace("$1", instance); + } client.searchAsync( - GET_PROVIDER_AND_RESOURCES, + providerAndResources, docIndex, getCatRecords -> { if (getCatRecords.succeeded()) { @@ -2125,6 +2132,22 @@ private void datasets(Promise datasetResult, JsonArray highestCountR // sorting resource group based on the time of creation. Collections.sort(resourceGroupArray, comapratorForLatestDataset()); + // getting count of providers of a particular instance + ArrayList providerList = new ArrayList(); + if (!instance.isBlank()) { + for (int i = 0; i < getCatRecords.result().getJsonArray(RESULTS).size(); i++) { + JsonObject record = getCatRecords.result().getJsonArray(RESULTS).getJsonObject(i); + if (record.getJsonArray(TYPE).getString(0).equals("iudx:ResourceGroup") + && !providerList.contains(record.getString("provider")) + && !(record.getString("provider")).equals(null)) { + providerList.add(record.getString("provider")); + + } + + } + typeCount.put("iudx:Provider", providerList.size()); + } + ArrayList latestResourceGroup = new ArrayList<>(); int resourceGroupSize = 0; if (resourceGroupArray.size() < 6) { @@ -2151,17 +2174,13 @@ private void datasets(Promise datasetResult, JsonArray highestCountR if (resourceGroupArray .get(i) .getString("id") - .equals(highestCountResource.getJsonObject(j).getString("rgid"))) { - String datasetId = highestCountResource.getJsonObject(j).getString("rgid"); - int index = datasetId.indexOf("/", datasetId.indexOf("/") + 1); - String providerId = datasetId.substring(0, index); + .equals(highestCountResource.getJsonObject(j) + .getString("resourcegroup"))) { JsonObject resource = resourceGroupArray.get(i); resource .put( "totalResources", - resourceGroupCount.get(resourceGroupArray.get(i).getString("id"))) - .put("providerDescription", providerDescription.get(providerId)); - + resourceGroupCount.get(resourceGroupArray.get(i).getString("id"))); featuredResourceGroup.add(resource); resource = new JsonObject(); } diff --git a/src/main/java/iudx/catalogue/server/mlayer/MlayerService.java b/src/main/java/iudx/catalogue/server/mlayer/MlayerService.java index c61593a5..8df230d8 100644 --- a/src/main/java/iudx/catalogue/server/mlayer/MlayerService.java +++ b/src/main/java/iudx/catalogue/server/mlayer/MlayerService.java @@ -54,6 +54,6 @@ static MlayerService createProxy(Vertx vertx, String address) { MlayerService getMlayerDataset(String datasetId, Handler> handler); @Fluent - MlayerService getMlayerPopularDatasets(Handler> handler); + MlayerService getMlayerPopularDatasets(String instance, Handler> handler); } diff --git a/src/main/java/iudx/catalogue/server/mlayer/MlayerServiceImpl.java b/src/main/java/iudx/catalogue/server/mlayer/MlayerServiceImpl.java index 4f70c5f2..c56a8fd6 100644 --- a/src/main/java/iudx/catalogue/server/mlayer/MlayerServiceImpl.java +++ b/src/main/java/iudx/catalogue/server/mlayer/MlayerServiceImpl.java @@ -247,8 +247,8 @@ public MlayerService getMlayerDataset( } @Override - public MlayerService getMlayerPopularDatasets(Handler> handler) { - + public MlayerService getMlayerPopularDatasets(String instance, + Handler> handler) { String query = GET_HIGH_COUNT_DATASET.replace("$1", databaseTable); LOGGER.debug("postgres query" + query); postgresService.executeQuery( @@ -256,7 +256,9 @@ public MlayerService getMlayerPopularDatasets(Handler> h dbHandler -> { if (dbHandler.succeeded()) { JsonArray popularDataset = dbHandler.result().getJsonArray("results"); + LOGGER.debug("popular datasets are {}", popularDataset); databaseService.getMlayerPopularDatasets( + instance, popularDataset, getPopularDatasetsHandler -> { if (getPopularDatasetsHandler.succeeded()) { diff --git a/src/main/java/iudx/catalogue/server/mlayer/util/Constants.java b/src/main/java/iudx/catalogue/server/mlayer/util/Constants.java index 336418e0..6d5a3143 100644 --- a/src/main/java/iudx/catalogue/server/mlayer/util/Constants.java +++ b/src/main/java/iudx/catalogue/server/mlayer/util/Constants.java @@ -7,9 +7,7 @@ public class Constants { public static final String INSTANCE_ID = "instanceId"; public static final String DOMAIN_ID = "domainId"; public static final String GET_HIGH_COUNT_DATASET = - "with auditing_rs_view as (select resourceid, count(*) as hits, " - + "(select count(*) from regexp_matches(resourceid, '/', 'g')) as " - + "idtype from $1 group by resourceid) select left(resourceid,length(resourceid) " - + "-strpos(reverse(resourceid),'/')) as rgid, sum(hits) as totalhits from " - + "auditing_rs_view where idtype=4 group by rgid order by totalhits desc limit 6"; + "select resourcegroup, count(id) as totalhits from $1 " + + "group by resourcegroup order by totalhits " + + "desc limit 6"; } diff --git a/src/test/java/iudx/catalogue/server/database/DatabaseServiceImplTest.java b/src/test/java/iudx/catalogue/server/database/DatabaseServiceImplTest.java index 7a10db5c..0365a37d 100644 --- a/src/test/java/iudx/catalogue/server/database/DatabaseServiceImplTest.java +++ b/src/test/java/iudx/catalogue/server/database/DatabaseServiceImplTest.java @@ -2981,10 +2981,11 @@ public void testGetMlayerPopularDatasetsSuccess(VertxTestContext testContext) { mlayerDomainIndex, nlpService, geoService); + String instanceName =""; JsonObject json = - new JsonObject().put("rgid", "abcd/abcd/abcd/abcd").put("instance", "instance"); + new JsonObject().put("resourcegroup", "abcd/abcd/abcd/abcd").put("instance", "instance"); JsonObject json2 = - new JsonObject().put("rgid", "abcd/abcd/abcd/abcd").put("instance", "instance"); + new JsonObject().put("resourcegroup", "abcd/abcd/abcd/abcd").put("instance", "instance"); JsonArray highestCountResource = new JsonArray().add(json).add(json2); DatabaseServiceImpl.client = mock(ElasticClient.class); @@ -3030,6 +3031,7 @@ public AsyncResult answer(InvocationOnMock arg0) throws Throwable { .when(DatabaseServiceImpl.client) .searchAsync(any(), any(), any()); databaseService.getMlayerPopularDatasets( + instanceName, highestCountResource, handler -> { if (handler.succeeded()) { @@ -3042,9 +3044,11 @@ public AsyncResult answer(InvocationOnMock arg0) throws Throwable { }); } + @Test @Description("test getMlayerPopularDatasets method when DB Request fails") public void testGetMlayerPopularDatasetsFailed(VertxTestContext testContext) { + String instance =""; DatabaseServiceImpl databaseService = new DatabaseServiceImpl( client, @@ -3071,6 +3075,7 @@ public AsyncResult answer(InvocationOnMock arg0) throws Throwable { .searchAsync(any(), any(), any()); databaseService.getMlayerPopularDatasets( + instance, highestCountResource, handler -> { if (handler.failed()) { @@ -3091,6 +3096,7 @@ public AsyncResult answer(InvocationOnMock arg0) throws Throwable { @Description( "test getMlayerPopularDatasets method when DB Request is successful and type equals iudx:Provider") public void testGetMlayerPopularDatasetsProviderSuccess(VertxTestContext testContext) { + String instanceName =""; DatabaseServiceImpl databaseService = new DatabaseServiceImpl( client, @@ -3131,6 +3137,7 @@ public AsyncResult answer(InvocationOnMock arg0) throws Throwable { .when(DatabaseServiceImpl.client) .searchAsync(any(), any(), any()); databaseService.getMlayerPopularDatasets( + instanceName, highestCountResource, handler -> { if (handler.succeeded()) { @@ -3156,6 +3163,7 @@ public void testGetMlayerPopularDatasetsResourceSuccess(VertxTestContext testCon mlayerDomainIndex, nlpService, geoService); + String instanceName ="dummy"; JsonObject json = new JsonObject().put("rgid", "duumy-id"); JsonObject json2 = new JsonObject().put("rgid", "duumy-id"); @@ -3187,6 +3195,7 @@ public AsyncResult answer(InvocationOnMock arg0) throws Throwable { .when(DatabaseServiceImpl.client) .searchAsync(any(), any(), any()); databaseService.getMlayerPopularDatasets( + instanceName, highestCountResource, handler -> { if (handler.succeeded()) { diff --git a/src/test/java/iudx/catalogue/server/database/postgres/PostgresServiceTest.java b/src/test/java/iudx/catalogue/server/database/postgres/PostgresServiceTest.java index d1fa0e6e..43b52e50 100644 --- a/src/test/java/iudx/catalogue/server/database/postgres/PostgresServiceTest.java +++ b/src/test/java/iudx/catalogue/server/database/postgres/PostgresServiceTest.java @@ -69,11 +69,12 @@ public void testExecuteQuerySuccess(VertxTestContext testContext) { StringBuilder stringBuilder = new StringBuilder(GET_HIGH_COUNT_DATASET.replace("$1", table)); String expected = - "{\"type\":\"urn:dx:cat:Success\",\"title\":\"Success\",\"results\":[{\"rgid\":\"iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/rs.iudx.io/surat-itms-realtime-information\",\"totalhits\":1}]}"; + "{\"type\":\"urn:dx:cat:Success\",\"title\":\"Success\",\"results\":[{\"resourcegroup\":\"dummy-rg-id\",\"totalhits\":1}]}"; pgService.executeQuery( stringBuilder.toString(), handler -> { if (handler.succeeded()) { + LOGGER.debug(handler.result().toString()); assertEquals(expected, handler.result().toString()); assertTrue(handler.result().containsKey("type")); assertTrue(handler.result().containsKey("title")); diff --git a/src/test/java/iudx/catalogue/server/mlayer/MlayerServiceTest.java b/src/test/java/iudx/catalogue/server/mlayer/MlayerServiceTest.java index 690214e2..2dfc0d85 100644 --- a/src/test/java/iudx/catalogue/server/mlayer/MlayerServiceTest.java +++ b/src/test/java/iudx/catalogue/server/mlayer/MlayerServiceTest.java @@ -821,6 +821,7 @@ void successfulGetMlayerOverviewTest(VertxTestContext testContext) { JsonObject json = new JsonObject(); json.put("results", jsonArray); jsonArray.add("dataset"); + String instanceName ="dummy"; when(asyncResult.result()).thenReturn(json); when(asyncResult.succeeded()).thenReturn(true); @@ -829,12 +830,12 @@ void successfulGetMlayerOverviewTest(VertxTestContext testContext) { @SuppressWarnings("unchecked") @Override public AsyncResult answer(InvocationOnMock arg0) throws Throwable { - ((Handler>) arg0.getArgument(1)).handle(asyncResult); + ((Handler>) arg0.getArgument(2)).handle(asyncResult); return null; } }) .when(databaseService) - .getMlayerPopularDatasets(any(), any()); + .getMlayerPopularDatasets(any(), any(),any()); doAnswer( new Answer>() { @SuppressWarnings("unchecked") @@ -847,10 +848,10 @@ public AsyncResult answer(InvocationOnMock arg0) throws Throwable { .when(postgresService) .executeQuery(any(), any()); - mlayerService.getMlayerPopularDatasets( + mlayerService.getMlayerPopularDatasets(instanceName, handler -> { if (handler.succeeded()) { - verify(databaseService, times(1)).getMlayerPopularDatasets(any(), any()); + verify(databaseService, times(1)).getMlayerPopularDatasets(any(),any(), any()); verify(postgresService, times(1)).executeQuery(any(), any()); testContext.completeNow(); @@ -870,6 +871,7 @@ void failedGetMlayerOverviewTest(VertxTestContext testContext) { json.put("results", jsonArray); jsonArray.add("dataset"); // when(asyncResult.result()).thenReturn(json); + String instance =""; doAnswer( new Answer>() { @@ -883,7 +885,7 @@ public AsyncResult answer(InvocationOnMock arg0) throws Throwable { .when(postgresService) .executeQuery(any(), any()); - mlayerService.getMlayerPopularDatasets( + mlayerService.getMlayerPopularDatasets(instance, handler -> { if (handler.failed()) { verify(postgresService, times(1)).executeQuery(any(), any()); diff --git a/src/test/resources/iudx-catalogue-server-v5.0.0.postman_collection.json b/src/test/resources/iudx-catalogue-server-v5.0.0.postman_collection.json index c6fc5138..cad66a98 100644 --- a/src/test/resources/iudx-catalogue-server-v5.0.0.postman_collection.json +++ b/src/test/resources/iudx-catalogue-server-v5.0.0.postman_collection.json @@ -1,6 +1,6 @@ { "info": { - "_postman_id": "f6073aac-49ba-431f-8d3a-69c5ca08e137", + "_postman_id": "17ffa0ab-2bb6-4aa7-a68c-393f68f437c4", "name": "iudx-catalogue-server v5.0.0", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", "_exporter_id": "17194681" @@ -13093,6 +13093,56 @@ { "name": "Mlayer Popular Datasets", "item": [ + { + "name": "200 (Success) - Get landing page details through instance", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Response status\", function () {\r", + " pm.response.to.have.status(200);\r", + "});\r", + "\r", + "pm.test(\"Check response body\", function () { \r", + " const body = pm.response.json();\r", + " pm.expect(body).to.have.property(\"type\", \"urn:dx:cat:Success\");\r", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "", + "value": "", + "type": "text", + "disabled": true + } + ], + "url": { + "raw": "{{host}}{{base}}/internal/ui/popularDatasets?instance=surat", + "host": [ + "{{host}}{{base}}" + ], + "path": [ + "internal", + "ui", + "popularDatasets" + ], + "query": [ + { + "key": "instance", + "value": "surat" + } + ] + } + }, + "response": [] + }, { "name": "200 (Success) - Get landing page details", "event": [ diff --git a/src/test/resources/pg_test_schema.sql b/src/test/resources/pg_test_schema.sql index 0da191df..3eb657a5 100644 --- a/src/test/resources/pg_test_schema.sql +++ b/src/test/resources/pg_test_schema.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS auditing_rs +CREATE TABLE IF NOT EXISTS auditing_rs_withtype_temp ( id varchar NOT NULL, api varchar NOT NULL, @@ -8,7 +8,8 @@ CREATE TABLE IF NOT EXISTS auditing_rs isotime varchar NOT NULL, time timestamp without time zone, providerid varchar NOT NULL, + resourcegroup varchar, size numeric NOT NULL ); -INSERT INTO auditing_rs (id, api, userid, epochtime, resourceid, isotime, providerid, size, time) VALUES ('a462ae274cc242688486adfdb2c853cf', '/ngsi-ld/v1/entities', '15c7506f-c800-48d6-adeb-0542b03947c6', 1671443074549, 'iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/rs.iudx.io/surat-itms-realtime-information/surat-itms-live-eta', '2022-12-19T15:14:34+05:30[Asia/Kolkata]', 'iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86', 79, '2022-12-19 09:44:34'); \ No newline at end of file +INSERT INTO auditing_rs_withtype_temp (id, api, userid, epochtime, resourceid, isotime, providerid, resourcegroup, size, time) VALUES ('a462ae274cc242688486adfdb2c853cf', '/ngsi-ld/v1/entities', '15c7506f-c800-48d6-adeb-0542b03947c6', 1671443074549, 'iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/rs.iudx.io/surat-itms-realtime-information/surat-itms-live-eta', '2022-12-19T15:14:34+05:30[Asia/Kolkata]', 'iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86', 'dummy-rg-id', 79, '2022-12-19 09:44:34'); \ No newline at end of file