Skip to content

Commit

Permalink
Merge pull request #251 from tanvi029/popular-dataset
Browse files Browse the repository at this point in the history 
popular dataset fixes
  • Loading branch information
@pranavrd
pranavrd authored Jul 27, 2023
2 parents fec2800 + 6946474 commit 42e77f0
Show file tree
Hide file tree
Showing 12 changed files with 129 additions and 32 deletions.
7 changes: 6 additions & 1 deletion src/main/java/iudx/catalogue/server/apiserver/MlayerApis.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -574,9 +574,14 @@ public void getMlayerDatasetHandler(RoutingContext routingContext) {
*/
public void getMlayerPopularDatasetsHandler(RoutingContext routingContext) {
LOGGER.debug("Info : fetching the data for the landing Page");
String instance = "";
if (routingContext.request().getParam(INSTANCE) != null) {
instance = routingContext.request().getParam(INSTANCE);
}
LOGGER.debug("Instance {}", instance);
HttpServerResponse response = routingContext.response();
response.putHeader(HEADER_CONTENT_TYPE, MIME_APPLICATION_JSON);
mlayerService.getMlayerPopularDatasets(
mlayerService.getMlayerPopularDatasets(instance,
handler -> {
if (handler.succeeded()) {
response.setStatusCode(200).end(handler.result().toString());
Expand Down
10 changes: 10 additions & 0 deletions src/main/java/iudx/catalogue/server/database/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,16 @@ public class Constants {
+ " \"iudx:Provider\"}}]}}]}},\"_source\":{\"includes\": [\"id\",\"description\","
+ "\"type\",\"resourceGroup\",\"accessPolicy\",\"provider\",\"itemCreatedAt\","
+ "\"instance\",\"label\"]},\"size\":10000}";

public static final String GET_DATASET_BY_INSTANCE =
"{\"query\":{\"bool\":{\"should\":[{\"bool\":{\"must\":[{\"match\":{\"type.keyword\":"
+ "\"iudx:ResourceGroup\"}},{\"match\": {\"instance.keyword\": \"$1\"}}]}},"
+ "{\"bool\":"
+ "{\"must\":[{\"match\":{\"type.keyword\":\"iudx:Resource\"}},{\"match\":"
+ " {\"instance.keyword\": \"$1\"}}]}},{\"bool\":{\"must\":[{\"match\":"
+ "{\"type.keyword\": \"iudx:Provider\"}}]}}]}},\"_source\":{\"includes\": "
+ "[\"id\",\"description\",\"type\",\"resourceGroup\",\"accessPolicy\","
+ "\"provider\",\"itemCreatedAt\",\"instance\",\"label\"]},\"size\":10000}\n";
public static final String GET_SORTED_MLAYER_INSTANCES =
"{\"query\": {\"match_all\":{}},\"sort\":[{\"name\":\"asc\"}],\"_source\": "
+ "{\"includes\": [\"name\",\"cover\",\"icon\"]},\"size\":10000}";
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/iudx/catalogue/server/database/DatabaseService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ DatabaseService updateMlayerInstance(
DatabaseService getMlayerDataset(String datasetId, Handler<AsyncResult<JsonObject>> handler);

@Fluent
DatabaseService getMlayerPopularDatasets(JsonArray highestCountResource,
DatabaseService getMlayerPopularDatasets(String instance, JsonArray highestCountResource,
Handler<AsyncResult<JsonObject>> handler);

/* create db service with nlp and geocoding */
Expand Down
41 changes: 30 additions & 11 deletions src/main/java/iudx/catalogue/server/database/DatabaseServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1925,7 +1925,7 @@ public DatabaseService getMlayerDataset(
}

@Override
public DatabaseService getMlayerPopularDatasets(
public DatabaseService getMlayerPopularDatasets(String instance,
JsonArray highestCountResource, Handler<AsyncResult<JsonObject>> handler) {
Promise<JsonObject> instanceResult = Promise.promise();

Expand All @@ -1935,7 +1935,7 @@ public DatabaseService getMlayerPopularDatasets(
searchSortedMlayerInstances(instanceResult);

allMlayerDomains(domainResult);
datasets(datasetResult, highestCountResource);
datasets(instance, datasetResult, highestCountResource);
CompositeFuture.all(instanceResult.future(), domainResult.future(), datasetResult.future())
.onComplete(
ar -> {
Expand Down Expand Up @@ -2081,9 +2081,16 @@ public int compare(JsonObject record1, JsonObject record2) {
return jsonComparator;
}

private void datasets(Promise<JsonObject> datasetResult, JsonArray highestCountResource) {
private void datasets(String instance, Promise<JsonObject> datasetResult,
JsonArray highestCountResource) {
String providerAndResources = "";
if (instance.isBlank()) {
providerAndResources = GET_PROVIDER_AND_RESOURCES;
} else {
providerAndResources = GET_DATASET_BY_INSTANCE.replace("$1", instance);
}
client.searchAsync(
GET_PROVIDER_AND_RESOURCES,
providerAndResources,
docIndex,
getCatRecords -> {
if (getCatRecords.succeeded()) {
Expand Down Expand Up @@ -2125,6 +2132,22 @@ private void datasets(Promise<JsonObject> datasetResult, JsonArray highestCountR
// sorting resource group based on the time of creation.
Collections.sort(resourceGroupArray, comapratorForLatestDataset());

// getting count of providers of a particular instance
ArrayList<String> providerList = new ArrayList<String>();
if (!instance.isBlank()) {
for (int i = 0; i < getCatRecords.result().getJsonArray(RESULTS).size(); i++) {
JsonObject record = getCatRecords.result().getJsonArray(RESULTS).getJsonObject(i);
if (record.getJsonArray(TYPE).getString(0).equals("iudx:ResourceGroup")
&& !providerList.contains(record.getString("provider"))
&& !(record.getString("provider")).equals(null)) {
providerList.add(record.getString("provider"));

}

}
typeCount.put("iudx:Provider", providerList.size());
}

ArrayList<JsonObject> latestResourceGroup = new ArrayList<>();
int resourceGroupSize = 0;
if (resourceGroupArray.size() < 6) {
Expand All @@ -2151,17 +2174,13 @@ private void datasets(Promise<JsonObject> datasetResult, JsonArray highestCountR
if (resourceGroupArray
.get(i)
.getString("id")
.equals(highestCountResource.getJsonObject(j).getString("rgid"))) {
String datasetId = highestCountResource.getJsonObject(j).getString("rgid");
int index = datasetId.indexOf("/", datasetId.indexOf("/") + 1);
String providerId = datasetId.substring(0, index);
.equals(highestCountResource.getJsonObject(j)
.getString("resourcegroup"))) {
JsonObject resource = resourceGroupArray.get(i);
resource
.put(
"totalResources",
resourceGroupCount.get(resourceGroupArray.get(i).getString("id")))
.put("providerDescription", providerDescription.get(providerId));

resourceGroupCount.get(resourceGroupArray.get(i).getString("id")));
featuredResourceGroup.add(resource);
resource = new JsonObject();
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/iudx/catalogue/server/mlayer/MlayerService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,6 @@ static MlayerService createProxy(Vertx vertx, String address) {
MlayerService getMlayerDataset(String datasetId, Handler<AsyncResult<JsonObject>> handler);

@Fluent
MlayerService getMlayerPopularDatasets(Handler<AsyncResult<JsonObject>> handler);
MlayerService getMlayerPopularDatasets(String instance, Handler<AsyncResult<JsonObject>> handler);
}

6 changes: 4 additions & 2 deletions src/main/java/iudx/catalogue/server/mlayer/MlayerServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,16 +247,18 @@ public MlayerService getMlayerDataset(
}

@Override
public MlayerService getMlayerPopularDatasets(Handler<AsyncResult<JsonObject>> handler) {

public MlayerService getMlayerPopularDatasets(String instance,
Handler<AsyncResult<JsonObject>> handler) {
String query = GET_HIGH_COUNT_DATASET.replace("$1", databaseTable);
LOGGER.debug("postgres query" + query);
postgresService.executeQuery(
query,
dbHandler -> {
if (dbHandler.succeeded()) {
JsonArray popularDataset = dbHandler.result().getJsonArray("results");
LOGGER.debug("popular datasets are {}", popularDataset);
databaseService.getMlayerPopularDatasets(
instance,
popularDataset,
getPopularDatasetsHandler -> {
if (getPopularDatasetsHandler.succeeded()) {
Expand Down
8 changes: 3 additions & 5 deletions src/main/java/iudx/catalogue/server/mlayer/util/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@ public class Constants {
public static final String INSTANCE_ID = "instanceId";
public static final String DOMAIN_ID = "domainId";
public static final String GET_HIGH_COUNT_DATASET =
"with auditing_rs_view as (select resourceid, count(*) as hits, "
+ "(select count(*) from regexp_matches(resourceid, '/', 'g')) as "
+ "idtype from $1 group by resourceid) select left(resourceid,length(resourceid) "
+ "-strpos(reverse(resourceid),'/')) as rgid, sum(hits) as totalhits from "
+ "auditing_rs_view where idtype=4 group by rgid order by totalhits desc limit 6";
"select resourcegroup, count(id) as totalhits from $1 "
+ "group by resourcegroup order by totalhits "
+ "desc limit 6";
}
13 changes: 11 additions & 2 deletions src/test/java/iudx/catalogue/server/database/DatabaseServiceImplTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2981,10 +2981,11 @@ public void testGetMlayerPopularDatasetsSuccess(VertxTestContext testContext) {
mlayerDomainIndex,
nlpService,
geoService);
String instanceName ="";
JsonObject json =
new JsonObject().put("rgid", "abcd/abcd/abcd/abcd").put("instance", "instance");
new JsonObject().put("resourcegroup", "abcd/abcd/abcd/abcd").put("instance", "instance");
JsonObject json2 =
new JsonObject().put("rgid", "abcd/abcd/abcd/abcd").put("instance", "instance");
new JsonObject().put("resourcegroup", "abcd/abcd/abcd/abcd").put("instance", "instance");

JsonArray highestCountResource = new JsonArray().add(json).add(json2);
DatabaseServiceImpl.client = mock(ElasticClient.class);
Expand Down Expand Up @@ -3030,6 +3031,7 @@ public AsyncResult<JsonObject> answer(InvocationOnMock arg0) throws Throwable {
.when(DatabaseServiceImpl.client)
.searchAsync(any(), any(), any());
databaseService.getMlayerPopularDatasets(
instanceName,
highestCountResource,
handler -> {
if (handler.succeeded()) {
Expand All @@ -3042,9 +3044,11 @@ public AsyncResult<JsonObject> answer(InvocationOnMock arg0) throws Throwable {
});
}


@Test
@Description("test getMlayerPopularDatasets method when DB Request fails")
public void testGetMlayerPopularDatasetsFailed(VertxTestContext testContext) {
String instance ="";
DatabaseServiceImpl databaseService =
new DatabaseServiceImpl(
client,
Expand All @@ -3071,6 +3075,7 @@ public AsyncResult<JsonObject> answer(InvocationOnMock arg0) throws Throwable {
.searchAsync(any(), any(), any());

databaseService.getMlayerPopularDatasets(
instance,
highestCountResource,
handler -> {
if (handler.failed()) {
Expand All @@ -3091,6 +3096,7 @@ public AsyncResult<JsonObject> answer(InvocationOnMock arg0) throws Throwable {
@Description(
"test getMlayerPopularDatasets method when DB Request is successful and type equals iudx:Provider")
public void testGetMlayerPopularDatasetsProviderSuccess(VertxTestContext testContext) {
String instanceName ="";
DatabaseServiceImpl databaseService =
new DatabaseServiceImpl(
client,
Expand Down Expand Up @@ -3131,6 +3137,7 @@ public AsyncResult<JsonObject> answer(InvocationOnMock arg0) throws Throwable {
.when(DatabaseServiceImpl.client)
.searchAsync(any(), any(), any());
databaseService.getMlayerPopularDatasets(
instanceName,
highestCountResource,
handler -> {
if (handler.succeeded()) {
Expand All @@ -3156,6 +3163,7 @@ public void testGetMlayerPopularDatasetsResourceSuccess(VertxTestContext testCon
mlayerDomainIndex,
nlpService,
geoService);
String instanceName ="dummy";
JsonObject json = new JsonObject().put("rgid", "duumy-id");
JsonObject json2 = new JsonObject().put("rgid", "duumy-id");

Expand Down Expand Up @@ -3187,6 +3195,7 @@ public AsyncResult<JsonObject> answer(InvocationOnMock arg0) throws Throwable {
.when(DatabaseServiceImpl.client)
.searchAsync(any(), any(), any());
databaseService.getMlayerPopularDatasets(
instanceName,
highestCountResource,
handler -> {
if (handler.succeeded()) {
Expand Down
3 changes: 2 additions & 1 deletion src/test/java/iudx/catalogue/server/database/postgres/PostgresServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,12 @@ public void testExecuteQuerySuccess(VertxTestContext testContext) {
StringBuilder stringBuilder = new StringBuilder(GET_HIGH_COUNT_DATASET.replace("$1", table));

String expected =
"{\"type\":\"urn:dx:cat:Success\",\"title\":\"Success\",\"results\":[{\"rgid\":\"iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/rs.iudx.io/surat-itms-realtime-information\",\"totalhits\":1}]}";
"{\"type\":\"urn:dx:cat:Success\",\"title\":\"Success\",\"results\":[{\"resourcegroup\":\"dummy-rg-id\",\"totalhits\":1}]}";
pgService.executeQuery(
stringBuilder.toString(),
handler -> {
if (handler.succeeded()) {
LOGGER.debug(handler.result().toString());
assertEquals(expected, handler.result().toString());
assertTrue(handler.result().containsKey("type"));
assertTrue(handler.result().containsKey("title"));
Expand Down
12 changes: 7 additions & 5 deletions src/test/java/iudx/catalogue/server/mlayer/MlayerServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -821,6 +821,7 @@ void successfulGetMlayerOverviewTest(VertxTestContext testContext) {
JsonObject json = new JsonObject();
json.put("results", jsonArray);
jsonArray.add("dataset");
String instanceName ="dummy";
when(asyncResult.result()).thenReturn(json);

when(asyncResult.succeeded()).thenReturn(true);
Expand All @@ -829,12 +830,12 @@ void successfulGetMlayerOverviewTest(VertxTestContext testContext) {
@SuppressWarnings("unchecked")
@Override
public AsyncResult<JsonObject> answer(InvocationOnMock arg0) throws Throwable {
((Handler<AsyncResult<JsonObject>>) arg0.getArgument(1)).handle(asyncResult);
((Handler<AsyncResult<JsonObject>>) arg0.getArgument(2)).handle(asyncResult);
return null;
}
})
.when(databaseService)
.getMlayerPopularDatasets(any(), any());
.getMlayerPopularDatasets(any(), any(),any());
doAnswer(
new Answer<AsyncResult<JsonObject>>() {
@SuppressWarnings("unchecked")
Expand All @@ -847,10 +848,10 @@ public AsyncResult<JsonObject> answer(InvocationOnMock arg0) throws Throwable {
.when(postgresService)
.executeQuery(any(), any());

mlayerService.getMlayerPopularDatasets(
mlayerService.getMlayerPopularDatasets(instanceName,
handler -> {
if (handler.succeeded()) {
verify(databaseService, times(1)).getMlayerPopularDatasets(any(), any());
verify(databaseService, times(1)).getMlayerPopularDatasets(any(),any(), any());
verify(postgresService, times(1)).executeQuery(any(), any());

testContext.completeNow();
Expand All @@ -870,6 +871,7 @@ void failedGetMlayerOverviewTest(VertxTestContext testContext) {
json.put("results", jsonArray);
jsonArray.add("dataset");
// when(asyncResult.result()).thenReturn(json);
String instance ="";

doAnswer(
new Answer<AsyncResult<JsonObject>>() {
Expand All @@ -883,7 +885,7 @@ public AsyncResult<JsonObject> answer(InvocationOnMock arg0) throws Throwable {
.when(postgresService)
.executeQuery(any(), any());

mlayerService.getMlayerPopularDatasets(
mlayerService.getMlayerPopularDatasets(instance,
handler -> {
if (handler.failed()) {
verify(postgresService, times(1)).executeQuery(any(), any());
Expand Down
52 changes: 51 additions & 1 deletion src/test/resources/iudx-catalogue-server-v5.0.0.postman_collection.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"info": {
"_postman_id": "f6073aac-49ba-431f-8d3a-69c5ca08e137",
"_postman_id": "17ffa0ab-2bb6-4aa7-a68c-393f68f437c4",
"name": "iudx-catalogue-server v5.0.0",
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
"_exporter_id": "17194681"
Expand Down Expand Up @@ -13093,6 +13093,56 @@
{
"name": "Mlayer Popular Datasets",
"item": [
{
"name": "200 (Success) - Get landing page details through instance",
"event": [
{
"listen": "test",
"script": {
"exec": [
"pm.test(\"Response status\", function () {\r",
" pm.response.to.have.status(200);\r",
"});\r",
"\r",
"pm.test(\"Check response body\", function () { \r",
" const body = pm.response.json();\r",
" pm.expect(body).to.have.property(\"type\", \"urn:dx:cat:Success\");\r",
"});"
],
"type": "text/javascript"
}
}
],
"request": {
"method": "GET",
"header": [
{
"key": "",
"value": "",
"type": "text",
"disabled": true
}
],
"url": {
"raw": "{{host}}{{base}}/internal/ui/popularDatasets?instance=surat",
"host": [
"{{host}}{{base}}"
],
"path": [
"internal",
"ui",
"popularDatasets"
],
"query": [
{
"key": "instance",
"value": "surat"
}
]
}
},
"response": []
},
{
"name": "200 (Success) - Get landing page details",
"event": [
Expand Down
5 changes: 3 additions & 2 deletions src/test/resources/pg_test_schema.sql
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
CREATE TABLE IF NOT EXISTS auditing_rs
CREATE TABLE IF NOT EXISTS auditing_rs_withtype_temp
(
id varchar NOT NULL,
api varchar NOT NULL,
Expand All @@ -8,7 +8,8 @@ CREATE TABLE IF NOT EXISTS auditing_rs
isotime varchar NOT NULL,
time timestamp without time zone,
providerid varchar NOT NULL,
resourcegroup varchar,
size numeric NOT NULL
);

INSERT INTO auditing_rs (id, api, userid, epochtime, resourceid, isotime, providerid, size, time) VALUES ('a462ae274cc242688486adfdb2c853cf', '/ngsi-ld/v1/entities', '15c7506f-c800-48d6-adeb-0542b03947c6', 1671443074549, 'iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/rs.iudx.io/surat-itms-realtime-information/surat-itms-live-eta', '2022-12-19T15:14:34+05:30[Asia/Kolkata]', 'iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86', 79, '2022-12-19 09:44:34');
INSERT INTO auditing_rs_withtype_temp (id, api, userid, epochtime, resourceid, isotime, providerid, resourcegroup, size, time) VALUES ('a462ae274cc242688486adfdb2c853cf', '/ngsi-ld/v1/entities', '15c7506f-c800-48d6-adeb-0542b03947c6', 1671443074549, 'iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86/rs.iudx.io/surat-itms-realtime-information/surat-itms-live-eta', '2022-12-19T15:14:34+05:30[Asia/Kolkata]', 'iisc.ac.in/89a36273d77dac4cf38114fca1bbe64392547f86', 'dummy-rg-id', 79, '2022-12-19 09:44:34');

0 comments on commit 42e77f0

Please sign in to comment.