Add: added elastic search aggregation to query facet objects (#859)

* fix: improved elastic search query filter * fix: updated mathJS package version && improved number check * fix: fixed inconsistency between data list and count number * fix: minor naming change * fix: improved mapping for digit with floating points * fix: revert back to mongo query if es db doesnt return any value * fix: fixed pagination conflict between ES and mongo query * fix: remove empty space in string value of condition filter with trim() * fix: minor change for object name and removed redundant type * fix: improved elastic search query filter (#837) * fix: improved elastic search query filter * fix: updated mathJS package version && improved number check * fix: fixed inconsistency between data list and count number * fix: minor naming change * fix: improved mapping for digit with floating points * fix: revert back to mongo query if es db doesnt return any value * build(deps): bump actions/setup-node from 3 to 4 Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3 to 4. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](actions/setup-node@v3...v4) --- updated-dependencies: - dependency-name: actions/setup-node dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <[email protected]> * fix: fixed pagination conflict between ES and mongo query * fix: remove empty space in string value of condition filter with trim() * fix: minor change for object name and removed redundant type --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * build(deps-dev): bump @types/express-session from 1.17.8 to 1.17.9 Bumps [@types/express-session](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/express-session) from 1.17.8 to 1.17.9. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/express-session) --- updated-dependencies: - dependency-name: "@types/express-session" dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <[email protected]> * build(deps-dev): bump @nestjs/cli from 10.2.0 to 10.2.1 Bumps [@nestjs/cli](https://github.com/nestjs/nest-cli) from 10.2.0 to 10.2.1. - [Release notes](https://github.com/nestjs/nest-cli/releases) - [Changelog](https://github.com/nestjs/nest-cli/blob/master/.release-it.json) - [Commits](nestjs/nest-cli@10.2.0...10.2.1) --- updated-dependencies: - dependency-name: "@nestjs/cli" dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <[email protected]> * build(deps): bump @elastic/elasticsearch from 8.9.0 to 8.10.0 Bumps [@elastic/elasticsearch](https://github.com/elastic/elasticsearch-js) from 8.9.0 to 8.10.0. - [Release notes](https://github.com/elastic/elasticsearch-js/releases) - [Changelog](https://github.com/elastic/elasticsearch-js/blob/main/docs/changelog.asciidoc) - [Commits](elastic/elasticsearch-js@v8.9.0...v8.10.0) --- updated-dependencies: - dependency-name: "@elastic/elasticsearch" dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <[email protected]> * build(deps-dev): bump concurrently from 8.2.1 to 8.2.2 Bumps [concurrently](https://github.com/open-cli-tools/concurrently) from 8.2.1 to 8.2.2. - [Release notes](https://github.com/open-cli-tools/concurrently/releases) - [Commits](open-cli-tools/concurrently@v8.2.1...v8.2.2) --- updated-dependencies: - dependency-name: concurrently dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <[email protected]> * build(deps-dev): bump @types/node-fetch from 2.6.6 to 2.6.7 Bumps [@types/node-fetch](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node-fetch) from 2.6.6 to 2.6.7. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/node-fetch) --- updated-dependencies: - dependency-name: "@types/node-fetch" dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <[email protected]> * build(deps): bump @nestjs/axios from 3.0.0 to 3.0.1 Bumps [@nestjs/axios](https://github.com/nestjs/axios) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/nestjs/axios/releases) - [Changelog](https://github.com/nestjs/axios/blob/master/.release-it.json) - [Commits](nestjs/axios@3.0.0...3.0.1) --- updated-dependencies: - dependency-name: "@nestjs/axios" dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <[email protected]> * build(deps-dev): bump @types/luxon from 3.3.2 to 3.3.3 Bumps [@types/luxon](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/luxon) from 3.3.2 to 3.3.3. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/luxon) --- updated-dependencies: - dependency-name: "@types/luxon" dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <[email protected]> * build(deps-dev): bump @types/mocha from 10.0.2 to 10.0.3 Bumps [@types/mocha](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/mocha) from 10.0.2 to 10.0.3. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/mocha) --- updated-dependencies: - dependency-name: "@types/mocha" dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <[email protected]> * build(deps-dev): bump @types/passport-local from 1.0.36 to 1.0.37 Bumps [@types/passport-local](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/passport-local) from 1.0.36 to 1.0.37. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/passport-local) --- updated-dependencies: - dependency-name: "@types/passport-local" dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <[email protected]> * build(deps): bump mathjs from 11.11.2 to 12.0.0 Bumps [mathjs](https://github.com/josdejong/mathjs) from 11.11.2 to 12.0.0. - [Changelog](https://github.com/josdejong/mathjs/blob/develop/HISTORY.md) - [Commits](josdejong/mathjs@v11.11.2...v12.0.0) --- updated-dependencies: - dependency-name: mathjs dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <[email protected]> * build(deps-dev): bump sinon from 16.1.0 to 17.0.0 Bumps [sinon](https://github.com/sinonjs/sinon) from 16.1.0 to 17.0.0. - [Release notes](https://github.com/sinonjs/sinon/releases) - [Changelog](https://github.com/sinonjs/sinon/blob/main/docs/changelog.md) - [Commits](sinonjs/sinon@v16.1.0...v17.0.0) --- updated-dependencies: - dependency-name: sinon dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <[email protected]> * fix: improved elastic search query filter (#837) * fix: improved elastic search query filter * fix: updated mathJS package version && improved number check * fix: fixed inconsistency between data list and count number * fix: minor naming change * fix: improved mapping for digit with floating points * fix: revert back to mongo query if es db doesnt return any value * build(deps): bump actions/setup-node from 3 to 4 Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3 to 4. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](actions/setup-node@v3...v4) --- updated-dependencies: - dependency-name: actions/setup-node dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <[email protected]> * fix: fixed pagination conflict between ES and mongo query * fix: remove empty space in string value of condition filter with trim() * fix: minor change for object name and removed redundant type --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix: included ES aggregation for fullfacet * fix: removed esEnabled flag from createFullFacetPipeline function * fix: minor clean up * fix: minor cleanup --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
SciCatProject · Nov 7, 2023 · c76c6f7 · c76c6f7
1 parent ec60892
commit c76c6f7
Show file tree

Hide file tree

Showing 7 changed files with 130 additions and 31 deletions.
diff --git a/src/common/utils.ts b/src/common/utils.ts
@@ -612,7 +612,6 @@ export const createFullfacetPipeline = <T, Y extends object>(
   fields: Y,
   facets: string[],
   subField = "",
-  esEnabled = false,
 ): PipelineStage[] => {
   const pipeline: PipelineStage[] = [];
   const facetMatch: Record<string, unknown> = {};
@@ -622,13 +621,6 @@ export const createFullfacetPipeline = <T, Y extends object>(
       facetMatch[key] = searchExpression<T>(model, key, fields[key as keyof Y]);
     }
 
-    if (esEnabled) {
-      if (key === "mode") {
-        pipelineHandler.handleModeSearch(pipeline, fields, key, idField);
-      }
-      return;
-    }
-
     switch (key) {
       case "text":
         pipelineHandler.handleTextSearch(pipeline, model, fields, key);

diff --git a/src/datasets/datasets.service.ts b/src/datasets/datasets.service.ts
@@ -157,27 +157,10 @@ export class DatasetsService {
 
       data = await this.datasetModel.aggregate(pipeline).exec();
     } else {
-      const { count: initialCount } = await this.ESClient.getCount();
-      const { totalCount, data: esPids } = await this.ESClient.search(
-        fields as IDatasetFields,
-        initialCount,
-      );
+      const facetResult = await this.ESClient.aggregate(fields);
 
-      fields.mode = { _id: { $in: esPids } };
-      const pipeline = createFullfacetPipeline<DatasetDocument, IDatasetFields>(
-        this.datasetModel,
-        "pid",
-        fields,
-        facets,
-        "",
-        !!this.ESClient,
-      );
-      data = await this.datasetModel.aggregate(pipeline).exec();
-
-      // NOTE: below code is to overwrite totalCount with ES result
-      data[0].all = [{ totalSets: totalCount }];
+      data = facetResult;
     }
-
     return data;
   }
 

diff --git a/src/elastic-search/elastic-search.service.ts b/src/elastic-search/elastic-search.service.ts
@@ -11,6 +11,7 @@ import { SearchQueryService } from "./providers/query-builder.service";
 import {
   SearchTotalHits,
   SearchRequest,
+  AggregationsAggregate,
 } from "@elastic/elasticsearch/lib/api/types";
 import { IDatasetFields } from "src/datasets/interfaces/dataset-filters.interface";
 import {
@@ -24,7 +25,11 @@ import {
 } from "src/datasets/schemas/dataset.schema";
 import { ConfigService } from "@nestjs/config";
 import { sleep } from "src/common/utils";
-import { transformKeysInObject, initialSyncTransform } from "./helpers/utils";
+import {
+  transformKeysInObject,
+  initialSyncTransform,
+  transformFacets,
+} from "./helpers/utils";
 
 @Injectable()
 export class ElasticSearchService implements OnModuleInit {
@@ -287,6 +292,37 @@ export class ElasticSearchService implements OnModuleInit {
     }
   }
 
+  async aggregate(searchParam: IDatasetFields) {
+    try {
+      const searchQuery = this.searchService.buildSearchQuery(searchParam);
+      const facetPipeline = this.searchService.buildFullFacetPipeline();
+
+      const searchOptions = {
+        query: searchQuery.query,
+        size: 0,
+        aggs: facetPipeline,
+        _source: [""],
+      } as SearchRequest;
+
+      const body = await this.esService.search(searchOptions);
+
+      const transformedFacets = transformFacets(
+        body.aggregations as AggregationsAggregate,
+      );
+
+      return transformedFacets;
+    } catch (error) {
+      Logger.error(
+        "SearchService || aggregate query issue || -> aggregate",
+        error,
+      );
+
+      throw new HttpException(
+        `SearchService || aggregate query issue || -> aggregate ${error}`,
+        HttpStatus.BAD_REQUEST,
+      );
+    }
+  }
   async updateInsertDocument(data: DatasetDocument) {
     //NOTE: Replace all keys with lower case, also replace spaces and dot with underscore
     delete data._id;
@@ -349,7 +385,10 @@ export class ElasticSearchService implements OnModuleInit {
       },
       onDrop(doc) {
         console.error(doc.document._id, doc.error?.reason);
-        Logger.error("data insert faield: ", doc.document._id);
+        Logger.error(
+          "SearchService-> performBulkOperation-> data insert faield: ",
+          doc.document._id,
+        );
       },
     });
   }

diff --git a/src/elastic-search/helpers/utils.ts b/src/elastic-search/helpers/utils.ts
@@ -1,5 +1,9 @@
+import {
+  AggregationsAggregate,
+  AggregationsFrequentItemSetsBucketKeys,
+} from "@elastic/elasticsearch/lib/api/types";
 import { DatasetClass } from "src/datasets/schemas/dataset.schema";
-import { IFilter } from "../interfaces/es-common.type";
+import { IFilter, ITransformedFullFacets } from "../interfaces/es-common.type";
 
 export const transformKey = (key: string): string => {
   return key.trim().replace(/[.]/g, "\\.").replace(/ /g, "_").toLowerCase();
@@ -109,3 +113,27 @@ export const convertToElasticSearchQuery = (
 
   return filters;
 };
+
+export const transformFacets = (
+  aggregation: AggregationsAggregate,
+): Record<string, unknown>[] => {
+  const transformed = Object.entries(aggregation).reduce(
+    (acc, [key, value]) => {
+      const isBucketArray = Array.isArray(value.buckets);
+
+      acc[key] = isBucketArray
+        ? value.buckets.map(
+            (bucket: AggregationsFrequentItemSetsBucketKeys) => ({
+              _id: bucket.key,
+              count: bucket.doc_count,
+            }),
+          )
+        : [{ totalSets: value.value }];
+
+      return acc;
+    },
+    {} as ITransformedFullFacets,
+  );
+
+  return [transformed];
+};
diff --git a/src/elastic-search/interfaces/es-common.type.ts b/src/elastic-search/interfaces/es-common.type.ts
@@ -57,3 +57,27 @@ export interface IFilter {
     };
   };
 }
+
+export interface IFullFacets {
+  [key: string]: {
+    terms?: {
+      field: string;
+      order: {
+        _key?: string;
+        _count?: string;
+      };
+    };
+    value_count?: {
+      field: "pid";
+    };
+  };
+}
+
+export interface ITransformedFullFacets {
+  [key: string]:
+    | {
+        _id: string;
+        count: number;
+      }
+    | { totalSets: number };
+}
diff --git a/src/elastic-search/providers/fields.enum.ts b/src/elastic-search/providers/fields.enum.ts
@@ -10,6 +10,14 @@ export enum FilterFields {
   IsPublished = "isPublished",
 }
 
+export enum FacetFields {
+  Type = "type",
+  CreationLocation = "creationLocation",
+  OwnerGroup = "ownerGroup",
+  AccessGroups = "accessGroups",
+  Keywords = "keywords",
+}
+
 export enum QueryFields {
   DatasetName = "datasetName",
   Description = "description",

diff --git a/src/elastic-search/providers/query-builder.service.ts b/src/elastic-search/providers/query-builder.service.ts
@@ -4,10 +4,12 @@ import { IDatasetFields } from "src/datasets/interfaces/dataset-filters.interfac
 import {
   IBoolShould,
   IFilter,
+  IFullFacets,
   IShould,
   ObjectType,
 } from "../interfaces/es-common.type";
-import { FilterFields, QueryFields } from "./fields.enum";
+import { FilterFields, QueryFields, FacetFields } from "./fields.enum";
+
 import { mapScientificQuery } from "src/common/utils";
 import { IScientificFilter } from "src/common/interfaces/common.interface";
 import { convertToElasticSearchQuery } from "../helpers/utils";
@@ -16,6 +18,7 @@ import { convertToElasticSearchQuery } from "../helpers/utils";
 export class SearchQueryService {
   readonly filterFields = [...Object.values(FilterFields)];
   readonly queryFields = [...Object.values(QueryFields)];
+  readonly facetFields = [...Object.values(FacetFields)];
   readonly textQuerySplitMethod = /[ ,]+/;
 
   public buildSearchQuery(searchParam: IDatasetFields) {
@@ -179,4 +182,26 @@ export class SearchQueryService {
     };
     return finalQuery;
   }
+
+  public buildFullFacetPipeline(facetFields = this.facetFields) {
+    const pipeline: IFullFacets = {
+      all: {
+        value_count: {
+          field: "pid",
+        },
+      },
+    };
+
+    for (const field of facetFields) {
+      pipeline[field] = {
+        terms: {
+          field: field,
+          order: {
+            _count: "desc",
+          },
+        },
+      };
+    }
+    return pipeline;
+  }
 }