fix: behavior of Elasticsearch query differs from mongoDB query (#1184)

* fix: improved elasticsearch query * In elasticsearch datasetMapping ,ignore_above 256 option for keywords type is added * minor fix * fix e2e test fail * elasticsearch variable naming improvement
SciCatProject · May 21, 2024 · 5f237c2 · 5f237c2
1 parent 4201fff
commit 5f237c2
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 24 deletions.
diff --git a/src/elastic-search/configuration/datasetFieldMapping.ts b/src/elastic-search/configuration/datasetFieldMapping.ts
@@ -25,6 +25,7 @@ export const datasetMappings: MappingObject = {
   },
   pid: {
     type: "keyword",
+    ignore_above: 256,
   },
   creationTime: {
     type: "date",
@@ -48,29 +49,45 @@ export const datasetMappings: MappingObject = {
   },
   proposalId: {
     type: "keyword",
+    ignore_above: 256,
+  },
+  sampleId: {
+    type: "keyword",
+    ignore_above: 256,
   },
   sourceFolder: {
     type: "keyword",
+    ignore_above: 256,
   },
   isPublished: {
     type: "boolean",
   },
   type: {
     type: "keyword",
+    ignore_above: 256,
   },
   keywords: {
     type: "keyword",
+    ignore_above: 256,
   },
   creationLocation: {
     type: "keyword",
+    ignore_above: 256,
   },
   ownerGroup: {
     type: "keyword",
+    ignore_above: 256,
   },
   accessGroups: {
     type: "keyword",
+    ignore_above: 256,
   },
   sharedWith: {
     type: "keyword",
+    ignore_above: 256,
+  },
+  ownerEmail: {
+    type: "keyword",
+    ignore_above: 256,
   },
 };
diff --git a/src/elastic-search/elastic-search.service.ts b/src/elastic-search/elastic-search.service.ts
@@ -428,7 +428,7 @@ export class ElasticSearchService implements OnModuleInit {
         ];
       },
       onDrop(doc) {
-        console.debug(`${doc.document._id}`, doc.error?.reason);
+        console.debug(`${doc.document.pid}`, doc.error?.reason);
       },
     });
   }

diff --git a/src/elastic-search/providers/fields.enum.ts b/src/elastic-search/providers/fields.enum.ts
@@ -12,15 +12,7 @@ export enum FilterFields {
   Mode = "mode",
 }
 
-export enum FacetFields {
-  Type = "type",
-  CreationLocation = "creationLocation",
-  OwnerGroup = "ownerGroup",
-  AccessGroups = "accessGroups",
-  Keywords = "keywords",
-}
-
-export enum QueryFields {
+export enum MustFields {
   DatasetName = "datasetName",
   Description = "description",
 }
@@ -30,6 +22,14 @@ export enum ShouldFields {
   UserGroups = "userGroups",
 }
 
+export enum FacetFields {
+  Type = "type",
+  CreationLocation = "creationLocation",
+  OwnerGroup = "ownerGroup",
+  AccessGroups = "accessGroups",
+  Keywords = "keywords",
+}
+
 export enum SortFields {
   DatasetName = "datasetName",
   DatasetNameKeyword = "datasetName.keyword",

diff --git a/src/elastic-search/providers/query-builder.service.ts b/src/elastic-search/providers/query-builder.service.ts
@@ -10,7 +10,7 @@ import {
 } from "../interfaces/es-common.type";
 import {
   FilterFields,
-  QueryFields,
+  MustFields,
   FacetFields,
   ShouldFields,
 } from "./fields.enum";
@@ -22,7 +22,7 @@ import { convertToElasticSearchQuery } from "../helpers/utils";
 @Injectable()
 export class SearchQueryService {
   readonly filterFields = [...Object.values(FilterFields)];
-  readonly queryFields = [...Object.values(QueryFields)];
+  readonly mustFields = [...Object.values(MustFields)];
   readonly shouldFields = [...Object.values(ShouldFields)];
   readonly facetFields = [...Object.values(FacetFields)];
   readonly textQuerySplitMethod = /[ ,]+/;
@@ -33,9 +33,13 @@ export class SearchQueryService {
 
       const filter = this.buildFilterFields(fields);
       const should = this.buildShouldFields(fields);
-      const query = this.buildTextQuery(fields);
+      const must = this.buildTextQuery(fields);
 
-      return this.constructFinalQuery(filter, should, query);
+      // NOTE: The final query flow is as follows:
+      // step 1. Build filter fields conditions must match all filter fields
+      // step 2. Build should fields conditions must match at least one should field
+      // step 3. Build text query conditions must match all text query fields
+      return this.constructFinalQuery(filter, should, must);
     } catch (err) {
       Logger.error("Elastic search build search query failed");
       throw err;
@@ -44,15 +48,14 @@ export class SearchQueryService {
   private buildFilterFields(fields: Partial<IDatasetFields>): IFilter[] {
     const filter: IFilter[] = [];
 
-    for (const fieldName of this.filterFields) {
-      if (fields[fieldName]) {
-        const filterQueries = this.buildTermsFilter(
-          fieldName,
-          fields[fieldName],
-        );
-        filter.push(...filterQueries);
+    Object.entries(fields).forEach(([key, value]) => {
+      if (this.shouldFields.includes(key as ShouldFields) || key === "text") {
+        return;
       }
-    }
+
+      const filterQueries = this.buildTermsFilter(key, value);
+      filter.push(...filterQueries);
+    });
 
     return filter;
   }
@@ -101,7 +104,7 @@ export class SearchQueryService {
   private buildWildcardQueries(text: string): QueryDslQueryContainer[] {
     const terms = this.splitSearchText(text);
     return terms.flatMap((term) =>
-      this.queryFields.map((fieldName) => ({
+      this.mustFields.map((fieldName) => ({
         wildcard: { [fieldName]: { value: `*${term}*` } },
       })),
     );
@@ -173,7 +176,7 @@ export class SearchQueryService {
         if (typeof values === "string") {
           filterArray.push({
             match: {
-              [`${fieldName}.keyword`]: values as string,
+              [fieldName]: values as string,
             },
           });
         }