From 5f237c2a564bec4c82ff2604bc19e501984e239f Mon Sep 17 00:00:00 2001 From: Jay Date: Tue, 21 May 2024 13:46:31 +0200 Subject: [PATCH] fix: behavior of Elasticsearch query differs from mongoDB query (#1184) * fix: improved elasticsearch query * In elasticsearch datasetMapping ,ignore_above 256 option for keywords type is added * minor fix * fix e2e test fail * elasticsearch variable naming improvement --- .../configuration/datasetFieldMapping.ts | 17 ++++++++++ src/elastic-search/elastic-search.service.ts | 2 +- src/elastic-search/providers/fields.enum.ts | 18 +++++------ .../providers/query-builder.service.ts | 31 ++++++++++--------- 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/src/elastic-search/configuration/datasetFieldMapping.ts b/src/elastic-search/configuration/datasetFieldMapping.ts index 24c78ec15..e28d8b983 100644 --- a/src/elastic-search/configuration/datasetFieldMapping.ts +++ b/src/elastic-search/configuration/datasetFieldMapping.ts @@ -25,6 +25,7 @@ export const datasetMappings: MappingObject = { }, pid: { type: "keyword", + ignore_above: 256, }, creationTime: { type: "date", @@ -48,29 +49,45 @@ export const datasetMappings: MappingObject = { }, proposalId: { type: "keyword", + ignore_above: 256, + }, + sampleId: { + type: "keyword", + ignore_above: 256, }, sourceFolder: { type: "keyword", + ignore_above: 256, }, isPublished: { type: "boolean", }, type: { type: "keyword", + ignore_above: 256, }, keywords: { type: "keyword", + ignore_above: 256, }, creationLocation: { type: "keyword", + ignore_above: 256, }, ownerGroup: { type: "keyword", + ignore_above: 256, }, accessGroups: { type: "keyword", + ignore_above: 256, }, sharedWith: { type: "keyword", + ignore_above: 256, + }, + ownerEmail: { + type: "keyword", + ignore_above: 256, }, }; diff --git a/src/elastic-search/elastic-search.service.ts b/src/elastic-search/elastic-search.service.ts index 59d9b4244..fc3dca187 100644 --- a/src/elastic-search/elastic-search.service.ts +++ b/src/elastic-search/elastic-search.service.ts @@ -428,7 +428,7 @@ export class ElasticSearchService implements OnModuleInit { ]; }, onDrop(doc) { - console.debug(`${doc.document._id}`, doc.error?.reason); + console.debug(`${doc.document.pid}`, doc.error?.reason); }, }); } diff --git a/src/elastic-search/providers/fields.enum.ts b/src/elastic-search/providers/fields.enum.ts index 13718e921..8030407ab 100644 --- a/src/elastic-search/providers/fields.enum.ts +++ b/src/elastic-search/providers/fields.enum.ts @@ -12,15 +12,7 @@ export enum FilterFields { Mode = "mode", } -export enum FacetFields { - Type = "type", - CreationLocation = "creationLocation", - OwnerGroup = "ownerGroup", - AccessGroups = "accessGroups", - Keywords = "keywords", -} - -export enum QueryFields { +export enum MustFields { DatasetName = "datasetName", Description = "description", } @@ -30,6 +22,14 @@ export enum ShouldFields { UserGroups = "userGroups", } +export enum FacetFields { + Type = "type", + CreationLocation = "creationLocation", + OwnerGroup = "ownerGroup", + AccessGroups = "accessGroups", + Keywords = "keywords", +} + export enum SortFields { DatasetName = "datasetName", DatasetNameKeyword = "datasetName.keyword", diff --git a/src/elastic-search/providers/query-builder.service.ts b/src/elastic-search/providers/query-builder.service.ts index 08070f3e1..4813479e6 100644 --- a/src/elastic-search/providers/query-builder.service.ts +++ b/src/elastic-search/providers/query-builder.service.ts @@ -10,7 +10,7 @@ import { } from "../interfaces/es-common.type"; import { FilterFields, - QueryFields, + MustFields, FacetFields, ShouldFields, } from "./fields.enum"; @@ -22,7 +22,7 @@ import { convertToElasticSearchQuery } from "../helpers/utils"; @Injectable() export class SearchQueryService { readonly filterFields = [...Object.values(FilterFields)]; - readonly queryFields = [...Object.values(QueryFields)]; + readonly mustFields = [...Object.values(MustFields)]; readonly shouldFields = [...Object.values(ShouldFields)]; readonly facetFields = [...Object.values(FacetFields)]; readonly textQuerySplitMethod = /[ ,]+/; @@ -33,9 +33,13 @@ export class SearchQueryService { const filter = this.buildFilterFields(fields); const should = this.buildShouldFields(fields); - const query = this.buildTextQuery(fields); + const must = this.buildTextQuery(fields); - return this.constructFinalQuery(filter, should, query); + // NOTE: The final query flow is as follows: + // step 1. Build filter fields conditions must match all filter fields + // step 2. Build should fields conditions must match at least one should field + // step 3. Build text query conditions must match all text query fields + return this.constructFinalQuery(filter, should, must); } catch (err) { Logger.error("Elastic search build search query failed"); throw err; @@ -44,15 +48,14 @@ export class SearchQueryService { private buildFilterFields(fields: Partial): IFilter[] { const filter: IFilter[] = []; - for (const fieldName of this.filterFields) { - if (fields[fieldName]) { - const filterQueries = this.buildTermsFilter( - fieldName, - fields[fieldName], - ); - filter.push(...filterQueries); + Object.entries(fields).forEach(([key, value]) => { + if (this.shouldFields.includes(key as ShouldFields) || key === "text") { + return; } - } + + const filterQueries = this.buildTermsFilter(key, value); + filter.push(...filterQueries); + }); return filter; } @@ -101,7 +104,7 @@ export class SearchQueryService { private buildWildcardQueries(text: string): QueryDslQueryContainer[] { const terms = this.splitSearchText(text); return terms.flatMap((term) => - this.queryFields.map((fieldName) => ({ + this.mustFields.map((fieldName) => ({ wildcard: { [fieldName]: { value: `*${term}*` } }, })), ); @@ -173,7 +176,7 @@ export class SearchQueryService { if (typeof values === "string") { filterArray.push({ match: { - [`${fieldName}.keyword`]: values as string, + [fieldName]: values as string, }, }); }