Skip to content

Commit

Permalink
Merge pull request #1228 from SciCatProject/elastic-search-filter-que…
Browse files Browse the repository at this point in the history 
…ry-for-numbers

fix: ES missing number value filter query
  • Loading branch information
@nitrosx
nitrosx authored May 28, 2024
2 parents 78a9119 + 03aa70d commit 1bc0cf5
Show file tree
Hide file tree
Showing 4 changed files with 129 additions and 71 deletions.
3 changes: 3 additions & 0 deletions src/elastic-search/configuration/datasetFieldMapping.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,4 +90,7 @@ export const datasetMappings: MappingObject = {
type: "keyword",
ignore_above: 256,
},
size: {
type: "long",
},
};
4 changes: 2 additions & 2 deletions src/elastic-search/providers/query-builder.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,10 +173,10 @@ export class SearchQueryService {
},
});
}
if (typeof values === "string") {
if (typeof values === "string" || typeof values === "number") {
filterArray.push({
match: {
[fieldName]: values as string,
[fieldName]: values as string | number,
},
});
}
Expand Down
191 changes: 122 additions & 69 deletions test/ElasticSearch.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/* eslint-disable @typescript-eslint/no-var-requires */
"use strict";

const { faker } = require("@faker-js/faker");
const utils = require("./LoginUtils");
const { TestData } = require("./TestData");
require("dotenv").config();
Expand All @@ -23,21 +24,18 @@ const scientificMetadataFieldName = {
string: "with_string",
};

const scientificMetadata = ({
lhs = "",
relation = "",
rhs = "",
unit = "",
}) => {
const scientificMetadata = (values) => {
const scientificQuery = values.map((value) => {
return {
lhs: value.lhs,
relation: value.relation,
rhs: value.rhs,
unit: value.unit,
};
});

return {
scientific: [
{
lhs,
relation,
rhs,
unit,
},
],
scientific: scientificQuery,
};
};

Expand Down Expand Up @@ -71,7 +69,7 @@ const scientificMetadata = ({
);
});

it("0010: adds a new raw dataset with scientificMetadata", function () {
it("0010: adds a new raw dataset with scientificMetadata", async () => {
return request(appUrl)
.post("/api/v3/Datasets")
.send(TestData.ScientificMetadataForElasticSearch)
Expand All @@ -92,42 +90,74 @@ const scientificMetadata = ({
});
});

it("0020: should fetch dataset with unitSI and ValueSI condition filter", function () {
request(appUrl)
it("0020: should fetch dataset with correct unitSI and ValueSI condition for scientific filter", async () => {
return request(appUrl)
.post("/api/v3/elastic-search/search")
.send(
scientificMetadata({
lhs: scientificMetadataFieldName.unitAndValue,
relation: Relation.GREATER_THAN,
rhs: 99,
unit: "m",
}),
scientificMetadata([
{
lhs: scientificMetadataFieldName.unitAndValue,
relation: Relation.GREATER_THAN,
rhs: 99,
unit: "m",
},
{
lhs: scientificMetadataFieldName.unitAndValue,
relation: Relation.EQUAL_TO_NUMERIC,
rhs: 100,
unit: "m",
},
{
lhs: scientificMetadataFieldName.unitAndValue,
relation: Relation.LESS_THAN,
rhs: 101,
unit: "m",
},
]),
)
.set("Accept", "application/json")
.set({ Authorization: `Bearer ${accessTokenAdminIngestor}` });
.set({ Authorization: `Bearer ${accessTokenAdminIngestor}` })
.expect(TestData.SuccessfulPostStatusCode)
.expect("Content-Type", /json/)
.then((res) => {
res.body.data.should.include(decodeURIComponent(pid));
});
});

request(appUrl)
it("0021: should fetch dataset with correct numeric value for scientific filter", async () => {
return request(appUrl)
.post("/api/v3/elastic-search/search")
.send(
scientificMetadata({
lhs: scientificMetadataFieldName.unitAndValue,
relation: Relation.LESS_THAN,
rhs: 101,
unit: "m",
}),
scientificMetadata([
{
lhs: scientificMetadataFieldName.number,
relation: Relation.EQUAL_TO_NUMERIC,
rhs: 111,
unit: "",
},
]),
)
.set("Accept", "application/json")
.set({ Authorization: `Bearer ${accessTokenAdminIngestor}` });
.set({ Authorization: `Bearer ${accessTokenAdminIngestor}` })
.expect(TestData.SuccessfulPostStatusCode)
.expect("Content-Type", /json/)
.then((res) => {
res.body.data.should.include(decodeURIComponent(pid));
});
});

it("0022: should fetch dataset with correct string value for the scientific filter", async () => {
return request(appUrl)
.post("/api/v3/elastic-search/search")
.send(
scientificMetadata({
lhs: scientificMetadataFieldName.unitAndValue,
relation: Relation.EQUAL_TO_NUMERIC,
rhs: 100,
unit: "m",
}),
scientificMetadata([
{
lhs: scientificMetadataFieldName.string,
relation: Relation.EQUAL_TO_STRING,
rhs: "222",
unit: "",
},
]),
)
.set("Accept", "application/json")
.set({ Authorization: `Bearer ${accessTokenAdminIngestor}` })
Expand All @@ -137,37 +167,36 @@ const scientificMetadata = ({
res.body.data.should.include(decodeURIComponent(pid));
});
});
it("0030: should fetch dataset with numeric value filter", async () => {

it("0023: should fail when fetching dataset with incorrect relation type and value type for the scientific filter", async () => {
return request(appUrl)
.post("/api/v3/elastic-search/search")
.send(
scientificMetadata({
lhs: scientificMetadataFieldName.number,
relation: Relation.EQUAL_TO_NUMERIC,
rhs: 111,
unit: "",
}),
scientificMetadata([
{
lhs: scientificMetadataFieldName.number,
relation: Relation.EQUAL_TO_NUMERIC,
rhs: "111",
unit: "",
},
]),
)
.set("Accept", "application/json")
.set({ Authorization: `Bearer ${accessTokenAdminIngestor}` })
.expect(TestData.SuccessfulPostStatusCode)
.expect("Content-Type", /json/)
.then((res) => {
res.body.data.should.include(decodeURIComponent(pid));
res.body.data.should.be.length(0);
});
});

it("0040: should fetch dataset with string value filter", async () => {
it("0030: should fetching dataset with correct proposalId and size", async () => {
return request(appUrl)
.post("/api/v3/elastic-search/search")
.send(
scientificMetadata({
lhs: scientificMetadataFieldName.string,
relation: Relation.EQUAL_TO_STRING,
rhs: "222",
unit: "",
}),
)
.send({
proposalId: TestData.ScientificMetadataForElasticSearch.proposalId,
size: TestData.ScientificMetadataForElasticSearch.size,
})
.set("Accept", "application/json")
.set({ Authorization: `Bearer ${accessTokenAdminIngestor}` })
.expect(TestData.SuccessfulPostStatusCode)
Expand All @@ -177,30 +206,54 @@ const scientificMetadata = ({
});
});

it("0050: should fail when fetching dataset with incorrect relation type and value type", async () => {
it("0031: should fail fetching dataset with correct proposalId but wrong size", async () => {
return request(appUrl)
.post("/api/v3/elastic-search/search")
.send(
scientificMetadata({
lhs: scientificMetadataFieldName.number,
relation: Relation.EQUAL_TO_NUMERIC,
rhs: "111",
unit: "",
}),
)
.send({
proposalId: TestData.ScientificMetadataForElasticSearch.proposalId,
size: faker.number.int({ min: 100000001, max: 100400000 }),
})
.set("Accept", "application/json")
.set({ Authorization: `Bearer ${accessTokenAdminIngestor}` })
.expect(TestData.SuccessfulPostStatusCode)
.expect("Content-Type", /json/)
.then((res) => {
res.body.should.have
.property("data")
.which.is.an("array")
.that.has.lengthOf(0);
res.body.data.should.be.length(0);
});
});
it("0032: should fail fetching dataset with wrong proposalId but correct size", async () => {
return request(appUrl)
.post("/api/v3/elastic-search/search")
.send({
proposalId: "wrongProposalId",
size: TestData.ScientificMetadataForElasticSearch.size,
})
.set("Accept", "application/json")
.set({ Authorization: `Bearer ${accessTokenAdminIngestor}` })
.expect(TestData.SuccessfulPostStatusCode)
.expect("Content-Type", /json/)
.then((res) => {
res.body.data.should.be.length(0);
});
});

it("0033: should fail fetching dataset with incorrect proposalId and size", async () => {
return request(appUrl)
.post("/api/v3/elastic-search/search")
.send({
proposalId: "wrongProposalId",
size: faker.number.int({ min: 100000001, max: 100400000 }),
})
.set("Accept", "application/json")
.set({ Authorization: `Bearer ${accessTokenAdminIngestor}` })
.expect(TestData.SuccessfulPostStatusCode)
.expect("Content-Type", /json/)
.then((res) => {
res.body.data.should.be.length(0);
});
});

it("0060: should delete this raw dataset", async () => {
it("0034: should delete this raw dataset", async () => {
return request(appUrl)
.delete("/api/v3/datasets/" + pid)
.set("Accept", "application/json")
Expand Down
2 changes: 2 additions & 0 deletions test/TestData.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -809,6 +809,8 @@ const TestData = {
creationTime: faker.date.past(),
sourceFolder: faker.system.directoryPath(),
owner: faker.internet.userName(),
size: faker.number.int({ min: 0, max: 100000000 }),
proposalId: faker.string.numeric(6),
contactEmail: faker.internet.email(),
scientificMetadata: {
with_unit_and_value_si: {
Expand Down

0 comments on commit 1bc0cf5

Please sign in to comment.