Merge pull request #794 from SciCatProject/SWAP-3516-elastic-search-i…

…nclude-filter-for-creationtime

feat : introduced new filters to elastic search, creationtime & scientific metadata condition

.github/workflows/test.yml

@@ -102,9 +102,15 @@ jobs:
         run: npm run build
 
   api_tests:
-    name: API tests
+    name: API tests with ElasticSearch enabled - ${{ matrix.elasticsearch_enabled }}
     needs: [install-and-cache]
     runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        elasticsearch_enabled: ["yes", "no"]
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -123,6 +129,7 @@ jobs:
 
       - name: API tests
         env:
+          ELASTICSEARCH_ENABLED: ${{ matrix.elasticsearch_enabled }}
           MONGODB_URI: mongodb://localhost:27017/scicat
           EXPRESS_SESSION_SECRET: a_scicat_secret
           JWT_SECRET: a_scicat_secret
@@ -146,6 +153,17 @@ jobs:
           CREATE_DATASET_WITH_PID_GROUPS: "group2,group3"
           DATASET_CREATION_VALIDATION_ENABLED: true
           DATASET_CREATION_VALIDATION_REGEX: "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-4[0-9A-Fa-f]{3}-[89ABab][0-9A-Fa-f]{3}-[0-9A-Fa-f]{12}$"
+          ES_HOST: https://localhost:9200
+          ES_USERNAME: elastic
+          ES_PASSWORD: duo-password
+          MONGODB_COLLECTION: Dataset
+          ES_MAX_RESULT: 210000
+          ES_FIELDS_LIMIT: 400000
+          ES_INDEX: "dataset"
+          ES_REFRESH: "wait_for"
+          STACK_VERSION: 8.8.2
+          CLUSTER_NAME: es-cluster
+          MEM_LIMIT: 4G
 
         # Start mongo container and app before running api tests
         run: |
@@ -166,6 +184,13 @@ jobs:
           node-version: ${{ env.NODE_VERSION }}
 
       - name: Run stack in docker-compose
+        env:
+          STACK_VERSION: 8.8.2
+          CLUSTER_NAME: es-cluster
+          MEM_LIMIT: 4G
+          ES_USERNAME: elastic
+          ES_PASSWORD: duo-password
+
         run: |
           cp CI/E2E/docker-compose.yaml docker-compose.yaml
           docker-compose pull

.prettierrc

@@ -4,4 +4,4 @@
   "printWidth": 80,
   "trailingComma": "all",
   "arrowParens": "always"
-}
+}

CI/E2E/.env.elastic-search

@@ -0,0 +1,6 @@
+STACK_VERSION=8.8.2
+ES_PORT=9200
+CLUSTER_NAME=es-cluster
+MEM_LIMIT=4G
+ES_USERNAME=elastic
+ES_PASSWORD=duo-password

CI/E2E/backend.env

@@ -32,4 +32,12 @@ OAI_PROVIDER_ROUTER=
 CREATE_DATASET_WITH_PID_GROUPS=""
 DATASET_CREATION_VALIDATION_ENABLED=true
 DATASET_CREATION_VALIDATION_REGEX="^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$"
-
+ELASTICSEARCH_ENABLED='yes'
+ES_HOST=https://localhost:9200
+ES_USERNAME=elastic
+ES_PASSWORD=duo-password
+MONGODB_COLLECTION=Dataset
+ES_MAX_RESULT=210000
+ES_FIELDS_LIMIT=400000
+ES_INDEX="dataset"
+ES_REFRESH="wait_for"

CI/E2E/docker-compose-local-no-es.yaml

@@ -0,0 +1,27 @@
+version: "3.2"
+services:
+  mongodb:
+    image: bitnami/mongodb:latest
+    volumes:
+      - "mongodb_data:/bitnami"
+    ports:
+      - "27017:27017"
+  scichat-loopback:
+    image: dacat/scichat-loopback:e2e
+    command:
+      [
+        "./wait-for-it.sh",
+        "mongodb:27017",
+        "--",
+        "node",
+        "-r",
+        "dotenv/config",
+        "."
+      ]
+    volumes:
+      - ".env.scichat-loopback:/home/node/app/.env"
+    depends_on:
+      - mongodb
+volumes:
+  mongodb_data:
+    driver: local

CI/E2E/docker-compose-local.yaml

@@ -16,12 +16,35 @@ services:
         "node",
         "-r",
         "dotenv/config",
-        ".",
+        "."
       ]
     volumes:
       - ".env.scichat-loopback:/home/node/app/.env"
     depends_on:
       - mongodb
+
+  es01:
+    depends_on:
+      - mongodb
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    ports:
+      - ${ES_PORT}:9200
+    environment:
+      - node.name=es01
+      - ES_JAVA_OPTS=-Xms2g -Xmx2g
+      - cluster.name=${CLUSTER_NAME}
+      - cluster.initial_master_nodes=es01
+      - ELASTIC_PASSWORD=${ES_PASSWORD}
+      - bootstrap.memory_lock=true
+    mem_limit: ${MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+
 volumes:
   mongodb_data:
     driver: local
+
+  es01:
+    driver: local

CI/E2E/docker-compose.yaml

@@ -49,6 +49,26 @@ services:
       - "traefik.http.routers.frontend.rule=PathPrefix(`/`)"
       - "traefik.http.routers.frontend.entrypoints=web"
 
+  es01:
+    depends_on:
+      - mongodb
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    environment:
+      - node.name=es01
+      - ES_JAVA_OPTS=-Xms2g -Xmx2g
+      - cluster.name=${CLUSTER_NAME}
+      - cluster.initial_master_nodes=es01
+      - ELASTIC_PASSWORD=${ES_PASSWORD}
+      - bootstrap.memory_lock=true
+
+    mem_limit: ${MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+
 volumes:
   mongodb_data:
     driver: local
+  es01:
+    driver: local

CI/ESS/docker-compose.api.yaml

@@ -5,7 +5,26 @@ services:
       - mongodb_data:/bitnami
     ports:
       - "27017:27017"
-
+  es01:
+    depends_on:
+      - mongodb
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    ports:
+      - "9200:9200"
+    environment:
+      - node.name=es01
+      - ES_JAVA_OPTS=-Xms2g -Xmx2g
+      - cluster.name=${CLUSTER_NAME}
+      - cluster.initial_master_nodes=es01
+      - ELASTIC_PASSWORD=${ES_PASSWORD}
+      - bootstrap.memory_lock=true
+    mem_limit: ${MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
 volumes:
   mongodb_data:
     driver: local
+  es01:
+    driver: local

CI/ESS/docker-compose.yaml

@@ -24,9 +24,25 @@ services:
       - RABBITMQ_USERNAME=admin
     volumes:
       - './rabbitmq:/bitnami'
-    depends_on: 
+    depends_on:
       - mongodb
 
+  es01:
+    depends_on:
+      - mongodb
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    environment:
+      - node.name=es01
+      - ES_JAVA_OPTS=-Xms2g -Xmx2g
+      - cluster.name=${CLUSTER_NAME}
+      - cluster.initial_master_nodes=es01
+      - ELASTIC_PASSWORD=${ES_PASSWORD}
+      - bootstrap.memory_lock=true
+    mem_limit: ${MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
 #  catanie:
 #    build: 
 #      context: catanie
@@ -83,18 +99,20 @@ services:
 #    build: nodered
 #    ports:
 #     - "1880:1880"
-# 
+#
 #  myshell:
 #    build:
 #      context: myshell
 #    image: garethcmurphy/myshell
 #    depends_on: 
 #      - mongodb
 #      - local-rabbit-rabbitmq.dev
-    
+
 
 volumes:
   mongodb_data:
     driver: local
   rabbitmq_data:
     driver: local
+  es01:
+    driver: local

package.json

@@ -27,14 +27,16 @@
     "test:api": "npm run test:api:jest --maxWorkers=50% && concurrently -k -s first \"wait-on http://localhost:3000/explorer/ && npm run test:api:mocha\" \"npm run start\"",
     "test:api:jest": "jest --config ./test/config/jest-e2e.json --maxWorkers=50%",
     "test:api:mocha": "mocha --config ./test/config/.mocharc.json",
-    "prepare:local": "docker-compose -f CI/E2E/docker-compose-local.yaml up -d && cp functionalAccounts.json.example functionalAccounts.json"
+    "prepare:local": "docker-compose -f CI/E2E/docker-compose-local.yaml --env-file CI/E2E/.env.elastic-search up  -d && cp functionalAccounts.json.example functionalAccounts.json"
   },
   "dependencies": {
     "@casl/ability": "^6.3.2",
+    "@elastic/elasticsearch": "^8.9.0",
     "@nestjs-modules/mailer": "^1.8.1",
     "@nestjs/axios": "^3.0.0",
     "@nestjs/common": "^9.0.0",
     "@nestjs/config": "^3.0.0",
+    "@nestjs/elasticsearch": "^10.0.1",
     "@nestjs/core": "^9.0.0",
     "@nestjs/event-emitter": "^2.0.1",
     "@nestjs/jwt": "^10.0.1",

src/app.module.ts

@@ -111,6 +111,7 @@ export class AppModule implements OnApplicationBootstrap {
       this.configService.get<string>("rabbitMq.enabled") === "yes"
         ? true
         : false;
+
     if (rabbitMqEnabled) {
       const hostname = this.configService.get<string>("rabbitMq.hostname");
       const username = this.configService.get<string>("rabbitMq.username");