build query parameters using data_end_time (#741)

This PR addresses a data non-population issue observed in HC detectors. When setting the time horizon in the anomaly overview to the past hour, two boxes appeared in the heatmap. However, clicking on both resulted in no data being populated. Extending the time horizon to three hours increased the number of boxes to six, but similarly, clicking on these boxes also resulted in no data appearing. The root cause of the issue is a mismatch in time references: the time displayed in the HC heatmap cells is calculated based on the anomaly plot time, which corresponds to data_end_time. However, when querying data within the HC heatmap cell's time range, data_start_time was used instead. This PR updates sorting and querying fields from `DATA_START_TIME` to `DATA_END_TIME` to align with the data displayed in HC heatmap cells and ensure accuracy in temporal data analysis. Testing done: 1. reproduced the issue and verified the fix. 2. added unit tests. 3. Confirmed that single stream detector result views remain functional post-changes. Signed-off-by: Kaituo Li <[email protected]>
opensearch-project · Apr 30, 2024 · 0a748c7 · 0a748c7
1 parent 0335282
commit 0a748c7
Show file tree

Hide file tree

Showing 7 changed files with 239 additions and 99 deletions.
diff --git a/.github/workflows/build-and-test-workflow.yml b/.github/workflows/build-and-test-workflow.yml
@@ -88,6 +88,12 @@ jobs:
         run: |
           cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
           yarn osd bootstrap --single-version=loose
+      - name: Set npm to use bash for shell
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: |
+          # Sets Windows to use bash for npm shell so the script (e.g., environment variable resolution in package.json postbuild script)
+          # commands work as intended
+          npm config set script-shell "C:\\Program Files\\git\\bin\\bash.exe"
       - name: Build the plugin
         run: |
           cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin

diff --git a/.github/workflows/remote-integ-tests-workflow.yml b/.github/workflows/remote-integ-tests-workflow.yml
@@ -1,133 +1,177 @@
-# Running AD integ tests stored in https://github.com/opensearch-project/opensearch-dashboards-functional-test
-# In the future we should pull dependencies from bundled build snapshots. Because that is not available
-# yet we build the cluster from source (besides core Opensearch, which is a pulled min artifact).
-name: Remote integ tests workflow
-on:
-  push:
-    branches:
-      - "*"
-  pull_request:
-    branches:
-      - "*"
+name: FTR E2E AD Workbench Test
+
+on: [pull_request, push]
+
+env:
+  CI: 1
+  # avoid warnings like "tput: No value for $TERM and no -T specified"
+  TERM: xterm
+  OPENSEARCH_DASHBOARDS_VERSION: 'main'
+  OPENSEARCH_VERSION: '3.0.0'
+  OPENSEARCH_PLUGIN_VERSION: '3.0.0.0'
+
 jobs:
-  test-without-security:
-    name: Run integ tests without security
+  tests:
+    name: Run FTR E2E AD Workbench Tests
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest]
-        java: [11]
-        include:
-          - os: windows-latest
-            cypress_cache_folder: ~/AppData/Local/Cypress/Cache
-          - os: ubuntu-latest
-            cypress_cache_folder: ~/.cache/Cypress
+        os: [ ubuntu-latest ]
+        jdk: [ 11 ]
     runs-on: ${{ matrix.os }}
+
     steps:
-      - name: Set up Java 11
-        uses: actions/setup-java@v3
+      - name: Set up JDK
+        uses: actions/setup-java@v1
         with:
-          distribution: 'corretto'
-          java-version: '11'
-
-      - name: Enable longer filenames
-        if: ${{ matrix.os == 'windows-latest' }}
-        run: git config --system core.longpaths true
+          java-version: ${{ matrix.jdk }}
 
-      - name: Checkout OpenSearch Dashboards
+      - name: Checkout Anomaly-Detection
         uses: actions/checkout@v2
         with:
-          repository: opensearch-project/OpenSearch-Dashboards
+          path: anomaly-detection
+          repository: opensearch-project/anomaly-detection
           ref: '${{ github.base_ref }}'
-          path: OpenSearch-Dashboards
 
-      - name: Checkout Anomaly Detection OpenSearch Dashboards plugin
+      - name: Run OpenSearch with plugin
+        run: |
+          cd anomaly-detection
+          ./gradlew run &
+          timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
+        shell: bash
+
+      - name: Check OpenSearch Running on Linux
+        if: ${{ runner.os != 'Windows'}}
+        run: curl http://localhost:9200/
+        shell: bash
+
+      - name: Checkout OpenSearch Dashboards
         uses: actions/checkout@v2
         with:
-          path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
+          path: OpenSearch-Dashboards
+          repository: opensearch-project/OpenSearch-Dashboards
+          ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
+          fetch-depth: 0
+          filter: |
+            cypress
+            test
 
-      - name: Setup Node
-        uses: actions/setup-node@v3
+      - name: Checkout AD in OpenSearch Dashboards Plugins Dir
+        uses: actions/checkout@v2
         with:
-          node-version-file: './OpenSearch-Dashboards/.nvmrc'
-          registry-url: 'https://registry.npmjs.org'
+            path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
 
-      - name: Install Yarn
-        # Need to use bash to avoid having a windows/linux specific step
-        shell: bash
+      - id: tool-versions
         run: |
-          YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn")
-          echo "Installing yarn@$YARN_VERSION"
-          npm i -g yarn@$YARN_VERSION
-
-      - run: node -v
-      - run: yarn -v
+          echo "node_version=$(cat .node-version)" >> $GITHUB_OUTPUT
+          echo "yarn_version=$(jq -r '.engines.yarn' package.json)" >> $GITHUB_OUTPUT
+        working-directory: OpenSearch-Dashboards
+        shell: bash
 
-      - name: Checkout Anomaly-Detection
-        uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
         with:
-          path: anomaly-detection
-          repository: opensearch-project/anomaly-detection
-          ref: '${{ github.base_ref }}'
+          node-version: ${{ steps.tool-versions.outputs.node_version }}
+          registry-url: 'https://registry.npmjs.org'
 
-      - name: Run OpenSearch with plugin
+      - name: Setup Opensearch Dashboards
         run: |
-          cd anomaly-detection
-          CONFIG_PATH=../OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin/opensearch_dashboards.json
-          OPENSEARCH_VERSION=$(node -p "require('$CONFIG_PATH').opensearchDashboardsVersion")-SNAPSHOT
-          echo "Using OpenSearch version $OPENSEARCH_VERSION"
-          ./gradlew run -Dopensearch.version=$OPENSEARCH_VERSION &
-          timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
+          npm uninstall -g yarn
+          echo "Installing yarn ${{ steps.tool-versions.outputs.yarn_version }}"
+          npm i -g yarn@${{ steps.tool-versions.outputs.yarn_version }}
+          yarn cache clean
+          yarn add sha.js
+        working-directory: OpenSearch-Dashboards
         shell: bash
 
-      - name: Bootstrap the plugin
+      - name: Boodstrap Opensearch Dashboards
         run: |
-          cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
           yarn osd bootstrap --single-version=loose
+        working-directory: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
+
+      - name: Run Opensearch Dashboards with AD Installed
+        run: |
+          nohup yarn start --no-base-path --no-watch --server.host="0.0.0.0" | tee dashboard.log &
+        working-directory: OpenSearch-Dashboards
 
-      - name: Run OpenSearch Dashboards server
+      - name : Check If OpenSearch Dashboards Is Ready
+        if: ${{ runner.os == 'Linux' }}
         run: |
-          cd OpenSearch-Dashboards
-          yarn start --no-base-path --no-watch &
+          if timeout 600 grep -q "bundles compiled successfully after" <(tail -n0 -f dashboard.log); then
+            echo "OpenSearch Dashboards compiled successfully."
+          else
+            echo "Timeout for 600 seconds reached. OpenSearch Dashboards did not finish compiling."
+            exit 1
+          fi
+        working-directory: OpenSearch-Dashboards
+
+      - name: Show OpenSearch Dashboards Logs
+        if: always()
+        run: cat dashboard.log
+        working-directory: OpenSearch-Dashboards
+
+      - name: Health check 
+        run: |
+          timeout 600 bash -c 'while [[ "$(curl -k http://localhost:5601/api/status | jq -r '.status.overall.state')" != "green" ]]; do sleep 5; done'
         shell: bash
 
-      # Window is slow so wait longer
-      - name: Sleep until OSD server starts - windows
-        if: ${{ matrix.os == 'windows-latest' }}
-        run: Start-Sleep -s 400
-        shell: powershell
-
-      - name: Sleep until OSD server starts - non-windows
-        if: ${{ matrix.os != 'windows-latest' }}
-        run: sleep 300
+      - name: Check OpenSearch Dashboards Running on Linux
+        if: ${{ runner.os != 'Windows'}}
+        run: curl http://localhost:5601/api/status
         shell: bash
 
-      - name: Checkout opensearch-dashboards-functional-test
+      - name: Checkout Dashboards Functional Test Repo
         uses: actions/checkout@v2
         with:
           path: opensearch-dashboards-functional-test
           repository: opensearch-project/opensearch-dashboards-functional-test
-          ref: '${{ github.base_ref }}'
+          ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
+          fetch-depth: 0
+
+      - name: Install Cypress
+        run: |
+          npm install cypress --save-dev
+        shell: bash
+        working-directory: opensearch-dashboards-functional-test
 
       - name: Get Cypress version
         id: cypress_version
         run: |
-          echo "::set-output name=cypress_version::$(cat ./opensearch-dashboards-functional-test/package.json | jq '.devDependencies.cypress' | tr -d '"')"
+          echo "::set-output name=cypress_version::$(cat ./package.json | jq '.dependencies.cypress' | tr -d '"')"
+        working-directory: opensearch-dashboards-functional-test
 
-      - name: Cache Cypress
-        id: cache-cypress
-        uses: actions/cache@v1
+      - name: Finding spec files and store to output
+        id: finding-files
+        run: |
+          echo "::set-output name=FILELIST::$(find cypress/integration/plugins/anomaly-detection-dashboards-plugin -name '*.js' -print)"
+        working-directory: opensearch-dashboards-functional-test
+
+      - name: Print spec files from output
+        run: |
+          IFS="," read -a myarray <<< ${{ steps.finding-files.outputs.FILELIST }}
+          for i in "${myarray[@]}"; do
+            echo "${i}"
+          done
+        working-directory: opensearch-dashboards-functional-test
+
+      - name: Run spec files from output
+        run: |
+          IFS="," read -a myarray <<< ${{ steps.finding-files.outputs.FILELIST }}
+          for i in "${myarray[@]}"; do
+            yarn cypress:run-without-security --browser electron --spec "${i}"
+            sleep 60
+          done
+        working-directory: opensearch-dashboards-functional-test
+
+      - name: Capture failure screenshots
+        uses: actions/upload-artifact@v1
+        if: failure()
         with:
-          path: ${{ matrix.cypress_cache_folder }}
-          key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }}
-        env:
-          CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }}
-      - run: npx cypress cache list
-      - run: npx cypress cache path
-
-      - name: Run AD cypress tests
-        uses: cypress-io/github-action@v2
+          name: cypress-screenshots-${{ matrix.os }}
+          path: opensearch-dashboards-functional-test/cypress/screenshots
+
+      - name: Capture failure test video
+        uses: actions/upload-artifact@v1
+        if: failure()
         with:
-          working-directory: opensearch-dashboards-functional-test
-          command: yarn run cypress run --env SECURITY_ENABLED=false --spec cypress/integration/plugins/anomaly-detection-dashboards-plugin/**/*.js
-        env:
-          CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }}   
+          name: cypress-videos-${{ matrix.os }}
+          path: opensearch-dashboards-functional-test/cypress/videos
diff --git a/package.json b/package.json
@@ -4,17 +4,17 @@
   "description": "OpenSearch Anomaly Detection Dashboards Plugin",
   "main": "index.js",
   "config": {
-    "plugin_version": "3.0.0.0",
-    "plugin_name": "anomalyDetectionDashboards",
-    "plugin_zip_name": "anomaly-detection-dashboards"
+    "id": "anomalyDetectionDashboards",
+    "zip_name": "anomaly-detection-dashboards"
   },
   "scripts": {
     "osd": "node ../../scripts/osd",
     "opensearch": "node ../../scripts/opensearch",
     "lint": "node ../../scripts/eslint .",
     "plugin-helpers": "node ../../scripts/plugin_helpers",
     "test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js",
-    "build": "yarn plugin-helpers build && echo Renaming artifact to $npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip && mv ./build/$npm_package_config_plugin_name*.zip ./build/$npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip"
+    "build": "yarn plugin-helpers build",
+    "postbuild": "echo Renaming artifact to [$npm_package_config_zip_name-$npm_package_version.zip] && mv build/$npm_package_config_id*.zip build/$npm_package_config_zip_name-$npm_package_version.zip"
   },
   "lint-staged": {
     "*.{ts,tsx,js,jsx,json,css,md}": [
@@ -56,4 +56,4 @@
     "browserify-sign": "^4.2.2",
     "axios": "^1.6.1"
   }
-}
+}
diff --git a/public/pages/DetectorResults/containers/AnomalyResults.tsx b/public/pages/DetectorResults/containers/AnomalyResults.tsx
@@ -255,6 +255,7 @@ export function AnomalyResults(props: AnomalyResultsProps) {
       endDate: adjustedCurrentTime.valueOf(),
     } as DateRange;
 
+    // build result search query params relative to data end time
     const params = buildParamsForGetAnomalyResultsWithDateRange(
       featureDataPointsRange.startDate,
       featureDataPointsRange.endDate