From 0a748c7dff74e5b1168e1774d81cebf52dec3d52 Mon Sep 17 00:00:00 2001 From: Kaituo Li Date: Tue, 30 Apr 2024 11:12:51 -0700 Subject: [PATCH] build query parameters using data_end_time (#741) This PR addresses a data non-population issue observed in HC detectors. When setting the time horizon in the anomaly overview to the past hour, two boxes appeared in the heatmap. However, clicking on both resulted in no data being populated. Extending the time horizon to three hours increased the number of boxes to six, but similarly, clicking on these boxes also resulted in no data appearing. The root cause of the issue is a mismatch in time references: the time displayed in the HC heatmap cells is calculated based on the anomaly plot time, which corresponds to data_end_time. However, when querying data within the HC heatmap cell's time range, data_start_time was used instead. This PR updates sorting and querying fields from `DATA_START_TIME` to `DATA_END_TIME` to align with the data displayed in HC heatmap cells and ensure accuracy in temporal data analysis. Testing done: 1. reproduced the issue and verified the fix. 2. added unit tests. 3. Confirmed that single stream detector result views remain functional post-changes. Signed-off-by: Kaituo Li --- .github/workflows/build-and-test-workflow.yml | 6 + .../workflows/remote-integ-tests-workflow.yml | 226 +++++++++++------- package.json | 10 +- .../containers/AnomalyResults.tsx | 1 + .../__tests__/anomalyResultUtils.test.ts | 57 +++++ public/pages/utils/anomalyResultUtils.ts | 36 ++- ...ion-kibana-plugin.release-notes-1.7.0.0.md | 2 +- 7 files changed, 239 insertions(+), 99 deletions(-) diff --git a/.github/workflows/build-and-test-workflow.yml b/.github/workflows/build-and-test-workflow.yml index 4410e06c..7a1de2d8 100644 --- a/.github/workflows/build-and-test-workflow.yml +++ b/.github/workflows/build-and-test-workflow.yml @@ -88,6 +88,12 @@ jobs: run: | cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin yarn osd bootstrap --single-version=loose + - name: Set npm to use bash for shell + if: ${{ matrix.os == 'windows-latest' }} + run: | + # Sets Windows to use bash for npm shell so the script (e.g., environment variable resolution in package.json postbuild script) + # commands work as intended + npm config set script-shell "C:\\Program Files\\git\\bin\\bash.exe" - name: Build the plugin run: | cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin diff --git a/.github/workflows/remote-integ-tests-workflow.yml b/.github/workflows/remote-integ-tests-workflow.yml index 7bce9cb6..129bafda 100644 --- a/.github/workflows/remote-integ-tests-workflow.yml +++ b/.github/workflows/remote-integ-tests-workflow.yml @@ -1,133 +1,177 @@ -# Running AD integ tests stored in https://github.com/opensearch-project/opensearch-dashboards-functional-test -# In the future we should pull dependencies from bundled build snapshots. Because that is not available -# yet we build the cluster from source (besides core Opensearch, which is a pulled min artifact). -name: Remote integ tests workflow -on: - push: - branches: - - "*" - pull_request: - branches: - - "*" +name: FTR E2E AD Workbench Test + +on: [pull_request, push] + +env: + CI: 1 + # avoid warnings like "tput: No value for $TERM and no -T specified" + TERM: xterm + OPENSEARCH_DASHBOARDS_VERSION: 'main' + OPENSEARCH_VERSION: '3.0.0' + OPENSEARCH_PLUGIN_VERSION: '3.0.0.0' + jobs: - test-without-security: - name: Run integ tests without security + tests: + name: Run FTR E2E AD Workbench Tests strategy: + fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] - java: [11] - include: - - os: windows-latest - cypress_cache_folder: ~/AppData/Local/Cypress/Cache - - os: ubuntu-latest - cypress_cache_folder: ~/.cache/Cypress + os: [ ubuntu-latest ] + jdk: [ 11 ] runs-on: ${{ matrix.os }} + steps: - - name: Set up Java 11 - uses: actions/setup-java@v3 + - name: Set up JDK + uses: actions/setup-java@v1 with: - distribution: 'corretto' - java-version: '11' - - - name: Enable longer filenames - if: ${{ matrix.os == 'windows-latest' }} - run: git config --system core.longpaths true + java-version: ${{ matrix.jdk }} - - name: Checkout OpenSearch Dashboards + - name: Checkout Anomaly-Detection uses: actions/checkout@v2 with: - repository: opensearch-project/OpenSearch-Dashboards + path: anomaly-detection + repository: opensearch-project/anomaly-detection ref: '${{ github.base_ref }}' - path: OpenSearch-Dashboards - - name: Checkout Anomaly Detection OpenSearch Dashboards plugin + - name: Run OpenSearch with plugin + run: | + cd anomaly-detection + ./gradlew run & + timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done' + shell: bash + + - name: Check OpenSearch Running on Linux + if: ${{ runner.os != 'Windows'}} + run: curl http://localhost:9200/ + shell: bash + + - name: Checkout OpenSearch Dashboards uses: actions/checkout@v2 with: - path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin + path: OpenSearch-Dashboards + repository: opensearch-project/OpenSearch-Dashboards + ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }} + fetch-depth: 0 + filter: | + cypress + test - - name: Setup Node - uses: actions/setup-node@v3 + - name: Checkout AD in OpenSearch Dashboards Plugins Dir + uses: actions/checkout@v2 with: - node-version-file: './OpenSearch-Dashboards/.nvmrc' - registry-url: 'https://registry.npmjs.org' + path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin - - name: Install Yarn - # Need to use bash to avoid having a windows/linux specific step - shell: bash + - id: tool-versions run: | - YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn") - echo "Installing yarn@$YARN_VERSION" - npm i -g yarn@$YARN_VERSION - - - run: node -v - - run: yarn -v + echo "node_version=$(cat .node-version)" >> $GITHUB_OUTPUT + echo "yarn_version=$(jq -r '.engines.yarn' package.json)" >> $GITHUB_OUTPUT + working-directory: OpenSearch-Dashboards + shell: bash - - name: Checkout Anomaly-Detection - uses: actions/checkout@v2 + - uses: actions/setup-node@v1 with: - path: anomaly-detection - repository: opensearch-project/anomaly-detection - ref: '${{ github.base_ref }}' + node-version: ${{ steps.tool-versions.outputs.node_version }} + registry-url: 'https://registry.npmjs.org' - - name: Run OpenSearch with plugin + - name: Setup Opensearch Dashboards run: | - cd anomaly-detection - CONFIG_PATH=../OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin/opensearch_dashboards.json - OPENSEARCH_VERSION=$(node -p "require('$CONFIG_PATH').opensearchDashboardsVersion")-SNAPSHOT - echo "Using OpenSearch version $OPENSEARCH_VERSION" - ./gradlew run -Dopensearch.version=$OPENSEARCH_VERSION & - timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done' + npm uninstall -g yarn + echo "Installing yarn ${{ steps.tool-versions.outputs.yarn_version }}" + npm i -g yarn@${{ steps.tool-versions.outputs.yarn_version }} + yarn cache clean + yarn add sha.js + working-directory: OpenSearch-Dashboards shell: bash - - name: Bootstrap the plugin + - name: Boodstrap Opensearch Dashboards run: | - cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin yarn osd bootstrap --single-version=loose + working-directory: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin + + - name: Run Opensearch Dashboards with AD Installed + run: | + nohup yarn start --no-base-path --no-watch --server.host="0.0.0.0" | tee dashboard.log & + working-directory: OpenSearch-Dashboards - - name: Run OpenSearch Dashboards server + - name : Check If OpenSearch Dashboards Is Ready + if: ${{ runner.os == 'Linux' }} run: | - cd OpenSearch-Dashboards - yarn start --no-base-path --no-watch & + if timeout 600 grep -q "bundles compiled successfully after" <(tail -n0 -f dashboard.log); then + echo "OpenSearch Dashboards compiled successfully." + else + echo "Timeout for 600 seconds reached. OpenSearch Dashboards did not finish compiling." + exit 1 + fi + working-directory: OpenSearch-Dashboards + + - name: Show OpenSearch Dashboards Logs + if: always() + run: cat dashboard.log + working-directory: OpenSearch-Dashboards + + - name: Health check + run: | + timeout 600 bash -c 'while [[ "$(curl -k http://localhost:5601/api/status | jq -r '.status.overall.state')" != "green" ]]; do sleep 5; done' shell: bash - # Window is slow so wait longer - - name: Sleep until OSD server starts - windows - if: ${{ matrix.os == 'windows-latest' }} - run: Start-Sleep -s 400 - shell: powershell - - - name: Sleep until OSD server starts - non-windows - if: ${{ matrix.os != 'windows-latest' }} - run: sleep 300 + - name: Check OpenSearch Dashboards Running on Linux + if: ${{ runner.os != 'Windows'}} + run: curl http://localhost:5601/api/status shell: bash - - name: Checkout opensearch-dashboards-functional-test + - name: Checkout Dashboards Functional Test Repo uses: actions/checkout@v2 with: path: opensearch-dashboards-functional-test repository: opensearch-project/opensearch-dashboards-functional-test - ref: '${{ github.base_ref }}' + ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }} + fetch-depth: 0 + + - name: Install Cypress + run: | + npm install cypress --save-dev + shell: bash + working-directory: opensearch-dashboards-functional-test - name: Get Cypress version id: cypress_version run: | - echo "::set-output name=cypress_version::$(cat ./opensearch-dashboards-functional-test/package.json | jq '.devDependencies.cypress' | tr -d '"')" + echo "::set-output name=cypress_version::$(cat ./package.json | jq '.dependencies.cypress' | tr -d '"')" + working-directory: opensearch-dashboards-functional-test - - name: Cache Cypress - id: cache-cypress - uses: actions/cache@v1 + - name: Finding spec files and store to output + id: finding-files + run: | + echo "::set-output name=FILELIST::$(find cypress/integration/plugins/anomaly-detection-dashboards-plugin -name '*.js' -print)" + working-directory: opensearch-dashboards-functional-test + + - name: Print spec files from output + run: | + IFS="," read -a myarray <<< ${{ steps.finding-files.outputs.FILELIST }} + for i in "${myarray[@]}"; do + echo "${i}" + done + working-directory: opensearch-dashboards-functional-test + + - name: Run spec files from output + run: | + IFS="," read -a myarray <<< ${{ steps.finding-files.outputs.FILELIST }} + for i in "${myarray[@]}"; do + yarn cypress:run-without-security --browser electron --spec "${i}" + sleep 60 + done + working-directory: opensearch-dashboards-functional-test + + - name: Capture failure screenshots + uses: actions/upload-artifact@v1 + if: failure() with: - path: ${{ matrix.cypress_cache_folder }} - key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }} - env: - CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }} - - run: npx cypress cache list - - run: npx cypress cache path - - - name: Run AD cypress tests - uses: cypress-io/github-action@v2 + name: cypress-screenshots-${{ matrix.os }} + path: opensearch-dashboards-functional-test/cypress/screenshots + + - name: Capture failure test video + uses: actions/upload-artifact@v1 + if: failure() with: - working-directory: opensearch-dashboards-functional-test - command: yarn run cypress run --env SECURITY_ENABLED=false --spec cypress/integration/plugins/anomaly-detection-dashboards-plugin/**/*.js - env: - CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }} + name: cypress-videos-${{ matrix.os }} + path: opensearch-dashboards-functional-test/cypress/videos diff --git a/package.json b/package.json index 8ac2f912..be4f3767 100644 --- a/package.json +++ b/package.json @@ -4,9 +4,8 @@ "description": "OpenSearch Anomaly Detection Dashboards Plugin", "main": "index.js", "config": { - "plugin_version": "3.0.0.0", - "plugin_name": "anomalyDetectionDashboards", - "plugin_zip_name": "anomaly-detection-dashboards" + "id": "anomalyDetectionDashboards", + "zip_name": "anomaly-detection-dashboards" }, "scripts": { "osd": "node ../../scripts/osd", @@ -14,7 +13,8 @@ "lint": "node ../../scripts/eslint .", "plugin-helpers": "node ../../scripts/plugin_helpers", "test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js", - "build": "yarn plugin-helpers build && echo Renaming artifact to $npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip && mv ./build/$npm_package_config_plugin_name*.zip ./build/$npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip" + "build": "yarn plugin-helpers build", + "postbuild": "echo Renaming artifact to [$npm_package_config_zip_name-$npm_package_version.zip] && mv build/$npm_package_config_id*.zip build/$npm_package_config_zip_name-$npm_package_version.zip" }, "lint-staged": { "*.{ts,tsx,js,jsx,json,css,md}": [ @@ -56,4 +56,4 @@ "browserify-sign": "^4.2.2", "axios": "^1.6.1" } -} +} \ No newline at end of file diff --git a/public/pages/DetectorResults/containers/AnomalyResults.tsx b/public/pages/DetectorResults/containers/AnomalyResults.tsx index b09b18b2..366573b6 100644 --- a/public/pages/DetectorResults/containers/AnomalyResults.tsx +++ b/public/pages/DetectorResults/containers/AnomalyResults.tsx @@ -255,6 +255,7 @@ export function AnomalyResults(props: AnomalyResultsProps) { endDate: adjustedCurrentTime.valueOf(), } as DateRange; + // build result search query params relative to data end time const params = buildParamsForGetAnomalyResultsWithDateRange( featureDataPointsRange.startDate, featureDataPointsRange.endDate diff --git a/public/pages/utils/__tests__/anomalyResultUtils.test.ts b/public/pages/utils/__tests__/anomalyResultUtils.test.ts index 914c4c1f..7c393a7e 100644 --- a/public/pages/utils/__tests__/anomalyResultUtils.test.ts +++ b/public/pages/utils/__tests__/anomalyResultUtils.test.ts @@ -13,6 +13,7 @@ import { getFeatureMissingDataAnnotations, getFeatureDataPointsForDetector, parsePureAnomalies, + buildParamsForGetAnomalyResultsWithDateRange, } from '../anomalyResultUtils'; import { getRandomDetector } from '../../../redux/reducers/__tests__/utils'; import { @@ -22,11 +23,16 @@ import { AnomalyData, } from '../../../models/interfaces'; import { ANOMALY_RESULT_SUMMARY, PARSED_ANOMALIES } from './constants'; +import { MAX_ANOMALIES } from '../../../utils/constants'; +import { SORT_DIRECTION, AD_DOC_FIELDS } from '../../../../server/utils/constants'; describe('anomalyResultUtils', () => { let randomDetector_20_min: Detector; let randomDetector_20_sec: Detector; let feature_id = 'deny_max'; + const startTime = 1609459200000; // January 1, 2021 + const endTime = 1609545600000; // January 2, 2021 + beforeAll(() => { randomDetector_20_min = { ...getRandomDetector(true), @@ -569,6 +575,57 @@ describe('anomalyResultUtils', () => { ) ).toEqual([]); }); + test('should correctly build parameters with default options', () => { + const expected = { + from: 0, + size: MAX_ANOMALIES, + sortDirection: SORT_DIRECTION.DESC, + sortField: AD_DOC_FIELDS.DATA_END_TIME, + startTime: startTime, + endTime: endTime, + fieldName: AD_DOC_FIELDS.DATA_END_TIME, + anomalyThreshold: -1, + entityList: undefined, // Default as an empty array stringified + }; + + const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime); + expect(result).toEqual(expected); + }); + + test('should correctly handle `anomalyOnly` and non-empty `entityList`', () => { + const entities = [{ id: '1', name: 'Entity1' }, { id: '2', name: 'Entity2' }]; + const expected = { + from: 0, + size: MAX_ANOMALIES, + sortDirection: SORT_DIRECTION.DESC, + sortField: AD_DOC_FIELDS.DATA_END_TIME, + startTime: startTime, + endTime: endTime, + fieldName: AD_DOC_FIELDS.DATA_END_TIME, + anomalyThreshold: 0, // because anomalyOnly is true + entityList: JSON.stringify(entities), + }; + + const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, true, entities); + expect(result).toEqual(expected); + }); + + test('should handle undefined `entityList` as an empty array JSON string', () => { + const expected = { + from: 0, + size: MAX_ANOMALIES, + sortDirection: SORT_DIRECTION.DESC, + sortField: AD_DOC_FIELDS.DATA_END_TIME, + startTime: startTime, + endTime: endTime, + fieldName: AD_DOC_FIELDS.DATA_END_TIME, + anomalyThreshold: -1, // default as anomalyOnly is false + entityList: undefined, // Default for undefined entityList + }; + + const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, false, undefined); + expect(result).toEqual(expected); + }); }); describe('parsePureAnomalies()', () => { diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts index cc3408b6..1ff35b9a 100644 --- a/public/pages/utils/anomalyResultUtils.ts +++ b/public/pages/utils/anomalyResultUtils.ts @@ -118,6 +118,38 @@ export const getLiveAnomalyResults = ( ); }; +/** + * Builds search query parameters for retrieving anomaly results within a specified date range. + * + * This function constructs a parameter object for querying an anomaly detection system, filtering results + * by a given start and end time. It supports filtering anomalies based on a threshold and can limit results to + * specific entities if provided. + * + * In the context of anomaly results, the startTime and endTime parameters are used to compare against the data_end_time. + * Using data_end_time instead of data_start_time is crucial because, within HC heatmap cells, the startTime and + * endTime are derived from each cell's start and end times, which are determined based on the plotTime—coinciding + * with the data_end_time. This alignment ensures that the temporal data within each heatmap cell accurately + * reflects the intervals intended for analysis. + * + * @param startTime - The epoch time (in milliseconds) marking the start of the date range for the query. + * @param endTime - The epoch time (in milliseconds) marking the end of the date range for the query. + * @param anomalyOnly - Optional. If true, the query will return only results where anomalies are detected + * (anomaly threshold is set to 0). If false or omitted, it will include all results + * (anomaly threshold is set to -1). Default is `false`. + * @param entityList - Optional. An array of entities to filter the results. If omitted, results are not filtered + * by entities. Default is `undefined`. + * + * @returns An object containing the necessary parameters for the anomaly results search query. This object includes: + * - `from`: The starting index for fetching results (always set to 0). + * - `size`: The maximum number of anomalies to return (`MAX_ANOMALIES`). + * - `sortDirection`: The sorting order of results, set to descending (`SORT_DIRECTION.DESC`). + * - `sortField`: The field used to sort the data, set to data end time (`AD_DOC_FIELDS.DATA_END_TIME`). + * - `startTime`: Passed start time for the search range. + * - `endTime`: Passed end time for the search range. + * - `fieldName`: Field used to query the data, set to data end time (`AD_DOC_FIELDS.DATA_END_TIME`). + * - `anomalyThreshold`: The minimum score threshold for anomalies, dependent on `anomalyOnly` parameter. + * - `entityList`: A JSON string representing the list of entities to filter the results by. + */ export const buildParamsForGetAnomalyResultsWithDateRange = ( startTime: number, endTime: number, @@ -128,10 +160,10 @@ export const buildParamsForGetAnomalyResultsWithDateRange = ( from: 0, size: MAX_ANOMALIES, sortDirection: SORT_DIRECTION.DESC, - sortField: AD_DOC_FIELDS.DATA_START_TIME, + sortField: AD_DOC_FIELDS.DATA_END_TIME, startTime: startTime, endTime: endTime, - fieldName: AD_DOC_FIELDS.DATA_START_TIME, + fieldName: AD_DOC_FIELDS.DATA_END_TIME, anomalyThreshold: anomalyOnly ? 0 : -1, entityList: JSON.stringify(entityList), }; diff --git a/release-notes/opendistro-for-elasticsearch.anomaly-detection-kibana-plugin.release-notes-1.7.0.0.md b/release-notes/opendistro-for-elasticsearch.anomaly-detection-kibana-plugin.release-notes-1.7.0.0.md index 834a442b..a481d7fd 100644 --- a/release-notes/opendistro-for-elasticsearch.anomaly-detection-kibana-plugin.release-notes-1.7.0.0.md +++ b/release-notes/opendistro-for-elasticsearch.anomaly-detection-kibana-plugin.release-notes-1.7.0.0.md @@ -61,7 +61,7 @@ You can use the plugin with the same version of the [Open Distro for Elasticsear - Tune AD result charts [PR #102](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/102) - Use annotation for live chart [PR #119](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/119) - Set fixed height for anomalies live chart [PR #123](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/123) -- Use scientific notation when number less than 0.01 on live chart [PR #124](https://github.com/opendistro-for-elasticsearchanomaly-detection-kibana-plugin/pull/124) +- Use scientific notation when number less than 0.01 on live chart [PR #124](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/124) - Use bucket aggregation for anomaly distribution [PR #126](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/126) ## Bug Fixes