Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Increased timeout thresholds for alerting cypress tests to reduce flakiness when running on slow domains. #1093

Merged
merged 3 commits into from
Feb 17, 2024
Merged

Increased timeout thresholds for alerting cypress tests to reduce flakiness when running on slow domains. #1093

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
78214f9
Increased timeout thresholds for alerting cypress tests to reduce fla…
AWSHurneyt Feb 16, 2024
500e97f
Made spec less flaky.
AWSHurneyt Feb 17, 2024
9641904
Merge branch 'main' into 3.0-v2.12-integ-test-failures
AWSHurneyt Feb 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cypress/integration/plugins/alerting-dashboards-plugin/acknowledge_alerts_modal_spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ const BUCKET_TRIGGER = 'sample_alerts_flyout_bucket_level_trigger';
const QUERY_MONITOR = 'sample_alerts_flyout_query_level_monitor';
const QUERY_TRIGGER = 'sample_alerts_flyout_query_level_trigger';

const TWENTY_SECONDS = 20000;
const TWENTY_SECONDS = 60000;

describe('AcknowledgeAlertsModal', () => {
before(() => {
Expand Down
7 changes: 5 additions & 2 deletions cypress/integration/plugins/alerting-dashboards-plugin/alert_spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@
* SPDX-License-Identifier: Apache-2.0
*/

import { ALERTING_PLUGIN_NAME } from '../../../utils/plugins/alerting-dashboards-plugin/constants';
import {
ALERTING_PLUGIN_NAME,
ALERTING_PLUGIN_TIMEOUT,
} from '../../../utils/plugins/alerting-dashboards-plugin/constants';
import sampleQueryLevelMonitorWithAlwaysTrueTrigger from '../../../fixtures/plugins/alerting-dashboards-plugin/sample_query_level_monitor_with_always_true_trigger';
import sampleQueryLevelMonitorWorkflow from '../../../fixtures/plugins/alerting-dashboards-plugin/sample_query_level_monitor_workflow';
import { BASE_PATH } from '../../../utils/base_constants';
Expand All @@ -19,7 +22,7 @@ describe('Alerts', () => {
cy.visit(`${BASE_PATH}/app/${ALERTING_PLUGIN_NAME}#/dashboard`);

// Common text to wait for to confirm page loaded, give up to 30 seconds for initial load
cy.contains('Acknowledge', { timeout: 30000 });
cy.contains('Acknowledge', { timeout: ALERTING_PLUGIN_TIMEOUT });
});

describe("can be in 'Active' state", () => {
Expand Down
281 changes: 181 additions & 100 deletions cypress/integration/plugins/alerting-dashboards-plugin/alerts_dashboard_flyout_spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import {
ALERTING_INDEX,
ALERTING_PLUGIN_NAME,
ALERTING_PLUGIN_TIMEOUT,
} from '../../../utils/plugins/alerting-dashboards-plugin/constants';
import sampleAlertsFlyoutBucketMonitor from '../../../fixtures/plugins/alerting-dashboards-plugin/sample_alerts_flyout_bucket_level_monitor.json';
import sampleAlertsFlyoutQueryMonitor from '../../../fixtures/plugins/alerting-dashboards-plugin/sample_alerts_flyout_query_level_monitor.json';
Expand Down Expand Up @@ -44,8 +45,8 @@ describe('Alerts by trigger flyout', () => {
cy.visit(`${BASE_PATH}/app/${ALERTING_PLUGIN_NAME}#/monitors`);

// Confirm test monitors were created successfully
cy.contains(BUCKET_MONITOR_NAME);
cy.contains(QUERY_MONITOR_NAME);
cy.contains(BUCKET_MONITOR_NAME, { timeout: ALERTING_PLUGIN_TIMEOUT });
cy.contains(QUERY_MONITOR_NAME, { timeout: ALERTING_PLUGIN_TIMEOUT });

// Wait 1 minutes for the test monitors to trigger alerts,
// then go to the 'Alerts by trigger' dashboard page to view alerts
Expand All @@ -57,8 +58,8 @@ describe('Alerts by trigger flyout', () => {
cy.visit(`${BASE_PATH}/app/${ALERTING_PLUGIN_NAME}#/dashboard`);

// Confirm dashboard is displaying rows for the test monitors.
cy.contains(BUCKET_MONITOR_NAME);
cy.contains(QUERY_MONITOR_NAME);
cy.contains(BUCKET_MONITOR_NAME, { timeout: ALERTING_PLUGIN_TIMEOUT });
cy.contains(QUERY_MONITOR_NAME, { timeout: ALERTING_PLUGIN_TIMEOUT });

// Waiting 5 seconds for alerts to finish loading.
// This short wait period alleviates flakiness observed during these tests.
Expand All @@ -67,91 +68,144 @@ describe('Alerts by trigger flyout', () => {

it('Bucket-level monitor flyout test', () => {
// Click the link for the flyout.
cy.get(`[data-test-subj="euiLink_${BUCKET_TRIGGER}"]`).click();
cy.get(`[data-test-subj="euiLink_${BUCKET_TRIGGER}"]`, {
timeout: ALERTING_PLUGIN_TIMEOUT,
}).click();

// Perform the test checks within the flyout component.
cy.get(`[data-test-subj="alertsDashboardFlyout_${BUCKET_TRIGGER}"]`).within(
() => {
// Confirm flyout header contains expected text.
cy.get(
`[data-test-subj="alertsDashboardFlyout_header_${BUCKET_TRIGGER}"]`
).contains(`Alerts by ${BUCKET_TRIGGER}`);

// Confirm 'Trigger name' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_triggerName_${BUCKET_TRIGGER}"]`
).as('triggerName');
cy.get('@triggerName').contains('Trigger name');
cy.get('@triggerName').contains(BUCKET_TRIGGER);

// Confirm 'Severity' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_severity_${BUCKET_TRIGGER}"]`
).as('severity');
cy.get('@severity').contains('Severity');
cy.get('@severity').contains('4 (Low)');

// Confirm 'Monitor' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_monitor_${BUCKET_TRIGGER}"]`
).as('monitor');
cy.get('@monitor').contains('Monitor');
cy.get('@monitor').contains(BUCKET_MONITOR_NAME);

// Confirm 'Conditions' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_conditions_${BUCKET_TRIGGER}"]`
).as('conditions');
cy.get('@conditions').contains('Conditions');

// Confirm the 'Conditions' sections renders with all of the expected conditions.
[
'params._count < 10000',
'OR',
'params.avg_products_price == 10',
].forEach((entry) => cy.get('@conditions').contains(entry));

// Confirm 'Time range for the last' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_timeRange_${BUCKET_TRIGGER}"]`
).as('timeRange');
cy.get('@timeRange').contains('Time range for the last');
cy.get('@timeRange').contains('10 day(s)');

// Confirm 'Filters' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_filters_${BUCKET_TRIGGER}"]`
).as('filters');
cy.get('@filters').contains('Filters');
cy.get('@filters').contains('All fields are included');

// Confirm 'Group by' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_groupBy_${BUCKET_TRIGGER}"]`
).as('groupBy');
cy.get('@groupBy').contains('Group by');
cy.get('@groupBy').contains('customer_gender, user');

// Set the 'severity' filter to only display ACTIVE alerts.
cy.get('[data-test-subj="dashboardAlertStateFilter"]').select('Active');

// This monitor configuration consistently returns 46 alerts when testing locally.
// Confirm the flyout dashboard contains more than 1 ACTIVE alert.
cy.get('tbody > tr').should(($tr) =>
expect($tr).to.have.length.greaterThan(1)
);

// Select the first and last alerts in the table.
cy.get('input[data-test-subj^="checkboxSelectRow-"]').first().click();
cy.get('input[data-test-subj^="checkboxSelectRow-"]').last().click();

// Press the flyout 'Acknowledge button, and wait for the AcknowledgeAlerts API call to complete.
cy.get('[data-test-subj="flyoutAcknowledgeAlertsButton"]').click();
}
);
cy.get(`[data-test-subj="alertsDashboardFlyout_${BUCKET_TRIGGER}"]`, {
timeout: ALERTING_PLUGIN_TIMEOUT,
}).within(() => {
// Confirm flyout header contains expected text.
cy.get(
`[data-test-subj="alertsDashboardFlyout_header_${BUCKET_TRIGGER}"]`,
{ timeout: ALERTING_PLUGIN_TIMEOUT }
).contains(`Alerts by ${BUCKET_TRIGGER}`, {
timeout: ALERTING_PLUGIN_TIMEOUT,
});

// Confirm 'Trigger name' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_triggerName_${BUCKET_TRIGGER}"]`,
{ timeout: ALERTING_PLUGIN_TIMEOUT }
).as('triggerName');
cy.get('@triggerName').contains('Trigger name', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});
cy.get('@triggerName').contains(BUCKET_TRIGGER, {
timeout: ALERTING_PLUGIN_TIMEOUT,
});

// Confirm 'Severity' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_severity_${BUCKET_TRIGGER}"]`,
{ timeout: ALERTING_PLUGIN_TIMEOUT }
).as('severity');
cy.get('@severity', { timeout: ALERTING_PLUGIN_TIMEOUT }).contains(
'Severity',
{ timeout: ALERTING_PLUGIN_TIMEOUT }
);
cy.get('@severity', { timeout: ALERTING_PLUGIN_TIMEOUT }).contains(
'4 (Low)',
{ timeout: ALERTING_PLUGIN_TIMEOUT }
);

// Confirm 'Monitor' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_monitor_${BUCKET_TRIGGER}"]`
).as('monitor');
cy.get('@monitor').contains('Monitor', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});
cy.get('@monitor').contains(BUCKET_MONITOR_NAME, {
timeout: ALERTING_PLUGIN_TIMEOUT,
});

// Confirm 'Conditions' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_conditions_${BUCKET_TRIGGER}"]`
).as('conditions');
cy.get('@conditions').contains('Conditions', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});

// Confirm the 'Conditions' sections renders with all of the expected conditions.
[
'params._count < 10000',
'OR',
'params.avg_products_price == 10',
].forEach((entry) =>
cy
.get('@conditions')
.contains(entry, { timeout: ALERTING_PLUGIN_TIMEOUT })
);

// Confirm 'Time range for the last' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_timeRange_${BUCKET_TRIGGER}"]`
).as('timeRange');
cy.get('@timeRange').contains('Time range for the last', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});
cy.get('@timeRange').contains('10 day(s)', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});

// Confirm 'Filters' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_filters_${BUCKET_TRIGGER}"]`
).as('filters');
cy.get('@filters').contains('Filters', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});
cy.get('@filters').contains('All fields are included', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});

// Confirm 'Group by' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_groupBy_${BUCKET_TRIGGER}"]`
).as('groupBy');
cy.get('@groupBy').contains('Group by', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});
cy.get('@groupBy').contains('customer_gender, user', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});

// Set the 'severity' filter to only display ACTIVE alerts.
cy.get('[data-test-subj="dashboardAlertStateFilter"]', {
timeout: ALERTING_PLUGIN_TIMEOUT,
}).select('Active');

// This monitor configuration consistently returns 46 alerts when testing locally.
// Confirm the flyout dashboard contains more than 1 ACTIVE alert.
cy.get('tbody > tr').should(($tr) =>
expect($tr).to.have.length.greaterThan(1)
);

// Select the first and last alerts in the table.
cy.get('input[data-test-subj^="checkboxSelectRow-"]', {
timeout: ALERTING_PLUGIN_TIMEOUT,
})
.first()
.click();
cy.get('input[data-test-subj^="checkboxSelectRow-"]', {
timeout: ALERTING_PLUGIN_TIMEOUT,
})
.last()
.click();

// Press the flyout 'Acknowledge button, and wait for the AcknowledgeAlerts API call to complete.
cy.get('[data-test-subj="flyoutAcknowledgeAlertsButton"]', {
timeout: ALERTING_PLUGIN_TIMEOUT,
}).click();
});

// Confirm acknowledge alerts toast displays expected text.
cy.contains('Successfully acknowledged 2 alerts.');
cy.contains('Successfully acknowledged 2 alerts.', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});

// Confirm alerts were acknowledged as expected.
cy.get(`[data-test-subj="alertsDashboardFlyout_${BUCKET_TRIGGER}"]`).within(
Expand Down Expand Up @@ -185,58 +239,85 @@ describe('Alerts by trigger flyout', () => {
// Confirm flyout header contains expected text.
cy.get(
`[data-test-subj="alertsDashboardFlyout_header_${QUERY_TRIGGER}"]`
).contains(`Alerts by ${QUERY_TRIGGER}`);
).contains(`Alerts by ${QUERY_TRIGGER}`, {
timeout: ALERTING_PLUGIN_TIMEOUT,
});

// Confirm 'Trigger name' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_triggerName_${QUERY_TRIGGER}"]`
).as('triggerName');
cy.get('@triggerName').contains('Trigger name');
cy.get('@triggerName').contains(QUERY_TRIGGER);
cy.get('@triggerName').contains('Trigger name', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});
cy.get('@triggerName').contains(QUERY_TRIGGER, {
timeout: ALERTING_PLUGIN_TIMEOUT,
});

// Confirm 'Severity' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_severity_${QUERY_TRIGGER}"]`
).as('severity');
cy.get('@severity').contains('Severity');
cy.get('@severity').contains('2 (High)');
cy.get('@severity').contains('Severity', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});
cy.get('@severity').contains('2 (High)', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});

// Confirm 'Monitor' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_monitor_${QUERY_TRIGGER}"]`
).as('monitor');
cy.get('@monitor').contains('Monitor');
cy.get('@monitor').contains(QUERY_MONITOR_NAME);
cy.get('@monitor').contains('Monitor', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});
cy.get('@monitor').contains(QUERY_MONITOR_NAME, {
timeout: ALERTING_PLUGIN_TIMEOUT,
});

// Confirm 'Conditions' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_conditions_${QUERY_TRIGGER}"]`
).as('conditions');
cy.get('@conditions').contains('Condition');
cy.get('@conditions').contains('Condition', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});
cy.get('@conditions').contains(
`ctx.results[0].hits.total.value < 10000`
`ctx.results[0].hits.total.value < 10000`,
{ timeout: ALERTING_PLUGIN_TIMEOUT }
);

// Confirm 'Time range for the last' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_timeRange_${QUERY_TRIGGER}"]`
).as('timeRange');
cy.get('@timeRange').contains('Time range for the last');
cy.get('@timeRange').contains('10 day(s)');
cy.get('@timeRange').contains('Time range for the last', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});
cy.get('@timeRange').contains('10 day(s)', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});

// Confirm 'Filters' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_filters_${QUERY_TRIGGER}"]`
).as('filters');
cy.get('@filters').contains('Filters');
cy.get('@filters').contains('-');
cy.get('@filters').contains('Filters', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});
cy.get('@filters').contains('-', { timeout: ALERTING_PLUGIN_TIMEOUT });

// Confirm 'Group by' sections renders as expected.
cy.get(
`[data-test-subj="alertsDashboardFlyout_groupBy_${QUERY_TRIGGER}"]`
).as('groupBy');
cy.get('@groupBy').contains('Group by');
cy.get('@groupBy').contains('user');
cy.get('@groupBy').contains('Group by', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});
cy.get('@groupBy').contains('user', {
timeout: ALERTING_PLUGIN_TIMEOUT,
});

// Set the 'severity' filter to only display ACTIVE alerts.
cy.get('[data-test-subj="dashboardAlertStateFilter"]').select('Active');
Expand Down
Loading
Loading