diff --git a/infrastructure/lib/constructs/canarySns.ts b/infrastructure/lib/constructs/canarySns.ts index 766be9a..381da74 100644 --- a/infrastructure/lib/constructs/canarySns.ts +++ b/infrastructure/lib/constructs/canarySns.ts @@ -4,6 +4,7 @@ import {Construct} from "constructs"; import {Alarm} from "aws-cdk-lib/aws-cloudwatch"; import { Canary } from 'aws-cdk-lib/aws-synthetics'; import * as cloudwatch from "aws-cdk-lib/aws-cloudwatch"; +import {Duration} from "aws-cdk-lib"; interface canarySnsProps extends SnsMonitorsProps { readonly canaryAlarms: Array<{ alertName: string, canary: Canary }>; @@ -24,10 +25,12 @@ export class canarySns extends SnsMonitors { private canaryFailed(alertName: string, canary: Canary): [Alarm, string] { const alarmObject = new cloudwatch.Alarm(this, `error_alarm_${alertName}`, { - metric: canary.metricSuccessPercent(), - threshold: 50, + metric: canary.metricSuccessPercent({ + period: Duration.minutes(15) + }), + threshold: 0, evaluationPeriods: 1, - comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD, + comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD, datapointsToAlarm: 1, treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING, alarmDescription: "Detect Canary failure", diff --git a/infrastructure/lib/infrastructure-stack.ts b/infrastructure/lib/infrastructure-stack.ts index 5137cb2..a107043 100644 --- a/infrastructure/lib/infrastructure-stack.ts +++ b/infrastructure/lib/infrastructure-stack.ts @@ -10,7 +10,7 @@ import {ArnPrincipal} from "aws-cdk-lib/aws-iam"; import {OpenSearchWAF} from "./stacks/waf"; import {OpenSearchMetricsNginxCognito} from "./constructs/opensearchNginxProxyCognito"; import {OpenSearchMetricsMonitoringStack} from "./stacks/monitoringDashboard"; -import {OpenSearchMetricsSecrets} from "./stacks/secrets"; +import {OpenSearchMetricsSecretsStack} from "./stacks/secrets"; // import * as sqs from 'aws-cdk-lib/aws-sqs'; export class InfrastructureStack extends Stack { @@ -43,7 +43,7 @@ export class InfrastructureStack extends Stack { // Create Secrets Manager - const openSearchMetricsSecretsStack = new OpenSearchMetricsSecrets(app, "OpenSearchMetrics-Secrets", { + const openSearchMetricsSecretsStack = new OpenSearchMetricsSecretsStack(app, "OpenSearchMetrics-Secrets", { secretName: 'metrics-creds' }); diff --git a/infrastructure/lib/stacks/opensearchNginxProxyReadonly.ts b/infrastructure/lib/stacks/opensearchNginxProxyReadonly.ts index ca4f3ad..df4c672 100644 --- a/infrastructure/lib/stacks/opensearchNginxProxyReadonly.ts +++ b/infrastructure/lib/stacks/opensearchNginxProxyReadonly.ts @@ -59,15 +59,15 @@ export class OpenSearchMetricsNginxReadonly extends Stack { readonly asg: AutoScalingGroup; constructor(scope: Construct, id: string, props: NginxProps) { - const { vpc, securityGroup } = props; + const {vpc, securityGroup} = props; super(scope, id); const instanceRole = this.createNginxReadonlyInstanceRole(props); - this.asg = new AutoScalingGroup(this, 'OpenSearchMetricsReadonly-MetricsProxyAsg', { + this.asg = new AutoScalingGroup(this, 'OpenSearchMetricsReadonly-MetricsProxyAsg', { instanceType: InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE), - blockDevices: [{ deviceName: '/dev/xvda', volume: BlockDeviceVolume.ebs(50) }], // GB - healthCheck: HealthCheck.ec2({ grace: Duration.seconds(90) }), + blockDevices: [{deviceName: '/dev/xvda', volume: BlockDeviceVolume.ebs(50)}], // GB + healthCheck: HealthCheck.ec2({grace: Duration.seconds(90)}), machineImage: props && props.ami ? MachineImage.fromSsmParameter(props.ami) : MachineImage.latestAmazonLinux2(), @@ -148,7 +148,6 @@ export class OpenSearchMetricsNginxReadonly extends Stack { } - private buildOpenSearchDashboardConf(nginxProps: NginxProps): string { return `'# See for reference template for opensearchdashboard: resolver 10.0.0.2 ipv6=off; @@ -206,7 +205,17 @@ export class OpenSearchMetricsNginxReadonly extends Stack { 'sudo yum install docker -y', 'sudo systemctl enable docker', 'sudo systemctl start docker', - `docker run --rm -tid -v ~/.aws:/root/.aws -p 8081:8080 --log-opt max-size=50m --log-opt max-file=5 public.ecr.aws/aws-observability/aws-sigv4-proxy:1.8 -v --name es --region ${nginxProps.region}` + `docker run --rm -tid -v ~/.aws:/root/.aws \ +-p 8081:8080 \ +--log-driver=awslogs \ +--log-opt awslogs-group=OpenSearchMetrics/aws-sigv4-proxy.log \ +--log-opt awslogs-create-group=true \ +--log-opt awslogs-region=${nginxProps.region} \ +--log-opt awslogs-multiline-pattern='(INFO|DEBU|ERRO)' \ +--log-opt tag='{{ with split .ImageName ":" }}{{join . "_"}}{{end}}-{{.ID}}' \ +public.ecr.aws/aws-observability/aws-sigv4-proxy:1.8 \ +-v --name es --region ${nginxProps.region} +` ]; } @@ -230,6 +239,20 @@ export class OpenSearchMetricsNginxReadonly extends Stack { ], resources: [domainArn] })); + + role.addToPolicy(new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:DescribeLogStreams" + ], + resources: [ + `arn:aws:logs:${Project.REGION}:${Project.AWS_ACCOUNT}:log-group:OpenSearchMetrics/aws-sigv4-proxy.log:*` + ] + })); + return role; } } diff --git a/infrastructure/lib/stacks/secrets.ts b/infrastructure/lib/stacks/secrets.ts index 84cd497..7d6d473 100644 --- a/infrastructure/lib/stacks/secrets.ts +++ b/infrastructure/lib/stacks/secrets.ts @@ -6,7 +6,7 @@ export interface SecretProps { readonly secretName: string } -export class OpenSearchMetricsSecrets extends Stack { +export class OpenSearchMetricsSecretsStack extends Stack { readonly secret: Secret; constructor(scope: Construct, id: string, props: SecretProps ) { diff --git a/infrastructure/package-lock.json b/infrastructure/package-lock.json index 88b12f8..15a063a 100644 --- a/infrastructure/package-lock.json +++ b/infrastructure/package-lock.json @@ -3964,6 +3964,7 @@ "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, "bin": { "semver": "bin/semver.js" } diff --git a/infrastructure/test/monitoring-stack.test.ts b/infrastructure/test/monitoring-stack.test.ts index ed39220..5cb22f6 100644 --- a/infrastructure/test/monitoring-stack.test.ts +++ b/infrastructure/test/monitoring-stack.test.ts @@ -6,7 +6,7 @@ import {OpenSearchDomainStack} from "../lib/stacks/opensearch"; import {VpcStack} from "../lib/stacks/vpc"; import {ArnPrincipal} from "aws-cdk-lib/aws-iam"; import {OpenSearchMetricsMonitoringStack} from "../lib/stacks/monitoringDashboard"; -import {OpenSearchMetricsSecrets} from "../lib/stacks/secrets"; +import {OpenSearchMetricsSecretsStack} from "../lib/stacks/secrets"; test('Monitoring Stack Test', () => { const app = new App(); @@ -27,7 +27,7 @@ test('Monitoring Stack Test', () => { vpcStack: vpcStack, lambdaPackage: Project.LAMBDA_PACKAGE }); - const openSearchMetricsSecretsStack = new OpenSearchMetricsSecrets(app, "OpenSearchMetrics-Secrets", { + const openSearchMetricsSecretsStack = new OpenSearchMetricsSecretsStack(app, "OpenSearchMetrics-Secrets", { secretName: 'metrics-creds' }); const openSearchMetricsMonitoringStack = new OpenSearchMetricsMonitoringStack(app, "OpenSearchMetrics-Monitoring", { @@ -211,7 +211,7 @@ test('Monitoring Stack Test', () => { ], "AlarmDescription": "Detect Canary failure", "AlarmName": "Canary_failed_MetricsWorkflow", - "ComparisonOperator": "LessThanThreshold", + "ComparisonOperator": "LessThanOrEqualToThreshold", "DatapointsToAlarm": 1, "Dimensions": [ { @@ -224,9 +224,9 @@ test('Monitoring Stack Test', () => { "EvaluationPeriods": 1, "MetricName": "SuccessPercent", "Namespace": "CloudWatchSynthetics", - "Period": 300, + "Period": 900, "Statistic": "Average", - "Threshold": 50, + "Threshold": 0, "TreatMissingData": "notBreaching" }); }); diff --git a/infrastructure/test/nginx.test.ts b/infrastructure/test/nginx.test.ts index 6c74c14..c54691d 100644 --- a/infrastructure/test/nginx.test.ts +++ b/infrastructure/test/nginx.test.ts @@ -79,148 +79,51 @@ test('OpenSearchMetricsNginxReadonly Stack Test', () => { } ], }); -}); - -test('OpenSearchMetricsNginxCognito Test', () => { - const app = new App(); - const openSearchDomainStack = new OpenSearchDomainStack(app, 'Test-OpenSearchHealth-OpenSearch', { - region: "us-east-1", - account: "test-account", - vpcStack: new VpcStack(app, 'OpenSearchHealth-VPC', {}), - enableNginxCognito: true, - jenkinsAccess: { - jenkinsAccountRoles: [ - new ArnPrincipal(Project.JENKINS_MASTER_ROLE), - new ArnPrincipal(Project.JENKINS_AGENT_ROLE) - ] - } - }); - const openSearchDomainStackTemplate = Template.fromStack(openSearchDomainStack); - openSearchDomainStackTemplate.resourceCountIs('AWS::Route53::RecordSet', 1); - openSearchDomainStackTemplate.hasResourceProperties('AWS::Route53::RecordSet', { - "Name": `${Project.METRICS_COGNITO_HOSTED_ZONE}.`, - "Type": "A" - }); - - openSearchDomainStackTemplate.resourceCountIs('AWS::AutoScaling::LaunchConfiguration', 1); - openSearchDomainStackTemplate.resourceCountIs('AWS::EC2::SecurityGroup', 2); - openSearchDomainStackTemplate.hasResourceProperties('AWS::EC2::SecurityGroup', { - "SecurityGroupEgress": [ - { - "CidrIp": "0.0.0.0/0", - "Description": "Allow all outbound traffic by default", - "IpProtocol": "-1" - } - ] - }); - openSearchDomainStackTemplate.hasResourceProperties('AWS::EC2::SecurityGroup', { - "SecurityGroupIngress": [ - { - "CidrIp": "0.0.0.0/0", - "Description": "Allow from anyone on port 443", - "FromPort": 443, - "IpProtocol": "tcp", - "ToPort": 443 - } - ] - }); - openSearchDomainStackTemplate.hasResourceProperties('AWS::IAM::Role', { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "ManagedPolicyArns": [ + template.resourceCountIs('AWS::IAM::Policy', 1); + template.hasResourceProperties('AWS::IAM::Policy', { + "PolicyDocument": { + "Statement": [ { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/AmazonSSMManagedInstanceCore" + "Action": [ + "es:Describe*", + "es:List*", + "es:Get*", + "es:ESHttpGet", + "es:ESHttpPost", + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:aws:es:::domain/", + { + "Fn::ImportValue": "OpenSearchHealth-OpenSearch:ExportsOutputRefOpenSearchHealthDomainD942887BFEBF5289" + }, + "/*" + ] ] - ] - } - ], - "RoleName": "OpenSearchCognitoUserAccess" - }) - openSearchDomainStackTemplate.resourceCountIs('AWS::ElasticLoadBalancingV2::LoadBalancer', 1); - openSearchDomainStackTemplate.hasResourceProperties('AWS::ElasticLoadBalancingV2::LoadBalancer', { - "LoadBalancerAttributes": [ - { - "Key": "deletion_protection.enabled", - "Value": "false" - } - ], - "Scheme": "internet-facing", - "Type": "application" - }); - openSearchDomainStackTemplate.resourceCountIs('AWS::ElasticLoadBalancingV2::Listener', 1); - openSearchDomainStackTemplate.hasResourceProperties('AWS::ElasticLoadBalancingV2::Listener', { - "Port": 443, - "Protocol": "HTTPS" - }); - openSearchDomainStackTemplate.resourceCountIs('AWS::ElasticLoadBalancingV2::TargetGroup', 1); - openSearchDomainStackTemplate.hasResourceProperties('AWS::ElasticLoadBalancingV2::TargetGroup', { - "HealthCheckPath": "/", - "HealthCheckPort": "80", - "Port": 443, - "Protocol": "HTTPS", - "TargetGroupAttributes": [ - { - "Key": "stickiness.enabled", - "Value": "false" - } - ], - "TargetType": "instance", - "VpcId": { - "Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputRefOpenSearchHealthVpcB885AABED860B3EB" - } - }); - openSearchDomainStackTemplate.resourceCountIs('AWS::AutoScaling::AutoScalingGroup', 1); - openSearchDomainStackTemplate.hasResourceProperties('AWS::AutoScaling::AutoScalingGroup', { - "DesiredCapacity": "1", - "HealthCheckGracePeriod": 90, - "HealthCheckType": "EC2", - "LaunchConfigurationName": { - "Ref": "OpenSearchMetricsNginxOpenSearchMetricsCognitoMetricsProxyAsgLaunchConfig8D060946" - }, - "MaxSize": "1", - "MinSize": "1", - "Tags": [ - { - "Key": "name", - "PropagateAtLaunch": true, - "Value": "OpenSearchMetricsCognito-NginxProxyHost" + } }, { - "Key": "Name", - "PropagateAtLaunch": true, - "Value": "OpenSearchMetricsCognito" - } - ], - "TargetGroupARNs": [ - { - "Ref": "OpenSearchMetricsNginxOpenSearchMetricsCognitoNginxProxyAlbOpenSearchMetricsCognitoNginxProxyAlbListenerOpenSearchMetricsCognitoNginxProxyAlbTargetGroup8E449B4A" + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:DescribeLogStreams" + ], + "Effect": "Allow", + "Resource": "arn:aws:logs:::log-group:OpenSearchMetrics/aws-sigv4-proxy.log:*" } ], - "VPCZoneIdentifier": [ - { - "Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputRefOpenSearchHealthVpcPrivateSubnet1Subnet529349B600974078" - }, - { - "Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputRefOpenSearchHealthVpcPrivateSubnet2SubnetBA599EDB2BEEEA30" - } - ] + "Version": "2012-10-17" + }, + "PolicyName": "OpenSearchMetricsReadonlyNginxProxyRoleDefaultPolicy8EDC749D", + "Roles": [ + { + "Ref": "OpenSearchMetricsReadonlyNginxProxyRoleE26CC937" + } + ] }); - }); + diff --git a/infrastructure/test/opensearch-stack.test.ts b/infrastructure/test/opensearch-stack.test.ts index 698d44a..4899718 100644 --- a/infrastructure/test/opensearch-stack.test.ts +++ b/infrastructure/test/opensearch-stack.test.ts @@ -7,10 +7,10 @@ import {VpcStack} from "../lib/stacks/vpc"; test('OpenSearchDomain Stack Test', () => { const app = new App(); - const openSearchDomainStack = new OpenSearchDomainStack(app, 'Test-OpenSearchHealth-OpenSearch', { + const openSearchDomainStack = new OpenSearchDomainStack(app, 'OpenSearchHealth-OpenSearch', { region: "us-east-1", account: "test-account", - vpcStack: new VpcStack(app, 'Test-OpenSearchHealth-VPC', {}), + vpcStack: new VpcStack(app, 'OpenSearchHealth-VPC', {}), enableNginxCognito: true, jenkinsAccess: { jenkinsAccountRoles: [ @@ -106,4 +106,131 @@ test('OpenSearchDomain Stack Test', () => { }, "EngineVersion": "OpenSearch_2.13", }); + + openSearchDomainStackTemplate.resourceCountIs('AWS::Route53::RecordSet', 1); + openSearchDomainStackTemplate.hasResourceProperties('AWS::Route53::RecordSet', { + "Name": `${Project.METRICS_COGNITO_HOSTED_ZONE}.`, + "Type": "A" + }); + + openSearchDomainStackTemplate.resourceCountIs('AWS::AutoScaling::LaunchConfiguration', 1); + openSearchDomainStackTemplate.resourceCountIs('AWS::EC2::SecurityGroup', 2); + openSearchDomainStackTemplate.hasResourceProperties('AWS::EC2::SecurityGroup', { + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ] + }); + openSearchDomainStackTemplate.hasResourceProperties('AWS::EC2::SecurityGroup', { + "SecurityGroupIngress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow from anyone on port 443", + "FromPort": 443, + "IpProtocol": "tcp", + "ToPort": 443 + } + ] + }); + openSearchDomainStackTemplate.hasResourceProperties('AWS::IAM::Role', { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/AmazonSSMManagedInstanceCore" + ] + ] + } + ], + "RoleName": "OpenSearchCognitoUserAccess" + }) + openSearchDomainStackTemplate.resourceCountIs('AWS::ElasticLoadBalancingV2::LoadBalancer', 1); + openSearchDomainStackTemplate.hasResourceProperties('AWS::ElasticLoadBalancingV2::LoadBalancer', { + "LoadBalancerAttributes": [ + { + "Key": "deletion_protection.enabled", + "Value": "false" + } + ], + "Scheme": "internet-facing", + "Type": "application" + }); + openSearchDomainStackTemplate.resourceCountIs('AWS::ElasticLoadBalancingV2::Listener', 1); + openSearchDomainStackTemplate.hasResourceProperties('AWS::ElasticLoadBalancingV2::Listener', { + "Port": 443, + "Protocol": "HTTPS" + }); + openSearchDomainStackTemplate.resourceCountIs('AWS::ElasticLoadBalancingV2::TargetGroup', 1); + openSearchDomainStackTemplate.hasResourceProperties('AWS::ElasticLoadBalancingV2::TargetGroup', { + "HealthCheckPath": "/", + "HealthCheckPort": "80", + "Port": 443, + "Protocol": "HTTPS", + "TargetGroupAttributes": [ + { + "Key": "stickiness.enabled", + "Value": "false" + } + ], + "TargetType": "instance", + "VpcId": { + "Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputRefOpenSearchHealthVpcB885AABED860B3EB" + } + }); + openSearchDomainStackTemplate.resourceCountIs('AWS::AutoScaling::AutoScalingGroup', 1); + openSearchDomainStackTemplate.hasResourceProperties('AWS::AutoScaling::AutoScalingGroup', { + "DesiredCapacity": "1", + "HealthCheckGracePeriod": 90, + "HealthCheckType": "EC2", + "LaunchConfigurationName": { + "Ref": "OpenSearchMetricsNginxOpenSearchMetricsCognitoMetricsProxyAsgLaunchConfig8D060946" + }, + "MaxSize": "1", + "MinSize": "1", + "Tags": [ + { + "Key": "name", + "PropagateAtLaunch": true, + "Value": "OpenSearchMetricsCognito-NginxProxyHost" + }, + { + "Key": "Name", + "PropagateAtLaunch": true, + "Value": "OpenSearchMetricsCognito" + } + ], + "TargetGroupARNs": [ + { + "Ref": "OpenSearchMetricsNginxOpenSearchMetricsCognitoNginxProxyAlbOpenSearchMetricsCognitoNginxProxyAlbListenerOpenSearchMetricsCognitoNginxProxyAlbTargetGroup8E449B4A" + } + ], + "VPCZoneIdentifier": [ + { + "Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputRefOpenSearchHealthVpcPrivateSubnet1Subnet529349B600974078" + }, + { + "Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputRefOpenSearchHealthVpcPrivateSubnet2SubnetBA599EDB2BEEEA30" + } + ] + }); }); diff --git a/infrastructure/test/secrets-stack.test.ts b/infrastructure/test/secrets-stack.test.ts index edbd6f1..3334c0e 100644 --- a/infrastructure/test/secrets-stack.test.ts +++ b/infrastructure/test/secrets-stack.test.ts @@ -1,10 +1,10 @@ import {App} from "aws-cdk-lib"; import {Template} from "aws-cdk-lib/assertions"; -import {OpenSearchMetricsSecrets} from "../lib/stacks/secrets"; +import {OpenSearchMetricsSecretsStack} from "../lib/stacks/secrets"; test('Secrets Stack Test', () => { const app = new App(); - const openSearchMetricsSecretsStack = new OpenSearchMetricsSecrets(app, "OpenSearchMetrics-Secrets", { + const openSearchMetricsSecretsStack = new OpenSearchMetricsSecretsStack(app, "OpenSearchMetrics-Secrets", { secretName: 'metrics-creds' }); const template = Template.fromStack(openSearchMetricsSecretsStack);