Added docker pushing logs to cloudwatch functionality

Signed-off-by: Brandon Shien <[email protected]>
opensearch-project · Jun 28, 2024 · 3c1ce5c · 3c1ce5c
1 parent 50d617b
commit 3c1ce5c
Show file tree

Hide file tree

Showing 9 changed files with 215 additions and 158 deletions.
diff --git a/infrastructure/lib/constructs/canarySns.ts b/infrastructure/lib/constructs/canarySns.ts
@@ -4,6 +4,7 @@ import {Construct} from "constructs";
 import {Alarm} from "aws-cdk-lib/aws-cloudwatch";
 import { Canary } from 'aws-cdk-lib/aws-synthetics';
 import * as cloudwatch from "aws-cdk-lib/aws-cloudwatch";
+import {Duration} from "aws-cdk-lib";
 
 interface canarySnsProps extends SnsMonitorsProps {
     readonly canaryAlarms: Array<{ alertName: string, canary: Canary }>;
@@ -24,10 +25,12 @@ export class canarySns extends SnsMonitors {
 
     private canaryFailed(alertName: string, canary: Canary): [Alarm, string] {
         const alarmObject = new cloudwatch.Alarm(this, `error_alarm_${alertName}`, {
-            metric: canary.metricSuccessPercent(),
-            threshold: 50,
+            metric: canary.metricSuccessPercent({
+                period: Duration.minutes(15)
+            }),
+            threshold: 0,
             evaluationPeriods: 1,
-            comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD,
+            comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD,
             datapointsToAlarm: 1,
             treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
             alarmDescription: "Detect Canary failure",

diff --git a/infrastructure/lib/infrastructure-stack.ts b/infrastructure/lib/infrastructure-stack.ts
@@ -10,7 +10,7 @@ import {ArnPrincipal} from "aws-cdk-lib/aws-iam";
 import {OpenSearchWAF} from "./stacks/waf";
 import {OpenSearchMetricsNginxCognito} from "./constructs/opensearchNginxProxyCognito";
 import {OpenSearchMetricsMonitoringStack} from "./stacks/monitoringDashboard";
-import {OpenSearchMetricsSecrets} from "./stacks/secrets";
+import {OpenSearchMetricsSecretsStack} from "./stacks/secrets";
 
 // import * as sqs from 'aws-cdk-lib/aws-sqs';
 export class InfrastructureStack extends Stack {
@@ -43,7 +43,7 @@ export class InfrastructureStack extends Stack {
 
     // Create Secrets Manager
 
-    const openSearchMetricsSecretsStack = new OpenSearchMetricsSecrets(app, "OpenSearchMetrics-Secrets", {
+    const openSearchMetricsSecretsStack = new OpenSearchMetricsSecretsStack(app, "OpenSearchMetrics-Secrets", {
       secretName: 'metrics-creds'
     });
 

diff --git a/infrastructure/lib/stacks/opensearchNginxProxyReadonly.ts b/infrastructure/lib/stacks/opensearchNginxProxyReadonly.ts
@@ -59,15 +59,15 @@ export class OpenSearchMetricsNginxReadonly extends Stack {
     readonly asg: AutoScalingGroup;
 
     constructor(scope: Construct, id: string, props: NginxProps) {
-        const { vpc, securityGroup } = props;
+        const {vpc, securityGroup} = props;
 
         super(scope, id);
 
         const instanceRole = this.createNginxReadonlyInstanceRole(props);
-            this.asg = new AutoScalingGroup(this, 'OpenSearchMetricsReadonly-MetricsProxyAsg', {
+        this.asg = new AutoScalingGroup(this, 'OpenSearchMetricsReadonly-MetricsProxyAsg', {
             instanceType: InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE),
-            blockDevices: [{ deviceName: '/dev/xvda', volume: BlockDeviceVolume.ebs(50) }], // GB
-            healthCheck: HealthCheck.ec2({ grace: Duration.seconds(90) }),
+            blockDevices: [{deviceName: '/dev/xvda', volume: BlockDeviceVolume.ebs(50)}], // GB
+            healthCheck: HealthCheck.ec2({grace: Duration.seconds(90)}),
             machineImage: props && props.ami ?
                 MachineImage.fromSsmParameter(props.ami) :
                 MachineImage.latestAmazonLinux2(),
@@ -148,7 +148,6 @@ export class OpenSearchMetricsNginxReadonly extends Stack {
     }
 
 
-
     private buildOpenSearchDashboardConf(nginxProps: NginxProps): string {
         return `'# See for reference template for opensearchdashboard:
             resolver 10.0.0.2 ipv6=off;
@@ -206,7 +205,17 @@ export class OpenSearchMetricsNginxReadonly extends Stack {
             'sudo yum install docker -y',
             'sudo systemctl enable docker',
             'sudo systemctl start docker',
-            `docker run --rm -tid -v ~/.aws:/root/.aws -p 8081:8080 --log-opt max-size=50m --log-opt max-file=5 public.ecr.aws/aws-observability/aws-sigv4-proxy:1.8 -v --name es --region ${nginxProps.region}`
+            `docker run --rm -tid -v ~/.aws:/root/.aws \
+-p 8081:8080 \
+--log-driver=awslogs \
+--log-opt awslogs-group=OpenSearchMetrics/aws-sigv4-proxy.log \
+--log-opt awslogs-create-group=true \
+--log-opt awslogs-region=${nginxProps.region} \
+--log-opt awslogs-multiline-pattern='(INFO|DEBU|ERRO)' \
+--log-opt tag='{{ with split .ImageName ":" }}{{join . "_"}}{{end}}-{{.ID}}' \
+public.ecr.aws/aws-observability/aws-sigv4-proxy:1.8 \
+-v --name es --region ${nginxProps.region}
+`
         ];
     }
 
@@ -230,6 +239,20 @@ export class OpenSearchMetricsNginxReadonly extends Stack {
             ],
             resources: [domainArn]
         }));
+
+        role.addToPolicy(new PolicyStatement({
+            effect: Effect.ALLOW,
+            actions: [
+                "logs:CreateLogGroup",
+                "logs:CreateLogStream",
+                "logs:PutLogEvents",
+                "logs:DescribeLogStreams"
+            ],
+            resources: [
+                `arn:aws:logs:${Project.REGION}:${Project.AWS_ACCOUNT}:log-group:OpenSearchMetrics/aws-sigv4-proxy.log:*`
+            ]
+        }));
+
         return role;
     }
 }
diff --git a/infrastructure/lib/stacks/secrets.ts b/infrastructure/lib/stacks/secrets.ts
@@ -6,7 +6,7 @@ export interface SecretProps {
     readonly secretName: string
 }
 
-export class OpenSearchMetricsSecrets extends Stack {
+export class OpenSearchMetricsSecretsStack extends Stack {
     readonly secret: Secret;
 
     constructor(scope: Construct, id: string, props: SecretProps ) {

diff --git a/infrastructure/package-lock.json b/infrastructure/package-lock.json
diff --git a/infrastructure/test/monitoring-stack.test.ts b/infrastructure/test/monitoring-stack.test.ts
@@ -6,7 +6,7 @@ import {OpenSearchDomainStack} from "../lib/stacks/opensearch";
 import {VpcStack} from "../lib/stacks/vpc";
 import {ArnPrincipal} from "aws-cdk-lib/aws-iam";
 import {OpenSearchMetricsMonitoringStack} from "../lib/stacks/monitoringDashboard";
-import {OpenSearchMetricsSecrets} from "../lib/stacks/secrets";
+import {OpenSearchMetricsSecretsStack} from "../lib/stacks/secrets";
 
 test('Monitoring Stack Test', () => {
     const app = new App();
@@ -27,7 +27,7 @@ test('Monitoring Stack Test', () => {
         vpcStack: vpcStack,
         lambdaPackage: Project.LAMBDA_PACKAGE
     });
-    const openSearchMetricsSecretsStack = new OpenSearchMetricsSecrets(app, "OpenSearchMetrics-Secrets", {
+    const openSearchMetricsSecretsStack = new OpenSearchMetricsSecretsStack(app, "OpenSearchMetrics-Secrets", {
         secretName: 'metrics-creds'
     });
     const openSearchMetricsMonitoringStack = new OpenSearchMetricsMonitoringStack(app, "OpenSearchMetrics-Monitoring", {
@@ -211,7 +211,7 @@ test('Monitoring Stack Test', () => {
         ],
         "AlarmDescription": "Detect Canary failure",
         "AlarmName": "Canary_failed_MetricsWorkflow",
-        "ComparisonOperator": "LessThanThreshold",
+        "ComparisonOperator": "LessThanOrEqualToThreshold",
         "DatapointsToAlarm": 1,
         "Dimensions": [
             {
@@ -224,9 +224,9 @@ test('Monitoring Stack Test', () => {
         "EvaluationPeriods": 1,
         "MetricName": "SuccessPercent",
         "Namespace": "CloudWatchSynthetics",
-        "Period": 300,
+        "Period": 900,
         "Statistic": "Average",
-        "Threshold": 50,
+        "Threshold": 0,
         "TreatMissingData": "notBreaching"
     });
 });
diff --git a/infrastructure/test/nginx.test.ts b/infrastructure/test/nginx.test.ts
@@ -79,148 +79,51 @@ test('OpenSearchMetricsNginxReadonly Stack Test', () => {
             }
         ],
     });
-});
-
-test('OpenSearchMetricsNginxCognito Test', () => {
-    const app = new App();
-    const openSearchDomainStack = new OpenSearchDomainStack(app, 'Test-OpenSearchHealth-OpenSearch', {
-        region: "us-east-1",
-        account: "test-account",
-        vpcStack: new VpcStack(app, 'OpenSearchHealth-VPC', {}),
-        enableNginxCognito: true,
-        jenkinsAccess: {
-            jenkinsAccountRoles:  [
-                new ArnPrincipal(Project.JENKINS_MASTER_ROLE),
-                new ArnPrincipal(Project.JENKINS_AGENT_ROLE)
-            ]
-        }
-    });
-    const openSearchDomainStackTemplate = Template.fromStack(openSearchDomainStack);
-    openSearchDomainStackTemplate.resourceCountIs('AWS::Route53::RecordSet', 1);
-    openSearchDomainStackTemplate.hasResourceProperties('AWS::Route53::RecordSet', {
-        "Name": `${Project.METRICS_COGNITO_HOSTED_ZONE}.`,
-        "Type": "A"
-    });
-
-    openSearchDomainStackTemplate.resourceCountIs('AWS::AutoScaling::LaunchConfiguration', 1);
-    openSearchDomainStackTemplate.resourceCountIs('AWS::EC2::SecurityGroup', 2);
-    openSearchDomainStackTemplate.hasResourceProperties('AWS::EC2::SecurityGroup', {
-        "SecurityGroupEgress": [
-            {
-                "CidrIp": "0.0.0.0/0",
-                "Description": "Allow all outbound traffic by default",
-                "IpProtocol": "-1"
-            }
-        ]
-    });
-    openSearchDomainStackTemplate.hasResourceProperties('AWS::EC2::SecurityGroup', {
-        "SecurityGroupIngress": [
-            {
-                "CidrIp": "0.0.0.0/0",
-                "Description": "Allow from anyone on port 443",
-                "FromPort": 443,
-                "IpProtocol": "tcp",
-                "ToPort": 443
-            }
-        ]
-    });
-    openSearchDomainStackTemplate.hasResourceProperties('AWS::IAM::Role', {
-            "AssumeRolePolicyDocument": {
-                "Statement": [
-                    {
-                        "Action": "sts:AssumeRole",
-                        "Effect": "Allow",
-                        "Principal": {
-                            "Service": "ec2.amazonaws.com"
-                        }
-                    }
-                ],
-                "Version": "2012-10-17"
-            },
-            "ManagedPolicyArns": [
+    template.resourceCountIs('AWS::IAM::Policy', 1);
+    template.hasResourceProperties('AWS::IAM::Policy', {
+        "PolicyDocument": {
+            "Statement": [
                 {
-                    "Fn::Join": [
-                        "",
-                        [
-                            "arn:",
-                            {
-                                "Ref": "AWS::Partition"
-                            },
-                            ":iam::aws:policy/AmazonSSMManagedInstanceCore"
+                    "Action": [
+                        "es:Describe*",
+                        "es:List*",
+                        "es:Get*",
+                        "es:ESHttpGet",
+                        "es:ESHttpPost",
+                    ],
+                    "Effect": "Allow",
+                    "Resource": {
+                        "Fn::Join": [
+                            "",
+                            [
+                                "arn:aws:es:::domain/",
+                                {
+                                    "Fn::ImportValue": "OpenSearchHealth-OpenSearch:ExportsOutputRefOpenSearchHealthDomainD942887BFEBF5289"
+                                },
+                                "/*"
+                            ]
                         ]
-                    ]
-                }
-            ],
-            "RoleName": "OpenSearchCognitoUserAccess"
-    })
-    openSearchDomainStackTemplate.resourceCountIs('AWS::ElasticLoadBalancingV2::LoadBalancer', 1);
-    openSearchDomainStackTemplate.hasResourceProperties('AWS::ElasticLoadBalancingV2::LoadBalancer', {
-        "LoadBalancerAttributes": [
-            {
-                "Key": "deletion_protection.enabled",
-                "Value": "false"
-            }
-        ],
-        "Scheme": "internet-facing",
-        "Type": "application"
-    });
-    openSearchDomainStackTemplate.resourceCountIs('AWS::ElasticLoadBalancingV2::Listener', 1);
-    openSearchDomainStackTemplate.hasResourceProperties('AWS::ElasticLoadBalancingV2::Listener', {
-        "Port": 443,
-        "Protocol": "HTTPS"
-    });
-    openSearchDomainStackTemplate.resourceCountIs('AWS::ElasticLoadBalancingV2::TargetGroup', 1);
-    openSearchDomainStackTemplate.hasResourceProperties('AWS::ElasticLoadBalancingV2::TargetGroup', {
-        "HealthCheckPath": "/",
-        "HealthCheckPort": "80",
-        "Port": 443,
-        "Protocol": "HTTPS",
-        "TargetGroupAttributes": [
-            {
-                "Key": "stickiness.enabled",
-                "Value": "false"
-            }
-        ],
-        "TargetType": "instance",
-        "VpcId": {
-            "Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputRefOpenSearchHealthVpcB885AABED860B3EB"
-        }
-    });
-    openSearchDomainStackTemplate.resourceCountIs('AWS::AutoScaling::AutoScalingGroup', 1);
-    openSearchDomainStackTemplate.hasResourceProperties('AWS::AutoScaling::AutoScalingGroup', {
-            "DesiredCapacity": "1",
-            "HealthCheckGracePeriod": 90,
-            "HealthCheckType": "EC2",
-            "LaunchConfigurationName": {
-                "Ref": "OpenSearchMetricsNginxOpenSearchMetricsCognitoMetricsProxyAsgLaunchConfig8D060946"
-            },
-            "MaxSize": "1",
-            "MinSize": "1",
-            "Tags": [
-                {
-                    "Key": "name",
-                    "PropagateAtLaunch": true,
-                    "Value": "OpenSearchMetricsCognito-NginxProxyHost"
+                    }
                 },
                 {
-                    "Key": "Name",
-                    "PropagateAtLaunch": true,
-                    "Value": "OpenSearchMetricsCognito"
-                }
-            ],
-            "TargetGroupARNs": [
-                {
-                    "Ref": "OpenSearchMetricsNginxOpenSearchMetricsCognitoNginxProxyAlbOpenSearchMetricsCognitoNginxProxyAlbListenerOpenSearchMetricsCognitoNginxProxyAlbTargetGroup8E449B4A"
+                    "Action": [
+                        "logs:CreateLogGroup",
+                        "logs:CreateLogStream",
+                        "logs:PutLogEvents",
+                        "logs:DescribeLogStreams"
+                    ],
+                    "Effect": "Allow",
+                    "Resource": "arn:aws:logs:::log-group:OpenSearchMetrics/aws-sigv4-proxy.log:*"
                 }
             ],
-            "VPCZoneIdentifier": [
-                {
-                    "Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputRefOpenSearchHealthVpcPrivateSubnet1Subnet529349B600974078"
-                },
-                {
-                    "Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputRefOpenSearchHealthVpcPrivateSubnet2SubnetBA599EDB2BEEEA30"
-                }
-            ]
+            "Version": "2012-10-17"
+        },
+        "PolicyName": "OpenSearchMetricsReadonlyNginxProxyRoleDefaultPolicy8EDC749D",
+        "Roles": [
+            {
+                "Ref": "OpenSearchMetricsReadonlyNginxProxyRoleE26CC937"
+            }
+        ]
     });
-
 });
+