Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added docker pushing logs to cloudwatch functionality #50

Merged
merged 1 commit into from
Jul 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions infrastructure/lib/constructs/canarySns.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {Construct} from "constructs";
import {Alarm} from "aws-cdk-lib/aws-cloudwatch";
import { Canary } from 'aws-cdk-lib/aws-synthetics';
import * as cloudwatch from "aws-cdk-lib/aws-cloudwatch";
import {Duration} from "aws-cdk-lib";

interface canarySnsProps extends SnsMonitorsProps {
readonly canaryAlarms: Array<{ alertName: string, canary: Canary }>;
Expand All @@ -24,10 +25,12 @@ export class canarySns extends SnsMonitors {

private canaryFailed(alertName: string, canary: Canary): [Alarm, string] {
const alarmObject = new cloudwatch.Alarm(this, `error_alarm_${alertName}`, {
metric: canary.metricSuccessPercent(),
threshold: 50,
metric: canary.metricSuccessPercent({
period: Duration.minutes(15)
}),
threshold: 0,
evaluationPeriods: 1,
comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD,
comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD,
datapointsToAlarm: 1,
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
alarmDescription: "Detect Canary failure",
Expand Down
4 changes: 2 additions & 2 deletions infrastructure/lib/infrastructure-stack.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import {ArnPrincipal} from "aws-cdk-lib/aws-iam";
import {OpenSearchWAF} from "./stacks/waf";
import {OpenSearchMetricsNginxCognito} from "./constructs/opensearchNginxProxyCognito";
import {OpenSearchMetricsMonitoringStack} from "./stacks/monitoringDashboard";
import {OpenSearchMetricsSecrets} from "./stacks/secrets";
import {OpenSearchMetricsSecretsStack} from "./stacks/secrets";

// import * as sqs from 'aws-cdk-lib/aws-sqs';
export class InfrastructureStack extends Stack {
Expand Down Expand Up @@ -43,7 +43,7 @@ export class InfrastructureStack extends Stack {

// Create Secrets Manager

const openSearchMetricsSecretsStack = new OpenSearchMetricsSecrets(app, "OpenSearchMetrics-Secrets", {
const openSearchMetricsSecretsStack = new OpenSearchMetricsSecretsStack(app, "OpenSearchMetrics-Secrets", {
secretName: 'metrics-creds'
});

Expand Down
2 changes: 1 addition & 1 deletion infrastructure/lib/stacks/opensearch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -196,4 +196,4 @@ export class OpenSearchDomainStack extends Stack {
});
}
}
}
}
35 changes: 29 additions & 6 deletions infrastructure/lib/stacks/opensearchNginxProxyReadonly.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,15 +59,15 @@ export class OpenSearchMetricsNginxReadonly extends Stack {
readonly asg: AutoScalingGroup;

constructor(scope: Construct, id: string, props: NginxProps) {
const { vpc, securityGroup } = props;
const {vpc, securityGroup} = props;

super(scope, id);

const instanceRole = this.createNginxReadonlyInstanceRole(props);
this.asg = new AutoScalingGroup(this, 'OpenSearchMetricsReadonly-MetricsProxyAsg', {
this.asg = new AutoScalingGroup(this, 'OpenSearchMetricsReadonly-MetricsProxyAsg', {
instanceType: InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE),
blockDevices: [{ deviceName: '/dev/xvda', volume: BlockDeviceVolume.ebs(50) }], // GB
healthCheck: HealthCheck.ec2({ grace: Duration.seconds(90) }),
blockDevices: [{deviceName: '/dev/xvda', volume: BlockDeviceVolume.ebs(50)}], // GB
healthCheck: HealthCheck.ec2({grace: Duration.seconds(90)}),
machineImage: props && props.ami ?
MachineImage.fromSsmParameter(props.ami) :
MachineImage.latestAmazonLinux2(),
Expand Down Expand Up @@ -148,7 +148,6 @@ export class OpenSearchMetricsNginxReadonly extends Stack {
}



private buildOpenSearchDashboardConf(nginxProps: NginxProps): string {
return `'# See for reference template for opensearchdashboard:
resolver 10.0.0.2 ipv6=off;
Expand Down Expand Up @@ -206,7 +205,17 @@ export class OpenSearchMetricsNginxReadonly extends Stack {
'sudo yum install docker -y',
'sudo systemctl enable docker',
'sudo systemctl start docker',
`docker run --rm -tid -v ~/.aws:/root/.aws -p 8081:8080 --log-opt max-size=50m --log-opt max-file=5 public.ecr.aws/aws-observability/aws-sigv4-proxy:1.8 -v --name es --region ${nginxProps.region}`
`docker run --rm -tid -v ~/.aws:/root/.aws \
-p 8081:8080 \
--log-driver=awslogs \
--log-opt awslogs-group=OpenSearchMetrics/aws-sigv4-proxy.log \
--log-opt awslogs-create-group=true \
--log-opt awslogs-region=${nginxProps.region} \
--log-opt awslogs-multiline-pattern='(INFO|DEBU|ERRO)' \
--log-opt tag='{{ with split .ImageName ":" }}{{join . "_"}}{{end}}-{{.ID}}' \
public.ecr.aws/aws-observability/aws-sigv4-proxy:1.8 \
-v --name es --region ${nginxProps.region}
`
];
}

Expand All @@ -230,6 +239,20 @@ export class OpenSearchMetricsNginxReadonly extends Stack {
],
resources: [domainArn]
}));

role.addToPolicy(new PolicyStatement({
effect: Effect.ALLOW,
actions: [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents",
"logs:DescribeLogStreams"
],
resources: [
`arn:aws:logs:${Project.REGION}:${Project.AWS_ACCOUNT}:log-group:OpenSearchMetrics/aws-sigv4-proxy.log:*`
]
}));

return role;
}
}
2 changes: 1 addition & 1 deletion infrastructure/lib/stacks/secrets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ export interface SecretProps {
readonly secretName: string
}

export class OpenSearchMetricsSecrets extends Stack {
export class OpenSearchMetricsSecretsStack extends Stack {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The OpenSearchMetricsSecrets should be fine here right ? as it already extends stack do we need OpenSearchMetricsSecretsStack?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since the other stacks have the Stack suffix, I thought it would be better to keep that style

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

does this name change destroy the stack and recreate ?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tested on local, it didn't destroy the stack, I think that only happens if the id changes

readonly secret: Secret;

constructor(scope: Construct, id: string, props: SecretProps ) {
Expand Down
1 change: 1 addition & 0 deletions infrastructure/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 5 additions & 5 deletions infrastructure/test/monitoring-stack.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import {OpenSearchDomainStack} from "../lib/stacks/opensearch";
import {VpcStack} from "../lib/stacks/vpc";
import {ArnPrincipal} from "aws-cdk-lib/aws-iam";
import {OpenSearchMetricsMonitoringStack} from "../lib/stacks/monitoringDashboard";
import {OpenSearchMetricsSecrets} from "../lib/stacks/secrets";
import {OpenSearchMetricsSecretsStack} from "../lib/stacks/secrets";

test('Monitoring Stack Test', () => {
const app = new App();
Expand All @@ -27,7 +27,7 @@ test('Monitoring Stack Test', () => {
vpcStack: vpcStack,
lambdaPackage: Project.LAMBDA_PACKAGE
});
const openSearchMetricsSecretsStack = new OpenSearchMetricsSecrets(app, "OpenSearchMetrics-Secrets", {
const openSearchMetricsSecretsStack = new OpenSearchMetricsSecretsStack(app, "OpenSearchMetrics-Secrets", {
secretName: 'metrics-creds'
});
const openSearchMetricsMonitoringStack = new OpenSearchMetricsMonitoringStack(app, "OpenSearchMetrics-Monitoring", {
Expand Down Expand Up @@ -211,7 +211,7 @@ test('Monitoring Stack Test', () => {
],
"AlarmDescription": "Detect Canary failure",
"AlarmName": "Canary_failed_MetricsWorkflow",
"ComparisonOperator": "LessThanThreshold",
"ComparisonOperator": "LessThanOrEqualToThreshold",
"DatapointsToAlarm": 1,
"Dimensions": [
{
Expand All @@ -224,9 +224,9 @@ test('Monitoring Stack Test', () => {
"EvaluationPeriods": 1,
"MetricName": "SuccessPercent",
"Namespace": "CloudWatchSynthetics",
"Period": 300,
"Period": 900,
"Statistic": "Average",
"Threshold": 50,
"Threshold": 0,
"TreatMissingData": "notBreaching"
});
});
177 changes: 40 additions & 137 deletions infrastructure/test/nginx.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,148 +79,51 @@ test('OpenSearchMetricsNginxReadonly Stack Test', () => {
}
],
});
});

test('OpenSearchMetricsNginxCognito Test', () => {
const app = new App();
const openSearchDomainStack = new OpenSearchDomainStack(app, 'Test-OpenSearchHealth-OpenSearch', {
region: "us-east-1",
account: "test-account",
vpcStack: new VpcStack(app, 'OpenSearchHealth-VPC', {}),
enableNginxCognito: true,
jenkinsAccess: {
jenkinsAccountRoles: [
new ArnPrincipal(Project.JENKINS_MASTER_ROLE),
new ArnPrincipal(Project.JENKINS_AGENT_ROLE)
]
}
});
const openSearchDomainStackTemplate = Template.fromStack(openSearchDomainStack);
openSearchDomainStackTemplate.resourceCountIs('AWS::Route53::RecordSet', 1);
openSearchDomainStackTemplate.hasResourceProperties('AWS::Route53::RecordSet', {
"Name": `${Project.METRICS_COGNITO_HOSTED_ZONE}.`,
"Type": "A"
});

openSearchDomainStackTemplate.resourceCountIs('AWS::AutoScaling::LaunchConfiguration', 1);
openSearchDomainStackTemplate.resourceCountIs('AWS::EC2::SecurityGroup', 2);
openSearchDomainStackTemplate.hasResourceProperties('AWS::EC2::SecurityGroup', {
"SecurityGroupEgress": [
{
"CidrIp": "0.0.0.0/0",
"Description": "Allow all outbound traffic by default",
"IpProtocol": "-1"
}
]
});
openSearchDomainStackTemplate.hasResourceProperties('AWS::EC2::SecurityGroup', {
"SecurityGroupIngress": [
{
"CidrIp": "0.0.0.0/0",
"Description": "Allow from anyone on port 443",
"FromPort": 443,
"IpProtocol": "tcp",
"ToPort": 443
}
]
});
openSearchDomainStackTemplate.hasResourceProperties('AWS::IAM::Role', {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "ec2.amazonaws.com"
}
}
],
"Version": "2012-10-17"
},
"ManagedPolicyArns": [
template.resourceCountIs('AWS::IAM::Policy', 1);
template.hasResourceProperties('AWS::IAM::Policy', {
"PolicyDocument": {
"Statement": [
{
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":iam::aws:policy/AmazonSSMManagedInstanceCore"
"Action": [
"es:Describe*",
"es:List*",
"es:Get*",
"es:ESHttpGet",
"es:ESHttpPost",
],
"Effect": "Allow",
"Resource": {
"Fn::Join": [
"",
[
"arn:aws:es:::domain/",
{
"Fn::ImportValue": "OpenSearchHealth-OpenSearch:ExportsOutputRefOpenSearchHealthDomainD942887BFEBF5289"
},
"/*"
]
]
]
}
],
"RoleName": "OpenSearchCognitoUserAccess"
})
openSearchDomainStackTemplate.resourceCountIs('AWS::ElasticLoadBalancingV2::LoadBalancer', 1);
openSearchDomainStackTemplate.hasResourceProperties('AWS::ElasticLoadBalancingV2::LoadBalancer', {
"LoadBalancerAttributes": [
{
"Key": "deletion_protection.enabled",
"Value": "false"
}
],
"Scheme": "internet-facing",
"Type": "application"
});
openSearchDomainStackTemplate.resourceCountIs('AWS::ElasticLoadBalancingV2::Listener', 1);
openSearchDomainStackTemplate.hasResourceProperties('AWS::ElasticLoadBalancingV2::Listener', {
"Port": 443,
"Protocol": "HTTPS"
});
openSearchDomainStackTemplate.resourceCountIs('AWS::ElasticLoadBalancingV2::TargetGroup', 1);
openSearchDomainStackTemplate.hasResourceProperties('AWS::ElasticLoadBalancingV2::TargetGroup', {
"HealthCheckPath": "/",
"HealthCheckPort": "80",
"Port": 443,
"Protocol": "HTTPS",
"TargetGroupAttributes": [
{
"Key": "stickiness.enabled",
"Value": "false"
}
],
"TargetType": "instance",
"VpcId": {
"Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputRefOpenSearchHealthVpcB885AABED860B3EB"
}
});
openSearchDomainStackTemplate.resourceCountIs('AWS::AutoScaling::AutoScalingGroup', 1);
openSearchDomainStackTemplate.hasResourceProperties('AWS::AutoScaling::AutoScalingGroup', {
"DesiredCapacity": "1",
"HealthCheckGracePeriod": 90,
"HealthCheckType": "EC2",
"LaunchConfigurationName": {
"Ref": "OpenSearchMetricsNginxOpenSearchMetricsCognitoMetricsProxyAsgLaunchConfig8D060946"
},
"MaxSize": "1",
"MinSize": "1",
"Tags": [
{
"Key": "name",
"PropagateAtLaunch": true,
"Value": "OpenSearchMetricsCognito-NginxProxyHost"
}
},
{
"Key": "Name",
"PropagateAtLaunch": true,
"Value": "OpenSearchMetricsCognito"
}
],
"TargetGroupARNs": [
{
"Ref": "OpenSearchMetricsNginxOpenSearchMetricsCognitoNginxProxyAlbOpenSearchMetricsCognitoNginxProxyAlbListenerOpenSearchMetricsCognitoNginxProxyAlbTargetGroup8E449B4A"
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents",
"logs:DescribeLogStreams"
],
"Effect": "Allow",
"Resource": "arn:aws:logs:::log-group:OpenSearchMetrics/aws-sigv4-proxy.log:*"
}
],
"VPCZoneIdentifier": [
{
"Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputRefOpenSearchHealthVpcPrivateSubnet1Subnet529349B600974078"
},
{
"Fn::ImportValue": "OpenSearchHealth-VPC:ExportsOutputRefOpenSearchHealthVpcPrivateSubnet2SubnetBA599EDB2BEEEA30"
}
]
"Version": "2012-10-17"
},
"PolicyName": "OpenSearchMetricsReadonlyNginxProxyRoleDefaultPolicy8EDC749D",
"Roles": [
{
"Ref": "OpenSearchMetricsReadonlyNginxProxyRoleE26CC937"
}
]
});

});

Loading
Loading