From 51420b983dccbf750696ca3be6513e684d1c3fb4 Mon Sep 17 00:00:00 2001 From: Phil Snyder Date: Wed, 18 Sep 2024 15:02:12 -0700 Subject: [PATCH] add stacks for raw sync lambda --- .../namespaced/lambda-raw-sync-role.yaml | 15 ++++ .../develop/namespaced/lambda-raw-sync.yaml | 18 +++++ src/lambda_function/raw_sync/template.yaml | 60 ++++++++++++++++ templates/lambda-raw-sync-role.yaml | 71 +++++++++++++++++++ 4 files changed, 164 insertions(+) create mode 100644 config/develop/namespaced/lambda-raw-sync-role.yaml create mode 100644 config/develop/namespaced/lambda-raw-sync.yaml create mode 100644 src/lambda_function/raw_sync/template.yaml create mode 100644 templates/lambda-raw-sync-role.yaml diff --git a/config/develop/namespaced/lambda-raw-sync-role.yaml b/config/develop/namespaced/lambda-raw-sync-role.yaml new file mode 100644 index 0000000..9c9ca0c --- /dev/null +++ b/config/develop/namespaced/lambda-raw-sync-role.yaml @@ -0,0 +1,15 @@ +template: + path: lambda-raw-role.yaml +stack_name: "{{ stack_group_config.namespace }}-lambda-raw-role" +dependencies: + - develop/namespaced/sns-dispatch.yaml + - develop/namespaced/sqs-input-to-dispatch.yaml + - develop/s3-cloudformation-bucket.yaml + - develop/s3-input-bucket.yaml + - develop/s3-raw-bucket.yaml +parameters: + S3SourceBucketName: {{ stack_group_config.input_bucket_name }} + S3TargetBucketName: {{ stack_group_config.raw_bucket_name }} + SNSTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn" +stack_tags: + {{ stack_group_config.default_stack_tags }} diff --git a/config/develop/namespaced/lambda-raw-sync.yaml b/config/develop/namespaced/lambda-raw-sync.yaml new file mode 100644 index 0000000..e86e9f8 --- /dev/null +++ b/config/develop/namespaced/lambda-raw-sync.yaml @@ -0,0 +1,18 @@ +template: + type: sam + path: src/lambda_function/raw_sync/template.yaml + artifact_bucket_name: {{ stack_group_config.template_bucket_name }} + artifact_prefix: "{{ stack_group_config.namespace }}/src/lambda" +dependencies: + - develop/namespaced/lambda-raw-sync-role.yaml + - develop/s3-cloudformation-bucket.yaml + - develop/s3-raw-bucket.yaml + - develop/s3-input-bucket.yaml +stack_name: "{{ stack_group_config.namespace }}-lambda-raw-sync" +parameters: + RoleArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-raw-sync-role::RoleArn" + S3InputBucket: {{ stack_group_config.input_bucket_name }} + S3InputKeyPrefix: "{{ stack_group_config.namespace }}/" + S3RawBucket: {{ stack_group_config.raw_bucket_name }} + S3RawKeyPrefix: "{{ stack_group_config.namespace }}/json/" +stack_tags: {{ stack_group_config.default_stack_tags }} diff --git a/src/lambda_function/raw_sync/template.yaml b/src/lambda_function/raw_sync/template.yaml new file mode 100644 index 0000000..7c9d4d3 --- /dev/null +++ b/src/lambda_function/raw_sync/template.yaml @@ -0,0 +1,60 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 + +Description: > + SAM Template for the raw sync Lambda. The raw sync Lambda ensures that + the input and raw S3 buckets are synchronized by verifying that all non-zero + sized JSON in the exports in the input bucket have a corresponding object in + the raw bucket. + +Parameters: + + RoleArn: + Type: String + Description: ARN of the raw sync Lambda role. + + S3InputBucket: + Type: String + Description: Name of the input S3 bucket. + + S3InputKeyPrefix: + Type: String + Description: S3 key prefix where exports are written. + + S3RawBucket: + Type: String + Description: Name of the Raw S3 bucket. + + S3RawKeyPrefix: + Type: String + Description: S3 key prefix where files are written. + + LambdaPythonVersion: + Type: String + Description: Python version to use for this lambda function + Default: "3.9" + +Resources: + RawSyncFunction: + Type: AWS::Serverless::Function + Properties: + PackageType: Zip + CodeUri: ./ + Handler: app.lambda_handler + Runtime: !Sub "python${LambdaPythonVersion}" + Role: !Ref RoleArn + MemorySize: 1024 + Timeout: 900 + Environment: + Variables: + INPUT_S3_BUCKET: !Ref S3InputBucket + INPUT_S3_KEY_PREFIX: !Ref S3InputKeyPrefix + RAW_S3_BUCKET: !Ref S3RawBucket + RAW_S3_KEY_PREFIX: !Ref S3RawKeyPrefix + +Outputs: + RawSyncFunctionArn: + Description: Arn of the raw sync Lambda. + Value: !GetAtt RawSyncFunction.Arn + Export: + Name: !Sub "${AWS::Region}-${AWS::StackName}-RawSyncFunctionArn" diff --git a/templates/lambda-raw-sync-role.yaml b/templates/lambda-raw-sync-role.yaml new file mode 100644 index 0000000..5c4da5d --- /dev/null +++ b/templates/lambda-raw-sync-role.yaml @@ -0,0 +1,71 @@ +AWSTemplateFormatVersion: '2010-09-09' + +Transform: AWS::Serverless-2016-10-31 + +Description: > + An IAM Role for the raw sync Lambda + +Parameters: + S3SourceBucketName: + Type: String + Description: Name of the S3 bucket where exports are deposited. + + S3TargetBucketName: + Type: String + Description: Name of the S3 bucket where raw JSON is written to. + + SNSTopicArn: + Type: String + Description: > + ARN of the SNS topic where files found not to have a corresponding + object in the target bucket will be published to for processing. + +Resources: + RawRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + Policies: + - PolicyName: ReadS3 + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - s3:Get* + - s3:List* + Resource: + - !Sub arn:aws:s3:::${S3SourceBucketName} + - !Sub arn:aws:s3:::${S3SourceBucketName}/* + - !Sub arn:aws:s3:::${S3TargetBucketName} + - !Sub arn:aws:s3:::${S3TargetBucketName}/* + - PolicyName: PublishToSNS + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - sns:Publish + Resource: + - !Ref SNSTopicArn + +Outputs: + RoleName: + Value: !Ref RawRole + Export: + Name: !Sub '${AWS::Region}-${AWS::StackName}-RoleName' + + RoleArn: + Value: !GetAtt RawRole.Arn + Export: + Name: !Sub '${AWS::Region}-${AWS::StackName}-RoleArn'