add stacks for raw sync lambda

config/develop/namespaced/lambda-raw-sync-role.yaml

@@ -0,0 +1,15 @@
+template:
+  path: lambda-raw-role.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-raw-role"
+dependencies:
+  - develop/namespaced/sns-dispatch.yaml
+  - develop/namespaced/sqs-input-to-dispatch.yaml
+  - develop/s3-cloudformation-bucket.yaml
+  - develop/s3-input-bucket.yaml
+  - develop/s3-raw-bucket.yaml
+parameters:
+  S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
+  S3TargetBucketName: {{ stack_group_config.raw_bucket_name }}
+  SNSTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/lambda-raw-sync.yaml

@@ -0,0 +1,18 @@
+template:
+  type: sam
+  path: src/lambda_function/raw_sync/template.yaml
+  artifact_bucket_name: {{ stack_group_config.template_bucket_name }}
+  artifact_prefix: "{{ stack_group_config.namespace }}/src/lambda"
+dependencies:
+  - develop/namespaced/lambda-raw-sync-role.yaml
+  - develop/s3-cloudformation-bucket.yaml
+  - develop/s3-raw-bucket.yaml
+  - develop/s3-input-bucket.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-raw-sync"
+parameters:
+  RoleArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-raw-sync-role::RoleArn"
+  S3InputBucket: {{ stack_group_config.input_bucket_name }}
+  S3InputKeyPrefix: "{{ stack_group_config.namespace }}/"
+  S3RawBucket: {{ stack_group_config.raw_bucket_name }}
+  S3RawKeyPrefix: "{{ stack_group_config.namespace }}/json/"
+stack_tags: {{ stack_group_config.default_stack_tags }}

src/lambda_function/raw_sync/template.yaml

@@ -0,0 +1,60 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+
+Description: >
+  SAM Template for the raw sync Lambda. The raw sync Lambda ensures that
+  the input and raw S3 buckets are synchronized by verifying that all non-zero
+  sized JSON in the exports in the input bucket have a corresponding object in
+  the raw bucket.
+
+Parameters:
+
+  RoleArn:
+    Type: String
+    Description: ARN of the raw sync Lambda role.
+
+  S3InputBucket:
+    Type: String
+    Description: Name of the input S3 bucket.
+
+  S3InputKeyPrefix:
+    Type: String
+    Description: S3 key prefix where exports are written.
+
+  S3RawBucket:
+    Type: String
+    Description: Name of the Raw S3 bucket.
+
+  S3RawKeyPrefix:
+    Type: String
+    Description: S3 key prefix where files are written.
+
+  LambdaPythonVersion:
+    Type: String
+    Description: Python version to use for this lambda function
+    Default: "3.9"
+
+Resources:
+  RawSyncFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      PackageType: Zip
+      CodeUri: ./
+      Handler: app.lambda_handler
+      Runtime: !Sub "python${LambdaPythonVersion}"
+      Role: !Ref RoleArn
+      MemorySize: 1024
+      Timeout: 900
+      Environment:
+        Variables:
+          INPUT_S3_BUCKET: !Ref S3InputBucket
+          INPUT_S3_KEY_PREFIX: !Ref S3InputKeyPrefix
+          RAW_S3_BUCKET: !Ref S3RawBucket
+          RAW_S3_KEY_PREFIX: !Ref S3RawKeyPrefix
+
+Outputs:
+  RawSyncFunctionArn:
+    Description: Arn of the raw sync Lambda.
+    Value: !GetAtt RawSyncFunction.Arn
+    Export:
+      Name: !Sub "${AWS::Region}-${AWS::StackName}-RawSyncFunctionArn"

templates/lambda-raw-sync-role.yaml

@@ -0,0 +1,71 @@
+AWSTemplateFormatVersion: '2010-09-09'
+
+Transform: AWS::Serverless-2016-10-31
+
+Description: >
+  An IAM Role for the raw sync Lambda
+
+Parameters:
+  S3SourceBucketName:
+    Type: String
+    Description: Name of the S3 bucket where exports are deposited.
+
+  S3TargetBucketName:
+    Type: String
+    Description: Name of the S3 bucket where raw JSON is written to.
+
+  SNSTopicArn:
+    Type: String
+    Description: >
+      ARN of the SNS topic where files found not to have a corresponding
+      object in the target bucket will be published to for processing.
+
+Resources:
+  RawRole:
+      Type: AWS::IAM::Role
+      Properties:
+        AssumeRolePolicyDocument:
+          Version: '2012-10-17'
+          Statement:
+          - Effect: Allow
+            Principal:
+              Service:
+              - lambda.amazonaws.com
+            Action:
+            - sts:AssumeRole
+        ManagedPolicyArns:
+          - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
+        Policies:
+        - PolicyName: ReadS3
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+            - Effect: Allow
+              Action:
+              - s3:Get*
+              - s3:List*
+              Resource:
+              - !Sub arn:aws:s3:::${S3SourceBucketName}
+              - !Sub arn:aws:s3:::${S3SourceBucketName}/*
+              - !Sub arn:aws:s3:::${S3TargetBucketName}
+              - !Sub arn:aws:s3:::${S3TargetBucketName}/*
+        - PolicyName: PublishToSNS
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - sns:Publish
+                Resource:
+                  - !Ref SNSTopicArn
+
+Outputs:
+  RoleName:
+    Value: !Ref RawRole
+    Export:
+      Name: !Sub '${AWS::Region}-${AWS::StackName}-RoleName'
+
+  RoleArn:
+    Value: !GetAtt RawRole.Arn
+    Export:
+      Name: !Sub '${AWS::Region}-${AWS::StackName}-RoleArn'