Merge pull request #29 from cormorack/update-aws-lambda

Update AWS lambda deployment from using cdk v1 to v2

.github/workflows/pr.yaml

@@ -20,25 +20,27 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Set Job Environment Variables
         run: |
           DOCKER_TAG=PR
           IMAGE_SPEC="${DOCKER_ORG}/${{ env.IMAGE_NAME }}:${DOCKER_TAG}"
-          echo "IMAGE_SPEC=${IMAGE_SPEC}" >> $GITHUB_ENV
+          echo "IMAGE_SPEC=${IMAGE_SPEC}" >> "$GITHUB_ENV"
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Build Docker Image
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: ./resources/docker/Dockerfile
           push: false
           build-args: |
             CONDA_VERSION=${{ env.CONDA_VERSION }}
+            echo "CONDA_VERSION=${CONDA_VERSION}" >> "$GITHUB_ENV"
             PYTHON_VERSION=${{ env.PYTHON_VERSION }}
+            echo "PYTHON_VERSION=${PYTHON_VERSION}" >> "$GITUB_ENV"
           tags: |
             ${{ env.IMAGE_SPEC }}

resources/aws/README.md

@@ -2,6 +2,8 @@
 
 The Lambda stack is also deployed by the [AWS CDK](https://aws.amazon.com/cdk/) utility. Under the hood, CDK will create the deployment package required for AWS Lambda, upload it to AWS, and handle the creation of the Lambda and API Gateway resources.
 
+This Lambda deployment is adapted from [Titiler](https://github.com/developmentseed/titiler/tree/main/deployment). 
+
 1. Install CDK and connect to your AWS account. This step is only necessary once per AWS account.
 
     ```bash
@@ -19,7 +21,7 @@ The Lambda stack is also deployed by the [AWS CDK](https://aws.amazon.com/cdk/)
     npm run cdk synth  # Synthesizes and prints the CloudFormation template for this stack
     ```
 
-3. Update settings in a `.env` file.
+3. Update settings in a `.env` file. This is already set in `env.example` so you can go ahead and deploy the stack unless you want to add other settings. 
 
     ```env
     CAVA_DATA_STACK_NAME="cava-data"

resources/aws/cdk/app.py

@@ -1,59 +1,53 @@
 """Construct App."""
 
 import os
-from typing import Any, Dict, List, Optional, Set
+from typing import Any, Dict, List, Optional
 
-from aws_cdk import aws_certificatemanager as acm
-from aws_cdk import aws_apigatewayv2 as apigw
-from aws_cdk import aws_apigatewayv2_integrations as apigw_integrations
-from aws_cdk import aws_iam as iam
+from aws_cdk import App, CfnOutput, Duration, Environment, Stack, Tag
+from aws_cdk import aws_apigatewayv2_alpha as apigw
 from aws_cdk import aws_ec2 as ec2
+from aws_cdk import aws_iam as iam
 from aws_cdk import aws_lambda
 from aws_cdk import aws_logs as logs
-from aws_cdk import core
+from aws_cdk.aws_apigatewayv2_integrations_alpha import HttpLambdaIntegration
 from config import StackSettings
+from constructs import Construct
 
 settings = StackSettings()
 
 
-class cavaDataLambdaStack(core.Stack):
+class cavaDataLambdaStack(Stack):
     """
-    Cava Data Lambda Stack
+    Titiler Lambda Stack
 
     This code is freely adapted from
-    - https://github.com/leothomas/titiler/blob/10df64fbbdd342a0762444eceebaac18d8867365/stack/app.py author: @leothomas
-    - https://github.com/ciaranevans/titiler/blob/3a4e04cec2bd9b90e6f80decc49dc3229b6ef569/stack/app.py author: @ciaranevans
-
+    - https://developmentseed.org/titiler/deployment/aws/lambda/ @developmentseed
+    - https://github.com/developmentseed/titiler/tree/main/deployment (Using the aws example)
     """
 
     def __init__(
         self,
-        scope: core.Construct,
+        scope: Construct,
         id: str,
         memory: int = 1024,
         timeout: int = 30,
         concurrent: Optional[int] = None,
         permissions: Optional[List[iam.PolicyStatement]] = None,
         environment: Optional[Dict] = None,
-        code_dir: str = "../../",
+        code_dir: str = "./",
         vpc: Optional[str] = None,
         security_group: Optional[str] = None,
         user_role: Optional[str] = None,
-        env: Optional[core.Environment] = None,
-        services_elb: Optional[str] = None,
-        extra_routes: Optional[Set] = None,
-        domain_name: Optional[str] = None,
-        certificate_arn: Optional[str] = None,
         **kwargs: Any,
     ) -> None:
         """Define stack."""
-        super().__init__(scope, id, env=env, **kwargs)
+        super().__init__(scope, id, **kwargs)
 
         permissions = permissions or []
         environment = environment or {}
         # Get IVpc
         security_groups = None
-        if vpc is not None:
+        if vpc is not None: 
             vpc = ec2.Vpc.from_lookup(self, f"{id}-vpc", vpc_id=vpc)
             if security_group is not None:
                 sg = ec2.SecurityGroup.from_lookup_by_id(self, f"{id}-sg", security_group_id=security_group)
@@ -63,11 +57,11 @@ def __init__(
             f"{id}-lambda",
             code=aws_lambda.DockerImageCode.from_image_asset(
                 directory=os.path.abspath(code_dir),
-                file="resources/aws/lambda/Dockerfile",
+                file="lambda/Dockerfile",
             ),
             memory_size=memory,
             reserved_concurrent_executions=concurrent,
-            timeout=core.Duration.seconds(timeout),
+            timeout=Duration.seconds(timeout),
             environment=environment,
             log_retention=logs.RetentionDays.ONE_WEEK,
             vpc=vpc,
@@ -81,40 +75,16 @@ def __init__(
         for perm in permissions:
             lambda_function.add_to_role_policy(perm)
 
-        # Setup custom domain mapping if available
-        default_domain_mapping = None
-        if domain_name is not None:
-            if certificate_arn is None:
-                raise ValueError("Certificate arn is missing, but you have provided domain name!")
-
-            custom_domain = apigw.DomainName(self, f"{id}-domain-name",
-                domain_name=domain_name,
-                certificate=acm.Certificate.from_certificate_arn(self, f"{id}-cert", certificate_arn)
-            )
-            default_domain_mapping = apigw.DomainMappingOptions(domain_name=custom_domain)
-
-        # Core API that combines all the services
-        core_api = apigw.HttpApi(self, f"{id}-core-api", default_domain_mapping=default_domain_mapping)
-        core_api.add_routes(
-            path="/data/{proxy+}",
-            integration=apigw_integrations.HttpLambdaIntegration(
-                f"{id}-data-proxy-integration", handler=lambda_function
+        api = apigw.HttpApi(
+            self,
+            f"{id}-endpoint",
+            default_integration=HttpLambdaIntegration(
+                f"{id}-integration", handler=lambda_function
             ),
         )
-        if services_elb is not None:
-            if extra_routes is not None:
-                for route in extra_routes:
-                    core_api.add_routes(
-                        path=f"/{route}/{{proxy+}}",
-                        integration=apigw_integrations.HttpUrlIntegration(
-                            f"{id}-{route}-integration",
-                            url=f"http://{services_elb}/{route}/{{proxy}}"
-                        )
-                    )
-        core.CfnOutput(self, "Endpoint", value=core_api.url)
-
+        CfnOutput(self, "Endpoint", value=api.url)
 
-app = core.App()
+app = App()
 
 perms = [
     iam.PolicyStatement(
@@ -134,45 +104,28 @@ def __init__(
         )
     )
 
-
 # Tag infrastructure
 for key, value in {
-    "Name": settings.name,
+    "Project": settings.name,
     "Environment": settings.stage,
     "Owner": settings.owner,
     "Project": settings.project,
 }.items():
     if value:
-        core.Tag.add(app, key, value)
-
-extra_routes = {
-    'metadata',
-    'media',
-    'feed',
-}
-
+        Tag(key, value)
 
-lambda_stackname = f"{settings.name}-lambda-{settings.stage}"
-stack_env = core.Environment(
+stack_env = Environment(
     account=settings.account_id,
     region=settings.region
 )
-cavaDataLambdaStack(
+lambda_stack = cavaDataLambdaStack(
     app,
-    lambda_stackname,
+    f"{settings.name}-lambda-{settings.stage}",
     memory=settings.memory,
     timeout=settings.timeout,
     concurrent=settings.max_concurrent,
-    environment=settings.env,
     permissions=perms,
-    vpc=settings.vpc,
-    security_group=settings.security_group,
-    user_role=settings.user_role,
-    env=stack_env,
-    services_elb=settings.services_elb,
-    extra_routes=extra_routes,
-    domain_name=settings.domain_name,
-    certificate_arn=settings.certificate_arn,
+    environment=settings.env,
 )
 
 app.synth()

resources/aws/cdk/config.py

@@ -1,34 +1,31 @@
 """CAVA_DATA_STACK Configs."""
-from typing import Dict, Optional
 
-import pydantic
+from typing import Dict, List, Optional
 
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
-class StackSettings(pydantic.BaseSettings):
-    """Application settings"""
-
-    class Config:
-        """model config"""
 
-        env_file = ".env"
-        env_prefix = "CAVA_DATA_STACK_"
+class StackSettings(BaseSettings):
+    """Application settings"""
 
     name: str = "cava-data"
     stage: str = "production"
 
-    owner: Optional[str]
+    owner: Optional[str] = None
+    client: Optional[str] = None
+    owner: Optional[str] = None
     project: str = "CAVA"
 
-    vpc: Optional[str]
-    security_group: Optional[str]
-    user_role: Optional[str]
+    vpc: Optional[str] = None
+    security_group: Optional[str] = None
+    user_role: Optional[str] = None
 
     # Stack environment
     region: str = "us-west-2"
     account_id: str = "123556123145"
-    services_elb: str
-    domain_name: Optional[str]
-    certificate_arn: Optional[str]
+    # services_elb: str
+    domain_name: Optional[str] = None
+    certificate_arn: Optional[str] = None
 
     # Default options for cava-data service
     env: Dict = {
@@ -40,8 +37,6 @@ class Config:
         "GOOGLE_SERVICE_JSON": "mybucket/service-json.json"
     }
     is_sqs: bool = False
-
-
     ###########################################################################
     # AWS LAMBDA
     # The following settings only apply to AWS Lambda deployment
@@ -51,22 +46,7 @@ class Config:
 
     # The maximum of concurrent executions you want to reserve for the function.
     # Default: - No specific limit - account limit.
-    max_concurrent: Optional[int]
+    max_concurrent: Optional[int] = None
+
+    model_config = SettingsConfigDict(env_prefix="CAVA_DATA_STACK_", env_file=".env")
 
-    @pydantic.root_validator
-    def set_sqs(cls, values):
-        env_values = values.get('env')
-        rabbitmq_uri = env_values.get('RABBITMQ_URI')
-        if not isinstance(rabbitmq_uri, str):
-            raise TypeError("RABBITMQ_URI must be a string!")
-
-        if rabbitmq_uri.startswith('sqs://'):
-            env_values.update({
-                'REGION': values.get('region'),
-            })
-            values.update({
-                'env': env_values,
-                'is_sqs': True
-            })
-            return values
-        return values

resources/aws/lambda/Dockerfile

@@ -1,16 +1,19 @@
+ARG PYTHON_VERSION=3.11
+
 FROM amazon/aws-lambda-python:3.8
 
-LABEL maintainer="Landung Setiawan"
-RUN yum update -y && \
-    yum install -y python3 python3-dev python3-pip gcc libcurl-devel libssh2-devel nss-devel which git && \
-    rm -Rf /var/cache/yum
-RUN which curl-config
-ENV LD_LIBRARY_PATH=/usr/lib
-ENV PYCURL_CURL_CONFIG=/usr/bin/curl-config
-ENV PYCURL_SSL_LIBRARY=nss
-COPY ./ /tmp/app
-RUN pip install /tmp/app
-RUN pip install mangum>=0.10.0
-RUN rm -rf /tmp/app
-COPY resources/aws/lambda/handler.py ./
-CMD ["handler.handler"]
+WORKDIR /tmp
+
+RUN pip install pip -U
+RUN pip install "titiler.application==0.17.0" "mangum>=0.10.0" -t /asset --no-binary pydantic
+
+# Reduce package size and remove useless files
+RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done;
+RUN cd /asset && find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
+RUN cd /asset && find . -type f -a -name '*.py' -print0 | xargs -0 rm -f
+RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf
+RUN rm -rdf /asset/numpy/doc/ /asset/boto3* /asset/botocore* /asset/bin /asset/geos_license /asset/Misc
+
+COPY lambda/handler.py /asset/handler.py
+
+CMD ["echo", "hello world"]