feat: add mlflow modules

README.md

@@ -8,7 +8,7 @@ The modules in this repository are decoupled from each other and can be aggregat
 
 The modules in this repository are / must be generic for reuse without affiliation to any one particular project in Machine Learning Operations domain.
 
-All modules in this repository adhere to the module strutucture defined in the the [SeedFarmer Guide](https://seed-farmer.readthedocs.io/en/latest)
+All modules in this repository adhere to the module structure defined in the the [SeedFarmer Guide](https://seed-farmer.readthedocs.io/en/latest)
 
 - [Project Structure](https://seed-farmer.readthedocs.io/en/latest/project_development.html)
 - [Module Development](https://seed-farmer.readthedocs.io/en/latest/module_development.html)
@@ -22,6 +22,13 @@ All modules in this repository adhere to the module strutucture defined in the t
 |-----------------------------------------------------------------------------|-------------------------------------------------|
 | [SageMaker Endpoint Module](modules/sagemaker/sagemaker-endpoint/README.md) | Creates SageMaker real-time inference endpoint. |
 
+### Mlflow Modules
+
+| Type                                                                | Description                     |
+|---------------------------------------------------------------------|---------------------------------|
+| [Mlflow Image Module](modules/mlflow/mlflow-image/README.md)        | Creates Mlflow container image. |
+| [Mlflow on Fargate Module](modules/mlflow/mlflow-fargate/README.md) | Runs Mlflow on AWS Fargate.     |
+
 ### Industry Data Framework (IDF) Modules
 
 The modules in this repository are compatible with [Industry Data Framework (IDF) Modules](https://github.com/awslabs/idf-modules) and can be used together within the same deployment. Refer to `examples/manifests` for examples.

manifests/deployment.yaml

@@ -4,8 +4,14 @@ forceDependencyRedeploy: true
 groups:
   - name: networking
     path: manifests/networking-modules.yaml
+  - name: storage
+    path: manifests/storage-modules.yaml
   - name: sagemaker-studio
     path: manifests/sagemaker-studio-modules.yaml
+  - name: images
+    path: manifests/images-modules.yaml
+  - name: mlflow
+    path: manifests/mlflow-modules.yaml
 targetAccountMappings:
   - alias: primary
     accountId:

manifests/images-modules.yaml

@@ -0,0 +1,9 @@
+name: mlflow-image
+path: modules/mlflow/mlflow-image
+parameters:
+  - name: ecr-repository-name
+    valueFrom:
+      moduleMetadata:
+        group: storage
+        name: ecr-mlflow
+        key: EcrRepositoryName

manifests/mlflow-modules.yaml

@@ -0,0 +1,27 @@
+name: mlflow-fargate
+path: modules/mlflow/mlflow-fargate
+parameters:
+  - name: vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: VpcId
+  - name: subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: PrivateSubnetIds
+  - name: ecr-repository-name
+    valueFrom:
+      moduleMetadata:
+        group: storage
+        name: ecr-mlflow
+        key: EcrRepositoryName
+  - name: artifacts-bucket-name
+    valueFrom:
+      moduleMetadata:
+        group: storage
+        name: buckets
+        key: ArtifactsBucketName

manifests/sagemaker-studio-modules.yaml

@@ -3,7 +3,7 @@ path: git::https://github.com/awslabs/idf-modules.git//modules/ml/sagemaker-stud
 targetAccount: primary
 parameters:
   - name: studio_domain_name
-    value: mlops
+    value: mlops-studio
   - name: vpc_id
     valueFrom:
       moduleMetadata:

manifests/storage-modules.yaml

@@ -0,0 +1,15 @@
+name: ecr-mlflow
+path: git::https://github.com/awslabs/idf-modules.git//modules/storage/ecr?ref=release/1.3.0&depth=1
+parameters:
+  - name: repository_name
+    value: "ecr-mlflow"
+---
+name: buckets
+path: git::https://github.com/awslabs/idf-modules.git//modules/storage/buckets?ref=release/1.3.0&depth=1
+targetAccount: primary
+targetRegion: us-east-1
+parameters:
+  - name: encryption-type
+    value: SSE
+  - name: retention-type
+    value: RETAIN

modules/mlflow/mlflow-fargate/README.md

@@ -0,0 +1,71 @@
+# Mlflow on Fargate module
+
+## Description
+
+This module runs Mlflow on AWS Fargate.
+
+## Inputs/Outputs
+
+### Input Parameters
+
+#### Required
+
+- `vpc-id`: The VPC-ID that the ECS cluster will be created in.
+- `subnet-ids`: The subnets that the Fargate task will use.
+- `ecr-repository-name`: The name of the ECR repository to pull the image from.
+- `artifacts-bucket-name`: Name of the artifacts store bucket
+
+#### Optional
+
+- `ecs-cluster-name`: Name of the ECS cluster.
+- `service-name`: Name of the service.
+- `task-cpu-units`: The number of cpu units used by the Fargate task.
+- `task-memory-limit-mb`: The amount (in MiB) of memory used by the Fargate task.
+
+### Sample manifest declaration
+
+```yaml
+name: mlflow-fargate
+path: modules/mlflow/mlflow-fargate
+parameters:
+  - name: vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: VpcId
+  - name: subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: PrivateSubnetIds
+  - name: ecr-repository-name
+    valueFrom:
+      moduleMetadata:
+        group: storage
+        name: ecr-mlflow
+        key: EcrRepositoryName
+  - name: artifacts-bucket-name
+    valueFrom:
+      moduleMetadata:
+        group: storage
+        name: buckets
+        key: ArtifactsBucketName
+```
+
+### Module Metadata Outputs
+
+- `ECSClusterName`: Name of the ECS cluster.
+- `ServiceName`: Name of the service.
+- `LoadBalancerDNSName`: Load balancer DNS name.
+
+#### Output Example
+
+```
+{
+  "ECSClusterName": "mlops-mlops-mlflow-mlflow-fargate-EcsCluster97242B84-xxxxxxxxxxxx",
+  "ServiceName": "mlops-mlops-mlflow-mlflow-fargate-MlflowLBServiceEBACC043-xxxxxxxxxxxx",
+  "LoadBalancerDNSName": "xxxxxxxxxxxx.elb.us-east-1.amazonaws.com"
+}
+```

modules/mlflow/mlflow-fargate/app.py

@@ -0,0 +1,79 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import os
+
+import aws_cdk
+
+from stack import MlflowFargateStack
+
+
+def _param(name: str) -> str:
+    return f"SEEDFARMER_PARAMETER_{name}"
+
+
+project_name = os.getenv("SEEDFARMER_PROJECT_NAME", "")
+deployment_name = os.getenv("SEEDFARMER_DEPLOYMENT_NAME", "")
+module_name = os.getenv("SEEDFARMER_MODULE_NAME", "")
+app_prefix = f"{project_name}-{deployment_name}-{module_name}"
+
+DEFAULT_ECS_CLUSTER_NAME = None
+DEFAULT_SERVICE_NAME = None
+DEFAULT_TASK_CPU_UNITS = 4 * 1024
+DEFAULT_TASK_MEMORY_LIMIT_MB = 8 * 1024
+
+environment = aws_cdk.Environment(
+    account=os.environ["CDK_DEFAULT_ACCOUNT"],
+    region=os.environ["CDK_DEFAULT_REGION"],
+)
+
+vpc_id = os.getenv(_param("VPC_ID"))
+subnet_ids = json.loads(os.getenv(_param("SUBNET_IDS"), "[]"))
+ecs_cluster_name = os.getenv(_param("ECS_CLUSTER_NAME"), DEFAULT_ECS_CLUSTER_NAME)
+service_name = os.getenv(_param("SERVICE_NAME"), DEFAULT_SERVICE_NAME)
+ecr_repo_name = os.getenv(_param("ECR_REPOSITORY_NAME"))
+task_cpu_units = os.getenv(_param("TASK_CPU_UNITS"), DEFAULT_TASK_CPU_UNITS)
+task_memory_limit_mb = os.getenv(_param("TASK_MEMORY_LIMIT_MB"), DEFAULT_TASK_MEMORY_LIMIT_MB)
+artifacts_bucket_name = os.getenv(_param("ARTIFACTS_BUCKET_NAME"))
+# TODO: add persistent backend store
+
+if not ecr_repo_name:
+    raise ValueError("Missing input parameter ecr-repository-name")
+
+if not vpc_id:
+    raise ValueError("Missing input parameter vpc-id")
+
+app = aws_cdk.App()
+stack = MlflowFargateStack(
+    scope=app,
+    id=app_prefix,
+    app_prefix=app_prefix,
+    vpc_id=vpc_id,
+    subnet_ids=subnet_ids,
+    ecs_cluster_name=ecs_cluster_name,
+    service_name=service_name,
+    ecr_repo_name=ecr_repo_name,
+    task_cpu_units=int(task_cpu_units),
+    task_memory_limit_mb=int(task_memory_limit_mb),
+    artifacts_bucket_name=artifacts_bucket_name,
+    env=aws_cdk.Environment(
+        account=os.environ["CDK_DEFAULT_ACCOUNT"],
+        region=os.environ["CDK_DEFAULT_REGION"],
+    ),
+)
+
+
+aws_cdk.CfnOutput(
+    scope=stack,
+    id="metadata",
+    value=stack.to_json_string(
+        {
+            "ECSClusterName": stack.cluster.cluster_name,
+            "ServiceName": stack.service.service.service_name,
+            "LoadBalancerDNSName": stack.service.load_balancer.load_balancer_dns_name,
+        }
+    ),
+)
+
+app.synth()

modules/mlflow/mlflow-fargate/deployspec.yaml

@@ -0,0 +1,35 @@
+publishGenericEnvVariables: true
+deploy:
+  phases:
+    install:
+      commands:
+      - npm install -g [email protected]
+      - pip install -r requirements.txt
+    pre_build:
+      commands:
+      - echo "Prebuild stage"
+    build:
+      commands:
+      - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json
+      # Export metadata
+      - seedfarmer metadata convert -f cdk-exports.json || true
+    post_build:
+      commands:
+      - echo "Build successful"
+destroy:
+  phases:
+    install:
+      commands:
+      - npm install -g [email protected]
+      - pip install -r requirements.txt
+    pre_build:
+      commands:
+      - echo "Prebuild stage"
+    build:
+      commands:
+      - cdk destroy --force --app "python app.py"
+    post_build:
+      commands:
+      - echo "Destroy successful"
+
+build_type: BUILD_GENERAL1_SMALL

modules/mlflow/mlflow-fargate/pyproject.toml

@@ -0,0 +1,35 @@
+[tool.black]
+line-length = 120
+target-version = ["py36", "py37", "py38"]
+exclude = '''
+/(
+    \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | \.env
+  | _build
+  | buck-out
+  | build
+  | dist
+  | codeseeder.out
+)/
+'''
+
+[tool.isort]
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+use_parentheses = true
+ensure_newline_before_comments = true
+line_length = 120
+py_version = 36
+skip_gitignore = false
+
+[tool.pytest.ini_options]
+addopts = "-v --cov=. --cov-report term --cov-config=coverage.ini --cov-fail-under=80"
+pythonpath = [
+  "."
+]

modules/mlflow/mlflow-fargate/requirements-dev.in

@@ -0,0 +1,14 @@
+awscli
+black
+cdk-nag
+cfn-lint
+check-manifest
+flake8
+isort
+mypy
+pip-tools
+pydot
+pyroma
+pytest
+types-PyYAML
+types-setuptools

modules/mlflow/mlflow-fargate/requirements.in

@@ -0,0 +1,4 @@
+aws-cdk-lib==2.126.0
+cdk-nag==2.28.27
+boto3==1.34.35
+cdk_ecr_deployment==2.5.30