Merge branch 'main' into main

.github/workflows/release.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.8
 
       - name: Install packages
         run: |

.github/workflows/tests.yml

@@ -9,10 +9,10 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.7"
           - "3.8"
           - "3.9"
           - "3.10"
+          - "3.11"
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
@@ -33,9 +33,7 @@ jobs:
         env:
           PREFECT_SERVER_DATABASE_CONNECTION_URL: "sqlite+aiosqlite:///./collection-tests.db"
         run: |
-          prefect server database reset -y
-          coverage run --branch -m pytest tests -vv
-          coverage report
+          pytest --cov=prefect_aws --no-cov-on-fail --cov-report=term-missing --cov-branch tests -n auto -vv
 
       - name: Run mkdocs build
         run: |

CHANGELOG.md

@@ -9,15 +9,54 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
-- Added retries to ECS task run creation for ECS worker - [#303](https://github.com/PrefectHQ/prefect-aws/pull/303)
-- Added support to `ECSWorker` for `awsvpcConfiguration` [#304](https://github.com/PrefectHQ/prefect-aws/pull/304)
-
 ### Changed
-- Added a check to AwsSecretsManager.load() for 'SecretString' as well as 'SecretBinary' - [#274](https://github.com/PrefectHQ/prefect-aws/pull/274)
+- Added 'SecretBrinary' suport to `AwsSecret` block - [#274](https://github.com/PrefectHQ/prefect-aws/pull/274)
+
+### Fixed
+
 ### Deprecated
 
 ### Removed
 
+## 0.4.2
+
+Released November 6th, 2023.
+
+### Fixed
+
+- Fixed use_ssl default for s3 client.
+
+## 0.4.1
+
+Released October 13th, 2023.
+
+### Added
+
+- AWS S3 copy and move tasks and `S3Bucket` methods - [#316](https://github.com/PrefectHQ/prefect-aws/pull/316)
+
+### Fixed
+
+- `ECSWorker` issue where defining a custom network configuration with a subnet would erroneously report it as missing from the VPC when more than one subnet exists in the VPC. [#321](https://github.com/PrefectHQ/prefect-aws/pull/321)
+- Updated `push_to_s3` and `pull_from_s3` deployment steps to properly create a boto3 session client if the passed credentials are a referenced `AwsCredentials` block [#322](https://github.com/PrefectHQ/prefect-aws/pull/322)
+
+## 0.4.0
+
+Released October 5th, 2023.
+
+### Changed
+
+- Changed `push_to_s3` deployment step function to write paths `as_posix()` to allow support for deploying from windows [#314](https://github.com/PrefectHQ/prefect-aws/pull/314)
+- Conditional imports to support operating with pydantic>2 installed - [#317](https://github.com/PrefectHQ/prefect-aws/pull/317)
+
+## 0.3.7
+
+Released August 31st, 2023.
+
+### Added
+
+- Added retries to ECS task run creation for ECS worker - [#303](https://github.com/PrefectHQ/prefect-aws/pull/303)
+- Added support to `ECSWorker` for `awsvpcConfiguration` [#304](https://github.com/PrefectHQ/prefect-aws/pull/304)
+
 ## 0.3.6
 
 Released July 20th, 2023.

README.md

@@ -1,26 +1,24 @@
-# Incorporate AWS into your Prefect workflows with `prefect-aws`
+# `prefect-aws`
 
 <p align="center">
-    <img src="https://user-images.githubusercontent.com/15331990/214123296-4cfa69ed-d105-4ca2-a351-4c21917086c7.png">
-    <br>
     <a href="https://pypi.python.org/pypi/prefect-aws/" alt="PyPI version">
-        <img alt="PyPI" src="https://img.shields.io/pypi/v/prefect-aws?color=0052FF&labelColor=090422"></a>
+        <img alt="PyPI" src="https://img.shields.io/pypi/v/prefect-aws?color=26272B&labelColor=090422"></a>
     <a href="https://github.com/prefecthq/prefect-aws/" alt="Stars">
-        <img src="https://img.shields.io/github/stars/prefecthq/prefect-aws?color=0052FF&labelColor=090422" /></a>
+        <img src="https://img.shields.io/github/stars/prefecthq/prefect-aws?color=26272B&labelColor=090422" /></a>
     <a href="https://pepy.tech/badge/prefect-aws/" alt="Downloads">
-        <img src="https://img.shields.io/pypi/dm/prefect-aws?color=0052FF&labelColor=090422" /></a>
+        <img src="https://img.shields.io/pypi/dm/prefect-aws?color=26272B&labelColor=090422" /></a>
     <a href="https://github.com/prefecthq/prefect-aws/pulse" alt="Activity">
-        <img src="https://img.shields.io/github/commit-activity/m/prefecthq/prefect-aws?color=0052FF&labelColor=090422" /></a>
+        <img src="https://img.shields.io/github/commit-activity/m/prefecthq/prefect-aws?color=26272B&labelColor=090422" /></a>
     <br>
     <a href="https://prefect-community.slack.com" alt="Slack">
-        <img src="https://img.shields.io/badge/slack-join_community-red.svg?color=0052FF&labelColor=090422&logo=slack" /></a>
+        <img src="https://img.shields.io/badge/slack-join_community-red.svg?color=26272B&labelColor=090422&logo=slack" /></a>
     <a href="https://discourse.prefect.io/" alt="Discourse">
-        <img src="https://img.shields.io/badge/discourse-browse_forum-red.svg?color=0052FF&labelColor=090422&logo=discourse" /></a>
+        <img src="https://img.shields.io/badge/discourse-browse_forum-red.svg?color=26272B&labelColor=090422&logo=discourse" /></a>
 </p>
 
 ## Welcome!
 
-The `prefect-aws` collection makes it easy to leverage the capabilities of AWS in your flows, featuring support for ECS, S3, Secrets Manager, Batch Job, and Client Waiter.
+`prefect-aws` makes it easy to leverage the capabilities of AWS in your flows, featuring support for ECS, S3, Secrets Manager, and Batch.
 
 Visit the full docs [here](https://PrefectHQ.github.io/prefect-aws).
 

docs/index.md

@@ -1,26 +1,24 @@
-# Coordinate and incorporate AWS in your dataflow with `prefect-aws`
+# `prefect-aws`
 
 <p align="center">
-    <img src="https://user-images.githubusercontent.com/15331990/214123296-4cfa69ed-d105-4ca2-a351-4c21917086c7.png">
-    <br>
     <a href="https://pypi.python.org/pypi/prefect-aws/" alt="PyPI version">
-        <img alt="PyPI" src="https://img.shields.io/pypi/v/prefect-aws?color=0052FF&labelColor=090422"></a>
+        <img alt="PyPI" src="https://img.shields.io/pypi/v/prefect-aws?color=26272B&labelColor=090422"></a>
     <a href="https://github.com/prefecthq/prefect-aws/" alt="Stars">
-        <img src="https://img.shields.io/github/stars/prefecthq/prefect-aws?color=0052FF&labelColor=090422" /></a>
+        <img src="https://img.shields.io/github/stars/prefecthq/prefect-aws?color=26272B&labelColor=090422" /></a>
     <a href="https://pepy.tech/badge/prefect-aws/" alt="Downloads">
-        <img src="https://img.shields.io/pypi/dm/prefect-aws?color=0052FF&labelColor=090422" /></a>
+        <img src="https://img.shields.io/pypi/dm/prefect-aws?color=26272B&labelColor=090422" /></a>
     <a href="https://github.com/prefecthq/prefect-aws/pulse" alt="Activity">
-        <img src="https://img.shields.io/github/commit-activity/m/prefecthq/prefect-aws?color=0052FF&labelColor=090422" /></a>
+        <img src="https://img.shields.io/github/commit-activity/m/prefecthq/prefect-aws?color=26272B&labelColor=090422" /></a>
     <br>
     <a href="https://prefect-community.slack.com" alt="Slack">
-        <img src="https://img.shields.io/badge/slack-join_community-red.svg?color=0052FF&labelColor=090422&logo=slack" /></a>
+        <img src="https://img.shields.io/badge/slack-join_community-red.svg?color=26272B&labelColor=090422&logo=slack" /></a>
     <a href="https://discourse.prefect.io/" alt="Discourse">
-        <img src="https://img.shields.io/badge/discourse-browse_forum-red.svg?color=0052FF&labelColor=090422&logo=discourse" /></a>
+        <img src="https://img.shields.io/badge/discourse-browse_forum-red.svg?color=26272B&labelColor=090422&logo=discourse" /></a>
 </p>
 
 ## Welcome!
 
-The `prefect-aws` collection makes it easy to leverage the capabilities of AWS in your flows, featuring support for ECSTask, S3, Secrets Manager, Batch Job, and Client Waiter.
+`prefect-aws` makes it easy to leverage the capabilities of AWS in your flows, featuring support for ECSTask, S3, Secrets Manager, Batch Job, and Client Waiter.
 
 
 ## Getting Started

docs/stylesheets/extra.css

@@ -1,9 +1,9 @@
 /* theme */ 
 :root > * {
     /* theme */ 
-    --md-primary-fg-color: #115AF4;
-    --md-primary-fg-color--light: #115AF4;
-    --md-primary-fg-color--dark:  #115AF4;  
+    --md-primary-fg-color: #26272B;
+    --md-primary-fg-color--light: #26272B;
+    --md-primary-fg-color--dark:  #26272B;  
 }
 
 /* Table formatting */
@@ -72,7 +72,7 @@ to force column width */
 /* dark mode slate theme */
 /* dark mode code overrides */
 [data-md-color-scheme="slate"] {
-    --md-code-bg-color: #252a33;
+    --md-code-bg-color: #1c1d20;
     --md-code-fg-color: #eee;
     --md-code-hl-color: #3b3d54;
     --md-code-hl-name-color: #eee;
@@ -100,15 +100,15 @@ to force column width */
 
 /* dark mode collection catalog overrides */
 [data-md-color-scheme="slate"] .collection-item {
-    background-color: #3b3d54;
+    background-color: #26272B;
 }
 
 /* dark mode recipe collection overrides */
 [data-md-color-scheme="slate"] .recipe-item {
-    background-color: #3b3d54;
+    background-color: #26272B;
 }
 
 /* dark mode API doc overrides */
 [data-md-color-scheme="slate"] .prefect-table th {
-    background-color: #3b3d54;
+    background-color: #26272B;
 }

mkdocs.yml

@@ -28,7 +28,7 @@ theme:
   icon:
     repo: fontawesome/brands/github
   logo:
-    img/prefect-logo-mark-solid-white-500.png
+    img/prefect-logo-mark.png
   font:
     text: Inter
     code: Source Code Pro

prefect_aws/client_parameters.py

@@ -5,7 +5,12 @@
 
 from botocore import UNSIGNED
 from botocore.client import Config
-from pydantic import BaseModel, Field, FilePath, root_validator, validator
+from pydantic import VERSION as PYDANTIC_VERSION
+
+if PYDANTIC_VERSION.startswith("2."):
+    from pydantic.v1 import BaseModel, Field, FilePath, root_validator, validator
+else:
+    from pydantic import BaseModel, Field, FilePath, root_validator, validator
 
 
 class AwsClientParameters(BaseModel):

prefect_aws/credentials.py

@@ -7,7 +7,12 @@
 from mypy_boto3_s3 import S3Client
 from mypy_boto3_secretsmanager import SecretsManagerClient
 from prefect.blocks.abstract import CredentialsBlock
-from pydantic import Field, SecretStr
+from pydantic import VERSION as PYDANTIC_VERSION
+
+if PYDANTIC_VERSION.startswith("2."):
+    from pydantic.v1 import Field, SecretStr
+else:
+    from pydantic import Field, SecretStr
 
 from prefect_aws.client_parameters import AwsClientParameters
 
@@ -35,7 +40,7 @@ class AwsCredentials(CredentialsBlock):
         ```
     """  # noqa E501
 
-    _logo_url = "https://images.ctfassets.net/gm98wzqotmnx/1jbV4lceHOjGgunX15lUwT/db88e184d727f721575aeb054a37e277/aws.png?h=250"  # noqa
+    _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/d74b16fe84ce626345adf235a47008fea2869a60-225x225.png"  # noqa
     _block_type_name = "AWS Credentials"
     _documentation_url = "https://prefecthq.github.io/prefect-aws/credentials/#prefect_aws.credentials.AwsCredentials"  # noqa
 
@@ -159,7 +164,7 @@ class MinIOCredentials(CredentialsBlock):
         ```
     """  # noqa E501
 
-    _logo_url = "https://images.ctfassets.net/gm98wzqotmnx/22vXcxsOrVeFrUwHfSoaeT/7607b876eb589a9028c8126e78f4c7b4/imageedit_7_2837870043.png?h=250"  # noqa
+    _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/676cb17bcbdff601f97e0a02ff8bcb480e91ff40-250x250.png"  # noqa
     _block_type_name = "MinIO Credentials"
     _description = (
         "Block used to manage authentication with MinIO. Refer to the MinIO "

prefect_aws/deployments/steps.py

@@ -91,14 +91,7 @@ def push_to_s3(
         ```
 
     """
-    if credentials is None:
-        credentials = {}
-    if client_parameters is None:
-        client_parameters = {}
-    advanced_config = client_parameters.pop("config", {})
-    client = boto3.client(
-        "s3", **credentials, **client_parameters, config=Config(**advanced_config)
-    )
+    s3 = get_s3_client(credentials=credentials, client_parameters=client_parameters)
 
     local_path = Path.cwd()
 
@@ -117,7 +110,9 @@ def push_to_s3(
             continue
         elif not local_file_path.is_dir():
             remote_file_path = Path(folder) / local_file_path.relative_to(local_path)
-            client.upload_file(str(local_file_path), bucket, str(remote_file_path))
+            s3.upload_file(
+                str(local_file_path), bucket, str(remote_file_path.as_posix())
+            )
 
     return {
         "bucket": bucket,
@@ -172,14 +167,7 @@ def pull_from_s3(
                 credentials: "{{ prefect.blocks.aws-credentials.dev-credentials }}"
         ```
     """
-    if credentials is None:
-        credentials = {}
-    if client_parameters is None:
-        client_parameters = {}
-    advanced_config = client_parameters.pop("config", {})
-
-    session = boto3.Session(**credentials)
-    s3 = session.client("s3", **client_parameters, config=Config(**advanced_config))
+    s3 = get_s3_client(credentials=credentials, client_parameters=client_parameters)
 
     local_path = Path.cwd()
 
@@ -204,3 +192,51 @@ def pull_from_s3(
         "folder": folder,
         "directory": str(local_path),
     }
+
+
+def get_s3_client(
+    credentials: Optional[Dict] = None,
+    client_parameters: Optional[Dict] = None,
+) -> dict:
+    if credentials is None:
+        credentials = {}
+    if client_parameters is None:
+        client_parameters = {}
+
+    # Get credentials from credentials (regardless if block or not)
+    aws_access_key_id = credentials.get("aws_access_key_id", None)
+    aws_secret_access_key = credentials.get("aws_secret_access_key", None)
+    aws_session_token = credentials.get("aws_session_token", None)
+
+    # Get remaining session info from credentials, or client_parameters
+    profile_name = credentials.get(
+        "profile_name", client_parameters.get("profile_name", None)
+    )
+    region_name = credentials.get(
+        "region_name", client_parameters.get("region_name", None)
+    )
+
+    # Get additional info from client_parameters, otherwise credentials input (if block)
+    aws_client_parameters = credentials.get("aws_client_parameters", client_parameters)
+    api_version = aws_client_parameters.get("api_version", None)
+    endpoint_url = aws_client_parameters.get("endpoint_url", None)
+    use_ssl = aws_client_parameters.get("use_ssl", True)
+    verify = aws_client_parameters.get("verify", None)
+    config_params = aws_client_parameters.get("config", {})
+    config = Config(**config_params)
+
+    session = boto3.Session(
+        aws_access_key_id=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key,
+        aws_session_token=aws_session_token,
+        profile_name=profile_name,
+        region_name=region_name,
+    )
+    return session.client(
+        "s3",
+        api_version=api_version,
+        endpoint_url=endpoint_url,
+        use_ssl=use_ssl,
+        verify=verify,
+        config=config,
+    )

prefect_aws/ecs.py

@@ -121,7 +121,13 @@
 from prefect.utilities.asyncutils import run_sync_in_worker_thread, sync_compatible
 from prefect.utilities.dockerutils import get_prefect_image_name
 from prefect.utilities.pydantic import JsonPatch
-from pydantic import Field, root_validator, validator
+from pydantic import VERSION as PYDANTIC_VERSION
+
+if PYDANTIC_VERSION.startswith("2."):
+    from pydantic.v1 import Field, root_validator, validator
+else:
+    from pydantic import Field, root_validator, validator
+
 from slugify import slugify
 from typing_extensions import Literal, Self
 
@@ -270,7 +276,7 @@ class ECSTask(Infrastructure):
 
     _block_type_slug = "ecs-task"
     _block_type_name = "ECS Task"
-    _logo_url = "https://images.ctfassets.net/gm98wzqotmnx/1jbV4lceHOjGgunX15lUwT/db88e184d727f721575aeb054a37e277/aws.png?h=250"  # noqa
+    _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/d74b16fe84ce626345adf235a47008fea2869a60-225x225.png"  # noqa
     _description = "Run a command as an ECS task."  # noqa
     _documentation_url = (
         "https://prefecthq.github.io/prefect-aws/ecs/#prefect_aws.ecs.ECSTask"  # noqa