From 50ad8f881480bd4ccab34b3b0f7c5730bbc3d473 Mon Sep 17 00:00:00 2001 From: markbruning <125313351+markbruning@users.noreply.github.com> Date: Tue, 3 Oct 2023 10:47:03 -0500 Subject: [PATCH] Add as_posix to s3 push file paths in deployment step (#314) --- CHANGELOG.md | 2 ++ prefect_aws/deployments/steps.py | 4 ++- tests/deploments/test_steps.py | 42 ++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0a447c8..7967e69f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Changed `push_to_s3` deployment step function to write paths `as_posix()` to allow support for deploying from windows [#314](https://github.com/PrefectHQ/prefect-aws/pull/314) + ### Deprecated ### Removed diff --git a/prefect_aws/deployments/steps.py b/prefect_aws/deployments/steps.py index 805b4648..5609a044 100644 --- a/prefect_aws/deployments/steps.py +++ b/prefect_aws/deployments/steps.py @@ -117,7 +117,9 @@ def push_to_s3( continue elif not local_file_path.is_dir(): remote_file_path = Path(folder) / local_file_path.relative_to(local_path) - client.upload_file(str(local_file_path), bucket, str(remote_file_path)) + client.upload_file( + str(local_file_path), bucket, str(remote_file_path.as_posix()) + ) return { "bucket": bucket, diff --git a/tests/deploments/test_steps.py b/tests/deploments/test_steps.py index 5e2d2efd..a2312d18 100644 --- a/tests/deploments/test_steps.py +++ b/tests/deploments/test_steps.py @@ -1,4 +1,5 @@ import os +import sys from pathlib import Path, PurePath, PurePosixPath import boto3 @@ -40,6 +41,24 @@ def tmp_files(tmp_path: Path): return tmp_path +@pytest.fixture +def tmp_files_win(tmp_path: Path): + files = [ + "testfile1.txt", + "testfile2.txt", + "testfile3.txt", + r"testdir1\testfile4.txt", + r"testdir2\testfile5.txt", + ] + + for file in files: + filepath = tmp_path / file + filepath.parent.mkdir(parents=True, exist_ok=True) + filepath.write_text("Sample text") + + return tmp_path + + @pytest.fixture def mock_aws_credentials(monkeypatch): # Set mock environment variables for AWS credentials @@ -77,6 +96,29 @@ def test_push_to_s3(s3_setup, tmp_files, mock_aws_credentials): assert set(object_keys) == set(expected_keys) +@pytest.mark.skipif(sys.platform != "win32", reason="requires Windows") +def test_push_to_s3_as_posix(s3_setup, tmp_files_win, mock_aws_credentials): + s3, bucket_name = s3_setup + folder = "my-project" + + os.chdir(tmp_files_win) + + push_to_s3(bucket_name, folder) + + s3_objects = s3.list_objects_v2(Bucket=bucket_name) + object_keys = [item["Key"] for item in s3_objects["Contents"]] + + expected_keys = [ + f"{folder}/testfile1.txt", + f"{folder}/testfile2.txt", + f"{folder}/testfile3.txt", + f"{folder}/testdir1/testfile4.txt", + f"{folder}/testdir2/testfile5.txt", + ] + + assert set(object_keys) == set(expected_keys) + + def test_pull_from_s3(s3_setup, tmp_path, mock_aws_credentials): s3, bucket_name = s3_setup folder = "my-project"