Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Commit

Permalink
Fix S3Bucket.stream_from source path resolution (#291)
Browse files Browse the repository at this point in the history
  • Loading branch information
dominictarro authored Jun 28, 2023
1 parent 8ec724f commit 917f452
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 3 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Changed

- Fixed `S3Bucket.stream_from` path resolution - [#291](https://github.com/PrefectHQ/prefect-aws/pull/291)

### Deprecated

### Removed
Expand Down
2 changes: 1 addition & 1 deletion prefect_aws/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -828,7 +828,7 @@ async def stream_from(
to_path = Path(from_path).name

# Get the source object's StreamingBody
from_path: str = self._join_bucket_folder(from_path)
from_path: str = bucket._join_bucket_folder(from_path)
from_client = bucket.credentials.get_s3_client()
obj = await run_sync_in_worker_thread(
from_client.get_object, Bucket=bucket.bucket_name, Key=from_path
Expand Down
19 changes: 17 additions & 2 deletions tests/test_s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -622,11 +622,26 @@ def s3_bucket_empty(self, credentials, bucket):
_s3_bucket = S3Bucket(bucket_name="bucket", credentials=credentials)
return _s3_bucket

@pytest.fixture
def s3_bucket_2_empty(self, credentials, bucket):
_s3_bucket = S3Bucket(
bucket_name="bucket",
credentials=credentials,
bucket_folder="subfolder",
)
return _s3_bucket

@pytest.fixture
def s3_bucket_with_object(self, s3_bucket_empty, object):
_s3_bucket_with_object = s3_bucket_empty # object will be added
return _s3_bucket_with_object

@pytest.fixture
def s3_bucket_2_with_object(self, s3_bucket_2_empty):
_s3_bucket_with_object = s3_bucket_2_empty
s3_bucket_2_empty.write_path("object", content=b"TEST")
return _s3_bucket_with_object

@pytest.fixture
def s3_bucket_with_objects(self, s3_bucket_with_object, object_in_folder):
_s3_bucket_with_objects = (
Expand Down Expand Up @@ -719,12 +734,12 @@ def test_download_folder_to_path(
@pytest.mark.parametrize("client_parameters", aws_clients[-1:], indirect=True)
def test_stream_from(
self,
s3_bucket_with_object: S3Bucket,
s3_bucket_2_with_object: S3Bucket,
s3_bucket_empty: S3Bucket,
client_parameters,
to_path,
):
path = s3_bucket_empty.stream_from(s3_bucket_with_object, "object", to_path)
path = s3_bucket_empty.stream_from(s3_bucket_2_with_object, "object", to_path)
data: bytes = s3_bucket_empty.read_path(path)
assert data == b"TEST"

Expand Down

0 comments on commit 917f452

Please sign in to comment.