Skip to content

Commit

Permalink
[BUG] Fix write_deltalake add action file path prefix (#3053)
Browse files Browse the repository at this point in the history
  • Loading branch information
kevinzwang authored Oct 16, 2024
1 parent e93dd94 commit 1aaf3cf
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 2 deletions.
5 changes: 4 additions & 1 deletion daft/table/table_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -672,9 +672,12 @@ def __call__(self, written_file):
else:
size = 0

# remove leading slash
path = written_file.path[1:] if written_file.path.startswith("/") else written_file.path

self.parent.add_actions.append(
AddAction(
written_file.path,
path,
size,
self.partition_values,
int(datetime.now().timestamp() * 1000),
Expand Down
2 changes: 1 addition & 1 deletion src/daft-parquet/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ pub enum Error {

#[snafu(display("Parquet reader timed out while trying to read: {path} with a time budget of {duration_ms} ms"))]
FileReadTimeout { path: String, duration_ms: i64 },
#[snafu(display("Internal IO Error when Opening: {path}:\nDetails:\n{source}"))]
#[snafu(display("Internal IO Error when opening: {path}:\nDetails:\n{source}"))]
InternalIOError {
path: String,
source: std::io::Error,
Expand Down
10 changes: 10 additions & 0 deletions tests/io/delta_lake/test_table_write.py
Original file line number Diff line number Diff line change
Expand Up @@ -300,3 +300,13 @@ def test_deltalake_write_partitioned_existing_table(tmp_path):
assert result["rows"] == [1, 1]

check_equal_both_daft_and_delta_rs(df1.concat(df2), path, [("int", "ascending"), ("string", "ascending")])


def test_deltalake_write_roundtrip(tmp_path):
path = tmp_path / "some_table"
df = daft.from_pydict({"a": [1, 2, 3, 4]})
df.write_deltalake(str(path))

read_df = daft.read_deltalake(str(path))
assert df.schema() == read_df.schema()
assert df.to_arrow() == read_df.to_arrow()

0 comments on commit 1aaf3cf

Please sign in to comment.