Merge branch 'main' into jay/parquet-pyarrow-switch-to-table

.github/workflows/python-package.yml

@@ -243,7 +243,7 @@ jobs:
         mkdir -p /tmp/daft-integration-testing/nginx
         chmod +rw /tmp/daft-integration-testing/nginx
     - name: Spin up IO services
-      uses: isbang/compose-action@v1.4.1
+      uses: isbang/compose-action@v1.5.0
       with:
         compose-file: ./tests/integration/docker-compose/docker-compose.yml
         down-flags: --volumes

Cargo.toml

@@ -51,13 +51,14 @@ members = [
 ]
 
 [workspace.dependencies]
+bytes = "1.4.0"
 futures = "0.3.28"
 html-escape = "0.2.13"
 num-derive = "0.3.3"
 num-traits = "0.2"
 prettytable-rs = "0.10"
 rand = "^0.8"
-serde_json = "1.0.100"
+serde_json = "1.0.103"
 snafu = "0.7.4"
 tokio = {version = "1.29.1", features = ["net", "time", "bytes", "process", "signal", "macros", "rt", "rt-multi-thread"]}
 
@@ -82,7 +83,7 @@ features = ["extension-module", "abi3-py37"]
 version = "0.19.1"
 
 [workspace.dependencies.pyo3-log]
-version = "0.8.2"
+version = "0.8.3"
 
 [workspace.dependencies.serde]
 features = ["derive", "rc"]

requirements-dev.txt

@@ -16,12 +16,12 @@ pytest-benchmark==4.0.0
 pytest-cov==4.1.0
 
 # Testing dependencies
-lxml==4.9.2
+lxml==4.9.3
 dask==2022.2.0; python_version < '3.8'
 dask==2023.5.0; python_version == '3.8'
 dask==2023.6.0; python_version >= '3.9'
 numpy; python_version < '3.9'
-numpy==1.25.0; python_version >= '3.9'
+numpy==1.25.1; python_version >= '3.9'
 pandas==1.3.5; python_version < '3.8'
 pandas==2.0.3; python_version >= '3.8'
 xxhash>=3.0.0

src/daft-core/Cargo.toml

@@ -5,7 +5,7 @@ bincode = {workspace = true}
 chrono = "0.4.26"
 chrono-tz = "0.8.3"
 common-error = {path = "../common/error", default-features = false}
-dyn-clone = "1.0.11"
+dyn-clone = "1.0.12"
 fnv = "1.0.7"
 html-escape = {workspace = true}
 lazy_static = {workspace = true}

src/daft-io/Cargo.toml

@@ -6,8 +6,7 @@ aws-credential-types = {version = "0.55.3", features = ["hardcoded-credentials"]
 aws-sdk-s3 = "0.28.0"
 aws-sig-auth = "0.55.3"
 aws-sigv4 = "0.55.3"
-bincode = {workspace = true}
-bytes = "1.4.0"
+bytes = {workspace = true}
 common-error = {path = "../common/error", default-features = false}
 daft-core = {path = "../daft-core", default-features = false}
 futures = {workspace = true}

src/daft-io/src/lib.rs

@@ -86,6 +86,12 @@ impl From<Error> for DaftError {
     }
 }
 
+impl From<Error> for std::io::Error {
+    fn from(err: Error) -> std::io::Error {
+        std::io::Error::new(std::io::ErrorKind::Other, err)
+    }
+}
+
 type Result<T, E = Error> = std::result::Result<T, E>;
 
 #[derive(Default)]

src/daft-parquet/Cargo.toml

@@ -1,5 +1,8 @@
 [dependencies]
 arrow2 = {workspace = true, features = ["io_parquet", "io_parquet_compression"]}
+async-compat = "0.2.1"
+async-stream = "0.3.5"
+bytes = {workspace = true}
 common-error = {path = "../common/error", default-features = false}
 daft-core = {path = "../daft-core", default-features = false}
 daft-io = {path = "../daft-io", default-features = false}
@@ -9,8 +12,11 @@ log = {workspace = true}
 parquet2 = "0.17.2"
 pyo3 = {workspace = true, optional = true}
 pyo3-log = {workspace = true, optional = true}
+rayon = "1.7.0"
 snafu = {workspace = true}
 tokio = {workspace = true}
+tokio-stream = "0.1.14"
+tokio-util = "0.7.8"
 
 [features]
 default = ["python"]