diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5857d2d53f..6a98988ea0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -101,7 +101,6 @@ repos: pass_filenames: false args: [--workspace, --no-default-features] - - id: clippy name: clippy description: Lint rust sources @@ -111,6 +110,13 @@ repos: types: [rust] pass_filenames: false + - id: check-for-rustls + name: check-for-rustls + description: Lint rust sources + entry: ./tools/check_for_rustls.sh + language: system + types: [rust] + pass_filenames: false - repo: https://github.com/abravalheri/validate-pyproject diff --git a/Cargo.lock b/Cargo.lock index 15742171a2..cb0fac6809 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -451,10 +451,8 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls 0.23.2", - "lazy_static", + "hyper-tls", "pin-project-lite", - "rustls 0.20.8", "tokio", "tower", "tracing", @@ -1080,6 +1078,7 @@ dependencies = [ "lazy_static", "log", "md5", + "openssl-sys", "pyo3", "pyo3-log", "reqwest", @@ -1310,6 +1309,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "foreign_vec" version = "0.1.0" @@ -1718,32 +1732,16 @@ dependencies = [ ] [[package]] -name = "hyper-rustls" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" -dependencies = [ - "http", - "hyper", - "log", - "rustls 0.20.8", - "rustls-native-certs", - "tokio", - "tokio-rustls 0.23.4", -] - -[[package]] -name = "hyper-rustls" -version = "0.24.1" +name = "hyper-tls" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "futures-util", - "http", + "bytes", "hyper", - "rustls 0.21.5", + "native-tls", "tokio", - "tokio-rustls 0.24.1", + "tokio-native-tls", ] [[package]] @@ -2131,6 +2129,24 @@ dependencies = [ "target-features", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "ndarray" version = "0.15.6" @@ -2283,12 +2299,60 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "openssl" +version = "0.10.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.25", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "300.1.3+3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd2c101a165fff9935e34def4669595ab1c7847943c42be86e21503e482be107" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + [[package]] name = "outref" version = "0.5.1" @@ -2804,22 +2868,21 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls 0.24.1", + "hyper-tls", "ipnet", "js-sys", "log", "mime", "mime_guess", + "native-tls", "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.5", - "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", - "tokio-rustls 0.24.1", + "tokio-native-tls", "tokio-util", "tower-service", "url", @@ -2827,7 +2890,6 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", "winreg", ] @@ -2900,61 +2962,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "rustls" -version = "0.20.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" -dependencies = [ - "log", - "ring", - "sct", - "webpki", -] - -[[package]] -name = "rustls" -version = "0.21.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36" -dependencies = [ - "log", - "ring", - "rustls-webpki", - "sct", -] - -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" -dependencies = [ - "base64 0.21.2", -] - -[[package]] -name = "rustls-webpki" -version = "0.101.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "rustversion" version = "1.0.13" @@ -2982,16 +2989,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" -[[package]] -name = "sct" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "security-framework" version = "2.9.1" @@ -3420,23 +3417,12 @@ dependencies = [ ] [[package]] -name = "tokio-rustls" -version = "0.23.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" -dependencies = [ - "rustls 0.20.8", - "tokio", - "webpki", -] - -[[package]] -name = "tokio-rustls" -version = "0.24.1" +name = "tokio-native-tls" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ - "rustls 0.21.5", + "native-tls", "tokio", ] @@ -3619,6 +3605,12 @@ dependencies = [ "getrandom 0.2.10", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -3753,25 +3745,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "webpki-roots" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" -dependencies = [ - "webpki", -] - [[package]] name = "weezl" version = "0.1.7" diff --git a/daft/filesystem.py b/daft/filesystem.py index c1cf0c9190..13df24d715 100644 --- a/daft/filesystem.py +++ b/daft/filesystem.py @@ -16,7 +16,6 @@ import fsspec import pyarrow as pa -from fsspec.implementations.http import HTTPFileSystem from fsspec.registry import get_filesystem_class from loguru import logger from pyarrow.fs import ( @@ -254,6 +253,8 @@ def _resolve_path_and_filesystem( def _is_http_fs(fs: FileSystem) -> bool: """Returns whether the provided pyarrow filesystem is an HTTP filesystem.""" + from fsspec.implementations.http import HTTPFileSystem + return ( isinstance(fs, PyFileSystem) and isinstance(fs.handler, FSSpecHandler) diff --git a/src/daft-io/Cargo.toml b/src/daft-io/Cargo.toml index 07fa06c551..446418ad5f 100644 --- a/src/daft-io/Cargo.toml +++ b/src/daft-io/Cargo.toml @@ -1,22 +1,23 @@ [dependencies] async-recursion = "1.0.4" async-trait = "0.1.71" -aws-config = {version = "0.55.3"} +aws-config = {version = "0.55.3", features = ["native-tls", "rt-tokio", "client-hyper", "credentials-sso"], default-features = false} aws-credential-types = {version = "0.55.3", features = ["hardcoded-credentials"]} -aws-sdk-s3 = "0.28.0" +aws-sdk-s3 = {version = "0.28.0", features = ["native-tls", "rt-tokio"], default-features = false} aws-sig-auth = "0.55.3" aws-sigv4 = "0.55.3" aws-smithy-async = "0.55.3" -azure_storage = {version = "0.13.0", features = ["enable_reqwest_rustls"], default-features = false} -azure_storage_blobs = {version = "0.13.1", features = ["enable_reqwest_rustls"], default-features = false} +azure_storage = {version = "0.13.0", features = ["enable_reqwest"], default-features = false} +azure_storage_blobs = {version = "0.13.1", features = ["enable_reqwest"], default-features = false} bytes = {workspace = true} common-error = {path = "../common/error", default-features = false} common-io-config = {path = "../common/io-config", default-features = false} daft-core = {path = "../daft-core", default-features = false} futures = {workspace = true} -google-cloud-storage = {version = "0.13.0", default-features = false, features = ["rustls-tls", "auth"]} +google-cloud-storage = {version = "0.13.0", default-features = false, features = ["default-tls", "auth"]} lazy_static = {workspace = true} log = {workspace = true} +openssl-sys = {version = "0.9.93", features = ["vendored"]} pyo3 = {workspace = true, optional = true} pyo3-log = {workspace = true, optional = true} serde = {workspace = true} @@ -27,7 +28,7 @@ url = "2.4.0" [dependencies.reqwest] default-features = false -features = ["stream", "rustls-tls"] +features = ["stream", "native-tls"] version = "0.11.18" [dev-dependencies] diff --git a/tools/check_for_rustls.sh b/tools/check_for_rustls.sh new file mode 100755 index 0000000000..210f9833f0 --- /dev/null +++ b/tools/check_for_rustls.sh @@ -0,0 +1,2 @@ +#!/bin/bash +cargo tree --workspace --all-features | grep -vzq rustls