From 62bc3bddbacf3603fac1e8724ed988c7f8d0cb75 Mon Sep 17 00:00:00 2001
From: Sammy Sidhu <samster25@users.noreply.github.com>
Date: Tue, 5 Sep 2023 17:51:56 -0700
Subject: [PATCH] [CHORE] Switch to openssl (#1344)

* Switch over from rustls to vendored openssl
---
 .pre-commit-config.yaml   |   8 +-
 Cargo.lock                | 231 +++++++++++++++++---------------------
 daft/filesystem.py        |   3 +-
 src/daft-io/Cargo.toml    |  13 ++-
 tools/check_for_rustls.sh |   2 +
 5 files changed, 120 insertions(+), 137 deletions(-)
 create mode 100755 tools/check_for_rustls.sh

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 5857d2d53f..6a98988ea0 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -101,7 +101,6 @@ repos:
     pass_filenames: false
     args: [--workspace, --no-default-features]
 
-
   - id: clippy
     name: clippy
     description: Lint rust sources
@@ -111,6 +110,13 @@ repos:
     types: [rust]
     pass_filenames: false
 
+  - id: check-for-rustls
+    name: check-for-rustls
+    description: Lint rust sources
+    entry: ./tools/check_for_rustls.sh
+    language: system
+    types: [rust]
+    pass_filenames: false
 
 
 - repo: https://github.com/abravalheri/validate-pyproject
diff --git a/Cargo.lock b/Cargo.lock
index 15742171a2..cb0fac6809 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -451,10 +451,8 @@ dependencies = [
  "http",
  "http-body",
  "hyper",
- "hyper-rustls 0.23.2",
- "lazy_static",
+ "hyper-tls",
  "pin-project-lite",
- "rustls 0.20.8",
  "tokio",
  "tower",
  "tracing",
@@ -1080,6 +1078,7 @@ dependencies = [
  "lazy_static",
  "log",
  "md5",
+ "openssl-sys",
  "pyo3",
  "pyo3-log",
  "reqwest",
@@ -1310,6 +1309,21 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
 [[package]]
 name = "foreign_vec"
 version = "0.1.0"
@@ -1718,32 +1732,16 @@ dependencies = [
 ]
 
 [[package]]
-name = "hyper-rustls"
-version = "0.23.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c"
-dependencies = [
- "http",
- "hyper",
- "log",
- "rustls 0.20.8",
- "rustls-native-certs",
- "tokio",
- "tokio-rustls 0.23.4",
-]
-
-[[package]]
-name = "hyper-rustls"
-version = "0.24.1"
+name = "hyper-tls"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97"
+checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
 dependencies = [
- "futures-util",
- "http",
+ "bytes",
  "hyper",
- "rustls 0.21.5",
+ "native-tls",
  "tokio",
- "tokio-rustls 0.24.1",
+ "tokio-native-tls",
 ]
 
 [[package]]
@@ -2131,6 +2129,24 @@ dependencies = [
  "target-features",
 ]
 
+[[package]]
+name = "native-tls"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
+dependencies = [
+ "lazy_static",
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
 [[package]]
 name = "ndarray"
 version = "0.15.6"
@@ -2283,12 +2299,60 @@ version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
+[[package]]
+name = "openssl"
+version = "0.10.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e"
+dependencies = [
+ "bitflags 1.3.2",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.25",
+]
+
 [[package]]
 name = "openssl-probe"
 version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
+[[package]]
+name = "openssl-src"
+version = "300.1.3+3.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd2c101a165fff9935e34def4669595ab1c7847943c42be86e21503e482be107"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.93"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d"
+dependencies = [
+ "cc",
+ "libc",
+ "openssl-src",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "outref"
 version = "0.5.1"
@@ -2804,22 +2868,21 @@ dependencies = [
  "http",
  "http-body",
  "hyper",
- "hyper-rustls 0.24.1",
+ "hyper-tls",
  "ipnet",
  "js-sys",
  "log",
  "mime",
  "mime_guess",
+ "native-tls",
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
- "rustls 0.21.5",
- "rustls-pemfile",
  "serde",
  "serde_json",
  "serde_urlencoded",
  "tokio",
- "tokio-rustls 0.24.1",
+ "tokio-native-tls",
  "tokio-util",
  "tower-service",
  "url",
@@ -2827,7 +2890,6 @@ dependencies = [
  "wasm-bindgen-futures",
  "wasm-streams",
  "web-sys",
- "webpki-roots",
  "winreg",
 ]
 
@@ -2900,61 +2962,6 @@ dependencies = [
  "windows-sys",
 ]
 
-[[package]]
-name = "rustls"
-version = "0.20.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f"
-dependencies = [
- "log",
- "ring",
- "sct",
- "webpki",
-]
-
-[[package]]
-name = "rustls"
-version = "0.21.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36"
-dependencies = [
- "log",
- "ring",
- "rustls-webpki",
- "sct",
-]
-
-[[package]]
-name = "rustls-native-certs"
-version = "0.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"
-dependencies = [
- "openssl-probe",
- "rustls-pemfile",
- "schannel",
- "security-framework",
-]
-
-[[package]]
-name = "rustls-pemfile"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2"
-dependencies = [
- "base64 0.21.2",
-]
-
-[[package]]
-name = "rustls-webpki"
-version = "0.101.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e"
-dependencies = [
- "ring",
- "untrusted",
-]
-
 [[package]]
 name = "rustversion"
 version = "1.0.13"
@@ -2982,16 +2989,6 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
-[[package]]
-name = "sct"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"
-dependencies = [
- "ring",
- "untrusted",
-]
-
 [[package]]
 name = "security-framework"
 version = "2.9.1"
@@ -3420,23 +3417,12 @@ dependencies = [
 ]
 
 [[package]]
-name = "tokio-rustls"
-version = "0.23.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"
-dependencies = [
- "rustls 0.20.8",
- "tokio",
- "webpki",
-]
-
-[[package]]
-name = "tokio-rustls"
-version = "0.24.1"
+name = "tokio-native-tls"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
 dependencies = [
- "rustls 0.21.5",
+ "native-tls",
  "tokio",
 ]
 
@@ -3619,6 +3605,12 @@ dependencies = [
  "getrandom 0.2.10",
 ]
 
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
 [[package]]
 name = "version_check"
 version = "0.9.4"
@@ -3753,25 +3745,6 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "webpki"
-version = "0.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"
-dependencies = [
- "ring",
- "untrusted",
-]
-
-[[package]]
-name = "webpki-roots"
-version = "0.22.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87"
-dependencies = [
- "webpki",
-]
-
 [[package]]
 name = "weezl"
 version = "0.1.7"
diff --git a/daft/filesystem.py b/daft/filesystem.py
index c1cf0c9190..13df24d715 100644
--- a/daft/filesystem.py
+++ b/daft/filesystem.py
@@ -16,7 +16,6 @@
 
 import fsspec
 import pyarrow as pa
-from fsspec.implementations.http import HTTPFileSystem
 from fsspec.registry import get_filesystem_class
 from loguru import logger
 from pyarrow.fs import (
@@ -254,6 +253,8 @@ def _resolve_path_and_filesystem(
 
 def _is_http_fs(fs: FileSystem) -> bool:
     """Returns whether the provided pyarrow filesystem is an HTTP filesystem."""
+    from fsspec.implementations.http import HTTPFileSystem
+
     return (
         isinstance(fs, PyFileSystem)
         and isinstance(fs.handler, FSSpecHandler)
diff --git a/src/daft-io/Cargo.toml b/src/daft-io/Cargo.toml
index 07fa06c551..446418ad5f 100644
--- a/src/daft-io/Cargo.toml
+++ b/src/daft-io/Cargo.toml
@@ -1,22 +1,23 @@
 [dependencies]
 async-recursion = "1.0.4"
 async-trait = "0.1.71"
-aws-config = {version = "0.55.3"}
+aws-config = {version = "0.55.3", features = ["native-tls", "rt-tokio", "client-hyper", "credentials-sso"], default-features = false}
 aws-credential-types = {version = "0.55.3", features = ["hardcoded-credentials"]}
-aws-sdk-s3 = "0.28.0"
+aws-sdk-s3 = {version = "0.28.0", features = ["native-tls", "rt-tokio"], default-features = false}
 aws-sig-auth = "0.55.3"
 aws-sigv4 = "0.55.3"
 aws-smithy-async = "0.55.3"
-azure_storage = {version = "0.13.0", features = ["enable_reqwest_rustls"], default-features = false}
-azure_storage_blobs = {version = "0.13.1", features = ["enable_reqwest_rustls"], default-features = false}
+azure_storage = {version = "0.13.0", features = ["enable_reqwest"], default-features = false}
+azure_storage_blobs = {version = "0.13.1", features = ["enable_reqwest"], default-features = false}
 bytes = {workspace = true}
 common-error = {path = "../common/error", default-features = false}
 common-io-config = {path = "../common/io-config", default-features = false}
 daft-core = {path = "../daft-core", default-features = false}
 futures = {workspace = true}
-google-cloud-storage = {version = "0.13.0", default-features = false, features = ["rustls-tls", "auth"]}
+google-cloud-storage = {version = "0.13.0", default-features = false, features = ["default-tls", "auth"]}
 lazy_static = {workspace = true}
 log = {workspace = true}
+openssl-sys = {version = "0.9.93", features = ["vendored"]}
 pyo3 = {workspace = true, optional = true}
 pyo3-log = {workspace = true, optional = true}
 serde = {workspace = true}
@@ -27,7 +28,7 @@ url = "2.4.0"
 
 [dependencies.reqwest]
 default-features = false
-features = ["stream", "rustls-tls"]
+features = ["stream", "native-tls"]
 version = "0.11.18"
 
 [dev-dependencies]
diff --git a/tools/check_for_rustls.sh b/tools/check_for_rustls.sh
new file mode 100755
index 0000000000..210f9833f0
--- /dev/null
+++ b/tools/check_for_rustls.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+cargo tree --workspace --all-features | grep -vzq rustls