feat: Introduce Google Cloud Storage Implementation (#4344)

Introduce native support for GCS using OpenDAL Closes #4236 Signed-off-by: Xuanwo <[email protected]>
quickwit-oss · Jan 10, 2024 · d47b6bd · d47b6bd
1 parent 173f647
commit d47b6bd
Show file tree

Hide file tree

Showing 14 changed files with 772 additions and 9 deletions.
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -146,6 +146,11 @@ jobs:
       - name: Run Azurite service
         run: DOCKER_SERVICES=azurite make docker-compose-up
 
+      # GitHub Actions does not allow services to be started with a custom command,
+      # so we are running fake gcs server as a container manually.
+      - name: Run Fake GCS Server service
+        run: DOCKER_SERVICES=fake-gcs-server make docker-compose-up
+
       - name: Run Pulsar service
         run: DOCKER_SERVICES=pulsar make docker-compose-up
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -148,6 +148,18 @@ services:
         - azurite_data:/data
     command: azurite --blobHost 0.0.0.0 --loose
 
+  fake-gcs-server:
+    image: fsouza/fake-gcs-server:${FAKE_GCS_SERVER_VERSION:-1.47.7}
+    container_name: fake-gcs-server
+    ports:
+      - "${MAP_HOST_FAKE_GCS_SERVER:-127.0.0.1}:4443:4443" # Blob store port
+    profiles:
+      - all
+      - fake-gcs-server
+    volumes:
+      - fake_gcs_server_data:/data/sample-bucket
+    command: -scheme http
+
   grafana:
     image: grafana/grafana-oss:${GRAFANA_VERSION:-9.4.7}
     container_name: grafana
@@ -223,3 +235,4 @@ volumes:
   localstack_data:
   postgres_data:
   azurite_data:
+  fake_gcs_server_data:
diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock
diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml
@@ -247,6 +247,9 @@ azure_storage_blobs = { version = "0.13.0", default-features = false, features =
   "enable_reqwest_rustls",
 ] }
 
+opendal = { version = "0.44", default-features = false }
+reqsign = { version = "0.14", default-features = false }
+
 quickwit-actors = { version = "0.7.0", path = "./quickwit-actors" }
 quickwit-aws = { version = "0.7.0", path = "./quickwit-aws" }
 quickwit-cluster = { version = "0.7.0", path = "./quickwit-cluster" }