Fix make test-all command

.github/workflows/ci.yml

@@ -13,10 +13,10 @@ on:
 env:
   CARGO_INCREMENTAL: 0
   QW_DISABLE_TELEMETRY: 1
+  QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev
   RUST_BACKTRACE: 1
-  RUSTFLAGS: -Dwarnings
   RUSTDOCFLAGS: -Dwarnings -Arustdoc::private_intra_doc_links
-  TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev
+  RUSTFLAGS: -Dwarnings
 
 # Ensures that we cancel running jobs for the same PR / same workflow.
 concurrency:

.github/workflows/coverage.yml

@@ -12,15 +12,15 @@ on:
       - quickwit/quickwit-*/**
 
 env:
+  AWS_REGION: us-east-1
   AWS_ACCESS_KEY_ID: "placeholder"
   AWS_SECRET_ACCESS_KEY: "placeholder"
-  AWS_REGION: us-east-1
   CARGO_INCREMENTAL: 0
+  PUBSUB_EMULATOR_HOST: "localhost:9898"
   QW_DISABLE_TELEMETRY: 1
   QW_S3_ENDPOINT: "http://localhost:4566" # Services are exposed as localhost because we are not running coverage in a container.
   QW_S3_FORCE_PATH_STYLE_ACCESS: 1
-  TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev
-  PUBSUB_EMULATOR_HOST: "localhost:9898"
+  QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev
 
 jobs:
   test:
@@ -111,6 +111,9 @@ jobs:
           - "9898:8681"
         env:
           PUBSUB_PROJECT1: "quickwit-emulator,emulator_topic:emulator_subscription"
+        credentials:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/ui-ci.yml

@@ -54,7 +54,7 @@ jobs:
       RUST_BACKTRACE: 1
       RUSTFLAGS: -Dwarnings -C lto=off
       RUSTDOCFLAGS: -Dwarnings -Arustdoc::private_intra_doc_links
-      TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@postgres:5432/quickwit-metastore-dev
+      QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@postgres:5432/quickwit-metastore-dev
     steps:
       - uses: actions/checkout@v3
       - uses: c-hive/gha-yarn-cache@v2

quickwit/Makefile

@@ -19,9 +19,10 @@ test-all:
 	AWS_ACCESS_KEY_ID=ignored \
 	AWS_SECRET_ACCESS_KEY=ignored \
 	AWS_REGION=us-east-1 \
+	PUBSUB_EMULATOR_HOST=localhost:8681 \
 	QW_S3_ENDPOINT=http://localhost:4566 \
 	QW_S3_FORCE_PATH_STYLE_ACCESS=1 \
-	TEST_DATABASE_URL=postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev \
+	QW_TEST_DATABASE_URL=postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev \
 	cargo test --all-features
 	cargo test --test failpoints --features fail/failpoints
 

quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs

@@ -118,7 +118,7 @@ impl GcpPubSubSource {
                     })?;
                 ClientConfig::default().with_credentials(credentials).await
             }
-            None => ClientConfig::default().with_auth().await,
+            _ => ClientConfig::default().with_auth().await,
         }
         .context("Failed to create GCP PubSub client config.")?;
 
@@ -141,11 +141,9 @@ impl GcpPubSubSource {
             max_messages_per_pull=%max_messages_per_pull,
             "Starting GCP PubSub source."
         );
-
         if !subscription.exists(Some(RetrySetting::default())).await? {
             anyhow::bail!("GCP PubSub subscription `{subscription_name}` does not exist.");
         }
-
         Ok(Self {
             ctx,
             subscription_name,
@@ -181,7 +179,6 @@ impl Source for GcpPubSubSource {
                     if let Err(err) = resp {
                         warn!("Failed to pull messages from subscription `{}`: {:?}", self.subscription_name, err);
                     }
-
                     if batch.num_bytes >= BATCH_NUM_BYTES_LIMIT {
                         break;
                     }
@@ -314,7 +311,10 @@ mod gcp_pubsub_emulator_tests {
     static GCP_TEST_PROJECT: &str = "quickwit-emulator";
 
     fn get_source_config(subscription: &str) -> SourceConfig {
-        var("PUBSUB_EMULATOR_HOST").expect("test should be run with PUBSUB_EMULATOR_HOST env");
+        var("PUBSUB_EMULATOR_HOST").expect(
+            "environment variable `PUBSUB_EMULATOR_HOST` should be set when running GCP PubSub \
+             source tests",
+        );
         let source_id = append_random_suffix("test-gcp-pubsub-source--source");
         SourceConfig {
             source_id,
@@ -352,18 +352,18 @@ mod gcp_pubsub_emulator_tests {
     }
 
     #[tokio::test]
-    async fn test_gcp_source_invalid_subscription() {
-        quickwit_common::setup_logging_for_tests();
-        let sub = append_random_suffix("test-gcp-pubsub-source--sub");
-        let source_config = get_source_config(&sub);
+    async fn test_gcp_pubsub_source_invalid_subscription() {
+        let subscription =
+            append_random_suffix("test-gcp-pubsub-source--invalid-subscription--subscription");
+        let source_config = get_source_config(&subscription);
 
-        let index_id = append_random_suffix("test-gcp-pubsub-source--process-message--index");
+        let index_id = append_random_suffix("test-gcp-pubsub-source--invalid-subscription--index");
         let index_uid = IndexUid::new(&index_id);
         let metastore = metastore_for_test();
         let SourceParams::GcpPubSub(params) = source_config.clone().source_params else {
             panic!(
                 "Expected `SourceParams::GcpPubSub` source params, got {:?}",
-                source_config.clone().source_params
+                source_config.source_params
             );
         };
         let ctx = SourceExecutionContext::for_test(
@@ -376,12 +376,11 @@ mod gcp_pubsub_emulator_tests {
     }
 
     #[tokio::test]
-    async fn test_gcp_source() {
-        quickwit_common::setup_logging_for_tests();
+    async fn test_gcp_pubsub_source() {
         let universe = Universe::with_accelerated_time();
 
         let topic = append_random_suffix("test-gcp-pubsub-source--topic");
-        let subscription = append_random_suffix("test-gcp-pubsub-source--sub");
+        let subscription = append_random_suffix("test-gcp-pubsub-source--subscription");
         let publisher = create_topic_and_subscription(&topic, &subscription).await;
 
         let source_config = get_source_config(&subscription);
@@ -392,20 +391,18 @@ mod gcp_pubsub_emulator_tests {
         let index_id: String = append_random_suffix("test-gcp-pubsub-source--index");
         let index_uid = IndexUid::new(&index_id);
 
-        let mut msgs = Vec::new();
+        let mut pubsub_messages = Vec::with_capacity(6);
         for i in 0..6 {
-            let msg = PubsubMessage {
+            let pubsub_message = PubsubMessage {
                 data: format!("Message {}", i).into(),
                 ..Default::default()
             };
-            msgs.push(msg);
+            pubsub_messages.push(pubsub_message);
         }
-
-        let publish_resp = publisher.publish_bulk(msgs).await;
-        for val in publish_resp {
-            val.get().await.unwrap();
+        let awaiters = publisher.publish_bulk(pubsub_messages).await;
+        for awaiter in awaiters {
+            awaiter.get().await.unwrap();
         }
-
         let source = source_loader
             .load_source(
                 SourceExecutionContext::for_test(
@@ -439,7 +436,7 @@ mod gcp_pubsub_emulator_tests {
             "Message 5",
         ];
         assert_eq!(messages[0].docs, expected_docs);
-        let expected_state: JsonValue = json!({
+        let expected_exit_state = json!({
             "index_id": index_id,
             "source_id": source_id,
             "subscription": subscription,
@@ -448,6 +445,6 @@ mod gcp_pubsub_emulator_tests {
             "num_invalid_messages": 0,
             "num_consecutive_empty_batches": 4,
         });
-        assert_eq!(exit_state, expected_state);
+        assert_eq!(exit_state, expected_exit_state);
     }
 }

quickwit/quickwit-metastore/src/metastore/postgresql_metastore.rs

@@ -1360,10 +1360,10 @@ impl crate::tests::test_suite::DefaultForTest for PostgresqlMetastore {
         // too catastrophic, as it is limited by the number of concurrent
         // unit tests running (= number of test-threads).
         dotenv::dotenv().ok();
-        let uri: Uri = std::env::var("TEST_DATABASE_URL")
-            .expect("Environment variable `TEST_DATABASE_URL` should be set.")
+        let uri: Uri = std::env::var("QW_TEST_DATABASE_URL")
+            .expect("Environment variable `QW_TEST_DATABASE_URL` should be set.")
             .parse()
-            .expect("Environment variable `TEST_DATABASE_URL` should be a valid URI.");
+            .expect("Environment variable `QW_TEST_DATABASE_URL` should be a valid URI.");
         PostgresqlMetastore::new(&PostgresMetastoreConfig::default(), &uri)
             .await
             .expect("Failed to initialize test PostgreSQL metastore.")

quickwit/quickwit-metastore/src/metastore_resolver.rs

@@ -190,7 +190,7 @@ mod tests {
         let metastore_resolver = MetastoreResolver::unconfigured();
         // If the database defined in the env var or the default one is not up, the
         // test block after making 10 attempts with a timeout of 10s each = 100s.
-        let test_database_url = env::var("TEST_DATABASE_URL").unwrap_or_else(|_| {
+        let test_database_url = env::var("QW_TEST_DATABASE_URL").unwrap_or_else(|_| {
             "postgres://quickwit-dev:quickwit-dev@localhost/quickwit-metastore-dev".to_string()
         });
         let (_uri_protocol, uri_path) = test_database_url.split_once("://").unwrap();