From 50875d01fff86a929453fa6768150451ab7e5783 Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Thu, 17 Aug 2023 13:10:53 -0400 Subject: [PATCH] Fix `make test-all` command --- .github/workflows/ci.yml | 4 +- .github/workflows/coverage.yml | 9 ++-- .github/workflows/ui-ci.yml | 2 +- quickwit/Makefile | 3 +- .../src/source/gcp_pubsub_source.rs | 45 +++++++++---------- .../src/metastore/postgresql_metastore.rs | 6 +-- .../src/metastore_resolver.rs | 2 +- 7 files changed, 36 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6da350bbc9..9a9d184067e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,10 +13,10 @@ on: env: CARGO_INCREMENTAL: 0 QW_DISABLE_TELEMETRY: 1 + QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev RUST_BACKTRACE: 1 - RUSTFLAGS: -Dwarnings RUSTDOCFLAGS: -Dwarnings -Arustdoc::private_intra_doc_links - TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev + RUSTFLAGS: -Dwarnings # Ensures that we cancel running jobs for the same PR / same workflow. concurrency: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 69e1542909c..23b2765edec 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -12,15 +12,15 @@ on: - quickwit/quickwit-*/** env: + AWS_REGION: us-east-1 AWS_ACCESS_KEY_ID: "placeholder" AWS_SECRET_ACCESS_KEY: "placeholder" - AWS_REGION: us-east-1 CARGO_INCREMENTAL: 0 + PUBSUB_EMULATOR_HOST: "localhost:9898" QW_DISABLE_TELEMETRY: 1 QW_S3_ENDPOINT: "http://localhost:4566" # Services are exposed as localhost because we are not running coverage in a container. QW_S3_FORCE_PATH_STYLE_ACCESS: 1 - TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev - PUBSUB_EMULATOR_HOST: "localhost:9898" + QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev jobs: test: @@ -111,6 +111,9 @@ jobs: - "9898:8681" env: PUBSUB_PROJECT1: "quickwit-emulator,emulator_topic:emulator_subscription" + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/ui-ci.yml b/.github/workflows/ui-ci.yml index d8e1c8d8cca..9e384220085 100644 --- a/.github/workflows/ui-ci.yml +++ b/.github/workflows/ui-ci.yml @@ -54,7 +54,7 @@ jobs: RUST_BACKTRACE: 1 RUSTFLAGS: -Dwarnings -C lto=off RUSTDOCFLAGS: -Dwarnings -Arustdoc::private_intra_doc_links - TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@postgres:5432/quickwit-metastore-dev + QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@postgres:5432/quickwit-metastore-dev steps: - uses: actions/checkout@v3 - uses: c-hive/gha-yarn-cache@v2 diff --git a/quickwit/Makefile b/quickwit/Makefile index 6bdfc8a7429..396725d7e03 100644 --- a/quickwit/Makefile +++ b/quickwit/Makefile @@ -19,9 +19,10 @@ test-all: AWS_ACCESS_KEY_ID=ignored \ AWS_SECRET_ACCESS_KEY=ignored \ AWS_REGION=us-east-1 \ + PUBSUB_EMULATOR_HOST=localhost:8681 \ QW_S3_ENDPOINT=http://localhost:4566 \ QW_S3_FORCE_PATH_STYLE_ACCESS=1 \ - TEST_DATABASE_URL=postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev \ + QW_TEST_DATABASE_URL=postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev \ cargo test --all-features cargo test --test failpoints --features fail/failpoints diff --git a/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs b/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs index f42f715bd99..27ce24eb5b4 100644 --- a/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs +++ b/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs @@ -118,7 +118,7 @@ impl GcpPubSubSource { })?; ClientConfig::default().with_credentials(credentials).await } - None => ClientConfig::default().with_auth().await, + _ => ClientConfig::default().with_auth().await, } .context("Failed to create GCP PubSub client config.")?; @@ -141,11 +141,9 @@ impl GcpPubSubSource { max_messages_per_pull=%max_messages_per_pull, "Starting GCP PubSub source." ); - if !subscription.exists(Some(RetrySetting::default())).await? { anyhow::bail!("GCP PubSub subscription `{subscription_name}` does not exist."); } - Ok(Self { ctx, subscription_name, @@ -181,7 +179,6 @@ impl Source for GcpPubSubSource { if let Err(err) = resp { warn!("Failed to pull messages from subscription `{}`: {:?}", self.subscription_name, err); } - if batch.num_bytes >= BATCH_NUM_BYTES_LIMIT { break; } @@ -314,7 +311,10 @@ mod gcp_pubsub_emulator_tests { static GCP_TEST_PROJECT: &str = "quickwit-emulator"; fn get_source_config(subscription: &str) -> SourceConfig { - var("PUBSUB_EMULATOR_HOST").expect("test should be run with PUBSUB_EMULATOR_HOST env"); + var("PUBSUB_EMULATOR_HOST").expect( + "environment variable `PUBSUB_EMULATOR_HOST` should be set when running GCP PubSub \ + source tests", + ); let source_id = append_random_suffix("test-gcp-pubsub-source--source"); SourceConfig { source_id, @@ -352,18 +352,18 @@ mod gcp_pubsub_emulator_tests { } #[tokio::test] - async fn test_gcp_source_invalid_subscription() { - quickwit_common::setup_logging_for_tests(); - let sub = append_random_suffix("test-gcp-pubsub-source--sub"); - let source_config = get_source_config(&sub); + async fn test_gcp_pubsub_source_invalid_subscription() { + let subscription = + append_random_suffix("test-gcp-pubsub-source--invalid-subscription--subscription"); + let source_config = get_source_config(&subscription); - let index_id = append_random_suffix("test-gcp-pubsub-source--process-message--index"); + let index_id = append_random_suffix("test-gcp-pubsub-source--invalid-subscription--index"); let index_uid = IndexUid::new(&index_id); let metastore = metastore_for_test(); let SourceParams::GcpPubSub(params) = source_config.clone().source_params else { panic!( "Expected `SourceParams::GcpPubSub` source params, got {:?}", - source_config.clone().source_params + source_config.source_params ); }; let ctx = SourceExecutionContext::for_test( @@ -376,12 +376,11 @@ mod gcp_pubsub_emulator_tests { } #[tokio::test] - async fn test_gcp_source() { - quickwit_common::setup_logging_for_tests(); + async fn test_gcp_pubsub_source() { let universe = Universe::with_accelerated_time(); let topic = append_random_suffix("test-gcp-pubsub-source--topic"); - let subscription = append_random_suffix("test-gcp-pubsub-source--sub"); + let subscription = append_random_suffix("test-gcp-pubsub-source--subscription"); let publisher = create_topic_and_subscription(&topic, &subscription).await; let source_config = get_source_config(&subscription); @@ -392,20 +391,18 @@ mod gcp_pubsub_emulator_tests { let index_id: String = append_random_suffix("test-gcp-pubsub-source--index"); let index_uid = IndexUid::new(&index_id); - let mut msgs = Vec::new(); + let mut pubsub_messages = Vec::with_capacity(6); for i in 0..6 { - let msg = PubsubMessage { + let pubsub_message = PubsubMessage { data: format!("Message {}", i).into(), ..Default::default() }; - msgs.push(msg); + pubsub_messages.push(pubsub_message); } - - let publish_resp = publisher.publish_bulk(msgs).await; - for val in publish_resp { - val.get().await.unwrap(); + let awaiters = publisher.publish_bulk(pubsub_messages).await; + for awaiter in awaiters { + awaiter.get().await.unwrap(); } - let source = source_loader .load_source( SourceExecutionContext::for_test( @@ -439,7 +436,7 @@ mod gcp_pubsub_emulator_tests { "Message 5", ]; assert_eq!(messages[0].docs, expected_docs); - let expected_state: JsonValue = json!({ + let expected_exit_state = json!({ "index_id": index_id, "source_id": source_id, "subscription": subscription, @@ -448,6 +445,6 @@ mod gcp_pubsub_emulator_tests { "num_invalid_messages": 0, "num_consecutive_empty_batches": 4, }); - assert_eq!(exit_state, expected_state); + assert_eq!(exit_state, expected_exit_state); } } diff --git a/quickwit/quickwit-metastore/src/metastore/postgresql_metastore.rs b/quickwit/quickwit-metastore/src/metastore/postgresql_metastore.rs index b83898ffe36..cebd2ed0ee1 100644 --- a/quickwit/quickwit-metastore/src/metastore/postgresql_metastore.rs +++ b/quickwit/quickwit-metastore/src/metastore/postgresql_metastore.rs @@ -1360,10 +1360,10 @@ impl crate::tests::test_suite::DefaultForTest for PostgresqlMetastore { // too catastrophic, as it is limited by the number of concurrent // unit tests running (= number of test-threads). dotenv::dotenv().ok(); - let uri: Uri = std::env::var("TEST_DATABASE_URL") - .expect("Environment variable `TEST_DATABASE_URL` should be set.") + let uri: Uri = std::env::var("QW_TEST_DATABASE_URL") + .expect("Environment variable `QW_TEST_DATABASE_URL` should be set.") .parse() - .expect("Environment variable `TEST_DATABASE_URL` should be a valid URI."); + .expect("Environment variable `QW_TEST_DATABASE_URL` should be a valid URI."); PostgresqlMetastore::new(&PostgresMetastoreConfig::default(), &uri) .await .expect("Failed to initialize test PostgreSQL metastore.") diff --git a/quickwit/quickwit-metastore/src/metastore_resolver.rs b/quickwit/quickwit-metastore/src/metastore_resolver.rs index d44a4084f6e..780aee86095 100644 --- a/quickwit/quickwit-metastore/src/metastore_resolver.rs +++ b/quickwit/quickwit-metastore/src/metastore_resolver.rs @@ -190,7 +190,7 @@ mod tests { let metastore_resolver = MetastoreResolver::unconfigured(); // If the database defined in the env var or the default one is not up, the // test block after making 10 attempts with a timeout of 10s each = 100s. - let test_database_url = env::var("TEST_DATABASE_URL").unwrap_or_else(|_| { + let test_database_url = env::var("QW_TEST_DATABASE_URL").unwrap_or_else(|_| { "postgres://quickwit-dev:quickwit-dev@localhost/quickwit-metastore-dev".to_string() }); let (_uri_protocol, uri_path) = test_database_url.split_once("://").unwrap();