From f0b3ebc8c8c9ff6591ea43f22d3d6a5828d49e14 Mon Sep 17 00:00:00 2001 From: Remi Dettai Date: Tue, 20 Aug 2024 14:02:25 +0200 Subject: [PATCH] Stop recording deltas when already EOF --- quickwit/quickwit-cli/tests/cli.rs | 2 +- quickwit/quickwit-indexing/src/source/doc_file_reader.rs | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/quickwit/quickwit-cli/tests/cli.rs b/quickwit/quickwit-cli/tests/cli.rs index 2eb7fe76041..63a7b00d29c 100644 --- a/quickwit/quickwit-cli/tests/cli.rs +++ b/quickwit/quickwit-cli/tests/cli.rs @@ -286,7 +286,7 @@ async fn test_ingest_docs_cli() { } #[tokio::test] -async fn test_ingest_docs_cli_twice() { +async fn test_reingest_same_file_cli() { quickwit_common::setup_logging_for_tests(); let index_id = append_random_suffix("test-index-simple"); let test_env = create_test_env(index_id.clone(), TestStorageType::LocalFileSystem) diff --git a/quickwit/quickwit-indexing/src/source/doc_file_reader.rs b/quickwit/quickwit-indexing/src/source/doc_file_reader.rs index f8fc3c0b8d3..071aa1d6cad 100644 --- a/quickwit/quickwit-indexing/src/source/doc_file_reader.rs +++ b/quickwit/quickwit-indexing/src/source/doc_file_reader.rs @@ -190,9 +190,12 @@ impl ObjectUriBatchReader { source_progress: &Progress, source_type: SourceType, ) -> anyhow::Result { + let mut batch_builder = BatchBuilder::new(source_type); + if self.is_eof { + return Ok(batch_builder); + } let limit_num_bytes = self.current_offset + BATCH_NUM_BYTES_LIMIT as usize; let mut new_offset = self.current_offset; - let mut batch_builder = BatchBuilder::new(source_type); while new_offset < limit_num_bytes { if let Some(record) = source_progress .protect_future(self.reader.next_record())