Skip to content

Commit

Permalink
Fix missing end in local file storage get_slice_stream
Browse files Browse the repository at this point in the history
  • Loading branch information
rdettai committed Oct 4, 2023
1 parent 704dfac commit a2bc35d
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 8 deletions.
8 changes: 4 additions & 4 deletions quickwit/quickwit-indexing/src/source/file_source.rs
Original file line number Diff line number Diff line change
Expand Up @@ -175,16 +175,16 @@ impl TypedSourceFactory for FileSourceFactory {
}
}

pub fn dir_and_filename(filepath: &Path) -> anyhow::Result<(Uri, &Path)> {
pub(crate) fn dir_and_filename(filepath: &Path) -> anyhow::Result<(Uri, &Path)> {
let dir_uri: Uri = filepath
.parent()
.ok_or_else(|| anyhow::anyhow!("Parent directory could not be resolved"))?
.context("Parent directory could not be resolved")?
.to_str()
.ok_or_else(|| anyhow::anyhow!("Path cannot be turned to string"))?
.context("Path cannot be turned to string")?
.parse()?;
let file_name = filepath
.file_name()
.ok_or_else(|| anyhow::anyhow!("Path does not appear to be a file"))?;
.context("Path does not appear to be a file")?;
Ok((dir_uri, file_name.as_ref()))
}

Expand Down
21 changes: 19 additions & 2 deletions quickwit/quickwit-storage/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ pub(crate) mod test_suite {
use std::path::Path;

use anyhow::Context;
use tokio::io::AsyncReadExt;

use crate::{Storage, StorageErrorKind};

Expand All @@ -174,6 +175,21 @@ pub(crate) mod test_suite {
Ok(())
}

async fn test_write_and_get_slice_stream(storage: &mut dyn Storage) -> anyhow::Result<()> {
let test_path = Path::new("write_and_read_slice_stream");
storage
.put(
test_path,
Box::new(b"abcdefghiklmnopqrstuvxyz"[..].to_vec()),
)
.await?;
let mut reader = storage.get_slice_stream(test_path, 3..6).await?;
let mut buf = vec![0; 3];
reader.read_exact(&mut buf).await?;
assert_eq!(&buf[..], b"def");
Ok(())
}

async fn test_write_get_all(storage: &mut dyn Storage) -> anyhow::Result<()> {
let test_path = Path::new("write_and_read_all");
storage
Expand Down Expand Up @@ -294,6 +310,9 @@ pub(crate) mod test_suite {
test_write_and_get_slice(storage)
.await
.context("write_and_get_slice")?;
test_write_and_get_slice_stream(storage)
.await
.context("write_and_get_slice_stream")?;
test_write_get_all(storage)
.await
.context("write_and_get_all")?;
Expand All @@ -320,8 +339,6 @@ pub(crate) mod test_suite {
pub async fn storage_test_single_part_upload(storage: &mut dyn Storage) -> anyhow::Result<()> {
use std::ops::Range;

use tokio::io::AsyncReadExt;

let test_path = Path::new("hello_small.txt");
let data = b"hello, happy tax payer!";
let data_size = data.len() as u64;
Expand Down
4 changes: 2 additions & 2 deletions quickwit/quickwit-storage/src/local_file_storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ use futures::StreamExt;
use quickwit_common::ignore_error_kind;
use quickwit_common::uri::Uri;
use quickwit_config::StorageBackend;
use tokio::io::{AsyncRead, AsyncSeekExt, AsyncWriteExt};
use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
use tracing::warn;

use crate::storage::SendableAsync;
Expand Down Expand Up @@ -239,7 +239,7 @@ impl Storage for LocalFileStorage {
let full_path = self.full_path(path)?;
let mut file = tokio::fs::File::open(&full_path).await?;
file.seek(SeekFrom::Start(range.start as u64)).await?;
Ok(Box::new(file))
Ok(Box::new(file.take(range.len() as u64)))
}

async fn delete(&self, path: &Path) -> StorageResult<()> {
Expand Down

0 comments on commit a2bc35d

Please sign in to comment.