diff --git a/src/daft-io/src/object_io.rs b/src/daft-io/src/object_io.rs index 3e624cb27f..98bc23de31 100644 --- a/src/daft-io/src/object_io.rs +++ b/src/daft-io/src/object_io.rs @@ -116,17 +116,21 @@ pub(crate) async fn recursive_iter( source: Arc, uri: &str, ) -> super::Result>> { + log::debug!(target: "recursive_iter", "starting recursive_iter: with top level of: {uri}"); let (to_rtn_tx, mut to_rtn_rx) = tokio::sync::mpsc::channel(16 * 1024); fn add_to_channel( source: Arc, tx: Sender>, dir: String, ) { + log::debug!(target: "recursive_iter", "recursive_iter: spawning task to list: {dir}"); tokio::spawn(async move { let s = source.iter_dir(&dir, None, None).await; + log::debug!(target: "recursive_iter", "started listing task for {dir}"); let mut s = match s { Ok(s) => s, Err(e) => { + log::debug!(target: "recursive_iter", "Error occurred when listing {dir}\nerror:\n{e}"); tx.send(Err(e)).await.map_err(|se| { super::Error::UnableToSendDataOverChannel { source: se.into() } })?; @@ -142,6 +146,7 @@ pub(crate) async fn recursive_iter( .await .map_err(|e| super::Error::UnableToSendDataOverChannel { source: e.into() })?; } + log::debug!(target: "recursive_iter", "completed listing task for {dir}"); super::Result::Ok(()) }); } diff --git a/src/daft-io/src/s3_like.rs b/src/daft-io/src/s3_like.rs index 22f4db2967..2b4614e110 100644 --- a/src/daft-io/src/s3_like.rs +++ b/src/daft-io/src/s3_like.rs @@ -352,6 +352,7 @@ impl S3LikeSource { range: Option>, region: &Region, ) -> super::Result { + log::debug!("S3 get at {uri}, range: {range:?}, in region: {region}"); let parsed = url::Url::parse(uri).with_context(|_| InvalidUrlSnafu { path: uri })?; let bucket = match parsed.host_str() { Some(s) => Ok(s), @@ -362,6 +363,7 @@ impl S3LikeSource { }?; let key = parsed.path(); if let Some(key) = key.strip_prefix('/') { + log::debug!("S3 get parsed uri: {uri} into Bucket: {bucket}, Key: {key}"); let request = self .get_s3_client(region) .await? @@ -461,6 +463,7 @@ impl S3LikeSource { uri: &str, region: &Region, ) -> super::Result { + log::debug!("S3 head at {uri} in region: {region}"); let parsed = url::Url::parse(uri).with_context(|_| InvalidUrlSnafu { path: uri })?; let bucket = match parsed.host_str() { @@ -472,6 +475,7 @@ impl S3LikeSource { }?; let key = parsed.path(); if let Some(key) = key.strip_prefix('/') { + log::debug!("S3 head parsed uri: {uri} into Bucket: {bucket}, Key: {key}"); let request = self .get_s3_client(region) .await? @@ -547,6 +551,7 @@ impl S3LikeSource { continuation_token: Option, region: &Region, ) -> super::Result { + log::debug!("S3 list_objects: Bucket: {bucket}, Key: {key}, continuation_token: {continuation_token:?} in region: {region}"); let request = self .get_s3_client(region) .await?