diff --git a/Cargo.lock b/Cargo.lock index 643d70c..b6c6fe3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,9 +175,9 @@ dependencies = [ [[package]] name = "attohttpc" -version = "0.22.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7" +checksum = "0f77d243921b0979fbbd728dd2d5162e68ac8252976797c24eb5b3a6af9090dc" dependencies = [ "http", "log", @@ -206,12 +206,12 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-creds" -version = "0.34.1" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3776743bb68d4ad02ba30ba8f64373f1be4e082fe47651767171ce75bb2f6cf5" +checksum = "390ad3b77f3e21e01a4a0355865853b681daf1988510b0b15e31c0c4ae7eb0f6" dependencies = [ "attohttpc", - "dirs", + "home", "log", "quick-xml", "rust-ini", @@ -223,9 +223,9 @@ dependencies = [ [[package]] name = "aws-region" -version = "0.25.3" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "056557a61427d0e5ba29dd931031c8ffed4ee7a550e7cd55692a9d8deb0a9dba" +checksum = "42fed2b9fca70f2908268d057a607f2a906f47edbf856ea8587de9038d264e22" dependencies = [ "thiserror", ] @@ -438,6 +438,26 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "const-random" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11df32a13d7892ec42d51d3d175faba5211ffe13ed25d4fb348ac9e9ce835593" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", +] + [[package]] name = "constant_time_eq" version = "0.3.0" @@ -503,6 +523,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -559,31 +585,14 @@ dependencies = [ ] [[package]] -name = "dirs" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.3.7" +name = "dlv-list" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" dependencies = [ - "libc", - "redox_users", - "winapi", + "const-random", ] -[[package]] -name = "dlv-list" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" - [[package]] name = "doc-comment" version = "0.3.3" @@ -888,6 +897,12 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" + [[package]] name = "hashlink" version = "0.7.0" @@ -936,6 +951,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys", +] + [[package]] name = "http" version = "0.2.9" @@ -1368,12 +1392,12 @@ dependencies = [ [[package]] name = "ordered-multimap" -version = "0.4.3" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" +checksum = "4ed8acf08e98e744e5384c8bc63ceb0364e68a6854187221c18df61c4797690e" dependencies = [ "dlv-list", - "hashbrown 0.12.3", + "hashbrown 0.13.2", ] [[package]] @@ -1557,9 +1581,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.26.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" dependencies = [ "memchr", "serde", @@ -1641,17 +1665,6 @@ dependencies = [ "bitflags 1.3.2", ] -[[package]] -name = "redox_users" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" -dependencies = [ - "getrandom", - "redox_syscall 0.2.16", - "thiserror", -] - [[package]] name = "regex" version = "1.9.6" @@ -1711,12 +1724,10 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", - "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", "web-sys", "winreg", ] @@ -1776,9 +1787,9 @@ dependencies = [ [[package]] name = "rust-ini" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" +checksum = "7e2a3bcec1f113553ef1c88aae6c020a369d03d55b58de9869a0908930385091" dependencies = [ "cfg-if", "ordered-multimap", @@ -1786,33 +1797,37 @@ dependencies = [ [[package]] name = "rust-s3" -version = "0.33.0" +version = "0.34.0-rc4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b2ac5ff6acfbe74226fa701b5ef793aaa054055c13ebb7060ad36942956e027" +checksum = "0533896b025761b23147ca1a168c436e1b87d14e460b1f19a6442882d2a3e07f" dependencies = [ "async-trait", "aws-creds", "aws-region", - "base64 0.13.1", + "base64 0.21.4", "bytes", "cfg-if", "futures", "hex", "hmac", "http", + "hyper", + "hyper-tls", "log", "maybe-async", "md5", "minidom", + "native-tls", "percent-encoding", "quick-xml", - "reqwest", "serde", "serde_derive", + "serde_json", "sha2", "thiserror", "time", "tokio", + "tokio-native-tls", "tokio-stream", "url", ] @@ -2303,6 +2318,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -2617,19 +2641,6 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" -[[package]] -name = "wasm-streams" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "web-sys" version = "0.3.64" diff --git a/Cargo.toml b/Cargo.toml index 71bcf23..a5107d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,7 @@ simple_logger = {version = "1.0.1", optional = true} daemonize = { version = "0.5", optional = true } tempfile = { version = "3.3.0", optional = true } workers = { git="https://github.com/threefoldtech/tokio-worker-pool.git" } -rust-s3 = "0.33" +rust-s3 = "0.34.0-rc3" openssl = { version = "0.10", features = ["vendored"] } regex = "1.9.6" diff --git a/src/fungi/meta.rs b/src/fungi/meta.rs index 1c6cce9..78bb265 100644 --- a/src/fungi/meta.rs +++ b/src/fungi/meta.rs @@ -5,6 +5,8 @@ use std::{ use sqlx::{sqlite::SqliteRow, FromRow, Row, SqlitePool}; +use crate::store; + const ID_LEN: usize = 32; const KEY_LEN: usize = 32; const TYPE_MASK: u32 = nix::libc::S_IFMT; @@ -41,7 +43,7 @@ static SCHEMA: &str = include_str!("../../schema/schema.sql"); #[derive(thiserror::Error, Debug)] pub enum Error { - #[error("failed to execute query: {0:#}")] + #[error("failed to execute query: {0}")] SqlError(#[from] sqlx::Error), #[error("invalid hash length")] @@ -53,7 +55,10 @@ pub enum Error { #[error("io error: {0:#}")] IO(#[from] std::io::Error), - #[error("{0:#}")] + #[error("store error: {0}")] + Store(#[from] store::Error), + + #[error("unknown meta error: {0}")] Anyhow(#[from] anyhow::Error), } diff --git a/src/main.rs b/src/main.rs index f08a551..86132f4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -69,9 +69,9 @@ struct PackOptions { #[clap(short, long, action=ArgAction::Append)] store: Vec, - /// no_strip_password strips password from store url, otherwise password will be stored in the fl and then shipped. - /// Some stores like ZDB has a public namespace which means writing requires a password - #[clap(long, default_value_t = false)] + /// no_strip_password strips password from store url, otherwise password will be stored in the fl and then shipped. + /// Some stores like ZDB has a public namespace which means writing requires a password + #[clap(long, default_value_t = false)] no_strip_password: bool, /// target directory to upload @@ -168,11 +168,11 @@ fn mount(opts: MountOptions) -> Result<()> { } match daemon.execute() { - daemonize::Outcome::Parent(Ok(_)) => { + daemonize::Outcome::Parent(result) => { + result.context("daemonize")?; wait_child(target, pid_file); return Ok(()); } - daemonize::Outcome::Parent(Err(err)) => anyhow::bail!("failed to daemonize: {}", err), _ => {} } } @@ -203,11 +203,11 @@ fn wait_child(target: String, mut pid_file: tempfile::NamedTempFile) { } let mut buf = String::new(); if let Err(e) = pid_file.read_to_string(&mut buf) { - error!("failed to read pid_file: {}", e); + error!("failed to read pid_file: {:#}", e); } let pid = buf.parse::(); match pid { - Err(e) => error!("failed to parse pid_file contents {}: {}", buf, e), + Err(e) => error!("failed to parse pid_file contents {}: {:#}", buf, e), Ok(v) => { let _ = signal::kill(Pid::from_raw(v), Signal::SIGTERM); } // probably the child exited on its own diff --git a/src/pack.rs b/src/pack.rs index 3545e5d..a574d59 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -210,11 +210,7 @@ where } // write block to remote store - let block = self - .store - .set(&self.buffer[..size]) - .await - .context("failed to store blob")?; + let block = self.store.set(&self.buffer[..size]).await?; // write block info to meta self.writer.block(ino, &block.id, &block.key).await?; @@ -233,6 +229,7 @@ where type Output = (); async fn run(&mut self, (ino, path): Self::Input) -> Self::Output { + log::info!("uploading {:?}", path); if let Err(err) = self.upload(ino, &path).await { log::error!("failed to upload file ({:?}): {:#}", path, err); } diff --git a/src/store/mod.rs b/src/store/mod.rs index a2e715b..d0b9de6 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -48,7 +48,7 @@ pub enum Error { #[error("store is not available")] Unavailable, - #[error("compression error: {0:#}")] + #[error("compression error: {0}")] Compression(#[from] snap::Error), #[error("encryption error")] @@ -58,7 +58,7 @@ pub enum Error { #[error("multiple error: {0:?}")] Multiple(Box>), - #[error("io error: {0:#}")] + #[error("io error: {0}")] IO(#[from] std::io::Error), #[error("url parse error: {0}")] @@ -67,7 +67,8 @@ pub enum Error { UnknownStore(String), #[error("invalid schema '{0}' expected '{1}'")] InvalidScheme(String, String), - #[error("{0:#}")] + + #[error("unknown store error {0:#}")] Other(#[from] anyhow::Error), } diff --git a/src/store/s3store.rs b/src/store/s3store.rs index 18c7194..ffd192e 100644 --- a/src/store/s3store.rs +++ b/src/store/s3store.rs @@ -2,9 +2,8 @@ use super::{Error, Result, Route, Store}; use anyhow::Context; use futures::Future; -use std::pin::Pin; - use s3::{creds::Credentials, error::S3Error, Bucket, Region}; +use std::pin::Pin; use url::Url; fn get_config>(u: U) -> Result<(Credentials, Region, String)> { @@ -60,6 +59,11 @@ pub fn make(url: &str) -> Pin>>>> struct S3Store { bucket: Bucket, url: String, + // this is only here as a work around for this bug https://github.com/durch/rust-s3/issues/337 + // because rfs uses the store in async (and parallel) matter to upload/download blobs + // we need to synchronize this locally in that store which will hurt performance + // the 2 solutions now is to either wait until this bug is fixed, or switch to another client + // but for now we keep this work around } impl S3Store { @@ -80,17 +84,19 @@ impl Store for S3Store { async fn get(&self, key: &[u8]) -> super::Result> { match self.bucket.get_object(hex::encode(key)).await { Ok(res) => Ok(res.to_vec()), - Err(S3Error::Http(404, _)) => Err(Error::KeyNotFound), + Err(S3Error::HttpFailWithBody(404, _)) => Err(Error::KeyNotFound), Err(S3Error::Io(err)) => Err(Error::IO(err)), - Err(err) => Err(Error::Other(anyhow::Error::from(err))), + Err(err) => Err(anyhow::Error::from(err).into()), } } async fn set(&self, key: &[u8], blob: &[u8]) -> Result<()> { - match self.bucket.put_object(hex::encode(key), blob).await { - Ok(_) => Ok(()), - Err(err) => Err(Error::Other(anyhow::Error::from(err))), - } + self.bucket + .put_object(hex::encode(key), blob) + .await + .context("put object over s3 storage")?; + + Ok(()) } fn routes(&self) -> Vec { @@ -105,8 +111,7 @@ mod test { #[test] fn test_get_config() { let (cred, region, bucket_name) = - get_config("s3s://minioadmin:minioadmin@127.0.0.1:9000/mybucket?region=minio") - .unwrap(); + get_config("s3s://minioadmin:minioadmin@127.0.0.1:9000/mybucket?region=minio").unwrap(); assert_eq!( cred, Credentials {