diff --git a/src/cli.rs b/src/cli.rs index dde71420..9a43e77a 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -843,6 +843,42 @@ pub fn cli() -> Command { .help("Set timeout for download in seconds") .value_parser(clap::value_parser!(u64)) ) + + .arg(Arg::new("recursive") + .action(ArgAction::SetTrue) + .required(false) + .long("recursive") + .help("Download the sources and all the dependency sources") + ) + + .arg(Arg::new("image") + .required(false) + .value_name("IMAGE NAME") + .short('I') + .long("image") + .help("Name of the Docker image to use") + .long_help(indoc::indoc!(r#" + Name of the Docker image to use. + + Required because tree might look different on different images because of + conditions on dependencies. + "#)) + ) + + .arg(Arg::new("env") + .required(false) + .action(ArgAction::Append) + .short('E') + .long("env") + .value_parser(env_pass_validator) + .help("Additional env to be passed when building packages") + .long_help(indoc::indoc!(r#" + Additional env to be passed when building packages. + + Required because tree might look different on different images because of + conditions on dependencies. + "#)) + ) ) .subcommand(Command::new("of") .about("Get the paths of the sources of a package") diff --git a/src/commands/source/download.rs b/src/commands/source/download.rs index ff9ba9a0..a2a670c4 100644 --- a/src/commands/source/download.rs +++ b/src/commands/source/download.rs @@ -8,7 +8,10 @@ // SPDX-License-Identifier: EPL-2.0 // +use std::collections::HashSet; use std::concat; +use std::fmt; +use std::fmt::Display; use std::path::PathBuf; use std::sync::Arc; @@ -16,18 +19,24 @@ use anyhow::anyhow; use anyhow::Context; use anyhow::Error; use anyhow::Result; +use ascii_table::{Align, AsciiTable}; use clap::ArgMatches; use futures::stream::{FuturesUnordered, StreamExt}; use regex::Regex; use tokio::io::AsyncWriteExt; use tokio::sync::{Mutex, Semaphore}; -use tracing::{info, trace, warn}; +use tracing::{debug, error, info, trace, warn}; -use crate::config::*; -use crate::package::Package; +use crate::config::Configuration; +use crate::package::condition::ConditionData; +use crate::package::Dag; use crate::package::PackageName; use crate::package::PackageVersionConstraint; use crate::repository::Repository; +use crate::util::docker::ImageNameLookup; +use crate::util::EnvironmentVariableName; + +use crate::package::Package; use crate::source::*; use crate::util::progress::ProgressBars; @@ -42,6 +51,17 @@ enum DownloadResult { MarkedManual, } +impl fmt::Display for DownloadResult { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + DownloadResult::Forced => write!(f, "forced"), + DownloadResult::Skipped => write!(f, "skipped"), + DownloadResult::Succeeded => write!(f, "succeeded"), + DownloadResult::MarkedManual => write!(f, "marked manual"), + } + } +} + /// A wrapper around the indicatif::ProgressBar /// /// A wrapper around the indicatif::ProgressBar that is used to synchronize status information from @@ -270,6 +290,7 @@ pub async fn download( progressbars: ProgressBars, ) -> Result<()> { let force = matches.get_flag("force"); + let recursive = matches.get_flag("recursive"); let timeout = matches.get_one::("timeout").copied(); let cache = PathBuf::from(config.source_cache_root()); let sc = SourceCache::new(cache); @@ -294,9 +315,45 @@ pub async fn download( NUMBER_OF_MAX_CONCURRENT_DOWNLOADS, )); - let r = find_packages(&repo, pname, pvers, matching_regexp)?; + let found_packages = find_packages(&repo, pname, pvers, matching_regexp)?; + + let packages_to_download: HashSet = match recursive { + true => { + debug!("Finding package dependencies recursively"); + + let image_name_lookup = ImageNameLookup::create(config.docker().images())?; + let image_name = matches + .get_one::("image") + .map(|s| image_name_lookup.expand(s)) + .transpose()?; + + let additional_env = matches + .get_many::("env") + .unwrap_or_default() + .map(AsRef::as_ref) + .map(crate::util::env::parse_to_env) + .collect::>>()?; + + let condition_data = ConditionData { + image_name: image_name.as_ref(), + env: &additional_env, + }; + + let dependencies: Vec = found_packages + .iter() + .flat_map(|package| { + Dag::for_root_package((*package).clone(), &repo, None, &condition_data) + .map(|d| d.dag().graph().node_weights().cloned().collect::>()) + .unwrap() + }) + .collect(); + + HashSet::from_iter(dependencies) + } + false => HashSet::from_iter(found_packages.into_iter().cloned()), + }; - let r: Vec<(SourceEntry, Result)> = r + let mut r: Vec<(SourceEntry, Result)> = packages_to_download .iter() .flat_map(|p| { sc.sources_for(p).into_iter().map(|source| { @@ -314,6 +371,71 @@ pub async fn download( .collect() .await; + { + let mut ascii_table = AsciiTable::default(); + ascii_table.set_max_width( + terminal_size::terminal_size() + .map(|tpl| tpl.0 .0 as usize) + .unwrap_or(80), + ); + ascii_table.column(0).set_header("#").set_align(Align::Left); + ascii_table + .column(1) + .set_header("Package name") + .set_align(Align::Left); + ascii_table + .column(2) + .set_header("Version") + .set_align(Align::Left); + ascii_table + .column(3) + .set_header("Source name") + .set_align(Align::Left); + ascii_table + .column(4) + .set_header("Status") + .set_align(Align::Left); + ascii_table + .column(5) + .set_header("Path") + .set_align(Align::Left); + + let numbers: Vec = (0..r.len()).map(|n| n + 1).collect(); + r.sort_by(|(a, _), (b, _)| a.package_name().partial_cmp(b.package_name()).unwrap()); + let source_paths: Vec = r.iter().map(|v| v.0.path_as_string()).collect(); + + let data: Vec> = r + .iter() + .enumerate() + .map(|(i, v)| { + debug!("source_entry: {:#?}", v.0); + let n = &numbers[i]; + let mut row: Vec<&dyn Display> = vec![ + n, + v.0.package_name(), + v.0.package_version(), + v.0.package_source_name(), + ]; + if v.1.is_ok() { + let result = v.1.as_ref().unwrap() as &dyn Display; + row.push(result); + } else { + row.push(&"failed"); + } + row.push(&source_paths[i]); + row + }) + .collect(); + + ascii_table.print(data); + } + + for p in r { + if p.1.is_err() { + error!("{}: {:?}", p.0.package_name(), p.1); + } + } + super::verify(matches, config, repo, progressbars).await?; Ok(()) diff --git a/src/source/mod.rs b/src/source/mod.rs index 8e1fa830..8e9306ed 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -14,6 +14,7 @@ use anyhow::anyhow; use anyhow::Context; use anyhow::Error; use anyhow::Result; +use getset::Getters; use tracing::trace; use url::Url; @@ -37,11 +38,14 @@ impl SourceCache { } } -#[derive(Debug)] +#[derive(Debug, Getters)] pub struct SourceEntry { cache_root: PathBuf, + #[getset(get = "pub")] package_name: PackageName, + #[getset(get = "pub")] package_version: PackageVersion, + #[getset(get = "pub")] package_source_name: String, package_source: Source, } @@ -73,6 +77,10 @@ impl SourceEntry { }) } + pub fn path_as_string(&self) -> String { + self.path().to_string_lossy().to_string() + } + pub fn url(&self) -> &Url { self.package_source.url() }