diff --git a/CHANGES.md b/CHANGES.md index 1cd63db..8c6af7d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,15 @@ # Release Notes +## 0.12.0 + +This release adds support for using placeholders in the `scie.lift.base` lift manifest value as well +as the corresponding `SCIE_BASE` runtime control env var. A new placeholder is exposed in support of +establishing custom scie base `nce` cache directories that respect the target OS user cache dir +structure in the form of `{scie.user.cache_dir=}`. Typically, this placeholder will expand +to `~/Library/Caches` on macOS, `~\AppData\Local` on Windows and `~/.cache` on all other Unix +systems unless over-ridden via OS-specific means or else unavailable for some reason, in which case +the supplied `` is used. + ## 0.11.1 This release fixes a bug handling environment variable removal via regex when the environment diff --git a/Cargo.lock b/Cargo.lock index 48ebb04..f6a35d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -427,7 +427,7 @@ dependencies = [ [[package]] name = "jump" -version = "0.11.1" +version = "0.12.0" dependencies = [ "bstr", "byteorder", @@ -749,7 +749,7 @@ dependencies = [ [[package]] name = "scie-jump" -version = "0.11.1" +version = "0.12.0" dependencies = [ "bstr", "env_logger", diff --git a/Cargo.toml b/Cargo.toml index 470c382..79c7e0d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ members = [ [package] name = "scie-jump" -version = "0.11.1" +version = "0.12.0" description = "The self contained interpreted executable launcher." authors = [ "John Sirois ", diff --git a/docs/packaging.md b/docs/packaging.md index f8a5d8d..abf29a5 100644 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -118,6 +118,10 @@ A scie "lift" can opt in to loading `.env` files via the "load_dotenv" boolean f https://crates.io/crates/dotenv) crate handles this loading. A lift's files and commands can also have additional configuration metadata described. +A scie "lift" can also establish a custom `nce` cache directory via the "base" string field. Any +placeholders present in the custom value will be expanded save for the `{scie.lift}` placeholder +which will lead to a scie jump boot error. + For files, you can supply a "size" and sha256 "hash". Without these the boot-pack will calculate them, but you may want to set them in advance as a security precaution. The `scie-jump` will refuse to operate on any file whose size or hash do not match those specified. You can also manually @@ -342,7 +346,9 @@ Runtime external control variables: + `SCIE_BOOT=`: This can be used to select a non-default command for execution in a scie that defines named commands. One way to discover these is via invoking `SCIE=list `. + `SCIE_BASE`: This can be used to override the default scie base of `~/.cache/nce` on Linux, - `~/Library/Caches/nce` on Mac and `~\AppData\Local\nce` on Windows. + `~/Library/Caches/nce` on Mac and `~\AppData\Local\nce` on Windows. Any placeholders save for + `{scie.lift}` will be expanded. If the `{scie.lift}` placeholder is encountered expanding the + `SCIE_BASE` value, a runtime error will abort the scie jump boot. Runtime read-only variables: @@ -376,6 +382,7 @@ Further placeholders you can use in command "exe", "args" and "env" values inclu `linux`, `macos` or `windows` and `` is either `aarch64` or `x86_64`. + `{scie.platform.arch}`: The current chip architecture as described by `` above. + `{scie.platform.os}`: The current operating system as described by `` above. ++ `{scie.user.cache_dir=}`: The default user cache dir or `` if there is none. [^1]: The binaries that Coursier releases are single-file true native binaries that do not require a JVM at all. As such they are ~1/3 the size of the scie we build here, which contains a full JDK diff --git a/jump/Cargo.toml b/jump/Cargo.toml index 249831d..f2f1de8 100644 --- a/jump/Cargo.toml +++ b/jump/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jump" -version = "0.11.1" +version = "0.12.0" description = "The bulk of the scie-jump binary logic." authors = [ "John Sirois ", diff --git a/jump/src/cmd_env.rs b/jump/src/cmd_env.rs index 2c4e66b..a7e897e 100644 --- a/jump/src/cmd_env.rs +++ b/jump/src/cmd_env.rs @@ -78,6 +78,9 @@ impl EnvParser { Item::Placeholder(Placeholder::FileName(name)) => { reified.push_str(&format!("{{scie.files.{name}}}")) } + Item::Placeholder(Placeholder::UserCacheDir(fallback)) => { + reified.push_str(&format!("{{scie.user.cache_dir={fallback}}}")) + } Item::Placeholder(Placeholder::Scie) => reified.push_str("{scie}"), Item::Placeholder(Placeholder::ScieBase) => reified.push_str("{scie.base}"), Item::Placeholder(Placeholder::ScieBindings) => reified.push_str("{scie.bindings}"), @@ -397,4 +400,25 @@ mod tests { assert_eq!(expected, env_parser.parse_env().unwrap()); } + + #[test] + fn test_user_cache_dir_placeholder() { + let env_parser = EnvParser::new( + &[( + EnvVar::Replace("SCIE_BASE".to_string()), + Some("{scie.user.cache_dir=foo}".to_string()), + )] + .into_iter() + .collect::>(), + ); + + let expected = [( + "SCIE_BASE".to_string(), + "{scie.user.cache_dir=foo}".to_string(), + )] + .into_iter() + .collect::>(); + + assert_eq!(expected, env_parser.parse_env().unwrap()); + } } diff --git a/jump/src/config.rs b/jump/src/config.rs index e3997f1..045f272 100644 --- a/jump/src/config.rs +++ b/jump/src/config.rs @@ -3,7 +3,6 @@ use std::fmt::Formatter; use std::io::Write; -use std::path::PathBuf; use indexmap::IndexMap; use serde::de::{Error, Unexpected, Visitor}; @@ -240,7 +239,7 @@ pub struct Lift { pub description: Option, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] - pub base: Option, + pub base: Option, pub files: Vec, pub boot: Boot, #[serde(default)] diff --git a/jump/src/context.rs b/jump/src/context.rs index 0ea6b7f..168646d 100644 --- a/jump/src/context.rs +++ b/jump/src/context.rs @@ -197,6 +197,7 @@ pub(crate) struct SelectedCmd { pub(crate) argv1_consumed: bool, } +#[derive(Clone, Debug)] pub(crate) struct Context<'a> { scie: &'a Path, lift: &'a Lift, @@ -233,15 +234,14 @@ impl<'a> Context<'a> { let base = if let Ok(base) = env::var("SCIE_BASE") { PathBuf::from(base) } else if let Some(base) = &lift.base { - base.clone() + PathBuf::from(base) } else if let Some(dir) = dirs::cache_dir() { dir.join("nce") } else { PathBuf::from("~/.nce") }; let base = expanduser(base.as_path())?; - let lift_manifest = base.join(&lift.hash).join("lift.json"); - Ok(Context { + let mut context = Context { scie, lift, base, @@ -249,7 +249,7 @@ impl<'a> Context<'a> { files_by_name, replacements: HashSet::new(), lift_manifest: LiftManifest { - path: lift_manifest, + path: PathBuf::new(), // N.B.: We replace this empty value below. jump: jump.clone(), lift: lift.clone(), }, @@ -257,7 +257,41 @@ impl<'a> Context<'a> { lift_manifest_installed: false, bound: HashMap::new(), installed: HashSet::new(), - }) + }; + + // Now patch up the base and the lift path (which is derived from it) with any placeholder + // resolving that may be required. + let mut env = IndexMap::new(); + for (key, value) in env::vars_os() { + // TODO(John Sirois): Move from IndexMap to a trait with the method + // `get(&str) -> Option<&str>`. That will allow detecting key accesses with non-UTF-8 + // values to warn or error. + if let (Some(key), Some(value)) = (key.into_string().ok(), value.into_string().ok()) { + env.insert(key, value); + } + } + let (parsed_base, needs_lift_manifest) = context.reify_string( + &env, + &context + .base + .clone() + .into_os_string() + .into_string() + .map_err(|e| { + format!("Failed to interpret the scie.lift.base as a utf-8 string: {e:?}") + })?, + )?; + if needs_lift_manifest { + return Err(format!( + "The scie.lift.base cannot use the placeholder {{scie.lift}} since that \ + placeholder is calculated from the resolved location of the scie.lift.base, \ + given: {base}", + base = context.base.display() + )); + } + context.base = PathBuf::from(parsed_base.clone()); + context.lift_manifest.path = context.base.join(&lift.hash).join("lift.json"); + Ok(context) } fn prepare_process(&mut self, cmd: &'a Cmd) -> Result { @@ -555,6 +589,17 @@ impl<'a> Context<'a> { }; reified.push_str(&value) } + Item::Placeholder(Placeholder::UserCacheDir(fallback)) => { + let (parsed_fallback, needs_manifest) = self.reify_string(env, fallback)?; + lift_manifest_required |= needs_manifest; + reified.push_str(if let Some(user_cache_dir) = dirs::cache_dir() { + user_cache_dir.into_os_string().into_string().map_err(|e| { + format!("Could not interpret the user cache directory as a utf-8 string: {e:?}") + })? + } else { + parsed_fallback + }.as_str()) + } Item::Placeholder(Placeholder::Scie) => reified.push_str(path_to_str(self.scie)?), Item::Placeholder(Placeholder::ScieBase) => { reified.push_str(path_to_str(&self.base)?) @@ -615,7 +660,7 @@ pub(crate) fn select_command( #[cfg(test)] mod tests { use std::env; - use std::path::Path; + use std::path::{Path, PathBuf}; use indexmap::IndexMap; @@ -630,11 +675,16 @@ mod tests { size: 42, version: "0.1.0".to_string(), }; - let tempdir = tempfile::tempdir().unwrap(); let lift = Lift { name: "test".to_string(), description: None, - base: Some(tempdir.path().to_path_buf()), + base: Some( + PathBuf::from("{scie.user.cache_dir={scie.env.USER_CACHE_DIR=/tmp/nce}}") + .join("example") + .into_os_string() + .into_string() + .unwrap(), + ), load_dotenv: true, size: 137, hash: "abc".to_string(), @@ -693,10 +743,23 @@ mod tests { ); env.clear(); + let expected_scie_base = dirs::cache_dir() + .unwrap_or_else(|| PathBuf::from("/tmp/nce")) + .join("example"); + assert_eq!( + ( + expected_scie_base + .clone() + .into_os_string() + .into_string() + .unwrap(), + false + ), + context.reify_string(&mut env, "{scie.base}").unwrap() + ); assert_eq!( ( - tempdir - .path() + expected_scie_base .join("abc") .join("lift.json") .to_str() @@ -709,11 +772,20 @@ mod tests { .unwrap() ); + let mut invalid_lift = lift.clone(); + invalid_lift.base = Some("{scie.lift}/circular".to_string()); + assert_eq!( + "The scie.lift.base cannot use the placeholder {scie.lift} since that placeholder is \ + calculated from the resolved location of the scie.lift.base, given: \ + {scie.lift}/circular" + .to_string(), + Context::new(Path::new("scie_path"), &jump, &invalid_lift, &installer).unwrap_err() + ); + env.clear(); assert_eq!( ( - tempdir - .path() + expected_scie_base .join("def") .join("file") .to_str() @@ -750,11 +822,10 @@ mod tests { size: 42, version: "0.1.0".to_string(), }; - let tempdir = tempfile::tempdir().unwrap(); let lift = Lift { name: "test".to_string(), description: None, - base: Some(tempdir.path().to_path_buf()), + base: Some("/tmp/nce".to_string()), load_dotenv: true, size: 137, hash: "abc".to_string(), @@ -824,7 +895,10 @@ mod tests { assert_eq!( Process { env: expected_env.clone(), - exe: tempdir.path().join("def").join("dist-v1/v2/binary").into(), + exe: PathBuf::from("/tmp/nce") + .join("def") + .join("dist-v1/v2/binary") + .into(), args: vec![], }, process @@ -835,7 +909,10 @@ mod tests { assert_eq!( Process { env: expected_env.clone(), - exe: tempdir.path().join("def").join("dist-v1/v1/exe").into(), + exe: PathBuf::from("/tmp/nce") + .join("def") + .join("dist-v1/v1/exe") + .into(), args: vec![], }, process @@ -847,7 +924,10 @@ mod tests { assert_eq!( Process { env: expected_env, - exe: tempdir.path().join("ghi").join("dist-v2/v2/binary").into(), + exe: PathBuf::from("/tmp/nce") + .join("ghi") + .join("dist-v2/v2/binary") + .into(), args: vec![], }, process @@ -861,11 +941,10 @@ mod tests { size: 42, version: "0.1.0".to_string(), }; - let tempdir = tempfile::tempdir().unwrap(); let lift = Lift { name: "test".to_string(), description: None, - base: Some(tempdir.path().to_path_buf()), + base: Some("/tmp/nce".to_string()), load_dotenv: true, size: 137, hash: "abc".to_string(), diff --git a/jump/src/installer.rs b/jump/src/installer.rs index eaa09c0..1fe6911 100644 --- a/jump/src/installer.rs +++ b/jump/src/installer.rs @@ -164,6 +164,7 @@ where } } +#[derive(Debug)] pub(crate) struct Installer<'a> { payload: &'a [u8], } diff --git a/jump/src/lift.rs b/jump/src/lift.rs index 8cae7d8..1f4a6ac 100644 --- a/jump/src/lift.rs +++ b/jump/src/lift.rs @@ -1,7 +1,7 @@ // Copyright 2022 Science project contributors. // Licensed under the Apache License, Version 2.0 (see LICENSE). -use std::path::{Path, PathBuf}; +use std::path::Path; use bstr::ByteSlice; use logging_timer::time; @@ -52,7 +52,7 @@ impl From for crate::config::File { pub struct Lift { pub name: String, pub description: Option, - pub base: Option, + pub base: Option, pub(crate) load_dotenv: bool, pub size: usize, pub hash: String, diff --git a/jump/src/placeholders.rs b/jump/src/placeholders.rs index 0545244..56129cd 100644 --- a/jump/src/placeholders.rs +++ b/jump/src/placeholders.rs @@ -14,6 +14,7 @@ pub(crate) enum Placeholder<'a> { Env(&'a str), FileHash(&'a str), FileName(&'a str), + UserCacheDir(&'a str), Scie, ScieBase, ScieBindings, @@ -106,6 +107,26 @@ pub(crate) fn parse(text: &str) -> Result { ["scie", "files:hash", name] => { items.push(Item::Placeholder(Placeholder::FileHash(name))) } + ["scie", "user", cache_dir] => { + match cache_dir.splitn(2, '=').collect::>()[..] { + ["cache_dir", fallback] => { + items.push(Item::Placeholder(Placeholder::UserCacheDir(fallback))) + } + ["cache_dir"] => { + return Err( + "The {{scie.user.cache_dir}} requires a fallback value; e.g.: \ + {{scie.user.cache_dir=~/.cache}}" + .to_string(), + ) + } + _ => { + return Err(format!( + "Unrecognized placeholder in the {{scie.user.*}} \ + namespace: {{scie.user.{cache_dir}}}" + )) + } + } + } ["scie", "lift"] => items.push(Item::Placeholder(Placeholder::ScieLift)), ["scie", "platform"] => { items.push(Item::Placeholder(Placeholder::SciePlatform)) @@ -314,6 +335,24 @@ mod tests { ); } + #[test] + fn user_cache_dir() { + assert!(parse("{scie.user.config_dir}").is_err()); + assert!(parse("{scie.user.cache_dir}").is_err()); + assert_eq!( + vec![Item::Placeholder(Placeholder::UserCacheDir("~/fall/back"))], + parse("{scie.user.cache_dir=~/fall/back}").unwrap().items + ); + assert_eq!( + vec![Item::Placeholder(Placeholder::UserCacheDir( + "{scie.env.CACHE_DIR=~/fall/back}" + ))], + parse("{scie.user.cache_dir={scie.env.CACHE_DIR=~/fall/back}}") + .unwrap() + .items + ); + } + #[test] fn platform() { assert_eq!(