Add support for scie base value placeholders. (#137)

Both the lift manifest `scie.lift.base` field and the `SCIE_BASE` env
var now have their values expanded, accepting all placeholders except
for the `{scie.lift}` which depends on the fully evaluated value of the
scie base.

In order to make OS-convention-friendly values feasible, a new
`{scie.user.cache_dir=<fallback>}` placeholder is added that expands to
the OS-appropriate user cache dir when defined and `<fallback>` when
not.

CHANGES.md

@@ -1,5 +1,15 @@
 # Release Notes
 
+## 0.12.0
+
+This release adds support for using placeholders in the `scie.lift.base` lift manifest value as well
+as the corresponding `SCIE_BASE` runtime control env var. A new placeholder is exposed in support of
+establishing custom scie base `nce` cache directories that respect the target OS user cache dir
+structure in the form of `{scie.user.cache_dir=<fallback>}`. Typically, this placeholder will expand
+to `~/Library/Caches` on macOS, `~\AppData\Local` on Windows and `~/.cache` on all other Unix
+systems unless over-ridden via OS-specific means or else unavailable for some reason, in which case
+the supplied `<fallback>` is used.
+
 ## 0.11.1
 
 This release fixes a bug handling environment variable removal via regex when the environment

Cargo.toml

@@ -5,7 +5,7 @@ members = [
 
 [package]
 name = "scie-jump"
-version = "0.11.1"
+version = "0.12.0"
 description = "The self contained interpreted executable launcher."
 authors = [
     "John Sirois <[email protected]>",

docs/packaging.md

@@ -118,6 +118,10 @@ A scie "lift" can opt in to loading `.env` files via the "load_dotenv" boolean f
 https://crates.io/crates/dotenv) crate handles this loading. A lift's files and commands can also
 have additional configuration metadata described.
 
+A scie "lift" can also establish a custom `nce` cache directory via the "base" string field. Any
+placeholders  present in the custom value will be expanded save for the `{scie.lift}` placeholder
+which will lead to a scie jump boot error.
+
 For files, you can supply a "size" and sha256 "hash". Without these the boot-pack will calculate
 them, but you may want to set them in advance as a security precaution. The `scie-jump` will refuse
 to operate on any file whose size or hash do not match those specified. You can also manually
@@ -342,7 +346,9 @@ Runtime external control variables:
 + `SCIE_BOOT=<command>`: This can be used to select a non-default command for execution in a scie
   that defines named commands. One way to discover these is via invoking `SCIE=list <scie path>`.
 + `SCIE_BASE`: This can be used to override the default scie base of `~/.cache/nce` on Linux,
-   `~/Library/Caches/nce` on Mac and `~\AppData\Local\nce` on Windows.
+   `~/Library/Caches/nce` on Mac and `~\AppData\Local\nce` on Windows. Any placeholders save for
+   `{scie.lift}` will be expanded. If the `{scie.lift}` placeholder is encountered expanding the
+   `SCIE_BASE` value, a runtime error will abort the scie jump boot.
 
 Runtime read-only variables:
 
@@ -376,6 +382,7 @@ Further placeholders you can use in command "exe", "args" and "env" values inclu
   `linux`, `macos` or `windows` and `<ARCH>` is either `aarch64` or `x86_64`.
 + `{scie.platform.arch}`: The current chip architecture as described by `<ARCH>` above.
 + `{scie.platform.os}`: The current operating system as described by `<OS>` above.
++ `{scie.user.cache_dir=<fallback>}`: The default user cache dir or `<fallback>` if there is none.
 
 [^1]: The binaries that Coursier releases are single-file true native binaries that do not require a
 JVM at all. As such they are ~1/3 the size of the scie we build here, which contains a full JDK

jump/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "jump"
-version = "0.11.1"
+version = "0.12.0"
 description = "The bulk of the scie-jump binary logic."
 authors = [
     "John Sirois <[email protected]>",

jump/src/cmd_env.rs

@@ -78,6 +78,9 @@ impl EnvParser {
                 Item::Placeholder(Placeholder::FileName(name)) => {
                     reified.push_str(&format!("{{scie.files.{name}}}"))
                 }
+                Item::Placeholder(Placeholder::UserCacheDir(fallback)) => {
+                    reified.push_str(&format!("{{scie.user.cache_dir={fallback}}}"))
+                }
                 Item::Placeholder(Placeholder::Scie) => reified.push_str("{scie}"),
                 Item::Placeholder(Placeholder::ScieBase) => reified.push_str("{scie.base}"),
                 Item::Placeholder(Placeholder::ScieBindings) => reified.push_str("{scie.bindings}"),
@@ -397,4 +400,25 @@ mod tests {
 
         assert_eq!(expected, env_parser.parse_env().unwrap());
     }
+
+    #[test]
+    fn test_user_cache_dir_placeholder() {
+        let env_parser = EnvParser::new(
+            &[(
+                EnvVar::Replace("SCIE_BASE".to_string()),
+                Some("{scie.user.cache_dir=foo}".to_string()),
+            )]
+            .into_iter()
+            .collect::<IndexMap<_, _>>(),
+        );
+
+        let expected = [(
+            "SCIE_BASE".to_string(),
+            "{scie.user.cache_dir=foo}".to_string(),
+        )]
+        .into_iter()
+        .collect::<IndexMap<_, _>>();
+
+        assert_eq!(expected, env_parser.parse_env().unwrap());
+    }
 }

jump/src/config.rs

@@ -3,7 +3,6 @@
 
 use std::fmt::Formatter;
 use std::io::Write;
-use std::path::PathBuf;
 
 use indexmap::IndexMap;
 use serde::de::{Error, Unexpected, Visitor};
@@ -240,7 +239,7 @@ pub struct Lift {
     pub description: Option<String>,
     #[serde(default)]
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub base: Option<PathBuf>,
+    pub base: Option<String>,
     pub files: Vec<File>,
     pub boot: Boot,
     #[serde(default)]

jump/src/context.rs

@@ -197,6 +197,7 @@ pub(crate) struct SelectedCmd {
     pub(crate) argv1_consumed: bool,
 }
 
+#[derive(Clone, Debug)]
 pub(crate) struct Context<'a> {
     scie: &'a Path,
     lift: &'a Lift,
@@ -233,31 +234,64 @@ impl<'a> Context<'a> {
         let base = if let Ok(base) = env::var("SCIE_BASE") {
             PathBuf::from(base)
         } else if let Some(base) = &lift.base {
-            base.clone()
+            PathBuf::from(base)
         } else if let Some(dir) = dirs::cache_dir() {
             dir.join("nce")
         } else {
             PathBuf::from("~/.nce")
         };
         let base = expanduser(base.as_path())?;
-        let lift_manifest = base.join(&lift.hash).join("lift.json");
-        Ok(Context {
+        let mut context = Context {
             scie,
             lift,
             base,
             installer,
             files_by_name,
             replacements: HashSet::new(),
             lift_manifest: LiftManifest {
-                path: lift_manifest,
+                path: PathBuf::new(), // N.B.: We replace this empty value below.
                 jump: jump.clone(),
                 lift: lift.clone(),
             },
             lift_manifest_dependants: HashSet::new(),
             lift_manifest_installed: false,
             bound: HashMap::new(),
             installed: HashSet::new(),
-        })
+        };
+
+        // Now patch up the base and the lift path (which is derived from it) with any placeholder
+        // resolving that may be required.
+        let mut env = IndexMap::new();
+        for (key, value) in env::vars_os() {
+            // TODO(John Sirois): Move from IndexMap to a trait with the method
+            //  `get(&str) -> Option<&str>`. That will allow detecting key accesses with non-UTF-8
+            //  values to warn or error.
+            if let (Some(key), Some(value)) = (key.into_string().ok(), value.into_string().ok()) {
+                env.insert(key, value);
+            }
+        }
+        let (parsed_base, needs_lift_manifest) = context.reify_string(
+            &env,
+            &context
+                .base
+                .clone()
+                .into_os_string()
+                .into_string()
+                .map_err(|e| {
+                    format!("Failed to interpret the scie.lift.base as a utf-8 string: {e:?}")
+                })?,
+        )?;
+        if needs_lift_manifest {
+            return Err(format!(
+                "The scie.lift.base cannot use the placeholder {{scie.lift}} since that \
+                placeholder is calculated from the resolved location of the scie.lift.base, \
+                given: {base}",
+                base = context.base.display()
+            ));
+        }
+        context.base = PathBuf::from(parsed_base.clone());
+        context.lift_manifest.path = context.base.join(&lift.hash).join("lift.json");
+        Ok(context)
     }
 
     fn prepare_process(&mut self, cmd: &'a Cmd) -> Result<Process, String> {
@@ -555,6 +589,17 @@ impl<'a> Context<'a> {
                     };
                     reified.push_str(&value)
                 }
+                Item::Placeholder(Placeholder::UserCacheDir(fallback)) => {
+                    let (parsed_fallback, needs_manifest) = self.reify_string(env, fallback)?;
+                    lift_manifest_required |= needs_manifest;
+                    reified.push_str(if let Some(user_cache_dir) = dirs::cache_dir() {
+                        user_cache_dir.into_os_string().into_string().map_err(|e| {
+                            format!("Could not interpret the user cache directory as a utf-8 string: {e:?}")
+                        })?
+                    } else {
+                        parsed_fallback
+                    }.as_str())
+                }
                 Item::Placeholder(Placeholder::Scie) => reified.push_str(path_to_str(self.scie)?),
                 Item::Placeholder(Placeholder::ScieBase) => {
                     reified.push_str(path_to_str(&self.base)?)
@@ -615,7 +660,7 @@ pub(crate) fn select_command(
 #[cfg(test)]
 mod tests {
     use std::env;
-    use std::path::Path;
+    use std::path::{Path, PathBuf};
 
     use indexmap::IndexMap;
 
@@ -630,11 +675,16 @@ mod tests {
             size: 42,
             version: "0.1.0".to_string(),
         };
-        let tempdir = tempfile::tempdir().unwrap();
         let lift = Lift {
             name: "test".to_string(),
             description: None,
-            base: Some(tempdir.path().to_path_buf()),
+            base: Some(
+                PathBuf::from("{scie.user.cache_dir={scie.env.USER_CACHE_DIR=/tmp/nce}}")
+                    .join("example")
+                    .into_os_string()
+                    .into_string()
+                    .unwrap(),
+            ),
             load_dotenv: true,
             size: 137,
             hash: "abc".to_string(),
@@ -693,10 +743,23 @@ mod tests {
         );
 
         env.clear();
+        let expected_scie_base = dirs::cache_dir()
+            .unwrap_or_else(|| PathBuf::from("/tmp/nce"))
+            .join("example");
+        assert_eq!(
+            (
+                expected_scie_base
+                    .clone()
+                    .into_os_string()
+                    .into_string()
+                    .unwrap(),
+                false
+            ),
+            context.reify_string(&mut env, "{scie.base}").unwrap()
+        );
         assert_eq!(
             (
-                tempdir
-                    .path()
+                expected_scie_base
                     .join("abc")
                     .join("lift.json")
                     .to_str()
@@ -709,11 +772,20 @@ mod tests {
                 .unwrap()
         );
 
+        let mut invalid_lift = lift.clone();
+        invalid_lift.base = Some("{scie.lift}/circular".to_string());
+        assert_eq!(
+            "The scie.lift.base cannot use the placeholder {scie.lift} since that placeholder is \
+            calculated from the resolved location of the scie.lift.base, given: \
+            {scie.lift}/circular"
+                .to_string(),
+            Context::new(Path::new("scie_path"), &jump, &invalid_lift, &installer).unwrap_err()
+        );
+
         env.clear();
         assert_eq!(
             (
-                tempdir
-                    .path()
+                expected_scie_base
                     .join("def")
                     .join("file")
                     .to_str()
@@ -750,11 +822,10 @@ mod tests {
             size: 42,
             version: "0.1.0".to_string(),
         };
-        let tempdir = tempfile::tempdir().unwrap();
         let lift = Lift {
             name: "test".to_string(),
             description: None,
-            base: Some(tempdir.path().to_path_buf()),
+            base: Some("/tmp/nce".to_string()),
             load_dotenv: true,
             size: 137,
             hash: "abc".to_string(),
@@ -824,7 +895,10 @@ mod tests {
         assert_eq!(
             Process {
                 env: expected_env.clone(),
-                exe: tempdir.path().join("def").join("dist-v1/v2/binary").into(),
+                exe: PathBuf::from("/tmp/nce")
+                    .join("def")
+                    .join("dist-v1/v2/binary")
+                    .into(),
                 args: vec![],
             },
             process
@@ -835,7 +909,10 @@ mod tests {
         assert_eq!(
             Process {
                 env: expected_env.clone(),
-                exe: tempdir.path().join("def").join("dist-v1/v1/exe").into(),
+                exe: PathBuf::from("/tmp/nce")
+                    .join("def")
+                    .join("dist-v1/v1/exe")
+                    .into(),
                 args: vec![],
             },
             process
@@ -847,7 +924,10 @@ mod tests {
         assert_eq!(
             Process {
                 env: expected_env,
-                exe: tempdir.path().join("ghi").join("dist-v2/v2/binary").into(),
+                exe: PathBuf::from("/tmp/nce")
+                    .join("ghi")
+                    .join("dist-v2/v2/binary")
+                    .into(),
                 args: vec![],
             },
             process
@@ -861,11 +941,10 @@ mod tests {
             size: 42,
             version: "0.1.0".to_string(),
         };
-        let tempdir = tempfile::tempdir().unwrap();
         let lift = Lift {
             name: "test".to_string(),
             description: None,
-            base: Some(tempdir.path().to_path_buf()),
+            base: Some("/tmp/nce".to_string()),
             load_dotenv: true,
             size: 137,
             hash: "abc".to_string(),

jump/src/installer.rs

@@ -164,6 +164,7 @@ where
     }
 }
 
+#[derive(Debug)]
 pub(crate) struct Installer<'a> {
     payload: &'a [u8],
 }

jump/src/lift.rs

@@ -1,7 +1,7 @@
 // Copyright 2022 Science project contributors.
 // Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-use std::path::{Path, PathBuf};
+use std::path::Path;
 
 use bstr::ByteSlice;
 use logging_timer::time;
@@ -52,7 +52,7 @@ impl From<File> for crate::config::File {
 pub struct Lift {
     pub name: String,
     pub description: Option<String>,
-    pub base: Option<PathBuf>,
+    pub base: Option<String>,
     pub(crate) load_dotenv: bool,
     pub size: usize,
     pub hash: String,