Implement boot bindings support. (#21)

Now commands can depend on `{boot.bindings.<binding name>}` placeholders
which are satisfied by boot bindings commands that are run exactly once.

This is used in the Python example to produce a scie-pants Pants binary
that prepares a fully pre-compiled Pants venv that runs at the full
speed of a native Pants venv. The boot binding command that creates the
venv also uses a private PEX_ROOT and cleans this up in addition to the
Pants PEX used to build the venv. The result is an ~/.nce with just the
CPython interpreter and a venv that symlinks to it. There is no wasted
speed or space as compared to a traditional Pants Python venv (save for
the size of the scie binary itself which may be addressed by #9 or #19).

Closes #7
Closes #20

examples/python/lift.linux-x86_64.json

@@ -10,34 +10,49 @@
             "env": {
               "=PATH": "{cpython}/python/bin:{scie.env.PATH}"
             },
-            "exe": "{cpython}/python/bin/python3.9",
+            "exe": "{scie.bindings.venv}/venv/bin/python3.9",
             "args": [
-              "{pants.2.14.0.pex}"
+              "{scie.bindings.venv}/venv/pex"
             ]
           },
-          "inspect": {
-            "description": "PEX tools for Pants.",
+          "repl": {
+            "description": "A Python repl with Pants (minus plugins) available for inspection.",
             "env": {
               "=PATH": "{cpython}/python/bin:{scie.env.PATH}",
               "PEX_TOOLS": "1"
             },
+            "exe": "{scie.bindings.venv}/venv/bin/python3.9"
+          }
+        },
+        "bindings": {
+          "venv": {
+            "description": "Installs Pants in a venv and pre-compiles .pyc.",
+            "env": {
+              "=PATH": "{cpython}/python/bin:{scie.env.PATH}",
+              "PEX_TOOLS": "1",
+              "PEX_ROOT": "{scie.bindings}/pex_root"
+            },
             "exe": "{cpython}/python/bin/python3.9",
             "args": [
-              "{pants.2.14.0.pex}"
+              "{pants.2.14.0.pex}",
+              "venv",
+              "--bin-path",
+              "prepend",
+              "--compile",
+              "--rm",
+              "all",
+              "{scie.bindings}/venv"
             ]
           }
         }
       },
       "files": [
         {
           "name": "cpython-3.9.14+20221002-x86_64-unknown-linux-gnu-install_only.tar.gz",
-          "key": "cpython",
-          "hash": "e63d0c00a499e0202ba7a0f53ce69fca6d30237af39af9bc3c76bce6c7bf14d7",
-          "size": 26761725
+          "key": "cpython"
         },
         {
-          "name": "pants.2.14.0.pex",
-          "type": "zip"
+          "name": "pants.2.14.0.pex"
         }
       ]
     }

examples/python/lift.macos-x86_64.json

@@ -10,20 +10,38 @@
             "env": {
               "=PATH": "{cpython}/python/bin:{scie.env.PATH}"
             },
-            "exe": "{cpython}/python/bin/python3.9",
+            "exe": "{scie.bindings.venv}/venv/bin/python3.9",
             "args": [
-              "{pants.2.14.0.pex}"
+              "{scie.bindings.venv}/venv/pex"
             ]
           },
-          "inspect": {
-            "description": "PEX tools for Pants.",
+          "repl": {
+            "description": "A Python repl with Pants (minus plugins) available for inspection.",
             "env": {
               "=PATH": "{cpython}/python/bin:{scie.env.PATH}",
               "PEX_TOOLS": "1"
             },
+            "exe": "{scie.bindings.venv}/venv/bin/python3.9"
+          }
+        },
+        "bindings": {
+          "venv": {
+            "description": "Installs Pants in a venv and pre-compiles .pyc.",
+            "env": {
+              "=PATH": "{cpython}/python/bin:{scie.env.PATH}",
+              "PEX_TOOLS": "1",
+              "PEX_ROOT": "{scie.bindings}/pex_root"
+            },
             "exe": "{cpython}/python/bin/python3.9",
             "args": [
-              "{pants.2.14.0.pex}"
+              "{pants.2.14.0.pex}",
+              "venv",
+              "--bin-path",
+              "prepend",
+              "--compile",
+              "--rm",
+              "all",
+              "{scie.bindings}/venv"
             ]
           }
         }

examples/python/test.sh

@@ -9,7 +9,7 @@ trap gc EXIT
 gc "${PWD}/pants" "${PWD}/.pants.d" "${PWD}/.pids"
 
 time RUST_LOG=trace ./pants --no-pantsd -V
-time ./pants --no-pantsd -V
+time RUST_LOG=debug ./pants --no-pantsd -V
 
 # Use the built-in BusyBox functionality via env var.
-SCIE_BOOT="inspect" ./pants interpreter --verbose --indent 2
+SCIE_BOOT=repl ./pants -c 'from pants.util import strutil; print(strutil.__file__)'

jump/Cargo.toml

@@ -15,6 +15,7 @@ bzip2 = "0.4"
 dirs = "4.0"
 fd-lock = "3.0"
 flate2 = "1.0"  # For gz support.
+indexmap = "1.9"
 itertools = "0.10"
 log = { workspace = true }
 logging_timer = { workspace = true }

jump/src/atomic.rs

@@ -10,14 +10,14 @@ use serde::Serializer;
 #[derive(Copy, Clone, Eq, PartialEq)]
 pub(crate) enum Target {
     Directory,
-    _File, // TODO(John Sirois): Use for run-once boot bindings.
+    File, // TODO(John Sirois): Use for run-once boot bindings.
 }
 
 impl Display for Target {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
             Target::Directory => f.serialize_str("directory"),
-            Target::_File => f.serialize_str("file"),
+            Target::File => f.serialize_str("file"),
         }
     }
 }
@@ -32,7 +32,7 @@ impl Target {
                     return Ok(false);
                 }
             }
-            Target::_File => {
+            Target::File => {
                 if target.is_file() {
                     return Ok(true);
                 } else if !target.exists() {
@@ -67,6 +67,10 @@ where
 
     // First check.
     if target_type.check_exists(target)? {
+        debug!(
+            "The atomic {target_type} at {path} has already been established.",
+            path = target.display()
+        );
         return Ok(());
     }
 
@@ -102,6 +106,11 @@ where
 
     // Second check.
     if target_type.check_exists(target)? {
+        debug!(
+            "The atomic {target_type} at {path} has already been established \
+            (lost double-check race).",
+            path = target.display()
+        );
         return Ok(());
     }
 

jump/src/config.rs

@@ -216,7 +216,7 @@ pub struct Jump {
     pub version: String,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct Boot {
     pub commands: BTreeMap<String, Cmd>,
@@ -287,7 +287,7 @@ impl Default for Fmt {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct Other {
     #[serde(flatten)]
     other: BTreeMap<String, Value>,