[fud2] Migrate operations into Rhai (#2084)

* [fud2] Make plugins optional

Very minor fix: #2078 added plugins, controlled by a `plugins` key in
`fud2.toml`. However, it inadvertently made this option required, i.e.,
fud2 would crash if this option was not present. Now it's optional! So
we just silently proceed with no plugins if none are configured.

* Fix a doc typo

* Start refactoring plugin loading

* Start refactoring builder functions

* Refactor the rest of the functions

* Gather up a `register` function

* Use a fresh engine for each plugin??

Maybe this is wasteful, but it simplifies things a lot.

* Run one script at a time

* Refector script contexts

* Refactor runner struct

* Further beef up the runner

* Further simplify interface

* Fix a borrow that lasts too long

* Fix a method name

Clippy was thrown for a loop because of the `to_*` name.

* Move plugin loading to DriverBuilder?

Now the top-level thing is in the builder itself... no need for an extra
trait? Not sure this is a good idea.

* translate a bulk of operations into rhai

* use single engine to module resolution caching, enabling imports

* switched all plugins to use import system

* include plugins in debug build

* we can now load scripts embedded in the binary

* if a module fails once, don't retry it

* sort states and operations in list mode

* migrate entirely to plugins

* gate migration behind a feature

* move sorting to when we run files

* rename plugins to scripts internally

* refactored to have more things live in the resolver

* rename plugins/ to scripts/

---------

Co-authored-by: Adrian Sampson <[email protected]>

fud2/Cargo.toml

@@ -12,6 +12,10 @@ readme = "README.md"
 categories = ["build-tool"]
 description = "Compiler driver for the Calyx infrastructure"
 
+[features]
+migrate_to_scripts = []
+default = []
+
 [dependencies]
 fud-core = { path = "fud-core", version = "0.0.2" }
 anyhow.workspace = true

fud2/fud-core/Cargo.toml

@@ -22,4 +22,5 @@ env_logger.workspace = true
 rhai = "1.18.0"
 once_cell = "1.19.0"
 ariadne = "0.4.1"
+itertools.workspace = true
 rand = "0.8.5"

fud2/fud-core/src/exec/driver.rs

@@ -3,7 +3,7 @@ use crate::{config, run, script, utils};
 use camino::{Utf8Path, Utf8PathBuf};
 use cranelift_entity::{PrimaryMap, SecondaryMap};
 use rand::distributions::{Alphanumeric, DistString};
-use std::{collections::HashMap, error::Error, fmt::Display};
+use std::{collections::HashMap, error::Error, ffi::OsStr, fmt::Display};
 
 #[derive(PartialEq)]
 enum Destination {
@@ -246,6 +246,8 @@ pub struct DriverBuilder {
     ops: PrimaryMap<OpRef, Operation>,
     rsrc_dir: Option<Utf8PathBuf>,
     rsrc_files: Option<FileData>,
+    scripts_dir: Option<Utf8PathBuf>,
+    script_files: Option<FileData>,
 }
 
 #[derive(Debug)]
@@ -278,6 +280,8 @@ impl DriverBuilder {
             ops: Default::default(),
             rsrc_dir: None,
             rsrc_files: None,
+            scripts_dir: None,
+            script_files: None,
         }
     }
 
@@ -373,26 +377,51 @@ impl DriverBuilder {
         self.rsrc_files = Some(files);
     }
 
+    pub fn scripts_dir(&mut self, path: &str) {
+        self.scripts_dir = Some(path.into());
+    }
+
+    pub fn script_files(&mut self, files: FileData) {
+        self.script_files = Some(files);
+    }
+
     /// Load any plugin scripts specified in the configuration file.
-    pub fn load_plugins(self) -> Self {
-        // Get a list of plugins (paths to Rhai scripts) from the config file, if any.
+    pub fn load_plugins(mut self) -> Self {
+        // pull out things from self that we need
+        let plugin_dir = self.scripts_dir.take();
+        let plugin_files = self.script_files.take();
+
         // TODO: Let's try to avoid loading/parsing the configuration file here and
         // somehow reusing it from wherever we do that elsewhere.
         let config = config::load_config(&self.name);
-        let plugin_files =
-            match config.extract_inner::<Vec<std::path::PathBuf>>("plugins") {
-                Ok(v) => v,
-                Err(_) => {
-                    // No plugins to load.
-                    return self;
-                }
-            };
 
-        let mut bld = self;
-        for path in plugin_files {
-            bld = script::ScriptRunner::run_file(bld, path.as_path());
+        let mut runner = script::ScriptRunner::new(self);
+
+        // add system plugins
+        if let Some(plugin_dir) = plugin_dir {
+            runner.add_files(
+                std::fs::read_dir(plugin_dir)
+                    .unwrap()
+                    // filter out invalid paths
+                    .filter_map(|dir_entry| dir_entry.map(|p| p.path()).ok())
+                    // filter out paths that don't have `.rhai` extension
+                    .filter(|p| p.extension() == Some(OsStr::new("rhai"))),
+            );
+        }
+
+        // add static plugins (where string is included in binary)
+        if let Some(plugin_files) = plugin_files {
+            runner.add_static_files(plugin_files.into_iter());
         }
-        bld
+
+        // add user plugins defined in config
+        if let Ok(plugins) =
+            config.extract_inner::<Vec<std::path::PathBuf>>("plugins")
+        {
+            runner.add_files(plugins.into_iter());
+        }
+
+        runner.run()
     }
 
     pub fn build(self) -> Driver {

fud2/fud-core/src/script/exec_scripts.rs

@@ -119,9 +119,9 @@ impl RhaiEmitter {
         todo!()
     }
 
-    fn external_path(&mut self, path: &str) -> Utf8PathBuf {
+    fn external_path(&mut self, path: &str) -> String {
         let utf8_path = Utf8PathBuf::from(path);
-        self.0.borrow().external_path(&utf8_path)
+        self.0.borrow().external_path(&utf8_path).into_string()
     }
 
     fn arg(&mut self, name: &str, value: &str) -> RhaiResult<()> {
@@ -162,10 +162,10 @@ thread_local! {
     });
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub(super) struct RhaiSetupCtx {
-    pub path: PathBuf,
-    pub ast: rhai::AST,
+    pub path: Rc<PathBuf>,
+    pub ast: Rc<rhai::AST>,
     pub name: String,
 }
 
@@ -179,7 +179,7 @@ impl EmitSetup for RhaiSetupCtx {
                     &self.name,
                     (rhai_emit.clone(),),
                 )
-                .report(&self.path)
+                .report(self.path.as_ref())
             });
         })?;
 
@@ -202,7 +202,7 @@ impl EmitBuild for RhaiSetupCtx {
                     &self.name,
                     (rhai_emit.clone(), input.to_string(), output.to_string()),
                 )
-                .report(&self.path)
+                .report(self.path.as_ref())
             });
         })?;
 

fud2/fud-core/src/script/mod.rs

@@ -2,5 +2,6 @@ mod error;
 mod exec_scripts;
 mod plugin;
 mod report;
+mod resolver;
 
 pub use plugin::ScriptRunner;