From bf9634d52d77309401b697639fe636061e54888e Mon Sep 17 00:00:00 2001 From: maciektr Date: Mon, 29 Jan 2024 11:21:42 +0100 Subject: [PATCH] Implement loading shared libraries for proc macro plugins commit-id:a3155bbf --- Cargo.lock | 11 ++ Cargo.toml | 1 + scarb/Cargo.toml | 2 + scarb/src/compiler/db.rs | 5 +- scarb/src/compiler/plugin/mod.rs | 3 +- scarb/src/compiler/plugin/proc_macro/ffi.rs | 158 ++++++++++++++++++ .../host.rs} | 61 +------ scarb/src/compiler/plugin/proc_macro/mod.rs | 5 + 8 files changed, 190 insertions(+), 56 deletions(-) create mode 100644 scarb/src/compiler/plugin/proc_macro/ffi.rs rename scarb/src/compiler/plugin/{proc_macro_host.rs => proc_macro/host.rs} (79%) create mode 100644 scarb/src/compiler/plugin/proc_macro/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 1db957ea3..5c303e839 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3313,6 +3313,16 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "libloading" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "libmimalloc-sys" version = "0.1.35" @@ -4478,6 +4488,7 @@ dependencies = [ "io_tee", "itertools 0.12.1", "libc", + "libloading", "ntest", "once_cell", "pathdiff", diff --git a/Cargo.toml b/Cargo.toml index 209993faa..ee8616492 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,6 +124,7 @@ xshell = "0.2" xxhash-rust = { version = "0.8", features = ["xxh3"] } zip = { version = "0.6", default-features = false, features = ["deflate"] } zstd = "0.13" +libloading = "0.8.1" [profile.release] lto = true diff --git a/scarb/Cargo.toml b/scarb/Cargo.toml index fd18eb814..3d9fcad6f 100644 --- a/scarb/Cargo.toml +++ b/scarb/Cargo.toml @@ -80,6 +80,8 @@ xxhash-rust.workspace = true zip.workspace = true zstd.workspace = true scarb-proc-macro-interface = { path = "../utils/scarb-proc-macro-interface" } +libloading.workspace = true +libc.workspace = true [target.'cfg(not(target_os = "linux"))'.dependencies] reqwest = { workspace = true, default-features = true} diff --git a/scarb/src/compiler/db.rs b/scarb/src/compiler/db.rs index b05ae0702..f9b05bb63 100644 --- a/scarb/src/compiler/db.rs +++ b/scarb/src/compiler/db.rs @@ -11,7 +11,8 @@ use smol_str::SmolStr; use std::sync::Arc; use tracing::trace; -use crate::compiler::plugin::{CairoPlugin, ProcMacroHostPlugin}; +use crate::compiler::plugin::proc_macro::ProcMacroHostPlugin; +use crate::compiler::plugin::CairoPlugin; use crate::compiler::{CompilationUnit, CompilationUnitComponent}; use crate::core::Workspace; use crate::DEFAULT_MODULE_MAIN_FILE; @@ -43,7 +44,7 @@ fn load_plugins( let instance = plugin.instantiate()?; builder.with_plugin_suite(instance.plugin_suite()); } else { - proc_macros.register(plugin_info.package.clone()); + proc_macros.register(plugin_info.package.clone())?; } } builder.with_plugin_suite(proc_macros.instantiate()?.plugin_suite()); diff --git a/scarb/src/compiler/plugin/mod.rs b/scarb/src/compiler/plugin/mod.rs index ea7f2abeb..4023e5988 100644 --- a/scarb/src/compiler/plugin/mod.rs +++ b/scarb/src/compiler/plugin/mod.rs @@ -11,8 +11,7 @@ use crate::core::PackageId; use self::builtin::{BuiltinStarkNetPlugin, BuiltinTestPlugin}; pub mod builtin; -pub(crate) mod proc_macro_host; -pub use proc_macro_host::ProcMacroHostPlugin; +pub mod proc_macro; pub trait CairoPlugin: Sync { fn id(&self) -> PackageId; diff --git a/scarb/src/compiler/plugin/proc_macro/ffi.rs b/scarb/src/compiler/plugin/proc_macro/ffi.rs new file mode 100644 index 000000000..c19183a89 --- /dev/null +++ b/scarb/src/compiler/plugin/proc_macro/ffi.rs @@ -0,0 +1,158 @@ +use crate::core::Package; +use anyhow::Result; +use cairo_lang_defs::patcher::PatchBuilder; +use cairo_lang_syntax::node::db::SyntaxGroup; +use cairo_lang_syntax::node::{ast, TypedSyntaxNode}; +use camino::Utf8PathBuf; +use libloading::{Library, Symbol}; +use scarb_proc_macro_interface::shared::{FfiProcMacroResult, FfiTokenStream}; +use std::ffi::CString; +use std::fmt::Debug; + +#[cfg(not(windows))] +use libloading::os::unix::Symbol as RawSymbol; +#[cfg(windows)] +use libloading::os::windows::Symbol as RawSymbol; + +#[derive(Debug, Default, Clone)] +pub struct TokenStream(String); + +impl TokenStream { + /// Convert to struct with stable ABI, `FfiTokenStream`. + /// + pub fn to_ffi(&self) -> FfiTokenStream { + let cstring = CString::new(self.0.clone()).expect("CString::new failed"); + FfiTokenStream(cstring.into_raw()) + } + + /// Convert from struct with stable ABI, `FfiTokenStream`. + /// + /// # Safety + pub unsafe fn from_ffi(token_stream: FfiTokenStream) -> Self { + Self(token_stream.to_string()) + } +} + +impl TokenStream { + pub fn from_item_ast(db: &dyn SyntaxGroup, item_ast: ast::ModuleItem) -> Self { + let mut builder = PatchBuilder::new(db); + builder.add_node(item_ast.as_syntax_node()); + let cairo = builder.code.clone(); + Self(cairo) + } + + pub fn collect(self) -> String { + self.0 + } +} + +#[derive(Debug)] +#[allow(dead_code)] +pub enum ProcMacroResult { + /// Plugin has not taken any action. + Leave, + /// Plugin generated TokenStream replacement. + Replace(TokenStream), + /// Plugin ordered item removal. + Remove, +} + +impl ProcMacroResult { + /// Convert from struct with stable ABI, `FfiProcMacroResult`. + /// + /// # Safety + pub unsafe fn from_ffi(ffi_result: FfiProcMacroResult) -> Self { + match ffi_result { + FfiProcMacroResult::Leave => Self::Leave, + FfiProcMacroResult::Remove => Self::Remove, + FfiProcMacroResult::Replace(token_stream) => { + Self::Replace(TokenStream::from_ffi(token_stream)) + } + } + } +} + +#[non_exhaustive] +pub struct ProcMacroInstance { + plugin: Plugin, +} + +impl Debug for ProcMacroInstance { + fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Ok(()) + } +} + +fn shared_lib_path(package: &Package) -> Utf8PathBuf { + let lib_name = format!( + "{}{}.{}", + shared_lib_prefix(), + package.id.name, + shared_lib_ext() + ); + package.root().join("target").join("release").join(lib_name) +} + +fn shared_lib_prefix() -> &'static str { + #[cfg(windows)] + return ""; + #[cfg(not(windows))] + return "lib"; +} + +fn shared_lib_ext() -> &'static str { + #[cfg(target_os = "windows")] + return "dll"; + #[cfg(target_os = "macos")] + return "dylib"; + #[cfg(not(target_os = "windows"))] + #[cfg(not(target_os = "macos"))] + return "so"; +} + +impl ProcMacroInstance { + /// Load shared library + pub fn try_new(package: Package) -> Result { + let plugin = unsafe { Plugin::try_new(shared_lib_path(&package))? }; + Ok(Self { plugin }) + } + + /// Apply expansion to token stream. + pub(crate) fn generate_code(&self, token_stream: TokenStream) -> ProcMacroResult { + let ffi_token_stream = token_stream.to_ffi(); + let result = (self.plugin.vtable.expand)(ffi_token_stream); + unsafe { ProcMacroResult::from_ffi(result) } + } +} + +type ExpandCode = extern "C" fn(FfiTokenStream) -> FfiProcMacroResult; + +struct VTableV0 { + expand: RawSymbol, +} + +impl VTableV0 { + unsafe fn try_new(library: &Library) -> Result { + println!("Loading plugin API version 0..."); + + let expand: Symbol<'_, ExpandCode> = library.get(b"expand\0")?; + let expand = expand.into_raw(); + + Ok(VTableV0 { expand }) + } +} + +struct Plugin { + #[allow(dead_code)] + library: Library, + vtable: VTableV0, +} + +impl Plugin { + unsafe fn try_new(library_name: Utf8PathBuf) -> Result { + let library = Library::new(library_name)?; + let vtable = VTableV0::try_new(&library)?; + + Ok(Plugin { library, vtable }) + } +} diff --git a/scarb/src/compiler/plugin/proc_macro_host.rs b/scarb/src/compiler/plugin/proc_macro/host.rs similarity index 79% rename from scarb/src/compiler/plugin/proc_macro_host.rs rename to scarb/src/compiler/plugin/proc_macro/host.rs index 50bb9cd2b..b81a71c9b 100644 --- a/scarb/src/compiler/plugin/proc_macro_host.rs +++ b/scarb/src/compiler/plugin/proc_macro/host.rs @@ -1,68 +1,24 @@ +use crate::compiler::plugin::proc_macro::{ProcMacroInstance, ProcMacroResult, TokenStream}; use crate::compiler::plugin::{CairoPlugin, CairoPluginInstance}; use crate::core::{Package, PackageId, PackageName, SourceId}; use crate::internal::to_version::ToVersion; use anyhow::Result; -use cairo_lang_defs::patcher::PatchBuilder; use cairo_lang_defs::plugin::{ MacroPlugin, MacroPluginMetadata, PluginGeneratedFile, PluginResult, }; use cairo_lang_semantic::plugin::PluginSuite; use cairo_lang_syntax::attribute::structured::AttributeListStructurize; +use cairo_lang_syntax::node::ast; use cairo_lang_syntax::node::db::SyntaxGroup; -use cairo_lang_syntax::node::{ast, TypedSyntaxNode}; use itertools::Itertools; use smol_str::SmolStr; use std::collections::HashMap; use std::sync::Arc; use typed_builder::TypedBuilder; -#[derive(Debug, Default, Clone)] -pub struct TokenStream(String); - -impl TokenStream { - pub fn from_item_ast(db: &dyn SyntaxGroup, item_ast: ast::ModuleItem) -> Self { - let mut builder = PatchBuilder::new(db); - builder.add_node(item_ast.as_syntax_node()); - let cairo = builder.code.clone(); - Self(cairo) - } - - pub fn collect(self) -> String { - self.0 - } -} - -#[derive(Debug)] -#[allow(dead_code)] -pub enum ProcMacroResult { - /// Plugin has not taken any action. - Leave, - /// Plugin generated TokenStream replacement. - Replace(TokenStream), - /// Plugin ordered item removal. - Remove, -} - -#[derive(Debug, Clone)] -pub struct ProcMacroInstance {} - -impl ProcMacroInstance { - pub fn new(_package: Package) -> Self { - // Load shared library - // TODO(maciektr): Implement - Self {} - } - - fn generate_code(&self, _token_stream: TokenStream) -> ProcMacroResult { - // Apply expansion to token stream. - // TODO(maciektr): Implement - ProcMacroResult::Leave - } -} - #[derive(Debug, TypedBuilder)] pub struct ProcMacroHost { - macros: HashMap>, + macros: HashMap>, } pub type ProcMacroId = SmolStr; @@ -191,7 +147,7 @@ impl MacroPlugin for ProcMacroHost { #[derive(Default)] pub struct ProcMacroHostPlugin { - macros: HashMap>, + macros: HashMap>, } impl ProcMacroHostPlugin { @@ -203,12 +159,13 @@ impl ProcMacroHostPlugin { ) } - pub fn register(&mut self, package: Package) { + pub fn register(&mut self, package: Package) -> Result<()> { // Create instance // Register instance in hash map let name = package.id.name.to_smol_str(); - let instance = ProcMacroInstance::new(package); - self.macros.insert(name, Box::new(instance)); + let instance = ProcMacroInstance::try_new(package)?; + self.macros.insert(name, Arc::new(instance)); + Ok(()) } } @@ -227,7 +184,7 @@ impl CairoPlugin for ProcMacroHostPlugin { #[derive(TypedBuilder)] pub struct ProcMacroHostPluginInstance { - macros: HashMap>, + macros: HashMap>, } impl ProcMacroHostPluginInstance { diff --git a/scarb/src/compiler/plugin/proc_macro/mod.rs b/scarb/src/compiler/plugin/proc_macro/mod.rs new file mode 100644 index 000000000..7e563afe8 --- /dev/null +++ b/scarb/src/compiler/plugin/proc_macro/mod.rs @@ -0,0 +1,5 @@ +mod ffi; +mod host; + +pub use ffi::*; +pub use host::*;