From 71efffa9f4bfaedd2b708416198f2dac3aaa1202 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 19 Aug 2024 12:51:01 +0200 Subject: [PATCH 1/2] Initial attempt to use dynamically acquired symbols on Unix --- rustler/src/codegen_runtime.rs | 3 +- rustler_codegen/src/init.rs | 5 +- rustler_sys/Cargo.toml | 2 +- rustler_sys/build.rs | 154 +++++++----------- rustler_sys/src/functions.rs | 36 ++++ rustler_sys/src/lib.rs | 8 +- rustler_sys/src/nif_filler.rs | 51 ++++++ .../src/{rustler_sys_api.rs => types.rs} | 27 +-- 8 files changed, 156 insertions(+), 130 deletions(-) create mode 100644 rustler_sys/src/functions.rs create mode 100644 rustler_sys/src/nif_filler.rs rename rustler_sys/src/{rustler_sys_api.rs => types.rs} (91%) diff --git a/rustler/src/codegen_runtime.rs b/rustler/src/codegen_runtime.rs index e1f4d3bf..6c86cb23 100644 --- a/rustler/src/codegen_runtime.rs +++ b/rustler/src/codegen_runtime.rs @@ -18,8 +18,7 @@ pub use crate::wrapper::{ NIF_ENV, NIF_MAJOR_VERSION, NIF_MINOR_VERSION, NIF_TERM, }; -#[cfg(windows)] -pub use rustler_sys::{TWinDynNifCallbacks, WIN_DYN_NIF_CALLBACKS}; +pub use rustler_sys::{internal_set_symbols, internal_write_symbols, DynNifCallbacks}; pub unsafe trait NifReturnable { unsafe fn into_returned(self, env: Env) -> NifReturned; diff --git a/rustler_codegen/src/init.rs b/rustler_codegen/src/init.rs index d4881b60..1f258e75 100644 --- a/rustler_codegen/src/init.rs +++ b/rustler_codegen/src/init.rs @@ -142,14 +142,15 @@ impl From for proc_macro2::TokenStream { #[cfg(unix)] #[no_mangle] extern "C" fn nif_init() -> *const rustler::codegen_runtime::DEF_NIF_ENTRY { + unsafe { rustler::codegen_runtime::internal_write_symbols() }; #inner } #[cfg(windows)] #[no_mangle] - extern "C" fn nif_init(callbacks: *mut rustler::codegen_runtime::TWinDynNifCallbacks) -> *const rustler::codegen_runtime::DEF_NIF_ENTRY { + extern "C" fn nif_init(callbacks: *mut rustler::codegen_runtime::DynNifCallbacks) -> *const rustler::codegen_runtime::DEF_NIF_ENTRY { unsafe { - rustler::codegen_runtime::WIN_DYN_NIF_CALLBACKS = Some(*callbacks); + rustler::codegen_runtime::internal_set_symbols(*callbacks); } #inner diff --git a/rustler_sys/Cargo.toml b/rustler_sys/Cargo.toml index df4d3167..59c065f5 100644 --- a/rustler_sys/Cargo.toml +++ b/rustler_sys/Cargo.toml @@ -42,7 +42,7 @@ nif_version_2_16 = ["nif_version_2_15"] nif_version_2_17 = ["nif_version_2_16"] [dependencies] -unreachable = "1.0" +libloading = "0.8" [build-dependencies] regex-lite = "0.1" diff --git a/rustler_sys/build.rs b/rustler_sys/build.rs index c45cb65d..f4be5255 100644 --- a/rustler_sys/build.rs +++ b/rustler_sys/build.rs @@ -11,24 +11,25 @@ use std::{env, fs}; pub const MIN_SUPPORTED_VERSION: (u32, u32) = (2, 14); pub const MAX_SUPPORTED_VERSION: (u32, u32) = (2, 17); -const SNIPPET_NAME: &str = "nif_api.snippet"; +const SNIPPET_NAME: &str = "nif_api.snippet.rs"; trait ApiBuilder { + fn init(&mut self) {} + fn finish(&mut self) {} + fn func(&mut self, ret: &str, name: &str, args: &str); fn variadic_func(&mut self, ret: &str, name: &str, args: &str); fn dummy(&mut self, name: &str); } -#[derive(PartialEq, Eq, Clone, Copy)] -pub enum OsFamily { - Unix, - Win, -} - pub struct GenerateOptions { pub ulong_size: usize, pub nif_version: (u32, u32), - pub target_family: OsFamily, +} + +enum OsFamily { + Win, + Unix, } fn write_ret(out: &mut String, ret: &str) { @@ -47,81 +48,37 @@ fn write_variadic_fn_type(out: &mut String, args: &str, ret: &str) { write_ret(out, ret); } -pub struct BasicApiBuilder<'a>(&'a mut String); - -impl<'a> ApiBuilder for BasicApiBuilder<'a> { - fn func(&mut self, ret: &str, name: &str, args: &str) { - writeln!(self.0, "extern \"C\" {{").unwrap(); - writeln!( - self.0, - " /// See [{}](http://www.erlang.org/doc/man/erl_nif.html#{}) in the Erlang docs.", - name, name - ) - .unwrap(); - - write!(self.0, " pub fn {}({})", name, args).unwrap(); - write_ret(self.0, ret); - writeln!(self.0, ";").unwrap(); - - writeln!(self.0, "}}").unwrap(); +pub struct CallbacksApiBuilder<'a>(&'a mut String); +impl<'a> ApiBuilder for CallbacksApiBuilder<'a> { + fn init(&mut self) { + writeln!(self.0, "#[allow(dead_code)]").unwrap(); + writeln!(self.0, "#[derive(Default, Copy, Clone)]").unwrap(); + writeln!(self.0, "pub struct DynNifCallbacks {{").unwrap(); } - fn variadic_func(&mut self, ret: &str, name: &str, args: &str) { - writeln!(self.0, "extern \"C\" {{").unwrap(); - writeln!(self.0, " #[doc(hidden)]").unwrap(); - writeln!(self.0, " #[link_name = \"{}\"]", name).unwrap(); - - write!(self.0, " pub fn _{}({}, ...)", name, args).unwrap(); - write_ret(self.0, ret); - writeln!(self.0, ";").unwrap(); - writeln!(self.0, "}}\n").unwrap(); - - writeln!( - self.0, - "/// See [{}](http://www.erlang.org/doc/man/erl_nif.html#{}) in the Erlang docs.", - name, name - ) - .unwrap(); - writeln!(self.0, "#[macro_export]").unwrap(); - writeln!(self.0, "macro_rules! {} {{", name).unwrap(); - writeln!( - self.0, - " ( $( $arg:expr ),* ) => {{ $crate::_{}($($arg),*) }};", - name - ) - .unwrap(); - writeln!( - self.0, - " ( $( $arg:expr ),+, ) => {{ {}!($($arg),*) }};", - name - ) - .unwrap(); - writeln!(self.0, "}}\n").unwrap(); + fn finish(&mut self) { + writeln!(self.0, "}}").unwrap(); } - fn dummy(&mut self, _name: &str) {} -} -pub struct WinCallbacksApiBuilder<'a>(&'a mut String); -impl<'a> ApiBuilder for WinCallbacksApiBuilder<'a> { fn func(&mut self, ret: &str, name: &str, args: &str) { - write!(self.0, " {}: ", name).unwrap(); + write!(self.0, " {}: Option<", name).unwrap(); write_fn_type(self.0, args, ret); - writeln!(self.0, ",").unwrap(); + writeln!(self.0, ">,").unwrap(); } fn variadic_func(&mut self, ret: &str, name: &str, args: &str) { - write!(self.0, " {}: ", name).unwrap(); + write!(self.0, " {}: Option<", name).unwrap(); write_variadic_fn_type(self.0, args, ret); - writeln!(self.0, ",").unwrap(); + writeln!(self.0, ">,").unwrap(); } fn dummy(&mut self, name: &str) { - write!(self.0, " {}: ", name).unwrap(); + write!(self.0, " {}: Option<", name).unwrap(); write_fn_type(self.0, "", ""); - writeln!(self.0, ",").unwrap(); + writeln!(self.0, ">,").unwrap(); } } -pub struct WinForwardersApiBuilder<'a>(&'a mut String); -impl<'a> ApiBuilder for WinForwardersApiBuilder<'a> { +pub struct ForwardersApiBuilder<'a>(&'a mut String); +impl<'a> ApiBuilder for ForwardersApiBuilder<'a> { fn func(&mut self, ret: &str, name: &str, args: &str) { // This regex takes a list of args with types and return only the name of the args. // @@ -144,7 +101,7 @@ impl<'a> ApiBuilder for WinForwardersApiBuilder<'a> { writeln!(self.0, "{{").unwrap(); writeln!( self.0, - " (WIN_DYN_NIF_CALLBACKS.unchecked_unwrap().{})({})", + " (DYN_NIF_CALLBACKS.{}.unwrap_unchecked())({})", name, args_names ) .unwrap(); @@ -170,13 +127,35 @@ impl<'a> ApiBuilder for WinForwardersApiBuilder<'a> { write!(self.0, "pub unsafe fn get_{}() -> ", name).unwrap(); write_variadic_fn_type(self.0, args, ret); writeln!(self.0, " {{").unwrap(); + writeln!(self.0, " DYN_NIF_CALLBACKS.{}.unwrap_unchecked()", name).unwrap(); + writeln!(self.0, "}}\n").unwrap(); + } + fn dummy(&mut self, _name: &str) {} +} + +pub struct WriterBuilder<'a>(&'a mut String); + +impl<'a> ApiBuilder for WriterBuilder<'a> { + fn init(&mut self) { + write!( + self.0, + "impl DynNifCallbacks {{\n fn write_symbols(&mut self, filler: T) {{\n" + ) + .unwrap(); + } + fn finish(&mut self) { + writeln!(self.0, " }}\n}}").unwrap(); + } + fn func(&mut self, _ret: &str, name: &str, _args: &str) { writeln!( self.0, - " WIN_DYN_NIF_CALLBACKS.unchecked_unwrap().{}", - name + " filler.write(&mut self.{}, \"{}\0\");", + name, name ) .unwrap(); - writeln!(self.0, "}}\n").unwrap(); + } + fn variadic_func(&mut self, ret: &str, name: &str, args: &str) { + self.func(ret, name, args); } fn dummy(&mut self, _name: &str) {} } @@ -202,31 +181,10 @@ fn generate(opts: &GenerateOptions) -> String { ) .unwrap(); - // Basic - if opts.target_family == OsFamily::Win { - writeln!(out, "#[allow(dead_code)]").unwrap(); - writeln!(out, "#[derive(Copy, Clone)]").unwrap(); - writeln!(out, "pub struct TWinDynNifCallbacks {{").unwrap(); - build_api(&mut WinCallbacksApiBuilder(&mut out), opts); - writeln!(out, "}}").unwrap(); - - // The line below would be the "faithful" reproduction of the NIF Win API, but Rust - // is currently not allowing statics to be uninitialized (1.3 beta). Revisit this when - // RFC911 is implemented (or some other mechanism) - // writeln!(out, "pub static mut WIN_DYN_NIF_CALLBACKS: TWinDynNifCallbacks = unsafe {{ std::mem::uninitialized() }};\n").unwrap(); - - // The work-around is to use Option. The problem here is that we have to do an unwrap() for - // each API call which is extra work. - writeln!( - out, - "pub static mut WIN_DYN_NIF_CALLBACKS:Option = None;\n" - ) - .unwrap(); + build_api(&mut CallbacksApiBuilder(&mut out), opts); - build_api(&mut WinForwardersApiBuilder(&mut out), opts); - } else { - build_api(&mut BasicApiBuilder(&mut out), opts); - } + build_api(&mut ForwardersApiBuilder(&mut out), opts); + build_api(&mut WriterBuilder(&mut out), opts); if opts.ulong_size == 4 { writeln!(out, "use std::os::raw::{{c_ulonglong, c_longlong}};").unwrap(); @@ -258,6 +216,7 @@ pub unsafe fn enif_get_uint64(env: *mut ErlNifEnv, term: ERL_NIF_TERM, ip: *mut } fn build_api(b: &mut dyn ApiBuilder, opts: &GenerateOptions) { + b.init(); b.func("*mut c_void", "enif_priv_data", "arg1: *mut ErlNifEnv"); b.func("*mut c_void", "enif_alloc", "size: size_t"); b.func("", "enif_free", "ptr: *mut c_void"); @@ -883,6 +842,8 @@ fn build_api(b: &mut dyn ApiBuilder, opts: &GenerateOptions) { // handling uses the `TWinDynNifCallbacks` struct. // // The correct order can (currently) by derived from the `erl_nif_api_funcs.h` header. + + b.finish(); } fn get_nif_version_from_features() -> (u32, u32) { @@ -936,7 +897,6 @@ fn main() { let opts = GenerateOptions { ulong_size, nif_version, - target_family, }; let api = generate(&opts); diff --git a/rustler_sys/src/functions.rs b/rustler_sys/src/functions.rs new file mode 100644 index 00000000..27117639 --- /dev/null +++ b/rustler_sys/src/functions.rs @@ -0,0 +1,36 @@ +use crate::nif_filler::{self, DynNifFiller}; +use crate::types::*; + +static mut DYN_NIF_CALLBACKS: DynNifCallbacks = + unsafe { std::mem::MaybeUninit::zeroed().assume_init() }; + +pub unsafe fn internal_set_symbols(callbacks: DynNifCallbacks) { + DYN_NIF_CALLBACKS = callbacks; +} + +pub unsafe fn internal_write_symbols() { + let filler = nif_filler::new(); + DYN_NIF_CALLBACKS.write_symbols(filler); +} + +/// See [enif_make_pid](http://erlang.org/doc/man/erl_nif.html#enif_make_pid) in the Erlang docs +pub unsafe fn enif_make_pid(_env: *mut ErlNifEnv, pid: ErlNifPid) -> ERL_NIF_TERM { + pid.pid +} + +/// See [enif_compare_pids](http://erlang.org/doc/man/erl_nif.html#enif_compare_pids) in the Erlang docs +pub unsafe fn enif_compare_pids(pid1: *const ErlNifPid, pid2: *const ErlNifPid) -> c_int { + // Mimics the implementation of the enif_compare_pids macro + enif_compare((*pid1).pid, (*pid2).pid) +} + +// Include the file generated by `build.rs`. +include!(concat!(env!("OUT_DIR"), "/nif_api.snippet.rs")); +// example of included content: +// extern "C" { +// pub fn enif_priv_data(arg1: *mut ErlNifEnv) -> *mut c_void; +// pub fn enif_alloc(size: size_t) -> *mut c_void; +// pub fn enif_free(ptr: *mut c_void); +// pub fn enif_is_atom(arg1: *mut ErlNifEnv, term: ERL_NIF_TERM) -> c_int; +// pub fn enif_is_binary(arg1: *mut ErlNifEnv, term: ERL_NIF_TERM) -> c_int; +// ... diff --git a/rustler_sys/src/lib.rs b/rustler_sys/src/lib.rs index 2218ac46..4190e047 100644 --- a/rustler_sys/src/lib.rs +++ b/rustler_sys/src/lib.rs @@ -4,7 +4,11 @@ Low level Rust bindings to the [Erlang NIF API](http://www.erlang.org/doc/man/er // Don't throw warnings on NIF naming conventions #![allow(non_camel_case_types)] +#![allow(clippy::missing_safety_doc)] -pub mod rustler_sys_api; +mod functions; +mod nif_filler; +mod types; -pub use crate::rustler_sys_api::*; +pub use crate::functions::*; +pub use crate::types::*; diff --git a/rustler_sys/src/nif_filler.rs b/rustler_sys/src/nif_filler.rs new file mode 100644 index 00000000..f4f767d6 --- /dev/null +++ b/rustler_sys/src/nif_filler.rs @@ -0,0 +1,51 @@ +pub(crate) trait DynNifFiller { + fn write(&self, field: &mut Option, name: &str); +} + +pub struct NullNifFiller; +impl DynNifFiller for NullNifFiller { + fn write(&self, _field: &mut Option, _name: &str) {} +} + +#[cfg(not(target_os = "windows"))] +mod internal { + use std::ffi::OsStr; + + use super::DynNifFiller; + use libloading::os::unix::{Library, RTLD_GLOBAL, RTLD_NOW}; + + pub(crate) struct DlsymNifFiller { + lib: libloading::Library, + } + + impl DlsymNifFiller { + pub(crate) fn new() -> Self { + let lib = unsafe { Library::open(None::<&OsStr>, RTLD_NOW | RTLD_GLOBAL) }; + DlsymNifFiller { + lib: lib.unwrap().into(), + } + } + } + + impl DynNifFiller for DlsymNifFiller { + fn write(&self, field: &mut Option, name: &str) { + let symbol = unsafe { self.lib.get::(name.as_bytes()).unwrap() }; + *field = Some(*symbol); + } + } + + pub(crate) fn new() -> impl DynNifFiller { + DlsymNifFiller::new() + } +} + +#[cfg(target_os = "windows")] +mod internal { + use super::*; + + pub fn new() -> impl DynNifFiller { + NullNifFiller + } +} + +pub(crate) use internal::new; diff --git a/rustler_sys/src/rustler_sys_api.rs b/rustler_sys/src/types.rs similarity index 91% rename from rustler_sys/src/rustler_sys_api.rs rename to rustler_sys/src/types.rs index befee7d0..09db65f9 100644 --- a/rustler_sys/src/rustler_sys_api.rs +++ b/rustler_sys/src/types.rs @@ -1,9 +1,6 @@ #![allow(clippy::missing_safety_doc)] #![allow(clippy::upper_case_acronyms)] -#[cfg(windows)] -use unreachable::UncheckedOptionExt; // unchecked unwrap used in generated Windows code - pub use std::ffi::{c_char, c_double, c_int, c_long, c_uchar, c_uint, c_ulong, c_void}; use std::os; @@ -195,18 +192,7 @@ pub enum ErlNifCharEncoding { #[derive(Debug, Copy, Clone)] #[repr(C)] pub struct ErlNifPid { - pid: ERL_NIF_TERM, -} - -/// See [enif_make_pid](http://erlang.org/doc/man/erl_nif.html#enif_make_pid) in the Erlang docs -pub unsafe fn enif_make_pid(_env: *mut ErlNifEnv, pid: ErlNifPid) -> ERL_NIF_TERM { - pid.pid -} - -/// See [enif_compare_pids](http://erlang.org/doc/man/erl_nif.html#enif_compare_pids) in the Erlang docs -pub unsafe fn enif_compare_pids(pid1: *const ErlNifPid, pid2: *const ErlNifPid) -> c_int { - // Mimics the implementation of the enif_compare_pids macro - enif_compare((*pid1).pid, (*pid2).pid) + pub(crate) pid: ERL_NIF_TERM, } /// See [ErlNifSysInfo](http://www.erlang.org/doc/man/erl_nif.html#ErlNifSysInfo) in the Erlang docs. @@ -346,14 +332,3 @@ pub enum ErlNifOption { ERL_NIF_OPT_DELAY_HALT = 1, ERL_NIF_OPT_ON_HALT = 2, } - -// Include the file generated by `build.rs`. -include!(concat!(env!("OUT_DIR"), "/nif_api.snippet")); -// example of included content: -// extern "C" { -// pub fn enif_priv_data(arg1: *mut ErlNifEnv) -> *mut c_void; -// pub fn enif_alloc(size: size_t) -> *mut c_void; -// pub fn enif_free(ptr: *mut c_void); -// pub fn enif_is_atom(arg1: *mut ErlNifEnv, term: ERL_NIF_TERM) -> c_int; -// pub fn enif_is_binary(arg1: *mut ErlNifEnv, term: ERL_NIF_TERM) -> c_int; -// ... From 30cc3bcddee68ab268c8675bc19011c987c841aa Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 19 Aug 2024 12:53:24 +0200 Subject: [PATCH 2/2] Drop now unnecessary config.toml --- .cargo/config.toml | 5 --- rustler_mix/lib/rustler/compiler.ex | 61 ----------------------------- 2 files changed, 66 deletions(-) delete mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index 20f03f3d..00000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,5 +0,0 @@ -[target.'cfg(target_os = "macos")'] -rustflags = [ - "-C", "link-arg=-undefined", - "-C", "link-arg=dynamic_lookup", -] diff --git a/rustler_mix/lib/rustler/compiler.ex b/rustler_mix/lib/rustler/compiler.ex index 14f764bd..5a48b729 100644 --- a/rustler_mix/lib/rustler/compiler.ex +++ b/rustler_mix/lib/rustler/compiler.ex @@ -61,69 +61,8 @@ defmodule Rustler.Compiler do ["rustup", "run", version, "cargo", "rustc"] end - defp ensure_platform_requirements!(crate_path, config, {:unix, :darwin}) do - # We attempt to find a .cargo/config upwards from the crate_path, which - # has a target configuration for macos. If any such config exists, we - # assume that the config correctly encodes the needed linker arguments. - - workspace_root = config.metadata["workspace_root"] - - components = - crate_path - |> Path.relative_to(workspace_root) - |> Path.split() - - {potential_config_files, _} = - Enum.map_reduce(["" | components], workspace_root, fn component, path -> - path = Path.join(path, component) - # See https://doc.rust-lang.org/cargo/reference/config.html, cargo - # accepts the config with and without a file extension of `.toml`. - file = Path.join([path, ".cargo", "config"]) - file_with_extension = file <> ".toml" - {[file, file_with_extension], path} - end) - - potential_config_files = List.flatten(potential_config_files) - - has_macos_target_os_configuration? = - potential_config_files - |> Enum.filter(&File.exists?/1) - |> Enum.reverse() - |> Stream.map(&Toml.decode_file!/1) - |> Enum.find(&macos_target_configuration/1) - - unless has_macos_target_os_configuration? do - raise """ - Compiling on macOS requires special link args in order to compile - correctly. - - To remove this error, please create .cargo/config.toml or .cargo/config - with the following content: - - [target.'cfg(target_os = "macos")'] - rustflags = [ - "-C", "link-arg=-undefined", - "-C", "link-arg=dynamic_lookup", - ] - - - See https://developer.apple.com/library/archive/documentation/DeveloperTools/Conceptual/MachOTopics/1-Articles/executing_files.html - for more details. - """ - end - end - defp ensure_platform_requirements!(_, _, _), do: :ok - defp macos_target_configuration(toml) do - toml - |> Map.get("target", []) - |> Enum.filter(fn {key, _} -> - String.match?(key, ~r/(.*macos.*)|(.*darwin.*)/) - end) - |> Map.new() - end - defp make_no_default_features_flag(args, true), do: args defp make_no_default_features_flag(args, false), do: args ++ ["--no-default-features"]