C Extension API (#381)

Initial experimental(!) implementation of loadable extensions with C extension API

See duckdb/duckdb#12682 for more info on the DuckDB C extension API

---------

Co-authored-by: martin <[email protected]>

.github/workflows/rust.yaml

@@ -33,12 +33,13 @@ jobs:
           rust-version: stable${{ matrix.host }}
           targets: ${{ matrix.target }}
           components: 'rustfmt, clippy'
+
       # download libduckdb
       - uses: robinraju/[email protected]
         name: Download duckdb
         with:
           repository: "duckdb/duckdb"
-          tag: "v1.0.0"
+          tag: "v1.1.1"
           fileName: ${{ matrix.duckdb }}
           out-file-path: .
 
@@ -49,15 +50,25 @@ jobs:
         with:
           file_path: ${{ github.workspace }}/${{ matrix.duckdb }}
           extract_dir: libduckdb
+
       - run: cargo fmt --all -- --check
         if: matrix.os == 'ubuntu-latest'
-      - run: cargo clippy --all-targets --workspace --all-features -- -D warnings -A clippy::redundant-closure
+
+      # TODO: remove
+      - name: Workaround for https://github.com/pola-rs/polars/issues/19063
+        run: |
+          cargo update [email protected] --precise 2.5.0
+
+      - name: run cargo clippy
         if: matrix.os == 'ubuntu-latest'
-        name: run cargo clippy
         env:
           DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb
           DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb
           LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb
+        run: |
+          cargo clippy --all-targets --workspace --all-features -- -D warnings -A clippy::redundant-closure
+          
+
       - name: Run cargo-tarpaulin
         if: matrix.os == 'ubuntu-latest'
         uses: actions-rs/[email protected]
@@ -70,6 +81,7 @@ jobs:
           DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb
           DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb
           LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb
+
       - name: Upload to codecov.io
         if: matrix.os == 'ubuntu-latest'
         uses: codecov/codecov-action@v1
@@ -88,19 +100,28 @@ jobs:
         with:
           name: PATH
           value: $env:PATH;${{ github.workspace }}/libduckdb
+
       - name: Run cargo-test
         if: matrix.os == 'windows-latest'
         run: cargo test --features "modern-full vtab-full vtab-loadable"
         env:
           DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb
           DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb
+
       - name: Build loadable extension
         run: cargo build --example hello-ext --features="vtab-loadable"
         env:
           DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb
           DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb
           LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb
 
+      - name: Build loadable extension
+        run: cargo build --example hello-ext-capi --features="vtab-loadable loadable-extension"
+        env:
+          DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb
+          DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb
+          LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb
+
   Windows:
     name: Windows build from source
     needs: test
@@ -117,6 +138,7 @@ jobs:
         with:
           rust-version: stable
           targets: x86_64-pc-windows-msvc
+
       - run: cargo install cargo-examples
 
   Sanitizer:
@@ -140,7 +162,9 @@ jobs:
           # leak sanitization, but we don't care about backtraces here, so long
           # as the other tests have them.
           RUST_BACKTRACE: "0"
-        run: cargo -Z build-std test --features "modern-full extensions-full" --target x86_64-unknown-linux-gnu
+        run: |
+          # TODO switch back to modern-full once polars is fixed
+          cargo -Z build-std test --features "chrono serde_json url r2d2 uuid extensions-full" --target x86_64-unknown-linux-gnu --package duckdb
       - name: publish crates --dry-run
         uses: katyo/publish-crates@v2
         with:

.gitignore

@@ -27,6 +27,6 @@ Cargo.lock
 
 *.db
 
-crates/libduckdb-sys/duckdb-sources/
+crates/libduckdb-sys/duckdb-sources/*
 crates/libduckdb-sys/duckdb/
 crates/libduckdb-sys/._duckdb

.gitmodules

@@ -1,4 +1,3 @@
 [submodule "crates/libduckdb-sys/duckdb-sources"]
 	path = crates/libduckdb-sys/duckdb-sources
 	url = https://github.com/duckdb/duckdb
-	update = none

Cargo.toml

@@ -7,7 +7,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.0.0"
+version = "1.1.1"
 authors = ["wangfenjin <[email protected]>"]
 edition = "2021"
 repository = "https://github.com/duckdb/duckdb-rs"
@@ -19,8 +19,8 @@ license = "MIT"
 categories = ["database"]
 
 [workspace.dependencies]
-duckdb = { version = "1.0.0", path = "crates/duckdb" }
-libduckdb-sys = { version = "1.0.0", path = "crates/libduckdb-sys" }
+duckdb = { version = "1.1.1", path = "crates/duckdb" }
+libduckdb-sys = { version = "1.1.1", path = "crates/libduckdb-sys" }
 duckdb-loadable-macros = { version = "0.1.2", path = "crates/duckdb-loadable-macros" }
 autocfg = "1.0"
 bindgen = { version = "0.69", default-features = false }
@@ -43,6 +43,7 @@ pkg-config = "0.3.24"
 polars = "0.35.4"
 polars-core = "0.35.4"
 pretty_assertions = "1.4.0"
+prettyplease = "0.2.20"
 proc-macro2 = "1.0.56"
 quote = "1.0.21"
 r2d2 = "0.8.9"

Makefile

@@ -11,4 +11,4 @@ all:
 	cargo clippy --all-targets --workspace --features buildtime_bindgen --features modern-full -- -D warnings -A clippy::redundant-closure
 
 test:
-	cargo test --features bundled --features modern-full -- --nocapture
+	cargo test --features bundled --features modern-full -- --nocapture

add_rustfmt_hook.sh

@@ -8,7 +8,7 @@ command -v rustfmt >/dev/null 2>&1 || { echo >&2 "Rustfmt is required but it's n
 # write a whole script to pre-commit hook
 # NOTE: it will overwrite pre-commit file!
 cat > .git/hooks/pre-commit <<'EOF'
-#!/bin/bash -e
+#!/bin/bash
 declare -a rust_files=()
 files=$(git diff-index --name-only --cached HEAD)
 echo 'Formatting source files'

crates/duckdb-loadable-macros/Cargo.toml

@@ -14,6 +14,7 @@ description = "Native bindings to the libduckdb library, C API; build loadable e
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+darling = "0.20.10"
 proc-macro2 = { workspace = true }
 quote = { workspace = true }
 syn = { workspace = true, features = ["extra-traits", "full", "fold", "parsing"] }

crates/duckdb-loadable-macros/src/lib.rs

@@ -6,6 +6,108 @@ use syn::{parse_macro_input, spanned::Spanned, Item};
 use proc_macro::TokenStream;
 use quote::quote_spanned;
 
+use darling::{ast::NestedMeta, Error, FromMeta};
+
+/// For parsing the arguments to the duckdb_entrypoint_c_api macro
+#[derive(Debug, FromMeta)]
+struct CEntryPointMacroArgs {
+    #[darling(default)]
+    /// The name to be given to this extension. This name is used in the entrypoint function called by duckdb
+    ext_name: String,
+    /// The minimum C API version this extension requires. It is recommended to set this to the lowest possible version
+    /// at which your extension still compiles
+    min_duckdb_version: Option<String>,
+}
+
+/// Wraps an entrypoint function to expose an unsafe extern "C" function of the same name.
+/// Warning: experimental!
+#[proc_macro_attribute]
+pub fn duckdb_entrypoint_c_api(attr: TokenStream, item: TokenStream) -> TokenStream {
+    let attr_args = match NestedMeta::parse_meta_list(attr.into()) {
+        Ok(v) => v,
+        Err(e) => {
+            return TokenStream::from(Error::from(e).write_errors());
+        }
+    };
+
+    let args = match CEntryPointMacroArgs::from_list(&attr_args) {
+        Ok(v) => v,
+        Err(e) => {
+            return TokenStream::from(e.write_errors());
+        }
+    };
+
+    // Set the minimum duckdb version (dev by default)
+    let minimum_duckdb_version = match args.min_duckdb_version {
+        Some(i) => i,
+        None => "dev".to_string(),
+    };
+
+    let ast = parse_macro_input!(item as syn::Item);
+
+    match ast {
+        Item::Fn(func) => {
+            let c_entrypoint = Ident::new(format!("{}_init_c_api", args.ext_name).as_str(), Span::call_site());
+            let prefixed_original_function = func.sig.ident.clone();
+            let c_entrypoint_internal = Ident::new(
+                format!("{}_init_c_api_internal", args.ext_name).as_str(),
+                Span::call_site(),
+            );
+
+            quote_spanned! {func.span()=>
+                /// # Safety
+                ///
+                /// Internal Entrypoint for error handling
+                pub unsafe fn #c_entrypoint_internal(info: ffi::duckdb_extension_info, access: *const ffi::duckdb_extension_access) -> Result<bool, Box<dyn std::error::Error>> {
+                    let have_api_struct = ffi::duckdb_rs_extension_api_init(info, access, #minimum_duckdb_version).unwrap();
+
+                    if !have_api_struct {
+                        // initialization failed to return an api struct, likely due to an API version mismatch, we can simply return here
+                        return Ok(false);
+                    }
+
+                    // TODO: handle error here?
+                    let db : ffi::duckdb_database = *(*access).get_database.unwrap()(info);
+                    let connection = Connection::open_from_raw(db.cast())?;
+
+                    #prefixed_original_function(connection)?;
+
+                    Ok(true)
+                }
+
+                /// # Safety
+                ///
+                /// Entrypoint that will be called by DuckDB
+                #[no_mangle]
+                pub unsafe extern "C" fn #c_entrypoint(info: ffi::duckdb_extension_info, access: *const ffi::duckdb_extension_access) -> bool {
+                    let init_result = #c_entrypoint_internal(info, access);
+
+                    if let Err(x) = init_result {
+                        let error_c_string = std::ffi::CString::new(x.to_string());
+
+                        match error_c_string {
+                            Ok(e) => {
+                                (*access).set_error.unwrap()(info, e.as_ptr());
+                            },
+                            Err(_e) => {
+                                let error_alloc_failure = c"An error occured but the extension failed to allocate memory for an error string";
+                                (*access).set_error.unwrap()(info, error_alloc_failure.as_ptr());
+                            }
+                        }
+                        return false;
+                    }
+
+                    init_result.unwrap()
+                }
+
+                #func
+            }
+            .into()
+        }
+        _ => panic!("Only function items are allowed on duckdb_entrypoint"),
+    }
+}
+
 /// Wraps an entrypoint function to expose an unsafe extern "C" function of the same name.
 #[proc_macro_attribute]
 pub fn duckdb_entrypoint(_attr: TokenStream, item: TokenStream) -> TokenStream {

crates/duckdb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "duckdb"
-version = "1.0.0"
+version = "1.1.1"
 authors.workspace = true
 edition.workspace = true
 repository.workspace = true
@@ -35,6 +35,8 @@ polars = ["dep:polars"]
 # FIXME: These were added to make clippy happy: these features appear unused and should perhaps be removed
 column_decltype = []
 extra_check = []
+# Warning: experimental feature
+loadable-extension = ["libduckdb-sys/loadable-extension"]
 
 [dependencies]
 libduckdb-sys = { workspace = true }
@@ -93,3 +95,8 @@ all-features = false
 name = "hello-ext"
 crate-type = ["cdylib"]
 required-features = ["vtab-loadable"]
+
+[[example]]
+name = "hello-ext-capi"
+crate-type = ["cdylib"]
+required-features = ["vtab-loadable", "loadable-extension"]

crates/duckdb/examples/hello-ext-capi/main.rs

@@ -0,0 +1,93 @@
+extern crate duckdb;
+extern crate duckdb_loadable_macros;
+extern crate libduckdb_sys;
+
+use duckdb::{
+    core::{DataChunkHandle, Inserter, LogicalTypeHandle, LogicalTypeId},
+    vtab::{BindInfo, Free, FunctionInfo, InitInfo, VTab},
+    Connection, Result,
+};
+use duckdb_loadable_macros::duckdb_entrypoint_c_api;
+use libduckdb_sys as ffi;
+use std::{
+    error::Error,
+    ffi::{c_char, CString},
+};
+
+#[repr(C)]
+struct HelloBindData {
+    name: *mut c_char,
+}
+
+impl Free for HelloBindData {
+    fn free(&mut self) {
+        unsafe {
+            if self.name.is_null() {
+                return;
+            }
+            drop(CString::from_raw(self.name));
+        }
+    }
+}
+
+#[repr(C)]
+struct HelloInitData {
+    done: bool,
+}
+
+struct HelloVTab;
+
+impl Free for HelloInitData {}
+
+impl VTab for HelloVTab {
+    type InitData = HelloInitData;
+    type BindData = HelloBindData;
+
+    unsafe fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box<dyn std::error::Error>> {
+        bind.add_result_column("column0", LogicalTypeHandle::from(LogicalTypeId::Varchar));
+        let param = bind.get_parameter(0).to_string();
+        unsafe {
+            (*data).name = CString::new(param).unwrap().into_raw();
+        }
+        Ok(())
+    }
+
+    unsafe fn init(_: &InitInfo, data: *mut HelloInitData) -> Result<(), Box<dyn std::error::Error>> {
+        unsafe {
+            (*data).done = false;
+        }
+        Ok(())
+    }
+
+    unsafe fn func(func: &FunctionInfo, output: &mut DataChunkHandle) -> Result<(), Box<dyn std::error::Error>> {
+        let init_info = func.get_init_data::<HelloInitData>();
+        let bind_info = func.get_bind_data::<HelloBindData>();
+
+        unsafe {
+            if (*init_info).done {
+                output.set_len(0);
+            } else {
+                (*init_info).done = true;
+                let vector = output.flat_vector(0);
+                let name = CString::from_raw((*bind_info).name);
+                let result = CString::new(format!("Hello {}", name.to_str()?))?;
+                // Can't consume the CString
+                (*bind_info).name = CString::into_raw(name);
+                vector.insert(0, result);
+                output.set_len(1);
+            }
+        }
+        Ok(())
+    }
+
+    fn parameters() -> Option<Vec<LogicalTypeHandle>> {
+        Some(vec![LogicalTypeHandle::from(LogicalTypeId::Varchar)])
+    }
+}
+
+#[duckdb_entrypoint_c_api(ext_name = "rusty_quack", min_duckdb_version = "v0.0.1")]
+pub fn extension_entrypoint(con: Connection) -> Result<(), Box<dyn Error>> {
+    con.register_table_function::<HelloVTab>("hello")
+        .expect("Failed to register hello table function");
+    Ok(())
+}