build loadable ext (#149)

* build loadable ext

* fix warning

* add loadable macros

* fix comment

* fix clippy error

* fix clippy

Author: wangfenjin

.github/workflows/rust.yaml

@@ -94,6 +94,8 @@ jobs:
         env:
           DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb
           DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb
+      - name: Build loadable extension
+        run: cargo build --example hello-ext --features="vtab-loadable bundled"
 
   Windows:
     name: Windows build from source
@@ -116,6 +118,8 @@ jobs:
           rust-version: stable
           targets: x86_64-pc-windows-msvc
       - run: cargo test --features "modern-full extensions-full"
+      - name: Build loadable extension
+        run: cargo build --example hello-ext --features="vtab-loadable bundled"
 
   Sanitizer:
     name: Address Sanitizer
@@ -139,7 +143,8 @@ jobs:
           # as the other tests have them.
           RUST_BACKTRACE: "0"
         run: cargo -Z build-std test --features "modern-full extensions-full" --target x86_64-unknown-linux-gnu
-
+      - name: Build loadable extension
+        run: cargo build --example hello-ext --features="vtab-loadable bundled"
       - uses: wangfenjin/publish-crates@main
         name: cargo publish --dry-run
         with:

Cargo.toml

@@ -18,7 +18,7 @@ categories = ["database"]
 name = "duckdb"
 
 [workspace]
-members = ["libduckdb-sys"]
+members = ["libduckdb-sys", "duckdb-loadable-macros"]
 
 [features]
 default = []
@@ -27,6 +27,7 @@ httpfs = ["libduckdb-sys/httpfs", "bundled"]
 json = ["libduckdb-sys/json", "bundled"]
 parquet = ["libduckdb-sys/parquet", "bundled"]
 vtab = []
+vtab-loadable = ["vtab", "duckdb-loadable-macros"]
 vtab-excel = ["vtab", "calamine"]
 vtab-arrow = ["vtab", "num"]
 vtab-full = ["vtab-excel", "vtab-arrow"]
@@ -61,6 +62,7 @@ strum = { version = "0.24", features = ["derive"] }
 r2d2 = { version = "0.8.9", optional = true }
 calamine = { version = "0.19.1", optional = true }
 num = { version = "0.4", optional = true, default-features = false, features = ["std"] }
+duckdb-loadable-macros = { version = "0.1.0", path="./duckdb-loadable-macros", optional = true }
 
 [dev-dependencies]
 doc-comment = "0.3"
@@ -90,3 +92,8 @@ default-target = "x86_64-unknown-linux-gnu"
 [package.metadata.playground]
 features = []
 all-features = false
+
+[[example]]
+name = "hello-ext"
+crate-type = ["cdylib"]
+required-features = ["vtab-loadable", "bundled"]

duckdb-loadable-macros/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "duckdb-loadable-macros"
+version = "0.1.0"
+authors = ["wangfenjin <[email protected]>"]
+edition = "2021"
+license = "MIT"
+repository = "https://github.com/wangfenjin/duckdb-rs"
+homepage = "https://github.com/wangfenjin/duckdb-rs"
+keywords = ["duckdb", "ffi", "database"]
+readme = "README.md"
+categories = ["external-ffi-bindings", "database"]
+description = "Native bindings to the libduckdb library, C API; build loadable extensions"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+proc-macro2 = { version = "1.0.56" }
+quote = { version = "1.0.21" }
+syn = { version = "1.0.95", features = [ "extra-traits", "full", "fold", "parsing" ] }
+
+[lib]
+proc-macro = true

duckdb-loadable-macros/LICENSE

@@ -0,0 +1 @@
+../LICENSE

duckdb-loadable-macros/README.md

@@ -0,0 +1 @@
+../README.md

duckdb-loadable-macros/src/lib.rs

@@ -0,0 +1,47 @@
+use proc_macro2::Ident;
+
+use syn::{parse_macro_input, spanned::Spanned, Item};
+
+use proc_macro::TokenStream;
+use quote::quote_spanned;
+
+/// Wraps an entrypoint function to expose an unsafe extern "C" function of the same name.
+#[proc_macro_attribute]
+pub fn duckdb_entrypoint(_attr: TokenStream, item: TokenStream) -> TokenStream {
+    let ast = parse_macro_input!(item as syn::Item);
+    match ast {
+        Item::Fn(mut func) => {
+            let c_entrypoint = func.sig.ident.clone();
+
+            let original_funcname = func.sig.ident.to_string();
+            func.sig.ident = Ident::new(format!("_{}", original_funcname).as_str(), func.sig.ident.span());
+
+            let prefixed_original_function = func.sig.ident.clone();
+
+            quote_spanned! {func.span()=>
+                #func
+
+                /// # Safety
+                ///
+                /// Will be called by duckdb
+                #[no_mangle]
+                pub unsafe extern "C" fn #c_entrypoint(db: *mut c_void) {
+                    let connection = Connection::open_from_raw(db.cast()).expect("can't open db connection");
+                    #prefixed_original_function(connection).expect("init failed");
+                }
+
+                /// # Safety
+                ///
+                /// Predefined function, don't need to change unless you are sure
+                #[no_mangle]
+                pub unsafe extern "C" fn libhello_ext_version() -> *const c_char {
+                    ffi::duckdb_library_version()
+                }
+
+
+            }
+            .into()
+        }
+        _ => panic!("Only function items are allowed on duckdb_entrypoint"),
+    }
+}

examples/hello-ext/main.rs

@@ -0,0 +1,93 @@
+extern crate duckdb;
+extern crate duckdb_loadable_macros;
+extern crate libduckdb_sys;
+
+use duckdb::{
+    vtab::{BindInfo, DataChunk, Free, FunctionInfo, InitInfo, Inserter, LogicalType, LogicalTypeId, VTab},
+    Connection, Result,
+};
+use duckdb_loadable_macros::duckdb_entrypoint;
+use libduckdb_sys as ffi;
+use std::{
+    error::Error,
+    ffi::{c_char, c_void, CString},
+};
+
+#[repr(C)]
+struct HelloBindData {
+    name: *mut c_char,
+}
+
+impl Free for HelloBindData {
+    fn free(&mut self) {
+        unsafe {
+            if self.name.is_null() {
+                return;
+            }
+            drop(CString::from_raw(self.name));
+        }
+    }
+}
+
+#[repr(C)]
+struct HelloInitData {
+    done: bool,
+}
+
+struct HelloVTab;
+
+impl Free for HelloInitData {}
+
+impl VTab for HelloVTab {
+    type InitData = HelloInitData;
+    type BindData = HelloBindData;
+
+    fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box<dyn std::error::Error>> {
+        bind.add_result_column("column0", LogicalType::new(LogicalTypeId::Varchar));
+        let param = bind.get_parameter(0).to_string();
+        unsafe {
+            (*data).name = CString::new(param).unwrap().into_raw();
+        }
+        Ok(())
+    }
+
+    fn init(_: &InitInfo, data: *mut HelloInitData) -> Result<(), Box<dyn std::error::Error>> {
+        unsafe {
+            (*data).done = false;
+        }
+        Ok(())
+    }
+
+    fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box<dyn std::error::Error>> {
+        let init_info = func.get_init_data::<HelloInitData>();
+        let bind_info = func.get_bind_data::<HelloBindData>();
+
+        unsafe {
+            if (*init_info).done {
+                output.set_len(0);
+            } else {
+                (*init_info).done = true;
+                let vector = output.flat_vector(0);
+                let name = CString::from_raw((*bind_info).name);
+                let result = CString::new(format!("Hello {}", name.to_str()?))?;
+                // Can't consume the CString
+                (*bind_info).name = CString::into_raw(name);
+                vector.insert(0, result);
+                output.set_len(1);
+            }
+        }
+        Ok(())
+    }
+
+    fn parameters() -> Option<Vec<LogicalType>> {
+        Some(vec![LogicalType::new(LogicalTypeId::Varchar)])
+    }
+}
+
+// Exposes a extern C function named "libhello_ext_init" in the compiled dynamic library,
+// the "entrypoint" that duckdb will use to load the extension.
+#[duckdb_entrypoint]
+pub fn libhello_ext_init(conn: Connection) -> Result<(), Box<dyn Error>> {
+    conn.register_table_function::<HelloVTab>("hello")?;
+    Ok(())
+}

src/lib.rs

@@ -249,6 +249,23 @@ impl Connection {
         Connection::open_in_memory_with_flags(Config::default())
     }
 
+    /// Open a new connection to an ffi database.
+    ///
+    /// # Failure
+    ///
+    /// Will return `Err` if the underlying DuckDB open call fails.
+    /// # Safety
+    ///
+    /// Need to pass in a valid db instance
+    #[inline]
+    pub unsafe fn open_from_raw(raw: ffi::duckdb_database) -> Result<Connection> {
+        InnerConnection::new(raw, false).map(|db| Connection {
+            db: RefCell::new(db),
+            cache: StatementCache::with_capacity(STATEMENT_CACHE_DEFAULT_CAPACITY),
+            path: None, // Can we know the path from connection?
+        })
+    }
+
     /// Open a new connection to a DuckDB database.
     ///
     /// # Failure
@@ -554,7 +571,7 @@ mod test {
     // this function is never called, but is still type checked; in
     // particular, calls with specific instantiations will require
     // that those types are `Send`.
-    #[allow(dead_code, unconditional_recursion)]
+    #[allow(dead_code, unconditional_recursion, clippy::extra_unused_type_parameters)]
     fn ensure_send<T: Send>() {
         ensure_send::<Connection>();
     }