diff --git a/README.md b/README.md
index e1a41af..66e87ea 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,10 @@
-# Pyo3 extensions for Polars
+## 1. Shared library plugins for Polars
+This is new functionality and not entirely stable, but should be preferred over `2.` as this
+will circumvent the GIL and will be the way we want to support extending polars.
+
+See more in `examples/derive_expression`.
+
+## 2. Pyo3 extensions for Polars
diff --git a/example/derive_expression/expression_lib/src/expressions.rs b/example/derive_expression/expression_lib/src/expressions.rs
index c0aeba2..273acc5 100644
--- a/example/derive_expression/expression_lib/src/expressions.rs
+++ b/example/derive_expression/expression_lib/src/expressions.rs
@@ -12,9 +12,7 @@ fn pig_latin_str(value: &str, output: &mut String) {
#[polars_expr(output_type=Utf8)]
fn pig_latinnify(inputs: &[Series]) -> PolarsResult {
let ca = inputs[0].utf8()?;
-
let out: Utf8Chunked = ca.apply_to_buffer(pig_latin_str);
-
Ok(out.into_series())
}
diff --git a/pyo3-polars-derive/src/attr.rs b/pyo3-polars-derive/src/attr.rs
index c6c2142..a5fecc3 100644
--- a/pyo3-polars-derive/src/attr.rs
+++ b/pyo3-polars-derive/src/attr.rs
@@ -1,21 +1,21 @@
-use std::fmt::Debug;
+use crate::keywords;
use proc_macro2::Ident;
+use std::fmt::Debug;
use syn::parse::{Parse, ParseStream};
use syn::Token;
-use crate::keywords;
#[derive(Clone, Debug)]
pub struct KeyWordAttribute {
pub kw: K,
- pub value: V
+ pub value: V,
}
impl Parse for KeyWordAttribute {
fn parse(input: ParseStream) -> syn::Result {
- let kw= input.parse()?;
+ let kw = input.parse()?;
let _: Token![=] = input.parse()?;
let value = input.parse()?;
- Ok(KeyWordAttribute{kw, value})
+ Ok(KeyWordAttribute { kw, value })
}
}
@@ -25,7 +25,7 @@ pub type OutputFuncAttribute = KeyWordAttribute;
#[derive(Default, Debug)]
pub struct ExprsFunctionOptions {
pub output_dtype: Option,
- pub output_type_fn: Option
+ pub output_type_fn: Option,
}
impl Parse for ExprsFunctionOptions {
@@ -47,4 +47,4 @@ impl Parse for ExprsFunctionOptions {
}
Ok(options)
}
-}
\ No newline at end of file
+}
diff --git a/pyo3-polars-derive/src/keywords.rs b/pyo3-polars-derive/src/keywords.rs
index 8bb6767..062baba 100644
--- a/pyo3-polars-derive/src/keywords.rs
+++ b/pyo3-polars-derive/src/keywords.rs
@@ -1,3 +1,2 @@
-
syn::custom_keyword!(output_type);
syn::custom_keyword!(type_func);
diff --git a/pyo3-polars-derive/src/lib.rs b/pyo3-polars-derive/src/lib.rs
index 21cccbc..f0de3d5 100644
--- a/pyo3-polars-derive/src/lib.rs
+++ b/pyo3-polars-derive/src/lib.rs
@@ -3,7 +3,7 @@ mod keywords;
use proc_macro::TokenStream;
use quote::quote;
-use syn::{parse_macro_input};
+use syn::parse_macro_input;
fn create_expression_function(ast: syn::ItemFn) -> proc_macro2::TokenStream {
let fn_name = &ast.sig.ident;
@@ -20,7 +20,6 @@ fn create_expression_function(ast: syn::ItemFn) -> proc_macro2::TokenStream {
// call the function
let output: polars_core::prelude::Series = #fn_name(&inputs).unwrap();
- // let output = polars_core::prelude::Series::full_null("", 3, &DataType::Null);
let out = polars_ffi::export_series(&output);
out
}
@@ -32,17 +31,18 @@ fn get_field_name(fn_name: &syn::Ident) -> syn::Ident {
}
fn get_inputs() -> proc_macro2::TokenStream {
- quote!(
- let inputs = std::slice::from_raw_parts(field, len);
- let inputs = inputs.iter().map(|field| {
- let field = polars_core::export::arrow::ffi::import_field_from_c(field).unwrap();
- polars_core::prelude::Field::from(&field)
- }).collect::>();
- )
+ quote!(
+ let inputs = std::slice::from_raw_parts(field, len);
+ let inputs = inputs.iter().map(|field| {
+ let field = polars_core::export::arrow::ffi::import_field_from_c(field).unwrap();
+ let out = polars_core::prelude::Field::from(&field);
+ out
+ }).collect::>();
+ )
}
fn create_field_function(fn_name: &syn::Ident) -> proc_macro2::TokenStream {
- let map_field_name = get_field_name(&fn_name);
+ let map_field_name = get_field_name(fn_name);
let inputs = get_inputs();
quote! (
@@ -55,7 +55,10 @@ fn create_field_function(fn_name: &syn::Ident) -> proc_macro2::TokenStream {
)
}
-fn create_field_function_from_with_dtype(fn_name: &syn::Ident, dtype: syn::Ident) -> proc_macro2::TokenStream {
+fn create_field_function_from_with_dtype(
+ fn_name: &syn::Ident,
+ dtype: syn::Ident,
+) -> proc_macro2::TokenStream {
let map_field_name = get_field_name(fn_name);
let inputs = get_inputs();
@@ -72,8 +75,6 @@ fn create_field_function_from_with_dtype(fn_name: &syn::Ident, dtype: syn::Ident
)
}
-
-
#[proc_macro_attribute]
pub fn polars_expr(attr: TokenStream, input: TokenStream) -> TokenStream {
let ast = parse_macro_input!(input as syn::ItemFn);
diff --git a/pyo3-polars-derive/tests/01.rs b/pyo3-polars-derive/tests/01.rs
index 111d123..cb0082b 100644
--- a/pyo3-polars-derive/tests/01.rs
+++ b/pyo3-polars-derive/tests/01.rs
@@ -1,7 +1,7 @@
use polars_core::error::PolarsResult;
-use pyo3_polars_derive::polars_expr;
-use polars_core::prelude::{Series, Field};
+use polars_core::prelude::{Field, Series};
use polars_plan::dsl::FieldsMapper;
+use pyo3_polars_derive::polars_expr;
fn horizontal_product_output(input_fields: &[Field]) -> PolarsResult {
FieldsMapper::new(input_fields).map_to_supertype()
@@ -16,7 +16,4 @@ fn horizontal_product(series: &[Series]) -> PolarsResult {
Ok(acc)
}
-
-fn main() {
-
-}
\ No newline at end of file
+fn main() {}
diff --git a/pyo3-polars-derive/tests/02.rs b/pyo3-polars-derive/tests/02.rs
index e6fce3e..fda4347 100644
--- a/pyo3-polars-derive/tests/02.rs
+++ b/pyo3-polars-derive/tests/02.rs
@@ -1,6 +1,6 @@
use polars_core::error::PolarsResult;
+use polars_core::prelude::Series;
use pyo3_polars_derive::polars_expr;
-use polars_core::prelude::{Series};
#[polars_expr(output_type=Int32)]
fn horizontal_product(series: &[Series]) -> PolarsResult {
@@ -11,7 +11,4 @@ fn horizontal_product(series: &[Series]) -> PolarsResult {
Ok(acc)
}
-
-fn main() {
-
-}
+fn main() {}
diff --git a/pyo3-polars/src/error.rs b/pyo3-polars/src/error.rs
index 16df9e2..6ff4720 100644
--- a/pyo3-polars/src/error.rs
+++ b/pyo3-polars/src/error.rs
@@ -3,7 +3,7 @@ use std::fmt::{Debug, Formatter};
use polars::prelude::PolarsError;
use polars_core::error::ArrowError;
use pyo3::create_exception;
-use pyo3::exceptions::{PyException, PyIOError, PyRuntimeError, PyValueError};
+use pyo3::exceptions::{PyException, PyIndexError, PyIOError, PyRuntimeError, PyValueError};
use pyo3::prelude::*;
use thiserror::Error;
@@ -29,6 +29,7 @@ impl std::convert::From for PyErr {
PolarsError::ShapeMismatch(err) => ShapeError::new_err(err.to_string()),
PolarsError::SchemaMismatch(err) => SchemaError::new_err(err.to_string()),
PolarsError::Io(err) => PyIOError::new_err(err.to_string()),
+ PolarsError::OutOfBounds(err) => PyIndexError::new_err(err.to_string()),
PolarsError::InvalidOperation(err) => PyValueError::new_err(err.to_string()),
PolarsError::ArrowError(err) => ArrowErrorException::new_err(format!("{:?}", err)),
PolarsError::Duplicate(err) => DuplicateError::new_err(err.to_string()),
diff --git a/pyo3-polars/src/export.rs b/pyo3-polars/src/export.rs
index 9ffd746..76551b3 100644
--- a/pyo3-polars/src/export.rs
+++ b/pyo3-polars/src/export.rs
@@ -1,3 +1,3 @@
+pub use polars_core;
pub use polars_ffi;
pub use polars_plan;
-pub use polars_core;
diff --git a/pyo3-polars/src/ffi/to_py.rs b/pyo3-polars/src/ffi/to_py.rs
index 7316aee..9b4fa3a 100644
--- a/pyo3-polars/src/ffi/to_py.rs
+++ b/pyo3-polars/src/ffi/to_py.rs
@@ -1,7 +1,7 @@
+use polars::export::arrow::ffi;
use polars::prelude::{ArrayRef, ArrowField};
use pyo3::ffi::Py_uintptr_t;
use pyo3::prelude::*;
-use polars::export::arrow::ffi;
/// Arrow array to Python.
pub(crate) fn to_py_array(array: ArrayRef, py: Python, pyarrow: &PyModule) -> PyResult {
diff --git a/pyo3-polars/src/lib.rs b/pyo3-polars/src/lib.rs
index 1fd3d04..237b796 100644
--- a/pyo3-polars/src/lib.rs
+++ b/pyo3-polars/src/lib.rs
@@ -41,10 +41,12 @@
//! })
//! out_df = my_cool_function(df)
//! ```
-pub mod error;
-mod ffi;
+#[cfg(feature = "derive")]
pub mod derive;
+pub mod error;
+#[cfg(feature = "derive")]
pub mod export;
+mod ffi;
use crate::error::PyPolarsErr;
use crate::ffi::to_py::to_py_array;