diff --git a/README.md b/README.md index e1a41af..66e87ea 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,10 @@ -# Pyo3 extensions for Polars +## 1. Shared library plugins for Polars +This is new functionality and not entirely stable, but should be preferred over `2.` as this +will circumvent the GIL and will be the way we want to support extending polars. + +See more in `examples/derive_expression`. + +## 2. Pyo3 extensions for Polars diff --git a/example/derive_expression/expression_lib/src/expressions.rs b/example/derive_expression/expression_lib/src/expressions.rs index c0aeba2..273acc5 100644 --- a/example/derive_expression/expression_lib/src/expressions.rs +++ b/example/derive_expression/expression_lib/src/expressions.rs @@ -12,9 +12,7 @@ fn pig_latin_str(value: &str, output: &mut String) { #[polars_expr(output_type=Utf8)] fn pig_latinnify(inputs: &[Series]) -> PolarsResult { let ca = inputs[0].utf8()?; - let out: Utf8Chunked = ca.apply_to_buffer(pig_latin_str); - Ok(out.into_series()) } diff --git a/pyo3-polars-derive/src/attr.rs b/pyo3-polars-derive/src/attr.rs index c6c2142..a5fecc3 100644 --- a/pyo3-polars-derive/src/attr.rs +++ b/pyo3-polars-derive/src/attr.rs @@ -1,21 +1,21 @@ -use std::fmt::Debug; +use crate::keywords; use proc_macro2::Ident; +use std::fmt::Debug; use syn::parse::{Parse, ParseStream}; use syn::Token; -use crate::keywords; #[derive(Clone, Debug)] pub struct KeyWordAttribute { pub kw: K, - pub value: V + pub value: V, } impl Parse for KeyWordAttribute { fn parse(input: ParseStream) -> syn::Result { - let kw= input.parse()?; + let kw = input.parse()?; let _: Token![=] = input.parse()?; let value = input.parse()?; - Ok(KeyWordAttribute{kw, value}) + Ok(KeyWordAttribute { kw, value }) } } @@ -25,7 +25,7 @@ pub type OutputFuncAttribute = KeyWordAttribute; #[derive(Default, Debug)] pub struct ExprsFunctionOptions { pub output_dtype: Option, - pub output_type_fn: Option + pub output_type_fn: Option, } impl Parse for ExprsFunctionOptions { @@ -47,4 +47,4 @@ impl Parse for ExprsFunctionOptions { } Ok(options) } -} \ No newline at end of file +} diff --git a/pyo3-polars-derive/src/keywords.rs b/pyo3-polars-derive/src/keywords.rs index 8bb6767..062baba 100644 --- a/pyo3-polars-derive/src/keywords.rs +++ b/pyo3-polars-derive/src/keywords.rs @@ -1,3 +1,2 @@ - syn::custom_keyword!(output_type); syn::custom_keyword!(type_func); diff --git a/pyo3-polars-derive/src/lib.rs b/pyo3-polars-derive/src/lib.rs index 21cccbc..f0de3d5 100644 --- a/pyo3-polars-derive/src/lib.rs +++ b/pyo3-polars-derive/src/lib.rs @@ -3,7 +3,7 @@ mod keywords; use proc_macro::TokenStream; use quote::quote; -use syn::{parse_macro_input}; +use syn::parse_macro_input; fn create_expression_function(ast: syn::ItemFn) -> proc_macro2::TokenStream { let fn_name = &ast.sig.ident; @@ -20,7 +20,6 @@ fn create_expression_function(ast: syn::ItemFn) -> proc_macro2::TokenStream { // call the function let output: polars_core::prelude::Series = #fn_name(&inputs).unwrap(); - // let output = polars_core::prelude::Series::full_null("", 3, &DataType::Null); let out = polars_ffi::export_series(&output); out } @@ -32,17 +31,18 @@ fn get_field_name(fn_name: &syn::Ident) -> syn::Ident { } fn get_inputs() -> proc_macro2::TokenStream { - quote!( - let inputs = std::slice::from_raw_parts(field, len); - let inputs = inputs.iter().map(|field| { - let field = polars_core::export::arrow::ffi::import_field_from_c(field).unwrap(); - polars_core::prelude::Field::from(&field) - }).collect::>(); - ) + quote!( + let inputs = std::slice::from_raw_parts(field, len); + let inputs = inputs.iter().map(|field| { + let field = polars_core::export::arrow::ffi::import_field_from_c(field).unwrap(); + let out = polars_core::prelude::Field::from(&field); + out + }).collect::>(); + ) } fn create_field_function(fn_name: &syn::Ident) -> proc_macro2::TokenStream { - let map_field_name = get_field_name(&fn_name); + let map_field_name = get_field_name(fn_name); let inputs = get_inputs(); quote! ( @@ -55,7 +55,10 @@ fn create_field_function(fn_name: &syn::Ident) -> proc_macro2::TokenStream { ) } -fn create_field_function_from_with_dtype(fn_name: &syn::Ident, dtype: syn::Ident) -> proc_macro2::TokenStream { +fn create_field_function_from_with_dtype( + fn_name: &syn::Ident, + dtype: syn::Ident, +) -> proc_macro2::TokenStream { let map_field_name = get_field_name(fn_name); let inputs = get_inputs(); @@ -72,8 +75,6 @@ fn create_field_function_from_with_dtype(fn_name: &syn::Ident, dtype: syn::Ident ) } - - #[proc_macro_attribute] pub fn polars_expr(attr: TokenStream, input: TokenStream) -> TokenStream { let ast = parse_macro_input!(input as syn::ItemFn); diff --git a/pyo3-polars-derive/tests/01.rs b/pyo3-polars-derive/tests/01.rs index 111d123..cb0082b 100644 --- a/pyo3-polars-derive/tests/01.rs +++ b/pyo3-polars-derive/tests/01.rs @@ -1,7 +1,7 @@ use polars_core::error::PolarsResult; -use pyo3_polars_derive::polars_expr; -use polars_core::prelude::{Series, Field}; +use polars_core::prelude::{Field, Series}; use polars_plan::dsl::FieldsMapper; +use pyo3_polars_derive::polars_expr; fn horizontal_product_output(input_fields: &[Field]) -> PolarsResult { FieldsMapper::new(input_fields).map_to_supertype() @@ -16,7 +16,4 @@ fn horizontal_product(series: &[Series]) -> PolarsResult { Ok(acc) } - -fn main() { - -} \ No newline at end of file +fn main() {} diff --git a/pyo3-polars-derive/tests/02.rs b/pyo3-polars-derive/tests/02.rs index e6fce3e..fda4347 100644 --- a/pyo3-polars-derive/tests/02.rs +++ b/pyo3-polars-derive/tests/02.rs @@ -1,6 +1,6 @@ use polars_core::error::PolarsResult; +use polars_core::prelude::Series; use pyo3_polars_derive::polars_expr; -use polars_core::prelude::{Series}; #[polars_expr(output_type=Int32)] fn horizontal_product(series: &[Series]) -> PolarsResult { @@ -11,7 +11,4 @@ fn horizontal_product(series: &[Series]) -> PolarsResult { Ok(acc) } - -fn main() { - -} +fn main() {} diff --git a/pyo3-polars/src/error.rs b/pyo3-polars/src/error.rs index 16df9e2..6ff4720 100644 --- a/pyo3-polars/src/error.rs +++ b/pyo3-polars/src/error.rs @@ -3,7 +3,7 @@ use std::fmt::{Debug, Formatter}; use polars::prelude::PolarsError; use polars_core::error::ArrowError; use pyo3::create_exception; -use pyo3::exceptions::{PyException, PyIOError, PyRuntimeError, PyValueError}; +use pyo3::exceptions::{PyException, PyIndexError, PyIOError, PyRuntimeError, PyValueError}; use pyo3::prelude::*; use thiserror::Error; @@ -29,6 +29,7 @@ impl std::convert::From for PyErr { PolarsError::ShapeMismatch(err) => ShapeError::new_err(err.to_string()), PolarsError::SchemaMismatch(err) => SchemaError::new_err(err.to_string()), PolarsError::Io(err) => PyIOError::new_err(err.to_string()), + PolarsError::OutOfBounds(err) => PyIndexError::new_err(err.to_string()), PolarsError::InvalidOperation(err) => PyValueError::new_err(err.to_string()), PolarsError::ArrowError(err) => ArrowErrorException::new_err(format!("{:?}", err)), PolarsError::Duplicate(err) => DuplicateError::new_err(err.to_string()), diff --git a/pyo3-polars/src/export.rs b/pyo3-polars/src/export.rs index 9ffd746..76551b3 100644 --- a/pyo3-polars/src/export.rs +++ b/pyo3-polars/src/export.rs @@ -1,3 +1,3 @@ +pub use polars_core; pub use polars_ffi; pub use polars_plan; -pub use polars_core; diff --git a/pyo3-polars/src/ffi/to_py.rs b/pyo3-polars/src/ffi/to_py.rs index 7316aee..9b4fa3a 100644 --- a/pyo3-polars/src/ffi/to_py.rs +++ b/pyo3-polars/src/ffi/to_py.rs @@ -1,7 +1,7 @@ +use polars::export::arrow::ffi; use polars::prelude::{ArrayRef, ArrowField}; use pyo3::ffi::Py_uintptr_t; use pyo3::prelude::*; -use polars::export::arrow::ffi; /// Arrow array to Python. pub(crate) fn to_py_array(array: ArrayRef, py: Python, pyarrow: &PyModule) -> PyResult { diff --git a/pyo3-polars/src/lib.rs b/pyo3-polars/src/lib.rs index 1fd3d04..237b796 100644 --- a/pyo3-polars/src/lib.rs +++ b/pyo3-polars/src/lib.rs @@ -41,10 +41,12 @@ //! }) //! out_df = my_cool_function(df) //! ``` -pub mod error; -mod ffi; +#[cfg(feature = "derive")] pub mod derive; +pub mod error; +#[cfg(feature = "derive")] pub mod export; +mod ffi; use crate::error::PyPolarsErr; use crate::ffi::to_py::to_py_array;