Merge branch 'main' of https://github.com/fkemeth/polars into feature…

…/Add_min_periods_to_polars.corr_#15458

Cargo.toml

@@ -14,7 +14,7 @@ default-members = [
 # ]
 
 [workspace.package]
-version = "0.40.0"
+version = "0.41.0"
 authors = ["Ritchie Vink <[email protected]>"]
 edition = "2021"
 homepage = "https://www.pola.rs/"
@@ -93,32 +93,33 @@ version_check = "0.9.4"
 xxhash-rust = { version = "0.8.6", features = ["xxh3"] }
 zstd = "0.13"
 
-polars = { version = "0.40.0", path = "crates/polars", default-features = false }
-polars-compute = { version = "0.40.0", path = "crates/polars-compute", default-features = false }
-polars-core = { version = "0.40.0", path = "crates/polars-core", default-features = false }
-polars-error = { version = "0.40.0", path = "crates/polars-error", default-features = false }
-polars-expr = { version = "0.40.0", path = "crates/polars-expr", default-features = false }
-polars-ffi = { version = "0.40.0", path = "crates/polars-ffi", default-features = false }
-polars-io = { version = "0.40.0", path = "crates/polars-io", default-features = false }
-polars-json = { version = "0.40.0", path = "crates/polars-json", default-features = false }
-polars-lazy = { version = "0.40.0", path = "crates/polars-lazy", default-features = false }
-polars-ops = { version = "0.40.0", path = "crates/polars-ops", default-features = false }
-polars-parquet = { version = "0.40.0", path = "crates/polars-parquet", default-features = false }
-polars-pipe = { version = "0.40.0", path = "crates/polars-pipe", default-features = false }
-polars-plan = { version = "0.40.0", path = "crates/polars-plan", default-features = false }
-polars-row = { version = "0.40.0", path = "crates/polars-row", default-features = false }
-polars-sql = { version = "0.40.0", path = "crates/polars-sql", default-features = false }
-polars-stream = { version = "0.40.0", path = "crates/polars-stream", default-features = false }
-polars-time = { version = "0.40.0", path = "crates/polars-time", default-features = false }
-polars-utils = { version = "0.40.0", path = "crates/polars-utils", default-features = false }
+polars = { version = "0.41.0", path = "crates/polars", default-features = false }
+polars-compute = { version = "0.41.0", path = "crates/polars-compute", default-features = false }
+polars-core = { version = "0.41.0", path = "crates/polars-core", default-features = false }
+polars-error = { version = "0.41.0", path = "crates/polars-error", default-features = false }
+polars-expr = { version = "0.41.0", path = "crates/polars-expr", default-features = false }
+polars-ffi = { version = "0.41.0", path = "crates/polars-ffi", default-features = false }
+polars-io = { version = "0.41.0", path = "crates/polars-io", default-features = false }
+polars-json = { version = "0.41.0", path = "crates/polars-json", default-features = false }
+polars-lazy = { version = "0.41.0", path = "crates/polars-lazy", default-features = false }
+polars-mem-engine = { version = "0.41.0", path = "crates/polars-mem-engine", default-features = false }
+polars-ops = { version = "0.41.0", path = "crates/polars-ops", default-features = false }
+polars-parquet = { version = "0.41.0", path = "crates/polars-parquet", default-features = false }
+polars-pipe = { version = "0.41.0", path = "crates/polars-pipe", default-features = false }
+polars-plan = { version = "0.41.0", path = "crates/polars-plan", default-features = false }
+polars-row = { version = "0.41.0", path = "crates/polars-row", default-features = false }
+polars-sql = { version = "0.41.0", path = "crates/polars-sql", default-features = false }
+polars-stream = { version = "0.41.0", path = "crates/polars-stream", default-features = false }
+polars-time = { version = "0.41.0", path = "crates/polars-time", default-features = false }
+polars-utils = { version = "0.41.0", path = "crates/polars-utils", default-features = false }
 
 [workspace.dependencies.arrow-format]
 package = "polars-arrow-format"
 version = "0.1.0"
 
 [workspace.dependencies.arrow]
 package = "polars-arrow"
-version = "0.40.0"
+version = "0.41.0"
 path = "crates/polars-arrow"
 default-features = false
 features = [

_typos.toml

@@ -10,13 +10,15 @@ ND = "ND"
 ba = "ba"
 nd = "nd"
 opt_nd = "opt_nd"
-ser = "ser"
+ANDed = "ANDed"
 
 [default.extend-words]
 arange = "arange"
 strat = "strat"
 '"r0ot"' = "r0ot"
 wee = "wee"
+ser = "ser"
+ND = "ND"
 
 [type.csv]
 extend-glob = ["*.csv"]

crates/Makefile

@@ -116,6 +116,8 @@ publish:  ## Publish Polars crates
 	cargo publish --allow-dirty -p polars-io
 	cargo publish --allow-dirty -p polars-plan
 	cargo publish --allow-dirty -p polars-expr
+	cargo publish --allow-dirty -p polars-mem-engine
+	cargo publish --allow-dirty -p polars-stream
 	cargo publish --allow-dirty -p polars-pipe
 	cargo publish --allow-dirty -p polars-lazy
 	cargo publish --allow-dirty -p polars-sql

crates/polars-arrow/src/array/binview/buffers.rs

@@ -0,0 +1,36 @@
+use std::hash::{Hash, Hasher};
+use std::sync::Arc;
+use polars_utils::aliases::{InitHashMaps, PlHashSet, PlIndexSet};
+use crate::buffer::Buffer;
+
+pub struct BufferKey<'a> {
+    pub inner: &'a Buffer<u8>,
+}
+
+impl Hash for BufferKey<'_> {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        state.write_u64(self.inner.as_ptr() as u64)
+    }
+}
+
+impl PartialEq for BufferKey<'_> {
+    #[inline]
+    fn eq(&self, other: &Self) -> bool {
+        self.inner.as_ptr() == other.inner.as_ptr()
+    }
+}
+
+impl Eq for BufferKey<'_> {}
+
+pub fn dedupe_view_buffers<'a, I: Iterator<Item=&'a Arc<[Buffer<u8>]>>>(iter: I) -> PlIndexSet<BufferKey<'a>> {
+    // Deduplicate a whole Arc<[buffer]> group
+    let mut processed_buffer_groups = PlHashSet::new();
+    // Deduplicate the separate Buffers.
+    let mut buffers = PlIndexSet::new();
+    for data_buffers in iter {
+        if processed_buffer_groups.insert(data_buffers.as_ptr() as usize) {
+            buffers.extend(data_buffers.iter().map(|buf| BufferKey { inner: buf }))
+        }
+    }
+    buffers
+}

crates/polars-arrow/src/array/equal/mod.rs

@@ -217,7 +217,7 @@ pub fn equal(lhs: &dyn Array, rhs: &dyn Array) -> bool {
             let rhs = rhs.as_any().downcast_ref().unwrap();
             boolean::equal(lhs, rhs)
         },
-        Primitive(primitive) => with_match_primitive_type!(primitive, |$T| {
+        Primitive(primitive) => with_match_primitive_type_full!(primitive, |$T| {
             let lhs = lhs.as_any().downcast_ref().unwrap();
             let rhs = rhs.as_any().downcast_ref().unwrap();
             primitive::equal::<$T>(lhs, rhs)

crates/polars-arrow/src/array/fmt.rs

@@ -2,7 +2,7 @@ use std::fmt::{Result, Write};
 
 use super::Array;
 use crate::bitmap::Bitmap;
-use crate::{match_integer_type, with_match_primitive_type};
+use crate::{match_integer_type, with_match_primitive_type_full};
 
 /// Returns a function that writes the value of the element of `array`
 /// at position `index` to a [`Write`],
@@ -17,7 +17,7 @@ pub fn get_value_display<'a, F: Write + 'a>(
         Boolean => Box::new(|f, index| {
             super::boolean::fmt::write_value(array.as_any().downcast_ref().unwrap(), index, f)
         }),
-        Primitive(primitive) => with_match_primitive_type!(primitive, |$T| {
+        Primitive(primitive) => with_match_primitive_type_full!(primitive, |$T| {
             let writer = super::primitive::fmt::get_write_value::<$T, _>(
                 array.as_any().downcast_ref().unwrap(),
             );