Skip to content

Commit

Permalink
chore: pushdown TRIM_DECIMAL_ZEROS into arrow
Browse files Browse the repository at this point in the history
  • Loading branch information
flisky committed Apr 3, 2024
1 parent bf2a00b commit 65661a7
Show file tree
Hide file tree
Showing 6 changed files with 21 additions and 11 deletions.
1 change: 0 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 11 additions & 0 deletions crates/polars-arrow/src/compute/decimal.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,16 @@
use std::sync::atomic::{AtomicBool, Ordering};

use atoi::FromRadix10SignedChecked;

static TRIM_DECIMAL_ZEROS: AtomicBool = AtomicBool::new(false);

pub fn get_trim_decimal_zeros() -> bool {
TRIM_DECIMAL_ZEROS.load(Ordering::Relaxed)
}
pub fn set_trim_decimal_zeros(trim: Option<bool>) {
TRIM_DECIMAL_ZEROS.store(trim.unwrap_or(false), Ordering::Relaxed)
}

/// Count the number of b'0's at the beginning of a slice.
fn leading_zeros(bytes: &[u8]) -> u8 {
bytes.iter().take_while(|byte| **byte == b'0').count() as u8
Expand Down
9 changes: 5 additions & 4 deletions crates/polars-core/src/fmt.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#[cfg(any(feature = "fmt", feature = "fmt_no_tty"))]
use std::borrow::Cow;
use std::fmt::{Debug, Display, Formatter, Write};
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use std::sync::atomic::{AtomicU8, Ordering};
use std::sync::RwLock;
use std::{fmt, str};

Expand Down Expand Up @@ -39,7 +39,6 @@ pub enum FloatFmt {
static FLOAT_PRECISION: RwLock<Option<usize>> = RwLock::new(None);
static FLOAT_FMT: AtomicU8 = AtomicU8::new(FloatFmt::Mixed as u8);

static TRIM_DECIMAL_ZEROS: AtomicBool = AtomicBool::new(false);
static THOUSANDS_SEPARATOR: AtomicU8 = AtomicU8::new(b'\0');
static DECIMAL_SEPARATOR: AtomicU8 = AtomicU8::new(b'.');

Expand All @@ -65,8 +64,9 @@ pub fn get_thousands_separator() -> String {
sep.to_string()
}
}
#[cfg(feature = "dtype-decimal")]
pub fn get_trim_decimal_zeros() -> bool {
TRIM_DECIMAL_ZEROS.load(Ordering::Relaxed)
arrow::compute::decimal::get_trim_decimal_zeros()
}

// Numeric formatting setters
Expand All @@ -82,8 +82,9 @@ pub fn set_decimal_separator(dec: Option<char>) {
pub fn set_thousands_separator(sep: Option<char>) {
THOUSANDS_SEPARATOR.store(sep.unwrap_or('\0') as u8, Ordering::Relaxed)
}
#[cfg(feature = "dtype-decimal")]
pub fn set_trim_decimal_zeros(trim: Option<bool>) {
TRIM_DECIMAL_ZEROS.store(trim.unwrap_or(false), Ordering::Relaxed)
arrow::compute::decimal::set_trim_decimal_zeros(trim)
}

macro_rules! format_array {
Expand Down
4 changes: 2 additions & 2 deletions crates/polars-io/src/csv/write_impl.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,8 +160,8 @@ unsafe fn write_any_value(
write!(f, "{quote}")?;
end_with_quote = true
}
let trim_zeros = polars_core::fmt::get_trim_decimal_zeros();
let buf = arrow::legacy::compute::decimal::format_decimal(v, scale, trim_zeros);
let trim_zeros = arrow::compute::decimal::get_trim_decimal_zeros();
let buf = arrow::compute::decimal::format_decimal(v, scale, trim_zeros);
write!(f, "{}", buf.as_str())
},
_ => {
Expand Down
1 change: 0 additions & 1 deletion crates/polars-json/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ repository = { workspace = true }
description = "JSON related logic for the Polars DataFrame library"

[dependencies]
polars-core = { workspace = true }
polars-error = { workspace = true }
polars-utils = { workspace = true }

Expand Down
6 changes: 3 additions & 3 deletions crates/polars-json/src/json/write/serialize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@ use std::io::Write;

use arrow::array::*;
use arrow::bitmap::utils::ZipValidity;
#[cfg(feature = "dtype-decimal")]
use arrow::compute::decimal::{format_decimal, get_trim_decimal_zeros};
use arrow::datatypes::{ArrowDataType, IntegerType, TimeUnit};
use arrow::io::iterator::BufStreamingIterator;
#[cfg(feature = "dtype-decimal")]
use arrow::legacy::compute::decimal::format_decimal;
use arrow::offset::Offset;
#[cfg(feature = "chrono-tz")]
use arrow::temporal_conversions::parse_offset_tz;
Expand Down Expand Up @@ -121,7 +121,7 @@ fn decimal_serializer<'a>(
offset: usize,
take: usize,
) -> Box<dyn StreamingIterator<Item = [u8]> + 'a + Send + Sync> {
let trim_zeros = polars_core::fmt::get_trim_decimal_zeros();
let trim_zeros = get_trim_decimal_zeros();
let f = move |x: Option<&i128>, buf: &mut Vec<u8>| {
if let Some(x) = x {
utf8::write_str(buf, format_decimal(*x, scale, trim_zeros).as_str()).unwrap()
Expand Down

0 comments on commit 65661a7

Please sign in to comment.