From c23ee7342c0c822f1458f892c60edfdd7ce857cc Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Fri, 1 Nov 2024 06:12:20 +0900 Subject: [PATCH 01/29] GH-44589: [GLib] Add GArrowDecimal32 class (#44597) ### Rationale for this change The `Decimal32` class has been introduced. It is necessary to support it in GLib. ### What changes are included in this PR? Implement `GArrowDecimal32` class ### Are these changes tested? YES ### Are there any user-facing changes? NO * GitHub Issue: #44589 Authored-by: Hiroyuki Sato Signed-off-by: Sutou Kouhei --- c_glib/arrow-glib/decimal.cpp | 434 +++++++++++++++++++++++++++++++++- c_glib/arrow-glib/decimal.h | 79 +++++++ c_glib/arrow-glib/decimal.hpp | 5 + c_glib/test/test-decimal32.rb | 222 +++++++++++++++++ 4 files changed, 739 insertions(+), 1 deletion(-) create mode 100644 c_glib/test/test-decimal32.rb diff --git a/c_glib/arrow-glib/decimal.cpp b/c_glib/arrow-glib/decimal.cpp index edc2af7a7e051..30b596f7a33c0 100644 --- a/c_glib/arrow-glib/decimal.cpp +++ b/c_glib/arrow-glib/decimal.cpp @@ -24,6 +24,24 @@ template struct DecimalConverter { }; +template <> struct DecimalConverter +{ + using ArrowType = arrow::Decimal32; + using GArrowType = GArrowDecimal32; + + GArrowType * + new_raw(std::shared_ptr *arrow_decimal32) + { + return garrow_decimal32_new_raw(arrow_decimal32); + } + + std::shared_ptr + get_raw(GArrowType *decimal32) + { + return garrow_decimal32_get_raw(decimal32); + } +}; + template <> struct DecimalConverter { using ArrowType = arrow::Decimal64; @@ -319,9 +337,11 @@ G_BEGIN_DECLS /** * SECTION: decimal * @section_id: decimal - * @title: 64-bit, 128-bit and 256-bit decimal classes + * @title: 32-bit, 64-bit, 128-bit and 256-bit decimal classes * @include: arrow-glib/arrow-glib.h * + * #GArrowDecimal32 is a 32-bit decimal class. + * * #GArrowDecimal64 is a 64-bit decimal class. * * #GArrowDecimal128 is a 128-bit decimal class. @@ -331,6 +351,403 @@ G_BEGIN_DECLS * Since: 0.10.0 */ +typedef struct GArrowDecimal32Private_ +{ + std::shared_ptr decimal32; +} GArrowDecimal32Private; + +enum { + PROP_DECIMAL32 = 1 +}; + +G_DEFINE_TYPE_WITH_PRIVATE(GArrowDecimal32, garrow_decimal32, G_TYPE_OBJECT) + +#define GARROW_DECIMAL32_GET_PRIVATE(obj) \ + static_cast( \ + garrow_decimal32_get_instance_private(GARROW_DECIMAL32(obj))) + +static void +garrow_decimal32_finalize(GObject *object) +{ + auto priv = GARROW_DECIMAL32_GET_PRIVATE(object); + + priv->decimal32.~shared_ptr(); + + G_OBJECT_CLASS(garrow_decimal32_parent_class)->finalize(object); +} + +static void +garrow_decimal32_set_property(GObject *object, + guint prop_id, + const GValue *value, + GParamSpec *pspec) +{ + auto priv = GARROW_DECIMAL32_GET_PRIVATE(object); + + switch (prop_id) { + case PROP_DECIMAL32: + priv->decimal32 = + *static_cast *>(g_value_get_pointer(value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void +garrow_decimal32_init(GArrowDecimal32 *object) +{ + auto priv = GARROW_DECIMAL32_GET_PRIVATE(object); + new (&priv->decimal32) std::shared_ptr; +} + +static void +garrow_decimal32_class_init(GArrowDecimal32Class *klass) +{ + auto gobject_class = G_OBJECT_CLASS(klass); + + gobject_class->finalize = garrow_decimal32_finalize; + gobject_class->set_property = garrow_decimal32_set_property; + + GParamSpec *spec; + spec = g_param_spec_pointer( + "decimal32", + "Decimal32", + "The raw std::shared *", + static_cast(G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY)); + g_object_class_install_property(gobject_class, PROP_DECIMAL32, spec); +} + +/** + * garrow_decimal32_new_string: + * @data: The data of the decimal. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (nullable): + * A newly created #GArrowDecimal32 on success, %NULL on error. + * + * Since: 19.0.0 + */ +GArrowDecimal32 * +garrow_decimal32_new_string(const gchar *data, GError **error) +{ + return garrow_decimal_new_string(data, + error, + "[decimal32][new][string]"); +} + +/** + * garrow_decimal32_new_integer: + * @data: The data of the decimal. + * + * Returns: A newly created #GArrowDecimal32. + * + * Since: 19.0.0 + */ +GArrowDecimal32 * +garrow_decimal32_new_integer(const gint64 data) +{ + return garrow_decimal_new_integer(data); +} + +/** + * garrow_decimal32_copy: + * @decimal: The decimal to be copied. + * + * Returns: (transfer full): A copied #GArrowDecimal32. + * + * Since: 19.0.0 + */ +GArrowDecimal32 * +garrow_decimal32_copy(GArrowDecimal32 *decimal) +{ + return garrow_decimal_copy(decimal); +} + +/** + * garrow_decimal32_equal: + * @decimal: A #GArrowDecimal32. + * @other_decimal: A #GArrowDecimal32 to be compared. + * + * Returns: %TRUE if the decimal is equal to the other decimal, %FALSE + * otherwise. + * + * Since: 19.0.0 + */ +gboolean +garrow_decimal32_equal(GArrowDecimal32 *decimal, GArrowDecimal32 *other_decimal) +{ + return garrow_decimal_equal(decimal, other_decimal); +} + +/** + * garrow_decimal32_not_equal: + * @decimal: A #GArrowDecimal32. + * @other_decimal: A #GArrowDecimal32 to be compared. + * + * Returns: %TRUE if the decimal isn't equal to the other decimal, + * %FALSE otherwise. + * + * Since: 19.0.0 + */ +gboolean +garrow_decimal32_not_equal(GArrowDecimal32 *decimal, GArrowDecimal32 *other_decimal) +{ + return garrow_decimal_not_equal(decimal, other_decimal); +} + +/** + * garrow_decimal32_less_than: + * @decimal: A #GArrowDecimal32. + * @other_decimal: A #GArrowDecimal32 to be compared. + * + * Returns: %TRUE if the decimal is less than the other decimal, + * %FALSE otherwise. + * + * Since: 19.0.0 + */ +gboolean +garrow_decimal32_less_than(GArrowDecimal32 *decimal, GArrowDecimal32 *other_decimal) +{ + return garrow_decimal_less_than(decimal, other_decimal); +} + +/** + * garrow_decimal32_less_than_or_equal: + * @decimal: A #GArrowDecimal32. + * @other_decimal: A #GArrowDecimal32 to be compared. + * + * Returns: %TRUE if the decimal is less than the other decimal + * or equal to the other decimal, %FALSE otherwise. + * + * Since: 19.0.0 + */ +gboolean +garrow_decimal32_less_than_or_equal(GArrowDecimal32 *decimal, + GArrowDecimal32 *other_decimal) +{ + return garrow_decimal_less_than_or_equal(decimal, other_decimal); +} + +/** + * garrow_decimal32_greater_than: + * @decimal: A #GArrowDecimal32. + * @other_decimal: A #GArrowDecimal32 to be compared. + * + * Returns: %TRUE if the decimal is greater than the other decimal, + * %FALSE otherwise. + * + * Since: 19.0.0 + */ +gboolean +garrow_decimal32_greater_than(GArrowDecimal32 *decimal, GArrowDecimal32 *other_decimal) +{ + return garrow_decimal_greater_than(decimal, other_decimal); +} + +/** + * garrow_decimal32_greater_than_or_equal: + * @decimal: A #GArrowDecimal32. + * @other_decimal: A #GArrowDecimal32 to be compared. + * + * Returns: %TRUE if the decimal is greater than the other decimal + * or equal to the other decimal, %FALSE otherwise. + * + * Since: 19.0.0 + */ +gboolean +garrow_decimal32_greater_than_or_equal(GArrowDecimal32 *decimal, + GArrowDecimal32 *other_decimal) +{ + return garrow_decimal_greater_than_or_equal(decimal, other_decimal); +} + +/** + * garrow_decimal32_to_string_scale: + * @decimal: A #GArrowDecimal32. + * @scale: The scale of the decimal. + * + * Returns: The string representation of the decimal. + * + * It should be freed with g_free() when no longer needed. + * + * Since: 19.0.0 + */ +gchar * +garrow_decimal32_to_string_scale(GArrowDecimal32 *decimal, gint32 scale) +{ + return garrow_decimal_to_string_scale(decimal, scale); +} + +/** + * garrow_decimal32_to_string: + * @decimal: A #GArrowDecimal32. + * + * Returns: The string representation of the decimal. + * + * It should be freed with g_free() when no longer needed. + * + * Since: 19.0.0 + */ +gchar * +garrow_decimal32_to_string(GArrowDecimal32 *decimal) +{ + return garrow_decimal_to_string(decimal); +} + +/** + * garrow_decimal32_to_bytes: + * @decimal: A #GArrowDecimal32. + * + * Returns: (transfer full): The binary representation of the decimal. + * + * Since: 19.0.0 + */ +GBytes * +garrow_decimal32_to_bytes(GArrowDecimal32 *decimal) +{ + return garrow_decimal_to_bytes(decimal); +} + +/** + * garrow_decimal32_abs: + * @decimal: A #GArrowDecimal32. + * + * Computes the absolute value of the @decimal destructively. + * + * Since: 19.0.0 + */ +void +garrow_decimal32_abs(GArrowDecimal32 *decimal) +{ + garrow_decimal_abs(decimal); +} + +/** + * garrow_decimal32_negate: + * @decimal: A #GArrowDecimal32. + * + * Negate the current value of the @decimal destructively. + * + * Since: 19.0.0 + */ +void +garrow_decimal32_negate(GArrowDecimal32 *decimal) +{ + garrow_decimal_negate(decimal); +} + +/** + * garrow_decimal32_to_integer: + * @decimal: A #GArrowDecimal32. + * + * Returns: The 64-bit integer representation of the decimal. + * + * Since: 19.0.0 + */ +gint64 +garrow_decimal32_to_integer(GArrowDecimal32 *decimal) +{ + auto arrow_decimal = garrow_decimal32_get_raw(decimal); + return static_cast(*arrow_decimal); +} + +/** + * garrow_decimal32_plus: + * @left: A #GArrowDecimal32. + * @right: A #GArrowDecimal32. + * + * Returns: (transfer full): The added value of these decimals. + * + * Since: 19.0.0 + */ +GArrowDecimal32 * +garrow_decimal32_plus(GArrowDecimal32 *left, GArrowDecimal32 *right) +{ + return garrow_decimal_plus(left, right); +} + +/** + * garrow_decimal32_minus: + * @left: A #GArrowDecimal32. + * @right: A #GArrowDecimal32. + * + * Returns: (transfer full): The subtracted value of these decimals. + * + * Since: 19.0.0 + */ +GArrowDecimal32 * +garrow_decimal32_minus(GArrowDecimal32 *left, GArrowDecimal32 *right) +{ + return garrow_decimal_minus(left, right); +} + +/** + * garrow_decimal32_multiply: + * @left: A #GArrowDecimal32. + * @right: A #GArrowDecimal32. + * + * Returns: (transfer full): The multiplied value of these decimals. + * + * Since: 19.0.0 + */ +GArrowDecimal32 * +garrow_decimal32_multiply(GArrowDecimal32 *left, GArrowDecimal32 *right) +{ + return garrow_decimal_multiply(left, right); +} + +/** + * garrow_decimal32_divide: + * @left: A #GArrowDecimal32. + * @right: A #GArrowDecimal32. + * @remainder: (out) (nullable): A return location for the remainder + * value of these decimals. The returned #GArrowDecimal32 be + * unreferred with g_object_unref() when no longer needed. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (nullable) (transfer full): The divided value of + * these decimals or %NULL on error. + * + * Since: 19.0.0 + */ +GArrowDecimal32 * +garrow_decimal32_divide(GArrowDecimal32 *left, + GArrowDecimal32 *right, + GArrowDecimal32 **remainder, + GError **error) +{ + return garrow_decimal_divide(left, + right, + remainder, + error, + "[decimal32][divide]"); +} + +/** + * garrow_decimal32_rescale: + * @decimal: A #GArrowDecimal32. + * @original_scale: A scale to be converted from. + * @new_scale: A scale to be converted to. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (nullable) (transfer full): The rescaled decimal or %NULL on error. + * + * Since: 19.0.0 + */ +GArrowDecimal32 * +garrow_decimal32_rescale(GArrowDecimal32 *decimal, + gint32 original_scale, + gint32 new_scale, + GError **error) +{ + return garrow_decimal_rescale(decimal, + original_scale, + new_scale, + error, + "[decimal32][rescale]"); +} typedef struct GArrowDecimal64Private_ { std::shared_ptr decimal64; @@ -1499,6 +1916,21 @@ garrow_decimal256_rescale(GArrowDecimal256 *decimal, G_END_DECLS +GArrowDecimal32 * +garrow_decimal32_new_raw(std::shared_ptr *arrow_decimal32) +{ + auto decimal32 = + g_object_new(garrow_decimal32_get_type(), "decimal32", arrow_decimal32, NULL); + return GARROW_DECIMAL32(decimal32); +} + +std::shared_ptr +garrow_decimal32_get_raw(GArrowDecimal32 *decimal32) +{ + auto priv = GARROW_DECIMAL32_GET_PRIVATE(decimal32); + return priv->decimal32; +} + GArrowDecimal64 * garrow_decimal64_new_raw(std::shared_ptr *arrow_decimal64) { diff --git a/c_glib/arrow-glib/decimal.h b/c_glib/arrow-glib/decimal.h index bb266424b4cc0..6f839a67d9b3b 100644 --- a/c_glib/arrow-glib/decimal.h +++ b/c_glib/arrow-glib/decimal.h @@ -25,6 +25,85 @@ G_BEGIN_DECLS +/* Disabled because it conflicts with GARROW_TYPE_DECIMAL32 in GArrowType. */ +/* #define GARROW_TYPE_DECIMAL32 (garrow_decimal32_get_type()) */ +GARROW_AVAILABLE_IN_19_0 +G_DECLARE_DERIVABLE_TYPE(GArrowDecimal32, garrow_decimal32, GARROW, DECIMAL32, GObject) + +struct _GArrowDecimal32Class +{ + GObjectClass parent_class; +}; + +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal32 * +garrow_decimal32_new_string(const gchar *data, GError **error); +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal32 * +garrow_decimal32_new_integer(const gint64 data); +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal32 * +garrow_decimal32_copy(GArrowDecimal32 *decimal); +GARROW_AVAILABLE_IN_19_0 +gboolean +garrow_decimal32_equal(GArrowDecimal32 *decimal, GArrowDecimal32 *other_decimal); +GARROW_AVAILABLE_IN_19_0 +gboolean +garrow_decimal32_not_equal(GArrowDecimal32 *decimal, GArrowDecimal32 *other_decimal); +GARROW_AVAILABLE_IN_19_0 +gboolean +garrow_decimal32_less_than(GArrowDecimal32 *decimal, GArrowDecimal32 *other_decimal); +GARROW_AVAILABLE_IN_19_0 +gboolean +garrow_decimal32_less_than_or_equal(GArrowDecimal32 *decimal, + GArrowDecimal32 *other_decimal); +GARROW_AVAILABLE_IN_19_0 +gboolean +garrow_decimal32_greater_than(GArrowDecimal32 *decimal, GArrowDecimal32 *other_decimal); +GARROW_AVAILABLE_IN_19_0 +gboolean +garrow_decimal32_greater_than_or_equal(GArrowDecimal32 *decimal, + GArrowDecimal32 *other_decimal); +GARROW_AVAILABLE_IN_19_0 +gchar * +garrow_decimal32_to_string_scale(GArrowDecimal32 *decimal, gint32 scale); +GARROW_AVAILABLE_IN_19_0 +gchar * +garrow_decimal32_to_string(GArrowDecimal32 *decimal); +GARROW_AVAILABLE_IN_19_0 +GBytes * +garrow_decimal32_to_bytes(GArrowDecimal32 *decimal); +GARROW_AVAILABLE_IN_19_0 +void +garrow_decimal32_abs(GArrowDecimal32 *decimal); +GARROW_AVAILABLE_IN_19_0 +void +garrow_decimal32_negate(GArrowDecimal32 *decimal); +GARROW_AVAILABLE_IN_19_0 +gint64 +garrow_decimal32_to_integer(GArrowDecimal32 *decimal); +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal32 * +garrow_decimal32_plus(GArrowDecimal32 *left, GArrowDecimal32 *right); +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal32 * +garrow_decimal32_minus(GArrowDecimal32 *left, GArrowDecimal32 *right); +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal32 * +garrow_decimal32_multiply(GArrowDecimal32 *left, GArrowDecimal32 *right); +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal32 * +garrow_decimal32_divide(GArrowDecimal32 *left, + GArrowDecimal32 *right, + GArrowDecimal32 **remainder, + GError **error); +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal32 * +garrow_decimal32_rescale(GArrowDecimal32 *decimal, + gint32 original_scale, + gint32 new_scale, + GError **error); + /* Disabled because it conflicts with GARROW_TYPE_DECIMAL64 in GArrowType. */ /* #define GARROW_TYPE_DECIMAL64 (garrow_decimal64_get_type()) */ GARROW_AVAILABLE_IN_ALL diff --git a/c_glib/arrow-glib/decimal.hpp b/c_glib/arrow-glib/decimal.hpp index dbfb7f30c60e0..09ac40a51297e 100644 --- a/c_glib/arrow-glib/decimal.hpp +++ b/c_glib/arrow-glib/decimal.hpp @@ -25,6 +25,11 @@ #include +GArrowDecimal32 * +garrow_decimal32_new_raw(std::shared_ptr *arrow_decimal32); +std::shared_ptr +garrow_decimal32_get_raw(GArrowDecimal32 *decimal); + GArrowDecimal64 * garrow_decimal64_new_raw(std::shared_ptr *arrow_decimal64); std::shared_ptr diff --git a/c_glib/test/test-decimal32.rb b/c_glib/test/test-decimal32.rb new file mode 100644 index 0000000000000..33b84ccc6b531 --- /dev/null +++ b/c_glib/test/test-decimal32.rb @@ -0,0 +1,222 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestDecimal32 < Test::Unit::TestCase + def test_new_string_invalid + message = + "[decimal32][new][string]: Invalid: " + + "The string '1,1' is not a valid decimal32 number" + error = assert_raise(Arrow::Error::Invalid) do + Arrow::Decimal32.new("1,1") + end + assert_equal(message, + error.message.lines.first.chomp) + end + + def test_copy + decimal = Arrow::Decimal32.new("234.23445") + assert_equal(decimal, decimal.copy) + end + + def test_to_string_scale + integer_data = 23423445 + string_data = "234.23445" + decimal = Arrow::Decimal32.new(integer_data) + assert_equal(string_data, decimal.to_string_scale(5)) + end + + def test_to_string + string_data = "999999999" + decimal = Arrow::Decimal32.new(string_data) + assert_equal(string_data, decimal.to_s) + end + + def test_to_bytes + decimal = Arrow::Decimal32.new("12.3") + assert_equal("\x7B\x00\x00\x00", + decimal.to_bytes.to_s) + end + + def test_abs + absolute_value = "230492239" + negative_value = "-230492239" + decimal = Arrow::Decimal32.new(negative_value) + decimal.abs + assert_equal(absolute_value, decimal.to_s) + end + + def test_negate + positive_value = "230492239" + negative_value = "-230492239" + decimal = Arrow::Decimal32.new(positive_value) + decimal.negate + assert_equal(negative_value, decimal.to_s) + decimal.negate + assert_equal(positive_value, decimal.to_s) + end + + def test_plus + integer_data1 = 23423445 + integer_data2 = 5443 + decimal1 = Arrow::Decimal32.new(integer_data1) + decimal2 = Arrow::Decimal32.new(integer_data2) + decimal3 = decimal1.plus(decimal2) + assert_equal((integer_data1 + integer_data2).to_s, + decimal3.to_s) + end + + def test_multiply + integer_data1 = 23423 + integer_data2 = 5443 + decimal1 = Arrow::Decimal32.new(integer_data1) + decimal2 = Arrow::Decimal32.new(integer_data2) + decimal3 = decimal1.multiply(decimal2) + assert_equal((integer_data1 * integer_data2).to_s, + decimal3.to_s) + end + + def test_divide + integer_data1 = 23423 + integer_data2 = -5443 + decimal1 = Arrow::Decimal32.new(integer_data1) + decimal2 = Arrow::Decimal32.new(integer_data2) + result, remainder = decimal1.divide(decimal2) + assert_equal([ + integer_data1.quo(integer_data2).truncate.to_s, + integer_data1.remainder(integer_data2).to_s, + ], + [result.to_s, remainder.to_s]) + end + + def test_divide_zero + decimal1 = Arrow::Decimal32.new(23423445) + decimal2 = Arrow::Decimal32.new(0) + message = + "[decimal32][divide]: Invalid: Division by 0 in Decimal32" + assert_raise(Arrow::Error::Invalid.new(message)) do + decimal1.divide(decimal2) + end + end + + def test_equal + decimal = Arrow::Decimal32.new(10) + other_decimal1 = Arrow::Decimal32.new(10) + other_decimal2 = Arrow::Decimal32.new(11) + assert_equal([ + true, + false, + ], + [ + decimal == other_decimal1, + decimal == other_decimal2, + ]) + end + + def test_not_equal + decimal = Arrow::Decimal32.new(10) + other_decimal1 = Arrow::Decimal32.new(10) + other_decimal2 = Arrow::Decimal32.new(11) + assert_equal([ + false, + true, + ], + [ + decimal != other_decimal1, + decimal != other_decimal2, + ]) + end + + def test_less_than + decimal = Arrow::Decimal32.new(10) + other_decimal1 = Arrow::Decimal32.new(11) + other_decimal2 = Arrow::Decimal32.new(9) + assert_equal([ + true, + false, + false + ], + [ + decimal < other_decimal1, + decimal < other_decimal2, + decimal < decimal, + ]) + end + + def test_less_than_or_equal + decimal = Arrow::Decimal32.new(10) + other_decimal1 = Arrow::Decimal32.new(11) + other_decimal2 = Arrow::Decimal32.new(9) + assert_equal([ + true, + false, + true + ], + [ + decimal <= other_decimal1, + decimal <= other_decimal2, + decimal <= decimal + ]) + end + + def test_greater_than + decimal = Arrow::Decimal32.new(10) + other_decimal1 = Arrow::Decimal32.new(11) + other_decimal2 = Arrow::Decimal32.new(9) + assert_equal([ + false, + true, + false + ], + [ + decimal > other_decimal1, + decimal > other_decimal2, + decimal > decimal + ]) + end + + def test_greater_than_or_equal + decimal = Arrow::Decimal32.new(10) + other_decimal1 = Arrow::Decimal32.new(11) + other_decimal2 = Arrow::Decimal32.new(9) + assert_equal([ + false, + true, + true + ], + [ + decimal >= other_decimal1, + decimal >= other_decimal2, + decimal >= decimal + ]) + end + + def test_rescale + decimal = Arrow::Decimal32.new(10) + assert_equal(Arrow::Decimal32.new(1000), + decimal.rescale(1, 3)) + end + + def test_rescale_fail + decimal = Arrow::Decimal32.new(10) + message = + "[decimal32][rescale]: Invalid: " + + "Rescaling Decimal32 value would cause data loss" + assert_raise(Arrow::Error::Invalid.new(message)) do + decimal.rescale(1, -1) + end + end +end From 191472d41714afbd9a2b0b3eb5e25be79ee6fdda Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 1 Nov 2024 06:16:56 +0900 Subject: [PATCH 02/29] GH-44570: [Release][R][Docs] Update `r/pkgdown/assets/versions.html` (#44572) ### Rationale for this change It's still used at https://arrow.apache.org/docs/r/versions.html . So we should update it too. ### What changes are included in this PR? Commit it. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * GitHub Issue: #44570 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- dev/release/01-prepare-test.rb | 23 +++++++++++++++++++++++ dev/release/post-12-bump-versions-test.rb | 23 +++++++++++++++++++++++ dev/release/utils-prepare.sh | 1 + r/pkgdown/assets/versions.html | 13 ++++++++++--- r/pkgdown/assets/versions.json | 4 ++++ 5 files changed, 61 insertions(+), 3 deletions(-) diff --git a/dev/release/01-prepare-test.rb b/dev/release/01-prepare-test.rb index ca53b7f8fdee5..27f9f5f869ea3 100644 --- a/dev/release/01-prepare-test.rb +++ b/dev/release/01-prepare-test.rb @@ -263,6 +263,18 @@ def test_version_pre_tag ] if next_release_type == :major expected_changes += [ + { + path: "r/pkgdown/assets/versions.html", + hunks: [ + [ + "-

#{@previous_version}.9000 (dev)

", + "-

#{@previous_r_version} (release)

", + "+

#{@release_version}.9000 (dev)

", + "+

#{@release_version} (release)

", + "+

#{@previous_r_version}

", + ] + ], + }, { path: "r/pkgdown/assets/versions.json", hunks: [ @@ -281,6 +293,17 @@ def test_version_pre_tag ] else expected_changes += [ + { + path: "r/pkgdown/assets/versions.html", + hunks: [ + [ + "-

#{@previous_version}.9000 (dev)

", + "-

#{@previous_r_version} (release)

", + "+

#{@release_version}.9000 (dev)

", + "+

#{@release_version} (release)

", + ] + ], + }, { path: "r/pkgdown/assets/versions.json", hunks: [ diff --git a/dev/release/post-12-bump-versions-test.rb b/dev/release/post-12-bump-versions-test.rb index 9af334c496fe6..9c4026584aeca 100644 --- a/dev/release/post-12-bump-versions-test.rb +++ b/dev/release/post-12-bump-versions-test.rb @@ -210,6 +210,18 @@ def test_version_post_tag ["+ (#{@next_major_version}, 0),"], ], }, + { + path: "r/pkgdown/assets/versions.html", + hunks: [ + [ + "-

#{@previous_version}.9000 (dev)

", + "-

#{@previous_r_version} (release)

", + "+

#{@release_version}.9000 (dev)

", + "+

#{@release_version} (release)

", + "+

#{@previous_r_version}

", + ], + ], + }, { path: "r/pkgdown/assets/versions.json", hunks: [ @@ -228,6 +240,17 @@ def test_version_post_tag ] else expected_changes += [ + { + path: "r/pkgdown/assets/versions.html", + hunks: [ + [ + "-

#{@previous_version}.9000 (dev)

", + "-

#{@previous_r_version} (release)

", + "+

#{@release_version}.9000 (dev)

", + "+

#{@release_version} (release)

", + ], + ], + }, { path: "r/pkgdown/assets/versions.json", hunks: [ diff --git a/dev/release/utils-prepare.sh b/dev/release/utils-prepare.sh index ecdd0a26dcb7a..19ffda578b4b0 100644 --- a/dev/release/utils-prepare.sh +++ b/dev/release/utils-prepare.sh @@ -194,6 +194,7 @@ update_versions() { "${base_version}" \ "${next_version}" git add docs/source/_static/versions.json + git add r/pkgdown/assets/versions.html git add r/pkgdown/assets/versions.json popd } diff --git a/r/pkgdown/assets/versions.html b/r/pkgdown/assets/versions.html index 8ba513a98c85b..e2f56772cf339 100644 --- a/r/pkgdown/assets/versions.html +++ b/r/pkgdown/assets/versions.html @@ -1,9 +1,16 @@ -

13.0.0.9000 (dev)

-

13.0.0.1 (release)

+

18.0.0.9000 (dev)

+

18.0.0 (release)

+

17.0.0

+

16.1.0

+

16.0.0

+

15.0.2

+

14.0.2

+

13.0.0.1

12.0.1.1

-

11.0.0.3

+

12.0.0

+

11.0.0

10.0.1

9.0.0

8.0.0

diff --git a/r/pkgdown/assets/versions.json b/r/pkgdown/assets/versions.json index cecbed7f32818..41c8469772f9d 100644 --- a/r/pkgdown/assets/versions.json +++ b/r/pkgdown/assets/versions.json @@ -15,6 +15,10 @@ "name": "16.1.0", "version": "16.1/" }, + { + "name": "16.0.0", + "version": "16.0/" + }, { "name": "15.0.2", "version": "15.0/" From e1fa7e5fb257fa9f83ffcc531efda8a4489b2961 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 1 Nov 2024 08:47:17 +0900 Subject: [PATCH 03/29] GH-44590: [C++] Add `const` and `&` to `arrow::Array::statistics()` return type (#44592) ### Rationale for this change It must be immutable. ### What changes are included in this PR? Add missing `const` and `&`. ### Are these changes tested? Yes. ### Are there any user-facing changes? Yes. **This PR includes breaking changes to public APIs.** * GitHub Issue: #44590 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- cpp/src/arrow/array/array_base.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/array/array_base.h b/cpp/src/arrow/array/array_base.h index e4af67d7e5f0b..21faa3f4279ea 100644 --- a/cpp/src/arrow/array/array_base.h +++ b/cpp/src/arrow/array/array_base.h @@ -237,8 +237,8 @@ class ARROW_EXPORT Array { /// This just delegates to calling statistics on the underlying ArrayData /// object which backs this Array. /// - /// \return const ArrayStatistics& - std::shared_ptr statistics() const { return data_->statistics; } + /// \return const std::shared_ptr& + const std::shared_ptr& statistics() const { return data_->statistics; } protected: Array() = default; From f3abc6802a94a1a4202c710c236a24a137e0a0d7 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Fri, 1 Nov 2024 07:58:33 -0700 Subject: [PATCH 04/29] GH-43547: [R][CI] Add recheck workflow for checking reverse dependencies on GHA (#43784) ### Rationale for this change See https://github.com/apache/arrow/issues/43547. ### What changes are included in this PR? Adds two new new crossbow tasks for performing reverse dependency checking using https://github.com/r-devel/recheck: - `r-recheck-most` - `r-recheck-strong` ### Are these changes tested? Yes. https://github.com/apache/arrow/pull/44523#issuecomment-2434122461. ### Are there any user-facing changes? No. * GitHub Issue: #43547 Fixes https://github.com/apache/arrow/issues/43547. Authored-by: Bryce Mecum Signed-off-by: Bryce Mecum --- dev/tasks/r/github.recheck.yml | 30 ++++++++++++++++++++++++++++++ dev/tasks/tasks.yml | 15 +++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 dev/tasks/r/github.recheck.yml diff --git a/dev/tasks/r/github.recheck.yml b/dev/tasks/r/github.recheck.yml new file mode 100644 index 0000000000000..5f0095fa22126 --- /dev/null +++ b/dev/tasks/r/github.recheck.yml @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + recheck: + name: Reverse check {{ which }} dependents + uses: r-devel/recheck/.github/workflows/recheck.yml@9fe04de60ebeadd505b8d76223a346617ccca836 + with: + which: {{ which }} + subdirectory: r + repository: {{ arrow.github_repo }} + ref: {{ arrow.branch }} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 30c1daecf7a31..8f542265fd02d 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -94,6 +94,7 @@ groups: r: - test*-r-* - r-binary-packages + - r-recheck-most ruby: - test-*ruby* @@ -901,6 +902,20 @@ tasks: - r-pkg__bin__macosx__big-sur-arm64__contrib__4.3__arrow_{no_rc_r_version}\.tgz - r-pkg__src__contrib__arrow_{no_rc_r_version}\.tar\.gz +{% for which in ["strong", "most"] %} + # strong and most used here are defined by ?tools::package_dependencies as: + # + # strong: Depends, Imports, LinkingTo + # most: Depends, Imports, LinkingTo, Suggests + # + # So the key difference between strong and most is whether you want to expand + # the reverse dependency checking to Suggests (most) or not. + r-recheck-{{which}}: + ci: github + template: r/github.recheck.yml + params: + which: {{which}} +{% endfor %} ########################### Release verification ############################ From 277df0ec4bdc0e9bc4557ce40d38fcf921396683 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sat, 2 Nov 2024 07:04:15 +0900 Subject: [PATCH 05/29] GH-44601: [GLib] Fix the wrong GARROW_AVAILABLE_IN declaration (#44602) ### Rationale for this change GArrowDecimal64 will be introduced in the 19.0.0 release. This part should be `GARROW_AVAILABLE_IN_19_0` instead of `GARROW_AVAILABLE_IN_ALL` ### What changes are included in this PR? Change the `GARROW_AVAILABLE_IN` declaration. ### Are these changes tested? YES ### Are there any user-facing changes? NO * GitHub Issue: #44601 Authored-by: Hiroyuki Sato Signed-off-by: Sutou Kouhei --- c_glib/arrow-glib/decimal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c_glib/arrow-glib/decimal.h b/c_glib/arrow-glib/decimal.h index 6f839a67d9b3b..e034f1bbdc84c 100644 --- a/c_glib/arrow-glib/decimal.h +++ b/c_glib/arrow-glib/decimal.h @@ -106,7 +106,7 @@ garrow_decimal32_rescale(GArrowDecimal32 *decimal, /* Disabled because it conflicts with GARROW_TYPE_DECIMAL64 in GArrowType. */ /* #define GARROW_TYPE_DECIMAL64 (garrow_decimal64_get_type()) */ -GARROW_AVAILABLE_IN_ALL +GARROW_AVAILABLE_IN_19_0 G_DECLARE_DERIVABLE_TYPE(GArrowDecimal64, garrow_decimal64, GARROW, DECIMAL64, GObject) struct _GArrowDecimal64Class From 11c11a48234a7f49e0585f5762b3a6332ac7622a Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sat, 2 Nov 2024 07:07:47 +0900 Subject: [PATCH 06/29] GH-44603: [GLib] Add GArrowDecimal64Array and GArrowDecimal64ArrayBuilder (#44605) ### Rationale for this change The `arrow::Decimal64Array` has been released. The `GArrowDecimal64Array` class must be implemented in the GLib. ### What changes are included in this PR? * Implement `GArrowDecimal64Array` * Implement `GArrowDecimal64ArrayBuilder` ### Are these changes tested? YES ### Are there any user-facing changes? NO * GitHub Issue: #44603 Authored-by: Hiroyuki Sato Signed-off-by: Sutou Kouhei --- c_glib/arrow-glib/array-builder.cpp | 105 ++++++++++++++++++++++++++++ c_glib/arrow-glib/array-builder.h | 30 ++++++++ c_glib/arrow-glib/basic-array.cpp | 62 ++++++++++++++++ c_glib/arrow-glib/basic-array.h | 20 ++++++ c_glib/test/test-decimal64-array.rb | 37 ++++++++++ 5 files changed, 254 insertions(+) create mode 100644 c_glib/test/test-decimal64-array.rb diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp index 1897562e13286..4f82ee2983b34 100644 --- a/c_glib/arrow-glib/array-builder.cpp +++ b/c_glib/arrow-glib/array-builder.cpp @@ -457,6 +457,9 @@ G_BEGIN_DECLS * #GArrowMapArrayBuilder is the class to create a new * #GArrowMapArray. * + * #GArrowDecimal64ArrayBuilder is the class to create a new + * #GArrowDecimal64Array. + * * #GArrowDecimal128ArrayBuilder is the class to create a new * #GArrowDecimal128Array. * @@ -6062,6 +6065,105 @@ garrow_map_array_builder_get_value_builder(GArrowMapArrayBuilder *builder) return priv->value_builder; } +G_DEFINE_TYPE(GArrowDecimal64ArrayBuilder, + garrow_decimal64_array_builder, + GARROW_TYPE_FIXED_SIZE_BINARY_ARRAY_BUILDER) + +static void +garrow_decimal64_array_builder_init(GArrowDecimal64ArrayBuilder *builder) +{ +} + +static void +garrow_decimal64_array_builder_class_init(GArrowDecimal64ArrayBuilderClass *klass) +{ +} + +/** + * garrow_decimal64_array_builder_new: + * @data_type: #GArrowDecimal64DataType for the decimal. + * + * Returns: A newly created #GArrowDecimal64ArrayBuilder. + * + * Since: 19.0.0 + */ +GArrowDecimal64ArrayBuilder * +garrow_decimal64_array_builder_new(GArrowDecimal64DataType *data_type) +{ + auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type)); + auto builder = + garrow_array_builder_new(arrow_data_type, NULL, "[decimal64-array-builder][new]"); + return GARROW_DECIMAL64_ARRAY_BUILDER(builder); +} + +/** + * garrow_decimal64_array_builder_append_value: + * @builder: A #GArrowDecimal64ArrayBuilder. + * @value: (nullable): A decimal value. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: %TRUE on success, %FALSE if there was an error. + * + * Since: 19.0.0 + */ +gboolean +garrow_decimal64_array_builder_append_value(GArrowDecimal64ArrayBuilder *builder, + GArrowDecimal64 *value, + GError **error) +{ + if (value) { + auto arrow_decimal = garrow_decimal64_get_raw(value); + return garrow_array_builder_append_value( + GARROW_ARRAY_BUILDER(builder), + *arrow_decimal, + error, + "[decimal64-array-builder][append-value]"); + } else { + return garrow_array_builder_append_null(GARROW_ARRAY_BUILDER(builder), error); + } +} + +/** + * garrow_decimal64_array_builder_append_values: + * @builder: A #GArrowDecimal64ArrayBuilder. + * @values: (array length=values_length): The array of #GArrowDecimal64. + * @values_length: The length of @values. + * @is_valids: (nullable) (array length=is_valids_length): The array of + * boolean that shows whether the Nth value is valid or not. If the + * Nth @is_valids is %TRUE, the Nth @values is valid value. Otherwise + * the Nth value is null value. + * @is_valids_length: The length of @is_valids. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Append multiple values at once. It's more efficient than multiple + * `append` and `append_null` calls. + * + * Returns: %TRUE on success, %FALSE if there was an error. + * + * Since: 19.0.0 + */ +gboolean +garrow_decimal64_array_builder_append_values(GArrowDecimal64ArrayBuilder *builder, + GArrowDecimal64 **values, + gint64 values_length, + const gboolean *is_valids, + gint64 is_valids_length, + GError **error) +{ + return garrow_array_builder_append_values( + GARROW_ARRAY_BUILDER(builder), + values, + values_length, + is_valids, + is_valids_length, + error, + "[decimal64-array-builder][append-values]", + [](guint8 *output, GArrowDecimal64 *value, gsize size) { + auto arrow_decimal = garrow_decimal64_get_raw(value); + arrow_decimal->ToBytes(output); + }); +} + G_DEFINE_TYPE(GArrowDecimal128ArrayBuilder, garrow_decimal128_array_builder, GARROW_TYPE_FIXED_SIZE_BINARY_ARRAY_BUILDER) @@ -6581,6 +6683,9 @@ garrow_array_builder_new_raw(std::shared_ptr *arrow_builder case arrow::Type::type::MAP: type = GARROW_TYPE_MAP_ARRAY_BUILDER; break; + case arrow::Type::type::DECIMAL64: + type = GARROW_TYPE_DECIMAL64_ARRAY_BUILDER; + break; case arrow::Type::type::DECIMAL128: type = GARROW_TYPE_DECIMAL128_ARRAY_BUILDER; break; diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h index da9e8748ee387..f653ddb7781a3 100644 --- a/c_glib/arrow-glib/array-builder.h +++ b/c_glib/arrow-glib/array-builder.h @@ -1729,6 +1729,36 @@ GARROW_AVAILABLE_IN_0_17 GArrowArrayBuilder * garrow_map_array_builder_get_value_builder(GArrowMapArrayBuilder *builder); +#define GARROW_TYPE_DECIMAL64_ARRAY_BUILDER (garrow_decimal64_array_builder_get_type()) +GARROW_AVAILABLE_IN_19_0 +G_DECLARE_DERIVABLE_TYPE(GArrowDecimal64ArrayBuilder, + garrow_decimal64_array_builder, + GARROW, + DECIMAL64_ARRAY_BUILDER, + GArrowFixedSizeBinaryArrayBuilder) +struct _GArrowDecimal64ArrayBuilderClass +{ + GArrowFixedSizeBinaryArrayBuilderClass parent_class; +}; + +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal64ArrayBuilder * +garrow_decimal64_array_builder_new(GArrowDecimal64DataType *data_type); + +GARROW_AVAILABLE_IN_19_0 +gboolean +garrow_decimal64_array_builder_append_value(GArrowDecimal64ArrayBuilder *builder, + GArrowDecimal64 *value, + GError **error); +GARROW_AVAILABLE_IN_19_0 +gboolean +garrow_decimal64_array_builder_append_values(GArrowDecimal64ArrayBuilder *builder, + GArrowDecimal64 **values, + gint64 values_length, + const gboolean *is_valids, + gint64 is_valids_length, + GError **error); + #define GARROW_TYPE_DECIMAL128_ARRAY_BUILDER (garrow_decimal128_array_builder_get_type()) GARROW_AVAILABLE_IN_ALL G_DECLARE_DERIVABLE_TYPE(GArrowDecimal128ArrayBuilder, diff --git a/c_glib/arrow-glib/basic-array.cpp b/c_glib/arrow-glib/basic-array.cpp index f102a252467a3..8c39715c384f0 100644 --- a/c_glib/arrow-glib/basic-array.cpp +++ b/c_glib/arrow-glib/basic-array.cpp @@ -171,6 +171,11 @@ G_BEGIN_DECLS * have Arrow format data, you need to use #GArrowMonthDayNanoIntervalArray * to create a new array. * + * #GArrowDecimal64Array is a class for 64-bit decimal array. It can + * store zero or more 64-bit decimal data. If you don't have Arrow + * format data, you need to use #GArrowDecimal64ArrayBuilder to + * create a new array. + * * #GArrowDecimal128Array is a class for 128-bit decimal array. It can * store zero or more 128-bit decimal data. If you don't have Arrow * format data, you need to use #GArrowDecimal128ArrayBuilder to @@ -3090,6 +3095,60 @@ garrow_fixed_size_binary_array_get_values_bytes(GArrowFixedSizeBinaryArray *arra arrow_binary_array->byte_width() * arrow_array->length()); } +G_DEFINE_TYPE(GArrowDecimal64Array, + garrow_decimal64_array, + GARROW_TYPE_FIXED_SIZE_BINARY_ARRAY) +static void +garrow_decimal64_array_init(GArrowDecimal64Array *object) +{ +} + +static void +garrow_decimal64_array_class_init(GArrowDecimal64ArrayClass *klass) +{ +} + +/** + * garrow_decimal64_array_format_value: + * @array: A #GArrowDecimal64Array. + * @i: The index of the target value. + * + * Returns: (transfer full): The formatted @i-th value. + * + * It should be freed with g_free() when no longer needed. + * + * Since: 19.0.0 + */ +gchar * +garrow_decimal64_array_format_value(GArrowDecimal64Array *array, gint64 i) +{ + auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array)); + auto arrow_decimal64_array = + std::static_pointer_cast(arrow_array); + auto value = arrow_decimal64_array->FormatValue(i); + return g_strndup(value.data(), value.size()); +} + +/** + * garrow_decimal64_array_get_value: + * @array: A #GArrowDecimal64Array. + * @i: The index of the target value. + * + * Returns: (transfer full): The @i-th value. + * + * Since: 19.0.0 + */ +GArrowDecimal64 * +garrow_decimal64_array_get_value(GArrowDecimal64Array *array, gint64 i) +{ + auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array)); + auto arrow_decimal64_array = + std::static_pointer_cast(arrow_array); + auto arrow_decimal64 = + std::make_shared(arrow_decimal64_array->GetValue(i)); + return garrow_decimal64_new_raw(&arrow_decimal64); +} + G_DEFINE_TYPE(GArrowDecimal128Array, garrow_decimal128_array, GARROW_TYPE_FIXED_SIZE_BINARY_ARRAY) @@ -3443,6 +3502,9 @@ garrow_array_new_raw_valist(std::shared_ptr *arrow_array, case arrow::Type::type::DICTIONARY: type = GARROW_TYPE_DICTIONARY_ARRAY; break; + case arrow::Type::type::DECIMAL64: + type = GARROW_TYPE_DECIMAL64_ARRAY; + break; case arrow::Type::type::DECIMAL128: type = GARROW_TYPE_DECIMAL128_ARRAY; break; diff --git a/c_glib/arrow-glib/basic-array.h b/c_glib/arrow-glib/basic-array.h index 95679aa37c57a..f70cf114a4a96 100644 --- a/c_glib/arrow-glib/basic-array.h +++ b/c_glib/arrow-glib/basic-array.h @@ -810,6 +810,26 @@ GARROW_AVAILABLE_IN_3_0 GBytes * garrow_fixed_size_binary_array_get_values_bytes(GArrowFixedSizeBinaryArray *array); +#define GARROW_TYPE_DECIMAL64_ARRAY (garrow_decimal64_array_get_type()) +GARROW_AVAILABLE_IN_19_0 +G_DECLARE_DERIVABLE_TYPE(GArrowDecimal64Array, + garrow_decimal64_array, + GARROW, + DECIMAL64_ARRAY, + GArrowFixedSizeBinaryArray) +struct _GArrowDecimal64ArrayClass +{ + GArrowFixedSizeBinaryArrayClass parent_class; +}; + +GARROW_AVAILABLE_IN_19_0 +gchar * +garrow_decimal64_array_format_value(GArrowDecimal64Array *array, gint64 i); + +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal64 * +garrow_decimal64_array_get_value(GArrowDecimal64Array *array, gint64 i); + #define GARROW_TYPE_DECIMAL128_ARRAY (garrow_decimal128_array_get_type()) GARROW_AVAILABLE_IN_ALL G_DECLARE_DERIVABLE_TYPE(GArrowDecimal128Array, diff --git a/c_glib/test/test-decimal64-array.rb b/c_glib/test/test-decimal64-array.rb new file mode 100644 index 0000000000000..ab7b9e2523481 --- /dev/null +++ b/c_glib/test/test-decimal64-array.rb @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestDecimal64Array < Test::Unit::TestCase + def test_format_value + data_type = Arrow::Decimal64DataType.new(8, 2) + builder = Arrow::Decimal64ArrayBuilder.new(data_type) + decimal = Arrow::Decimal64.new("23423445") + builder.append_value(decimal) + array = builder.finish + assert_equal("234234.45", array.format_value(0)) + end + + def test_value + data_type = Arrow::Decimal64DataType.new(8, 2) + builder = Arrow::Decimal64ArrayBuilder.new(data_type) + decimal = Arrow::Decimal64.new("23423445") + builder.append_value(decimal) + array = builder.finish + assert_equal("234234.45", + array.get_value(0).to_string_scale(array.value_data_type.scale)) + end +end From ab0c857695420f94cc83259494941a68b2762dee Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 3 Nov 2024 00:31:06 +0900 Subject: [PATCH 07/29] GH-44578: [Release][Packaging] Verify wheel version (#44593) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change We want to detect binary build from wrong source. ### What changes are included in this PR? Add version check. If we use wrong source, binary's version is `X.Y.Z-SNAPSHOT` not `X.Y.Z`. So the added check is failed. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * GitHub Issue: #44578 Authored-by: Sutou Kouhei Signed-off-by: Raúl Cumplido --- ci/scripts/python_wheel_unix_test.sh | 7 +++++++ dev/release/verify-release-candidate.sh | 27 ++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/ci/scripts/python_wheel_unix_test.sh b/ci/scripts/python_wheel_unix_test.sh index 1487581eaef51..3ce86b16116b0 100755 --- a/ci/scripts/python_wheel_unix_test.sh +++ b/ci/scripts/python_wheel_unix_test.sh @@ -88,6 +88,13 @@ import pyarrow.parquet fi fi +if [ "${CHECK_VERSION}" == "ON" ]; then + pyarrow_version=$(python -c "import pyarrow; print(pyarrow.__version__)") + [ "${pyarrow_version}" = "${ARROW_VERSION}" ] + arrow_cpp_version=$(python -c "import pyarrow; print(pyarrow.cpp_build_info.version)") + [ "${arrow_cpp_version}" = "${ARROW_VERSION}" ] +fi + if [ "${CHECK_WHEEL_CONTENT}" == "ON" ]; then python ${source_dir}/ci/scripts/python_wheel_validate_contents.py \ --path ${source_dir}/python/repaired_wheels diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index d9f973562aa78..17d10601d11d7 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -1052,6 +1052,12 @@ test_linux_wheels() { local wheel_content="OFF" fi + if [ "${SOURCE_KIND}" = "tarball" ]; then + local check_version="ON" + else + local check_version="OFF" + fi + for python in ${python_versions}; do local pyver=${python/m} for platform in ${platform_tags}; do @@ -1061,7 +1067,11 @@ test_linux_wheels() { continue fi pip install pyarrow-${TEST_PYARROW_VERSION:-${VERSION}}-cp${pyver/.}-cp${python/.}-${platform}.whl - CHECK_WHEEL_CONTENT=${wheel_content:-"ON"} INSTALL_PYARROW=OFF ARROW_GCS=${check_gcs} \ + ARROW_GCS=${check_gcs} \ + ARROW_VERSION=${VERSION} \ + CHECK_VERSION=${check_version} \ + CHECK_WHEEL_CONTENT=${wheel_content:-"ON"} \ + INSTALL_PYARROW=OFF \ ${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_SOURCE_DIR} done done @@ -1086,6 +1096,12 @@ test_macos_wheels() { local wheel_content="OFF" fi + if [ "${SOURCE_KIND}" = "tarball" ]; then + local check_version="ON" + else + local check_version="OFF" + fi + # verify arch-native wheels inside an arch-native conda environment for python in ${python_versions}; do local pyver=${python/m} @@ -1102,8 +1118,13 @@ test_macos_wheels() { fi pip install pyarrow-${VERSION}-cp${pyver/.}-cp${python/.}-${platform}.whl - CHECK_WHEEL_CONTENT=${wheel_content:-"ON"} INSTALL_PYARROW=OFF ARROW_FLIGHT=${check_flight} \ - ARROW_GCS=${check_gcs} ARROW_S3=${check_s3} \ + ARROW_FLIGHT=${check_flight} \ + ARROW_GCS=${check_gcs} \ + ARROW_S3=${check_s3} \ + ARROW_VERSION=${VERSION} \ + CHECK_WHEEL_CONTENT=${wheel_content:-"ON"} \ + CHECK_VERSION=${check_version} \ + INSTALL_PYARROW=OFF \ ${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_SOURCE_DIR} done done From e76082dc9b5d704b4f5f438c8e8b3bf68a361894 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 3 Nov 2024 06:26:28 +0900 Subject: [PATCH 08/29] MINOR: [JS] Bump memfs from 4.9.2 to 4.14.0 in /js (#44609) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [memfs](https://github.com/streamich/memfs) from 4.9.2 to 4.14.0.
Release notes

Sourced from memfs's releases.

v4.14.0

4.14.0 (2024-10-13)

Features

  • support stream as source in promises version of writeFile (#1069) (11f8a36)

v4.13.0

4.13.0 (2024-10-07)

Features

v4.12.0

4.12.0 (2024-09-19)

Features

v4.11.2

4.11.2 (2024-09-17)

Bug Fixes

v4.11.1

4.11.1 (2024-08-01)

Bug Fixes

v4.11.0

4.11.0 (2024-07-27)

Features

  • volume implementation of .opendir() method (7072fb7)

v4.10.0

4.10.0 (2024-07-27)

... (truncated)

Changelog

Sourced from memfs's changelog.

4.14.0 (2024-10-13)

Features

  • support stream as source in promises version of writeFile (#1069) (11f8a36)

4.13.0 (2024-10-07)

Features

4.12.0 (2024-09-19)

Features

4.11.2 (2024-09-17)

Bug Fixes

4.11.1 (2024-08-01)

Bug Fixes

4.11.0 (2024-07-27)

Features

  • volume implementation of .opendir() method (7072fb7)

4.10.0 (2024-07-27)

Features

  • 🎸 add IReadableWebStreamOptions type (99ebd64)
  • 🎸 implement FileHandle.readableWebStream() (c3ddc6c)

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=memfs&package-manager=npm_and_yarn&previous-version=4.9.2&new-version=4.14.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- js/package.json | 2 +- js/yarn.lock | 35 ++++++++++++++--------------------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/js/package.json b/js/package.json index a879814426ed7..2a0db0067bd5b 100644 --- a/js/package.json +++ b/js/package.json @@ -98,7 +98,7 @@ "ix": "7.0.0", "jest": "29.7.0", "jest-silent-reporter": "0.6.0", - "memfs": "4.9.2", + "memfs": "4.14.0", "mkdirp": "3.0.1", "multistream": "4.1.0", "regenerator-runtime": "0.14.1", diff --git a/js/yarn.lock b/js/yarn.lock index e237d09469f4d..2941ccf3c569d 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -948,10 +948,10 @@ hyperdyperid "^1.2.0" thingies "^1.20.0" -"@jsonjoy.com/util@^1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@jsonjoy.com/util/-/util-1.1.2.tgz#5072c27ecdb16d1ed7a2d125a1d0ed8aba01d652" - integrity sha512-HOGa9wtE6LEz2I5mMQ2pMSjth85PmD71kPbsecs02nEUq3/Kw0wRK3gmZn5BCEB8mFLXByqPxjHgApoMwIPMKQ== +"@jsonjoy.com/util@^1.1.2", "@jsonjoy.com/util@^1.3.0": + version "1.5.0" + resolved "https://registry.yarnpkg.com/@jsonjoy.com/util/-/util-1.5.0.tgz#6008e35b9d9d8ee27bc4bfaa70c8cbf33a537b4c" + integrity sha512-ojoNsrIuPI9g6o8UxhraZQSyF2ByJanAY4cTFbc8Mf2AXEF4aQRGY1dJxyJpuyav8r9FGflEt/Ff3u5Nt6YMPA== "@nodelib/fs.scandir@2.1.5": version "2.1.5" @@ -5168,14 +5168,14 @@ mdurl@^2.0.0: resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-2.0.0.tgz#80676ec0433025dd3e17ee983d0fe8de5a2237e0" integrity sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w== -memfs@4.9.2: - version "4.9.2" - resolved "https://registry.yarnpkg.com/memfs/-/memfs-4.9.2.tgz#42e7b48207268dad8c9c48ea5d4952c5d3840433" - integrity sha512-f16coDZlTG1jskq3mxarwB+fGRrd0uXWt+o1WIhRfOwbXQZqUDsTVxQBFK9JjRQHblg8eAG2JSbprDXKjc7ijQ== +memfs@4.14.0: + version "4.14.0" + resolved "https://registry.yarnpkg.com/memfs/-/memfs-4.14.0.tgz#48d5e85a03ea0b428280003212fbca3063531be3" + integrity sha512-JUeY0F/fQZgIod31Ja1eJgiSxLn7BfQlCnqhwXFBzFHEw63OdLK7VJUJ7bnzNsWgCyoUP5tEp1VRY8rDaYzqOA== dependencies: "@jsonjoy.com/json-pack" "^1.0.3" - "@jsonjoy.com/util" "^1.1.2" - sonic-forest "^1.0.0" + "@jsonjoy.com/util" "^1.3.0" + tree-dump "^1.0.1" tslib "^2.0.0" memoizee@0.4.X: @@ -6397,13 +6397,6 @@ snapdragon@^0.8.1: source-map-resolve "^0.5.0" use "^3.1.0" -sonic-forest@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/sonic-forest/-/sonic-forest-1.0.2.tgz#d80aa621d1cffe75a606ca44789ccff30f5b9ce6" - integrity sha512-2rICdwIJi5kVlehMUVtJeHn3ohh5YZV4pDv0P0c1M11cRz/gXNViItpM94HQwfvnXuzybpqK0LZJgTa3lEwtAw== - dependencies: - tree-dump "^1.0.0" - source-map-resolve@^0.5.0: version "0.5.3" resolved "https://registry.yarnpkg.com/source-map-resolve/-/source-map-resolve-0.5.3.tgz#190866bece7553e1f8f267a2ee82c606b5509a1a" @@ -6879,10 +6872,10 @@ totalist@^3.0.0: resolved "https://registry.yarnpkg.com/totalist/-/totalist-3.0.1.tgz#ba3a3d600c915b1a97872348f79c127475f6acf8" integrity sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ== -tree-dump@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/tree-dump/-/tree-dump-1.0.1.tgz#b448758da7495580e6b7830d6b7834fca4c45b96" - integrity sha512-WCkcRBVPSlHHq1dc/px9iOfqklvzCbdRwvlNfxGZsrHqf6aZttfPrd7DJTt6oR10dwUfpFFQeVTkPbBIZxX/YA== +tree-dump@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/tree-dump/-/tree-dump-1.0.2.tgz#c460d5921caeb197bde71d0e9a7b479848c5b8ac" + integrity sha512-dpev9ABuLWdEubk+cIaI9cHwRNNDjkBBLXTwI4UCUFdQ5xXKqNXoK4FEciw/vxf+NQ7Cb7sGUyeUtORvHIdRXQ== trim-newlines@^4.0.2: version "4.1.1" From 2312eff1acc8818abcf057ec1fea54205b8c8d2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 3 Nov 2024 06:26:54 +0900 Subject: [PATCH 09/29] MINOR: [JS] Bump rollup from 4.22.4 to 4.24.3 in /js (#44610) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [rollup](https://github.com/rollup/rollup) from 4.22.4 to 4.24.3.
Release notes

Sourced from rollup's releases.

v4.24.3

4.24.3

2024-10-29

Bug Fixes

  • Slightly reduce memory consumption by specifying fixed array sizes where possible (#5703)

Pull Requests

v4.24.2

4.24.2

2024-10-27

Bug Fixes

  • Add missing build dependency (#5705)

Pull Requests

  • #5705: Fix "Couldn't find package" error when installing rollup using yarn (@​tagattie)

v4.24.1

4.24.1

2024-10-27

Bug Fixes

  • Support running Rollup natively on FreeBSD (#5698)

Pull Requests

v4.24.0

4.24.0

2024-10-02

... (truncated)

Changelog

Sourced from rollup's changelog.

4.24.3

2024-10-29

Bug Fixes

  • Slightly reduce memory consumption by specifying fixed array sizes where possible (#5703)

Pull Requests

4.24.2

2024-10-27

Bug Fixes

  • Add missing build dependency (#5705)

Pull Requests

  • #5705: Fix "Couldn't find package" error when installing rollup using yarn (@​tagattie)

4.24.1

2024-10-27

Bug Fixes

  • Support running Rollup natively on FreeBSD (#5698)

Pull Requests

4.24.0

2024-10-02

Features

  • Support preserving and transpiling JSX syntax (#5668)

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=rollup&package-manager=npm_and_yarn&previous-version=4.22.4&new-version=4.24.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- js/package.json | 2 +- js/yarn.lock | 220 +++++++++++++++++++++++++----------------------- 2 files changed, 117 insertions(+), 105 deletions(-) diff --git a/js/package.json b/js/package.json index 2a0db0067bd5b..27132ff812b7e 100644 --- a/js/package.json +++ b/js/package.json @@ -102,7 +102,7 @@ "mkdirp": "3.0.1", "multistream": "4.1.0", "regenerator-runtime": "0.14.1", - "rollup": "4.22.4", + "rollup": "4.24.3", "rxjs": "7.8.1", "ts-jest": "29.1.4", "ts-node": "10.9.2", diff --git a/js/yarn.lock b/js/yarn.lock index 2941ccf3c569d..e7906b9aaad2d 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -1017,85 +1017,95 @@ estree-walker "^2.0.2" picomatch "^2.3.1" -"@rollup/rollup-android-arm-eabi@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.22.4.tgz#8b613b9725e8f9479d142970b106b6ae878610d5" - integrity sha512-Fxamp4aEZnfPOcGA8KSNEohV8hX7zVHOemC8jVBoBUHu5zpJK/Eu3uJwt6BMgy9fkvzxDaurgj96F/NiLukF2w== - -"@rollup/rollup-android-arm64@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.22.4.tgz#654ca1049189132ff602bfcf8df14c18da1f15fb" - integrity sha512-VXoK5UMrgECLYaMuGuVTOx5kcuap1Jm8g/M83RnCHBKOqvPPmROFJGQaZhGccnsFtfXQ3XYa4/jMCJvZnbJBdA== - -"@rollup/rollup-darwin-arm64@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.22.4.tgz#6d241d099d1518ef0c2205d96b3fa52e0fe1954b" - integrity sha512-xMM9ORBqu81jyMKCDP+SZDhnX2QEVQzTcC6G18KlTQEzWK8r/oNZtKuZaCcHhnsa6fEeOBionoyl5JsAbE/36Q== - -"@rollup/rollup-darwin-x64@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.22.4.tgz#42bd19d292a57ee11734c980c4650de26b457791" - integrity sha512-aJJyYKQwbHuhTUrjWjxEvGnNNBCnmpHDvrb8JFDbeSH3m2XdHcxDd3jthAzvmoI8w/kSjd2y0udT+4okADsZIw== - -"@rollup/rollup-linux-arm-gnueabihf@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.22.4.tgz#f23555ee3d8fe941c5c5fd458cd22b65eb1c2232" - integrity sha512-j63YtCIRAzbO+gC2L9dWXRh5BFetsv0j0va0Wi9epXDgU/XUi5dJKo4USTttVyK7fGw2nPWK0PbAvyliz50SCQ== - -"@rollup/rollup-linux-arm-musleabihf@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.22.4.tgz#f3bbd1ae2420f5539d40ac1fde2b38da67779baa" - integrity sha512-dJnWUgwWBX1YBRsuKKMOlXCzh2Wu1mlHzv20TpqEsfdZLb3WoJW2kIEsGwLkroYf24IrPAvOT/ZQ2OYMV6vlrg== - -"@rollup/rollup-linux-arm64-gnu@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.22.4.tgz#7abe900120113e08a1f90afb84c7c28774054d15" - integrity sha512-AdPRoNi3NKVLolCN/Sp4F4N1d98c4SBnHMKoLuiG6RXgoZ4sllseuGioszumnPGmPM2O7qaAX/IJdeDU8f26Aw== - -"@rollup/rollup-linux-arm64-musl@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.22.4.tgz#9e655285c8175cd44f57d6a1e8e5dedfbba1d820" - integrity sha512-Gl0AxBtDg8uoAn5CCqQDMqAx22Wx22pjDOjBdmG0VIWX3qUBHzYmOKh8KXHL4UpogfJ14G4wk16EQogF+v8hmA== - -"@rollup/rollup-linux-powerpc64le-gnu@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.22.4.tgz#9a79ae6c9e9d8fe83d49e2712ecf4302db5bef5e" - integrity sha512-3aVCK9xfWW1oGQpTsYJJPF6bfpWfhbRnhdlyhak2ZiyFLDaayz0EP5j9V1RVLAAxlmWKTDfS9wyRyY3hvhPoOg== - -"@rollup/rollup-linux-riscv64-gnu@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.22.4.tgz#67ac70eca4ace8e2942fabca95164e8874ab8128" - integrity sha512-ePYIir6VYnhgv2C5Xe9u+ico4t8sZWXschR6fMgoPUK31yQu7hTEJb7bCqivHECwIClJfKgE7zYsh1qTP3WHUA== - -"@rollup/rollup-linux-s390x-gnu@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.22.4.tgz#9f883a7440f51a22ed7f99e1d070bd84ea5005fc" - integrity sha512-GqFJ9wLlbB9daxhVlrTe61vJtEY99/xB3C8e4ULVsVfflcpmR6c8UZXjtkMA6FhNONhj2eA5Tk9uAVw5orEs4Q== - -"@rollup/rollup-linux-x64-gnu@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.22.4.tgz#70116ae6c577fe367f58559e2cffb5641a1dd9d0" - integrity sha512-87v0ol2sH9GE3cLQLNEy0K/R0pz1nvg76o8M5nhMR0+Q+BBGLnb35P0fVz4CQxHYXaAOhE8HhlkaZfsdUOlHwg== - -"@rollup/rollup-linux-x64-musl@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.22.4.tgz#f473f88219feb07b0b98b53a7923be716d1d182f" - integrity sha512-UV6FZMUgePDZrFjrNGIWzDo/vABebuXBhJEqrHxrGiU6HikPy0Z3LfdtciIttEUQfuDdCn8fqh7wiFJjCNwO+g== - -"@rollup/rollup-win32-arm64-msvc@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.22.4.tgz#4349482d17f5d1c58604d1c8900540d676f420e0" - integrity sha512-BjI+NVVEGAXjGWYHz/vv0pBqfGoUH0IGZ0cICTn7kB9PyjrATSkX+8WkguNjWoj2qSr1im/+tTGRaY+4/PdcQw== - -"@rollup/rollup-win32-ia32-msvc@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.22.4.tgz#a6fc39a15db618040ec3c2a24c1e26cb5f4d7422" - integrity sha512-SiWG/1TuUdPvYmzmYnmd3IEifzR61Tragkbx9D3+R8mzQqDBz8v+BvZNDlkiTtI9T15KYZhP0ehn3Dld4n9J5g== - -"@rollup/rollup-win32-x64-msvc@4.22.4": - version "4.22.4" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.22.4.tgz#3dd5d53e900df2a40841882c02e56f866c04d202" - integrity sha512-j8pPKp53/lq9lMXN57S8cFz0MynJk8OWNuUnXct/9KCpKU7DgU3bYMJhwWmcqC0UU29p8Lr0/7KEVcaM6bf47Q== +"@rollup/rollup-android-arm-eabi@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.24.3.tgz#49a2a9808074f2683667992aa94b288e0b54fc82" + integrity sha512-ufb2CH2KfBWPJok95frEZZ82LtDl0A6QKTa8MoM+cWwDZvVGl5/jNb79pIhRvAalUu+7LD91VYR0nwRD799HkQ== + +"@rollup/rollup-android-arm64@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.24.3.tgz#197e3bc01c228d3c23591e0fcedca91f8f398ec1" + integrity sha512-iAHpft/eQk9vkWIV5t22V77d90CRofgR2006UiCjHcHJFVI1E0oBkQIAbz+pLtthFw3hWEmVB4ilxGyBf48i2Q== + +"@rollup/rollup-darwin-arm64@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.24.3.tgz#16772c0309d0dc3cca716580cdac7a1c560ddf46" + integrity sha512-QPW2YmkWLlvqmOa2OwrfqLJqkHm7kJCIMq9kOz40Zo9Ipi40kf9ONG5Sz76zszrmIZZ4hgRIkez69YnTHgEz1w== + +"@rollup/rollup-darwin-x64@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.24.3.tgz#4e98120a1c4cda7d4043ccce72347cee53784140" + integrity sha512-KO0pN5x3+uZm1ZXeIfDqwcvnQ9UEGN8JX5ufhmgH5Lz4ujjZMAnxQygZAVGemFWn+ZZC0FQopruV4lqmGMshow== + +"@rollup/rollup-freebsd-arm64@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.24.3.tgz#27145e414986e216e0d9b9a8d488028f33c39566" + integrity sha512-CsC+ZdIiZCZbBI+aRlWpYJMSWvVssPuWqrDy/zi9YfnatKKSLFCe6fjna1grHuo/nVaHG+kiglpRhyBQYRTK4A== + +"@rollup/rollup-freebsd-x64@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.24.3.tgz#67e75fd87a903090f038b212273c492e5ca6b32f" + integrity sha512-F0nqiLThcfKvRQhZEzMIXOQG4EeX61im61VYL1jo4eBxv4aZRmpin6crnBJQ/nWnCsjH5F6J3W6Stdm0mBNqBg== + +"@rollup/rollup-linux-arm-gnueabihf@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.24.3.tgz#bb45ebadbb9496298ab5461373bde357e8f33e88" + integrity sha512-KRSFHyE/RdxQ1CSeOIBVIAxStFC/hnBgVcaiCkQaVC+EYDtTe4X7z5tBkFyRoBgUGtB6Xg6t9t2kulnX6wJc6A== + +"@rollup/rollup-linux-arm-musleabihf@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.24.3.tgz#384276c23feb0a4d6ffa603a9a760decce8b4118" + integrity sha512-h6Q8MT+e05zP5BxEKz0vi0DhthLdrNEnspdLzkoFqGwnmOzakEHSlXfVyA4HJ322QtFy7biUAVFPvIDEDQa6rw== + +"@rollup/rollup-linux-arm64-gnu@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.24.3.tgz#89e5a4570ddd9eca908324a6de60bd64f904e3f0" + integrity sha512-fKElSyXhXIJ9pqiYRqisfirIo2Z5pTTve5K438URf08fsypXrEkVmShkSfM8GJ1aUyvjakT+fn2W7Czlpd/0FQ== + +"@rollup/rollup-linux-arm64-musl@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.24.3.tgz#9ffd7cd6c6c6670d8c039056d6a49ad9f1f66949" + integrity sha512-YlddZSUk8G0px9/+V9PVilVDC6ydMz7WquxozToozSnfFK6wa6ne1ATUjUvjin09jp34p84milxlY5ikueoenw== + +"@rollup/rollup-linux-powerpc64le-gnu@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.24.3.tgz#4d32ce982e2d25e3b8116336ad5ce6e270b5a024" + integrity sha512-yNaWw+GAO8JjVx3s3cMeG5Esz1cKVzz8PkTJSfYzE5u7A+NvGmbVFEHP+BikTIyYWuz0+DX9kaA3pH9Sqxp69g== + +"@rollup/rollup-linux-riscv64-gnu@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.24.3.tgz#f43d4e0572397e3d3acd82d77d79ce021dea3310" + integrity sha512-lWKNQfsbpv14ZCtM/HkjCTm4oWTKTfxPmr7iPfp3AHSqyoTz5AgLemYkWLwOBWc+XxBbrU9SCokZP0WlBZM9lA== + +"@rollup/rollup-linux-s390x-gnu@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.24.3.tgz#264f8a4c206173945bdab2a676d638b7945106a9" + integrity sha512-HoojGXTC2CgCcq0Woc/dn12wQUlkNyfH0I1ABK4Ni9YXyFQa86Fkt2Q0nqgLfbhkyfQ6003i3qQk9pLh/SpAYw== + +"@rollup/rollup-linux-x64-gnu@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.24.3.tgz#e86172a407b2edd41540ec2ae636e497fadccff6" + integrity sha512-mnEOh4iE4USSccBOtcrjF5nj+5/zm6NcNhbSEfR3Ot0pxBwvEn5QVUXcuOwwPkapDtGZ6pT02xLoPaNv06w7KQ== + +"@rollup/rollup-linux-x64-musl@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.24.3.tgz#8ae9bf78986d1b16ccbc89ab6f2dfa96807d3178" + integrity sha512-rMTzawBPimBQkG9NKpNHvquIUTQPzrnPxPbCY1Xt+mFkW7pshvyIS5kYgcf74goxXOQk0CP3EoOC1zcEezKXhw== + +"@rollup/rollup-win32-arm64-msvc@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.24.3.tgz#11d6a59f651a3c2a9e5eaab0a99367b77a29c319" + integrity sha512-2lg1CE305xNvnH3SyiKwPVsTVLCg4TmNCF1z7PSHX2uZY2VbUpdkgAllVoISD7JO7zu+YynpWNSKAtOrX3AiuA== + +"@rollup/rollup-win32-ia32-msvc@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.24.3.tgz#7ff146e53dc6e388b60329b7ec3335501d2b0f98" + integrity sha512-9SjYp1sPyxJsPWuhOCX6F4jUMXGbVVd5obVpoVEi8ClZqo52ViZewA6eFz85y8ezuOA+uJMP5A5zo6Oz4S5rVQ== + +"@rollup/rollup-win32-x64-msvc@4.24.3": + version "4.24.3" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.24.3.tgz#7687335781efe6bee14d6ed8eff9746a9f24c9cd" + integrity sha512-HGZgRFFYrMrP3TJlq58nR1xy8zHKId25vhmm5S9jETEfDf6xybPxsavFTJaufe2zgOGYJBskGlj49CwtEuFhWQ== "@rollup/stream@3.0.1": version "3.0.1" @@ -1298,10 +1308,10 @@ "@types/estree" "*" "@types/json-schema" "*" -"@types/estree@*", "@types/estree@1.0.5", "@types/estree@^1.0.0", "@types/estree@^1.0.5": - version "1.0.5" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4" - integrity sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw== +"@types/estree@*", "@types/estree@1.0.6", "@types/estree@^1.0.0", "@types/estree@^1.0.5": + version "1.0.6" + resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.6.tgz#628effeeae2064a1b4e79f78e81d87b7e5fc7b50" + integrity sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw== "@types/expect@^1.20.4": version "1.20.4" @@ -6179,29 +6189,31 @@ rimraf@^3.0.2: dependencies: glob "^7.1.3" -rollup@4.22.4: - version "4.22.4" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.22.4.tgz#4135a6446671cd2a2453e1ad42a45d5973ec3a0f" - integrity sha512-vD8HJ5raRcWOyymsR6Z3o6+RzfEPCnVLMFJ6vRslO1jt4LO6dUo5Qnpg7y4RkZFM2DMe3WUirkI5c16onjrc6A== +rollup@4.24.3: + version "4.24.3" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.24.3.tgz#8b259063740af60b0030315f88665ba2041789b8" + integrity sha512-HBW896xR5HGmoksbi3JBDtmVzWiPAYqp7wip50hjQ67JbDz61nyoMPdqu1DvVW9asYb2M65Z20ZHsyJCMqMyDg== dependencies: - "@types/estree" "1.0.5" + "@types/estree" "1.0.6" optionalDependencies: - "@rollup/rollup-android-arm-eabi" "4.22.4" - "@rollup/rollup-android-arm64" "4.22.4" - "@rollup/rollup-darwin-arm64" "4.22.4" - "@rollup/rollup-darwin-x64" "4.22.4" - "@rollup/rollup-linux-arm-gnueabihf" "4.22.4" - "@rollup/rollup-linux-arm-musleabihf" "4.22.4" - "@rollup/rollup-linux-arm64-gnu" "4.22.4" - "@rollup/rollup-linux-arm64-musl" "4.22.4" - "@rollup/rollup-linux-powerpc64le-gnu" "4.22.4" - "@rollup/rollup-linux-riscv64-gnu" "4.22.4" - "@rollup/rollup-linux-s390x-gnu" "4.22.4" - "@rollup/rollup-linux-x64-gnu" "4.22.4" - "@rollup/rollup-linux-x64-musl" "4.22.4" - "@rollup/rollup-win32-arm64-msvc" "4.22.4" - "@rollup/rollup-win32-ia32-msvc" "4.22.4" - "@rollup/rollup-win32-x64-msvc" "4.22.4" + "@rollup/rollup-android-arm-eabi" "4.24.3" + "@rollup/rollup-android-arm64" "4.24.3" + "@rollup/rollup-darwin-arm64" "4.24.3" + "@rollup/rollup-darwin-x64" "4.24.3" + "@rollup/rollup-freebsd-arm64" "4.24.3" + "@rollup/rollup-freebsd-x64" "4.24.3" + "@rollup/rollup-linux-arm-gnueabihf" "4.24.3" + "@rollup/rollup-linux-arm-musleabihf" "4.24.3" + "@rollup/rollup-linux-arm64-gnu" "4.24.3" + "@rollup/rollup-linux-arm64-musl" "4.24.3" + "@rollup/rollup-linux-powerpc64le-gnu" "4.24.3" + "@rollup/rollup-linux-riscv64-gnu" "4.24.3" + "@rollup/rollup-linux-s390x-gnu" "4.24.3" + "@rollup/rollup-linux-x64-gnu" "4.24.3" + "@rollup/rollup-linux-x64-musl" "4.24.3" + "@rollup/rollup-win32-arm64-msvc" "4.24.3" + "@rollup/rollup-win32-ia32-msvc" "4.24.3" + "@rollup/rollup-win32-x64-msvc" "4.24.3" fsevents "~2.3.2" run-parallel@^1.1.9: From abda57c42123ab03d97fec7a10c91f231ba7ad64 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sun, 3 Nov 2024 06:53:24 +0900 Subject: [PATCH 10/29] GH-44604: [GLib] Add Decimal32Array (#44617) ### Rationale for this change The `arrow::Decimal32Array` has been released. The `GArrowDecimal32Array` class must be implemented in the GLib. ### What changes are included in this PR? * Implement `GArrowDecimal32Array` * Implement `GArrowDecimal32ArrayBuilder` ### Are these changes tested? YES ### Are there any user-facing changes? NO * GitHub Issue: #44604 Authored-by: Hiroyuki Sato Signed-off-by: Sutou Kouhei --- c_glib/arrow-glib/array-builder.cpp | 105 ++++++++++++++++++++++++++++ c_glib/arrow-glib/array-builder.h | 30 ++++++++ c_glib/arrow-glib/basic-array.cpp | 62 ++++++++++++++++ c_glib/arrow-glib/basic-array.h | 20 ++++++ c_glib/test/test-decimal32-array.rb | 37 ++++++++++ 5 files changed, 254 insertions(+) create mode 100644 c_glib/test/test-decimal32-array.rb diff --git a/c_glib/arrow-glib/array-builder.cpp b/c_glib/arrow-glib/array-builder.cpp index 4f82ee2983b34..87e22c7435209 100644 --- a/c_glib/arrow-glib/array-builder.cpp +++ b/c_glib/arrow-glib/array-builder.cpp @@ -457,6 +457,9 @@ G_BEGIN_DECLS * #GArrowMapArrayBuilder is the class to create a new * #GArrowMapArray. * + * #GArrowDecimal32ArrayBuilder is the class to create a new + * #GArrowDecimal32Array. + * * #GArrowDecimal64ArrayBuilder is the class to create a new * #GArrowDecimal64Array. * @@ -6065,6 +6068,105 @@ garrow_map_array_builder_get_value_builder(GArrowMapArrayBuilder *builder) return priv->value_builder; } +G_DEFINE_TYPE(GArrowDecimal32ArrayBuilder, + garrow_decimal32_array_builder, + GARROW_TYPE_FIXED_SIZE_BINARY_ARRAY_BUILDER) + +static void +garrow_decimal32_array_builder_init(GArrowDecimal32ArrayBuilder *builder) +{ +} + +static void +garrow_decimal32_array_builder_class_init(GArrowDecimal32ArrayBuilderClass *klass) +{ +} + +/** + * garrow_decimal32_array_builder_new: + * @data_type: #GArrowDecimal32DataType for the decimal. + * + * Returns: A newly created #GArrowDecimal32ArrayBuilder. + * + * Since: 19.0.0 + */ +GArrowDecimal32ArrayBuilder * +garrow_decimal32_array_builder_new(GArrowDecimal32DataType *data_type) +{ + auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type)); + auto builder = + garrow_array_builder_new(arrow_data_type, NULL, "[decimal32-array-builder][new]"); + return GARROW_DECIMAL32_ARRAY_BUILDER(builder); +} + +/** + * garrow_decimal32_array_builder_append_value: + * @builder: A #GArrowDecimal32ArrayBuilder. + * @value: (nullable): A decimal value. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: %TRUE on success, %FALSE if there was an error. + * + * Since: 19.0.0 + */ +gboolean +garrow_decimal32_array_builder_append_value(GArrowDecimal32ArrayBuilder *builder, + GArrowDecimal32 *value, + GError **error) +{ + if (value) { + auto arrow_decimal = garrow_decimal32_get_raw(value); + return garrow_array_builder_append_value( + GARROW_ARRAY_BUILDER(builder), + *arrow_decimal, + error, + "[decimal32-array-builder][append-value]"); + } else { + return garrow_array_builder_append_null(GARROW_ARRAY_BUILDER(builder), error); + } +} + +/** + * garrow_decimal32_array_builder_append_values: + * @builder: A #GArrowDecimal32ArrayBuilder. + * @values: (array length=values_length): The array of #GArrowDecimal32. + * @values_length: The length of @values. + * @is_valids: (nullable) (array length=is_valids_length): The array of + * boolean that shows whether the Nth value is valid or not. If the + * Nth @is_valids is %TRUE, the Nth @values is valid value. Otherwise + * the Nth value is null value. + * @is_valids_length: The length of @is_valids. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Append multiple values at once. It's more efficient than multiple + * `append` and `append_null` calls. + * + * Returns: %TRUE on success, %FALSE if there was an error. + * + * Since: 19.0.0 + */ +gboolean +garrow_decimal32_array_builder_append_values(GArrowDecimal32ArrayBuilder *builder, + GArrowDecimal32 **values, + gint64 values_length, + const gboolean *is_valids, + gint64 is_valids_length, + GError **error) +{ + return garrow_array_builder_append_values( + GARROW_ARRAY_BUILDER(builder), + values, + values_length, + is_valids, + is_valids_length, + error, + "[decimal32-array-builder][append-values]", + [](guint8 *output, GArrowDecimal32 *value, gsize size) { + auto arrow_decimal = garrow_decimal32_get_raw(value); + arrow_decimal->ToBytes(output); + }); +} + G_DEFINE_TYPE(GArrowDecimal64ArrayBuilder, garrow_decimal64_array_builder, GARROW_TYPE_FIXED_SIZE_BINARY_ARRAY_BUILDER) @@ -6683,6 +6785,9 @@ garrow_array_builder_new_raw(std::shared_ptr *arrow_builder case arrow::Type::type::MAP: type = GARROW_TYPE_MAP_ARRAY_BUILDER; break; + case arrow::Type::type::DECIMAL32: + type = GARROW_TYPE_DECIMAL32_ARRAY_BUILDER; + break; case arrow::Type::type::DECIMAL64: type = GARROW_TYPE_DECIMAL64_ARRAY_BUILDER; break; diff --git a/c_glib/arrow-glib/array-builder.h b/c_glib/arrow-glib/array-builder.h index f653ddb7781a3..c15c411503114 100644 --- a/c_glib/arrow-glib/array-builder.h +++ b/c_glib/arrow-glib/array-builder.h @@ -1729,6 +1729,36 @@ GARROW_AVAILABLE_IN_0_17 GArrowArrayBuilder * garrow_map_array_builder_get_value_builder(GArrowMapArrayBuilder *builder); +#define GARROW_TYPE_DECIMAL32_ARRAY_BUILDER (garrow_decimal32_array_builder_get_type()) +GARROW_AVAILABLE_IN_19_0 +G_DECLARE_DERIVABLE_TYPE(GArrowDecimal32ArrayBuilder, + garrow_decimal32_array_builder, + GARROW, + DECIMAL32_ARRAY_BUILDER, + GArrowFixedSizeBinaryArrayBuilder) +struct _GArrowDecimal32ArrayBuilderClass +{ + GArrowFixedSizeBinaryArrayBuilderClass parent_class; +}; + +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal32ArrayBuilder * +garrow_decimal32_array_builder_new(GArrowDecimal32DataType *data_type); + +GARROW_AVAILABLE_IN_19_0 +gboolean +garrow_decimal32_array_builder_append_value(GArrowDecimal32ArrayBuilder *builder, + GArrowDecimal32 *value, + GError **error); +GARROW_AVAILABLE_IN_19_0 +gboolean +garrow_decimal32_array_builder_append_values(GArrowDecimal32ArrayBuilder *builder, + GArrowDecimal32 **values, + gint64 values_length, + const gboolean *is_valids, + gint64 is_valids_length, + GError **error); + #define GARROW_TYPE_DECIMAL64_ARRAY_BUILDER (garrow_decimal64_array_builder_get_type()) GARROW_AVAILABLE_IN_19_0 G_DECLARE_DERIVABLE_TYPE(GArrowDecimal64ArrayBuilder, diff --git a/c_glib/arrow-glib/basic-array.cpp b/c_glib/arrow-glib/basic-array.cpp index 8c39715c384f0..2169f8a05c77b 100644 --- a/c_glib/arrow-glib/basic-array.cpp +++ b/c_glib/arrow-glib/basic-array.cpp @@ -171,6 +171,11 @@ G_BEGIN_DECLS * have Arrow format data, you need to use #GArrowMonthDayNanoIntervalArray * to create a new array. * + * #GArrowDecimal32Array is a class for 32-bit decimal array. It can + * store zero or more 32-bit decimal data. If you don't have Arrow + * format data, you need to use #GArrowDecimal32ArrayBuilder to + * create a new array. + * * #GArrowDecimal64Array is a class for 64-bit decimal array. It can * store zero or more 64-bit decimal data. If you don't have Arrow * format data, you need to use #GArrowDecimal64ArrayBuilder to @@ -3095,6 +3100,60 @@ garrow_fixed_size_binary_array_get_values_bytes(GArrowFixedSizeBinaryArray *arra arrow_binary_array->byte_width() * arrow_array->length()); } +G_DEFINE_TYPE(GArrowDecimal32Array, + garrow_decimal32_array, + GARROW_TYPE_FIXED_SIZE_BINARY_ARRAY) +static void +garrow_decimal32_array_init(GArrowDecimal32Array *object) +{ +} + +static void +garrow_decimal32_array_class_init(GArrowDecimal32ArrayClass *klass) +{ +} + +/** + * garrow_decimal32_array_format_value: + * @array: A #GArrowDecimal32Array. + * @i: The index of the target value. + * + * Returns: (transfer full): The formatted @i-th value. + * + * It should be freed with g_free() when no longer needed. + * + * Since: 19.0.0 + */ +gchar * +garrow_decimal32_array_format_value(GArrowDecimal32Array *array, gint64 i) +{ + auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array)); + auto arrow_decimal32_array = + std::static_pointer_cast(arrow_array); + auto value = arrow_decimal32_array->FormatValue(i); + return g_strndup(value.data(), value.size()); +} + +/** + * garrow_decimal32_array_get_value: + * @array: A #GArrowDecimal32Array. + * @i: The index of the target value. + * + * Returns: (transfer full): The @i-th value. + * + * Since: 19.0.0 + */ +GArrowDecimal32 * +garrow_decimal32_array_get_value(GArrowDecimal32Array *array, gint64 i) +{ + auto arrow_array = garrow_array_get_raw(GARROW_ARRAY(array)); + auto arrow_decimal32_array = + std::static_pointer_cast(arrow_array); + auto arrow_decimal32 = + std::make_shared(arrow_decimal32_array->GetValue(i)); + return garrow_decimal32_new_raw(&arrow_decimal32); +} + G_DEFINE_TYPE(GArrowDecimal64Array, garrow_decimal64_array, GARROW_TYPE_FIXED_SIZE_BINARY_ARRAY) @@ -3502,6 +3561,9 @@ garrow_array_new_raw_valist(std::shared_ptr *arrow_array, case arrow::Type::type::DICTIONARY: type = GARROW_TYPE_DICTIONARY_ARRAY; break; + case arrow::Type::type::DECIMAL32: + type = GARROW_TYPE_DECIMAL32_ARRAY; + break; case arrow::Type::type::DECIMAL64: type = GARROW_TYPE_DECIMAL64_ARRAY; break; diff --git a/c_glib/arrow-glib/basic-array.h b/c_glib/arrow-glib/basic-array.h index f70cf114a4a96..dbffedde28164 100644 --- a/c_glib/arrow-glib/basic-array.h +++ b/c_glib/arrow-glib/basic-array.h @@ -810,6 +810,26 @@ GARROW_AVAILABLE_IN_3_0 GBytes * garrow_fixed_size_binary_array_get_values_bytes(GArrowFixedSizeBinaryArray *array); +#define GARROW_TYPE_DECIMAL32_ARRAY (garrow_decimal32_array_get_type()) +GARROW_AVAILABLE_IN_19_0 +G_DECLARE_DERIVABLE_TYPE(GArrowDecimal32Array, + garrow_decimal32_array, + GARROW, + DECIMAL32_ARRAY, + GArrowFixedSizeBinaryArray) +struct _GArrowDecimal32ArrayClass +{ + GArrowFixedSizeBinaryArrayClass parent_class; +}; + +GARROW_AVAILABLE_IN_19_0 +gchar * +garrow_decimal32_array_format_value(GArrowDecimal32Array *array, gint64 i); + +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal32 * +garrow_decimal32_array_get_value(GArrowDecimal32Array *array, gint64 i); + #define GARROW_TYPE_DECIMAL64_ARRAY (garrow_decimal64_array_get_type()) GARROW_AVAILABLE_IN_19_0 G_DECLARE_DERIVABLE_TYPE(GArrowDecimal64Array, diff --git a/c_glib/test/test-decimal32-array.rb b/c_glib/test/test-decimal32-array.rb new file mode 100644 index 0000000000000..ee40f27e81179 --- /dev/null +++ b/c_glib/test/test-decimal32-array.rb @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestDecimal32Array < Test::Unit::TestCase + def test_format_value + data_type = Arrow::Decimal32DataType.new(8, 2) + builder = Arrow::Decimal32ArrayBuilder.new(data_type) + decimal = Arrow::Decimal32.new("23423445") + builder.append_value(decimal) + array = builder.finish + assert_equal("234234.45", array.format_value(0)) + end + + def test_value + data_type = Arrow::Decimal32DataType.new(8, 2) + builder = Arrow::Decimal32ArrayBuilder.new(data_type) + decimal = Arrow::Decimal32.new("23423445") + builder.append_value(decimal) + array = builder.finish + assert_equal("234234.45", + array.get_value(0).to_string_scale(array.value_data_type.scale)) + end +end From 405ed6fa84f79471bb57d58bb279584245106d19 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 3 Nov 2024 08:35:51 +0900 Subject: [PATCH 11/29] MINOR: [JS] Bump tslib from 2.6.3 to 2.8.1 in /js (#44611) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [tslib](https://github.com/Microsoft/tslib) from 2.6.3 to 2.8.1.
Release notes

Sourced from tslib's releases.

v2.8.1

What's Changed

Full Changelog: https://github.com/microsoft/tslib/compare/v2.8.0...v2.8.1

v2.8.0

What's Changed

Full Changelog: https://github.com/microsoft/tslib/compare/v2.7.0...v2.8.0

v2.7.0

What's Changed

Full Changelog: https://github.com/microsoft/tslib/compare/v2.6.3...v2.7.0

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=tslib&package-manager=npm_and_yarn&previous-version=2.6.3&new-version=2.8.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- js/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/js/yarn.lock b/js/yarn.lock index e7906b9aaad2d..2babda15b0ef1 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -6933,9 +6933,9 @@ ts-node@10.9.2: yn "3.1.1" tslib@^2.0.0, tslib@^2.1.0, tslib@^2.4.0, tslib@^2.6.2: - version "2.6.3" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.3.tgz#0438f810ad7a9edcde7a241c3d80db693c8cbfe0" - integrity sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ== + version "2.8.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f" + integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" From 5ff8cda475a7799d865a3741f32ace68f622a345 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 3 Nov 2024 08:36:45 +0900 Subject: [PATCH 12/29] MINOR: [JS] Bump webpack from 5.91.0 to 5.96.1 in /js (#44612) Bumps [webpack](https://github.com/webpack/webpack) from 5.91.0 to 5.96.1.
Release notes

Sourced from webpack's releases.

v5.96.1

Bug Fixes

  • [Types] Add @ types/eslint-scope to dependencieS
  • [Types] Fixed regression in validate

v5.96.0

Bug Fixes

  • Fixed Module Federation should track all referenced chunks
  • Handle Data URI without base64 word
  • HotUpdateChunk have correct runtime when modified with new runtime
  • Order of chunks ids in generated chunk code
  • No extra Javascript chunks when using asset module as an entrypoint
  • Use optimistically logic for output.environment.dynamicImport to determine chunk format when no browserslist or target
  • Collision with global variables for optimization.avoidEntryIife
  • Avoid through variables in inlined module
  • Allow chunk template strings in output.devtoolNamespace
  • No extra runtime for get javascript/css chunk filename
  • No extra runtime for prefetch and preload in JS runtime when it was unsed in CSS
  • Avoid cache invalidation using ProgressPlugin
  • Increase parallelism when using importModule on the execution stage
  • Correctly parsing string in export and import
  • Typescript types
  • [CSS] css/auto considers a module depending on its filename as css (pure CSS) or css/local, before it was css/global and css/local
  • [CSS] Always interpolate classes even if they are not involved in export
  • [CSS] No extra runtime in Javascript runtime chunks for asset modules used in CSS
  • [CSS] No extra runtime in Javascript runtime chunks for external asset modules used in CSS
  • [CSS] No extra runtime for the node target
  • [CSS] Fixed url()s and @ import parsing
  • [CSS] Fixed - emit a warning on broken :local and :global

New Features

  • Export CSS and ESM runtime modules
  • Single Runtime Chunk and Federation eager module hoisting
  • [CSS] Support /* webpackIgnore: true */ for CSS files
  • [CSS] Support src() support
  • [CSS] CSS nesting in CSS modules

v5.95.0

Bug Fixes

  • Fixed hanging when attempting to read a symlink-like file that it can't read
  • Handle default for import context element dependency
  • Merge duplicate chunks call after split chunks
  • Generate correctly code for dynamically importing the same file twice and destructuring
  • Use content hash as [base] and [name] for extracted DataURI's
  • Distinguish module and import in module-import for externals import's

... (truncated)

Commits
  • d4ced73 chore(release): 5.96.1
  • 7d6dbea fix: types regression in validate
  • 5c556e3 fix: types regression in validate
  • 2420eae fix: add @ types/eslint-scope to dependencies due types regression
  • ec45d2d fix: add @ types/eslint-scope to dependencies
  • aff0c3e chore(release): 5.96.0
  • 6f11ec1 refactor: module source types code
  • b07142f refactor: module source types code
  • 7d98b3c fix: Module Federation should track all referenced chunks
  • 6d09769 chore: linting
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=webpack&package-manager=npm_and_yarn&previous-version=5.91.0&new-version=5.96.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- js/package.json | 2 +- js/yarn.lock | 106 +++++++++++++++++++++++++----------------------- 2 files changed, 56 insertions(+), 52 deletions(-) diff --git a/js/package.json b/js/package.json index 27132ff812b7e..399a5c3b94486 100644 --- a/js/package.json +++ b/js/package.json @@ -112,7 +112,7 @@ "vinyl-named": "1.1.0", "vinyl-source-stream": "2.0.0", "web-streams-polyfill": "3.2.1", - "webpack": "5.91.0", + "webpack": "5.96.1", "webpack-bundle-analyzer": "4.10.2", "webpack-stream": "7.0.0", "xml2js": "0.6.2" diff --git a/js/yarn.lock b/js/yarn.lock index 2babda15b0ef1..1345eb97cf9fb 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -1292,7 +1292,7 @@ resolved "https://registry.yarnpkg.com/@types/command-line-usage/-/command-line-usage-5.0.4.tgz#374e4c62d78fbc5a670a0f36da10235af879a0d5" integrity sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg== -"@types/eslint-scope@^3.7.3": +"@types/eslint-scope@^3.7.7": version "3.7.7" resolved "https://registry.yarnpkg.com/@types/eslint-scope/-/eslint-scope-3.7.7.tgz#3108bd5f18b0cdb277c867b3dd449c9ed7079ac5" integrity sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg== @@ -1308,7 +1308,7 @@ "@types/estree" "*" "@types/json-schema" "*" -"@types/estree@*", "@types/estree@1.0.6", "@types/estree@^1.0.0", "@types/estree@^1.0.5": +"@types/estree@*", "@types/estree@1.0.6", "@types/estree@^1.0.0", "@types/estree@^1.0.6": version "1.0.6" resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.6.tgz#628effeeae2064a1b4e79f78e81d87b7e5fc7b50" integrity sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw== @@ -1683,11 +1683,6 @@ resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d" integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ== -acorn-import-assertions@^1.9.0: - version "1.9.0" - resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz#507276249d684797c84e0734ef84860334cfb1ac" - integrity sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA== - acorn-jsx@^5.3.2: version "5.3.2" resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" @@ -1703,10 +1698,10 @@ acorn@^6.4.1: resolved "https://registry.yarnpkg.com/acorn/-/acorn-6.4.2.tgz#35866fd710528e92de10cf06016498e47e39e1e6" integrity sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ== -acorn@^8.0.4, acorn@^8.4.1, acorn@^8.7.1, acorn@^8.8.2, acorn@^8.9.0: - version "8.11.3" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.3.tgz#71e0b14e13a4ec160724b38fb7b0f233b1b81d7a" - integrity sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg== +acorn@^8.0.4, acorn@^8.14.0, acorn@^8.4.1, acorn@^8.8.2, acorn@^8.9.0: + version "8.14.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.14.0.tgz#063e2c70cac5fb4f6467f0b11152e04c682795b0" + integrity sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA== aggregate-error@^4.0.0: version "4.0.1" @@ -2164,15 +2159,15 @@ braces@^3.0.2: dependencies: fill-range "^7.0.1" -browserslist@^4.21.10, browserslist@^4.22.2, browserslist@^4.23.0: - version "4.23.0" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.23.0.tgz#8f3acc2bbe73af7213399430890f86c63a5674ab" - integrity sha512-QW8HiM1shhT2GuzkvklfjcKDiWFXHOeFCIA/huJPwHsslwcydgk7X+z2zXpEijP98UCY7HbubZt5J2Zgvf0CaQ== +browserslist@^4.22.2, browserslist@^4.23.0, browserslist@^4.24.0: + version "4.24.2" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.24.2.tgz#f5845bc91069dbd55ee89faf9822e1d885d16580" + integrity sha512-ZIc+Q62revdMcqC6aChtW4jz3My3klmCO1fEmINZY/8J3EpBg5/A/D0AKmBveUh6pgoeycoMkVMko84tuYS+Gg== dependencies: - caniuse-lite "^1.0.30001587" - electron-to-chromium "^1.4.668" - node-releases "^2.0.14" - update-browserslist-db "^1.0.13" + caniuse-lite "^1.0.30001669" + electron-to-chromium "^1.5.41" + node-releases "^2.0.18" + update-browserslist-db "^1.1.1" bs-logger@0.x: version "0.2.6" @@ -2259,10 +2254,10 @@ camelcase@^6.2.0, camelcase@^6.3.0: resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== -caniuse-lite@^1.0.30001587: - version "1.0.30001605" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001605.tgz#ca12d7330dd8bcb784557eb9aa64f0037870d9d6" - integrity sha512-nXwGlFWo34uliI9z3n6Qc0wZaf7zaZWA1CPZ169La5mV3I/gem7bst0vr5XQH5TJXZIMfDeZyOrZnSlVzKxxHQ== +caniuse-lite@^1.0.30001669: + version "1.0.30001677" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001677.tgz#27c2e2c637e007cfa864a16f7dfe7cde66b38b5f" + integrity sha512-fmfjsOlJUpMWu+mAAtZZZHz7UEwsUxIIvu1TJfO1HqFQvB/B+ii0xr9B5HpbZY/mC4XZ8SvjHJqtAY6pDPQEog== chalk-template@^0.4.0: version "0.4.0" @@ -2831,10 +2826,10 @@ eastasianwidth@^0.2.0: resolved "https://registry.yarnpkg.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz#696ce2ec0aa0e6ea93a397ffcf24aa7840c827cb" integrity sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA== -electron-to-chromium@^1.4.668: - version "1.4.724" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.724.tgz#e0a86fe4d3d0e05a4d7b032549d79608078f830d" - integrity sha512-RTRvkmRkGhNBPPpdrgtDKvmOEYTrPlXDfc0J/Nfq5s29tEahAwhiX4mmhNzj6febWMleulxVYPh7QwCSL/EldA== +electron-to-chromium@^1.5.41: + version "1.5.50" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.50.tgz#d9ba818da7b2b5ef1f3dd32bce7046feb7e93234" + integrity sha512-eMVObiUQ2LdgeO1F/ySTXsvqvxb6ZH2zPGaMYsWzRDdOddUa77tdmI0ltg+L16UpbWdhPmuF3wIQYyQq65WfZw== emittery@^0.13.1: version "0.13.1" @@ -2858,10 +2853,10 @@ end-of-stream@^1.0.0, end-of-stream@^1.1.0, end-of-stream@^1.4.4: dependencies: once "^1.4.0" -enhanced-resolve@^5.16.0: - version "5.16.0" - resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.16.0.tgz#65ec88778083056cb32487faa9aef82ed0864787" - integrity sha512-O+QWCviPNSSLAD9Ucn8Awv+poAkqn3T1XY5/N7kR7rQO9yfSGWkYZDwpJ+iKF7B8rxaQKWngSqACpgzeapSyoA== +enhanced-resolve@^5.17.1: + version "5.17.1" + resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz#67bfbbcc2f81d511be77d686a90267ef7f898a15" + integrity sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg== dependencies: graceful-fs "^4.2.4" tapable "^2.2.0" @@ -3008,6 +3003,11 @@ escalade@^3.1.1: resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.2.tgz#54076e9ab29ea5bf3d8f1ed62acffbb88272df27" integrity sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA== +escalade@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.2.0.tgz#011a3f69856ba189dffa7dc8fcce99d2a87903e5" + integrity sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA== + escape-string-regexp@5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz#4683126b500b61762f2dbebace1806e8be31b1c8" @@ -5413,10 +5413,10 @@ node-int64@^0.4.0: resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" integrity sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw== -node-releases@^2.0.14: - version "2.0.14" - resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.14.tgz#2ffb053bceb8b2be8495ece1ab6ce600c4461b0b" - integrity sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw== +node-releases@^2.0.18: + version "2.0.18" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.18.tgz#f010e8d35e2fe8d6b2944f03f70213ecedc4ca3f" + integrity sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g== normalize-package-data@^2.3.2, normalize-package-data@^2.5.0: version "2.5.0" @@ -5747,6 +5747,11 @@ picocolors@^1.0.0: resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== +picocolors@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" + integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== + picomatch@^2.0.4, picomatch@^2.2.3, picomatch@^2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" @@ -7082,13 +7087,13 @@ upath@^1.1.1: resolved "https://registry.yarnpkg.com/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894" integrity sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg== -update-browserslist-db@^1.0.13: - version "1.0.13" - resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz#3c5e4f5c083661bd38ef64b6328c26ed6c8248c4" - integrity sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg== +update-browserslist-db@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.1.1.tgz#80846fba1d79e82547fb661f8d141e0945755fe5" + integrity sha512-R8UzCaa9Az+38REPiJ1tXlImTJXlVfgHZsglwBD/k6nj76ctsH1E3q4doGrukiLQd3sGQYu56r5+lo5r94l29A== dependencies: - escalade "^3.1.1" - picocolors "^1.0.0" + escalade "^3.2.0" + picocolors "^1.1.0" uri-js@^4.2.2: version "4.4.1" @@ -7292,21 +7297,20 @@ webpack-stream@7.0.0: through "^2.3.8" vinyl "^2.2.1" -webpack@5.91.0: - version "5.91.0" - resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.91.0.tgz#ffa92c1c618d18c878f06892bbdc3373c71a01d9" - integrity sha512-rzVwlLeBWHJbmgTC/8TvAcu5vpJNII+MelQpylD4jNERPwpBJOE2lEcko1zJX3QJeLjTTAnQxn/OJ8bjDzVQaw== +webpack@5.96.1: + version "5.96.1" + resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.96.1.tgz#3676d1626d8312b6b10d0c18cc049fba7ac01f0c" + integrity sha512-l2LlBSvVZGhL4ZrPwyr8+37AunkcYj5qh8o6u2/2rzoPc8gxFJkLj1WxNgooi9pnoc06jh0BjuXnamM4qlujZA== dependencies: - "@types/eslint-scope" "^3.7.3" - "@types/estree" "^1.0.5" + "@types/eslint-scope" "^3.7.7" + "@types/estree" "^1.0.6" "@webassemblyjs/ast" "^1.12.1" "@webassemblyjs/wasm-edit" "^1.12.1" "@webassemblyjs/wasm-parser" "^1.12.1" - acorn "^8.7.1" - acorn-import-assertions "^1.9.0" - browserslist "^4.21.10" + acorn "^8.14.0" + browserslist "^4.24.0" chrome-trace-event "^1.0.2" - enhanced-resolve "^5.16.0" + enhanced-resolve "^5.17.1" es-module-lexer "^1.2.1" eslint-scope "5.1.1" events "^3.2.0" From a08037f33f2fe00763032623e18ba049d19a024f Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Sun, 3 Nov 2024 11:01:49 +0900 Subject: [PATCH 13/29] GH-44618: [GLib] Add GArrowDecimal64Scalar (#44620) ### Rationale for this change The `arrow::Decimal64Scalar` has been released. GLib needs to implement `GArrowDecimal64Scalar`. ### What changes are included in this PR? Implement `GArrowDecimal64Scalar`. ### Are these changes tested? YES ### Are there any user-facing changes? NO * GitHub Issue: #44618 Authored-by: Hiroyuki Sato Signed-off-by: Sutou Kouhei --- c_glib/arrow-glib/scalar.cpp | 126 +++++++++++++++++++++++++++ c_glib/arrow-glib/scalar.h | 16 ++++ c_glib/test/test-decimal64-scalar.rb | 48 ++++++++++ 3 files changed, 190 insertions(+) create mode 100644 c_glib/test/test-decimal64-scalar.rb diff --git a/c_glib/arrow-glib/scalar.cpp b/c_glib/arrow-glib/scalar.cpp index f965b4970304b..57085a00c4b10 100644 --- a/c_glib/arrow-glib/scalar.cpp +++ b/c_glib/arrow-glib/scalar.cpp @@ -104,6 +104,8 @@ G_BEGIN_DECLS * #GArrowMonthDayNanoIntervalScalar is a class for the month day nano * intarval scalar. * + * #GArrowDecimal64Scalar is a class for a 64-bit decimal scalar. + * * #GArrowDecimal128Scalar is a class for a 128-bit decimal scalar. * * #GArrowDecimal256Scalar is a class for a 256-bit decimal scalar. @@ -1631,6 +1633,127 @@ garrow_month_day_nano_interval_scalar_get_value(GArrowMonthDayNanoIntervalScalar return priv->value; } +typedef struct GArrowDecimal64ScalarPrivate_ +{ + GArrowDecimal64 *value; +} GArrowDecimal64ScalarPrivate; + +G_DEFINE_TYPE_WITH_PRIVATE(GArrowDecimal64Scalar, + garrow_decimal64_scalar, + GARROW_TYPE_SCALAR) + +#define GARROW_DECIMAL64_SCALAR_GET_PRIVATE(obj) \ + static_cast( \ + garrow_decimal64_scalar_get_instance_private(GARROW_DECIMAL64_SCALAR(obj))) + +static void +garrow_decimal64_scalar_dispose(GObject *object) +{ + auto priv = GARROW_DECIMAL64_SCALAR_GET_PRIVATE(object); + + if (priv->value) { + g_object_unref(priv->value); + priv->value = NULL; + } + + G_OBJECT_CLASS(garrow_decimal64_scalar_parent_class)->dispose(object); +} + +static void +garrow_decimal64_scalar_set_property(GObject *object, + guint prop_id, + const GValue *value, + GParamSpec *pspec) +{ + auto priv = GARROW_DECIMAL64_SCALAR_GET_PRIVATE(object); + + switch (prop_id) { + case PROP_VALUE: + priv->value = GARROW_DECIMAL64(g_value_dup_object(value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void +garrow_decimal64_scalar_init(GArrowDecimal64Scalar *object) +{ +} + +static void +garrow_decimal64_scalar_class_init(GArrowDecimal64ScalarClass *klass) +{ + auto gobject_class = G_OBJECT_CLASS(klass); + + gobject_class->dispose = garrow_decimal64_scalar_dispose; + gobject_class->set_property = garrow_decimal64_scalar_set_property; + + GParamSpec *spec; + /** + * GArrowDecimal64Scalar:value: + * + * The value of the scalar. + * + * Since: 19.0.0 + */ + spec = g_param_spec_object( + "value", + "Value", + "The value of the scalar", + garrow_decimal64_get_type(), + static_cast(G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY)); + g_object_class_install_property(gobject_class, PROP_VALUE, spec); +} + +/** + * garrow_decimal64_scalar_new: + * @data_type: A #GArrowDecimal64DataType for this scalar. + * @value: The value of this scalar. + * + * Returns: A newly created #GArrowDecimal64Scalar. + * + * Since: 19.0.0 + */ +GArrowDecimal64Scalar * +garrow_decimal64_scalar_new(GArrowDecimal64DataType *data_type, GArrowDecimal64 *value) +{ + auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type)); + auto arrow_value = garrow_decimal64_get_raw(value); + auto arrow_scalar = std::static_pointer_cast( + std::make_shared(*arrow_value, arrow_data_type)); + return GARROW_DECIMAL64_SCALAR(garrow_scalar_new_raw(&arrow_scalar, + "scalar", + &arrow_scalar, + "data-type", + data_type, + "value", + value, + NULL)); +} + +/** + * garrow_decimal64_scalar_get_value: + * @scalar: A #GArrowDecimal64Scalar. + * + * Returns: (transfer none): The value of this scalar. + * + * Since: 19.0.0 + */ +GArrowDecimal64 * +garrow_decimal64_scalar_get_value(GArrowDecimal64Scalar *scalar) +{ + auto priv = GARROW_DECIMAL64_SCALAR_GET_PRIVATE(scalar); + if (!priv->value) { + auto arrow_scalar = std::static_pointer_cast( + garrow_scalar_get_raw(GARROW_SCALAR(scalar))); + auto arrow_value = std::make_shared(arrow_scalar->value); + priv->value = garrow_decimal64_new_raw(&arrow_value); + } + return priv->value; +} + typedef struct GArrowDecimal128ScalarPrivate_ { GArrowDecimal128 *value; @@ -2508,6 +2631,9 @@ garrow_scalar_new_raw_valist(std::shared_ptr *arrow_scalar, case arrow::Type::type::INTERVAL_MONTH_DAY_NANO: type = GARROW_TYPE_MONTH_DAY_NANO_INTERVAL_SCALAR; break; + case arrow::Type::type::DECIMAL64: + type = GARROW_TYPE_DECIMAL64_SCALAR; + break; case arrow::Type::type::DECIMAL128: type = GARROW_TYPE_DECIMAL128_SCALAR; break; diff --git a/c_glib/arrow-glib/scalar.h b/c_glib/arrow-glib/scalar.h index 5f9015d29c61c..c9de9958ad402 100644 --- a/c_glib/arrow-glib/scalar.h +++ b/c_glib/arrow-glib/scalar.h @@ -501,6 +501,22 @@ GARROW_AVAILABLE_IN_8_0 GArrowMonthDayNano * garrow_month_day_nano_interval_scalar_get_value(GArrowMonthDayNanoIntervalScalar *scalar); +#define GARROW_TYPE_DECIMAL64_SCALAR (garrow_decimal64_scalar_get_type()) +GARROW_AVAILABLE_IN_19_0 +G_DECLARE_DERIVABLE_TYPE( + GArrowDecimal64Scalar, garrow_decimal64_scalar, GARROW, DECIMAL64_SCALAR, GArrowScalar) +struct _GArrowDecimal64ScalarClass +{ + GArrowScalarClass parent_class; +}; + +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal64Scalar * +garrow_decimal64_scalar_new(GArrowDecimal64DataType *data_type, GArrowDecimal64 *value); +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal64 * +garrow_decimal64_scalar_get_value(GArrowDecimal64Scalar *scalar); + #define GARROW_TYPE_DECIMAL128_SCALAR (garrow_decimal128_scalar_get_type()) GARROW_AVAILABLE_IN_5_0 G_DECLARE_DERIVABLE_TYPE(GArrowDecimal128Scalar, diff --git a/c_glib/test/test-decimal64-scalar.rb b/c_glib/test/test-decimal64-scalar.rb new file mode 100644 index 0000000000000..fb6a308b6d19f --- /dev/null +++ b/c_glib/test/test-decimal64-scalar.rb @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestDecimal64Scalar < Test::Unit::TestCase + def setup + @data_type = Arrow::Decimal64DataType.new(8, 2) + @value = Arrow::Decimal64.new("23423445") + @scalar = Arrow::Decimal64Scalar.new(@data_type, @value) + end + + def test_data_type + assert_equal(@data_type, + @scalar.data_type) + end + + def test_valid? + assert do + @scalar.valid? + end + end + + def test_equal + assert_equal(Arrow::Decimal64Scalar.new(@data_type, @value), + @scalar) + end + + def test_to_s + assert_equal("234234.45", @scalar.to_s) + end + + def test_value + assert_equal(@value, @scalar.value) + end +end From 00e7c65e17f7d59f7c9954473b15b8ffae8dfd1a Mon Sep 17 00:00:00 2001 From: Hang Zheng <49890011+ripplehang@users.noreply.github.com> Date: Mon, 4 Nov 2024 02:21:50 +0800 Subject: [PATCH 14/29] GH-43535: [C++] Support the AWS S3 SSE-C encryption (#43601) ### Rationale for this change [server-side encryption with customer-provided keys](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) is an important security feature for aws s3, it's useful when user want to manager the encryption key themselves, say, they don't want the data to be exposed to the aws system admin, and ensure the object is safe even the ACCESS_KEY and SECRET_KEY is somehow leaked. Some comparison of S3 encryption options : https://www.linkedin.com/pulse/delusion-s3-encryption-benefits-ravi-ivaturi/ ### What changes are included in this PR? 1. Add the **sse_customer_key** member for S3Options to support [server-side encryption with customer-provided keys](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) (SSE-C keys). - The sse_customer_key was expected to be 256 bits (32 bytes) according to [aws doc](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html#specifying-s3-c-encryption) - The sse_customer_key was expected to be the raw key rather than base64 encoded value, arrow would calculate the base64 and MD5 on the fly. - By default the sse_customer_key is empty, and when the sse_customer_key is empty, there is no impact on the existing workflow. When the sse_customer_key is configured, it would require the aws sdk version to newer than 1.9.201. 2. Add the **tls_ca_file_path**, **tls_ca_dir_path** and **tls_verify_certificates** members for S3Options. - the tls_ca_file_path, tls_ca_dir_path member for S3Options would override the value configured by arrow::fs::FileSystemGlobalOptions. - for s3, according to [aws sdk doc](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/client-config.html), the tls_ca_file_path and tls_ca_dir_path only take effect in Linux, in order to support connect to the the storage server like minio with self-signed certificates on non-linux platform, we expose the tls_verify_certificates. 3. Refine the unit test to start the minio server with self-signed certificate on linux platform, so the unit test could cover the https case on linux, and http case on non-linux platform. ### Are these changes tested? Yes ### Are there any user-facing changes? Only additional members to S3Options. * GitHub Issue: #43535 Lead-authored-by: Hang Zheng Co-authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- cpp/src/arrow/filesystem/s3_internal.h | 87 ++++++++++ .../arrow/filesystem/s3_test_cert_internal.h | 77 +++++++++ cpp/src/arrow/filesystem/s3_test_util.cc | 84 ++++++++-- cpp/src/arrow/filesystem/s3_test_util.h | 11 +- cpp/src/arrow/filesystem/s3fs.cc | 69 ++++++-- cpp/src/arrow/filesystem/s3fs.h | 31 ++++ cpp/src/arrow/filesystem/s3fs_benchmark.cc | 4 +- cpp/src/arrow/filesystem/s3fs_test.cc | 150 +++++++++++++++++- cpp/src/arrow/testing/util.cc | 6 + cpp/src/arrow/testing/util.h | 4 + 10 files changed, 486 insertions(+), 37 deletions(-) create mode 100644 cpp/src/arrow/filesystem/s3_test_cert_internal.h diff --git a/cpp/src/arrow/filesystem/s3_internal.h b/cpp/src/arrow/filesystem/s3_internal.h index 54da3d5987e8a..772387e5fb66e 100644 --- a/cpp/src/arrow/filesystem/s3_internal.h +++ b/cpp/src/arrow/filesystem/s3_internal.h @@ -29,15 +29,38 @@ #include #include #include +#include #include #include "arrow/filesystem/filesystem.h" #include "arrow/filesystem/s3fs.h" #include "arrow/status.h" +#include "arrow/util/base64.h" #include "arrow/util/logging.h" #include "arrow/util/print.h" #include "arrow/util/string.h" +#ifndef ARROW_AWS_SDK_VERSION_CHECK +// AWS_SDK_VERSION_{MAJOR,MINOR,PATCH} are available since 1.9.7. +# if defined(AWS_SDK_VERSION_MAJOR) && defined(AWS_SDK_VERSION_MINOR) && \ + defined(AWS_SDK_VERSION_PATCH) +// Redundant "(...)" are for suppressing "Weird number of spaces at +// line-start. Are you using a 2-space indent? [whitespace/indent] +// [3]" errors... +# define ARROW_AWS_SDK_VERSION_CHECK(major, minor, patch) \ + ((AWS_SDK_VERSION_MAJOR > (major) || \ + (AWS_SDK_VERSION_MAJOR == (major) && AWS_SDK_VERSION_MINOR > (minor)) || \ + ((AWS_SDK_VERSION_MAJOR == (major) && AWS_SDK_VERSION_MINOR == (minor) && \ + AWS_SDK_VERSION_PATCH >= (patch))))) +# else +# define ARROW_AWS_SDK_VERSION_CHECK(major, minor, patch) 0 +# endif +#endif // !ARROW_AWS_SDK_VERSION_CHECK + +#if ARROW_AWS_SDK_VERSION_CHECK(1, 9, 201) +# define ARROW_S3_HAS_SSE_CUSTOMER_KEY +#endif + namespace arrow { namespace fs { namespace internal { @@ -291,6 +314,70 @@ class ConnectRetryStrategy : public Aws::Client::RetryStrategy { int32_t max_retry_duration_; }; +/// \brief calculate the MD5 of the input SSE-C key (raw key, not base64 encoded) +/// \param sse_customer_key is the input SSE-C key +/// \return the base64 encoded MD5 for the input key +inline Result CalculateSSECustomerKeyMD5( + const std::string& sse_customer_key) { + // The key needs to be 256 bits (32 bytes) according to + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html#specifying-s3-c-encryption + if (sse_customer_key.length() != 32) { + return Status::Invalid("32 bytes SSE-C key is expected"); + } + + // Convert the raw binary key to an Aws::String + Aws::String sse_customer_key_aws_string(sse_customer_key.data(), + sse_customer_key.length()); + + // Compute the MD5 hash of the raw binary key + Aws::Utils::ByteBuffer sse_customer_key_md5 = + Aws::Utils::HashingUtils::CalculateMD5(sse_customer_key_aws_string); + + // Base64-encode the MD5 hash + return arrow::util::base64_encode(std::string_view( + reinterpret_cast(sse_customer_key_md5.GetUnderlyingData()), + sse_customer_key_md5.GetLength())); +} + +struct SSECustomerKeyHeaders { + std::string sse_customer_key; + std::string sse_customer_key_md5; + std::string sse_customer_algorithm; +}; + +inline Result> GetSSECustomerKeyHeaders( + const std::string& sse_customer_key) { + if (sse_customer_key.empty()) { + return std::nullopt; + } +#ifdef ARROW_S3_HAS_SSE_CUSTOMER_KEY + ARROW_ASSIGN_OR_RAISE(auto md5, internal::CalculateSSECustomerKeyMD5(sse_customer_key)); + return SSECustomerKeyHeaders{arrow::util::base64_encode(sse_customer_key), md5, + "AES256"}; +#else + return Status::NotImplemented( + "SSE customer key not supported by this version of the AWS SDK"); +#endif +} + +template +Status SetSSECustomerKey(S3RequestType* request, const std::string& sse_customer_key) { + ARROW_ASSIGN_OR_RAISE(auto maybe_headers, GetSSECustomerKeyHeaders(sse_customer_key)); + if (!maybe_headers.has_value()) { + return Status::OK(); + } +#ifdef ARROW_S3_HAS_SSE_CUSTOMER_KEY + auto headers = std::move(maybe_headers).value(); + request->SetSSECustomerKey(headers.sse_customer_key); + request->SetSSECustomerKeyMD5(headers.sse_customer_key_md5); + request->SetSSECustomerAlgorithm(headers.sse_customer_algorithm); + return Status::OK(); +#else + return Status::NotImplemented( + "SSE customer key not supported by this version of the AWS SDK"); +#endif +} + } // namespace internal } // namespace fs } // namespace arrow diff --git a/cpp/src/arrow/filesystem/s3_test_cert_internal.h b/cpp/src/arrow/filesystem/s3_test_cert_internal.h new file mode 100644 index 0000000000000..0a69ade7d0e5c --- /dev/null +++ b/cpp/src/arrow/filesystem/s3_test_cert_internal.h @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +namespace arrow::fs { +// The below two static strings are generated according to +// https://github.com/minio/minio/tree/RELEASE.2024-09-22T00-33-43Z/docs/tls#323-generate-a-self-signed-certificate +// `openssl req -new -x509 -nodes -days 36500 -keyout private.key -out public.crt -config +// openssl.conf` +static constexpr const char* kMinioPrivateKey = R"(-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCqwKYHsTSciGqP +uU3qkTWpnXIi3iC0eeW7JSzJHGFs880WdR5JdK4WufPK+1xzgiYjMEPfAcuSWz3b +qYyCI61q+a9Iu2nj7cFTW9bfZrmWlnI0YOLJc+q0AAdAjF1lvRKenH8tbjz/2jyl +i/cYQ+I5Tg4nngrX8OmOfluNzwD/nwGLq6/DVbzDUdPI9q1XtVT/0Vf7qwbDG1HD +NkIzKT5B+YdSLaOCRYNK3x7RPsfazKIBrTmRy1v454wKe8TjTmTB7+m5wKqfCJcq +lI253WHcK0lsw6zCNtX/kahPAvm/8mniPolW4qxoD6xwebgMVkrNTs3ztcPIG9O4 +pmCbATijAgMBAAECggEACL5swiAU7Z8etdVrZAOjl9f0LEzrp9JGLVst++50Hrwt +WGUO8/wBnjBPh6lvhoq3oT2rfBP/dLMva7w28cMZ8kxu6W6PcZiPOdGOI0qDXm69 +0mjTtDU3Y5hMxsVpUvhnp6+j45Otk/x89o1ATgHL59tTZjv1mjFABIf78DsVdgF9 +CMi2q6Lv7NLftieyWmz1K3p109z9+xkDNSOkVrv1JFChviKqWgIS0rdFjySvTgoy +rHYT+TweDliKJrZCeoUJmNB0uVW/dM9lXhcvkvkJZKPPurylx1oH5a7K/sWFPf7A +Ed1vjvZQFlaXu/bOUUSOZtkErAir/oCxrUDsHxGsAQKBgQDZghyy7jNGNdjZe1Xs +On1ZVgIS3Nt+OLGCVH7tTsfZsCOb+SkrhB1RQva3YzPMfgoZScI9+bN/pRVf49Pj +qGEHkW/wozutUve7UMzeTOm1aWxUuaKSrmYST7muvAnlYEtO7agd0wrcusYXlMoG +KQwghkufO9I7wXcrudMKXZalIwKBgQDI+FaUwhgfThkgq6bRbdMEeosgohrCM9Wm +E5JMePQq4VaGcgGveWUoNOgT8kvJa0qQwQOqLZj7kUIdj+SCRt0u+Wu3p5IMqdOq +6tMnLNQ3wzUC2KGFLSfISR3L/bo5Bo6Jqz4hVtjMk3PV9bu50MNTNaofYb2xlf/f +/WgiEG0WgQKBgAr8RVLMMQ7EvXUOg6Jwuc//Rg+J1BQl7OE2P0rhBbr66HGCPhAS +liB6j1dnzT/wxbXNQeA7clNqFRBIw3TmFjB5qfuvYt44KIbvZ8l6fPtKncwRrCJY +aJNYL3qhyKYrHOKZojoPZKcNT9/1BdcVz6T842jhbpbSCKDOu9f0Lh2dAoGATZeM +Hh0eISAPFY0QeDV1znnds3jC6g4HQ/q0dnAQnWmo9XmY6v3sr2xV2jWnSxnwjRjo +aFD4itBXfYBr0ly30wYbr6mz+s2q2oeVhL+LJAhrNDEdk4SOooaQSY0p1BCTAdYq +w8Z7J+kaRRZ+J0zRzROgHkOncKQgSYPWK6i55YECgYAC+ECrHhUlPsfusjKpFsEe +stW1HCt3wXtKQn6SJ6IAesbxwALZS6Da/ZC2x1mdBHS3GwWvtGLc0BPnPVfJjr9V +m82qkgJ+p5d7qp7pRA7SFD+5809yVqRnEF3rSLafgGet9ah0ZjZvQ3fwnYZNnNH9 +t9pJcv2E5xY7/nFNIorpKg== +-----END PRIVATE KEY----- +)"; + +static constexpr const char* kMinioCert = R"(-----BEGIN CERTIFICATE----- +MIIDiTCCAnGgAwIBAgIUXbHZ6FAhKSXg4WSGUQySlSyE4U0wDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlZBMQ4wDAYDVQQHDAVBcnJvdzEO +MAwGA1UECgwFQXJyb3cxDjAMBgNVBAsMBUFycm93MRMwEQYDVQQDDApBcnJyb3dU +ZXN0MB4XDTI0MDkyNDA5MzUxNloXDTM0MDkyMjA5MzUxNlowXzELMAkGA1UEBhMC +VVMxCzAJBgNVBAgMAlZBMQ4wDAYDVQQHDAVBcnJvdzEOMAwGA1UECgwFQXJyb3cx +DjAMBgNVBAsMBUFycm93MRMwEQYDVQQDDApBcnJyb3dUZXN0MIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqsCmB7E0nIhqj7lN6pE1qZ1yIt4gtHnluyUs +yRxhbPPNFnUeSXSuFrnzyvtcc4ImIzBD3wHLkls926mMgiOtavmvSLtp4+3BU1vW +32a5lpZyNGDiyXPqtAAHQIxdZb0Snpx/LW48/9o8pYv3GEPiOU4OJ54K1/Dpjn5b +jc8A/58Bi6uvw1W8w1HTyPatV7VU/9FX+6sGwxtRwzZCMyk+QfmHUi2jgkWDSt8e +0T7H2syiAa05kctb+OeMCnvE405kwe/pucCqnwiXKpSNud1h3CtJbMOswjbV/5Go +TwL5v/Jp4j6JVuKsaA+scHm4DFZKzU7N87XDyBvTuKZgmwE4owIDAQABoz0wOzAa +BgNVHREEEzARhwR/AAABgglsb2NhbGhvc3QwHQYDVR0OBBYEFOUNqUSfROf1dz3o +hAVBhgd3UIvKMA0GCSqGSIb3DQEBCwUAA4IBAQBSwWJ2dSw3jlHU0l2V3ozqthTt +XFo07AyWGw8AWNCM6mQ+GKBf0JJ1d7e4lyTf2lCobknS94EgGPORWeiucKYAoCjS +dh1eKGsSevz1rNbp7wsO7DoiRPciK+S95DbsPowloGI6fvOeE12Cf1udeNIpEYWs +OBFwN0HxfYqdPALCtw7l0icpTrJ2Us06UfL9kbkdZwQhXvOscG7JDRtNjBxl9XNm +TFeMNKROmrEPCWaYr6MJ+ItHtb5Cawapea4THz9GCjR9eLq2CbMqLezZ8xBHPzc4 +ixI2l0uCfg7ZUSA+90yaScc7bhEQ8CMiPtJgNKaKIqB58DpY7028xJpW7Ma2 +-----END CERTIFICATE----- +)"; +} // namespace arrow::fs diff --git a/cpp/src/arrow/filesystem/s3_test_util.cc b/cpp/src/arrow/filesystem/s3_test_util.cc index db0c60f2e80f2..0cfe038599cfe 100644 --- a/cpp/src/arrow/filesystem/s3_test_util.cc +++ b/cpp/src/arrow/filesystem/s3_test_util.cc @@ -19,6 +19,7 @@ # include #endif +#include "arrow/filesystem/s3_test_cert_internal.h" #include "arrow/filesystem/s3_test_util.h" #include "arrow/filesystem/s3fs.h" #include "arrow/testing/process.h" @@ -31,6 +32,11 @@ namespace arrow { namespace fs { +using ::arrow::internal::FileClose; +using ::arrow::internal::FileDescriptor; +using ::arrow::internal::FileOpenWritable; +using ::arrow::internal::FileWrite; +using ::arrow::internal::PlatformFilename; using ::arrow::internal::TemporaryDir; namespace { @@ -44,16 +50,16 @@ const char* kEnvConnectString = "ARROW_TEST_S3_CONNECT_STRING"; const char* kEnvAccessKey = "ARROW_TEST_S3_ACCESS_KEY"; const char* kEnvSecretKey = "ARROW_TEST_S3_SECRET_KEY"; -std::string GenerateConnectString() { return GetListenAddress(); } - } // namespace struct MinioTestServer::Impl { std::unique_ptr temp_dir_; + std::unique_ptr temp_dir_ca_; std::string connect_string_; std::string access_key_ = kMinioAccessKey; std::string secret_key_ = kMinioSecretKey; std::unique_ptr server_process_; + std::string scheme_ = "http"; }; MinioTestServer::MinioTestServer() : impl_(new Impl) {} @@ -69,7 +75,41 @@ std::string MinioTestServer::access_key() const { return impl_->access_key_; } std::string MinioTestServer::secret_key() const { return impl_->secret_key_; } -Status MinioTestServer::Start() { +std::string MinioTestServer::ca_dir_path() const { + return impl_->temp_dir_ca_->path().ToString(); +} + +std::string MinioTestServer::ca_file_path() const { + return impl_->temp_dir_ca_->path().ToString() + "/public.crt"; +} + +std::string MinioTestServer::scheme() const { return impl_->scheme_; } + +Status MinioTestServer::GenerateCertificateFile() { + // create the dedicated folder for certificate file, rather than reuse the data + // folder, since there is test case to check whether the folder is empty. + ARROW_ASSIGN_OR_RAISE(impl_->temp_dir_ca_, TemporaryDir::Make("s3fs-test-ca-")); + + ARROW_ASSIGN_OR_RAISE(auto public_crt_file, + PlatformFilename::FromString(ca_dir_path() + "/public.crt")); + ARROW_ASSIGN_OR_RAISE(auto public_cert_fd, FileOpenWritable(public_crt_file)); + ARROW_RETURN_NOT_OK(FileWrite(public_cert_fd.fd(), + reinterpret_cast(kMinioCert), + strlen(kMinioCert))); + ARROW_RETURN_NOT_OK(public_cert_fd.Close()); + + ARROW_ASSIGN_OR_RAISE(auto private_key_file, + PlatformFilename::FromString(ca_dir_path() + "/private.key")); + ARROW_ASSIGN_OR_RAISE(auto private_key_fd, FileOpenWritable(private_key_file)); + ARROW_RETURN_NOT_OK(FileWrite(private_key_fd.fd(), + reinterpret_cast(kMinioPrivateKey), + strlen(kMinioPrivateKey))); + ARROW_RETURN_NOT_OK(private_key_fd.Close()); + + return Status::OK(); +} + +Status MinioTestServer::Start(bool enable_tls) { const char* connect_str = std::getenv(kEnvConnectString); const char* access_key = std::getenv(kEnvAccessKey); const char* secret_key = std::getenv(kEnvSecretKey); @@ -88,12 +128,27 @@ Status MinioTestServer::Start() { impl_->server_process_->SetEnv("MINIO_SECRET_KEY", kMinioSecretKey); // Disable the embedded console (one less listening address to care about) impl_->server_process_->SetEnv("MINIO_BROWSER", "off"); - impl_->connect_string_ = GenerateConnectString(); - ARROW_RETURN_NOT_OK(impl_->server_process_->SetExecutable(kMinioExecutableName)); // NOTE: --quiet makes startup faster by suppressing remote version check - impl_->server_process_->SetArgs({"server", "--quiet", "--compat", "--address", - impl_->connect_string_, - impl_->temp_dir_->path().ToString()}); + std::vector minio_args({"server", "--quiet", "--compat"}); + if (enable_tls) { + ARROW_RETURN_NOT_OK(GenerateCertificateFile()); + minio_args.emplace_back("--certs-dir"); + minio_args.emplace_back(ca_dir_path()); + impl_->scheme_ = "https"; + // With TLS enabled, we need the connection hostname to match the certificate's + // subject name. This also constrains the actual listening IP address. + impl_->connect_string_ = GetListenAddress("localhost"); + } else { + // Without TLS enabled, we want to minimize the likelihood of address collisions + // by varying the listening IP address (note that most tests don't enable TLS). + impl_->connect_string_ = GetListenAddress(); + } + minio_args.emplace_back("--address"); + minio_args.emplace_back(impl_->connect_string_); + minio_args.emplace_back(impl_->temp_dir_->path().ToString()); + + ARROW_RETURN_NOT_OK(impl_->server_process_->SetExecutable(kMinioExecutableName)); + impl_->server_process_->SetArgs(minio_args); ARROW_RETURN_NOT_OK(impl_->server_process_->Execute()); return Status::OK(); } @@ -105,24 +160,29 @@ Status MinioTestServer::Stop() { struct MinioTestEnvironment::Impl { std::function>()> server_generator_; + bool enable_tls_; + + explicit Impl(bool enable_tls) : enable_tls_(enable_tls) {} Result> LaunchOneServer() { auto server = std::make_shared(); - RETURN_NOT_OK(server->Start()); + RETURN_NOT_OK(server->Start(enable_tls_)); return server; } }; -MinioTestEnvironment::MinioTestEnvironment() : impl_(new Impl) {} +MinioTestEnvironment::MinioTestEnvironment(bool enable_tls) + : impl_(new Impl(enable_tls)) {} MinioTestEnvironment::~MinioTestEnvironment() = default; void MinioTestEnvironment::SetUp() { auto pool = ::arrow::internal::GetCpuThreadPool(); - auto launch_one_server = []() -> Result> { + auto launch_one_server = + [enable_tls = impl_->enable_tls_]() -> Result> { auto server = std::make_shared(); - RETURN_NOT_OK(server->Start()); + RETURN_NOT_OK(server->Start(enable_tls)); return server; }; impl_->server_generator_ = [pool, launch_one_server]() { diff --git a/cpp/src/arrow/filesystem/s3_test_util.h b/cpp/src/arrow/filesystem/s3_test_util.h index e270a6e1c469a..0a89a7a9d5a15 100644 --- a/cpp/src/arrow/filesystem/s3_test_util.h +++ b/cpp/src/arrow/filesystem/s3_test_util.h @@ -40,7 +40,7 @@ class MinioTestServer { MinioTestServer(); ~MinioTestServer(); - Status Start(); + Status Start(bool enable_tls = false); Status Stop(); @@ -50,7 +50,14 @@ class MinioTestServer { std::string secret_key() const; + std::string ca_dir_path() const; + + std::string ca_file_path() const; + + std::string scheme() const; + private: + Status GenerateCertificateFile(); struct Impl; std::unique_ptr impl_; }; @@ -60,7 +67,7 @@ class MinioTestServer { class MinioTestEnvironment : public ::testing::Environment { public: - MinioTestEnvironment(); + explicit MinioTestEnvironment(bool enable_tls = false); ~MinioTestEnvironment(); void SetUp() override; diff --git a/cpp/src/arrow/filesystem/s3fs.cc b/cpp/src/arrow/filesystem/s3fs.cc index 13d6ead6ef686..ee47e1c702073 100644 --- a/cpp/src/arrow/filesystem/s3fs.cc +++ b/cpp/src/arrow/filesystem/s3fs.cc @@ -160,6 +160,7 @@ using internal::IsNotFound; using internal::OutcomeToResult; using internal::OutcomeToStatus; using internal::S3Backend; +using internal::SetSSECustomerKey; using internal::ToAwsString; using internal::ToURLEncodedAwsString; @@ -403,6 +404,13 @@ Result S3Options::FromUri(const Uri& uri, std::string* out_path) { } else if (kv.first == "allow_bucket_deletion") { ARROW_ASSIGN_OR_RAISE(options.allow_bucket_deletion, ::arrow::internal::ParseBoolean(kv.second)); + } else if (kv.first == "tls_ca_file_path") { + options.tls_ca_file_path = kv.second; + } else if (kv.first == "tls_ca_dir_path") { + options.tls_ca_dir_path = kv.second; + } else if (kv.first == "tls_verify_certificates") { + ARROW_ASSIGN_OR_RAISE(options.tls_verify_certificates, + ::arrow::internal::ParseBoolean(kv.second)); } else { return Status::Invalid("Unexpected query parameter in S3 URI: '", kv.first, "'"); } @@ -439,7 +447,11 @@ bool S3Options::Equals(const S3Options& other) const { background_writes == other.background_writes && allow_bucket_creation == other.allow_bucket_creation && allow_bucket_deletion == other.allow_bucket_deletion && - default_metadata_equals && GetAccessKey() == other.GetAccessKey() && + tls_ca_file_path == other.tls_ca_file_path && + tls_ca_dir_path == other.tls_ca_dir_path && + tls_verify_certificates == other.tls_verify_certificates && + sse_customer_key == other.sse_customer_key && default_metadata_equals && + GetAccessKey() == other.GetAccessKey() && GetSecretKey() == other.GetSecretKey() && GetSessionToken() == other.GetSessionToken()); } @@ -1125,12 +1137,17 @@ class ClientBuilder { } else { client_config_.retryStrategy = std::make_shared(); } - if (!internal::global_options.tls_ca_file_path.empty()) { + if (!options_.tls_ca_file_path.empty()) { + client_config_.caFile = ToAwsString(options_.tls_ca_file_path); + } else if (!internal::global_options.tls_ca_file_path.empty()) { client_config_.caFile = ToAwsString(internal::global_options.tls_ca_file_path); } - if (!internal::global_options.tls_ca_dir_path.empty()) { + if (!options_.tls_ca_dir_path.empty()) { + client_config_.caPath = ToAwsString(options_.tls_ca_dir_path); + } else if (!internal::global_options.tls_ca_dir_path.empty()) { client_config_.caPath = ToAwsString(internal::global_options.tls_ca_dir_path); } + client_config_.verifySSL = options_.tls_verify_certificates; // Set proxy options if provided if (!options_.proxy_options.scheme.empty()) { @@ -1292,11 +1309,14 @@ Aws::IOStreamFactory AwsWriteableStreamFactory(void* data, int64_t nbytes) { } Result GetObjectRange(Aws::S3::S3Client* client, - const S3Path& path, int64_t start, - int64_t length, void* out) { + const S3Path& path, + const std::string& sse_customer_key, + int64_t start, int64_t length, + void* out) { S3Model::GetObjectRequest req; req.SetBucket(ToAwsString(path.bucket)); req.SetKey(ToAwsString(path.key)); + RETURN_NOT_OK(SetSSECustomerKey(&req, sse_customer_key)); req.SetRange(ToAwsString(FormatRange(start, length))); req.SetResponseStreamFactory(AwsWriteableStreamFactory(out, length)); return OutcomeToResult("GetObject", client->GetObject(req)); @@ -1433,11 +1453,13 @@ bool IsDirectory(std::string_view key, const S3Model::HeadObjectResult& result) class ObjectInputFile final : public io::RandomAccessFile { public: ObjectInputFile(std::shared_ptr holder, const io::IOContext& io_context, - const S3Path& path, int64_t size = kNoSize) + const S3Path& path, int64_t size = kNoSize, + const std::string& sse_customer_key = "") : holder_(std::move(holder)), io_context_(io_context), path_(path), - content_length_(size) {} + content_length_(size), + sse_customer_key_(sse_customer_key) {} Status Init() { // Issue a HEAD Object to get the content-length and ensure any @@ -1450,6 +1472,7 @@ class ObjectInputFile final : public io::RandomAccessFile { S3Model::HeadObjectRequest req; req.SetBucket(ToAwsString(path_.bucket)); req.SetKey(ToAwsString(path_.key)); + RETURN_NOT_OK(SetSSECustomerKey(&req, sse_customer_key_)); ARROW_ASSIGN_OR_RAISE(auto client_lock, holder_->Lock()); auto outcome = client_lock.Move()->HeadObject(req); @@ -1534,9 +1557,9 @@ class ObjectInputFile final : public io::RandomAccessFile { // Read the desired range of bytes ARROW_ASSIGN_OR_RAISE(auto client_lock, holder_->Lock()); - ARROW_ASSIGN_OR_RAISE( - S3Model::GetObjectResult result, - GetObjectRange(client_lock.get(), path_, position, nbytes, out)); + ARROW_ASSIGN_OR_RAISE(S3Model::GetObjectResult result, + GetObjectRange(client_lock.get(), path_, sse_customer_key_, + position, nbytes, out)); auto& stream = result.GetBody(); stream.ignore(nbytes); @@ -1584,6 +1607,7 @@ class ObjectInputFile final : public io::RandomAccessFile { int64_t pos_ = 0; int64_t content_length_ = kNoSize; std::shared_ptr metadata_; + std::string sse_customer_key_; }; // Upload size per part. While AWS and Minio support different sizes for each @@ -1620,7 +1644,8 @@ class ObjectOutputStream final : public io::OutputStream { metadata_(metadata), default_metadata_(options.default_metadata), background_writes_(options.background_writes), - allow_delayed_open_(options.allow_delayed_open) {} + allow_delayed_open_(options.allow_delayed_open), + sse_customer_key_(options.sse_customer_key) {} ~ObjectOutputStream() override { // For compliance with the rest of the IO stack, Close rather than Abort, @@ -1668,6 +1693,7 @@ class ObjectOutputStream final : public io::OutputStream { S3Model::CreateMultipartUploadRequest req; req.SetBucket(ToAwsString(path_.bucket)); req.SetKey(ToAwsString(path_.key)); + RETURN_NOT_OK(SetSSECustomerKey(&req, sse_customer_key_)); RETURN_NOT_OK(SetMetadataInRequest(&req)); auto outcome = client_lock.Move()->CreateMultipartUpload(req); @@ -1771,6 +1797,7 @@ class ObjectOutputStream final : public io::OutputStream { req.SetKey(ToAwsString(path_.key)); req.SetUploadId(multipart_upload_id_); req.SetMultipartUpload(std::move(completed_upload)); + RETURN_NOT_OK(SetSSECustomerKey(&req, sse_customer_key_)); auto outcome = client_lock.Move()->CompleteMultipartUploadWithErrorFixup(std::move(req)); @@ -1958,6 +1985,7 @@ class ObjectOutputStream final : public io::OutputStream { req.SetKey(ToAwsString(path_.key)); req.SetBody(std::make_shared(data, nbytes)); req.SetContentLength(nbytes); + RETURN_NOT_OK(SetSSECustomerKey(&req, sse_customer_key_)); if (!background_writes_) { req.SetBody(std::make_shared(data, nbytes)); @@ -2171,6 +2199,7 @@ class ObjectOutputStream final : public io::OutputStream { Future<> pending_uploads_completed = Future<>::MakeFinished(Status::OK()); }; std::shared_ptr upload_state_; + std::string sse_customer_key_; }; // This function assumes info->path() is already set @@ -2339,6 +2368,17 @@ class S3FileSystem::Impl : public std::enable_shared_from_this(holder_, fs->io_context(), path); + auto ptr = std::make_shared(holder_, fs->io_context(), path, kNoSize, + fs->options().sse_customer_key); RETURN_NOT_OK(ptr->Init()); return ptr; } @@ -3002,8 +3043,8 @@ class S3FileSystem::Impl : public std::enable_shared_from_this(holder_, fs->io_context(), path, info.size()); + auto ptr = std::make_shared( + holder_, fs->io_context(), path, info.size(), fs->options().sse_customer_key); RETURN_NOT_OK(ptr->Init()); return ptr; } diff --git a/cpp/src/arrow/filesystem/s3fs.h b/cpp/src/arrow/filesystem/s3fs.h index 85d5ff8fed553..ac6342f26a304 100644 --- a/cpp/src/arrow/filesystem/s3fs.h +++ b/cpp/src/arrow/filesystem/s3fs.h @@ -196,6 +196,37 @@ struct ARROW_EXPORT S3Options { /// delay between retries. std::shared_ptr retry_strategy; + /// Optional customer-provided key for server-side encryption (SSE-C). + /// + /// This should be the 32-byte AES-256 key, unencoded. + std::string sse_customer_key; + + /// Optional path to a single PEM file holding all TLS CA certificates + /// + /// If empty, global filesystem options will be used (see FileSystemGlobalOptions); + /// if the corresponding global filesystem option is also empty, the underlying + /// TLS library's defaults will be used. + /// + /// Note this option may be ignored on some systems (Windows, macOS). + std::string tls_ca_file_path; + + /// Optional path to a directory holding TLS CA + /// + /// The given directory should contain CA certificates as individual PEM files + /// named along the OpenSSL "hashed" format. + /// + /// If empty, global filesystem options will be used (see FileSystemGlobalOptions); + /// if the corresponding global filesystem option is also empty, the underlying + /// TLS library's defaults will be used. + /// + /// Note this option may be ignored on some systems (Windows, macOS). + std::string tls_ca_dir_path; + + /// Whether to verify the S3 endpoint's TLS certificate + /// + /// This option applies if the scheme is "https". + bool tls_verify_certificates = true; + S3Options(); /// Configure with the default AWS credentials provider chain. diff --git a/cpp/src/arrow/filesystem/s3fs_benchmark.cc b/cpp/src/arrow/filesystem/s3fs_benchmark.cc index 212164296398b..b7b6dda64192a 100644 --- a/cpp/src/arrow/filesystem/s3fs_benchmark.cc +++ b/cpp/src/arrow/filesystem/s3fs_benchmark.cc @@ -61,7 +61,7 @@ class MinioFixture : public benchmark::Fixture { public: void SetUp(const ::benchmark::State& state) override { minio_.reset(new MinioTestServer()); - ASSERT_OK(minio_->Start()); + ASSERT_OK(minio_->Start(/*enable_tls=*/false)); const char* region_str = std::getenv(kEnvAwsRegion); if (region_str) { @@ -110,7 +110,7 @@ class MinioFixture : public benchmark::Fixture { void MakeFileSystem() { options_.ConfigureAccessKey(minio_->access_key(), minio_->secret_key()); - options_.scheme = "http"; + options_.scheme = minio_->scheme(); if (!region_.empty()) { options_.region = region_; } diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index 43091aaa986d9..3082ecb7843b8 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -71,6 +71,12 @@ #include "arrow/util/range.h" #include "arrow/util/string.h" +// TLS tests require the ability to set a custom CA file when initiating S3 client +// connections, which the AWS SDK currently only supports on Linux. +#if defined(__linux__) +# define ENABLE_TLS_TESTS +#endif // Linux + namespace arrow { namespace fs { @@ -80,6 +86,7 @@ using ::arrow::internal::ToChars; using ::arrow::internal::Zip; using ::arrow::util::UriEscape; +using ::arrow::fs::internal::CalculateSSECustomerKeyMD5; using ::arrow::fs::internal::ConnectRetryStrategy; using ::arrow::fs::internal::ErrorToStatus; using ::arrow::fs::internal::OutcomeToStatus; @@ -94,8 +101,15 @@ ::testing::Environment* s3_env = ::testing::AddGlobalTestEnvironment(new S3Envir ::testing::Environment* minio_env = ::testing::AddGlobalTestEnvironment(new MinioTestEnvironment); -MinioTestEnvironment* GetMinioEnv() { - return ::arrow::internal::checked_cast(minio_env); +::testing::Environment* minio_env_https = + ::testing::AddGlobalTestEnvironment(new MinioTestEnvironment(/*enable_tls=*/true)); + +MinioTestEnvironment* GetMinioEnv(bool enable_tls) { + if (enable_tls) { + return ::arrow::internal::checked_cast(minio_env_https); + } else { + return ::arrow::internal::checked_cast(minio_env); + } } class ShortRetryStrategy : public S3RetryStrategy { @@ -202,10 +216,15 @@ class S3TestMixin : public AwsTestMixin { protected: Status InitServerAndClient() { - ARROW_ASSIGN_OR_RAISE(minio_, GetMinioEnv()->GetOneServer()); + ARROW_ASSIGN_OR_RAISE(minio_, GetMinioEnv(enable_tls_)->GetOneServer()); client_config_.reset(new Aws::Client::ClientConfiguration()); client_config_->endpointOverride = ToAwsString(minio_->connect_string()); - client_config_->scheme = Aws::Http::Scheme::HTTP; + if (minio_->scheme() == "https") { + client_config_->scheme = Aws::Http::Scheme::HTTPS; + client_config_->caFile = ToAwsString(minio_->ca_file_path()); + } else { + client_config_->scheme = Aws::Http::Scheme::HTTP; + } client_config_->retryStrategy = std::make_shared(kRetryInterval, kMaxRetryDuration); credentials_ = {ToAwsString(minio_->access_key()), ToAwsString(minio_->secret_key())}; @@ -224,6 +243,11 @@ class S3TestMixin : public AwsTestMixin { std::unique_ptr client_config_; Aws::Auth::AWSCredentials credentials_; std::unique_ptr client_; + // Use plain HTTP by default, as this allows us to listen on different loopback + // addresses and thus minimize the risk of address reuse (HTTPS requires the + // hostname to match the certificate's subject name, constraining us to a + // single loopback address). + bool enable_tls_ = false; }; void AssertGetObject(Aws::S3::Model::GetObjectResult& result, @@ -249,6 +273,27 @@ void AssertObjectContents(Aws::S3::S3Client* client, const std::string& bucket, AssertGetObject(result, expected); } +//////////////////////////////////////////////////////////////////////////// +// Misc tests + +class InternalsTest : public AwsTestMixin {}; + +TEST_F(InternalsTest, CalculateSSECustomerKeyMD5) { + ASSERT_RAISES(Invalid, CalculateSSECustomerKeyMD5("")); // invalid length + ASSERT_RAISES(Invalid, + CalculateSSECustomerKeyMD5( + "1234567890123456789012345678901234567890")); // invalid length + // valid case, with some non-ASCII character and a null byte in the sse_customer_key + char sse_customer_key[32] = {}; + sse_customer_key[0] = '\x40'; // '@' character + sse_customer_key[1] = '\0'; // null byte + sse_customer_key[2] = '\xFF'; // non-ASCII + sse_customer_key[31] = '\xFA'; // non-ASCII + std::string sse_customer_key_string(sse_customer_key, sizeof(sse_customer_key)); + ASSERT_OK_AND_ASSIGN(auto md5, CalculateSSECustomerKeyMD5(sse_customer_key_string)) + ASSERT_EQ(md5, "97FTa6lj0hE7lshKdBy61g=="); // valid case +} + //////////////////////////////////////////////////////////////////////////// // S3Options tests @@ -300,6 +345,17 @@ TEST_F(S3OptionsTest, FromUri) { ASSERT_EQ(options.scheme, "http"); ASSERT_EQ(options.endpoint_override, "localhost"); ASSERT_EQ(path, "mybucket/foo/bar"); + ASSERT_EQ(options.tls_verify_certificates, true); + + // Explicit tls related configuration + ASSERT_OK_AND_ASSIGN( + options, + S3Options::FromUri("s3://mybucket/foo/bar/?tls_ca_dir_path=/test&tls_ca_file_path=/" + "test/test.pem&tls_verify_certificates=false", + &path)); + ASSERT_EQ(options.tls_ca_dir_path, "/test"); + ASSERT_EQ(options.tls_ca_file_path, "/test/test.pem"); + ASSERT_EQ(options.tls_verify_certificates, false); // Missing bucket name ASSERT_RAISES(Invalid, S3Options::FromUri("s3:///foo/bar/", &path)); @@ -443,6 +499,9 @@ class TestS3FS : public S3TestMixin { // Most tests will create buckets options_.allow_bucket_creation = true; options_.allow_bucket_deletion = true; + if (enable_tls_) { + options_.tls_ca_file_path = minio_->ca_file_path(); + } MakeFileSystem(); // Set up test bucket { @@ -532,7 +591,7 @@ class TestS3FS : public S3TestMixin { Result> MakeNewFileSystem( io::IOContext io_context = io::default_io_context()) { options_.ConfigureAccessKey(minio_->access_key(), minio_->secret_key()); - options_.scheme = "http"; + options_.scheme = minio_->scheme(); options_.endpoint_override = minio_->connect_string(); if (!options_.retry_strategy) { options_.retry_strategy = std::make_shared(); @@ -1298,6 +1357,82 @@ TEST_F(TestS3FS, OpenInputFile) { ASSERT_RAISES(IOError, file->Seek(10)); } +// Minio only allows Server Side Encryption on HTTPS client connections. +#ifdef ENABLE_TLS_TESTS +class TestS3FSHTTPS : public TestS3FS { + public: + void SetUp() override { + enable_tls_ = true; + TestS3FS::SetUp(); + } +}; + +TEST_F(TestS3FSHTTPS, SSECustomerKeyMatch) { + // normal write/read with correct SSE-C key + std::shared_ptr stream; + options_.sse_customer_key = "12345678123456781234567812345678"; + for (const auto& allow_delayed_open : {false, true}) { + ARROW_SCOPED_TRACE("allow_delayed_open = ", allow_delayed_open); + options_.allow_delayed_open = allow_delayed_open; + MakeFileSystem(); + ASSERT_OK_AND_ASSIGN(stream, fs_->OpenOutputStream("bucket/newfile_with_sse_c")); + ASSERT_OK(stream->Write("some")); + ASSERT_OK(stream->Close()); + ASSERT_OK_AND_ASSIGN(auto file, fs_->OpenInputFile("bucket/newfile_with_sse_c")); + ASSERT_OK_AND_ASSIGN(auto buf, file->Read(5)); + AssertBufferEqual(*buf, "some"); + ASSERT_OK(RestoreTestBucket()); + } +} + +TEST_F(TestS3FSHTTPS, SSECustomerKeyMismatch) { + std::shared_ptr stream; + for (const auto& allow_delayed_open : {false, true}) { + ARROW_SCOPED_TRACE("allow_delayed_open = ", allow_delayed_open); + options_.allow_delayed_open = allow_delayed_open; + options_.sse_customer_key = "12345678123456781234567812345678"; + MakeFileSystem(); + ASSERT_OK_AND_ASSIGN(stream, fs_->OpenOutputStream("bucket/newfile_with_sse_c")); + ASSERT_OK(stream->Write("some")); + ASSERT_OK(stream->Close()); + options_.sse_customer_key = "87654321876543218765432187654321"; + MakeFileSystem(); + ASSERT_RAISES(IOError, fs_->OpenInputFile("bucket/newfile_with_sse_c")); + ASSERT_OK(RestoreTestBucket()); + } +} + +TEST_F(TestS3FSHTTPS, SSECustomerKeyMissing) { + std::shared_ptr stream; + for (const auto& allow_delayed_open : {false, true}) { + ARROW_SCOPED_TRACE("allow_delayed_open = ", allow_delayed_open); + options_.allow_delayed_open = allow_delayed_open; + options_.sse_customer_key = "12345678123456781234567812345678"; + MakeFileSystem(); + ASSERT_OK_AND_ASSIGN(stream, fs_->OpenOutputStream("bucket/newfile_with_sse_c")); + ASSERT_OK(stream->Write("some")); + ASSERT_OK(stream->Close()); + + options_.sse_customer_key = {}; + MakeFileSystem(); + ASSERT_RAISES(IOError, fs_->OpenInputFile("bucket/newfile_with_sse_c")); + ASSERT_OK(RestoreTestBucket()); + } +} + +TEST_F(TestS3FSHTTPS, SSECustomerKeyCopyFile) { + ASSERT_OK_AND_ASSIGN(auto stream, fs_->OpenOutputStream("bucket/newfile_with_sse_c")); + ASSERT_OK(stream->Write("some")); + ASSERT_OK(stream->Close()); + ASSERT_OK(fs_->CopyFile("bucket/newfile_with_sse_c", "bucket/copied_with_sse_c")); + + ASSERT_OK_AND_ASSIGN(auto file, fs_->OpenInputFile("bucket/copied_with_sse_c")); + ASSERT_OK_AND_ASSIGN(auto buf, file->Read(5)); + AssertBufferEqual(*buf, "some"); + ASSERT_OK(RestoreTestBucket()); +} +#endif // ENABLE_TLS_TESTS + struct S3OptionsTestParameters { bool background_writes{false}; bool allow_delayed_open{false}; @@ -1420,7 +1555,8 @@ TEST_F(TestS3FS, FileSystemFromUri) { std::stringstream ss; ss << "s3://" << minio_->access_key() << ":" << minio_->secret_key() << "@bucket/somedir/subdir/subfile" - << "?scheme=http&endpoint_override=" << UriEscape(minio_->connect_string()); + << "?scheme=" << minio_->scheme() + << "&endpoint_override=" << UriEscape(minio_->connect_string()); std::string path; ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri(ss.str(), &path)); @@ -1522,7 +1658,7 @@ class TestS3FSGeneric : public S3TestMixin, public GenericFileSystemTest { } options_.ConfigureAccessKey(minio_->access_key(), minio_->secret_key()); - options_.scheme = "http"; + options_.scheme = minio_->scheme(); options_.endpoint_override = minio_->connect_string(); options_.retry_strategy = std::make_shared(); ASSERT_OK_AND_ASSIGN(s3fs_, S3FileSystem::Make(options_)); diff --git a/cpp/src/arrow/testing/util.cc b/cpp/src/arrow/testing/util.cc index 7bef9f7d4756d..e5e53801df949 100644 --- a/cpp/src/arrow/testing/util.cc +++ b/cpp/src/arrow/testing/util.cc @@ -206,6 +206,12 @@ std::string GetListenAddress() { return ss.str(); } +std::string GetListenAddress(const std::string& host) { + std::stringstream ss; + ss << host << ":" << GetListenPort(); + return ss.str(); +} + const std::vector>& all_dictionary_index_types() { static std::vector> types = { int8(), uint8(), int16(), uint16(), int32(), uint32(), int64(), uint64()}; diff --git a/cpp/src/arrow/testing/util.h b/cpp/src/arrow/testing/util.h index b4b2785a36292..8cc28a8b073a4 100644 --- a/cpp/src/arrow/testing/util.h +++ b/cpp/src/arrow/testing/util.h @@ -128,6 +128,10 @@ ARROW_TESTING_EXPORT int GetListenPort(); // port conflicts. ARROW_TESTING_EXPORT std::string GetListenAddress(); +// Get a "host:port" to listen on. Compared to GetListenAddress(), this function would use +// the host passed in. +ARROW_TESTING_EXPORT std::string GetListenAddress(const std::string& host); + ARROW_TESTING_EXPORT const std::vector>& all_dictionary_index_types(); From b6e877208f575df53a424d449b4c04fa2f2e81a5 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Tue, 5 Nov 2024 06:28:18 +0900 Subject: [PATCH 15/29] GH-44619: [GLib] Add GArrowDecimal32Scalar (#44628) ### Rationale for this change The `arrow::Decimal32Scalar` has been released. GLib needs to implement `GArrowDecimal32Scalar`. ### What changes are included in this PR? Implement `GArrowDecimal32Scalar`. ### Are these changes tested? YES ### Are there any user-facing changes? NO * GitHub Issue: #44619 Lead-authored-by: Hiroyuki Sato Co-authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- c_glib/arrow-glib/scalar.cpp | 126 +++++++++++++++++++++++++++ c_glib/arrow-glib/scalar.h | 16 ++++ c_glib/test/test-decimal32-scalar.rb | 48 ++++++++++ 3 files changed, 190 insertions(+) create mode 100644 c_glib/test/test-decimal32-scalar.rb diff --git a/c_glib/arrow-glib/scalar.cpp b/c_glib/arrow-glib/scalar.cpp index 57085a00c4b10..f2093e3e41ae2 100644 --- a/c_glib/arrow-glib/scalar.cpp +++ b/c_glib/arrow-glib/scalar.cpp @@ -104,6 +104,8 @@ G_BEGIN_DECLS * #GArrowMonthDayNanoIntervalScalar is a class for the month day nano * intarval scalar. * + * #GArrowDecimal32Scalar is a class for a 32-bit decimal scalar. + * * #GArrowDecimal64Scalar is a class for a 64-bit decimal scalar. * * #GArrowDecimal128Scalar is a class for a 128-bit decimal scalar. @@ -1633,6 +1635,127 @@ garrow_month_day_nano_interval_scalar_get_value(GArrowMonthDayNanoIntervalScalar return priv->value; } +typedef struct GArrowDecimal32ScalarPrivate_ +{ + GArrowDecimal32 *value; +} GArrowDecimal32ScalarPrivate; + +G_DEFINE_TYPE_WITH_PRIVATE(GArrowDecimal32Scalar, + garrow_decimal32_scalar, + GARROW_TYPE_SCALAR) + +#define GARROW_DECIMAL32_SCALAR_GET_PRIVATE(obj) \ + static_cast( \ + garrow_decimal32_scalar_get_instance_private(GARROW_DECIMAL32_SCALAR(obj))) + +static void +garrow_decimal32_scalar_dispose(GObject *object) +{ + auto priv = GARROW_DECIMAL32_SCALAR_GET_PRIVATE(object); + + if (priv->value) { + g_object_unref(priv->value); + priv->value = NULL; + } + + G_OBJECT_CLASS(garrow_decimal32_scalar_parent_class)->dispose(object); +} + +static void +garrow_decimal32_scalar_set_property(GObject *object, + guint prop_id, + const GValue *value, + GParamSpec *pspec) +{ + auto priv = GARROW_DECIMAL32_SCALAR_GET_PRIVATE(object); + + switch (prop_id) { + case PROP_VALUE: + priv->value = GARROW_DECIMAL32(g_value_dup_object(value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void +garrow_decimal32_scalar_init(GArrowDecimal32Scalar *object) +{ +} + +static void +garrow_decimal32_scalar_class_init(GArrowDecimal32ScalarClass *klass) +{ + auto gobject_class = G_OBJECT_CLASS(klass); + + gobject_class->dispose = garrow_decimal32_scalar_dispose; + gobject_class->set_property = garrow_decimal32_scalar_set_property; + + GParamSpec *spec; + /** + * GArrowDecimal32Scalar:value: + * + * The value of the scalar. + * + * Since: 19.0.0 + */ + spec = g_param_spec_object( + "value", + "Value", + "The value of the scalar", + garrow_decimal32_get_type(), + static_cast(G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY)); + g_object_class_install_property(gobject_class, PROP_VALUE, spec); +} + +/** + * garrow_decimal32_scalar_new: + * @data_type: A #GArrowDecimal32DataType for this scalar. + * @value: The value of this scalar. + * + * Returns: A newly created #GArrowDecimal32Scalar. + * + * Since: 19.0.0 + */ +GArrowDecimal32Scalar * +garrow_decimal32_scalar_new(GArrowDecimal32DataType *data_type, GArrowDecimal32 *value) +{ + auto arrow_data_type = garrow_data_type_get_raw(GARROW_DATA_TYPE(data_type)); + auto arrow_value = garrow_decimal32_get_raw(value); + auto arrow_scalar = std::static_pointer_cast( + std::make_shared(*arrow_value, arrow_data_type)); + return GARROW_DECIMAL32_SCALAR(garrow_scalar_new_raw(&arrow_scalar, + "scalar", + &arrow_scalar, + "data-type", + data_type, + "value", + value, + NULL)); +} + +/** + * garrow_decimal32_scalar_get_value: + * @scalar: A #GArrowDecimal32Scalar. + * + * Returns: (transfer none): The value of this scalar. + * + * Since: 19.0.0 + */ +GArrowDecimal32 * +garrow_decimal32_scalar_get_value(GArrowDecimal32Scalar *scalar) +{ + auto priv = GARROW_DECIMAL32_SCALAR_GET_PRIVATE(scalar); + if (!priv->value) { + auto arrow_scalar = std::static_pointer_cast( + garrow_scalar_get_raw(GARROW_SCALAR(scalar))); + auto arrow_value = std::make_shared(arrow_scalar->value); + priv->value = garrow_decimal32_new_raw(&arrow_value); + } + return priv->value; +} + typedef struct GArrowDecimal64ScalarPrivate_ { GArrowDecimal64 *value; @@ -2631,6 +2754,9 @@ garrow_scalar_new_raw_valist(std::shared_ptr *arrow_scalar, case arrow::Type::type::INTERVAL_MONTH_DAY_NANO: type = GARROW_TYPE_MONTH_DAY_NANO_INTERVAL_SCALAR; break; + case arrow::Type::type::DECIMAL32: + type = GARROW_TYPE_DECIMAL32_SCALAR; + break; case arrow::Type::type::DECIMAL64: type = GARROW_TYPE_DECIMAL64_SCALAR; break; diff --git a/c_glib/arrow-glib/scalar.h b/c_glib/arrow-glib/scalar.h index c9de9958ad402..4f2c44199f43b 100644 --- a/c_glib/arrow-glib/scalar.h +++ b/c_glib/arrow-glib/scalar.h @@ -501,6 +501,22 @@ GARROW_AVAILABLE_IN_8_0 GArrowMonthDayNano * garrow_month_day_nano_interval_scalar_get_value(GArrowMonthDayNanoIntervalScalar *scalar); +#define GARROW_TYPE_DECIMAL32_SCALAR (garrow_decimal32_scalar_get_type()) +GARROW_AVAILABLE_IN_19_0 +G_DECLARE_DERIVABLE_TYPE( + GArrowDecimal32Scalar, garrow_decimal32_scalar, GARROW, DECIMAL32_SCALAR, GArrowScalar) +struct _GArrowDecimal32ScalarClass +{ + GArrowScalarClass parent_class; +}; + +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal32Scalar * +garrow_decimal32_scalar_new(GArrowDecimal32DataType *data_type, GArrowDecimal32 *value); +GARROW_AVAILABLE_IN_19_0 +GArrowDecimal32 * +garrow_decimal32_scalar_get_value(GArrowDecimal32Scalar *scalar); + #define GARROW_TYPE_DECIMAL64_SCALAR (garrow_decimal64_scalar_get_type()) GARROW_AVAILABLE_IN_19_0 G_DECLARE_DERIVABLE_TYPE( diff --git a/c_glib/test/test-decimal32-scalar.rb b/c_glib/test/test-decimal32-scalar.rb new file mode 100644 index 0000000000000..cb54a6bb6a1e5 --- /dev/null +++ b/c_glib/test/test-decimal32-scalar.rb @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestDecimal32Scalar < Test::Unit::TestCase + def setup + @data_type = Arrow::Decimal32DataType.new(8, 2) + @value = Arrow::Decimal32.new("23423445") + @scalar = Arrow::Decimal32Scalar.new(@data_type, @value) + end + + def test_data_type + assert_equal(@data_type, + @scalar.data_type) + end + + def test_valid? + assert do + @scalar.valid? + end + end + + def test_equal + assert_equal(Arrow::Decimal32Scalar.new(@data_type, @value), + @scalar) + end + + def test_to_s + assert_equal("234234.45", @scalar.to_s) + end + + def test_value + assert_equal(@value, @scalar.value) + end +end From dcd0ad4cd0670ae3f39b89ab40d3cc81122ffabd Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 5 Nov 2024 06:33:28 +0900 Subject: [PATCH 16/29] GH-44624: [CI][JS] Increase "AMD64 macOS 13 NodeJS 18" timeout (#44625) ### Rationale for this change It took about 25m when it succeeded. We need to increase timeout for stable CI. ### What changes are included in this PR? Increase timeout to 45m from 30m. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * GitHub Issue: #44624 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- .github/workflows/js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml index 810c154aa9c27..5ef5b37c98815 100644 --- a/.github/workflows/js.yml +++ b/.github/workflows/js.yml @@ -89,7 +89,7 @@ jobs: name: AMD64 macOS 13 NodeJS ${{ matrix.node }} runs-on: macos-13 if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 30 + timeout-minutes: 45 strategy: fail-fast: false matrix: From 13f4a1dd84f424b5b558434e6cd3cdf07a114334 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 08:48:57 +0900 Subject: [PATCH 17/29] MINOR: [Java] Bump checker.framework.version from 3.48.1 to 3.48.2 in /java (#44635) Bumps `checker.framework.version` from 3.48.1 to 3.48.2. Updates `org.checkerframework:checker-qual` from 3.48.1 to 3.48.2
Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 3.48.2 (November 1, 2024)

Closed issues:

#6371, #6867.

Commits
  • 59f4594 new release 3.48.2
  • 93e8fac Prep for release.
  • 013a76c Update lists of aliases for @ NonNull (#6883)
  • f23bf98 Update dependency com.amazonaws:aws-java-sdk-bom to v1.12.777 (#6882)
  • 07d8845 Don't re-compute the enclosing method (#6876)
  • a70e1e9 Update dependency org.plumelib:plume-util to v1.10.0 (#6877)
  • fc99f34 Update versions.errorprone to v2.35.1 (#6875)
  • b7d9092 Update versions.errorprone to v2.34.0 (#6870)
  • cfdd5c9 Expect crash due to javac bug
  • 19419ac Cleaner logic to handle types of extends and implements clauses and fixed `ge...
  • Additional commits viewable in compare view

Updates `org.checkerframework:checker` from 3.48.1 to 3.48.2
Changelog

Sourced from org.checkerframework:checker's changelog.

Version 3.48.2 (November 1, 2024)

Closed issues:

#6371, #6867.

Commits
  • 59f4594 new release 3.48.2
  • 93e8fac Prep for release.
  • 013a76c Update lists of aliases for @ NonNull (#6883)
  • f23bf98 Update dependency com.amazonaws:aws-java-sdk-bom to v1.12.777 (#6882)
  • 07d8845 Don't re-compute the enclosing method (#6876)
  • a70e1e9 Update dependency org.plumelib:plume-util to v1.10.0 (#6877)
  • fc99f34 Update versions.errorprone to v2.35.1 (#6875)
  • b7d9092 Update versions.errorprone to v2.34.0 (#6870)
  • cfdd5c9 Expect crash due to javac bug
  • 19419ac Cleaner logic to handle types of extends and implements clauses and fixed `ge...
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 84fb967f4f1f0..fa1662ad48344 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -109,7 +109,7 @@ under the License. true 2.31.0 5.14.2 - 3.48.1 + 3.48.2 1.5.11 none -Xdoclint:none From 7612d52231db4455ab2be0c338de5a16dbaf0d05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 08:49:58 +0900 Subject: [PATCH 18/29] MINOR: [Java] Bump com.puppycrawl.tools:checkstyle from 10.18.2 to 10.20.0 in /java (#44637) Bumps [com.puppycrawl.tools:checkstyle](https://github.com/checkstyle/checkstyle) from 10.18.2 to 10.20.0.
Release notes

Sourced from com.puppycrawl.tools:checkstyle's releases.

checkstyle-10.20.0

Checkstyle 10.20.0 - https://checkstyle.org/releasenotes.html#Release_10.20.0

Breaking backward compatibility:

#15687 - JavadocMethodCheck: removed unnecessary tokens from acceptable

New:

#14424 - HideUtilityClassConstructor - Add option to skip validation based on list of annotations

Bug fixes:

#15831 - google_checks.xml not allowing eol left curly for switch statement with lambda-like construct

checkstyle-10.19.0

Checkstyle 10.19.0 - https://checkstyle.org/releasenotes.html#Release_10.19.0

New:

#9540 - WhitespaceAround: new property allowEmptySwitchBlockStatements #15263 - UnnecessaryParenthesesCheck does not flag unnecessary parentheses in conditional expression

Bug fixes:

#15664 - false-negative in google_checks.xml for not being able to detect requirement of K & R style for FINALLY #15769 - google_checks.xml: remove xpath suppression and false-positive indentation violations for block codes #15685 - JavadocParagraph does not work when paragraphs have their corresponding closing tag #15324 - Enforce preceding line break for opening braces of a case/default under switch in google_checks.xml #15733 - JavadocParagraph: report violation with column #15503 - JavadocParagraph: violate preceding P tag before block-level HTML tags #15716 - google_checks.xml: JavadocParagraph should have allowNewlineParagraph as false

... (truncated)

Commits
  • 41e15b3 [maven-release-plugin] prepare release checkstyle-10.20.0
  • 719ae40 doc: release notes for 10.20.0
  • 4c67922 Issue #15831: enabled allowEmptySwitchBlockStatements property of WhitespaceA...
  • 67b98ab Issue #14814: refactor checkline into iterative method
  • 9db3909 Issue #14814: refactor findmatch into iteration method
  • 853e2ba Issue #13345: Enable examples tests for ExplicitInitializationCheck
  • 8e8df58 Issue #13345: Enable examples tests for CovariantEqualsCheck
  • ca693c7 Issue #15829: Added test class for ConstructorsDeclarationGrouping
  • c256c10 Issue #6207: Added XPath regression test for ClassTypeParameterName
  • 42cf0ad Issue #15456: Specify violation messages for ArrayTrailingComma
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.puppycrawl.tools:checkstyle&package-manager=maven&previous-version=10.18.2&new-version=10.20.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index fa1662ad48344..65d0ca5af47be 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -105,7 +105,7 @@ under the License. 1.12.0 2 - 10.18.2 + 10.20.0 true 2.31.0 5.14.2 From a5fe294b2a75ebfa2a5988223bf61440ac4230e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 19:59:38 -0500 Subject: [PATCH 19/29] MINOR: [Java] Bump org.codehaus.mojo:exec-maven-plugin from 3.4.1 to 3.5.0 in /java (#44552) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.codehaus.mojo:exec-maven-plugin](https://github.com/mojohaus/exec-maven-plugin) from 3.4.1 to 3.5.0.
Release notes

Sourced from org.codehaus.mojo:exec-maven-plugin's releases.

3.5.0

🚀 New features and improvements

🐛 Bug Fixes

📦 Dependency updates

👻 Maintenance

Commits
  • b80d3d6 [maven-release-plugin] prepare release 3.5.0
  • 226a8ce Update site descriptor to 2.0.0
  • 47eac15 #322, enable to control the exec:java interaction with JVM classloader more f...
  • 582aed0 Bump project version
  • 8e7fa52 Update src/main/java/org/codehaus/mojo/exec/ExecMojo.java
  • d2bdc9c Add toolchain java path to environment variables in ExecMojo - added tests an...
  • eb62d78 Add toolchain java path to environment variables in ExecMojo - added tests an...
  • 8dbbb07 Add toolchain java path to environment variables in ExecMojo - added tests an...
  • 168b368 Add toolchain java path to environment variables in ExecMojo - added tests an...
  • 491526a Add toolchain java path to environment variables in ExecMojo - added tests an...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.codehaus.mojo:exec-maven-plugin&package-manager=maven&previous-version=3.4.1&new-version=3.5.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: David Li --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 65d0ca5af47be..15c46be50d678 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -514,7 +514,7 @@ under the License. org.codehaus.mojo exec-maven-plugin - 3.4.1 + 3.5.0 org.codehaus.mojo From 640bdd8688a142bbfb32983db6597339026f3131 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 20:00:04 -0500 Subject: [PATCH 20/29] MINOR: [Java] Bump com.google.api.grpc:proto-google-common-protos from 2.46.0 to 2.48.0 in /java (#44553) Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.46.0 to 2.48.0.
Release notes

Sourced from com.google.api.grpc:proto-google-common-protos's releases.

v2.48.0

2.48.0 (2024-10-22)

Features

Bug Fixes

  • generator setting incorrect name/class for sample due to region tag (2nd attempt) (#3293) (771bd0e)

Dependencies

  • update dependency com.google.errorprone:error_prone_annotations to v2.34.0 (#3303) (5b01274)
  • update dependency com.google.errorprone:error_prone_annotations to v2.34.0 (#3304) (5bd6c9c)
  • update google api dependencies (#3282) (a9eac85)
  • update google auth library dependencies to v1.29.0 (#3302) (e64eda2)

v2.47.0

2.47.0 (2024-10-04)

Features

  • gax: add API key authentication to ClientSettings (#3137) (df08956)
  • gax: append cred-type header for auth metrics (#3186) (ca3ec24)

Bug Fixes

  • address incorrect universe domain validation when quota project id is set (#3257) (6e70c37), closes #3256
  • Disable automatically retrieving Universe Domain from Metadata Server (#3272) (f4402bf)

Dependencies

  • update dependency com.fasterxml.jackson:jackson-bom to v2.18.0 (#3248) (821e83d)
  • update dependency com.google.errorprone:error_prone_annotations to v2.33.0 (#3265) (94450a9)
  • update dependency com.google.errorprone:error_prone_annotations to v2.33.0 (#3266) (8235463)
  • update dependency com.google.guava:guava to v33.3.1-jre (#3228) (4e76207)
  • update dependency net.bytebuddy:byte-buddy to v1.15.3 (#3246) (2aad71d)
  • update google api dependencies (#3242) (02aae9d)
  • update google auth library dependencies to v1.28.0 (#3267) (6d85864)
  • update googleapis/java-cloud-bom digest to 0cd97b7 (#3260) (2d54a5d)
  • update grpc dependencies to v1.67.1 (#3258) (e08906c)
  • update grpc dependencies to v1.67.1 in dependencies.properties (#3279) (5b46e70)

... (truncated)

Changelog

Sourced from com.google.api.grpc:proto-google-common-protos's changelog.

2.48.0 (2024-10-22)

Features

Bug Fixes

  • generator setting incorrect name/class for sample due to region tag (2nd attempt) (#3293) (771bd0e)

Dependencies

  • update dependency com.google.errorprone:error_prone_annotations to v2.34.0 (#3303) (5b01274)
  • update dependency com.google.errorprone:error_prone_annotations to v2.34.0 (#3304) (5bd6c9c)
  • update google api dependencies (#3282) (a9eac85)
  • update google auth library dependencies to v1.29.0 (#3302) (e64eda2)

2.47.0 (2024-10-04)

Features

  • gax: add API key authentication to ClientSettings (#3137) (df08956)
  • gax: append cred-type header for auth metrics (#3186) (ca3ec24)

Bug Fixes

  • address incorrect universe domain validation when quota project id is set (#3257) (6e70c37), closes #3256
  • Disable automatically retrieving Universe Domain from Metadata Server (#3272) (f4402bf)

Dependencies

  • update dependency com.fasterxml.jackson:jackson-bom to v2.18.0 (#3248) (821e83d)
  • update dependency com.google.errorprone:error_prone_annotations to v2.33.0 (#3265) (94450a9)
  • update dependency com.google.errorprone:error_prone_annotations to v2.33.0 (#3266) (8235463)
  • update dependency com.google.guava:guava to v33.3.1-jre (#3228) (4e76207)
  • update dependency net.bytebuddy:byte-buddy to v1.15.3 (#3246) (2aad71d)
  • update google api dependencies (#3242) (02aae9d)
  • update google auth library dependencies to v1.28.0 (#3267) (6d85864)
  • update googleapis/java-cloud-bom digest to 0cd97b7 (#3260) (2d54a5d)
  • update grpc dependencies to v1.67.1 (#3258) (e08906c)
  • update grpc dependencies to v1.67.1 in dependencies.properties (#3279) (5b46e70)
  • update junit5 monorepo to v5.11.2 (#3276) (6b10f94)
  • update netty dependencies to v4.1.114.final (#3263) (8bd83d9)

... (truncated)

Commits
  • d5e74d9 chore(main): release 2.48.0 (#3295)
  • e64eda2 deps: update google auth library dependencies to v1.29.0 (#3302)
  • 5b01274 deps: update dependency com.google.errorprone:error_prone_annotations to v2.3...
  • 5bd6c9c deps: update dependency com.google.errorprone:error_prone_annotations to v2.3...
  • 7512cfa chore: update base image to latest (#3301)
  • fd0b291 chore: update googleapis commit at Sat Oct 5 02:25:08 UTC 2024 (#3283)
  • 16365db chore: suppress pull progress (#3296)
  • a9eac85 deps: update google api dependencies (#3282)
  • dfe1a50 feat: selectively generate libraries (#3290)
  • 771bd0e fix: generator setting incorrect name/class for sample due to region tag (2nd...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.api.grpc:proto-google-common-protos&package-manager=maven&previous-version=2.46.0&new-version=2.48.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: David Li --- java/flight/flight-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index 3127bc0d949f1..461c415535764 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -134,7 +134,7 @@ under the License. com.google.api.grpc proto-google-common-protos - 2.46.0 + 2.48.0 test From 1a6de9d6590c6cc9afb28d7d5590378ee3d0844f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 20:01:37 -0500 Subject: [PATCH 21/29] MINOR: [Java] Bump io.grpc:grpc-bom from 1.65.0 to 1.68.1 in /java (#44639) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [io.grpc:grpc-bom](https://github.com/grpc/grpc-java) from 1.65.0 to 1.68.1.
Release notes

Sourced from io.grpc:grpc-bom's releases.

v1.68.1

gRPC Java 1.68.1 Release Notes

v1.68.0 was a mistake. This is the first release of version 1.68.x

Bug Fixes

  • xds: Fix NullPointerException introduced in "Fix load reporting when pick first is used for locality-routing" (#11553). This was in 1.67.1 but not 1.68.0

Behavior Changes

  • core: JSON parsing rejects duplicate keys in objects (#11575) (4be69e3f8). This is the existing behavior in C core. Duplicate keys in objects are dangerous as which value takes effect is undefined. Previously, the last value was used
  • okhttp: Detect transport executors with no remaining threads (#11503) (3a6be9ca1). The transport uses two threads, but one is on-demand. If the executor provided to builder.transportExecutor() runs out of threads (e.g., it is a fixed-size thread pool), all transports can be wedged, unable to run on-demand tasks, until keepalive kills one of them. Two threads are now used when handshaking a new transport, and the transport will time out after 1 second with “Timed out waiting for second handshake thread” if two threads are unavailable
  • gcp-csm-o11y: Get mesh_id value from CSM_MESH_ID environment variable, instead of getting it from bootstrap file (84d30afad)

Improvements

  • New grpc-context-override-opentelemetry artifact (#11523) (782a44ad6) (#11599) (e59ae5fad). This is a io.grpc.Context storage override to store its state in io.opentelemetry.context.Context. Libraries should not add a dependency on this artifact, as applications can only have one storage override in their classpath
  • New grpc-s2a artifact. It is a transport that offloads the handshake similar to ALTS, but for TLS. It provides io.grpc.s2a.S2AChannelCredentials
  • api: Enhance name resolver `ResolutionResult` to hold addresses or error so the single listener API onResult2 is used to convey both success and error cases for name resolution (#11330) (1ded8aff8)
  • core: Handle NameResolver/LoadBalancer exceptions when panicking (b692b9d26). This expands the class of bugs that will fail RPCs with the panic error, versus some undefined behavior
  • core: Use the default service config in case of initial name resolver address resolution error (#11577) (fa26a8bc5)
  • core: StreamTracer.inboundMessageRead() now reports uncompressed message size when the message does not need compression (#11598) (2aae68e11). Previously it always reported -1 (unknown)
  • netty: Avoid TCP_USER_TIMEOUT warning when explicitly specifying a non-epoll channel type to use (#11564) (62f409810)
  • okhttp: Don't warn about missing Conscrypt (6f3542297). This is especially helpful when using TLS but not running on Android
  • android: For UdsChannelBuilder, use fake IP instead of localhost (a908b5e40). This avoids an unnecessary DNS lookup
  • xds: Add xDS node ID in select control plane errors to enable cross-referencing with control plane logs when debugging (f3cf7c3c7)
  • xds: Enhanced how ADS stream terminations are handled, specifically addressing cases where a response has or hasn't been received (#2e9c3e19f)
  • binder: Update status code documentation for Android 11's package visibility rules. (#11551) (99be6e985)
  • binder: Update binderDied() error description to spell out the possibilities for those unfamiliar with Android internals. (#11628) (46c1b387f)
  • example-gauth: Use application default creds instead of file argument (#11595) (94a0a0d1c)
  • opentelemetry: Experimental OpenTelemetry tracing is available. Set the GRPC_EXPERIMENTAL_ENABLE_OTEL_TRACING environment variable to true to enable tracing support in GrpcOpenTelemetry (#11409, #11477)(043ba55, 421e237)

Dependencies

  • Updated protobuf-java to 3.25.5. This helps avoid CVE-2024-7254 (2ff837ab6)

Thanks to:
@​Juneezee
@​lgalfaso
@​bestbeforetoday
@​hlx502
@​JoeCqupt

v1.68.0 MISTAKE

This was supposed to be v1.67.0, but there was a mistake during the release process. This has everything in v1.67.1, except for:

  • xds: Fix NullPointerException introduced in "Fix load reporting when pick first is used for locality-routing" (grpc/grpc-java#11553)

v1.67.1

gRPC Java 1.67.1 Release Notes

... (truncated)

Commits
  • 16f93c8 Bump version to 1.68.1
  • 2b53352 Update README etc to reference 1.68.1
  • 135f433 Revert "stub: Ignore unary response on server if status is not OK" (#11636) (...
  • 2d0c158 Bump to 1.68.1-SNAPSHOT (#11637)
  • 46c1b38 Update binderDied() error description to spell out the possibilities for thos...
  • b65cbf5 inprocess: Support tracing message sizes guarded by flag (#11629)
  • 62f4098 netty: Avoid TCP_USER_TIMEOUT warning when not using epoll (#11564)
  • 00c8bc7 Minor grammar fix in Javadoc (#11609)
  • 4be69e3 core: SpiffeUtil API for extracting Spiffe URI and loading TrustBundles (#11575)
  • 1e0928f api: fix javadoc of CallCredentials.applyRequestMetadata
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=io.grpc:grpc-bom&package-manager=maven&previous-version=1.65.0&new-version=1.68.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: David Li --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 15c46be50d678..7e3e908716d0b 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -97,7 +97,7 @@ under the License. 2.0.16 33.3.1-jre 4.1.114.Final - 1.65.0 + 1.68.1 3.25.4 2.18.0 3.4.1 From 447f27b85677e03d8d47984562de55a51a9dfb44 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 20:19:50 -0500 Subject: [PATCH 22/29] MINOR: [Java] Bump logback.version from 1.5.11 to 1.5.12 in /java (#44550) Bumps `logback.version` from 1.5.11 to 1.5.12. Updates `ch.qos.logback:logback-classic` from 1.5.11 to 1.5.12
Commits

Updates `ch.qos.logback:logback-core` from 1.5.11 to 1.5.12
Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: David Li --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 7e3e908716d0b..c4d6147d63e05 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -110,7 +110,7 @@ under the License. 2.31.0 5.14.2 3.48.2 - 1.5.11 + 1.5.12 none -Xdoclint:none From 593c75c48e8e1ad2a6dd4077d50712bb828abccc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 20:40:51 -0500 Subject: [PATCH 23/29] MINOR: [Java] Bump com.fasterxml.jackson:jackson-bom from 2.18.0 to 2.18.1 in /java (#44636) Bumps [com.fasterxml.jackson:jackson-bom](https://github.com/FasterXML/jackson-bom) from 2.18.0 to 2.18.1.
Commits
  • ef33ac7 [maven-release-plugin] prepare release jackson-bom-2.18.1
  • f43bf9f Prepare for 2.18.1 release
  • 6f5259d Change to snapshot version of jackson-parent
  • 3f21ec5 Back to snapshot dep
  • bb45933 [maven-release-plugin] prepare for next development iteration
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.fasterxml.jackson:jackson-bom&package-manager=maven&previous-version=2.18.0&new-version=2.18.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: David Li --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index c4d6147d63e05..516186a9230a5 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -99,7 +99,7 @@ under the License. 4.1.114.Final 1.68.1 3.25.4 - 2.18.0 + 2.18.1 3.4.1 24.3.25 1.12.0 From 39c64e045e9c0ce2a49fdb542f7d8d5a691e4b27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 22:08:33 -0500 Subject: [PATCH 24/29] MINOR: [Java] Bump com.github.luben:zstd-jni from 1.5.6-6 to 1.5.6-7 in /java (#44548) Bumps [com.github.luben:zstd-jni](https://github.com/luben/zstd-jni) from 1.5.6-6 to 1.5.6-7.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.luben:zstd-jni&package-manager=maven&previous-version=1.5.6-6&new-version=1.5.6-7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: David Li --- java/compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/compression/pom.xml b/java/compression/pom.xml index 094e31afa4738..8cc4909034abe 100644 --- a/java/compression/pom.xml +++ b/java/compression/pom.xml @@ -55,7 +55,7 @@ under the License. com.github.luben zstd-jni - 1.5.6-6 + 1.5.6-7 From 4274db81e607f833cd022902eb4d01e841a930c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 22:20:23 -0500 Subject: [PATCH 25/29] MINOR: [Java] Bump org.bouncycastle:bcpkix-jdk18on from 1.78.1 to 1.79 in /java (#44638) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.bouncycastle:bcpkix-jdk18on](https://github.com/bcgit/bc-java) from 1.78.1 to 1.79.
Changelog

Sourced from org.bouncycastle:bcpkix-jdk18on's changelog.

2.1.1 Version Release: 1.80 Date:      TBD.

2.2.1 Version Release: 1.79 Date:      2024, 30th October.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.bouncycastle:bcpkix-jdk18on&package-manager=maven&previous-version=1.78.1&new-version=1.79)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: David Li --- java/flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/flight/flight-sql-jdbc-core/pom.xml b/java/flight/flight-sql-jdbc-core/pom.xml index 3e99e4b77ae3b..fc033a5ea7ab1 100644 --- a/java/flight/flight-sql-jdbc-core/pom.xml +++ b/java/flight/flight-sql-jdbc-core/pom.xml @@ -121,7 +121,7 @@ under the License. org.bouncycastle bcpkix-jdk18on - 1.78.1 + 1.79 From 46e7f38af71ed701815593f4b6327ceb464d3686 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Tue, 5 Nov 2024 17:02:27 +0100 Subject: [PATCH 26/29] GH-23995: [C#] Make PrimitiveArrayBuilder constructor public (#44596) Fixes #23995. Making these constructors `public` allows for writing custom builders. ### Rationale for this change Allows for writing custom builders. ### What changes are included in this PR? Only change of visibility on the default constructors of `PrimitiveArrayBuilder`. ### Are these changes tested? There is not much to test :) ### Are there any user-facing changes? See above. * GitHub Issue: #23995 Authored-by: Benedikt Reinartz Signed-off-by: Curt Hagenlocher --- csharp/src/Apache.Arrow/Arrays/PrimitiveArrayBuilder.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/csharp/src/Apache.Arrow/Arrays/PrimitiveArrayBuilder.cs b/csharp/src/Apache.Arrow/Arrays/PrimitiveArrayBuilder.cs index ae02173fb0df4..b3583842c1ed2 100644 --- a/csharp/src/Apache.Arrow/Arrays/PrimitiveArrayBuilder.cs +++ b/csharp/src/Apache.Arrow/Arrays/PrimitiveArrayBuilder.cs @@ -30,7 +30,7 @@ public abstract class PrimitiveArrayBuilder : IArr public int Length => ArrayBuilder.Length; - internal PrimitiveArrayBuilder(IArrowArrayBuilder> builder) + public PrimitiveArrayBuilder(IArrowArrayBuilder> builder) { ArrayBuilder = builder ?? throw new ArgumentNullException(nameof(builder)); } @@ -110,7 +110,7 @@ public abstract class PrimitiveArrayBuilder : IArrowArrayBu public int Length => ValueBuffer.Length; protected int NullCount => ValidityBuffer.UnsetBitCount; - internal PrimitiveArrayBuilder() + public PrimitiveArrayBuilder() { ValueBuffer = new ArrowBuffer.Builder(); ValidityBuffer = new ArrowBuffer.BitmapBuilder(); From c6f076a7e6e85a10e4b61824c33b6a639ee92e13 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Tue, 5 Nov 2024 14:44:57 -0600 Subject: [PATCH 27/29] GH-44648: [CI] Remove autotune and rebase from commentbot (#44649) ### Rationale for this change Removing code that doesn't work ### What changes are included in this PR? Deleting from the workflow ### Are these changes tested? Changes are part of CI ### Are there any user-facing changes? No * GitHub Issue: #44648 Authored-by: Jonathan Keane Signed-off-by: Jonathan Keane --- .github/workflows/comment_bot.yml | 123 +----------------------------- 1 file changed, 1 insertion(+), 122 deletions(-) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 8885171f0ab3f..9e0e8ab47e102 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -26,8 +26,7 @@ on: permissions: contents: read - pull-requests: write - + jobs: crossbow: name: Listen! @@ -55,126 +54,6 @@ jobs: --event-name ${{ github.event_name }} \ --event-payload ${{ github.event_path }} - autotune: - name: "Fix all the things" - if: startsWith(github.event.comment.body, '@github-actions autotune') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - - uses: r-lib/actions/pr-fetch@11a22a908006c25fe054c4ef0ac0436b1de3edbe # v2.6.4 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: See what is different - run: | - set -ex - DEFAULT_BRANCH=${{ github.event.repository.default_branch }} - git remote add upstream https://github.com/apache/arrow - git fetch upstream - - changed() { - git diff --name-only upstream/$DEFAULT_BRANCH... | grep -e "$1" >/dev/null 2>&1 - } - if changed '^r/.*\.R$'; then - echo "R_DOCS=true" >> $GITHUB_ENV - echo "R_CODE=true" >> $GITHUB_ENV - fi - if changed 'cmake' || changed 'CMake'; then - echo "CMAKE_FORMAT=true" >> $GITHUB_ENV - fi - if changed '^cpp/src'; then - echo "CLANG_FORMAT_CPP=true" >> $GITHUB_ENV - fi - if changed '^r/src'; then - echo "CLANG_FORMAT_R=true" >> $GITHUB_ENV - fi - - name: Ensure clang-format has the appropriate version - if: env.CMAKE_FORMAT == 'true' || - env.CLANG_FORMAT_CPP == 'true' || - env.CLANG_FORMAT_R == 'true' || - endsWith(github.event.comment.body, 'everything') - run: | - set -e - . .env # To get the clang version we use - sudo apt update - sudo apt install -y clang-format-${CLANG_TOOLS} - - name: Run cmake_format - if: env.CMAKE_FORMAT == 'true' || endsWith(github.event.comment.body, 'everything') - run: | - set -ex - export PATH=/home/runner/.local/bin:$PATH - python3 -m pip install --upgrade pip setuptools wheel - python3 -m pip install -e dev/archery[lint] - archery lint --cmake-format --fix - - name: Run clang-format on cpp - if: env.CLANG_FORMAT_CPP == 'true' || endsWith(github.event.comment.body, 'everything') - run: | - . .env # To get the clang version we use - cpp/build-support/run_clang_format.py \ - --clang_format_binary=clang-format-${CLANG_TOOLS} \ - --exclude_glob=cpp/build-support/lint_exclusions.txt \ - --source_dir=cpp/src --quiet --fix - - name: Run clang-format on r - if: env.CLANG_FORMAT_R == 'true' || endsWith(github.event.comment.body, 'everything') - run: | - . .env # To get the clang version we use - cpp/build-support/run_clang_format.py \ - --clang_format_binary=clang-format-${CLANG_TOOLS} \ - --exclude_glob=cpp/build-support/lint_exclusions.txt \ - --source_dir=r/src --quiet --fix - - uses: r-lib/actions/setup-r@11a22a908006c25fe054c4ef0ac0436b1de3edbe # v2.6.4 - if: env.R_DOCS == 'true' || env.R_CODE == 'true' || endsWith(github.event.comment.body, 'everything') - - name: Update R docs - if: env.R_DOCS == 'true' || endsWith(github.event.comment.body, 'everything') - shell: Rscript {0} - run: | - source("ci/etc/rprofile") - install.packages(c("remotes", "roxygen2")) - remotes::install_deps("r") - roxygen2::roxygenize("r") - - name: Style R code - if: env.R_CODE == 'true' || endsWith(github.event.comment.body, 'everything') - shell: Rscript {0} - run: | - changed_files <- system("git diff --name-only upstream/${{ github.event.repository.default_branch }}... 2>&1", intern = TRUE) - # only grab the .R files under r/ - changed_files <- grep('^r/.*\\.R$', changed_files, value = TRUE) - # remove codegen.R and other possible exclusions - changed_files <- changed_files[!changed_files %in% file.path("r", source("r/.styler_excludes.R")$value)] - source("ci/etc/rprofile") - install.packages(c("remotes", "styler")) - remotes::install_deps("r") - styler::style_file(changed_files) - - name: Commit results - run: | - git config user.name "$(git log -1 --pretty=format:%an)" - git config user.email "$(git log -1 --pretty=format:%ae)" - git commit -a -m 'Autoformat/render all the things [automated commit]' || echo "No changes to commit" - - uses: r-lib/actions/pr-push@11a22a908006c25fe054c4ef0ac0436b1de3edbe # v2.6.4 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - rebase: - name: "Rebase" - if: startsWith(github.event.comment.body, '@github-actions rebase') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - - uses: r-lib/actions/pr-fetch@11a22a908006c25fe054c4ef0ac0436b1de3edbe # v2.6.4 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Rebase on ${{ github.repository }} default branch - run: | - set -ex - git config user.name "$(git log -1 --pretty=format:%an)" - git config user.email "$(git log -1 --pretty=format:%ae)" - git remote add upstream https://github.com/${{ github.repository }} - git fetch --unshallow upstream ${{ github.event.repository.default_branch }} - git rebase upstream/${{ github.event.repository.default_branch }} - - uses: r-lib/actions/pr-push@11a22a908006c25fe054c4ef0ac0436b1de3edbe # v2.6.4 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - args: "--force" - issue_assign: name: "Assign issue" permissions: From f3b8d6b45155b35c902a11443f1b29cd846832a6 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 5 Nov 2024 18:14:09 -0500 Subject: [PATCH 28/29] MINOR: [Java] Revert io.grpc:grpc-bom to 1.65.0 (#44645) ### Rationale for this change I missed that gRPC (Protobuf) cannot be upgraded due to one of our CI jobs using an older libstdc++. ### What changes are included in this PR? This reverts commit 1a6de9d6590c6cc9afb28d7d5590378ee3d0844f. ### Are these changes tested? Yes ### Are there any user-facing changes? No Authored-by: David Li Signed-off-by: David Li --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 516186a9230a5..5c6719ee0f2c8 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -97,7 +97,7 @@ under the License. 2.0.16 33.3.1-jre 4.1.114.Final - 1.68.1 + 1.65.0 3.25.4 2.18.1 3.4.1 From c3601a97a0718ae47726e6c134cbed4b98bd1a36 Mon Sep 17 00:00:00 2001 From: Maksim Yegorov <997437+myegorov@users.noreply.github.com> Date: Tue, 5 Nov 2024 19:07:06 -0500 Subject: [PATCH 29/29] GH-44344: [Java] fix VectorSchemaRoot.getTransferPair for NullVector (#44631) ### Rationale for this change Do not throw [UnsupportedOperationException("Tried to get allocator from NullVector")](https://github.com/apache/arrow/blob/release-18.0.0-rc0/java/vector/src/main/java/org/apache/arrow/vector/NullVector.java#L160) from [VectorSchemaRoot.slice()](https://github.com/apache/arrow/blob/release-18.0.0-rc0/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java#L341) when slicing a VSR containing a NullVector or ZeroVector. Details in https://github.com/apache/arrow/issues/44344 ### Are these changes tested? Added unit test that would trigger an UnsupportedOperationException on the legacy path. * GitHub Issue: #44344 Authored-by: Maksim Yegorov <59841139+maksimyego-db@users.noreply.github.com> Signed-off-by: David Li --- .../org/apache/arrow/vector/NullVector.java | 9 ++++++- .../org/apache/arrow/vector/ValueVector.java | 6 +++++ .../arrow/vector/TestSplitAndTransfer.java | 25 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullVector.java index 227ca716f6391..6bfe540d232fc 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullVector.java @@ -155,9 +155,16 @@ public boolean allocateNewSafe() { @Override public void reAlloc() {} + /* + * IMPORTANT NOTE + * It's essential that NullVector (and ZeroVector) do not require BufferAllocator for any data storage. + * However, some methods of the parent interface may require passing in a BufferAllocator, even if null. + * + * @return null + */ @Override public BufferAllocator getAllocator() { - throw new UnsupportedOperationException("Tried to get allocator from NullVector"); + return null; } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index 724941aa2a1e8..0a45409eb9860 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -80,6 +80,12 @@ public interface ValueVector extends Closeable, Iterable { */ void reAlloc(); + /** + * Get the allocator associated with the vector. CAVEAT: Some ValueVector subclasses (e.g. + * NullVector) do not require an allocator for data storage and may return null. + * + * @return Returns nullable allocator. + */ BufferAllocator getAllocator(); /** diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java index a3f25bc5207b6..6aace956214ff 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java @@ -198,6 +198,31 @@ public void testWithEmptyVector() { toDUV.clear(); } + @Test + public void testWithNullVector() { + int valueCount = 123; + int startIndex = 10; + NullVector fromNullVector = new NullVector("nullVector"); + fromNullVector.setValueCount(valueCount); + TransferPair transferPair = fromNullVector.getTransferPair(fromNullVector.getAllocator()); + transferPair.splitAndTransfer(startIndex, valueCount - startIndex); + NullVector toNullVector = (NullVector) transferPair.getTo(); + + assertEquals(valueCount - startIndex, toNullVector.getValueCount()); + // no allocations to clear for NullVector + } + + @Test + public void testWithZeroVector() { + ZeroVector fromZeroVector = new ZeroVector("zeroVector"); + TransferPair transferPair = fromZeroVector.getTransferPair(fromZeroVector.getAllocator()); + transferPair.splitAndTransfer(0, 0); + ZeroVector toZeroVector = (ZeroVector) transferPair.getTo(); + + assertEquals(0, toZeroVector.getValueCount()); + // no allocations to clear for ZeroVector + } + @Test /* VarCharVector */ public void test() throws Exception { try (final VarCharVector varCharVector = new VarCharVector("myvector", allocator)) {