Merge branch 'main' into arrow-bit-util-remove-precompute-table

apache · May 24, 2024 · 4a9bae8 · 4a9bae8
2 parents 270e1a2 + b275483
commit 4a9bae8
Show file tree

Hide file tree

Showing 399 changed files with 17,562 additions and 7,492 deletions.
diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
@@ -186,7 +186,7 @@ jobs:
         shell: bash
         run: ci/scripts/ruby_test.sh $(pwd) $(pwd)/build
 
-  windows:
+  windows-mingw:
     name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} GLib & Ruby
     runs-on: windows-2019
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
@@ -267,7 +267,6 @@ jobs:
           ridk exec bash ci\scripts\cpp_build.sh "${source_dir}" "${build_dir}"
       - name: Build GLib
         run: |
-          $Env:CMAKE_BUILD_PARALLEL_LEVEL = $Env:NUMBER_OF_PROCESSORS
           $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")"
           $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")"
           $ErrorActionPreference = "Continue"
@@ -305,3 +304,96 @@ jobs:
           $Env:MAKE = "ridk exec make"
           $ErrorActionPreference = "Continue"
           rake -f ruby\Rakefile
+
+  windows-msvc:
+    name: AMD64 Windows MSVC GLib
+    runs-on: windows-2019
+    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+    env:
+      ARROW_BOOST_USE_SHARED: OFF
+      ARROW_BUILD_BENCHMARKS: OFF
+      ARROW_BUILD_SHARED: ON
+      ARROW_BUILD_STATIC: OFF
+      ARROW_BUILD_TESTS: OFF
+      ARROW_ACERO: ON
+      ARROW_DATASET: ON
+      ARROW_FLIGHT: OFF
+      ARROW_FLIGHT_SQL: OFF
+      ARROW_GANDIVA: OFF
+      ARROW_HDFS: OFF
+      ARROW_HOME: "${{ github.workspace }}/dist"
+      ARROW_JEMALLOC: OFF
+      ARROW_MIMALLOC: ON
+      ARROW_ORC: OFF
+      ARROW_PARQUET: ON
+      ARROW_SUBSTRAIT: OFF
+      ARROW_USE_GLOG: OFF
+      ARROW_VERBOSE_THIRDPARTY_BUILD: OFF
+      ARROW_WITH_BROTLI: OFF
+      ARROW_WITH_BZ2: OFF
+      ARROW_WITH_LZ4: OFF
+      ARROW_WITH_OPENTELEMETRY: OFF
+      ARROW_WITH_SNAPPY: ON
+      ARROW_WITH_ZLIB: OFF
+      ARROW_WITH_ZSTD: ON
+      BOOST_SOURCE: BUNDLED
+      CMAKE_CXX_STANDARD: "17"
+      CMAKE_GENERATOR: Ninja
+      CMAKE_INSTALL_PREFIX: "${{ github.workspace }}/dist"
+      CMAKE_UNITY_BUILD: ON
+    steps:
+      - name: Disable Crash Dialogs
+        run: |
+          reg add `
+            "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" `
+            /v DontShowUI `
+            /t REG_DWORD `
+            /d 1 `
+            /f
+      - name: Checkout Arrow
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: recursive
+      - name: Install vcpkg
+        shell: bash
+        run: |
+          ci/scripts/install_vcpkg.sh ./vcpkg
+      - name: Install meson
+        run: |
+          python -m pip install meson
+      - name: Install ccache
+        shell: bash
+        run: |
+          ci/scripts/install_ccache.sh 4.6.3 /usr
+      - name: Setup ccache
+        shell: bash
+        run: |
+          ci/scripts/ccache_setup.sh
+      - name: ccache info
+        id: ccache-info
+        shell: bash
+        run: |
+          echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
+      - name: Cache ccache
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.ccache-info.outputs.cache-dir }}
+          key: glib-ccache-msvc-${{ env.CACHE_VERSION }}-${{ hashFiles('cpp/**') }}
+          restore-keys: glib-ccache-msvc-${{ env.CACHE_VERSION }}-
+        env:
+          # We can invalidate the current cache by updating this.
+          CACHE_VERSION: "2024-05-09"
+      - name: Build C++
+        shell: cmd
+        run: |
+          call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
+          bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build"
+      - name: Build GLib
+        shell: cmd
+        run: |
+          call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
+          bash -c "VCPKG_ROOT=\"$(pwd)/vcpkg\" ci/scripts/c_glib_build.sh $(pwd) $(pwd)/build"
diff --git a/.gitignore b/.gitignore
@@ -102,4 +102,4 @@ __debug_bin
 .envrc
 
 # Develocity
-.mvn/.gradle-enterprise/
+.mvn/.develocity.xml
diff --git a/.mvn/gradle-enterprise.xml → .mvn/develocity.xml b/.mvn/gradle-enterprise.xml → .mvn/develocity.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <!--
 
     Licensed to the Apache Software Foundation (ASF) under one
@@ -19,19 +19,19 @@
     under the License.
 
 -->
-<gradleEnterprise xmlns="https://www.gradle.com/gradle-enterprise-maven" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="https://www.gradle.com/gradle-enterprise-maven https://www.gradle.com/schema/gradle-enterprise-maven.xsd">
+<develocity xmlns="https://www.gradle.com/develocity-maven" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="https://www.gradle.com/develocity-maven https://www.gradle.com/schema/develocity-maven.xsd">
     <server>
         <url>https://ge.apache.org</url>
         <allowUntrusted>false</allowUntrusted>
     </server>
     <buildScan>
         <capture>
-            <goalInputFiles>true</goalInputFiles>
+            <fileFingerprints>true</fileFingerprints>
             <buildLogging>true</buildLogging>
             <testLogging>true</testLogging>
         </capture>
         <backgroundBuildScanUpload>#{isFalse(env['CI'])}</backgroundBuildScanUpload>
-        <publish>ALWAYS</publish>
+        <publishing><onlyIf>true</onlyIf></publishing>
         <publishIfAuthenticated>true</publishIfAuthenticated>
         <obfuscation>
             <ipAddresses>#{{'0.0.0.0'}}</ipAddresses>
@@ -42,4 +42,4 @@
             <enabled>false</enabled>
         </remote>
     </buildCache>
-</gradleEnterprise>
+</develocity>
diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml
@@ -22,12 +22,12 @@
 <extensions xmlns="http://maven.apache.org/EXTENSIONS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/EXTENSIONS/1.0.0 http://maven.apache.org/xsd/core-extensions-1.0.0.xsd">
     <extension>
         <groupId>com.gradle</groupId>
-        <artifactId>gradle-enterprise-maven-extension</artifactId>
-        <version>1.20</version>
+        <artifactId>develocity-maven-extension</artifactId>
+        <version>1.21.4</version>
     </extension>
     <extension>
         <groupId>com.gradle</groupId>
         <artifactId>common-custom-user-data-maven-extension</artifactId>
-        <version>1.12.5</version>
+        <version>2.0</version>
     </extension>
 </extensions>
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -40,7 +40,7 @@ repos:
     hooks:
       - id: flake8
         name: Python Format
-        files: ^(python|dev|integration)/
+        files: ^(python|dev|c_glib|integration)/
         types:
           - file
           - python
@@ -116,25 +116,28 @@ repos:
         name: CMake Format
         files: >-
           (
+          ?.*CMakeLists\.txt$|
           ?^ci/.*/.*\.cmake$|
           ?^cpp/.*/.*\.cmake\.in$|
           ?^cpp/.*/.*\.cmake$|
-          ?^cpp/.*/CMakeLists\.txt$|
-          ?^go/.*/CMakeLists\.txt$|
-          ?^java/.*/CMakeLists\.txt$|
-          ?^matlab/.*/CMakeLists\.txt$|
-          ?^python/.*/CMakeLists\.txt$|
           )
         exclude: >-
           (
+          ?^ci/conan/all/.*CMakeLists\.txt$|
           ?^cpp/cmake_modules/FindNumPy\.cmake$|
           ?^cpp/cmake_modules/FindPythonLibsNew\.cmake$|
           ?^cpp/cmake_modules/UseCython\.cmake$|
-          ?^cpp/src/arrow/util/config\.h\.cmake$|
+          ?^cpp/src/arrow/util/.*\.h\.cmake$|
           )
   - repo: https://github.com/sphinx-contrib/sphinx-lint
     rev: v0.9.1
     hooks:
       - id: sphinx-lint
-        files: ^docs/
-        args: ['--disable', 'all', '--enable', 'trailing-whitespace,missing-final-newline', 'docs']
+        files: ^docs/source
+        exclude: ^docs/source/python/generated
+        args: [
+          '--enable',
+          'all',
+          '--disable',
+          'dangling-hyphen,line-too-long',
+        ]
diff --git a/c_glib/arrow-cuda-glib/arrow-cuda-glib.h b/c_glib/arrow-cuda-glib/arrow-cuda-glib.h
@@ -21,4 +21,6 @@
 
 #include <arrow-glib/arrow-glib.h>
 
+#include <arrow-cuda-glib/version.h>
+
 #include <arrow-cuda-glib/cuda.h>
diff --git a/c_glib/arrow-cuda-glib/cuda.h b/c_glib/arrow-cuda-glib/cuda.h
@@ -21,9 +21,12 @@
 
 #include <arrow-glib/arrow-glib.h>
 
+#include <arrow-cuda-glib/version.h>
+
 G_BEGIN_DECLS
 
 #define GARROW_CUDA_TYPE_DEVICE_MANAGER (garrow_cuda_device_manager_get_type())
+GARROW_CUDA_AVAILABLE_IN_0_12
 G_DECLARE_DERIVABLE_TYPE(GArrowCUDADeviceManager,
                          garrow_cuda_device_manager,
                          GARROW_CUDA,
@@ -35,6 +38,7 @@ struct _GArrowCUDADeviceManagerClass
 };
 
 #define GARROW_CUDA_TYPE_CONTEXT (garrow_cuda_context_get_type())
+GARROW_CUDA_AVAILABLE_IN_0_12
 G_DECLARE_DERIVABLE_TYPE(
   GArrowCUDAContext, garrow_cuda_context, GARROW_CUDA, CONTEXT, GObject)
 struct _GArrowCUDAContextClass
@@ -43,6 +47,7 @@ struct _GArrowCUDAContextClass
 };
 
 #define GARROW_CUDA_TYPE_BUFFER (garrow_cuda_buffer_get_type())
+GARROW_CUDA_AVAILABLE_IN_0_12
 G_DECLARE_DERIVABLE_TYPE(
   GArrowCUDABuffer, garrow_cuda_buffer, GARROW_CUDA, BUFFER, GArrowBuffer)
 struct _GArrowCUDABufferClass
@@ -51,6 +56,7 @@ struct _GArrowCUDABufferClass
 };
 
 #define GARROW_CUDA_TYPE_HOST_BUFFER (garrow_cuda_host_buffer_get_type())
+GARROW_CUDA_AVAILABLE_IN_0_12
 G_DECLARE_DERIVABLE_TYPE(GArrowCUDAHostBuffer,
                          garrow_cuda_host_buffer,
                          GARROW_CUDA,
@@ -62,6 +68,7 @@ struct _GArrowCUDAHostBufferClass
 };
 
 #define GARROW_CUDA_TYPE_IPC_MEMORY_HANDLE (garrow_cuda_ipc_memory_handle_get_type())
+GARROW_CUDA_AVAILABLE_IN_0_12
 G_DECLARE_DERIVABLE_TYPE(GArrowCUDAIPCMemoryHandle,
                          garrow_cuda_ipc_memory_handle,
                          GARROW_CUDA,
@@ -73,6 +80,7 @@ struct _GArrowCUDAIPCMemoryHandleClass
 };
 
 #define GARROW_CUDA_TYPE_BUFFER_INPUT_STREAM (garrow_cuda_buffer_input_stream_get_type())
+GARROW_CUDA_AVAILABLE_IN_0_12
 G_DECLARE_DERIVABLE_TYPE(GArrowCUDABufferInputStream,
                          garrow_cuda_buffer_input_stream,
                          GARROW_CUDA,
@@ -85,6 +93,7 @@ struct _GArrowCUDABufferInputStreamClass
 
 #define GARROW_CUDA_TYPE_BUFFER_OUTPUT_STREAM                                            \
   (garrow_cuda_buffer_output_stream_get_type())
+GARROW_CUDA_AVAILABLE_IN_0_12
 G_DECLARE_DERIVABLE_TYPE(GArrowCUDABufferOutputStream,
                          garrow_cuda_buffer_output_stream,
                          GARROW_CUDA,
@@ -95,71 +104,100 @@ struct _GArrowCUDABufferOutputStreamClass
   GArrowOutputStreamClass parent_class;
 };
 
+GARROW_CUDA_AVAILABLE_IN_0_12
 GArrowCUDADeviceManager *
 garrow_cuda_device_manager_new(GError **error);
 
+GARROW_CUDA_AVAILABLE_IN_0_12
 GArrowCUDAContext *
 garrow_cuda_device_manager_get_context(GArrowCUDADeviceManager *manager,
                                        gint gpu_number,
                                        GError **error);
+
+GARROW_CUDA_AVAILABLE_IN_0_12
 gsize
 garrow_cuda_device_manager_get_n_devices(GArrowCUDADeviceManager *manager);
 
+GARROW_CUDA_AVAILABLE_IN_0_12
 gint64
 garrow_cuda_context_get_allocated_size(GArrowCUDAContext *context);
 
+GARROW_CUDA_AVAILABLE_IN_0_12
 GArrowCUDABuffer *
 garrow_cuda_buffer_new(GArrowCUDAContext *context, gint64 size, GError **error);
+
+GARROW_CUDA_AVAILABLE_IN_0_12
 GArrowCUDABuffer *
 garrow_cuda_buffer_new_ipc(GArrowCUDAContext *context,
                            GArrowCUDAIPCMemoryHandle *handle,
                            GError **error);
+
+GARROW_CUDA_AVAILABLE_IN_0_12
 GArrowCUDABuffer *
 garrow_cuda_buffer_new_record_batch(GArrowCUDAContext *context,
                                     GArrowRecordBatch *record_batch,
                                     GError **error);
+
+GARROW_CUDA_AVAILABLE_IN_0_12
 GBytes *
 garrow_cuda_buffer_copy_to_host(GArrowCUDABuffer *buffer,
                                 gint64 position,
                                 gint64 size,
                                 GError **error);
+
+GARROW_CUDA_AVAILABLE_IN_0_12
 gboolean
 garrow_cuda_buffer_copy_from_host(GArrowCUDABuffer *buffer,
                                   const guint8 *data,
                                   gint64 size,
                                   GError **error);
+
+GARROW_CUDA_AVAILABLE_IN_0_12
 GArrowCUDAIPCMemoryHandle *
 garrow_cuda_buffer_export(GArrowCUDABuffer *buffer, GError **error);
+
+GARROW_CUDA_AVAILABLE_IN_0_12
 GArrowCUDAContext *
 garrow_cuda_buffer_get_context(GArrowCUDABuffer *buffer);
+
+GARROW_CUDA_AVAILABLE_IN_0_12
 GArrowRecordBatch *
 garrow_cuda_buffer_read_record_batch(GArrowCUDABuffer *buffer,
                                      GArrowSchema *schema,
                                      GArrowReadOptions *options,
                                      GError **error);
 
+GARROW_CUDA_AVAILABLE_IN_0_12
 GArrowCUDAHostBuffer *
 garrow_cuda_host_buffer_new(gint gpu_number, gint64 size, GError **error);
 
+GARROW_CUDA_AVAILABLE_IN_0_12
 GArrowCUDAIPCMemoryHandle *
 garrow_cuda_ipc_memory_handle_new(const guint8 *data, gsize size, GError **error);
 
+GARROW_CUDA_AVAILABLE_IN_0_12
 GArrowBuffer *
 garrow_cuda_ipc_memory_handle_serialize(GArrowCUDAIPCMemoryHandle *handle,
                                         GError **error);
 
+GARROW_CUDA_AVAILABLE_IN_0_12
 GArrowCUDABufferInputStream *
 garrow_cuda_buffer_input_stream_new(GArrowCUDABuffer *buffer);
 
+GARROW_CUDA_AVAILABLE_IN_0_12
 GArrowCUDABufferOutputStream *
 garrow_cuda_buffer_output_stream_new(GArrowCUDABuffer *buffer);
 
+GARROW_CUDA_AVAILABLE_IN_0_12
 gboolean
 garrow_cuda_buffer_output_stream_set_buffer_size(GArrowCUDABufferOutputStream *stream,
                                                  gint64 size,
                                                  GError **error);
+GARROW_CUDA_AVAILABLE_IN_0_12
 gint64
 garrow_cuda_buffer_output_stream_get_buffer_size(GArrowCUDABufferOutputStream *stream);
+
+GARROW_CUDA_AVAILABLE_IN_0_12
 gint64
 garrow_cuda_buffer_output_stream_get_buffered_size(GArrowCUDABufferOutputStream *stream);