Merge pull request #274 from chdb-io/buildLinuxArm64

Fix jemalloc background_thread caused assert failed

.github/workflows/build_linux_arm64_wheels-gh.yml

@@ -0,0 +1,180 @@
+name: Build Linux arm64
+
+on:
+  workflow_dispatch:
+    inputs:
+      TAG_NAME:
+        description: 'Release Version Tag'
+        required: true
+  release:
+    types: [created]
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '**/*.md'
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - '**/*.md'
+
+
+jobs:
+  build_wheels_linux:
+    name: ${{ matrix.os }} py${{ matrix.python-version }}
+    runs-on: GH-Linux-ARM64
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-24.04 ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+        # python-version: [ "3.7" ]
+    env:
+      RUNNER_OS: ${{ matrix.os }}
+      PYTHON_VERSION: ${{ matrix.python-version }}
+    steps:
+      - name: Install python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "${{ matrix.python-version }}"
+      - name: Install clang++ for Ubuntu
+        if: matrix.os == 'ubuntu-24.04'
+        run: |
+          pwd
+          uname -a
+          wget https://apt.llvm.org/llvm.sh
+          chmod +x llvm.sh
+          sudo ./llvm.sh 18
+          which clang++-18
+          clang++-18 --version
+          sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget
+          ccache -s
+      - name: Update git
+        run: |
+          sudo add-apt-repository ppa:git-core/ppa -y
+          sudo apt-get update
+          sudo apt-get install -y git
+          git --version
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Restore submodules cache
+        uses: actions/cache/restore@v3
+        id: cache
+        with:
+          path: |
+            contrib/**
+            !contrib/*-cmake
+            !**/CMakeLists.txt
+          key: |
+            submodule-${{ hashFiles('.gitmodules') }}
+      - name: Update submodules if cache miss
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          git submodule update --init --recursive --jobs 4
+      - name: Save submodules cache
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v3
+        with:
+          path: |
+            contrib/**
+            !contrib/*-cmake
+            !**/CMakeLists.txt
+          key: |
+            submodule-${{ hashFiles('.gitmodules') }}
+      - name: ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          key: ${{ matrix.os }}
+          max-size: 5G
+          append-timestamp: true
+      - name: remove old clang and link clang-18 to clang
+        if: matrix.os == 'ubuntu-24.04'
+        run: |
+          sudo rm -f /usr/bin/clang || true
+          sudo ln -s /usr/bin/clang-18 /usr/bin/clang
+          sudo rm -f /usr/bin/clang++ || true
+          sudo ln -s /usr/bin/clang++-18 /usr/bin/clang++
+          which clang++
+          clang++ --version
+      - name: Run chdb/build.sh
+        run: |
+          python3 -m pip install pybind11 setuptools
+          export CC=/usr/bin/clang
+          export CXX=/usr/bin/clang++
+          bash ./chdb/build.sh
+          python3 -m pip install pandas pyarrow
+          bash -x ./chdb/test_smoke.sh
+        continue-on-error: false
+      - name: Run libchdb stub in examples dir
+        run: |
+          bash -x ./examples/runStub.sh
+      - name: Check ccache statistics
+        run: |
+          ccache -s
+          ls -lh chdb
+          df -h
+      - name: Install dependencies for building wheels
+        run: python3 -m pip install -U pip tox pybind11 twine setuptools wheel
+      - name: Build wheels
+        run: |
+          export CC=/usr/bin/clang
+          export CXX=/usr/bin/clang++
+          make wheel
+      - name: Install patchelf from github
+        run: |
+          wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz
+          tar -xvf patchelf.tar.gz
+          sudo cp bin/patchelf /usr/bin/
+          sudo chmod +x /usr/bin/patchelf
+          patchelf --version
+      - name: Audit wheels
+        run: |
+          python3 -m pip install auditwheel
+          auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl
+        continue-on-error: false
+      - name: Show files
+        run: |
+          # e.g: remove chdb-0.11.4-cp310-cp310-linux_aarch64.whl, keep chdb-0.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
+          sudo rm -f dist/*linux_aarch64.whl
+          ls -lh dist
+        shell: bash
+      - name: Run tests
+        run: |
+          python3 -m pip install dist/*.whl
+          python3 -m pip install pandas pyarrow psutil
+          python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)"
+          make test
+        continue-on-error: false
+      - name: Upload wheels to release
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          gh release upload ${{ github.ref_name }} dist/*.whl --clobber
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+      - name: Packege libchdb.so
+        if: matrix.python-version == '3.12'
+        run: |
+          cp programs/local/chdb.h chdb.h
+          tar -czvf linux-aarch64-libchdb.tar.gz libchdb.so chdb.h
+      - name: Upload libchdb.so to release
+        if: startsWith(github.ref, 'refs/tags/v') && matrix.python-version == '3.12'
+        run: |
+          gh release upload ${{ github.ref_name }} linux-aarch64-libchdb.tar.gz --clobber
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+      - uses: actions/upload-artifact@v3
+        with:
+          path: |
+            ./dist/*.whl
+            ./linux-aarch64-libchdb.tar.gz
+      - name: Upload pypi
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          python3 -m pip install twine
+          python3 -m twine upload dist/*.whl
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+

contrib/jemalloc-cmake/CMakeLists.txt

@@ -34,7 +34,11 @@ if (OS_LINUX)
     # avoid spurious latencies and additional work associated with
     # MADV_DONTNEED. See
     # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
-    set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true")
+
+    # chdb: set background_thread true will cause Python 3.8 assert error on Linux aarch64:
+    #   <jemalloc>: contrib/jemalloc/src/jemalloc.c:4201: Failed assertion: "malloc_initialized()"
+    # so, disable it for now.
+    set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:false")
 else()
     set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
 endif()