diff --git a/.github/workflows/build_linux_arm64_wheels-gh.yml b/.github/workflows/build_linux_arm64_wheels-gh.yml new file mode 100644 index 00000000000..7737a8a6a5a --- /dev/null +++ b/.github/workflows/build_linux_arm64_wheels-gh.yml @@ -0,0 +1,180 @@ +name: Build Linux arm64 + +on: + workflow_dispatch: + inputs: + TAG_NAME: + description: 'Release Version Tag' + required: true + release: + types: [created] + push: + branches: + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - main + paths-ignore: + - '**/*.md' + + +jobs: + build_wheels_linux: + name: ${{ matrix.os }} py${{ matrix.python-version }} + runs-on: GH-Linux-ARM64 + strategy: + fail-fast: false + matrix: + os: [ ubuntu-24.04 ] + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] + # python-version: [ "3.7" ] + env: + RUNNER_OS: ${{ matrix.os }} + PYTHON_VERSION: ${{ matrix.python-version }} + steps: + - name: Install python + uses: actions/setup-python@v4 + with: + python-version: "${{ matrix.python-version }}" + - name: Install clang++ for Ubuntu + if: matrix.os == 'ubuntu-24.04' + run: | + pwd + uname -a + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 18 + which clang++-18 + clang++-18 --version + sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget + ccache -s + - name: Update git + run: | + sudo add-apt-repository ppa:git-core/ppa -y + sudo apt-get update + sudo apt-get install -y git + git --version + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Restore submodules cache + uses: actions/cache/restore@v3 + id: cache + with: + path: | + contrib/** + !contrib/*-cmake + !**/CMakeLists.txt + key: | + submodule-${{ hashFiles('.gitmodules') }} + - name: Update submodules if cache miss + if: steps.cache.outputs.cache-hit != 'true' + run: | + git submodule update --init --recursive --jobs 4 + - name: Save submodules cache + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: | + contrib/** + !contrib/*-cmake + !**/CMakeLists.txt + key: | + submodule-${{ hashFiles('.gitmodules') }} + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.os }} + max-size: 5G + append-timestamp: true + - name: remove old clang and link clang-18 to clang + if: matrix.os == 'ubuntu-24.04' + run: | + sudo rm -f /usr/bin/clang || true + sudo ln -s /usr/bin/clang-18 /usr/bin/clang + sudo rm -f /usr/bin/clang++ || true + sudo ln -s /usr/bin/clang++-18 /usr/bin/clang++ + which clang++ + clang++ --version + - name: Run chdb/build.sh + run: | + python3 -m pip install pybind11 setuptools + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + bash ./chdb/build.sh + python3 -m pip install pandas pyarrow + bash -x ./chdb/test_smoke.sh + continue-on-error: false + - name: Run libchdb stub in examples dir + run: | + bash -x ./examples/runStub.sh + - name: Check ccache statistics + run: | + ccache -s + ls -lh chdb + df -h + - name: Install dependencies for building wheels + run: python3 -m pip install -U pip tox pybind11 twine setuptools wheel + - name: Build wheels + run: | + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + make wheel + - name: Install patchelf from github + run: | + wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz + tar -xvf patchelf.tar.gz + sudo cp bin/patchelf /usr/bin/ + sudo chmod +x /usr/bin/patchelf + patchelf --version + - name: Audit wheels + run: | + python3 -m pip install auditwheel + auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl + continue-on-error: false + - name: Show files + run: | + # e.g: remove chdb-0.11.4-cp310-cp310-linux_aarch64.whl, keep chdb-0.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + sudo rm -f dist/*linux_aarch64.whl + ls -lh dist + shell: bash + - name: Run tests + run: | + python3 -m pip install dist/*.whl + python3 -m pip install pandas pyarrow psutil + python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)" + make test + continue-on-error: false + - name: Upload wheels to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} dist/*.whl --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - name: Packege libchdb.so + if: matrix.python-version == '3.12' + run: | + cp programs/local/chdb.h chdb.h + tar -czvf linux-aarch64-libchdb.tar.gz libchdb.so chdb.h + - name: Upload libchdb.so to release + if: startsWith(github.ref, 'refs/tags/v') && matrix.python-version == '3.12' + run: | + gh release upload ${{ github.ref_name }} linux-aarch64-libchdb.tar.gz --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - uses: actions/upload-artifact@v3 + with: + path: | + ./dist/*.whl + ./linux-aarch64-libchdb.tar.gz + - name: Upload pypi + if: startsWith(github.ref, 'refs/tags/v') + run: | + python3 -m pip install twine + python3 -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + diff --git a/.github/workflows/build_linux_arm64_wheels.yml b/.github/workflows/build_linux_arm64_wheels.yml deleted file mode 100644 index 90481ddcafc..00000000000 --- a/.github/workflows/build_linux_arm64_wheels.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: Build ARM64 - -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - - -defaults: - run: - shell: bash -leo pipefail {0} - -jobs: - build_wheels_linux_arm64: - name: ${{ matrix.os }} py${{ matrix.python-version }} - runs-on: - - self-hosted - - ARM64 - strategy: - fail-fast: false - matrix: - os: [ ubuntu-22.04 ] - env: - RUNNER_OS: ${{ matrix.os }} - PYTHON_VERSIONS: "3.8 3.9 3.10 3.11 3.12" - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Restore submodules cache - run: | - cp -a /builder_cache/contrib ./ - - name: remove old clang and link clang-18 to clang - if: matrix.os == 'ubuntu-22.04' - run: | - sudo rm -f /usr/bin/clang || true - sudo ln -s /usr/bin/clang-18 /usr/bin/clang - sudo rm -f /usr/bin/clang++ || true - sudo ln -s /usr/bin/clang++-18 /usr/bin/clang++ - which clang++ - clang++ --version - - name: Make linux-arm64 - run: | - bash -l -e chdb/build_linux_arm64.sh - continue-on-error: false - - name: Show libchdb - run: | - ls -lh - nm libchdb.so | grep query_stable || true - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - - name: Check ccache statistics - run: | - ccache -s - ls -lh chdb - df -h - - name: Audit wheels - run: | - export PYENV_ROOT="$HOME/.pyenv" - [[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH" - eval "$(pyenv init -)" - pyenv local "${{ matrix.python-version }}" - python3 -m pip install auditwheel - auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl - continue-on-error: false - - name: Show files - run: | - # e.g: remove chdb-0.11.4-cp310-cp310-linux_aarch64.whl, keep chdb-0.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl - sudo rm -f dist/*linux_aarch64.whl - ls -lh dist - shell: bash - continue-on-error: false - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - cp programs/local/chdb.h chdb.h - tar -czvf linux-aarch64-libchdb.tar.gz libchdb.so chdb.h - gh release upload ${{ github.ref_name }} linux-aarch64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v3 - with: - path: | - ./dist/*.whl - ./linux-aarch64-libchdb.tar.gz - - name: Upload pypi - if: ${{ startsWith(github.ref, 'refs/tags/v') || startsWIth(github.event.inputs.TAG_NAME, 'v') }} - run: | - export PYENV_ROOT="$HOME/.pyenv" - [[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH" - eval "$(pyenv init -)" - pyenv local "${{ matrix.python-version }}" - python3 -m pip install twine - python3 -m twine upload ./dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - - diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index c74f532d29a..05dc95f7da4 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -34,7 +34,11 @@ if (OS_LINUX) # avoid spurious latencies and additional work associated with # MADV_DONTNEED. See # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. - set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true") + + # chdb: set background_thread true will cause Python 3.8 assert error on Linux aarch64: + # : contrib/jemalloc/src/jemalloc.c:4201: Failed assertion: "malloc_initialized()" + # so, disable it for now. + set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:false") else() set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000") endif()