[CI] Auto Format Checking and test checking. (#73)

* chore: Update support matrix in README * Move bitblas package to root * Remove unused code files * Create soft link for tvm * Create soft link for tvm * Update softlink paths for tvm in setup.py * Refactor import statements to use relative paths * fix test linear * Move bitblas package to root * Move bitblas package to root * refactor splitk test * Fix assert statement in ladder_permutate_impl.py * Refactor test_ladder_permutate_ops.py for improved readability and maintainability * Refactor test_ladder_permutate_ops.py for improved readability and maintainability * improve and evaluate the test scripts. * resolve security issue. * ci test * requirements install * enhance installation script. * make sure the origin/main branch exist. * fetch all history. * install * refactor script install with pip install * chore: Update installation script to include pip wheel installation * chore: Update pip installation in CI workflow * chore: Update Python version in CI workflow to 3.9 * chore: Update CI workflow to include pip wheel installation and Python 3.9 * chore: Update requirements-dev.txt with wheel and setuptools dependencies * chore: Update CI workflow to include pip wheel installation * chore: Update CI workflow to include pip wheel installation and Python 3.9 * wheel test * add * update setup.pt * chore: Update setup.py to improve compatibility with Python 3.9 and include pip wheel installation * trick invarent to make the test pass. * chore: Update CI workflow to include running tests with pytest * Lint Fix * chore: Update CI workflow to include running tests with pytest
microsoft · Jul 5, 2024 · 1b7e52d · 1b7e52d
1 parent 8804d77
commit 1b7e52d
Show file tree

Hide file tree

Showing 7 changed files with 256 additions and 73 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,67 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  format-check:
+    runs-on: self-hosted
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+
+    - name: Create virtual environment
+      run: python -m venv bitblas_ci
+
+    - name: Activate virtual environment and install dependencies
+      run: |
+        source bitblas_ci/bin/activate
+        python -m pip install --upgrade pip
+        if [ -f requirements-dev.txt ]; then python -m pip install -r requirements-dev.txt; fi
+
+    - name: Run format check
+      run: |
+        source bitblas_ci/bin/activate
+        ./format.sh
+
+  build-test:
+    runs-on: self-hosted
+    needs: format-check
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+
+    - name: Create virtual environment
+      run: python -m venv bitblas_ci
+
+    - name: Activate virtual environment and install dependencies
+      run: |
+        source bitblas_ci/bin/activate
+        python -m pip install --upgrade pip
+        if [ -f requirements-dev.txt ]; then python -m pip install -r requirements-dev.txt; fi
+
+    - name: Install project in wheel mode
+      run: |
+        source bitblas_ci/bin/activate
+        python -m pip install .
+
+    - name: Run tests
+      run: |
+        source bitblas_ci/bin/activate
+        cd testing/python
+        python -m pytest
diff --git a/format.sh b/format.sh
@@ -64,7 +64,13 @@ format_changed() {
     #
     # `diff-filter=ACM` and $MERGEBASE is to ensure we only format files that
     # exist on both branches.
-    MERGEBASE="$(git merge-base origin/main HEAD)"
+    if git show-ref --verify --quiet refs/remotes/origin/main; then
+        BASE_BRANCH="origin/main"
+    else
+        BASE_BRANCH="main"
+    fi
+
+    MERGEBASE="$(git merge-base $BASE_BRANCH HEAD)"
 
     if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
         git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs -P 5 \
@@ -110,7 +116,13 @@ spell_check_changed() {
     #
     # `diff-filter=ACM` and $MERGEBASE is to ensure we only lint files that
     # exist on both branches.
-    MERGEBASE="$(git merge-base origin/main HEAD)"
+    if git show-ref --verify --quiet refs/remotes/origin/main; then
+        BASE_BRANCH="origin/main"
+    else
+        BASE_BRANCH="main"
+    fi
+
+    MERGEBASE="$(git merge-base $BASE_BRANCH HEAD)"
 
     if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
         git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
@@ -148,7 +160,13 @@ lint_changed() {
     #
     # `diff-filter=ACM` and $MERGEBASE is to ensure we only lint files that
     # exist on both branches.
-    MERGEBASE="$(git merge-base origin/main HEAD)"
+    if git show-ref --verify --quiet refs/remotes/origin/main; then
+        BASE_BRANCH="origin/main"
+    else
+        BASE_BRANCH="main"
+    fi
+
+    MERGEBASE="$(git merge-base $BASE_BRANCH HEAD)"
 
     if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
         git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \

diff --git a/install.sh b/install.sh
@@ -7,7 +7,47 @@
 pip install -r requirements.txt
 
 # install llvm
-apt-get install llvm-10
+LLVM_VERSION="10.0.1"
+IS_AARCH64=false
+EXTRACT_PATH="3rdparty"
+
+UBUNTU_VERSION="16.04"
+if [[ "$LLVM_VERSION" > "16.0.0" ]]; then
+    UBUNTU_VERSION="20.04"
+elif [[ "$LLVM_VERSION" > "13.0.0" ]]; then
+    UBUNTU_VERSION="18.04"
+fi
+
+BASE_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}"
+if $IS_AARCH64; then
+    FILE_NAME="clang+llvm-${LLVM_VERSION}-aarch64-linux-gnu.tar.xz"
+else
+    FILE_NAME="clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-${UBUNTU_VERSION}.tar.xz"
+fi
+DOWNLOAD_URL="${BASE_URL}/${FILE_NAME}"
+
+mkdir -p "$EXTRACT_PATH"
+
+echo "Downloading $FILE_NAME from $DOWNLOAD_URL"
+curl -L -o "${EXTRACT_PATH}/${FILE_NAME}" "$DOWNLOAD_URL"
+
+if [ $? -ne 0 ]; then
+    echo "Download failed!"
+    exit 1
+fi
+
+echo "Extracting $FILE_NAME to $EXTRACT_PATH"
+tar -xJf "${EXTRACT_PATH}/${FILE_NAME}" -C "$EXTRACT_PATH"
+
+if [ $? -ne 0 ]; then
+    echo "Extraction failed!"
+    exit 1
+fi
+
+echo "Download and extraction completed successfully."
+
+LLVM_CONFIG_PATH="${EXTRACT_PATH}/$(basename ${FILE_NAME} .tar.xz)/bin/llvm-config"
+echo "LLVM config path: $LLVM_CONFIG_PATH"
 
 # clone and build tvm
 git submodule update --init --recursive
@@ -16,7 +56,7 @@ cd 3rdparty/tvm
 mkdir build
 cp cmake/config.cmake build
 cd build
-echo "set(USE_LLVM llvm-config-10)" >> config.cmake && echo "set(USE_CUDA ON)" >> config.cmake
+echo "set(USE_LLVM $LLVM_CONFIG_PATH)" >> config.cmake && echo "set(USE_CUDA ON)" >> config.cmake
 
 cmake .. && make -j && cd ../../..
 

diff --git a/maint/scripts/installation.sh b/maint/scripts/installation.sh
@@ -7,7 +7,47 @@
 pip install -r requirements.txt
 
 # install llvm
-apt-get install llvm-10
+LLVM_VERSION="10.0.1"
+IS_AARCH64=false
+EXTRACT_PATH="3rdparty"
+
+UBUNTU_VERSION="16.04"
+if [[ "$LLVM_VERSION" > "16.0.0" ]]; then
+    UBUNTU_VERSION="20.04"
+elif [[ "$LLVM_VERSION" > "13.0.0" ]]; then
+    UBUNTU_VERSION="18.04"
+fi
+
+BASE_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}"
+if $IS_AARCH64; then
+    FILE_NAME="clang+llvm-${LLVM_VERSION}-aarch64-linux-gnu.tar.xz"
+else
+    FILE_NAME="clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-${UBUNTU_VERSION}.tar.xz"
+fi
+DOWNLOAD_URL="${BASE_URL}/${FILE_NAME}"
+
+mkdir -p "$EXTRACT_PATH"
+
+echo "Downloading $FILE_NAME from $DOWNLOAD_URL"
+curl -L -o "${EXTRACT_PATH}/${FILE_NAME}" "$DOWNLOAD_URL"
+
+if [ $? -ne 0 ]; then
+    echo "Download failed!"
+    exit 1
+fi
+
+echo "Extracting $FILE_NAME to $EXTRACT_PATH"
+tar -xJf "${EXTRACT_PATH}/${FILE_NAME}" -C "$EXTRACT_PATH"
+
+if [ $? -ne 0 ]; then
+    echo "Extraction failed!"
+    exit 1
+fi
+
+echo "Download and extraction completed successfully."
+
+LLVM_CONFIG_PATH="${EXTRACT_PATH}/$(basename ${FILE_NAME} .tar.xz)/bin/llvm-config"
+echo "LLVM config path: $LLVM_CONFIG_PATH"
 
 # clone and build tvm
 git submodule update --init --recursive
@@ -16,7 +56,7 @@ cd 3rdparty/tvm
 mkdir build
 cp cmake/config.cmake build
 cd build
-echo "set(USE_LLVM llvm-config-10)" >> config.cmake && echo "set(USE_CUDA ON)" >> config.cmake
+echo "set(USE_LLVM $LLVM_CONFIG_PATH)" >> config.cmake && echo "set(USE_CUDA ON)" >> config.cmake
 
 cmake .. && make -j && cd ../../..
 

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -28,3 +28,5 @@ tornado
 torch
 thefuzz
 tabulate
+wheel
+setuptools
diff --git a/setup.py b/setup.py
@@ -8,7 +8,6 @@
 from setuptools.command.install import install
 from setuptools.command.build_py import build_py
 from setuptools.command.sdist import sdist
-from wheel.bdist_wheel import bdist_wheel
 import distutils.dir_util
 from typing import List
 import re
@@ -66,7 +65,7 @@ def get_nvcc_cuda_version():
 
 
 def get_bitblas_version(with_cuda=True, with_system_info=True) -> str:
-    version = find_version(get_path("python/bitblas", "__init__.py"))
+    version = find_version(get_path("bitblas", "__init__.py"))
     local_version_parts = []
     if with_system_info:
         local_version_parts.append(get_system_info().replace("-", "."))
@@ -209,8 +208,6 @@ def run(self):
         build_tvm(llvm_path)
         # Continue with the standard installation process
         install.run(self)
-        # Create softlink for bitblas
-        create_softlink(tvm_path="../3rdparty/tvm/python/tvm", bitblas_path="bitblas/tvm")
 
 
 class BitBLASBuilPydCommand(build_py):
@@ -224,8 +221,6 @@ def run(self):
         _, llvm_path = setup_llvm_for_tvm()
         # Build TVM
         build_tvm(llvm_path)
-        # Create softlink for bitblas
-        create_softlink(tvm_path="../3rdparty/tvm/python/tvm", bitblas_path="bitblas/tvm")
 
         # Copy the built TVM to the package directory
         TVM_PREBUILD_ITEMS = [
@@ -268,15 +263,15 @@ def make_distribution(self):
 
 setup(
     name=PACKAGE_NAME,
-    version=get_bitblas_version(with_cuda=False, with_system_info=False) if PYPI_BUILD else get_bitblas_version(),
-    packages=find_packages(where="python"),
-    package_dir={"": "python"},
+    version=get_bitblas_version(with_cuda=False, with_system_info=False)
+    if PYPI_BUILD else get_bitblas_version(),
+    packages=find_packages(where="."),
+    package_dir={"": "."},
     author="Microsoft Research",
     description="A light weight framework to generate high performance CUDA/HIP code for BLAS operators.",
     long_description=read_readme(),
     long_description_content_type='text/markdown',
-    platforms=["Environment :: GPU :: NVIDIA CUDA", 
-               "Operating System :: POSIX :: Linux"],
+    platforms=["Environment :: GPU :: NVIDIA CUDA", "Operating System :: POSIX :: Linux"],
     license="MIT",
     keywords="BLAS, CUDA, HIP, Code Generation, TVM",
     url="https://github.com/microsoft/BitBLAS",