Merge pull request #36 from pfxuan/hip-build

Support AMD GPUs via ROCm/HIP

.github/workflows/hip.yml

@@ -0,0 +1,191 @@
+name: OpenSplat (Ubuntu ROCm/HIP)
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [ assigned, opened, synchronize, reopened ]
+  release:
+    types: [ published, edited ]
+
+jobs:
+  build:
+    name: ${{ matrix.os }}-rocm-${{ matrix.rocm-version }}-torch-${{ matrix.torch-version }}-${{ matrix.cmake-build-type }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-22.04] # [ubuntu-22.04, ubuntu-20.04, ubuntu-18.04]
+        arch: [x64] # [x64, x86]
+        torch-version: [2.2.1] # [1.12.0, 1.13.0, 2.0.0, 2.1.0, 2.1.1, 2.1.2, 2.2.0, 2.2.1]
+        cuda-version: [11.8.0] # [12.3.2, 12.3.1, 12.1.1, 11.8.0, 11.7.1, 11.6.2, 11.5.2,11.4.4, 11.3.1, 11.2.2, 11.1.1, 11.0.3, cpu]
+        rocm-version: [5.7.1] # [5.4.2, 5.6.1, 5.7.1, 6.0.1, 6.0.2]
+        llvm-version: [16] # [12, 13, 14, 15, 16, 17]
+        cmake-build-type: [Release] # [Debug, ClangTidy]
+    env:
+      CCACHE_DIR: ${{ github.workspace }}/ccache
+      CCACHE_BASEDIR: ${{ github.workspace }}
+      CL: /MP
+      CMAKE_GENERATOR: Ninja
+      CMAKE_GENERATOR_PLATFORM: ${{matrix.arch}}
+      PYTORCH_ROCM_ARCH: gfx906
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Setup Ubuntu
+        run: |
+          wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+          sudo add-apt-repository "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${{ matrix.llvm-version }} main"
+          
+          sudo apt-get update
+          sudo apt-get install -y \
+            build-essential \
+            cmake \
+            ninja-build \
+            libopencv-dev \
+            llvm-${{ matrix.llvm-version }}-dev \
+            libclang-${{ matrix.llvm-version }}-dev \
+            clang-${{ matrix.llvm-version }} \
+            wget
+          sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${{ matrix.llvm-version }} 100
+          sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${{ matrix.llvm-version }} 100
+          sudo update-alternatives --set clang /usr/bin/clang-${{ matrix.llvm-version }}
+          sudo update-alternatives --set clang++ /usr/bin/clang++-${{ matrix.llvm-version }}
+          
+          wget -nv https://github.com/ccache/ccache/releases/download/v4.9.1/ccache-4.9.1-linux-x86_64.tar.xz
+          sudo tar xf ccache-4.9.1-linux-x86_64.tar.xz -C /usr/bin --strip-components=1 --no-same-owner ccache-4.9.1-linux-x86_64/ccache
+          rm -f ccache-*-linux-x86_64.tar.xz
+          ccache --version
+
+      - name: Free disk space
+        run: |
+          df -hT
+          # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/share/dotnet
+          # delete libraries for Android (12G), CodeQL (5.3G), PowerShell (1.3G), Swift (1.7G)
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf "${AGENT_TOOLSDIRECTORY}/CodeQL"
+          sudo rm -rf /usr/local/share/powershell
+          sudo rm -rf /usr/share/swift  
+          echo "Disk usage after cleanup:"
+          df -hT
+
+      - name: Sudo Tar Wrapper
+        run: |
+          # Workaround: https://github.com/containers/podman/discussions/17868
+          sudo mv -fv /usr/bin/tar /usr/bin/tar.orig
+          echo -e '#!/bin/sh\n\nsudo /usr/bin/tar.orig "$@"' | sudo tee -a /usr/bin/tar
+          sudo chmod +x /usr/bin/tar
+
+      - name: Restore CUDA Cache
+        uses: actions/cache@v4
+        id: cuda-cache
+        with:
+          key: cuda-${{matrix.CUDA-VERSION}}-ubuntu
+          path: |
+            /usr/local/cuda*
+
+      - name: Install CUDA
+        if: ${{ steps.cuda-cache.outputs.cache-hit != 'true' }}
+        run: |
+          bash .github/workflows/cuda/${{ runner.os }}.sh ${{ matrix.cuda-version }}
+
+      - name: Restore ROCm Cache
+        uses: actions/cache@v4
+        id: rocm-cache
+        with:
+          key: rocm-${{matrix.ROCM-VERSION}}-ubuntu
+          path: |
+            /opt/rocm*
+            /etc/alternatives/rocm
+
+      - name: Install ROCm
+        if: ${{ steps.rocm-cache.outputs.cache-hit != 'true' }}
+        run: |
+          bash .github/workflows/rocm/${{ runner.os }}.sh ${{ matrix.rocm-version }} ${{ matrix.os }}
+
+      - name: Setup ROCm And CUDA
+        shell: bash
+        run: |
+          ROCM_VER_FULL=${{ matrix.rocm-version }}
+          ROCM_VER_ARR=($(echo ${ROCM_VER_FULL} | tr "." " "))
+          ROCM_VER="${ROCM_VER_ARR[0]}.${ROCM_VER_ARR[1]}"
+          ROCM_VER_ID="${ROCM_VER_ARR[0]}_${ROCM_VER_ARR[1]}"
+          ROCM_VER_SHORT="rocm${ROCM_VER_ARR[0]}.${ROCM_VER_ARR[1]}"          
+          echo "ROCM_VER_SHORT=${ROCM_VER_SHORT}" >> ${GITHUB_ENV}
+
+          CUDA_VER_FULL=${{ matrix.cuda-version }}
+          CUDA_VER_ARR=($(echo ${CUDA_VER_FULL} | tr "." " "))
+          CUDA_VER="${CUDA_VER_ARR[0]}.${CUDA_VER_ARR[1]}"
+          CUDA_VER_ID="${CUDA_VER_ARR[0]}_${CUDA_VER_ARR[1]}"
+          CUDA_VER_SHORT="cu${CUDA_VER_ARR[0]}${CUDA_VER_ARR[1]}"          
+          echo "CUDA_VER_SHORT=${CUDA_VER_SHORT}" >> ${GITHUB_ENV}
+
+      - name: Restore LibTorch Cache
+        uses: actions/cache@v4
+        id: libtorch-cache
+        with:
+          key: libtorch-${{ matrix.torch-version }}-rocm5.7-ubuntu
+          path: |
+            /mnt/deps/libtorch
+
+      - name: Install LibTorch
+        if: ${{ steps.libtorch-cache.outputs.cache-hit != 'true' }}
+        run: |
+          sudo mkdir -p -m 777 /mnt/deps
+          wget --no-check-certificate -nv https://download.pytorch.org/libtorch/${ROCM_VER_SHORT}/libtorch-cxx11-abi-shared-with-deps-${{ matrix.torch-version }}%2B${ROCM_VER_SHORT}.zip -O /mnt/deps/libtorch.zip
+          unzip -q /mnt/deps/libtorch.zip -d /mnt/deps
+          rm /mnt/deps/libtorch.zip
+
+      - name: Cache Build
+        uses: actions/cache@v4
+        id: cache-builds
+        with:
+          key: ${{ matrix.os }}-rocm-${{ matrix.rocm-version }}-torch-${{ matrix.torch-version }}-${{ matrix.cmake-build-type }}-ccache-${{ github.run_id }}
+          restore-keys: ${{ matrix.os }}-rocm-${{ matrix.rocm-version }}-torch-${{ matrix.torch-version }}-${{ matrix.cmake-build-type }}-ccache-
+          path: ${{ env.CCACHE_DIR }}
+
+      - name: Configure And Build
+        run: |
+          set -x
+          source .github/workflows/cuda/${{ runner.os }}-env.sh ${CUDA_VER_SHORT}
+          export PATH=$PATH:/opt/rocm/bin
+          clang --version
+          find /opt/rocm -name __clang_cuda_runtime_wrapper.h
+          ls /opt/rocm/llvm/lib/clang
+          sudo ln -s -f /opt/rocm/llvm/lib/clang/16.0.0 /opt/rocm/llvm/lib/clang/16
+          
+          mkdir build
+          cd build
+          cmake .. \
+            -G${CMAKE_GENERATOR} \
+            -DCMAKE_BUILD_TYPE=${{ matrix.cmake-build-type }} \
+            -DCMAKE_C_COMPILER_LAUNCHER=$(which ccache) \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
+            -DCMAKE_PREFIX_PATH=/mnt/deps/libtorch \
+            -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/install \
+            -DGPU_RUNTIME=HIP \
+            -DHIP_ROOT_DIR=/opt/rocm \
+            -DOPENSPLAT_BUILD_SIMPLE_TRAINER=ON
+          ninja
+
+      - name: Save Converted Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          path: |
+            build/libgsplat.a
+            build/opensplat
+            build/simple_trainer
+
+      - name: Clean Compiler Cache
+        run: |
+          set -x
+          ccache --show-stats
+          ccache --evict-older-than 7d
+          ccache -s
+          ccache --show-stats

.github/workflows/rocm/Linux.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# ROCM major and minor version
+ROCM_VER_FULL=${1}
+ROCM_VER_ARR=($(echo ${ROCM_VER_FULL} | tr "." " "))
+ROCM_VER="${ROCM_VER_ARR[0]}.${ROCM_VER_ARR[1]}"
+
+if [[ ${2} == "ubuntu-20.04" ]]; then
+  OS_CODE_NAME=focal
+elif [[ ${2} == "ubuntu-22.04" ]]; then
+  OS_CODE_NAME=jammy
+else
+  echo "Unrecognized OS=${2}"
+  exit 1
+fi
+
+case ${ROCM_VER_FULL} in
+  6.0.2)
+    FILENAME=amdgpu-install_6.0.60002-1_all.deb
+    URL=https://repo.radeon.com/amdgpu-install/${ROCM_VER_FULL}/ubuntu/${OS_CODE_NAME}
+    ;;
+  6.0.1)
+    FILENAME=amdgpu-install_6.0.60001-1_all.deb
+    URL=https://repo.radeon.com/amdgpu-install/${ROCM_VER_FULL}/ubuntu/${OS_CODE_NAME}
+    ;;
+  5.7.1)
+    FILENAME=amdgpu-install_5.7.50701-1_all.deb
+    URL=https://repo.radeon.com/amdgpu-install/${ROCM_VER_FULL}/ubuntu/${OS_CODE_NAME}
+    ;;
+  5.6.1)
+    FILENAME=amdgpu-install_5.6.50601-1_all.deb
+    URL=https://repo.radeon.com/amdgpu-install/${ROCM_VER_FULL}/ubuntu/${OS_CODE_NAME}
+    ;;
+  5.4.2)
+    FILENAME=amdgpu-install_5.4.50402-1_all.deb
+    URL=https://repo.radeon.com/amdgpu-install/${ROCM_VER_FULL}/ubuntu/${OS_CODE_NAME}
+    ;;
+  *)
+    echo "Unrecognized ROCM_VERSION=${ROCM_VER}"
+    exit 1
+    ;;
+esac
+
+wget -nv ${URL}/${FILENAME}
+sudo dpkg -i ${FILENAME}
+sudo amdgpu-install -y --usecase=hip,rocm --no-dkms
+
+sudo apt-get -qq update
+sudo apt install -y hip-dev hipify-clang
+sudo apt clean
+
+rm -f ${FILENAME}

CMakeLists.txt

@@ -1,22 +1,49 @@
-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.21)
 project(opensplat)
 
 set(OPENSPLAT_BUILD_SIMPLE_TRAINER OFF CACHE BOOL "Build simple trainer applications")
+set(GPU_RUNTIME "CUDA" CACHE STRING "HIP or CUDA")
 set(OPENCV_DIR "OPENCV_DIR-NOTFOUND" CACHE "OPENCV_DIR" "Path to the OPENCV installation directory")
 
 if(NOT CMAKE_BUILD_TYPE)
 set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." FORCE)
 endif()
 
-set(CMAKE_CUDA_ARCHITECTURES 70 75)
-set(CMAKE_CXX_STANDARD 17)
-set(CUDA_STANDARD 17)
+enable_language(${GPU_RUNTIME})
+set(CMAKE_${GPU_RUNTIME}_STANDARD 17)
+set(${GPU_RUNTIME}_STANDARD 17)
+
+if(GPU_RUNTIME STREQUAL "CUDA")
+    set(CMAKE_CUDA_ARCHITECTURES 70 75)
+    find_package(CUDAToolkit REQUIRED)
+else()
+    set(USE_HIP ON CACHE BOOL "Use HIP for GPU acceleration")
+
+    if(NOT DEFINED HIP_PATH)
+        if(NOT DEFINED ENV{HIP_PATH})
+            set(HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed")
+        else()
+            set(HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to which HIP has been installed")
+        endif()
+    endif()
+    set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH})
+    find_package(HIP REQUIRED)
+
+    file(GLOB_RECURSE GSPLAT_GPU_SRC LIST_DIRECTORIES False vendor/gsplat/*.cu)
+    set_source_files_properties(${GSPLAT_GPU_SRC} PROPERTIES LANGUAGE HIP)
+
+    if(WIN32)
+        set(ROCM_ROOT "$ENV{HIP_PATH}" CACHE PATH "Root directory of the ROCm installation")
+    else()
+        set(ROCM_ROOT "/opt/rocm" CACHE PATH "Root directory of the ROCm installation")
+    endif()
+    list(APPEND CMAKE_PREFIX_PATH "${ROCM_ROOT}")
+endif()
 
 if (NOT WIN32 AND NOT APPLE)
     set(STDPPFS_LIBRARY stdc++fs)
 endif()
 
-find_package(CUDAToolkit REQUIRED)
 find_package(Torch REQUIRED)
 find_package(OpenCV HINTS "${OPENCV_DIR}" REQUIRED)
 
@@ -26,7 +53,14 @@ endif()
 set(OpenCV_LIBS opencv_core opencv_imgproc opencv_highgui opencv_calib3d)
 
 add_library(gsplat vendor/gsplat/forward.cu vendor/gsplat/backward.cu vendor/gsplat/bindings.cu vendor/gsplat/helpers.cuh)
-target_link_libraries(gsplat PUBLIC cuda)
+if(GPU_RUNTIME STREQUAL "CUDA")
+    set(GPU_LIBRARIES "cuda")
+    target_link_libraries(gsplat PUBLIC cuda)
+else(GPU_RUNTIME STREQUAL "HIP")
+    set(GPU_INCLUDE_DIRS "${ROCM_ROOT}/include")
+    target_compile_definitions(gsplat PRIVATE USE_HIP __HIP_PLATFORM_AMD__)
+endif()
+
 target_include_directories(gsplat PRIVATE
     ${PROJECT_SOURCE_DIR}/vendor/glm        
     ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}
@@ -36,13 +70,21 @@ set_target_properties(gsplat PROPERTIES LINKER_LANGUAGE CXX)
 set_target_properties(gsplat PROPERTIES CUDA_ARCHITECTURES "70;75")
 
 add_executable(opensplat opensplat.cpp point_io.cpp nerfstudio.cpp model.cpp kdtree_tensor.cpp spherical_harmonics.cpp cv_utils.cpp utils.cpp project_gaussians.cpp rasterize_gaussians.cpp ssim.cpp optim_scheduler.cpp colmap.cpp input_data.cpp tensor_math.cpp)
-target_include_directories(opensplat PRIVATE ${PROJECT_SOURCE_DIR}/vendor/glm)
-target_link_libraries(opensplat PUBLIC ${STDPPFS_LIBRARY} cuda gsplat ${TORCH_LIBRARIES} ${OpenCV_LIBS})
+set_property(TARGET opensplat PROPERTY CXX_STANDARD 17)
+target_include_directories(opensplat PRIVATE ${PROJECT_SOURCE_DIR}/vendor/glm ${GPU_INCLUDE_DIRS})
+target_link_libraries(opensplat PUBLIC ${STDPPFS_LIBRARY} ${GPU_LIBRARIES} gsplat ${TORCH_LIBRARIES} ${OpenCV_LIBS})
+if(GPU_RUNTIME STREQUAL "HIP")
+    target_compile_definitions(opensplat PRIVATE USE_HIP __HIP_PLATFORM_AMD__)
+endif()
 
 if(OPENSPLAT_BUILD_SIMPLE_TRAINER)
     add_executable(simple_trainer simple_trainer.cpp project_gaussians.cpp rasterize_gaussians.cpp)
-    target_include_directories(simple_trainer PRIVATE ${PROJECT_SOURCE_DIR}/vendor/glm)
-    target_link_libraries(simple_trainer PUBLIC cuda gsplat ${TORCH_LIBRARIES} ${OpenCV_LIBS})
+    target_include_directories(simple_trainer PRIVATE ${PROJECT_SOURCE_DIR}/vendor/glm ${GPU_INCLUDE_DIRS})
+    target_link_libraries(simple_trainer PUBLIC ${GPU_LIBRARIES} gsplat ${TORCH_LIBRARIES} ${OpenCV_LIBS})
+    set_property(TARGET simple_trainer PROPERTY CXX_STANDARD 17)
+    if(GPU_RUNTIME STREQUAL "HIP")
+        target_compile_definitions(simple_trainer PRIVATE USE_HIP __HIP_PLATFORM_AMD__)
+    endif()
 endif()
 
 # The following code block is suggested to be used on Windows.