Skip to content

github: Update the main github actions workflow for musl #1300

github: Update the main github actions workflow for musl

github: Update the main github actions workflow for musl #1300

Workflow file for this run

name: Builds
on:
push:
branches:
- '**'
pull_request:
schedule:
- cron: '0 0 * * *'
permissions:
contents: read
jobs:
# For scheduled builds, get the commit ids of the latest versions to build.
#
# Doing this as one single job, which then propagates the information further
# to the other jobs, so that all build jobs in one workflow builds the exact
# same version. This allows trusting builds without assertions enabled
# slightly more, when we know a separate build of the same version but with
# assertions enabled, has passed some amount of tests.
prepare:
if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw')
runs-on: ubuntu-latest
outputs:
LLVM_VERSION: ${{steps.get-versions.outputs.LLVM_VERSION}}
MUSL_VERSION: ${{steps.get-versions.outputs.MUSL_VERSION}}
TAG: ${{steps.get-tag.outputs.TAG}}
COMMIT_DATE_UNIX: ${{steps.get-tag.outputs.COMMIT_DATE_UNIX}}
BUILD_DATE: ${{steps.get-tag.outputs.BUILD_DATE}}
BUILD_DATE_UNIX: ${{steps.get-tag.outputs.BUILD_DATE_UNIX}}
steps:
- uses: actions/checkout@v4
with:
sparse-checkout: .
- name: Select build tag
id: get-tag
run: |
if ${{github.event_name == 'schedule'}}; then
TAG=nightly
else
TAG=$(TZ=UTC date +%Y%m%d)
fi
echo TAG=$TAG >> $GITHUB_OUTPUT
echo COMMIT_DATE_UNIX=$(git log -1 --pretty=%ct $GITHUB_SHA) >> $GITHUB_OUTPUT
BUILD_DATE=$(date -u '+%FT%TZ')
echo BUILD_DATE=${BUILD_DATE} >> $GITHUB_OUTPUT
echo BUILD_DATE_UNIX=$(date -d "${BUILD_DATE}" +%s) >> $GITHUB_OUTPUT
cat $GITHUB_OUTPUT
cat $GITHUB_OUTPUT >> parameters.txt
- name: Check latest version
if: github.event_name == 'schedule'
id: get-versions
run: |
echo LLVM_VERSION=$(git ls-remote https://github.com/llvm/llvm-project.git | grep 'refs/heads/main$' | awk '{print $1}') >> $GITHUB_OUTPUT
echo MUSL_VERSION=$(git ls-remote https://github.com/bminor/musl.git | grep 'refs/heads/master$' | awk '{print $1}') >> $GITHUB_OUTPUT
cat $GITHUB_OUTPUT
cat $GITHUB_OUTPUT >> parameters.txt
- uses: actions/upload-artifact@v4
with:
name: parameters
path: |
parameters.txt
retention-days: 7
# Build a cross compiler for Linux, targeting Windows.
linux:
if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw')
needs: [prepare]
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v4
- name: Build
env:
LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}}
MUSL_VERSION: ${{needs.prepare.outputs.MUSL_VERSION}}
TAG: ${{needs.prepare.outputs.TAG}}
SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}}
BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}}
run: |
sudo apt-get update && sudo apt-get install ninja-build
# Skip dynamic library dependencies that might make it harder to
# run the binaries on other distros (and that have little use within
# llvm-musl).
LLVM_CMAKEFLAGS="-DLLVM_ENABLE_LIBXML2=OFF -DLLVM_ENABLE_TERMINFO=OFF -DLLDB_ENABLE_PYTHON=OFF" ./build-all.sh $(pwd)/install/llvm-musl
.github/workflows/store-version.sh install/llvm-musl/versions.txt
- name: Test using the toolchain
run: |
sudo apt-get update && sudo apt-get install qemu-user-static libc6-arm64-cross libstdc++6-arm64-cross
#./test-libcxx-module.sh $(pwd)/install/llvm-musl
./run-tests.sh $(pwd)/install/llvm-musl
- name: Package the toolchain
env:
TAG: ${{needs.prepare.outputs.TAG}}
run: |
cd install
DISTRO=ubuntu-$(grep DISTRIB_RELEASE /etc/lsb-release | cut -f 2 -d =)-$(uname -m)
NAME=llvm-musl-$TAG-$DISTRO
mv llvm-musl $NAME
tar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 --sort=name --mtime="$BUILD_DATE" $NAME
- uses: actions/upload-artifact@v4
with:
name: linux-x86_64-toolchain
path: |
llvm-musl-*.tar.xz
retention-days: 7
# Crosscompile the toolchain for running on Linux on a different architecture, bundle the runtime
# libraries that were built in the 'linux' step above.
linux-cross-aarch64:
if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw')
needs: [linux, prepare]
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v4
- name: Build
env:
LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}}
SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}}
run: |
sudo apt-get update && sudo apt-get install ninja-build g++-aarch64-linux-gnu
./build-all.sh $(pwd)/install/llvm-musl --no-runtimes --host=aarch64-linux-gnu
.github/workflows/store-version.sh install/llvm-musl/versions.txt
- uses: actions/download-artifact@v4
with:
name: linux-x86_64-toolchain
- name: Unpack native toolchain
run: |
tar -Jxf llvm-musl-*.tar.xz
rm llvm-musl-*.tar.xz
mv llvm-musl* llvm-musl-native
- name: Assemble the cross-built toolchain
run: |
./prepare-cross-toolchain-unix.sh $(pwd)/llvm-musl-native $(pwd)/install/llvm-musl
- name: Test using the cross-built assembled toolchain
run: |
sudo apt-get update && sudo apt-get install qemu-user-static libc6-arm64-cross libstdc++6-arm64-cross
#QEMU_LD_PREFIX=/usr/aarch64-linux-gnu ./test-libcxx-module.sh $(pwd)/install/llvm-musl
QEMU_LD_PREFIX=/usr/aarch64-linux-gnu ./run-tests.sh $(pwd)/install/llvm-musl
- name: Package the toolchain
env:
TAG: ${{needs.prepare.outputs.TAG}}
BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}}
run: |
cd install
DISTRO=ubuntu-$(grep DISTRIB_RELEASE /etc/lsb-release | cut -f 2 -d =)-aarch64
NAME=llvm-musl-$TAG-$DISTRO
mv llvm-musl $NAME
tar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 --sort=name --mtime="$BUILD_DATE" $NAME
- uses: actions/upload-artifact@v4
with:
name: linux-aarch64-toolchain
path: |
llvm-musl-*.tar.xz
retention-days: 7
# Build a cross compiler for macOS, targeting Windows.
macos:
if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw')
needs: [linux, prepare]
runs-on: macos-14
steps:
- uses: actions/checkout@v4
- name: Build
env:
LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}}
TAG: ${{needs.prepare.outputs.TAG}}
SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}}
BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}}
run: |
brew install ninja
# Disable zstd and python. Both are available on the runners, but
# installed with homebrew, and only available in the native (x86_64)
# form. Therefore, autodetection will pick them up, but linking
# universal binaries fails as those libraries are unavailable in the
# other (arm64) architecture form.
MACOS_REDIST=1 LLVM_CMAKEFLAGS="-DLLVM_ENABLE_ZSTD=OFF -DLLDB_ENABLE_PYTHON=OFF" ./build-all.sh $(pwd)/install/llvm-musl --no-runtimes
.github/workflows/store-version.sh install/llvm-musl/versions.txt
- uses: actions/download-artifact@v4
with:
name: linux-x86_64-toolchain
- name: Unpack original toolchain
run: |
tar -Jxf llvm-musl-*.tar.xz
rm llvm-musl-*.tar.xz
mv llvm-musl* llvm-musl-original
- name: Assemble the toolchain
run: |
./prepare-cross-toolchain-unix.sh $(pwd)/llvm-musl-original $(pwd)/install/llvm-musl
- name: Test using the assembled toolchain
run: |
#./test-libcxx-module.sh $(pwd)/install/llvm-mingw
./run-tests.sh $(pwd)/install/llvm-musl
- name: Package the toolchain
env:
TAG: ${{needs.prepare.outputs.TAG}}
SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}}
BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}}
run: |
cd install
NAME=llvm-musl-$TAG-macos-universal
mv llvm-musl $NAME
gtar -Jcf ../$NAME.tar.xz --format=ustar --numeric-owner --owner=0 --group=0 --sort=name --mtime="$BUILD_DATE" $NAME
- uses: actions/upload-artifact@v4
with:
name: macos-toolchain
path: |
llvm-musl-*.tar.xz
retention-days: 7
# Use the Linux cross compilers built in the first step to cross compile
# llvm and make a proper standalone toolchain for Windows (for all 4
# architectures). The binaries built here match actual releases quite closely.
linux-cross-windows:
if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw')
needs: [linux, prepare]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
arch:
- i686
- x86_64
- armv7
- aarch64
steps:
- name: Unpack cross toolchain
run: |
curl -LO https://github.com/mstorsjo/llvm-mingw/releases/download/20240726/llvm-mingw-20240726-ucrt-ubuntu-20.04-x86_64.tar.xz
tar -Jxf llvm-mingw-*.tar.xz
rm llvm-mingw-*.tar.xz
sudo mv llvm-mingw* /opt/llvm-mingw
echo /opt/llvm-mingw/bin >> $GITHUB_PATH
- uses: actions/checkout@v4
- name: Build
env:
LLVM_VERSION: ${{needs.prepare.outputs.LLVM_VERSION}}
TAG: ${{needs.prepare.outputs.TAG}}
SOURCE_DATE_EPOCH: ${{needs.prepare.outputs.COMMIT_DATE_UNIX}}
BUILD_DATE: ${{needs.prepare.outputs.BUILD_DATE}}
run: |
sudo apt-get update && sudo apt-get install ninja-build
mkdir -p install/llvm-musl/bin
cp /opt/llvm-mingw/${{matrix.arch}}-w64-mingw32/bin/lib{c++,unwind}.dll install/llvm-musl/bin
./build-all.sh $(pwd)/install/llvm-musl --disable-clang-tools-extra --no-runtimes --host=${{matrix.arch}}-w64-mingw32
.github/workflows/store-version.sh install/llvm-musl/versions.txt
- uses: actions/download-artifact@v4
with:
name: linux-x86_64-toolchain
- name: Unpack original toolchain
run: |
tar -Jxf llvm-musl-*.tar.xz
rm llvm-musl-*.tar.xz
mv llvm-musl* llvm-musl-original
- name: Assemble the cross-built toolchain
run: |
./prepare-cross-toolchain-unix.sh $(pwd)/llvm-musl-original $(pwd)/install/llvm-musl
- name: Package the toolchain
env:
TAG: ${{needs.prepare.outputs.TAG}}
run: |
cd install
NAME=llvm-musl-$TAG-windows-${{matrix.arch}}
mv llvm-musl $NAME
zip -9rq ../$NAME.zip $NAME
- uses: actions/upload-artifact@v4
with:
name: windows-${{matrix.arch}}-toolchain
path: |
llvm-musl-*.zip
retention-days: 7
# Run llvm-musl's tests on x86_64 and i686 with the cross-built corresponding
# toolchains from above.
test-toolchain:
if: (github.event_name != 'schedule') || (github.repository == 'mstorsjo/llvm-mingw')
needs: [linux-cross-windows]
runs-on: windows-latest
defaults:
run:
shell: msys2 {0}
strategy:
fail-fast: false
matrix:
arch:
- x86_64
- i686
steps:
- uses: msys2/setup-msys2@v2
with:
msystem: mingw64
install: >-
unzip
make
- uses: actions/download-artifact@v4
with:
name: windows-${{matrix.arch}}-toolchain
- name: Unpack toolchain
run: |
unzip -q llvm-musl-*.zip
rm llvm-musl-*.zip
mv llvm-musl-* /llvm-musl
echo /llvm-musl/bin >> $GITHUB_PATH
- uses: actions/checkout@v4
- name: Run tests
run: |
./run-tests.sh /llvm-musl
# Test cross-building ffmpeg for all 4 targeted architectures from linux.
# This is done only on scheduled builds, with a toolchain with assertions
# enabled, to catch code generation bugs that might trigger asserts, to
# find such regressions early.
linux-test-cross-build-ffmpeg:
if: (github.event_name == 'schedule') && (github.repository == 'mstorsjo/llvm-mingw')
needs: [linux]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- { arch: i386, triple: i386-linux-musl }
- { arch: x86_64, triple: x86_64-linux-musl }
- { arch: arm, triple: arm-linux-musleabihf }
- { arch: aarch64, triple: aarch64-linux-musl }
steps:
- uses: actions/download-artifact@v4
with:
name: linux-x86_64-toolchain
- name: Unpack cross toolchain
run: |
tar -Jxf llvm-musl-*.tar.xz
rm llvm-musl-*.tar.xz
sudo mv llvm-musl* /opt/llvm-musl
echo /opt/llvm-musl/bin >> $GITHUB_PATH
- name: Checkout ffmpeg
uses: actions/checkout@v4
with:
repository: ffmpeg/ffmpeg
ref: n6.0
path: ffmpeg
- name: Build ffmpeg
run: |
sudo apt-get update && sudo apt-get install nasm
mkdir ffmpeg-build
cd ffmpeg-build
# Set a limit of 3 minutes per process/object file; with unproven
# versions from latest git, the compiler may occasionally hang.
# Make sure we exit in a reasonable time instead of waiting for
# the whole github actions timeout of 6h before exiting.
ulimit -t 180
../ffmpeg/configure --arch=${{matrix.arch}} --target-os=linux --cross-prefix=${{matrix.triple}}- --enable-gpl --extra-ldflags="-static" --prefix=../ffmpeg-install
make -j$(nproc)
make -j$(nproc) install
cd ../ffmpeg-install/bin
tar -Jcvf ../../ffmpeg-${{matrix.arch}}.tar.xz *
- uses: actions/upload-artifact@v4
with:
name: ffmpeg-${{matrix.arch}}
path: |
ffmpeg-*.tar.xz
retention-days: 7
upload-nightly:
if: (github.event_name == 'schedule') && (github.repository == 'mstorsjo/llvm-mingw')
permissions:
contents: write
needs: [linux, linux-cross-aarch64, macos, linux-cross-windows, test-toolchain, linux-test-cross-build-ffmpeg]
runs-on: ubuntu-latest
steps:
- name: Download all artifacts
uses: actions/download-artifact@v4
- name: Rearrange files
run: |
mv *-toolchain/*.zip *-toolchain/*.tar.xz .
- name: Upload binaries
env:
GITHUB_TOKEN: ${{github.token}}
run: |
gh release upload nightly *.tar.xz *.zip --clobber -R ${{github.repository}}