From 392035c16d5bbae37345ce729ddb2cc45e50ad87 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 11 Apr 2024 12:30:23 -0500 Subject: [PATCH] Update LLVM to 18.1.2 and add wasm-component-ld (#402) * Update LLVM to 18.1.2 and add wasm-component-ld This commit has two intertwined changes within it. First the LLVM submodule is updated to the 18.1.2 release branch. This alone would cause CI and tests to fail due to differing behavior for the `wasm32-wasip2` target. To fix these test failures the `wasm-component-ld` tool is added to the build. This tool is a Rust-written tool and installed via `cargo install` as part of the build at a pinned version written in the `Makefile`. This linker, used for components, is then used for the `wasm32-wasip2` target. Tests and CI are then updated to skip the need to have `wasm-tools` or the adapter for WASI when making components since that's now the job of `wasm-component-ld`. This then necessitated some changes to tests too. * Add back accidentally deleted CI * Inherit tools on PATH on Windows * Add Rust to docker build * Put rust in a different location in docker * Try to fix permissions * Review comments * Revert changes to test outputs --- .github/workflows/main.yml | 9 ++------- Dockerfile | 5 +++++ Makefile | 12 ++++++++++-- src/llvm-project | 2 +- tests/run.sh | 25 ++++++++----------------- tests/testcase.sh | 16 ++++------------ 6 files changed, 30 insertions(+), 39 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4303d08d6..0a2d00398 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -42,12 +42,6 @@ jobs: uses: bytecodealliance/actions/wasmtime/setup@v1 with: version: "18.0.2" - - name: Setup `wasm-tools` for tests - uses: bytecodealliance/actions/wasm-tools/setup@v1 - with: - version: "1.201.0" - - name: Download command adapter for tests - run: curl -f -L --retry 5 -o ${{ runner.temp }}/wasi_snapshot_preview1.command.wasm https://github.com/bytecodealliance/wasmtime/releases/download/v18.0.2/wasi_snapshot_preview1.command.wasm - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -67,7 +61,7 @@ jobs: run: NINJA_FLAGS=-v make package LLVM_CMAKE_FLAGS=-DLLVM_CCACHE_BUILD=ON shell: bash - name: Run the testsuite - run: NINJA_FLAGS=-v make check RUNTIME=wasmtime ADAPTER=${{ runner.temp }}/wasi_snapshot_preview1.command.wasm WASM_TOOLS=wasm-tools + run: NINJA_FLAGS=-v make check RUNTIME=wasmtime - name: Upload artifacts uses: actions/upload-artifact@v4 with: @@ -107,6 +101,7 @@ jobs: msystem: ${{ matrix.sys }} update: true release: false + path-type: inherit - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/Dockerfile b/Dockerfile index 9a864e63e..adddd029c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,11 @@ RUN curl -sSLO https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake- && mv cmake-3.25.1-linux-x86_64 /opt/cmake ENV PATH /opt/cmake/bin:$PATH +ENV RUSTUP_HOME=/rust/rustup CARGO_HOME=/rust/cargo PATH=$PATH:/rust/cargo/bin +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --profile=minimal && \ + chmod -R a+w /rust + RUN groupadd -g ${GID} builder && \ useradd --create-home --uid ${UID} --gid ${GID} builder USER builder diff --git a/Makefile b/Makefile index 9c7acffbf..fbe6af583 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ default: build @echo "Use -fdebug-prefix-map=$(ROOT_DIR)=wasisdk://v$(VERSION)" check: - TARGETS="$(TARGETS)" tests/run.sh "$(BUILD_PREFIX)" "$(RUNTIME)" "$(ADAPTER)" "$(WASM_TOOLS)" + TARGETS="$(TARGETS)" tests/run.sh "$(BUILD_PREFIX)" "$(RUNTIME)" clean: rm -rf build $(DESTDIR) @@ -122,6 +122,14 @@ build/llvm.BUILT: llvm-config touch build/llvm.BUILT +# Build the `wasm-component-ld` linker from source via `cargo install`. This is +# used for the `wasm32-wasip2` target natively by Clang. Note that `--root` +# passed to `cargo install` will place it in the output directory automatically. +build/wasm-component-ld.BUILT: build/llvm.BUILT + cargo install wasm-component-ld@0.1.5 --root $(BUILD_PREFIX) + touch build/wasm-component-ld.BUILT + + # Flags for running `make` in wasi-libc # $(1): the target that's being built WASI_LIBC_MAKEFLAGS = \ @@ -132,7 +140,7 @@ WASI_LIBC_MAKEFLAGS = \ SYSROOT=$(BUILD_PREFIX)/share/wasi-sysroot \ TARGET_TRIPLE=$(1) -build/wasi-libc.BUILT: build/compiler-rt.BUILT +build/wasi-libc.BUILT: build/compiler-rt.BUILT build/wasm-component-ld.BUILT $(MAKE) $(call WASI_LIBC_MAKEFLAGS,wasm32-wasi) default libc_so $(MAKE) $(call WASI_LIBC_MAKEFLAGS,wasm32-wasip1) default libc_so $(MAKE) $(call WASI_LIBC_MAKEFLAGS,wasm32-wasip2) WASI_SNAPSHOT=p2 default libc_so diff --git a/src/llvm-project b/src/llvm-project index 461274b81..26a1d6601 160000 --- a/src/llvm-project +++ b/src/llvm-project @@ -1 +1 @@ -Subproject commit 461274b81d8641eab64d494accddc81d7db8a09e +Subproject commit 26a1d6601d727a96f4301d0d8647b5a42760ae0c diff --git a/tests/run.sh b/tests/run.sh index 080eeaee5..bc8dd4d0b 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -17,17 +17,8 @@ wasi_sdk="$1" # Determine the wasm runtime to use, if one is provided. if [ $# -gt 1 ]; then runwasi="$2" - if [ $# -gt 3 ]; then - adapter="$3" - wasm_tools="$4" - else - adapter="" - wasm_tools="" - fi else runwasi="" - adapter="" - wasm_tools="" fi testdir=$(dirname $0) @@ -45,13 +36,13 @@ for target in $TARGETS; do echo "===== Testing compile-only with $options =====" for file in *.c; do echo "Testing compile-only $file..." - ../testcase.sh "$target" "" "" "" "$wasi_sdk/bin/clang" "$options --target=$target" "$file" - ../testcase.sh "$target" "" "" "" "$wasi_sdk/bin/$target-clang" "$options" "$file" + ../testcase.sh "$target" "" "$wasi_sdk/bin/clang" "$options --target=$target" "$file" + ../testcase.sh "$target" "" "$wasi_sdk/bin/$target-clang" "$options" "$file" done for file in *.cc; do echo "Testing compile-only $file..." - ../testcase.sh "$target" "" "" "" "$wasi_sdk/bin/clang++" "$options --target=$target -fno-exceptions" "$file" - ../testcase.sh "$target" "" "" "" "$wasi_sdk/bin/$target-clang++" "$options -fno-exceptions" "$file" + ../testcase.sh "$target" "" "$wasi_sdk/bin/clang++" "$options --target=$target -fno-exceptions" "$file" + ../testcase.sh "$target" "" "$wasi_sdk/bin/$target-clang++" "$options -fno-exceptions" "$file" done done cd - >/dev/null @@ -61,13 +52,13 @@ for target in $TARGETS; do echo "===== Testing with $options =====" for file in *.c; do echo "Testing $file..." - ../testcase.sh "$target" "$runwasi" "$adapter" "$wasm_tools" "$wasi_sdk/bin/clang" "$options --target=$target" "$file" - ../testcase.sh "$target" "$runwasi" "$adapter" "$wasm_tools" "$wasi_sdk/bin/$target-clang" "$options" "$file" + ../testcase.sh "$target" "$runwasi" "$wasi_sdk/bin/clang" "$options --target=$target" "$file" + ../testcase.sh "$target" "$runwasi" "$wasi_sdk/bin/$target-clang" "$options" "$file" done for file in *.cc; do echo "Testing $file..." - ../testcase.sh "$target" "$runwasi" "$adapter" "$wasm_tools" "$wasi_sdk/bin/clang++" "$options --target=$target -fno-exceptions" "$file" - ../testcase.sh "$target" "$runwasi" "$adapter" "$wasm_tools" "$wasi_sdk/bin/$target-clang++" "$options -fno-exceptions" "$file" + ../testcase.sh "$target" "$runwasi" "$wasi_sdk/bin/clang++" "$options --target=$target -fno-exceptions" "$file" + ../testcase.sh "$target" "$runwasi" "$wasi_sdk/bin/$target-clang++" "$options -fno-exceptions" "$file" done done cd - >/dev/null diff --git a/tests/testcase.sh b/tests/testcase.sh index 61c6612e5..3d8860464 100755 --- a/tests/testcase.sh +++ b/tests/testcase.sh @@ -8,18 +8,15 @@ set -ueo pipefail # script, so don't do anything fancy. target="$1" runwasi="$2" -adapter="$3" -wasm_tools="$4" -compiler="$5" -options="$6" -input="$7" +compiler="$3" +options="$4" +input="$5" # Compile names for generated files. wasm="$input.$options.wasm" stdout_observed="$input.$options.stdout.observed" stderr_observed="$input.$options.stderr.observed" exit_status_observed="$input.$options.exit_status.observed" -run_args="" # Optionally load compiler options from a file. if [ -e "$input.options" ]; then @@ -44,11 +41,6 @@ if [ "$runwasi" == "" ]; then exit 0 fi -if [ "$target" == "wasm32-wasip2" -a -n "$adapter" -a -n "$wasm_tools" ]; then - "$wasm_tools" component new --adapt "$adapter" "$wasm" -o "$wasm" - run_args="--wasm component-model" -fi - # Determine the input file to write to stdin. if [ -e "$input.stdin" ]; then stdin="$input.stdin" @@ -74,7 +66,7 @@ fi # Run the test, capturing stdout, stderr, and the exit status. exit_status=0 -"$runwasi" $run_args $env $dir "$wasm" $dirarg \ +"$runwasi" $env $dir "$wasm" $dirarg \ < "$stdin" \ > "$stdout_observed" \ 2> "$stderr_observed" \