Move compilation support image to wasm-corpus (#236)

* Move compilation support image to wasm-corpus

* fix tests

* Make scripts runnable on host or docker

* add readme

* reformat

pom.xml

@@ -40,6 +40,7 @@
     <module>wasm</module>
     <module>runtime</module>
     <module>wasi</module>
+    <module>wasm-corpus</module>
   </modules>
 
   <scm>

runtime/pom.xml

@@ -29,6 +29,12 @@
   </dependencies>
 
   <build>
+    <testResources>
+      <testResource>
+        <directory>${project.basedir}/../wasm-corpus/src/test/resources</directory>
+      </testResource>
+    </testResources>
+
     <plugins>
       <plugin>
         <groupId>com.dylibso.chicory</groupId>

wasi/pom.xml

@@ -28,4 +28,11 @@
     </dependency>
   </dependencies>
 
+  <build>
+    <testResources>
+      <testResource>
+        <directory>${project.basedir}/../wasm-corpus/src/test/resources</directory>
+      </testResource>
+    </testResources>
+  </build>
 </project>

wasi/src/test/java/com/dylibso/chicory/wasi/WasiPreview1Test.java

@@ -7,7 +7,6 @@
 import com.dylibso.chicory.runtime.HostImports;
 import com.dylibso.chicory.runtime.Module;
 import java.io.ByteArrayInputStream;
-import java.io.File;
 import org.junit.jupiter.api.Test;
 
 public class WasiPreview1Test {
@@ -20,8 +19,7 @@ public void shouldRunWasiModule() {
         var wasi =
                 new WasiPreview1(this.logger, WasiOptions.builder().withStdout(fakeStdout).build());
         var imports = new HostImports(wasi.toHostFunctions());
-        var module =
-                Module.builder(new File("src/test/resources/compiled/hello-wasi.wat.wasm")).build();
+        var module = Module.builder("compiled/hello-wasi.wat.wasm").build();
         module.instantiate(imports);
         assertEquals(fakeStdout.output().strip(), "hello world");
     }
@@ -33,8 +31,7 @@ public void shouldRunWasiRustModule() {
         var stdout = new MockPrintStream();
         var wasi = new WasiPreview1(this.logger, WasiOptions.builder().withStdout(stdout).build());
         var imports = new HostImports(wasi.toHostFunctions());
-        var module =
-                Module.builder(new File("src/test/resources/compiled/hello-wasi.rs.wasm")).build();
+        var module = Module.builder("compiled/hello-wasi.rs.wasm").build();
         module.instantiate(imports); // run _start and prints Hello, World!
         assertEquals(expected, stdout.output().strip());
     }
@@ -46,8 +43,7 @@ public void shouldRunWasiGreetRustModule() {
         var wasiOpts = WasiOptions.builder().withStdout(System.out).withStdin(fakeStdin).build();
         var wasi = new WasiPreview1(this.logger, wasiOpts);
         var imports = new HostImports(wasi.toHostFunctions());
-        var module =
-                Module.builder(new File("src/test/resources/compiled/greet-wasi.rs.wasm")).build();
+        var module = Module.builder("compiled/greet-wasi.rs.wasm").build();
         module.instantiate(imports);
     }
 
@@ -60,7 +56,7 @@ public void shouldRunWasiDemoJavyModule() {
         var wasiOpts = WasiOptions.builder().withStdout(fakeStdout).withStdin(fakeStdin).build();
         var wasi = new WasiPreview1(this.logger, wasiOpts);
         var imports = new HostImports(wasi.toHostFunctions());
-        var module = Module.builder("compiled/javy-demo.js.wasm").build();
+        var module = Module.builder("compiled/javy-demo.js.javy.wasm").build();
         module.instantiate(imports);
 
         assertEquals(fakeStdout.output(), "{\"foo\":3,\"newBar\":\"baz!\"}");

wasm-corpus/Dockerfile

@@ -0,0 +1,45 @@
+FROM ubuntu:jammy
+
+# Set the environment variable to avoid prompts
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y curl build-essential gcc make pkg-config libssl-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+RUN rustc --version && cargo --version
+RUN rustup target add wasm32-unknown-unknown
+RUN rustup target add wasm32-wasi
+
+WORKDIR /tmp
+
+# Install wasi-sdk
+RUN curl -L -O https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz && \
+    mkdir -p /opt/wasi-sdk && \
+    tar xvf wasi-sdk-20.0-linux.tar.gz && \
+    cp -r wasi-sdk-20.0/* /opt/wasi-sdk
+
+ENV WASI_SDK_PATH=/opt/wasi-sdk
+
+# Install wabt tools
+RUN curl -L -O https://github.com/WebAssembly/wabt/releases/download/1.0.34/wabt-1.0.34-ubuntu.tar.gz && \
+    mkdir -p /opt/wabt && \
+    tar xvf wabt-1.0.34-ubuntu.tar.gz && \
+    cp -r wabt-1.0.34/* /opt/wabt
+
+# Install Javy
+RUN curl -L -O https://github.com/bytecodealliance/javy/releases/download/v1.4.0/javy-x86_64-linux-v1.4.0.gz && \
+    mkdir -p /opt/javy && \
+    gunzip javy-x86_64-linux-v1.4.0.gz && \
+    mv javy-x86_64-linux-v1.4.0 /usr/bin/javy && \
+    chmod a+x /usr/bin/javy
+
+ENV PATH="/opt/wabt/bin:${PATH}"
+
+WORKDIR /usr/code
+
+ENTRYPOINT ["./compile.sh"]

wasm-corpus/README.md

@@ -0,0 +1,34 @@
+# Wasm Corpus
+
+
+This is a central place for all of our guest source code. It can be included in other maven projects as a test
+resources bundle.
+
+If you want to add or edit guest modules for tests, you'll need to compile here. Add the program to the subfolder
+for the appropriate source language. Example: src/test/resources/rust.
+
+You can use docker to compile everything. The run.sh script can orchestrate this for you:
+
+```bash
+cd wasm-corpus
+
+# rebuild (or build image for first time)
+./run.sh rebuild
+
+# compile all the submodules
+./run.sh
+
+# compile just one folder
+./run.sh rust
+```
+
+
+If you want to run on your host machine you can run the `compile.sh` subscript yourself.
+It has the same arguments (assumes you have WASI_SDK_PATH set for compiling c:
+
+```bash
+cd wasm-corpus/src/test/resources
+
+# example
+./compile.sh rust
+```

wasm-corpus/pom.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>com.dylibso.chicory</groupId>
+    <artifactId>chicory</artifactId>
+    <version>999-SNAPSHOT</version>
+  </parent>
+  <artifactId>wasm-corpus</artifactId>
+  <name>Chicory - WASM Corpus</name>
+  <description>A Corpus of guest modules for testing</description>
+</project>

wasm-corpus/run.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# run with `rebuild` to build the image
+# run with no args to compile everything
+
+if [[ "$1" = "rebuild" ]]; then
+  docker build . --platform linux/amd64 -t chicory/wasm-corpus
+else
+  # Optionally takes the args `lang` (ex: wat, rust) and `file` (ex: br.wat)
+  # both default to all
+  docker run --platform linux/amd64 -v ./src/test/resources:/usr/code --rm chicory/wasm-corpus $1 $2
+fi

wasm-corpus/src/test/resources/compile.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+set -e
+
+compileWat() {
+  filename=$(basename "$1")
+  (set -x; wat2wasm $1 -o "./compiled/$filename.wasm")
+}
+
+compileRust() {
+  filename=$(basename "$1")
+  # we can use the wasi targets for any file that end in -wasi.rs
+  if [[ "$filename" =~ wasi.rs$ ]]; then
+    target="wasm32-wasi"
+    crate_type="bin"
+  else
+    target="wasm32-unknown-unknown"
+    crate_type="cdylib"
+  fi
+  (set -x; rustc $1 --target=$target --crate-type=$crate_type -o "./compiled/$filename.wasm")
+}
+
+ENV_WASI_SDK_PATH="${WASK_SDK_PATH:-/opt/wasi-sdk}"
+compileC() {
+  filename=$(basename "$1")
+  (set -x; ${ENV_WASI_SDK_PATH}/bin/clang -g -o "./compiled/$filename.wasm" $1 -nostartfiles -Wl,--no-entry -Wl,--export=run)
+}
+
+compileJavy() {
+  filename=$(basename "$1")
+  (set -x; javy compile $1 -o "./compiled/$filename.javy.wasm")
+}
+
+compile() {
+  lang=$1
+  case "$lang" in
+    wat)
+      compileWat $2
+      ;;
+    rust)
+      compileRust $2
+      ;;
+    c)
+      compileC $2
+      ;;
+    javy)
+      compileJavy $2
+      ;;
+    *)
+      echo "Don't know how to compile language $lang"
+      exit 1
+      ;;
+  esac
+}
+
+lang="${1:-all}"
+path="${2:-all}"
+
+if [[ "$lang" == "all" ]]; then
+  langs=("wat" "rust" "c" "javy")
+else
+  langs=("$lang")
+fi
+
+for lang in "${langs[@]}"; do
+  echo "Compiling all modules in ./$lang/*"
+  for file in ./$lang/*; do
+    compile $lang $file
+  done
+done

wasi/src/test/resources/sources/javy-demo.js → wasm-corpus/src/test/resources/javy/javy-demo.js

@@ -48,4 +48,4 @@ function writeOutput(output) {
     // Stdout file descriptor
     const fd = 1;
     Javy.IO.writeSync(fd, buffer);
-}
+}

wasm/pom.xml

@@ -25,6 +25,11 @@
   </dependencies>
 
   <build>
+    <testResources>
+      <testResource>
+        <directory>${project.basedir}/../wasm-corpus/src/test/resources</directory>
+      </testResource>
+    </testResources>
     <plugins>
       <plugin>
         <groupId>com.dylibso.chicory</groupId>