CI experience (#25)

* added github actions workflow for integration test and image creation and push

.devcontainer/bootstrap.sh

@@ -6,6 +6,9 @@ $PATHRUSTUP/rustup update stable && $PATHRUSTUP/rustup default stable && $PATHRU
 # Installs wasm32 compiler targets
 $PATHRUSTUP/rustup target add wasm32-wasi wasm32-unknown-unknown
 
+# export path to cargo bin
+export PATH="$HOME/.cargo/bin:$PATH"
+
 # Install Azure CLI
 curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
 

.github/actions/build-devcontainer/action.yaml

@@ -0,0 +1,17 @@
+name: Create and setup devcontainer
+description: Create and setup devcontainer
+runs:
+  using: "composite"
+  steps:
+  - name: build the devcontainer image
+    run: |
+      docker build -t devcontainer -f .devcontainer/Dockerfile ./.devcontainer
+    shell: bash
+  - name: run the devcontainer image
+    run: |
+      docker run --name devcontainer -d --rm --privileged --mount type=bind,source=${{ github.workspace}},target=/workspace devcontainer
+    shell: bash
+  - name: install tools
+    run: |
+      docker exec devcontainer sh ./workspace/.devcontainer/bootstrap.sh
+    shell: bash

.github/workflows/PR.yaml

@@ -0,0 +1,19 @@
+name: Main Branch PR
+on: 
+  pull_request:
+    branches:
+      - main
+jobs:
+  Build-Code:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      name: download code
+      with:
+        fetch-depth: 0
+    - name: build the devcontainer
+      uses: ./.github/actions/build-devcontainer
+    # export of cargo path in each step is required because it is not when exported in bootstraph.sh script
+    - name: install cluster and workloads
+      run: |
+        docker exec devcontainer sh -c "export PATH="/root/.cargo/bin:$PATH" && cd ./workspace && make fmt && make"

.github/workflows/Release.yaml

@@ -0,0 +1,31 @@
+name: Main - Upload Apps Images/Packages
+
+on: 
+  push:
+    branches: 
+      - main
+    paths:
+      # Only run this when apps are updated.
+      - 'apps/**'
+      - '!apps/runtime/**'
+
+jobs:
+  Release-Apps-Images:
+    # grant packages write permission to GITHUB_TOKEN to push images outside the workflow (within devcontainer image)
+    permissions:
+      packages: write
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      name: download code
+      with:
+        fetch-depth: 0
+    - name: build the devcontainer
+      uses: ./.github/actions/build-devcontainer
+    - name: log in to the container registry
+      run: |
+        docker exec devcontainer sh -c "docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}"
+    # export of cargo path in each step is required because it is not when exported in bootstraph.sh script
+    - name: push workload images
+      run: |
+        docker exec devcontainer sh -c "export PATH="/root/.cargo/bin:$PATH" && cd ./workspace && make build_push_app_images GITHUBORG=${{ github.repository_owner }} GITHUBREPO=${{ github.repository }}"

Makefile

@@ -1,21 +1,13 @@
 K3DCLUSTERNAME := wasm-cluster
+K3DSHIMIMAGENAME := ghcr.io/deislabs/containerd-wasm-shims/examples/k3d:v0.9.0
 DOCKERDIR := ./wasm-shims/deployments/k3d
-TMPSPINDIR := ./wasm-shims/deployments/k3d/.tmp
 APPSDIR := ./apps
 
-all: build_k3d_node_image create_k3d_cluster install_redis deploy_app run_integrationtest
-
-build_k3d_node_image:
-	@echo "Copying spin shim..."
-	mkdir -p $(TMPSPINDIR)
-	cp -p ./apps/runtime/containerd-shim-spin-v1 $(TMPSPINDIR)/containerd-shim-spin-v1
-
-	@echo "Building k3d image..."
-	docker build -t k3d-shim -f $(DOCKERDIR)/Dockerfile $(DOCKERDIR)
+all: create_k3d_cluster install_redis deploy_app run_integrationtest
 
 create_k3d_cluster:
 	@echo "Creating k3d cluster..."
-	k3d cluster create $(K3DCLUSTERNAME) --image k3d-shim --api-port 6551 -p '8001:30010@loadbalancer' -p '8002:80@loadbalancer' --servers 1
+	k3d cluster create $(K3DCLUSTERNAME) --image $(K3DSHIMIMAGENAME) --api-port 6551 -p '8001:30010@loadbalancer' -p '8002:80@loadbalancer' --servers 1
 	@echo "Loading spin runtime..."
 	kubectl apply -f ./apps/runtime/runtime.yaml
 
@@ -43,10 +35,19 @@ run_integrationtest:
 
 test: clean all
 
+fmt:
+	@echo "Checking formatting of code..."
+	cargo fmt --all -- --check
+	cargo clippy --all-targets --all-features --workspace -- --deny=warnings
+
+build_push_app_images:
+	@echo "Build and save apps images to artifacts folder..."
+	sh ./deployment/build-push-workload-image.sh orderprocessor $(APPSDIR) $(GITHUBORG) $(GITHUBREPO)
+	sh ./deployment/build-push-workload-image.sh fulfilmentprocessor $(APPSDIR) $(GITHUBORG) $(GITHUBREPO)
+
 clean:
 	@echo "Cleaning up..."
 	k3d cluster delete $(K3DCLUSTERNAME)
-	rm -rf $(TMPSPINDIR)
 	cargo clean
 	rm -rf ./apps/orderprocessor/target
 	rm -rf ./apps/orderprocessor/.spin

apps/fulfilmentprocessor/src/lib.rs

@@ -1,7 +1,6 @@
 use anyhow::Result;
 use bytes::Bytes;
 use order_management::Order;
-use serde_json::Value;
 use spin_sdk::{redis, redis_component};
 use std::env::var;
 use std::str::from_utf8;

apps/orderprocessor/Makefile

@@ -10,4 +10,4 @@ clean:
 	rm -rf ./spin
 	cargo clean
 	kubectl delete -f ./deploy.yaml
-	kubectl delete -f ./service.yaml
+	kubectl delete -f ./service.yaml

apps/orderprocessor/receiver/src/lib.rs

@@ -5,7 +5,7 @@ use spin_sdk::{
     http::{Request, Response},
     http_component, redis,
 };
-use std::{env::var, io::Read};
+use std::env::var;
 
 // The environment variable is set in `spin.toml` that points to the
 // address of the Redis server that the component will publish

apps/runtime/runtime.yaml

@@ -3,4 +3,4 @@ apiVersion: node.k8s.io/v1
 kind: RuntimeClass
 metadata:
   name: wasmtime-spin
-handler: spin
+handler: spin

deployment/build-push-workload-image.sh

@@ -0,0 +1,25 @@
+#! /bin/bash
+# image name/tag must include the Github organization  to be pushed to the Github Container Registry (GHCR)
+IMAGENAME=$(echo ghcr.io/$3/$1 | tr '[:upper:]' '[:lower:]')
+# label including the Github organization and repository is required to connect it to the right Github repository
+LABEL=$(echo org.opencontainers.image.source=https://github.com/$4 | tr '[:upper:]' '[:lower:]')
+
+cd $2/$1
+
+# build the app
+spin build
+cp -r ../../target .
+
+# get the version from the spin.toml file
+VERSION=`grep ^version spin.toml | cut -d'"' -f 2`
+
+# build the docker image
+docker buildx build -f Dockerfile -t $IMAGENAME:$VERSION --label $LABEL --load --platform=wasi/wasm32 --provenance=false .
+
+# tag the docker image
+docker tag $IMAGENAME:$VERSION $IMAGENAME:latest
+
+docker push $IMAGENAME:$VERSION
+docker push $IMAGENAME:latest
+
+cd ../..

docs/ci.md

@@ -0,0 +1,39 @@
+# Continuous Integration (CI)
+
+This documents describes the CI process for the solution - in particular for the Spin apps.
+
+## Integration Tests
+
+The workflow [PR](../.github/workflows/PR.yaml) is used as a validation pipeline for pull requests created against the main branch.
+
+The job contains the generation and setup of the devcontainer including the docker-in-docker installation. That is required to build the wasm apps because it can be configured to use the wasm based containerd runtime for pulling and storing images as explaining in the [Docker-In-Docker](../docs/dev.md#docker-in-docker) section.
+The pipeline also configures the k3d cluster and deploys the apps into the cluster. The apps are validated by running the integration tests as part of the makefile experience.
+
+## Release Pipeline
+
+The workflow [Release](../.github/workflows/Release.yaml) is triggered when code is committed or merged into main branch (ideally as part of the PR merge) and is used to push the apps images into the GitHub Container Registry (GHCR).
+
+The pipeline runs the setup of the devcontainer and k3d cluster creation as well and logs in to the GHCR by using the default environment variables and secrets (GITHUB_TOKEN).
+
+Finally, the workflow executes a script via make command to build and push the images into the GHCR.
+
+## Configuration
+
+### GITHUB_TOKEN
+
+The GITHUB_TOKEN secret is required to be configured with "packages write" permission to be able to push the images into the GHCR from the Docker-In-Docker container.
+
+```yaml
+permissions:
+    packages: write
+```
+
+### Tagging and labeling
+
+Since the script [build-push-workload-image](../deployment/build-push-workload-image.sh) uses the "docker push" command to push the images into the GHCR, the images are tagged with the GHCR repository name (GitHub organization) and the image name.
+The right version is read from the spin.toml file and applied as one tag aside the "latest" tag.
+Additionally, the image is labeled with the GitHub repository name to connect it explicitly to the source code.
+
+`LABEL org.opencontainers.image.source=https://github.com/OWNER/REPO`
+
+More details can be found [here](https://docs.github.com/en/packages/learn-github-packages/connecting-a-repository-to-a-package#connecting-a-repository-to-a-container-image-using-the-command-line)