-
Notifications
You must be signed in to change notification settings - Fork 97
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #607 from apostasie/dev-fix-kube
CI fix, part II: kube
- Loading branch information
Showing
13 changed files
with
543 additions
and
157 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,6 +10,7 @@ on: | |
| env: | ||
| DOCKER_USER: testuser | ||
| DOCKER_PASSWORD: testpassword | ||
| NAMESPACE: nydus-system | ||
|
|
||
| jobs: | ||
| e2e_tests_k8s: | ||
|
|
@@ -25,115 +26,22 @@ jobs: | |
| with: | ||
| go-version-file: 'go.mod' | ||
| cache-dependency-path: "go.sum" | ||
| - name: Setup Kind | ||
| uses: engineerd/[email protected] | ||
| with: | ||
| version: v0.16.0 | ||
| config: tests/e2e/k8s/kind.yaml | ||
| - name: Build nydus snapshotter dev image | ||
| run: | | ||
| make | ||
| cp bin/containerd-nydus-grpc ./ | ||
| cp bin/nydus-overlayfs ./ | ||
| cp -r misc/snapshotter/* ./ | ||
| ls -tl ./ | ||
| NYDUS_VER=v$(curl -fsSL --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//') | ||
| docker build --build-arg NYDUS_VER=${NYDUS_VER} -t local-dev:e2e . | ||
| ## load local test image into kind node | ||
| kind load docker-image local-dev:e2e | ||
| - name: Setup registry | ||
| run: | | ||
| mkdir auth | ||
| docker run \ | ||
| --entrypoint htpasswd \ | ||
| httpd:2 -Bbn ${{ env.DOCKER_USER }} ${{ env.DOCKER_PASSWORD }} > auth/htpasswd | ||
| docker run -d \ | ||
| -p 5000:5000 \ | ||
| --restart=always \ | ||
| --name registry \ | ||
| -v "$(pwd)"/auth:/auth \ | ||
| -e "REGISTRY_AUTH=htpasswd" \ | ||
| -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \ | ||
| -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ | ||
| registry:2 | ||
| - name: Login to GitHub Container Registry | ||
| run: | | ||
| registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1) | ||
| ## sudo cat can't modify daemon.json | ||
| sudo chmod a+w /etc/docker/daemon.json | ||
| cat << EOF > /etc/docker/daemon.json | ||
| { | ||
| "exec-opts": ["native.cgroupdriver=cgroupfs"], | ||
| "cgroup-parent": "/actions_job", | ||
| "insecure-registries" : [ "${registry_ip}:5000" ] | ||
| } | ||
| EOF | ||
| sudo systemctl restart docker | ||
| docker login --username=${{ env.DOCKER_USER }} --password=${{ env.DOCKER_PASSWORD }} $registry_ip:5000 | ||
| - name: Setup nydus snapshotter | ||
| - name: Test | ||
| run: | | ||
| kubectl create -f tests/e2e/k8s/snapshotter-${{ inputs.auth-type }}.yaml | ||
| export ns=nydus-system | ||
| p=`kubectl -n $ns get pods --no-headers -o custom-columns=NAME:metadata.name` | ||
| echo "snapshotter pod name ${p}" | ||
| kubectl -n $ns wait po $p --for=condition=ready --timeout=2m | ||
| # change snapshotter to nydus after nydus snapshotter started | ||
| registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1) | ||
| docker cp kind-control-plane:/etc/containerd/config.toml containerd.config.toml.bak | ||
| cat << EOF >> containerd.config.toml.bak | ||
| [plugins."io.containerd.grpc.v1.cri".registry.mirrors."${registry_ip}:5000"] | ||
| endpoint = ["http://${registry_ip}:5000"] | ||
| EOF | ||
| docker cp containerd.config.toml.bak kind-control-plane:/etc/containerd/config.toml.bak | ||
| docker exec kind-control-plane sh -c "cat /etc/containerd/config.toml.bak > /etc/containerd/config.toml" | ||
| docker exec kind-control-plane systemctl restart containerd | ||
| - name: Install Nydus binaries and convert nydus image | ||
| run: | | ||
| NYDUS_VER=v$(curl -fsSL --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//') | ||
| wget -q https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz | ||
| tar xzf nydus-static-$NYDUS_VER-linux-amd64.tgz | ||
| sudo cp nydus-static/nydusify nydus-static/nydus-image /usr/local/bin/ | ||
| registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1) | ||
| sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ | ||
| --source busybox:latest \ | ||
| --target ${registry_ip}:5000/busybox:nydus-v6-latest \ | ||
| --fs-version 6 | ||
| - name: Run E2E test | ||
| run: | | ||
| registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1) | ||
| kubectl create --namespace nydus-system secret generic regcred \ | ||
| --from-file=.dockerconfigjson=$HOME/.docker/config.json \ | ||
| --type=kubernetes.io/dockerconfigjson | ||
| sed -e "s|REGISTRY_IP|${registry_ip}|" tests/e2e/k8s/test-pod.yaml.tpl > tests/e2e/k8s/test-pod.yaml | ||
| if [[ "${{ inputs.auth-type }}" == "cri" ]]; then | ||
| docker exec kind-control-plane sh -c 'echo " --image-service-endpoint=unix:///run/containerd-nydus/containerd-nydus-grpc.sock" >> /etc/default/kubelet' | ||
| docker exec kind-control-plane sh -c 'systemctl daemon-reload && systemctl restart kubelet' | ||
| fi | ||
| kubectl apply -f tests/e2e/k8s/test-pod.yaml | ||
| kubectl wait po test-pod -n nydus-system --for=condition=ready --timeout=1m | ||
| kubectl delete -f tests/e2e/k8s/test-pod.yaml | ||
| AUTH_TYPE='${{ inputs.auth-type }}' | ||
| ./tests/helpers/kind.sh | ||
| - name: Dump logs | ||
| if: failure() | ||
| continue-on-error: true | ||
| run: | | ||
| log_dir="/tmp/nydus-log" | ||
| mkdir -p $log_dir | ||
| export ns=nydus-system | ||
| for p in `kubectl -n $ns get pods --no-headers -o custom-columns=NAME:metadata.name`; do | ||
| kubectl -n $ns get pod $p -o yaml >> $log_dir/nydus-pods.conf | ||
| kubectl -n $ns describe pod $p >> $log_dir/nydus-pods.conf | ||
| kubectl -n $ns logs $p -c nydus-snapshotter >> $log_dir/nydus-snapshotter.log || echo "failed to get snapshotter log" | ||
| for p in `kubectl --namespace "$NAMESPACE" get pods --no-headers -o custom-columns=NAME:metadata.name`; do | ||
| kubectl --namespace "$NAMESPACE" get pod $p -o yaml >> $log_dir/nydus-pods.conf | ||
| kubectl --namespace "$NAMESPACE" describe pod $p >> $log_dir/nydus-pods.conf | ||
| kubectl --namespace "$NAMESPACE" logs $p -c nydus-snapshotter >> $log_dir/nydus-snapshotter.log || echo "failed to get snapshotter log" | ||
| done | ||
| kubectl -n $ns get secrets -o yaml >> $log_dir/nydus-secrets.log | ||
| kubectl --namespace "$NAMESPACE" get secrets -o yaml >> $log_dir/nydus-secrets.log | ||
| docker exec kind-control-plane cat /etc/containerd/config.toml >> $log_dir/containerd-config.toml | ||
| docker exec kind-control-plane containerd config dump >> $log_dir/containerd-config-dump.toml | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.