Merge pull request #570 from ChengyuZhu6/payload

A more easy way to setup nydus snapshotter by DaemonSet

.github/workflows/k8s-e2e.yml

@@ -81,13 +81,7 @@ jobs:
           # change snapshotter to nydus after nydus snapshotter started
           registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1)
           docker cp kind-control-plane:/etc/containerd/config.toml containerd.config.toml.bak
-          sed -i -e 's|snapshotter = "overlayfs"|snapshotter = "nydus"|' containerd.config.toml.bak
           cat << EOF >> containerd.config.toml.bak
-          [proxy_plugins]
-            [proxy_plugins.nydus]
-              type = "snapshot"
-              address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
-
           [plugins."io.containerd.grpc.v1.cri".registry.mirrors."${registry_ip}:5000"]
               endpoint = ["http://${registry_ip}:5000"]
           EOF

docs/setup_snapshotter_by_daemonset.md

@@ -0,0 +1,140 @@
+# Setup Nydus Snapshotter by DaemonSet
+
+This document will guide you through the simple steps of setting up and cleaning up the nydus snapshotter in a kubernetes cluster that runs on the host.
+
+## Steps for Setting up Nydus Snapshotter 
+
+To begin, let's clone the Nydus Snapshotter repository.
+
+```bash
+git clone https://github.com/containerd/nydus-snapshotter
+cd nydus-snapshotter
+```
+
+We can build the docker image locally. (optional)
+```bash
+$ export NYDUS_VER=$(curl -s "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name)
+$ make  # build snapshotter binaries
+$ cp bin/* misc/snapshotter/
+$ pushd misc/snapshotter/
+$ docker build --build-arg NYDUS_VER="${NYDUS_VER}" -t ghcr.io/containerd/nydus-snapshotter:latest .
+$ popd
+```
+**NOTE:** By default, the nydus snapshotter would use the latest release nydus version. If you want to use a specific version, you can set `NYDUS_VER` on your side.
+
+Next, we can configure access control for nydus snapshotter.
+```bash
+kubectl apply -f misc/snapshotter/nydus-snapshotter-rbac.yaml
+```
+
+Afterward, we can deploy a DaemonSet for nydus snapshotter.
+
+```bash
+kubectl apply -f misc/snapshotter/nydus-snapshotter.yaml
+```
+
+Then, we can confirm that nydus snapshotter is running through the DaemonSet.
+```bash
+$ kubectl get pods -n nydus-system 
+NAME                      READY   STATUS    RESTARTS   AGE
+nydus-snapshotter-26rf7   1/1     Running   0          18s
+```
+
+Finally, we can view the logs in the pod.
+```bash
+$ kubectl logs nydus-snapshotter-26rf7 -n nydus-system
+install nydus snapshotter artifacts
+there is no proxy plugin!
+Created symlink /etc/systemd/system/multi-user.target.wants/nydus-snapshotter.service → /etc/systemd/system/nydus-snapshotter.service.
+```
+
+And we can see the nydus snapshotter service on the host.
+```bash
+$ systemctl status nydus-snapshotter
+● nydus-snapshotter.service - nydus snapshotter
+     Loaded: loaded (/etc/systemd/system/nydus-snapshotter.service; enabled; vendor preset: enabled)
+    Drop-In: /etc/systemd/system/nydus-snapshotter.service.d
+             └─proxy.conf
+     Active: active (running) since Wed 2024-01-17 16:14:22 UTC; 56s ago
+   Main PID: 1100169 (containerd-nydu)
+      Tasks: 11 (limit: 96376)
+     Memory: 8.6M
+        CPU: 35ms
+     CGroup: /system.slice/nydus-snapshotter.service
+             └─1100169 /opt/nydus/bin/containerd-nydus-grpc --config /etc/nydus/config.toml
+
+Jan 17 16:14:22 worker systemd[1]: Started nydus snapshotter.
+Jan 17 16:14:22 worker containerd-nydus-grpc[1100169]: time="2024-01-17T16:14:22.998798369Z" level=info msg="Start nydus-snapshotter. Version: v0.7.0-308-g106a6cb, PID: 1100169, FsDriver: fusedev, DaemonMode: dedicated"
+Jan 17 16:14:23 worker containerd-nydus-grpc[1100169]: time="2024-01-17T16:14:23.000186538Z" level=info msg="Run daemons monitor..."
+```
+
+**NOTE:** By default, the nydus snapshotter operates as a systemd service. If you prefer to run nydus snapshotter as a standalone process, you can set `ENABLE_SYSTEMD_SERVICE` to `false` in `nydus-snapshotter.yaml`.
+
+## Steps for Cleaning up Nydus Snapshotter 
+
+We use `preStop`` hook in the DaemonSet to uninstall nydus snapshotter and roll back the containerd configuration.
+
+```bash
+$ kubectl delete -f misc/snapshotter/nydus-snapshotter.yaml 
+$ kubectl delete -f misc/snapshotter/nydus-snapshotter-rbac.yaml 
+$ systemd restart containerd.service
+```
+
+## Customized Setup
+
+As we know, nydus snapshotter supports four filesystem drivers (fs_driver): `fusedev`, `fscache`, `blockdev`, `proxy`. Within the container image, we have included configurations for these snapshotter drivers, as well as the corresponding nydusd configurations. By default, the fusedev driver is enabled in the nydus snapshotter, using the snapshotter configuration [`config-fusedev.toml`](../misc/snapshotter/config-fusedev.toml) and the nydusd configuration [`nydusd-config.fusedev.json`](../misc/snapshotter/nydusd-config.fusedev.json).
+
+### Other filesystem driver with related default configuration
+
+If we want to setup the nydus snapshotter with the default configuration for different fs_driver (such as `proxy`), we can modify the values in the `Configmap` in `nydus-snapshotter.yaml`:
+```yaml
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nydus-snapshotter-configs
+  labels:
+    app: nydus-snapshotter
+  namespace: nydus-snapshotter
+data:
+  FS_DRIVER: "proxy"
+  NYDUSD_DAEMON_MODE: "none"
+```
+
+Then we can run the nydus snapshotter enabling `proxy` `fs_driver` with the snapshotter configuration [`config-proxy.toml`](../misc/snapshotter/config-proxy.toml).
+
+**NOTE:** The fs_driver (`blockdev` and `proxy`) do not need nydusd, so they do not need nydusd config.
+
+### Same filesystem with different snapshotter configuration and different nydusd configuration
+
+If we want to setup the nydus snapshotter for the same fs_driver (such as `fusedev`) with different snapshotter configuration and different nydusd configuration, we can enable `ENABLE_CONFIG_FROM_VOLUME` and add the snapshotter configuration [`config.toml`](../misc/snapshotter/config.toml) and nydusd configuration [`nydusd-config.json`](../misc/snapshotter/nydusd-config.fusedev.json) in the `Configmap` in `nydus-snapshotter.yaml`:
+
+```yaml
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nydus-snapshotter-configs
+  labels:
+    app: nydus-snapshotter
+  namespace: nydus-snapshotter
+data:
+  ENABLE_CONFIG_FROM_VOLUME: "true"
+
+  config.toml: |-
+    # The snapshotter config content copied here
+
+  nydusd-config.json: |-
+    # The nydusd config content copied here
+```
+
+**NOTE:** We need to set `nydusd_config` to `/etc/nydus/nydusd-config.json` in the `config.toml`, so that snapshotter can find the nydusd configuration from configmap.
+
+### Customized Options
+
+| Options                             | Type   | Default                               | Comment                                                                                                                                         |
+| ----------------------------------- | ------ | ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
+| FS_DRIVER                           | string | "fusedev"                             | the filesystem driver of snapshotter                                                                                                            |
+| ENABLE_CONFIG_FROM_VOLUME           | bool   | false                                 | enabling to use the configurations from volume                                                                                                  |
+| ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER | bool   | false                                 | enabling to skip to set `plugins."io.containerd.grpc.v1.cri".containerd` to `nydus` for runtime specific snapshotter feature in containerd 1.7+ |
+| ENABLE_SYSTEMD_SERVICE              | bool   | true                                  | enabling to run nydus snapshotter as a systemd service                                                                                          |

misc/snapshotter/Dockerfile

@@ -2,30 +2,35 @@ FROM alpine:3.17.0 AS sourcer
 
 ARG NYDUS_VER=v2.1.5
 
-RUN apk add --no-cache curl
-RUN apk add --no-cache --upgrade grep
-RUN wget https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz && \
+RUN apk add --no-cache curl && \
+    apk add --no-cache --upgrade grep && \
+    curl -OL https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz && \
     echo $NYDUS_VER > /.nydus_version && \
     tar xzf nydus-static-$NYDUS_VER-linux-amd64.tgz && \
-    rm nydus-static-$NYDUS_VER-linux-amd64.tgz
-RUN mv nydus-static/* /
+    rm nydus-static-$NYDUS_VER-linux-amd64.tgz && \
+    mv nydus-static/* /
 
 FROM alpine:3.17.0
+ARG DESTINATION=/opt/nydus-artifacts
+ARG CONFIG_DESTINATION=${DESTINATION}/etc/nydus
+ARG BINARY_DESTINATION=${DESTINATION}/usr/local/bin
+ARG SCRIPT_DESTINATION=${DESTINATION}/opt/nydus
 
 WORKDIR /root/
-
-RUN apk add --no-cache libc6-compat
+RUN apk add --no-cache libc6-compat bash
 
 VOLUME /var/lib/containerd-nydus /run/containerd-nydus
 
 COPY --from=sourcer /.nydus_version /.nydus_version
 
-RUN mkdir -p /usr/local/bin/ /etc/nydus/ /var/lib/containerd-nydus/cache /tmp/blobs/
-COPY --from=sourcer /nydus* /usr/local/bin/
-COPY containerd-nydus-grpc /usr/local/bin/
-RUN chmod +x /usr/local/bin/containerd-nydus-grpc
-COPY nydusd-config.fusedev.json /etc/nydus/config.json
-COPY nydusd-config-localfs.json /etc/nydus/localfs.json
-COPY entrypoint.sh /
+RUN mkdir -p ${CONFIG_DESTINATION} ${BINARY_DESTINATION} ${SCRIPT_DESTINATION} /var/lib/containerd-nydus/cache /tmp/blobs/
+COPY --from=sourcer /nydus* ${BINARY_DESTINATION}/
+COPY containerd-nydus-grpc ${BINARY_DESTINATION}/
+COPY snapshotter.sh ${SCRIPT_DESTINATION}/snapshotter.sh
+RUN chmod +x ${BINARY_DESTINATION}/containerd-nydus-grpc ${SCRIPT_DESTINATION}/snapshotter.sh
+COPY nydusd-config.fusedev.json ${CONFIG_DESTINATION}/nydusd-fusedev.json
+COPY nydusd-config-localfs.json ${CONFIG_DESTINATION}/nydusd-localfs.json
+COPY nydusd-config.fscache.json ${CONFIG_DESTINATION}/nydusd-fscache.json
+COPY config.toml ${CONFIG_DESTINATION}/config.toml
+COPY nydus-snapshotter.service ${DESTINATION}/etc/systemd/system/nydus-snapshotter.service
 
-ENTRYPOINT ["/entrypoint.sh"]

misc/snapshotter/config-coco-host-sharing.toml → misc/snapshotter/config-blockdev.toml

@@ -36,4 +36,4 @@ mount_tarfs_on_host = false
 # - "image_block": generate a raw block disk image with tarfs for an image
 # - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info
 # - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info
-export_mode = "image_block_with_verity"
+export_mode = "image_block_with_verity"

misc/snapshotter/config-coco-guest-pulling.toml → misc/snapshotter/config-proxy.toml

@@ -12,4 +12,4 @@ fs_driver = "proxy"
 
 [snapshot]
 # Insert Kata volume information to `Mount.Options`
-enable_kata_volume = true
+enable_kata_volume = true

misc/snapshotter/config.toml

@@ -27,7 +27,8 @@ pprof_address = ""
 nydusd_config = "/etc/nydus/nydusd-config.fusedev.json"
 nydusd_path = "/usr/local/bin/nydusd"
 nydusimage_path = "/usr/local/bin/nydus-image"
-# fusedev or fscache
+# The fs driver can be one of the following options: fusedev, fscache, blockdev, proxy, or nodev. 
+# If `fs_driver` option is not specified, the default value is fusedev.
 fs_driver = "fusedev"
 # How to process when daemon dies: "none", "restart" or "failover"
 recover_policy = "restart"

misc/snapshotter/nydus-snapshotter-rbac.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: nydus-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nydus-snapshotter-sa
+  namespace: nydus-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nydus-snapshotter-role
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nydus-snapshotter-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: nydus-snapshotter-role
+subjects:
+- kind: ServiceAccount
+  name: nydus-snapshotter-sa
+  namespace: nydus-system

misc/snapshotter/nydus-snapshotter.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=nydus snapshotter
+After=network.target
+Before=containerd.service
+
+[Service]
+Type=simple
+Environment=HOME=/root
+ExecStart=/usr/local/bin/containerd-nydus-grpc --config /etc/nydus/config-proxy.toml
+Restart=always
+RestartSec=1
+KillMode=process
+OOMScoreAdjust=-999
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target