tests: update the e2e tests

update the e2e tests. Signed-off-by: ChengyuZhu6 <[email protected]>
containerd · Jan 26, 2024 · e0ba512 · e0ba512
1 parent ad4299e
commit e0ba512
Show file tree

Hide file tree

Showing 3 changed files with 270 additions and 191 deletions.
diff --git a/.github/workflows/k8s-e2e.yml b/.github/workflows/k8s-e2e.yml
@@ -81,13 +81,7 @@ jobs:
           # change snapshotter to nydus after nydus snapshotter started
           registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1)
           docker cp kind-control-plane:/etc/containerd/config.toml containerd.config.toml.bak
-          sed -i -e 's|snapshotter = "overlayfs"|snapshotter = "nydus"|' containerd.config.toml.bak
           cat << EOF >> containerd.config.toml.bak
-          [proxy_plugins]
-            [proxy_plugins.nydus]
-              type = "snapshot"
-              address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
-
           [plugins."io.containerd.grpc.v1.cri".registry.mirrors."${registry_ip}:5000"]
               endpoint = ["http://${registry_ip}:5000"]
           EOF

diff --git a/tests/e2e/k8s/snapshotter-cri.yaml b/tests/e2e/k8s/snapshotter-cri.yaml
@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Namespace
 metadata:
@@ -15,113 +16,126 @@ kind: Pod
 metadata:
   name: nydus-snapshotter
   namespace: nydus-system
+  labels:
+    app: nydus-snapshotter
 spec:
+  serviceAccountName: nydus-snapshotter-sa
+  hostNetwork: true
+  hostPID: true
   containers:
     - name: nydus-snapshotter
-      image: local-dev:e2e
+      image: "local-dev:e2e"
       imagePullPolicy: IfNotPresent
-      args:
-        - containerd-nydus-grpc
-        - --nydusd /usr/local/bin/nydusd
-        - --nydusd-config /etc/nydus/nydusd.json
-        - --config /etc/nydus/config.toml
-        - --log-level debug
-        - --daemon-mode shared
-        - --log-to-stdout
-      securityContext:
-        privileged: true
+      env:
+        - name: FS_DRIVER
+          valueFrom:
+            configMapKeyRef:
+              name: nydus-snapshotter-configs
+              key: FS_DRIVER
+              optional: true
+        - name: ENABLE_CONFIG_FROM_VOLUME
+          valueFrom:
+            configMapKeyRef:
+              name: nydus-snapshotter-configs
+              key: ENABLE_CONFIG_FROM_VOLUME
+              optional: true
+        - name: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER
+          valueFrom:
+            configMapKeyRef:
+              name: nydus-snapshotter-configs
+              key: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER
+              optional: true
+        - name: ENABLE_SYSTEMD_SERVICE
+          valueFrom:
+            configMapKeyRef:
+              name: nydus-snapshotter-configs
+              key: ENABLE_SYSTEMD_SERVICE
+              optional: true
+      lifecycle:
+        preStop:
+          exec:
+            command:
+              - "bash"
+              - "-c"
+              - |
+                /opt/nydus-artifacts/opt/nydus/snapshotter.sh cleanup
+      command:
+        - bash
+        - -c
+        - |-
+          /opt/nydus-artifacts/opt/nydus/snapshotter.sh deploy
       volumeMounts:
-        - mountPath: /etc/nydus/
-          name: nydus-all-config
-        - mountPath: /var/lib/containerd-nydus
+        - name: config-volume
+          mountPath: "/etc/nydus-snapshotter"
+        - name: nydus-lib
+          mountPath: "/var/lib/containerd-nydus"
           mountPropagation: Bidirectional
-          name: nydus-lib
-        - mountPath: /run/containerd-nydus
+        - name: nydus-run
+          mountPath: "/run/containerd-nydus"
           mountPropagation: Bidirectional
-          name: nydus-run
-        - mountPath: /run/containerd
+        - name: nydus-opt
+          mountPath: "/opt/nydus"
           mountPropagation: Bidirectional
-          name: containerd-run
-        - mountPath: /dev/fuse
-          name: fuse
-        - mountPath: /etc/containerd/config.toml
-          name: containerd-conf
-  hostNetwork: true
-  hostPID: true
-  serviceAccountName: nydus-snapshotter-sa
+        - name: nydus-etc
+          mountPath: "/etc/nydus"
+          mountPropagation: Bidirectional
+        - name: containerd-conf
+          mountPath: "/etc/containerd/config.toml"
+        - name: local-bin
+          mountPath: "/usr/local/bin/"
+        - name: etc-systemd-system
+          mountPath: "/etc/systemd/system/"
+        - name: fuse
+          mountPath: /dev/fuse
+      securityContext:
+        privileged: true
+
   volumes:
-    - name: nydus-all-config
-      projected:
-        sources:
-          - configMap:
-              name: nydusd-config
-          - configMap:
-              name: snapshotter-config
-    - hostPath:
+    - name: config-volume
+      configMap:
+        name: nydus-snapshotter-configs
+        optional: true
+    - name: nydus-run
+      hostPath:
         path: /run/containerd-nydus
         type: DirectoryOrCreate
-      name: nydus-run
-    - hostPath:
+    - name: nydus-lib
+      hostPath:
         path: /var/lib/containerd-nydus
         type: DirectoryOrCreate
-      name: nydus-lib
-    - hostPath:
-        path: /run/containerd
-        type: Directory
-      name: containerd-run
-    - hostPath:
-        path: /dev/fuse
-      name: fuse
-    - hostPath:
+    - name: nydus-etc
+      hostPath:
+          path: /etc/nydus
+          type: DirectoryOrCreate
+    - name: nydus-opt
+      hostPath:
+        path: /opt/nydus
+        type: DirectoryOrCreate
+    - name: containerd-conf
+      hostPath:
         path: /etc/containerd/config.toml
-        type: ""
-      name: containerd-conf
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: nydusd-config
-  namespace: nydus-system
-data:
-  nydusd.json: |-
-    {
-      "device": {
-        "backend": {
-          "type": "registry",
-          "config": {
-            "scheme": "",
-            "skip_verify": true,
-            "timeout": 10,
-            "connect_timeout": 10,
-            "retry_limit": 2
-          }
-        },
-        "cache": {
-          "type": "blobcache",
-          "config": {
-            "work_dir": "/var/lib/nydus/cache/"
-          }
-        }
-      },
-      "mode": "direct",
-      "digest_validate": false,
-      "iostats_files": false,
-      "enable_xattr": true,
-      "amplify_io": 1048576,
-      "fs_prefetch": {
-        "enable": true,
-        "threads_count": 10,
-        "merging_size": 131072,
-        "bandwidth_rate": 1048576
-      }
-    }
+    - name: local-bin
+      hostPath:
+        path: /usr/local/bin/
+    - name: etc-systemd-system
+      hostPath:
+        path: /etc/systemd/system/
+    - name: fuse
+      hostPath:
+        path: /dev/fuse
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: snapshotter-config
+  name: nydus-snapshotter-configs
+  labels:
+    app: nydus-snapshotter
   namespace: nydus-system
 data:
+  FS_DRIVER: "fusedev"
+  ENABLE_CONFIG_FROM_VOLUME: "true"
+  ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER: "false"
+  ENABLE_SYSTEMD_SERVICE: "true"
   config.toml: |-
     version = 1
     root = "/var/lib/containerd-nydus"
@@ -134,7 +148,6 @@ data:
     enable_stargz = false
     # Whether snapshotter should try to clean up resources when it is closed
     cleanup_on_close = false
-
     [daemon]
     nydusd_path = "/usr/local/bin/nydusd"
     nydusimage_path = "/usr/local/bin/nydus-image"
@@ -148,7 +161,6 @@ data:
     nydusd_config = "/etc/nydus/nydusd.json"
     # The fuse or fscache IO working threads started by nydusd
     threads_number = 4
-
     [log]
     # Snapshotter's log level
     level = "info"
@@ -160,10 +172,8 @@ data:
     # In unit MB(megabytes)
     log_rotation_max_size = 1
     log_to_stdout = false
-
     [remote]
     convert_vpc_registry = false
-
     [remote.auth]
     # Fetch the private registry auth by listening to K8s API server
     enable_kubeconfig_keychain = false
@@ -173,17 +183,47 @@ data:
     enable_cri_keychain = true
     # the target image service when using image proxy
     image_service_address = ""
-
     [snapshot]
     enable_nydus_overlayfs = false
     # Whether to remove resources when a snapshot is removed
     sync_remove = false
-
     [cache_manager]
     disable = false
     gc_period = "24h"
     cache_dir = ""
-
     [image]
     public_key_file = ""
     validate_signature = false
+
+  nydusd.json: |-
+    {
+      "device": {
+        "backend": {
+          "type": "registry",
+          "config": {
+            "scheme": "",
+            "skip_verify": true,
+            "timeout": 10,
+            "connect_timeout": 10,
+            "retry_limit": 2
+          }
+        },
+        "cache": {
+          "type": "blobcache",
+          "config": {
+            "work_dir": "/var/lib/nydus/cache/"
+          }
+        }
+      },
+      "mode": "direct",
+      "digest_validate": false,
+      "iostats_files": false,
+      "enable_xattr": true,
+      "amplify_io": 1048576,
+      "fs_prefetch": {
+        "enable": true,
+        "threads_count": 10,
+        "merging_size": 131072,
+        "bandwidth_rate": 1048576
+      }
+    }