From e0ba512a42cafb70219d0568c163786629f5821c Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Wed, 17 Jan 2024 17:10:34 +0800 Subject: [PATCH] tests: update the e2e tests update the e2e tests. Signed-off-by: ChengyuZhu6 --- .github/workflows/k8s-e2e.yml | 6 - tests/e2e/k8s/snapshotter-cri.yaml | 228 ++++++++++++++---------- tests/e2e/k8s/snapshotter-kubeconf.yaml | 227 +++++++++++++---------- 3 files changed, 270 insertions(+), 191 deletions(-) diff --git a/.github/workflows/k8s-e2e.yml b/.github/workflows/k8s-e2e.yml index fb46a55e31..d7a134890c 100644 --- a/.github/workflows/k8s-e2e.yml +++ b/.github/workflows/k8s-e2e.yml @@ -81,13 +81,7 @@ jobs: # change snapshotter to nydus after nydus snapshotter started registry_ip=$(ip addr show eth0 | grep 'inet ' | awk '{print $2}' | cut -d/ -f1) docker cp kind-control-plane:/etc/containerd/config.toml containerd.config.toml.bak - sed -i -e 's|snapshotter = "overlayfs"|snapshotter = "nydus"|' containerd.config.toml.bak cat << EOF >> containerd.config.toml.bak - [proxy_plugins] - [proxy_plugins.nydus] - type = "snapshot" - address = "/run/containerd-nydus/containerd-nydus-grpc.sock" - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."${registry_ip}:5000"] endpoint = ["http://${registry_ip}:5000"] EOF diff --git a/tests/e2e/k8s/snapshotter-cri.yaml b/tests/e2e/k8s/snapshotter-cri.yaml index 31641a5122..d011d26f67 100644 --- a/tests/e2e/k8s/snapshotter-cri.yaml +++ b/tests/e2e/k8s/snapshotter-cri.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Namespace metadata: @@ -15,113 +16,126 @@ kind: Pod metadata: name: nydus-snapshotter namespace: nydus-system + labels: + app: nydus-snapshotter spec: + serviceAccountName: nydus-snapshotter-sa + hostNetwork: true + hostPID: true containers: - name: nydus-snapshotter - image: local-dev:e2e + image: "local-dev:e2e" imagePullPolicy: IfNotPresent - args: - - containerd-nydus-grpc - - --nydusd /usr/local/bin/nydusd - - --nydusd-config /etc/nydus/nydusd.json - - --config /etc/nydus/config.toml - - --log-level debug - - --daemon-mode shared - - --log-to-stdout - securityContext: - privileged: true + env: + - name: FS_DRIVER + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: FS_DRIVER + optional: true + - name: ENABLE_CONFIG_FROM_VOLUME + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: ENABLE_CONFIG_FROM_VOLUME + optional: true + - name: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER + optional: true + - name: ENABLE_SYSTEMD_SERVICE + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: ENABLE_SYSTEMD_SERVICE + optional: true + lifecycle: + preStop: + exec: + command: + - "bash" + - "-c" + - | + /opt/nydus-artifacts/opt/nydus/snapshotter.sh cleanup + command: + - bash + - -c + - |- + /opt/nydus-artifacts/opt/nydus/snapshotter.sh deploy volumeMounts: - - mountPath: /etc/nydus/ - name: nydus-all-config - - mountPath: /var/lib/containerd-nydus + - name: config-volume + mountPath: "/etc/nydus-snapshotter" + - name: nydus-lib + mountPath: "/var/lib/containerd-nydus" mountPropagation: Bidirectional - name: nydus-lib - - mountPath: /run/containerd-nydus + - name: nydus-run + mountPath: "/run/containerd-nydus" mountPropagation: Bidirectional - name: nydus-run - - mountPath: /run/containerd + - name: nydus-opt + mountPath: "/opt/nydus" mountPropagation: Bidirectional - name: containerd-run - - mountPath: /dev/fuse - name: fuse - - mountPath: /etc/containerd/config.toml - name: containerd-conf - hostNetwork: true - hostPID: true - serviceAccountName: nydus-snapshotter-sa + - name: nydus-etc + mountPath: "/etc/nydus" + mountPropagation: Bidirectional + - name: containerd-conf + mountPath: "/etc/containerd/config.toml" + - name: local-bin + mountPath: "/usr/local/bin/" + - name: etc-systemd-system + mountPath: "/etc/systemd/system/" + - name: fuse + mountPath: /dev/fuse + securityContext: + privileged: true + volumes: - - name: nydus-all-config - projected: - sources: - - configMap: - name: nydusd-config - - configMap: - name: snapshotter-config - - hostPath: + - name: config-volume + configMap: + name: nydus-snapshotter-configs + optional: true + - name: nydus-run + hostPath: path: /run/containerd-nydus type: DirectoryOrCreate - name: nydus-run - - hostPath: + - name: nydus-lib + hostPath: path: /var/lib/containerd-nydus type: DirectoryOrCreate - name: nydus-lib - - hostPath: - path: /run/containerd - type: Directory - name: containerd-run - - hostPath: - path: /dev/fuse - name: fuse - - hostPath: + - name: nydus-etc + hostPath: + path: /etc/nydus + type: DirectoryOrCreate + - name: nydus-opt + hostPath: + path: /opt/nydus + type: DirectoryOrCreate + - name: containerd-conf + hostPath: path: /etc/containerd/config.toml - type: "" - name: containerd-conf ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: nydusd-config - namespace: nydus-system -data: - nydusd.json: |- - { - "device": { - "backend": { - "type": "registry", - "config": { - "scheme": "", - "skip_verify": true, - "timeout": 10, - "connect_timeout": 10, - "retry_limit": 2 - } - }, - "cache": { - "type": "blobcache", - "config": { - "work_dir": "/var/lib/nydus/cache/" - } - } - }, - "mode": "direct", - "digest_validate": false, - "iostats_files": false, - "enable_xattr": true, - "amplify_io": 1048576, - "fs_prefetch": { - "enable": true, - "threads_count": 10, - "merging_size": 131072, - "bandwidth_rate": 1048576 - } - } + - name: local-bin + hostPath: + path: /usr/local/bin/ + - name: etc-systemd-system + hostPath: + path: /etc/systemd/system/ + - name: fuse + hostPath: + path: /dev/fuse --- apiVersion: v1 kind: ConfigMap metadata: - name: snapshotter-config + name: nydus-snapshotter-configs + labels: + app: nydus-snapshotter namespace: nydus-system data: + FS_DRIVER: "fusedev" + ENABLE_CONFIG_FROM_VOLUME: "true" + ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER: "false" + ENABLE_SYSTEMD_SERVICE: "true" config.toml: |- version = 1 root = "/var/lib/containerd-nydus" @@ -134,7 +148,6 @@ data: enable_stargz = false # Whether snapshotter should try to clean up resources when it is closed cleanup_on_close = false - [daemon] nydusd_path = "/usr/local/bin/nydusd" nydusimage_path = "/usr/local/bin/nydus-image" @@ -148,7 +161,6 @@ data: nydusd_config = "/etc/nydus/nydusd.json" # The fuse or fscache IO working threads started by nydusd threads_number = 4 - [log] # Snapshotter's log level level = "info" @@ -160,10 +172,8 @@ data: # In unit MB(megabytes) log_rotation_max_size = 1 log_to_stdout = false - [remote] convert_vpc_registry = false - [remote.auth] # Fetch the private registry auth by listening to K8s API server enable_kubeconfig_keychain = false @@ -173,17 +183,47 @@ data: enable_cri_keychain = true # the target image service when using image proxy image_service_address = "" - [snapshot] enable_nydus_overlayfs = false # Whether to remove resources when a snapshot is removed sync_remove = false - [cache_manager] disable = false gc_period = "24h" cache_dir = "" - [image] public_key_file = "" validate_signature = false + + nydusd.json: |- + { + "device": { + "backend": { + "type": "registry", + "config": { + "scheme": "", + "skip_verify": true, + "timeout": 10, + "connect_timeout": 10, + "retry_limit": 2 + } + }, + "cache": { + "type": "blobcache", + "config": { + "work_dir": "/var/lib/nydus/cache/" + } + } + }, + "mode": "direct", + "digest_validate": false, + "iostats_files": false, + "enable_xattr": true, + "amplify_io": 1048576, + "fs_prefetch": { + "enable": true, + "threads_count": 10, + "merging_size": 131072, + "bandwidth_rate": 1048576 + } + } diff --git a/tests/e2e/k8s/snapshotter-kubeconf.yaml b/tests/e2e/k8s/snapshotter-kubeconf.yaml index 247155a914..7b1d4d60cb 100644 --- a/tests/e2e/k8s/snapshotter-kubeconf.yaml +++ b/tests/e2e/k8s/snapshotter-kubeconf.yaml @@ -1,15 +1,14 @@ +--- apiVersion: v1 kind: Namespace metadata: name: nydus-system - --- apiVersion: v1 kind: ServiceAccount metadata: name: nydus-snapshotter-sa namespace: nydus-system - --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -31,9 +30,9 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: nydus-snapshotter-role-binding roleRef: + apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: nydus-snapshotter-role - apiGroup: rbac.authorization.k8s.io subjects: - kind: ServiceAccount name: nydus-snapshotter-sa @@ -45,106 +44,126 @@ kind: Pod metadata: name: nydus-snapshotter namespace: nydus-system + labels: + app: nydus-snapshotter spec: + serviceAccountName: nydus-snapshotter-sa + hostNetwork: true + hostPID: true containers: - name: nydus-snapshotter - image: local-dev:e2e + image: "local-dev:e2e" imagePullPolicy: IfNotPresent - args: - - containerd-nydus-grpc - - --nydusd /usr/local/bin/nydusd - - --nydusd-config /etc/nydus/nydusd.json - - --config /etc/nydus/config.toml - - --log-level debug - - --daemon-mode shared - - --log-to-stdout - securityContext: - privileged: true + env: + - name: FS_DRIVER + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: FS_DRIVER + optional: true + - name: ENABLE_CONFIG_FROM_VOLUME + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: ENABLE_CONFIG_FROM_VOLUME + optional: true + - name: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER + optional: true + - name: ENABLE_SYSTEMD_SERVICE + valueFrom: + configMapKeyRef: + name: nydus-snapshotter-configs + key: ENABLE_SYSTEMD_SERVICE + optional: true + lifecycle: + preStop: + exec: + command: + - "bash" + - "-c" + - | + /opt/nydus-artifacts/opt/nydus/snapshotter.sh cleanup + command: + - bash + - -c + - |- + /opt/nydus-artifacts/opt/nydus/snapshotter.sh deploy volumeMounts: - - mountPath: /etc/nydus/ - name: nydus-all-config - - mountPath: /var/lib/containerd-nydus + - name: config-volume + mountPath: "/etc/nydus-snapshotter" + - name: nydus-lib + mountPath: "/var/lib/containerd-nydus" mountPropagation: Bidirectional - name: nydus-lib - - mountPath: /run/containerd-nydus + - name: nydus-run + mountPath: "/run/containerd-nydus" mountPropagation: Bidirectional - name: nydus-run - - mountPath: /dev/fuse - name: fuse - - mountPath: /etc/containerd/config.toml - name: containerd-conf - hostNetwork: true - hostPID: true - serviceAccountName: nydus-snapshotter-sa + - name: nydus-opt + mountPath: "/opt/nydus" + mountPropagation: Bidirectional + - name: nydus-etc + mountPath: "/etc/nydus" + mountPropagation: Bidirectional + - name: containerd-conf + mountPath: "/etc/containerd/config.toml" + - name: local-bin + mountPath: "/usr/local/bin/" + - name: etc-systemd-system + mountPath: "/etc/systemd/system/" + - name: fuse + mountPath: /dev/fuse + securityContext: + privileged: true + volumes: - - name: nydus-all-config - projected: - sources: - - configMap: - name: nydusd-config - - configMap: - name: snapshotter-config - - hostPath: + - name: config-volume + configMap: + name: nydus-snapshotter-configs + optional: true + - name: nydus-run + hostPath: path: /run/containerd-nydus type: DirectoryOrCreate - name: nydus-run - - hostPath: + - name: nydus-lib + hostPath: path: /var/lib/containerd-nydus type: DirectoryOrCreate - name: nydus-lib - - hostPath: - path: /dev/fuse - name: fuse - - hostPath: + - name: nydus-etc + hostPath: + path: /etc/nydus + type: DirectoryOrCreate + - name: nydus-opt + hostPath: + path: /opt/nydus + type: DirectoryOrCreate + - name: containerd-conf + hostPath: path: /etc/containerd/config.toml - type: "" - name: containerd-conf ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: nydusd-config - namespace: nydus-system -data: - nydusd.json: |- - { - "device": { - "backend": { - "type": "registry", - "config": { - "scheme": "", - "skip_verify": true, - "timeout": 10, - "connect_timeout": 10, - "retry_limit": 2 - } - }, - "cache": { - "type": "blobcache", - "config": { - "work_dir": "/var/lib/nydus/cache/" - } - } - }, - "mode": "direct", - "digest_validate": false, - "iostats_files": false, - "enable_xattr": true, - "amplify_io": 1048576, - "fs_prefetch": { - "enable": true, - "threads_count": 10, - "merging_size": 131072, - "bandwidth_rate": 1048576 - } - } + - name: local-bin + hostPath: + path: /usr/local/bin/ + - name: etc-systemd-system + hostPath: + path: /etc/systemd/system/ + - name: fuse + hostPath: + path: /dev/fuse --- apiVersion: v1 kind: ConfigMap metadata: - name: snapshotter-config + name: nydus-snapshotter-configs + labels: + app: nydus-snapshotter namespace: nydus-system data: + FS_DRIVER: "fusedev" + ENABLE_CONFIG_FROM_VOLUME: "true" + ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER: "false" + ENABLE_SYSTEMD_SERVICE: "false" config.toml: |- version = 1 root = "/var/lib/containerd-nydus" @@ -157,7 +176,6 @@ data: enable_stargz = false # Whether snapshotter should try to clean up resources when it is closed cleanup_on_close = false - [daemon] nydusd_path = "/usr/local/bin/nydusd" nydusimage_path = "/usr/local/bin/nydus-image" @@ -171,7 +189,6 @@ data: nydusd_config = "/etc/nydus/nydusd.json" # The fuse or fscache IO working threads started by nydusd threads_number = 4 - [log] # Snapshotter's log level level = "info" @@ -183,10 +200,8 @@ data: # In unit MB(megabytes) log_rotation_max_size = 1 log_to_stdout = false - [remote] convert_vpc_registry = false - [remote.auth] # Fetch the private registry auth by listening to K8s API server enable_kubeconfig_keychain = true @@ -196,17 +211,47 @@ data: enable_cri_keychain = false # the target image service when using image proxy image_service_address = "" - [snapshot] enable_nydus_overlayfs = false # Whether to remove resources when a snapshot is removed sync_remove = false - [cache_manager] disable = false gc_period = "24h" cache_dir = "" - [image] public_key_file = "" validate_signature = false + + nydusd.json: |- + { + "device": { + "backend": { + "type": "registry", + "config": { + "scheme": "", + "skip_verify": true, + "timeout": 10, + "connect_timeout": 10, + "retry_limit": 2 + } + }, + "cache": { + "type": "blobcache", + "config": { + "work_dir": "/var/lib/nydus/cache/" + } + } + }, + "mode": "direct", + "digest_validate": false, + "iostats_files": false, + "enable_xattr": true, + "amplify_io": 1048576, + "fs_prefetch": { + "enable": true, + "threads_count": 10, + "merging_size": 131072, + "bandwidth_rate": 1048576 + } + } \ No newline at end of file