add prefetch e2e test

Signed-off-by: billie60 <[email protected]>
containerd · Oct 31, 2023 · 1589530 · 1589530
1 parent 417a15d
commit 1589530
Show file tree

Hide file tree

Showing 3 changed files with 165 additions and 0 deletions.
diff --git a/integration/Dockerfile b/integration/Dockerfile
@@ -6,6 +6,7 @@ ARG NYDUS_SNAPSHOTTER_PROJECT=/nydus-snapshotter
 ARG DOWNLOADS_MIRROR="https://github.com"
 ARG NYDUS_VER=2.1.4
 ARG NERDCTL_VER=1.0.0
+ARG CRICTL_VER=1.26.0
 
 FROM golang:1.19.6-bullseye AS golang-base
 
@@ -16,6 +17,7 @@ ARG NYDUS_SNAPSHOTTER_PROJECT
 ARG DOWNLOADS_MIRROR
 ARG NYDUS_VER
 ARG NERDCTL_VER
+ARG CRICTL_VER
 
 # RUN echo '\
 # deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye main contrib non-free\n\
@@ -49,10 +51,16 @@ RUN wget ${DOWNLOADS_MIRROR}/containerd/nerdctl/releases/download/v${NERDCTL_VER
 tar xzf nerdctl-${NERDCTL_VER}-linux-amd64.tar.gz && \
 install -D -m 755 nerdctl /usr/local/bin/nerdctl
 
+# Install crictl
+RUN wget ${DOWNLOADS_MIRROR}/kubernetes-sigs/cri-tools/releases/download/v${CRICTL_VER}/crictl-v${CRICTL_VER}-linux-amd64.tar.gz && \
+tar xzf crictl-v${CRICTL_VER}-linux-amd64.tar.gz && \
+install -D -m 755 crictl /usr/local/bin/crictl
+
 # Install fscache driver configuration file
 COPY misc/snapshotter/nydusd-config.fscache.json /etc/nydus/nydusd-config.fscache.json
 COPY misc/snapshotter/nydusd-config-localfs.json /etc/nydus/nydusd-config-localfs.json
 COPY misc/snapshotter/config.toml /etc/nydus/config.toml
+COPY misc/prefetch/crictl.yaml /etc/crictl.yaml
 
 VOLUME [ "/var/lib" ]
 

diff --git a/integration/entrypoint.sh b/integration/entrypoint.sh
@@ -25,6 +25,10 @@ STARGZ_IMAGE=${STARGZ_IMAGE:-ghcr.io/stargz-containers/wordpress:5.9.2-esgz}
 REDIS_OCI_IMAGE=${REDIS_OCI_IMAGE:-ghcr.io/stargz-containers/redis:6.2.6-org}
 WORDPRESS_OCI_IMAGE=${WORDPRESS_OCI_IMAGE:-ghcr.io/dragonflyoss/image-service/wordpress:latest}
 
+WORDPRESS_PREFETCH=${WORDPRESS_PREFETCH:-http://192.168.253.128:8090/prefetch/wordpress}
+TOMCAT_PREFETCH=${TOMCAT_PREFETCH:-http://192.168.253.128:8090/prefetch/tomcat}
+WORDPRESS_IMAGE=210.30.96.107/images_size_test_r2/wordpress:nydus_latest
+
 PLUGIN=nydus
 
 RETRYNUM=30
@@ -47,6 +51,20 @@ function detect_go_race {
     fi
 }
 
+function stop_all_containers_crictl {
+     containers=$(crictl ps -aq | tr '\n' ' ')
+     if [[ ${containers} == "" ]]; then
+             return 0
+     else
+         echo "Remove containers ${containers}"
+         for C in ${containers}; do
+             crictl stop "${C}" || true
+             nerdctl rm "${C}" || true
+         done
+         return 1
+     fi
+}
+
 function stop_all_containers {
     containers=$(nerdctl ps -q | tr '\n' ' ')
     if [[ ${containers} == "" ]]; then
@@ -211,6 +229,17 @@ function is_cache_cleared {
     fi
 }
 
+function crictl_prune_images {
+    # Wait for containers observation.
+    sleep 1
+    func_retry stop_all_containers_crictl
+#    nerdctl container prune -f
+    crictl rmi --all
+    crictl rmp --all -f
+    crictl images
+    is_cache_cleared
+}
+
 function nerdctl_prune_images {
     # Wait for containers observation.
     sleep 1
@@ -573,10 +602,134 @@ function enable_nesting_for_cgroup_v2() {
     fi
 }
 
+# Function to create a Pod
+function start_single_container_prefetch {
+    echo "testing $FUNCNAME"
+    crictl_prune_images
+    reboot_containerd multiple
+
+
+    IFS="/" read -ra parts <<< "${1}"
+    image_name="${parts[-1]}"
+    IFS=':' read -ra parts <<< "$image_name"
+    image_name="${parts[0]}"
+    local pod_yaml="${image_name}-pod.yaml"
+    cat > "$pod_yaml" <<EOF
+    kind: pod
+    metadata:
+      name: "${image_name}-pod"
+      namespace: default
+      attempt: 1
+      uid: hdishd83djaidwnduwk28bcsb
+    log_directory: /tmp
+    annotations:
+      containerd.io/nydus-prefetch: |
+        [
+          {"image": "${1}", "prefetch": "${2}"}
+        ]
+    linux: {}
+EOF
+    local cmd_output=$(crictl runp "$pod_yaml" 2>&1)
+    if [[ "$cmd_output" == *"FATA"* ]] && [[ "$cmd_output" == *"error"* ]]; then
+        echo "Failed to create Pod, pod existed"
+        return 1
+    else
+        local pod_id
+        while read -r line; do
+          pod_id="$line"
+        done <<< "$cmd_output"
+    fi
+    local container_yaml="${image_name}-container.yaml"
+    cat > $container_yaml <<EOF
+    metadata:
+      name: $image_name
+    image:
+      image: ${1}
+    log_path: "${image_name}.0.log"
+    linux: {}
+EOF
+    crictl pull ${1}
+    output=$(crictl create ${pod_id} ${container_yaml} ${pod_yaml})
+    local container_id
+    while read -r line; do
+        container_id="$line"
+    done <<< "$output"
+    crictl start ${container_id}
+    return 0
+
+    detect_go_race
+}
+
+function start_multiple_containers_prefetch {
+    echo "testing $FUNCNAME"
+    crictl_prune_images
+    reboot_containerd multiple
+
+    local -n image_prefetch="${1}"
+    for key in "${!image_prefetch[@]}"; do
+        IFS="/" read -ra parts <<< "${key}"
+        image_name="${parts[-1]}"
+        IFS=':' read -ra parts <<< "$image_name"
+        image_name="${parts[0]}"
+        local pod_yaml="${image_name}-pod.yaml"
+        cat > "$pod_yaml" <<EOF
+        kind: pod
+        metadata:
+          name: "${image_name}-pod"
+          namespace: default
+          attempt: 1
+          uid: hdishd83djaidwnduwk28bcsb
+        log_directory: /tmp
+        annotations:
+          containerd.io/nydus-prefetch: |
+            [
+              {"image": "${key}", "prefetch": "${IMAGE_PREFETCH[$key]}"}
+            ]
+        linux: {}
+EOF
+        local cmd_output=$(crictl runp "$pod_yaml" 2>&1)
+        if [[ "$cmd_output" == *"FATA"* ]] && [[ "$cmd_output" == *"error"* ]]; then
+            echo "Failed to create Pod, pod existed"
+            return 1
+        else
+            local pod_id
+            while read -r line; do
+                pod_id="$line"
+            done <<< "$cmd_output"
+        fi
+        local container_yaml="${image_name}-container.yaml"
+        cat > $container_yaml <<EOF
+        metadata:
+          name: $image_name
+        image:
+          image: ${key}
+        log_path: "${image_name}.0.log"
+        linux: {}
+EOF
+        crictl pull ${key}
+        output=$(crictl create ${pod_id} ${container_yaml} ${pod_yaml})
+        local container_id
+        while read -r line; do
+          container_id="$line"
+        done <<< "$output"
+        crictl start ${container_id}
+      done
+      return 0
+
+      detect_go_race
+  }
+
+
+
+
+
 enable_nesting_for_cgroup_v2
 
 reboot_containerd multiple
 
+start_single_container_prefetch "${WORDPRESS_IMAGE}" "${WORDPRESS_PREFETCH}"
+start_multiple_containers_prefetch IMAGE_PREFETCH
+
 start_single_container_multiple_daemons
 start_multiple_containers_multiple_daemons
 start_multiple_containers_shared_daemon

diff --git a/misc/prefetch/crictl.yaml b/misc/prefetch/crictl.yaml
@@ -0,0 +1,4 @@
+runtime-endpoint: unix:///run/containerd/containerd.sock
+image-endpoint: unix:///run/containerd/containerd.sock
+timeout: 10
+debug: true