add prefetch integration test

Signed-off-by: billie60 <[email protected]>

Makefile

@@ -135,6 +135,7 @@ smoke:
 integration:
 	CGO_ENABLED=1 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags '-X "${PKG}/version.Version=${VERSION}" -extldflags "-static"' -race -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
 	CGO_ENABLED=1 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags '-X "${PKG}/version.Version=${VERSION}" -extldflags "-static"' -race -v -o bin/nydus-overlayfs ./cmd/nydus-overlayfs
+	CGO_ENABLED=1 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags '-X "${PKG}/version.Version=${VERSION}" -extldflags "-static"' -race -v -o bin/prefetchfiles-nri-plugin ./cmd/prefetchfiles-nri-plugin
 	$(SUDO) DOCKER_BUILDKIT=1 docker build ${BUILD_ARG_E2E_DOWNLOADS_MIRROR} -t nydus-snapshotter-e2e:0.1 -f integration/Dockerfile .
 	$(SUDO) docker run --cap-add SYS_ADMIN --security-opt seccomp=unconfined --cgroup-parent=system.slice --cgroupns private --name nydus-snapshotter_e2e --rm --privileged -v /root/.docker:/root/.docker -v `go env GOMODCACHE`:/go/pkg/mod \
 	-v `go env GOCACHE`:/root/.cache/go-build -v `pwd`:/nydus-snapshotter \

integration/Dockerfile

@@ -1,11 +1,13 @@
 
-ARG CONTAINERD_VER=1.6.15
+ARG CONTAINERD_VER=1.7.8
 ARG CONTAINERD_PROJECT=/containerd
 ARG RUNC_VERSION=1.1.4
 ARG NYDUS_SNAPSHOTTER_PROJECT=/nydus-snapshotter
 ARG DOWNLOADS_MIRROR="https://github.com"
 ARG NYDUS_VER=2.1.4
 ARG NERDCTL_VER=1.0.0
+ARG CRICTL_VER=1.26.0
+ARG CNI_VER=1.2.0
 
 FROM golang:1.19.6-bullseye AS golang-base
 
@@ -16,14 +18,16 @@ ARG NYDUS_SNAPSHOTTER_PROJECT
 ARG DOWNLOADS_MIRROR
 ARG NYDUS_VER
 ARG NERDCTL_VER
+ARG CRICTL_VER
+ARG CNI_VER
 
 # RUN echo '\
 # deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye main contrib non-free\n\
 # deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye-updates main contrib non-free\n\
 # deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye-backports main contrib non-free\n\
 # deb https://mirrors.tuna.tsinghua.edu.cn/debian-security bullseye-security main contrib non-free\n' > /etc/apt/sources.list
 
-RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev sudo psmisc jq lsof net-tools
+RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev sudo psmisc jq lsof net-tools iptables
 
 RUN go install github.com/go-delve/delve/cmd/dlv@latest
 
@@ -49,10 +53,29 @@ RUN wget ${DOWNLOADS_MIRROR}/containerd/nerdctl/releases/download/v${NERDCTL_VER
 tar xzf nerdctl-${NERDCTL_VER}-linux-amd64.tar.gz && \
 install -D -m 755 nerdctl /usr/local/bin/nerdctl
 
+# Install crictl
+RUN wget ${DOWNLOADS_MIRROR}/kubernetes-sigs/cri-tools/releases/download/v${CRICTL_VER}/crictl-v${CRICTL_VER}-linux-amd64.tar.gz && \
+tar xzf crictl-v${CRICTL_VER}-linux-amd64.tar.gz && \
+install -D -m 755 crictl /usr/local/bin/crictl
+
+# Startup CNI
+RUN wget ${DOWNLOADS_MIRROR}/containernetworking/plugins/releases/download/v${CNI_VER}/cni-plugins-linux-amd64-v${CNI_VER}.tgz && \
+mkdir -p /opt/cni/bin && \
+tar xzf cni-plugins-linux-amd64-v${CNI_VER}.tgz -C /opt/cni/bin/
+
+
+
 # Install fscache driver configuration file
 COPY misc/snapshotter/nydusd-config.fscache.json /etc/nydus/nydusd-config.fscache.json
 COPY misc/snapshotter/nydusd-config-localfs.json /etc/nydus/nydusd-config-localfs.json
 COPY misc/snapshotter/config.toml /etc/nydus/config.toml
+COPY misc/example/10-containerd-net.conflist /etc/cni/net.d/10-containerd-net.conflist
+COPY misc/prefetch/crictl.yaml /etc/crictl.yaml
+COPY misc/example/prefetchfiles-nri-plugin.conf /etc/nri/conf.d/03-prefetchfiles-nri-plugin.conf
+RUN chmod +x /etc/nri/conf.d/03-prefetchfiles-nri-plugin.conf
+COPY bin/prefetchfiles-nri-plugin /opt/nri/plugins/03-prefetchfiles-nri-plugin
+RUN chmod +x /opt/nri/plugins/03-prefetchfiles-nri-plugin
+COPY misc/prefetch/prefetch_server.go /go/prefetch_server.go
 
 VOLUME [ "/var/lib" ]
 

integration/entrypoint.sh

@@ -25,6 +25,12 @@ STARGZ_IMAGE=${STARGZ_IMAGE:-ghcr.io/stargz-containers/wordpress:5.9.2-esgz}
 REDIS_OCI_IMAGE=${REDIS_OCI_IMAGE:-ghcr.io/stargz-containers/redis:6.2.6-org}
 WORDPRESS_OCI_IMAGE=${WORDPRESS_OCI_IMAGE:-ghcr.io/dragonflyoss/image-service/wordpress:latest}
 
+WORDPRESS_PREFETCH=${WORDPRESS_PREFETCH:-http://localhost:8090/prefetch/wordpress}
+TOMCAT_PREFETCH=${TOMCAT_PREFETCH:-http://localhost:8090/prefetch/tomcat}
+declare -A IMAGE_PREFETCH
+IMAGE_PREFETCH=([${WORDPRESS_IMAGE}]=${WORDPRESS_PREFETCH} [${TOMCAT_IMAGE}]=${TOMCAT_PREFETCH})
+
+
 PLUGIN=nydus
 
 RETRYNUM=30
@@ -47,6 +53,33 @@ function detect_go_race {
     fi
 }
 
+function stop_all_containers_crictl {
+     containers=$(crictl ps -aq | tr '\n' ' ')
+     if [[ ${containers} == "" ]]; then
+             return 0
+     else
+         echo "Remove containers ${containers}"
+         for C in ${containers}; do
+             crictl stop "${C}" || true
+             crictl rm "${C}" || true
+         done
+         return 1
+     fi
+}
+function stop_all_pods_crictl {
+  pods=$(crictl pods -q | tr '\n' ' ')
+  if [[ ${pods} == "" ]]; then
+    return 0
+  else
+    echo "Remove pods ${pods}"
+    for P in ${pods}; do
+      crictl stopp "${P}" || true
+      crictl rmp "${P}" || true
+    done
+    return 1
+  fi
+}
+
 function stop_all_containers {
     containers=$(nerdctl ps -q | tr '\n' ' ')
     if [[ ${containers} == "" ]]; then
@@ -211,6 +244,16 @@ function is_cache_cleared {
     fi
 }
 
+function crictl_prune_images {
+    # Wait for containers observation.
+    sleep 1
+    func_retry stop_all_containers_crictl
+    crictl rmi --all
+    func_retry stop_all_pods_crictl
+    crictl images
+    is_cache_cleared
+}
+
 function nerdctl_prune_images {
     # Wait for containers observation.
     sleep 1
@@ -573,14 +616,149 @@ function enable_nesting_for_cgroup_v2() {
     fi
 }
 
+function set_prefetch {
+    echo "testing $FUNCNAME"
+
+    go run /go/prefetch_server.go &
+}
+
+# Function to create a Pod
+function start_single_container_prefetch {
+    echo "testing $FUNCNAME"
+    crictl_prune_images
+    reboot_containerd multiple
+
+    IFS="/" read -ra parts <<< "${1}"
+    image_name="${parts[-1]}"
+    IFS=':' read -ra parts <<< "$image_name"
+    image_name="${parts[0]}"
+    local pod_yaml="${image_name}-pod.yaml"
+    cat > "$pod_yaml" <<EOF
+    kind: pod
+    metadata:
+      name: "${image_name}-pod"
+      namespace: default
+      attempt: 1
+      uid: hdishd83djaidwnduwk28bcsb
+    log_directory: /tmp
+    annotations:
+      containerd.io/nydus-prefetch: |
+        [
+          {"image": "${1}", "prefetch": "${2}"}
+        ]
+    linux: {}
+EOF
+    local cmd_output=$(crictl runp "$pod_yaml" 2>&1)
+    if [[ "$cmd_output" == *"FATA"* ]] && [[ "$cmd_output" == *"error"* ]]; then
+        echo "Failed to create Pod, pod existed"
+        return 1
+    else
+        local pod_id
+        while read -r line; do
+          pod_id="$line"
+        done <<< "$cmd_output"
+    fi
+    local container_yaml="${image_name}-container.yaml"
+    cat > $container_yaml <<EOF
+    metadata:
+      name: $image_name
+    image:
+      image: ${1}
+    log_path: "${image_name}.0.log"
+    linux: {}
+EOF
+    crictl pull ${1}
+    echo "Running: crictl create ${pod_id} ${container_yaml} ${pod_yaml}"
+    output=$(crictl create ${pod_id} ${container_yaml} ${pod_yaml})
+    local container_id
+    while read -r line; do
+        container_id="$line"
+    done <<< "$output"
+    echo "Running: crictl start ${container_id}"
+    crictl start ${container_id}
+    return 0
+
+    detect_go_race
+}
+
+
+
+function start_multiple_containers_prefetch {
+    echo "testing $FUNCNAME"
+    crictl_prune_images
+    reboot_containerd multiple
+
+    local -n image_prefetch="${1}"
+    for key in "${!image_prefetch[@]}"; do
+        IFS="/" read -ra parts <<< "${key}"
+        image_name="${parts[-1]}"
+        IFS=':' read -ra parts <<< "$image_name"
+        image_name="${parts[0]}"
+        local pod_yaml="${image_name}-pod.yaml"
+        cat > "$pod_yaml" <<EOF
+        kind: pod
+        metadata:
+          name: "${image_name}-pod"
+          namespace: default
+          attempt: 1
+          uid: hdishd83djaidwnduwk28bcsb
+        log_directory: /tmp
+        annotations:
+          containerd.io/nydus-prefetch: |
+            [
+              {"image": "${key}", "prefetch": "${IMAGE_PREFETCH[$key]}"}
+            ]
+        linux: {}
+EOF
+        local cmd_output=$(crictl runp "$pod_yaml" 2>&1)
+        if [[ "$cmd_output" == *"FATA"* ]] && [[ "$cmd_output" == *"error"* ]]; then
+            echo "Failed to create Pod, pod existed"
+            return 1
+        else
+            local pod_id
+            while read -r line; do
+                pod_id="$line"
+            done <<< "$cmd_output"
+        fi
+        local container_yaml="${image_name}-container.yaml"
+        cat > $container_yaml <<EOF
+        metadata:
+          name: $image_name
+        image:
+          image: ${key}
+        log_path: "${image_name}.0.log"
+        linux: {}
+EOF
+        crictl pull ${key}
+        output=$(crictl create ${pod_id} ${container_yaml} ${pod_yaml})
+        local container_id
+        while read -r line; do
+          container_id="$line"
+        done <<< "$output"
+        echo "Running: crictl start ${container_id}"
+        crictl start ${container_id}
+      done
+      return 0
+
+      detect_go_race
+}
+
+
+
+
+
+
 enable_nesting_for_cgroup_v2
 
 reboot_containerd multiple
 
+
 start_single_container_multiple_daemons
 start_multiple_containers_multiple_daemons
 start_multiple_containers_shared_daemon
 
+
+
 pull_remove_one_image
 
 pull_remove_multiple_images shared
@@ -609,3 +787,10 @@ fi
 start_container_on_oci
 
 start_container_with_referrer_detect
+
+nerdctl_prune_images
+
+set_prefetch
+start_single_container_prefetch "${WORDPRESS_IMAGE}" "${WORDPRESS_PREFETCH}"
+start_multiple_containers_prefetch IMAGE_PREFETCH
+crictl_prune_images

misc/example/containerd-config.toml

@@ -19,6 +19,12 @@ oom_score = 0
       bin_dir = "/opt/cni/bin"
       conf_dir = "/etc/cni/net.d"
 
+[plugins."io.containerd.nri.v1.nri"]
+  config_file = "/etc/nri/nri.conf"
+  disable = false
+  plugin_path = "/opt/nri/plugins"
+  socket_path = "/var/run/nri.sock"
+
 [proxy_plugins]
   [proxy_plugins.nydus]
       type = "snapshot"

misc/prefetch/crictl.yaml

@@ -0,0 +1,4 @@
+runtime-endpoint: unix:///run/containerd/containerd.sock
+image-endpoint: unix:///run/containerd/containerd.sock
+timeout: 10
+debug: true