modify optimizer test

Signed-off-by: Guangyu Xu <[email protected]>

.github/workflows/optimizer.yml

@@ -16,6 +16,12 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
+  REGISTRY: ghcr.io
+  SNAPSHOTTER_CONFIG: /etc/nydus/config.toml
+  SOURCE_SNAPSHOTTER_CONFIG: misc/snapshotter/config_optimizer.toml
+  NYDUSD_CONFIG: /etc/nydus/nydusd-config.fusedev.json
+  SOURCE_NYDUSD_CONFIG: misc/snapshotter/nydusd-config.fusedev.json
+  SNAPSHOTTER_SYSTEMD_UNIT_SERVICE: misc/snapshotter/nydus-snapshotter.fusedev.service
 
 jobs:
   run_optimizer:
@@ -58,6 +64,34 @@ jobs:
           sudo mkdir -p /opt/cni/bin
           sudo tar xzf cni-plugins-linux-amd64-v1.2.0.tgz -C /opt/cni/bin/
           sudo install -D -m 755 misc/example/10-containerd-net.conflist /etc/cni/net.d/10-containerd-net.conflist
+      - name: Setup nydus-snapshotter and nydus service
+        run: |
+          GOOS=linux GOARCH=amd64 go build -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
+          sudo install -D -m 755 bin/containerd-nydus-grpc /usr/local/bin/containerd-nydus-grpc
+          if [ ! -e "${{ env.NYDUSD_CONFIG }}" ]; then
+          echo "Installing ${{ env.SOURCE_NYDUSD_CONFIG }} to ${{ env.NYDUSD_CONFIG }}"
+          sudo install -D -m 664 "${{ env.SOURCE_NYDUSD_CONFIG }}" "${{ env.NYDUSD_CONFIG }}"
+          fi
+          if [ ! -e "${{ env.SNAPSHOTTER_CONFIG }}" ]; then
+          echo "Installing ${{ env.SOURCE_SNAPSHOTTER_CONFIG }} to ${{ env.SNAPSHOTTER_CONFIG }}"
+          sudo install -D -m 664 "${{ env.SOURCE_SNAPSHOTTER_CONFIG }}" "${{ env.SNAPSHOTTER_CONFIG }}"
+          fi
+          sudo ln -f -s /etc/nydus/nydusd-config.fusedev.json /etc/nydus/nydusd-config.json
+          echo "Installing ${{ env.SNAPSHOTTER_SYSTEMD_UNIT_SERVICE }} to /etc/systemd/system/nydus-snapshotter.service"
+          sudo install -D -m 644 "${{ env.SNAPSHOTTER_SYSTEMD_UNIT_SERVICE }}" /etc/systemd/system/nydus-snapshotter.service
+          if command -v systemctl >/dev/null; then
+            sudo systemctl enable /etc/systemd/system/nydus-snapshotter.service
+            sudo systemctl restart nydus-snapshotter
+          fi
+          sleep 5
+          NYDUS_VER=v$(curl -s "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//')
+          wget https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz
+          tar xzvf nydus-static-$NYDUS_VER-linux-amd64.tgz
+          # sudo systemctl restart nydus-snapshotter.service
+          sudo install -D -m 755 nydus-static/nydusd /usr/local/bin/nydusd 
+          sudo install -D -m 755 nydus-static/nydus-image /usr/local/bin/nydus-image 
+          sudo install -D -m 755 nydus-static/nydusctl /usr/local/bin/nydusctl
+
       - name: Build and install optimizer
         run: |
           rustup component add rustfmt clippy
@@ -84,26 +118,48 @@ jobs:
             echo "containerd is not ready"
             exit 1
           fi
+      - name: Setup prefetch-distribution http server
+        run: |
+          go build -o prefetch-distribution tools/prefetch-distribution/main.go
+          nohup ./prefetch-distribution &  
+          sleep 5
+      - name: Log in to container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Generate accessed files list
         run: |
           sed -i "s|host_path: script|host_path: $(pwd)/misc/optimizer/script|g" misc/optimizer/nginx.yaml
           sudo crictl run misc/optimizer/nginx.yaml misc/optimizer/sandbox.yaml
           sleep 20
           sudo crictl rmp  -f --all
+          sudo crictl rmi --all
           tree /opt/nri/optimizer/results/
-          count=$(cat /opt/nri/optimizer/results/library/nginx:1.23.3 | wc -l)
+          count=$(cat /opt/nri/optimizer/results/dragonflyoss/image-service/nginx:nydus-latest | wc -l)
           expected=$(cat misc/optimizer/script/file_list.txt | wc -l)
           echo "count: $count expected minimum value: $expected"
           if [ $count -lt $expected ]; then
-            echo "failed to generate accessed files list for nginx:1.23.3"
+            echo "failed to generate accessed files list for nginx:nydus-latest"
             cat misc/optimizer/script/file_list.txt
             exit 1
           fi
-          cat /opt/nri/optimizer/results/library/nginx:1.23.3.csv
+          cat /opt/nri/optimizer/results/dragonflyoss/image-service/nginx:nydus-latest.csv
+      - name: Transmit the prefetch list to nydusd
+        run: |
+          sed -i "s|host_path: script|host_path: $(pwd)/misc/optimizer/script|g" misc/optimizer/nginx.yaml
+          sudo crictl run misc/optimizer/nginx.yaml misc/optimizer/sandbox.yaml
+          if sudo ps aux | grep  "[/]usr/local/bin/nydusd"  | grep "prefetch-files"; then
+              echo "Found --prefetch-files in running processes"
+          else
+              echo "Error: --prefetch-files not found in running processes"
+              exit 1
+          fi
       - name: Dump logs
         if: failure()
         continue-on-error: true
         run: |
           systemctl status containerd --no-pager -l
           journalctl -xeu containerd --no-pager
-          

config/config_test.go

@@ -31,6 +31,10 @@ func TestLoadSnapshotterTOMLConfig(t *testing.T) {
 			EnableStargz:         false,
 			EnableReferrerDetect: false,
 		},
+		PrefetchControllerConfig: PrefetchControllerConfig{
+			Enable:         false,
+			PrefetchConfig: "http://localhost:1323/api/v1/prefetch",
+		},
 		CleanupOnClose: false,
 		SystemControllerConfig: SystemControllerConfig{
 			Enable:  true,

misc/optimizer/containerd-config.toml

@@ -14,6 +14,11 @@
 
 version = 2
 
+[plugins."io.containerd.grpc.v1.cri"]
+  [plugins."io.containerd.grpc.v1.cri".containerd]
+      snapshotter = "nydus"
+      disable_snapshot_annotations = false
+
 [proxy_plugins]
   [proxy_plugins.nydus]
     type = "snapshot"

misc/optimizer/nginx.yaml

@@ -2,7 +2,7 @@ metadata:
   name: nginx
 
 image:
-  image: nginx:1.23.3
+  image: ghcr.io/dragonflyoss/image-service/nginx:nydus-latest
 
 mounts:
   - host_path: script

misc/snapshotter/config.toml

@@ -11,7 +11,7 @@ cleanup_on_close = false
 
 [prefetch]
 enable = false
-get_prefetch_endpoint = "http://localhost:1323/api/v1/prefetch/download"
+distribution_pull_endpoint = "http://localhost:1323/api/v1/prefetch"
 
 [system]
 # Snapshotter's debug and trace HTTP server interface
@@ -136,4 +136,4 @@ max_concurrent_proc = 0
 # - "image_block": generate a raw block disk image with tarfs for an image
 # - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info
 # - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info
-export_mode = ""
+export_mode = ""

misc/snapshotter/config_optimizer.toml

@@ -0,0 +1,139 @@
+version = 1
+# Snapshotter's own home directory where it stores and creates necessary resources
+root = "/var/lib/containerd-nydus"
+# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket
+address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
+# The nydus daemon mode can be one of the following options: multiple, dedicated, shared, or none. 
+# If `daemon_mode` option is not specified, the default value is multiple.
+daemon_mode = "dedicated"
+# Whether snapshotter should try to clean up resources when it is closed
+cleanup_on_close = false
+
+[prefetch]
+enable = true
+distribution_pull_endpoint = "http://localhost:1323/api/v1/prefetch"
+
+[system]
+# Snapshotter's debug and trace HTTP server interface
+enable = true
+# Unix domain socket path where system controller is listening on
+address = "/run/containerd-nydus/system.sock"
+
+[system.debug]
+# Snapshotter can profile the CPU utilization of each nydusd daemon when it is being started.
+# This option specifies the profile duration when nydusd is downloading and uncomproessing data.
+daemon_cpu_profile_duration_secs = 5
+# Enable by assigning an address, empty indicates pprof server is disabled
+pprof_address = ""
+
+[daemon]
+# Specify a configuration file for nydusd
+nydusd_config = "/etc/nydus/nydusd-config.fusedev.json"
+nydusd_path = "/usr/local/bin/nydusd"
+nydusimage_path = "/usr/local/bin/nydus-image"
+# The fs driver can be one of the following options: fusedev, fscache, blockdev, proxy, or nodev. 
+# If `fs_driver` option is not specified, the default value is fusedev.
+fs_driver = "fusedev"
+# How to process when daemon dies: "none", "restart" or "failover"
+recover_policy = "restart"
+# Nydusd worker thread number to handle FUSE or fscache requests, [0-1024].
+# Setting to 0 will use the default configuration of nydusd.
+threads_number = 4
+# Log rotation size for nydusd, in unit MB(megabytes). (default 100MB)
+log_rotation_size = 100
+
+[cgroup]
+# Whether to use separate cgroup for nydusd.
+enable = true
+# The memory limit for nydusd cgroup, which contains all nydusd processes.
+# Percentage is supported as well, please ensure it is end with "%".
+# The default unit is bytes. Acceptable values include "209715200", "200MiB", "200Mi" and "10%".
+memory_limit = ""
+
+[log]
+# Print logs to stdout rather than logging files
+log_to_stdout = false
+# Snapshotter's log level
+level = "info"
+log_rotation_compress = true
+log_rotation_local_time = true
+# Max number of days to retain logs
+log_rotation_max_age = 7
+log_rotation_max_backups = 5
+# In unit MB(megabytes)
+log_rotation_max_size = 100
+
+[metrics]
+# Enable by assigning an address, empty indicates metrics server is disabled
+address = ":9110"
+
+[remote]
+convert_vpc_registry = false
+
+[remote.mirrors_config]
+# Snapshotter will overwrite daemon's mirrors configuration
+# if the values loaded from this driectory are not null before starting a daemon.
+# Set to "" or an empty directory to disable it.
+#dir = "/etc/nydus/certs.d"
+
+[remote.auth]
+# Fetch the private registry auth by listening to K8s API server
+enable_kubeconfig_keychain = false
+# synchronize `kubernetes.io/dockerconfigjson` secret from kubernetes API server with specified kubeconfig (default `$KUBECONFIG` or `~/.kube/config`)
+kubeconfig_path = ""
+# Fetch the private registry auth as CRI image service proxy
+enable_cri_keychain = false
+# the target image service when using image proxy
+#image_service_address = "/run/containerd/containerd.sock"
+
+[snapshot]
+# Let containerd use nydus-overlayfs mount helper
+enable_nydus_overlayfs = false
+# Insert Kata Virtual Volume option to `Mount.Options`
+enable_kata_volume = false
+# Whether to remove resources when a snapshot is removed
+sync_remove = false
+
+[cache_manager]
+# Disable or enable recyclebin
+disable = false
+# How long to keep deleted files in recyclebin
+gc_period = "24h"
+# Directory to host cached files
+cache_dir = ""
+
+[image]
+public_key_file = ""
+validate_signature = false
+
+# The configuraions for features that are not production ready
+[experimental]
+# Whether to enable stargz support
+enable_stargz = false
+# Whether to enable referrers support
+# The option enables trying to fetch the Nydus image associated with the OCI image and run it.
+# Also see https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers
+enable_referrer_detect = false
+# Whether to enable authentication support
+# The option enables nydus snapshot to provide backend information to nydusd.
+enable_backend_source = false
+[experimental.tarfs]
+# Whether to enable nydus tarfs mode. Tarfs is supported by:
+# - The EROFS filesystem driver since Linux 6.4
+# - Nydus Image Service release v2.3
+enable_tarfs = false
+# Mount rafs on host by loopdev and EROFS
+mount_tarfs_on_host = false
+# Only enable nydus tarfs mode for images with `tarfs hint` label when true
+tarfs_hint = false
+# Maximum of concurrence to converting OCIv1 images to tarfs, 0 means default
+max_concurrent_proc = 0
+# Mode to export tarfs images:
+# - "none" or "": do not export tarfs
+# - "layer_verity_only": only generate disk verity information for a layer blob
+# - "image_verity_only": only generate disk verity information for all blobs of an image
+# - "layer_block": generate a raw block disk image with tarfs for a layer
+# - "image_block": generate a raw block disk image with tarfs for an image
+# - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info
+# - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info
+export_mode = ""