From f021603ec495afced4a10f74f91e2954cf37d5f2 Mon Sep 17 00:00:00 2001 From: Guangyu Xu Date: Thu, 18 Apr 2024 18:33:22 +0000 Subject: [PATCH] modify optimizer test Signed-off-by: Guangyu Xu --- .github/workflows/optimizer.yml | 64 +++++++++++- config/config_test.go | 4 + misc/optimizer/containerd-config.toml | 5 + misc/optimizer/nginx.yaml | 2 +- misc/snapshotter/config.toml | 4 +- misc/snapshotter/config_optimizer.toml | 139 +++++++++++++++++++++++++ 6 files changed, 211 insertions(+), 7 deletions(-) create mode 100644 misc/snapshotter/config_optimizer.toml diff --git a/.github/workflows/optimizer.yml b/.github/workflows/optimizer.yml index 4a90f13a83..a3c83b489a 100644 --- a/.github/workflows/optimizer.yml +++ b/.github/workflows/optimizer.yml @@ -16,6 +16,12 @@ on: env: CARGO_TERM_COLOR: always + REGISTRY: ghcr.io + SNAPSHOTTER_CONFIG: /etc/nydus/config.toml + SOURCE_SNAPSHOTTER_CONFIG: misc/snapshotter/config_optimizer.toml + NYDUSD_CONFIG: /etc/nydus/nydusd-config.fusedev.json + SOURCE_NYDUSD_CONFIG: misc/snapshotter/nydusd-config.fusedev.json + SNAPSHOTTER_SYSTEMD_UNIT_SERVICE: misc/snapshotter/nydus-snapshotter.fusedev.service jobs: run_optimizer: @@ -58,6 +64,34 @@ jobs: sudo mkdir -p /opt/cni/bin sudo tar xzf cni-plugins-linux-amd64-v1.2.0.tgz -C /opt/cni/bin/ sudo install -D -m 755 misc/example/10-containerd-net.conflist /etc/cni/net.d/10-containerd-net.conflist + - name: Setup nydus-snapshotter and nydus service + run: | + GOOS=linux GOARCH=amd64 go build -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc + sudo install -D -m 755 bin/containerd-nydus-grpc /usr/local/bin/containerd-nydus-grpc + if [ ! -e "${{ env.NYDUSD_CONFIG }}" ]; then + echo "Installing ${{ env.SOURCE_NYDUSD_CONFIG }} to ${{ env.NYDUSD_CONFIG }}" + sudo install -D -m 664 "${{ env.SOURCE_NYDUSD_CONFIG }}" "${{ env.NYDUSD_CONFIG }}" + fi + if [ ! -e "${{ env.SNAPSHOTTER_CONFIG }}" ]; then + echo "Installing ${{ env.SOURCE_SNAPSHOTTER_CONFIG }} to ${{ env.SNAPSHOTTER_CONFIG }}" + sudo install -D -m 664 "${{ env.SOURCE_SNAPSHOTTER_CONFIG }}" "${{ env.SNAPSHOTTER_CONFIG }}" + fi + sudo ln -f -s /etc/nydus/nydusd-config.fusedev.json /etc/nydus/nydusd-config.json + echo "Installing ${{ env.SNAPSHOTTER_SYSTEMD_UNIT_SERVICE }} to /etc/systemd/system/nydus-snapshotter.service" + sudo install -D -m 644 "${{ env.SNAPSHOTTER_SYSTEMD_UNIT_SERVICE }}" /etc/systemd/system/nydus-snapshotter.service + if command -v systemctl >/dev/null; then + sudo systemctl enable /etc/systemd/system/nydus-snapshotter.service + sudo systemctl restart nydus-snapshotter + fi + sleep 5 + NYDUS_VER=v$(curl -s "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//') + wget https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz + tar xzvf nydus-static-$NYDUS_VER-linux-amd64.tgz + # sudo systemctl restart nydus-snapshotter.service + sudo install -D -m 755 nydus-static/nydusd /usr/local/bin/nydusd + sudo install -D -m 755 nydus-static/nydus-image /usr/local/bin/nydus-image + sudo install -D -m 755 nydus-static/nydusctl /usr/local/bin/nydusctl + - name: Build and install optimizer run: | rustup component add rustfmt clippy @@ -84,26 +118,48 @@ jobs: echo "containerd is not ready" exit 1 fi + - name: Setup prefetch-distribution http server + run: | + go build -o prefetch-distribution tools/prefetch-distribution/main.go + nohup ./prefetch-distribution & + sleep 5 + - name: Log in to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Generate accessed files list run: | sed -i "s|host_path: script|host_path: $(pwd)/misc/optimizer/script|g" misc/optimizer/nginx.yaml sudo crictl run misc/optimizer/nginx.yaml misc/optimizer/sandbox.yaml sleep 20 sudo crictl rmp -f --all + sudo crictl rmi --all tree /opt/nri/optimizer/results/ - count=$(cat /opt/nri/optimizer/results/library/nginx:1.23.3 | wc -l) + count=$(cat /opt/nri/optimizer/results/dragonflyoss/image-service/nginx:nydus-latest | wc -l) expected=$(cat misc/optimizer/script/file_list.txt | wc -l) echo "count: $count expected minimum value: $expected" if [ $count -lt $expected ]; then - echo "failed to generate accessed files list for nginx:1.23.3" + echo "failed to generate accessed files list for nginx:nydus-latest" cat misc/optimizer/script/file_list.txt exit 1 fi - cat /opt/nri/optimizer/results/library/nginx:1.23.3.csv + cat /opt/nri/optimizer/results/dragonflyoss/image-service/nginx:nydus-latest.csv + - name: Transmit the prefetch list to nydusd + run: | + sed -i "s|host_path: script|host_path: $(pwd)/misc/optimizer/script|g" misc/optimizer/nginx.yaml + sudo crictl run misc/optimizer/nginx.yaml misc/optimizer/sandbox.yaml + if sudo ps aux | grep "[/]usr/local/bin/nydusd" | grep "prefetch-files"; then + echo "Found --prefetch-files in running processes" + else + echo "Error: --prefetch-files not found in running processes" + exit 1 + fi - name: Dump logs if: failure() continue-on-error: true run: | systemctl status containerd --no-pager -l journalctl -xeu containerd --no-pager - \ No newline at end of file diff --git a/config/config_test.go b/config/config_test.go index cd0c232387..fb20dfe3fb 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -31,6 +31,10 @@ func TestLoadSnapshotterTOMLConfig(t *testing.T) { EnableStargz: false, EnableReferrerDetect: false, }, + PrefetchControllerConfig: PrefetchControllerConfig{ + Enable: false, + PrefetchConfig: "http://localhost:1323/api/v1/prefetch", + }, CleanupOnClose: false, SystemControllerConfig: SystemControllerConfig{ Enable: true, diff --git a/misc/optimizer/containerd-config.toml b/misc/optimizer/containerd-config.toml index eda54c7cde..71bd0a1d29 100644 --- a/misc/optimizer/containerd-config.toml +++ b/misc/optimizer/containerd-config.toml @@ -14,6 +14,11 @@ version = 2 +[plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + snapshotter = "nydus" + disable_snapshot_annotations = false + [proxy_plugins] [proxy_plugins.nydus] type = "snapshot" diff --git a/misc/optimizer/nginx.yaml b/misc/optimizer/nginx.yaml index ee0bf5c995..eec62bdee0 100644 --- a/misc/optimizer/nginx.yaml +++ b/misc/optimizer/nginx.yaml @@ -2,7 +2,7 @@ metadata: name: nginx image: - image: nginx:1.23.3 + image: ghcr.io/dragonflyoss/image-service/nginx:nydus-latest mounts: - host_path: script diff --git a/misc/snapshotter/config.toml b/misc/snapshotter/config.toml index d5d625b358..b02881c00f 100644 --- a/misc/snapshotter/config.toml +++ b/misc/snapshotter/config.toml @@ -11,7 +11,7 @@ cleanup_on_close = false [prefetch] enable = false -get_prefetch_endpoint = "http://localhost:1323/api/v1/prefetch/download" +distribution_pull_endpoint = "http://localhost:1323/api/v1/prefetch" [system] # Snapshotter's debug and trace HTTP server interface @@ -136,4 +136,4 @@ max_concurrent_proc = 0 # - "image_block": generate a raw block disk image with tarfs for an image # - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info # - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info -export_mode = "" \ No newline at end of file +export_mode = "" diff --git a/misc/snapshotter/config_optimizer.toml b/misc/snapshotter/config_optimizer.toml new file mode 100644 index 0000000000..eb46d794be --- /dev/null +++ b/misc/snapshotter/config_optimizer.toml @@ -0,0 +1,139 @@ +version = 1 +# Snapshotter's own home directory where it stores and creates necessary resources +root = "/var/lib/containerd-nydus" +# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket +address = "/run/containerd-nydus/containerd-nydus-grpc.sock" +# The nydus daemon mode can be one of the following options: multiple, dedicated, shared, or none. +# If `daemon_mode` option is not specified, the default value is multiple. +daemon_mode = "dedicated" +# Whether snapshotter should try to clean up resources when it is closed +cleanup_on_close = false + +[prefetch] +enable = true +distribution_pull_endpoint = "http://localhost:1323/api/v1/prefetch" + +[system] +# Snapshotter's debug and trace HTTP server interface +enable = true +# Unix domain socket path where system controller is listening on +address = "/run/containerd-nydus/system.sock" + +[system.debug] +# Snapshotter can profile the CPU utilization of each nydusd daemon when it is being started. +# This option specifies the profile duration when nydusd is downloading and uncomproessing data. +daemon_cpu_profile_duration_secs = 5 +# Enable by assigning an address, empty indicates pprof server is disabled +pprof_address = "" + +[daemon] +# Specify a configuration file for nydusd +nydusd_config = "/etc/nydus/nydusd-config.fusedev.json" +nydusd_path = "/usr/local/bin/nydusd" +nydusimage_path = "/usr/local/bin/nydus-image" +# The fs driver can be one of the following options: fusedev, fscache, blockdev, proxy, or nodev. +# If `fs_driver` option is not specified, the default value is fusedev. +fs_driver = "fusedev" +# How to process when daemon dies: "none", "restart" or "failover" +recover_policy = "restart" +# Nydusd worker thread number to handle FUSE or fscache requests, [0-1024]. +# Setting to 0 will use the default configuration of nydusd. +threads_number = 4 +# Log rotation size for nydusd, in unit MB(megabytes). (default 100MB) +log_rotation_size = 100 + +[cgroup] +# Whether to use separate cgroup for nydusd. +enable = true +# The memory limit for nydusd cgroup, which contains all nydusd processes. +# Percentage is supported as well, please ensure it is end with "%". +# The default unit is bytes. Acceptable values include "209715200", "200MiB", "200Mi" and "10%". +memory_limit = "" + +[log] +# Print logs to stdout rather than logging files +log_to_stdout = false +# Snapshotter's log level +level = "info" +log_rotation_compress = true +log_rotation_local_time = true +# Max number of days to retain logs +log_rotation_max_age = 7 +log_rotation_max_backups = 5 +# In unit MB(megabytes) +log_rotation_max_size = 100 + +[metrics] +# Enable by assigning an address, empty indicates metrics server is disabled +address = ":9110" + +[remote] +convert_vpc_registry = false + +[remote.mirrors_config] +# Snapshotter will overwrite daemon's mirrors configuration +# if the values loaded from this driectory are not null before starting a daemon. +# Set to "" or an empty directory to disable it. +#dir = "/etc/nydus/certs.d" + +[remote.auth] +# Fetch the private registry auth by listening to K8s API server +enable_kubeconfig_keychain = false +# synchronize `kubernetes.io/dockerconfigjson` secret from kubernetes API server with specified kubeconfig (default `$KUBECONFIG` or `~/.kube/config`) +kubeconfig_path = "" +# Fetch the private registry auth as CRI image service proxy +enable_cri_keychain = false +# the target image service when using image proxy +#image_service_address = "/run/containerd/containerd.sock" + +[snapshot] +# Let containerd use nydus-overlayfs mount helper +enable_nydus_overlayfs = false +# Insert Kata Virtual Volume option to `Mount.Options` +enable_kata_volume = false +# Whether to remove resources when a snapshot is removed +sync_remove = false + +[cache_manager] +# Disable or enable recyclebin +disable = false +# How long to keep deleted files in recyclebin +gc_period = "24h" +# Directory to host cached files +cache_dir = "" + +[image] +public_key_file = "" +validate_signature = false + +# The configuraions for features that are not production ready +[experimental] +# Whether to enable stargz support +enable_stargz = false +# Whether to enable referrers support +# The option enables trying to fetch the Nydus image associated with the OCI image and run it. +# Also see https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers +enable_referrer_detect = false +# Whether to enable authentication support +# The option enables nydus snapshot to provide backend information to nydusd. +enable_backend_source = false +[experimental.tarfs] +# Whether to enable nydus tarfs mode. Tarfs is supported by: +# - The EROFS filesystem driver since Linux 6.4 +# - Nydus Image Service release v2.3 +enable_tarfs = false +# Mount rafs on host by loopdev and EROFS +mount_tarfs_on_host = false +# Only enable nydus tarfs mode for images with `tarfs hint` label when true +tarfs_hint = false +# Maximum of concurrence to converting OCIv1 images to tarfs, 0 means default +max_concurrent_proc = 0 +# Mode to export tarfs images: +# - "none" or "": do not export tarfs +# - "layer_verity_only": only generate disk verity information for a layer blob +# - "image_verity_only": only generate disk verity information for all blobs of an image +# - "layer_block": generate a raw block disk image with tarfs for a layer +# - "image_block": generate a raw block disk image with tarfs for an image +# - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info +# - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info +export_mode = ""