diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8adcc8db95..ae4ff194cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,8 @@ jobs: - name: Build run: | # Download nydus components - NYDUS_VER=v$(curl --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -s "https://api.github.com/repos/dragonflyoss/image-service/releases/latest" | jq -r .tag_name | sed 's/^v//') + # NYDUS_VER=v$(curl --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -s "https://api.github.com/repos/dragonflyoss/image-service/releases/latest" | jq -r .tag_name | sed 's/^v//') + NYDUS_VER=v$(curl --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -s "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//') wget https://github.com/dragonflyoss/image-service/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz tar xzvf nydus-static-$NYDUS_VER-linux-amd64.tgz mkdir -p /usr/bin diff --git a/.github/workflows/k8s-e2e.yml b/.github/workflows/k8s-e2e.yml index 542a372a32..7d72b134c7 100644 --- a/.github/workflows/k8s-e2e.yml +++ b/.github/workflows/k8s-e2e.yml @@ -35,7 +35,7 @@ jobs: cp bin/containerd-nydus-grpc ./ cp misc/snapshotter/* ./ ls -tl ./ - NYDUS_VER=v$(curl --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -s "https://api.github.com/repos/dragonflyoss/image-service/releases/latest" | jq -r .tag_name | sed 's/^v//') + NYDUS_VER=v$(curl --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -s "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//') docker build --build-arg NYDUS_VER=${NYDUS_VER} -t local-dev:e2e . ## load local test image into kind node @@ -97,7 +97,7 @@ jobs: docker exec kind-control-plane systemctl restart containerd - name: Install Nydus binaries and convert nydus image run: | - NYDUS_VER=v$(curl --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -s "https://api.github.com/repos/dragonflyoss/image-service/releases/latest" | jq -r .tag_name | sed 's/^v//') + NYDUS_VER=v$(curl --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -s "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//') wget -q https://github.com/dragonflyoss/image-service/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz tar xzf nydus-static-$NYDUS_VER-linux-amd64.tgz sudo cp nydus-static/nydusify nydus-static/nydus-image /usr/local/bin/ diff --git a/cmd/prefetchfiles-nri-plugin/main.go b/cmd/prefetchfiles-nri-plugin/main.go index 2cfdc73712..b6fe75f528 100644 --- a/cmd/prefetchfiles-nri-plugin/main.go +++ b/cmd/prefetchfiles-nri-plugin/main.go @@ -14,7 +14,6 @@ import ( "net" "net/http" "os" - "path/filepath" "strings" "github.com/containerd/nri/pkg/api" @@ -32,14 +31,17 @@ const ( endpointPrefetch = "/api/v1/prefetch" defaultEvents = "RunPodSandbox" defaultSystemControllerAddress = "/run/containerd-nydus/system.sock" - defaultPrefetchConfigDir = "/etc/nydus" nydusPrefetchAnnotation = "containerd.io/nydus-prefetch" ) +type PluginConfig struct { + SocketAddr string `toml:"socket_address"` +} + type PluginArgs struct { - PluginName string - PluginIdx string - SocketAddress string + PluginName string + PluginIdx string + Config PluginConfig } type Flags struct { @@ -62,8 +64,8 @@ func buildFlags(args *PluginArgs) []cli.Flag { &cli.StringFlag{ Name: "socket-addr", Value: defaultSystemControllerAddress, - Usage: "unix domain socket address. If defined in the configuration file, there is no need to add ", - Destination: &args.SocketAddress, + Usage: "unix domain socket address.", + Destination: &args.Config.SocketAddr, }, } } @@ -129,6 +131,30 @@ func (p *plugin) RunPodSandbox(pod *api.PodSandbox) error { return nil } +func (p *plugin) Configure(config, runtime, version string) (stub.EventMask, error) { + var cfg PluginConfig + log.Infof("got configuration data: %q from runtime %s %s", config, runtime, version) + if config == "" { + return p.mask, nil + } + + tree, err := toml.Load(config) + if err != nil { + return 0, err + } + if err := tree.Unmarshal(&cfg); err != nil { + return 0, err + } + p.mask, err = api.ParseEventMask(defaultEvents) + if err != nil { + return 0, errors.Wrap(err, "parse events in configuration") + } + + log.Infof("configuration: %#v", cfg) + globalSocket = cfg.SocketAddr + + return p.mask, nil +} func main() { @@ -148,24 +174,10 @@ func main() { log = logrus.StandardLogger() - configFileName := "prefetchConfig.toml" - configDir := defaultPrefetchConfigDir - configFilePath := filepath.Join(configDir, configFileName) - - config, err := toml.LoadFile(configFilePath) - if err != nil { - log.Warnf("failed to read config file: %v", err) - } - - configSocketAddrRaw := config.Get("file_prefetch.socket_address") - if configSocketAddrRaw != nil { - if configSocketAddr, ok := configSocketAddrRaw.(string); ok { - globalSocket = configSocketAddr - } else { - log.Warnf("failed to read config: 'file_prefetch.socket_address' is not a string") - } + if flags.Args.Config.SocketAddr != "" { + globalSocket = flags.Args.Config.SocketAddr } else { - globalSocket = flags.Args.SocketAddress + globalSocket = defaultSystemControllerAddress } log.SetFormatter(&logrus.TextFormatter{ diff --git a/docs/prefetchfiles-nri-plugin.md b/docs/prefetchfiles-nri-plugin.md new file mode 100644 index 0000000000..f4e768bb16 --- /dev/null +++ b/docs/prefetchfiles-nri-plugin.md @@ -0,0 +1,160 @@ +# User self-defined nydus image files prefetch +To improve the flexibility of nydus image files prefetch, for the k8s scenario, we can specify a prefetch files list when create a nydus daemon. The prefetch files list is user self-defined. Nydus-snapshotter has implemented a containerd NRI plugin to transmit the path of prefetch files list to nydus-snapshotter. The prefetch plugin requires NRI 2.0, which is available in containerd (>=v1.7.0). The prefetch plugin subscribes pod creation event, obtains the URL address containing the content of the files need to be prefetched, and forwards it to `nydus-snapshotter`. The `nydus-snapshotter` reads the data through the URL and stores it locally. Then when `nydusd` starts, it will pull the files defined in the prefetch files list through lazy loading. This allows the pull of the prefetch files to be done during container creation rather than image convert, improving the flexibility of file prefetching. + +## Requirements + +- [NRI 2.0](https://github.com/containerd/nri): Which has been integrated into containerd since [v1.7.0](https://github.com/containerd/containerd/tree/v1.7.0-beta.1). + +## Workflow + +1. Add information such as image reference and URL address containing prefetch files to annotations in pod configuration file. +2. Run the prefetch plugin to monitor RunPodSandbox events. +3. The prefetch plugin fetches image reference and URL and forwards them to nydus-snapshotter. +4. Nydus-snapshotter specifies the prefetch list when starting nydus daemon. +5. Nydusd completes the mounting of the nydus image. + + + + + +## Modify configuration file + +Modify containerd's toml configuration file to enable NRI. +```console +sudo tee -a /etc/containerd/config.toml <<- EOF +[plugins."io.containerd.nri.v1.nri"] + config_file = "/etc/nri/nri.conf" + disable = false + plugin_path = "/opt/nri/plugins" + socket_path = "/var/run/nri.sock" +EOF +``` +Containerd will load all NRI plugins in the `plugin_path` directory on startup. If you want to start an NRI plugin manually, please add the following configuration to allow other NRI plugins to connect via `socket_path`. + +```console +sudo tee /etc/nri/nri.conf <<- EOF +disableConnections: false +EOF +``` + +Restart the containerd service. + +```console +sudo systemctl restart containerd +``` +If you want to start the plugin using `pre-connection` mode, that is, `disableConnections: true`. You need to write a configuration file and place the plugin's binary file and configuration file in the correct directories. Here is an example of configuration file: +```console +sudo tee prefetchConfig.conf <<- EOF +# unix domain socket address +socket_address = "/run/containerd-nydus/system.sock" +EOF +``` +Then move the files to the correct directories and set permissions. +```console +go build +sudo install -D -m 755 ./prefetchfiles-nri-plugin /opt/nri/plugins/03-prefetchfiles-nri-plugin +sudo install -D -m 755 ./prefetchfiles-nri-plugin.conf /etc/nri/conf.d/03-prefetchfiles-nri-plugin.conf +``` +When manually starting the prefetch NRI plugin, the socket address can be modified through the command line parameter `socket-addr`. + + + +After start the prefetch plugin, it will monitor pod creation events. Note that NRI plugin can only be called from containerd/CRI. So creation a pod using crictl as below. +```console +sudo tee pod.yaml <<- EOF +kind: pod +metadata: + name: wordpress-sandbox + namespace: default + attempt: 1 + uid: hdishd83djaidwnduwk28bcsb +log_directory: /tmp +annotations: + containerd.io/nydus-prefetch: | + [ + {"image": "dockerhub.kubekey.local/dfns/wordpress:nydus_latest", "prefetch": "http://example.com/api/v1/resource/wordpress"} + ] + +linux: {} + +EOF + +crictl runp pod.yaml +``` +The files that require prefetching are written in a URL, and `nydus-snapshoter` will read the prefetch list based on the URL address and transfer it to a local text file. The specific content of the prefetch files can be customized by the user. +`http://example.com/api/v1/resource/wordpress` is just an example of URL address, which needs to be replaced with a real URL during actual operation. The following is an example of some prefetch files in URL: +```console +/usr/bin/env +/lib/x86_64-linux-gnu/ld-2.31.so +/etc/ld.so.cache +/lib/x86_64-linux-gnu/libc-2.31.so +/bin/bash +/lib/x86_64-linux-gnu/libtinfo.so.6.2 +/lib/x86_64-linux-gnu/libdl-2.31.so +/etc/nsswitch.conf +``` + +The prefetching NRI plugin also supports the case where the pod configuration file contains multiple containers. In this case, the definition of pod-mutlicontainers.yaml is as follows: + + +```console +sudo tee pod-mutlicontainers.yaml <<- EOF +apiVersion: v1 +kind: Pod +metadata: + name: multi-containers-pod + namespace: default + attempt: 1 + uid: hdishd83djaidwnduwk28bcsb +log_directory: /tmp +annotations: + containerd.io/nydus-prefetch: | + [ + {"image": "dockerhub.kubekey.local/dfns/busybox:nydus_latest", "prefetch": "http://example.com/api/v1/resource/busybox"}, + {"image": "dockerhub.kubekey.local/dfns/ubuntu:nydus_latest", "prefetch": "http://example.com/api/v1/resource/ubuntu"}, + {"image": "dockerhub.kubekey.local/dfns/wordpress:nydus_latest", "prefetch": "http://example.com/api/v1/resource/wordpress"} + ] + +linux: {} + +spec: + containers: + - name: container-1 + image: dockerhub.kubekey.local/dfns/busybox:nydus_latest + command: ["ls"] + - name: container-2 + image: dockerhub.kubekey.local/dfns/ubuntu:nydus_latest + command: ["ls"] + - name: container-3 + image: dockerhub.kubekey.local/dfns/wordpress:nydus_latest + command: ["ls"] +EOF +``` + + + +Note that the naming of keys in annotations is fixed, and the values in annotations are user self-defined. +After creating a pod, `nydus-snapshotter` will store the image references and paths of the prefetch list. + +## Nydus-snapshotter starts nydusd +Nydusd daemon will start when the container is created. We start a container like below. +```console +sudo tee wordpress.yaml <<- EOF +metadata: + name: wordpress +image: + image: dockerhub.kubekey.local/dfns/wordpress:nydus_latest +log_path: wordpress.0.log +linux: {} +EOF + +crictl pull dockerhub.kubekey.local/dfns/wordpress:nydus_latest +crictl create wordpress.yaml pod.yaml +``` + +Then nydus-snapshotter will start a daemon by command as follows. +```editorconfig + /usr/local/bin/nydusd fuse --thread-num 4 --config /var/lib/containerd-nydus/config/cjmnum3c3al8js5p3740/config.json --bootstrap /var/lib/containerd-nydus/snapshots/22/fs/image/image.boot --mountpoint /var/lib/containerd-nydus/snapshots/22/mnt --apisock /var/lib/containerd-nydus/socket/cjmnum3c3al8js5p3740/api.sock --log-level info --log-rotation-size 100 --prefetch-files /etc/nydus/PrefetchFiles/dockerhub.kubekey.local@dfns@wordpress:nydus_latest + ``` + +According to the parameter `--prefetch-files`, we have implemented file prefetching through lazy loading. diff --git a/go.mod b/go.mod index 799362bcf9..dc2f3fa408 100644 --- a/go.mod +++ b/go.mod @@ -47,18 +47,20 @@ require ( k8s.io/apimachinery v0.26.2 k8s.io/client-go v0.26.2 k8s.io/cri-api v0.27.0-alpha.3 -) -require github.com/freddierice/go-losetup v0.0.0-20220711213114-2a14873012db // indirect +) require ( github.com/cilium/ebpf v0.9.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/frankban/quicktest v1.14.4 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect ) +require github.com/freddierice/go-losetup v0.0.0-20220711213114-2a14873012db + +require github.com/frankban/quicktest v1.14.4 // indirect + require ( github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652 // indirect github.com/Microsoft/go-winio v0.6.0 // indirect @@ -155,7 +157,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.90.1 // indirect k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect - k8s.io/utils v0.0.0-20230308161112-d77c459e9343 // indirect + k8s.io/utils v0.0.0-20230308161112-d77c459e9343 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/go.sum b/go.sum index 37f0be0f7d..2644d89039 100644 --- a/go.sum +++ b/go.sum @@ -132,8 +132,6 @@ github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/freddierice/go-losetup v0.0.0-20220711213114-2a14873012db h1:StM6A9LvaVrFS2chAGcfRVDoBB6rHYPIGJ3GknpB25c= github.com/freddierice/go-losetup v0.0.0-20220711213114-2a14873012db/go.mod h1:pwuQfHWn6j2Fpl2AWw/bPLlKfojHxIIEa5TeKIgDFW4= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= diff --git a/misc/example/prefetchfiles-nri-plugin.conf b/misc/example/prefetchfiles-nri-plugin.conf new file mode 100644 index 0000000000..1b4cbc125f --- /dev/null +++ b/misc/example/prefetchfiles-nri-plugin.conf @@ -0,0 +1,3 @@ +# unix domain socket address +socket_address = "/run/containerd-nydus/system.sock" + diff --git a/misc/nri-prefetch/prefetchConfig.toml b/misc/nri-prefetch/prefetchConfig.toml deleted file mode 100644 index dcd09974d8..0000000000 --- a/misc/nri-prefetch/prefetchConfig.toml +++ /dev/null @@ -1,3 +0,0 @@ -[file_prefetch] -# This is used to configure the socket address for the file prefetch. -socket_address = "/run/containerd-nydus/system.sock" \ No newline at end of file diff --git a/pkg/filesystem/fs.go b/pkg/filesystem/fs.go index 4a95c4a3c6..8128265292 100644 --- a/pkg/filesystem/fs.go +++ b/pkg/filesystem/fs.go @@ -15,13 +15,13 @@ import ( "os" "path" - snpkg "github.com/containerd/containerd/pkg/snapshotters" "github.com/mohae/deepcopy" "github.com/opencontainers/go-digest" "github.com/pkg/errors" "golang.org/x/sync/errgroup" "github.com/containerd/containerd/log" + snpkg "github.com/containerd/containerd/pkg/snapshotters" "github.com/containerd/containerd/snapshots" "github.com/containerd/containerd/snapshots/storage" diff --git a/pkg/prefetch/prefetch.go b/pkg/prefetch/prefetch.go index 6f2d01cd3d..511f8e5b05 100644 --- a/pkg/prefetch/prefetch.go +++ b/pkg/prefetch/prefetch.go @@ -7,12 +7,22 @@ package prefetch import ( + "bytes" "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "strings" "sync" "github.com/containerd/containerd/log" + "golang.org/x/net/html" ) +const prefetchlistPath string = "/opt/PrefetchFiles" + type prefetchInfo struct { prefetchMap map[string]string prefetchMutex sync.Mutex @@ -34,7 +44,11 @@ func (p *prefetchInfo) SetPrefetchFiles(body []byte) error { } for _, item := range prefetchMsg { image := item["image"] - prefetchfiles := item["prefetch"] + url := item["prefetch"] + prefetchfiles, err := p.getprefetchConfigFile(url, image) + if err != nil { + return err + } p.prefetchMap[image] = prefetchfiles } @@ -42,6 +56,76 @@ func (p *prefetchInfo) SetPrefetchFiles(body []byte) error { return nil } +// Obtain the prefetch files through the URL and transfer them to a local txt file. +func (p *prefetchInfo) getprefetchConfigFile(url, image string) (string, error) { + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("HTTP response status code is not 200 OK: %s", resp.Status) + return "", errors.New(errMsg) + } + dirPath := prefetchlistPath + if _, err = os.Stat(dirPath); os.IsNotExist(err) { + err = os.MkdirAll(dirPath, 0755) + if err != nil { + return "", err + } + } + replaced := strings.ReplaceAll(image, "/", "@") + filePath := dirPath + "/" + replaced + file, err := os.Create(filePath) + if err != nil { + return "", err + } + + defer func() { + if err = file.Close(); err != nil { + log.L.Fatalf("Error while closing the file: %v", err) + } + }() + var htmlBuffer bytes.Buffer + + tokenizer := html.NewTokenizer(resp.Body) + inBody := false + + for { + tokenType := tokenizer.Next() + switch tokenType { + case html.ErrorToken: + _, err = io.Copy(file, &htmlBuffer) + if err != nil { + return "", err + } + return filePath, nil + case html.StartTagToken, html.SelfClosingTagToken: + token := tokenizer.Token() + if token.Data == "body" { + inBody = true + } + case html.TextToken: + if inBody { + text := strings.TrimSpace(tokenizer.Token().Data) + if text != "" { + htmlBuffer.WriteString(text + "\n") + } + } + case html.EndTagToken: + token := tokenizer.Token() + if token.Data == "body" { + inBody = false + } + case html.CommentToken: + // Placeholder for empty implementation, no actual action is taken here + case html.DoctypeToken: + // Placeholder for empty implementation, no actual action is taken here + } + } +} + func (p *prefetchInfo) GetPrefetchInfo(image string) string { p.prefetchMutex.Lock() defer p.prefetchMutex.Unlock()