add a doc for prefetch-nri-plugin

1. specify the prefetch value with an URL 2. Modify the reading mode of configuration file 3. add prefetchlist path in config.toml and so on Signed-off-by: billie60 <[email protected]>
containerd · Nov 14, 2023 · 4a7aac4 · 4a7aac4
1 parent 23c9caa
commit 4a7aac4
Show file tree

Hide file tree

Showing 12 changed files with 291 additions and 56 deletions.
diff --git a/cmd/prefetchfiles-nri-plugin/main.go b/cmd/prefetchfiles-nri-plugin/main.go
@@ -14,7 +14,6 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/containerd/nri/pkg/api"
@@ -32,14 +31,17 @@ const (
 	endpointPrefetch               = "/api/v1/prefetch"
 	defaultEvents                  = "RunPodSandbox"
 	defaultSystemControllerAddress = "/run/containerd-nydus/system.sock"
-	defaultPrefetchConfigDir       = "/etc/nydus"
 	nydusPrefetchAnnotation        = "containerd.io/nydus-prefetch"
 )
 
+type PluginConfig struct {
+	SocketAddr string `toml:"socket_address"`
+}
+
 type PluginArgs struct {
-	PluginName    string
-	PluginIdx     string
-	SocketAddress string
+	PluginName string
+	PluginIdx  string
+	Config     PluginConfig
 }
 
 type Flags struct {
@@ -62,8 +64,8 @@ func buildFlags(args *PluginArgs) []cli.Flag {
 		&cli.StringFlag{
 			Name:        "socket-addr",
 			Value:       defaultSystemControllerAddress,
-			Usage:       "unix domain socket address. If defined in the configuration file, there is no need to add ",
-			Destination: &args.SocketAddress,
+			Usage:       "UNIX domain socket address for connection to the nydus-snapshotter API.",
+			Destination: &args.Config.SocketAddr,
 		},
 	}
 }
@@ -129,6 +131,30 @@ func (p *plugin) RunPodSandbox(pod *api.PodSandbox) error {
 
 	return nil
 }
+func (p *plugin) Configure(config, runtime, version string) (stub.EventMask, error) {
+	var cfg PluginConfig
+	log.Infof("got configuration data: %q from runtime %s %s", config, runtime, version)
+	if config == "" {
+		return p.mask, nil
+	}
+
+	tree, err := toml.Load(config)
+	if err != nil {
+		return 0, err
+	}
+	if err := tree.Unmarshal(&cfg); err != nil {
+		return 0, err
+	}
+	p.mask, err = api.ParseEventMask(defaultEvents)
+	if err != nil {
+		return 0, errors.Wrap(err, "parse events in configuration")
+	}
+
+	log.Infof("configuration: %#v", cfg)
+	globalSocket = cfg.SocketAddr
+
+	return p.mask, nil
+}
 
 func main() {
 
@@ -148,25 +174,7 @@ func main() {
 
 			log = logrus.StandardLogger()
 
-			configFileName := "prefetchConfig.toml"
-			configDir := defaultPrefetchConfigDir
-			configFilePath := filepath.Join(configDir, configFileName)
-
-			config, err := toml.LoadFile(configFilePath)
-			if err != nil {
-				log.Warnf("failed to read config file: %v", err)
-			}
-
-			configSocketAddrRaw := config.Get("file_prefetch.socket_address")
-			if configSocketAddrRaw != nil {
-				if configSocketAddr, ok := configSocketAddrRaw.(string); ok {
-					globalSocket = configSocketAddr
-				} else {
-					log.Warnf("failed to read config: 'file_prefetch.socket_address' is not a string")
-				}
-			} else {
-				globalSocket = flags.Args.SocketAddress
-			}
+			globalSocket = flags.Args.Config.SocketAddr
 
 			log.SetFormatter(&logrus.TextFormatter{
 				PadLevelText: true,

diff --git a/config/global.go b/config/global.go
@@ -38,6 +38,8 @@ type GlobalConfig struct {
 	DaemonThreadsNum int
 	CacheGCPeriod    time.Duration
 	MirrorsConfig    MirrorsConfig
+
+	PrefetchRoot string
 }
 
 func IsFusedevSharedModeEnabled() bool {
@@ -64,6 +66,10 @@ func GetConfigRoot() string {
 	return globalConfig.ConfigRoot
 }
 
+func GetPrefetchRoot() string {
+	return globalConfig.PrefetchRoot
+}
+
 func GetMirrorsConfigDir() string {
 	return globalConfig.MirrorsConfig.Dir
 }
@@ -182,6 +188,8 @@ func ProcessConfigurations(c *SnapshotterConfig) error {
 	globalConfig.SocketRoot = filepath.Join(c.Root, "socket")
 	globalConfig.RootMountpoint = filepath.Join(c.Root, "mnt")
 
+	globalConfig.PrefetchRoot = filepath.Join(c.Root, "prefetch")
+
 	globalConfig.MirrorsConfig = c.RemoteConfig.MirrorsConfig
 
 	if c.CacheManagerConfig.GCPeriod != "" {

diff --git a/docs/prefetchfiles-nri-plugin.md b/docs/prefetchfiles-nri-plugin.md
@@ -0,0 +1,133 @@
+# User self-defined nydus image files prefetch
+In the on-demand loading mechanism, the dynamic loading of files or dependencies within the 
+container during runtime may generate considerable network IO, affecting performance. 
+The `--prefetch-patterns` parameter provided by `nydusify` enables you to specify a prefetch list 
+when converting an image to the Nydus format. This list is utilized during container startup to fetch 
+files according to the specified patterns. However, this method may lack flexibility as different 
+services using the same image might require access to distinct files.
+
+To improve the flexibility of nydus image files prefetch, for the k8s scenario, we can specify a prefetch files list when create a nydus daemon. The prefetch files list is user self-defined. Nydus-snapshotter has implemented a containerd NRI plugin to transmit the path of prefetch files list to nydus-snapshotter. The prefetch plugin requires NRI 2.0, which is available in containerd (>=v1.7.0). The prefetch plugin subscribes pod creation event, obtains the URL address containing the content of the files need to be prefetched, and forwards it to `nydus-snapshotter`. The `nydus-snapshotter` reads the data through the URL and stores it locally. Then when `nydusd` starts, it will pull the files defined in the prefetch files list through lazy loading. This allows the pull of the prefetch files to be done during container creation rather than image convert, improving the flexibility of file prefetching.
+
+## Requirements
+
+- [NRI 2.0](https://github.com/containerd/nri): Which has been integrated into containerd since [v1.7.0](https://github.com/containerd/containerd/tree/v1.7.0).
+
+## Workflow
+
+1. Add information such as image reference and URL address containing prefetch files to annotations in pod configuration file.
+2. Run the prefetch plugin to monitor RunPodSandbox events.
+3. The prefetch plugin fetches image reference and URL and forwards them to nydus-snapshotter.
+4. Nydus-snapshotter specifies the prefetch list when starting nydus daemon.
+5. Nydusd completes the mounting of the nydus image.
+
+## Modify configuration file 
+
+Modify containerd's toml configuration file to enable NRI.
+```console
+sudo tee -a /etc/containerd/config.toml <<- EOF
+[plugins."io.containerd.nri.v1.nri"]
+    config_file = "/etc/nri/nri.conf"
+    disable = false
+    plugin_path = "/opt/nri/plugins"
+    socket_path = "/var/run/nri.sock"
+EOF
+```
+Containerd will load all NRI plugins in the `plugin_path` directory on startup. If you want to start an NRI plugin manually, please add the following configuration to allow other NRI plugins to connect via `socket_path`.
+
+```console
+sudo tee /etc/nri/nri.conf <<- EOF
+disableConnections: false
+EOF
+```
+
+
+If you want to start the plugin using `pre-connection` mode. You need to write a configuration file and place the plugin's binary file and configuration file in the correct directories. Here is an example of configuration file:
+```console
+sudo tee prefetchfiles-nri-plugin.conf <<- EOF
+# UNIX domain socket address for connection to the nydus-snapshotter API
+socket_address = "/run/containerd-nydus/system.sock"
+EOF
+```
+Then move the files to the correct directories and set permissions. 
+```console
+go build 
+sudo install -D -m 755 ./prefetchfiles-nri-plugin /opt/nri/plugins/03-prefetchfiles-nri-plugin
+sudo install -D -m 755 ./prefetchfiles-nri-plugin.conf /etc/nri/conf.d/03-prefetchfiles-nri-plugin.conf
+```
+
+Restart the containerd service.
+
+```console
+sudo systemctl restart containerd
+```
+When manually starting the prefetch NRI plugin, the socket address can be modified through the command line parameter `socket-addr`. 
+
+
+
+After start the prefetch plugin, it will monitor pod creation events. Note that NRI plugin can only be called from containerd/CRI. So creation a pod using crictl as below.
+```console
+sudo tee pod.yaml <<- EOF
+kind: pod
+metadata:
+  name: wordpress-sandbox
+  namespace: default
+  attempt: 1
+  uid: hdishd83djaidwnduwk28bcsb
+log_directory: /tmp
+annotations:
+ containerd.io/nydus-prefetch: |
+    [
+        {
+            "image": "ghcr.io/dragonflyoss/image-service/wordpress:nydus-nightly-v6", 
+            "prefetch": "http://example.com/api/v1/resource/wordpress"
+        }
+    ]
+
+linux: {}
+
+EOF
+
+crictl runp pod.yaml
+```
+The list of files to be prefetched is written in a URL, and `nydus-snapshotter` will read the prefetch list based on the URL address and transfer it to a local file. The specific content of the prefetch files can be customized by the user.
+`http://example.com/api/v1/resource/wordpress` is just an example of URL address, which needs to be replaced with a real URL during actual operation. The following is an example of some prefetch files in URL:
+```console
+/usr/bin/env
+/lib/x86_64-linux-gnu/ld-2.31.so
+/etc/ld.so.cache
+/lib/x86_64-linux-gnu/libc-2.31.so
+/bin/bash
+/lib/x86_64-linux-gnu/libtinfo.so.6.2
+/lib/x86_64-linux-gnu/libdl-2.31.so
+/etc/nsswitch.conf
+```
+
+
+
+
+
+Note that the naming of keys in annotations is fixed, and the values in annotations are user self-defined.
+After creating a pod, `nydus-snapshotter` will store the image references and paths of the prefetch list.
+
+## Nydus-snapshotter starts nydusd
+Nydusd daemon will start when the container is created. We start a container like below.
+```console
+sudo tee wordpress.yaml <<- EOF
+metadata:
+  name: wordpress
+image:
+  image: ghcr.io/dragonflyoss/image-service/wordpress:nydus-nightly-v6
+log_path: wordpress.0.log
+linux: {}
+EOF
+
+crictl pull ghcr.io/dragonflyoss/image-service/wordpress:nydus-nightly-v6
+crictl create <pod_id> wordpress.yaml pod.yaml
+```
+
+Then nydus-snapshotter will start a daemon by command as follows.
+```editorconfig
+nydusd command: /usr/local/bin/nydusd fuse --thread-num 4 --config /var/lib/containerd-nydus/config/cl5kahug3d3s3k10qa0g/config.json --bootstrap /var/lib/containerd-nydus/snapshots/23/fs/image/image.boot --mountpoint /var/lib/containerd-nydus/snapshots/23/mnt --apisock /var/lib/containerd-nydus/socket/cl5kahug3d3s3k10qa0g/api.sock --log-level info --log-rotation-size 100 --prefetch-files /opt/prefetch/ghcr.io/dragonflyoss/image-service/wordpress:nydus-nightly-v6
+ ```
+
+According to the parameter `--prefetch-files`, we have implemented file prefetching through lazy loading. 
diff --git a/misc/example/prefetchfiles-nri-plugin.conf b/misc/example/prefetchfiles-nri-plugin.conf
@@ -0,0 +1,3 @@
+# UNIX domain socket address for connection to the nydus-snapshotter API
+socket_address = "/run/containerd-nydus/system.sock"
+
diff --git a/misc/nri-prefetch/prefetchConfig.toml b/misc/nri-prefetch/prefetchConfig.toml
diff --git a/pkg/daemon/config.go b/pkg/daemon/config.go
@@ -16,6 +16,7 @@ import (
 
 	"github.com/containerd/nydus-snapshotter/config"
 	"github.com/containerd/nydus-snapshotter/internal/constant"
+	"github.com/containerd/nydus-snapshotter/pkg/prefetch"
 )
 
 // Build runtime nydusd daemon object, which might be persisted later
@@ -31,6 +32,20 @@ func WithSocketDir(dir string) NewDaemonOpt {
 	}
 }
 
+func WithPrefetchDir(dir, imageID string) NewDaemonOpt {
+	return func(d *Daemon) error {
+		s := filepath.Join(dir, d.ID())
+		prefetchDir, err := prefetch.Pm.GetPrefetchListPath(s, imageID)
+		if err != nil {
+			return errors.Wrapf(err, "failed to get prefetchList for image %s in path %s", imageID, s)
+		}
+		if prefetchDir != "" {
+			d.States.PrefetchDir = prefetchDir
+		}
+		return nil
+	}
+}
+
 func WithRef(ref int32) NewDaemonOpt {
 	return func(d *Daemon) error {
 		d.ref = ref

diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go
@@ -55,6 +55,8 @@ type ConfigState struct {
 	ThreadNum       int
 	// Where the configuration file resides, all rafs instances share the same configuration template
 	ConfigDir string
+
+	PrefetchDir string
 }
 
 // TODO: Record queried nydusd state

diff --git a/pkg/filesystem/fs.go b/pkg/filesystem/fs.go
@@ -15,13 +15,13 @@ import (
 	"os"
 	"path"
 
-	snpkg "github.com/containerd/containerd/pkg/snapshotters"
 	"github.com/mohae/deepcopy"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/containerd/containerd/log"
+	snpkg "github.com/containerd/containerd/pkg/snapshotters"
 	"github.com/containerd/containerd/snapshots"
 	"github.com/containerd/containerd/snapshots/storage"
 
@@ -285,7 +285,7 @@ func (fs *Filesystem) Mount(ctx context.Context, snapshotID string, labels map[s
 			if err != nil {
 				return err
 			}
-			d, err = fs.createDaemon(fsManager, config.DaemonModeDedicated, mp, 0)
+			d, err = fs.createDaemon(fsManager, config.DaemonModeDedicated, mp, 0, imageID)
 			// if daemon already exists for snapshotID, just return
 			if err != nil && !errdefs.IsAlreadyExists(err) {
 				return err
@@ -578,7 +578,7 @@ func (fs *Filesystem) initSharedDaemon(fsManager *manager.Manager) (err error) {
 		return errors.Errorf("got null mountpoint for fsDriver %s", fsManager.FsDriver)
 	}
 
-	d, err := fs.createDaemon(fsManager, daemonMode, mp, 0)
+	d, err := fs.createDaemon(fsManager, daemonMode, mp, 0, "")
 	if err != nil {
 		return errors.Wrap(err, "initialize shared daemon")
 	}
@@ -612,7 +612,7 @@ func (fs *Filesystem) initSharedDaemon(fsManager *manager.Manager) (err error) {
 
 // createDaemon create new nydus daemon by snapshotID and imageID
 func (fs *Filesystem) createDaemon(fsManager *manager.Manager, daemonMode config.DaemonMode,
-	mountpoint string, ref int32) (d *daemon.Daemon, err error) {
+	mountpoint string, ref int32, imageID string) (d *daemon.Daemon, err error) {
 	opts := []daemon.NewDaemonOpt{
 		daemon.WithRef(ref),
 		daemon.WithSocketDir(config.GetSocketRoot()),
@@ -626,6 +626,10 @@ func (fs *Filesystem) createDaemon(fsManager *manager.Manager, daemonMode config
 		daemon.WithDaemonMode(daemonMode),
 	}
 
+	if imageID != "" {
+		opts = append(opts, daemon.WithPrefetchDir(config.GetPrefetchRoot(), imageID))
+	}
+
 	// For fscache driver, no need to provide mountpoint to nydusd daemon.
 	if mountpoint != "" {
 		opts = append(opts, daemon.WithMountpoint(mountpoint))