From 5e6e62675f3c1b461ad6264f5070169e81d4c293 Mon Sep 17 00:00:00 2001 From: Guangyu Xu Date: Thu, 18 Apr 2024 18:09:10 +0000 Subject: [PATCH] prefetch separate fanotify nydus-snapshotter side --- config/config.go | 7 ++++ config/global.go | 8 +++++ misc/snapshotter/config.toml | 4 +++ pkg/daemon/config.go | 2 +- pkg/prefetch/prefetch.go | 70 ++++++++++++++---------------------- 5 files changed, 47 insertions(+), 44 deletions(-) diff --git a/config/config.go b/config/config.go index 143b77f574..6b7f5f5731 100644 --- a/config/config.go +++ b/config/config.go @@ -209,6 +209,11 @@ type SystemControllerConfig struct { DebugConfig DebugConfig `toml:"debug"` } +type PrefetchControllerConfig struct { + Enable bool `toml:"enable"` + PrefetchConfig string `toml:"get_prefetch_endpoint"` +} + type SnapshotterConfig struct { // Configuration format version Version int `toml:"version"` @@ -229,6 +234,8 @@ type SnapshotterConfig struct { LoggingConfig LoggingConfig `toml:"log"` CgroupConfig CgroupConfig `toml:"cgroup"` Experimental Experimental `toml:"experimental"` + // Get prefetch list from http server + PrefetchControllerConfig PrefetchControllerConfig `toml:"prefetch"` } func LoadSnapshotterConfig(path string) (*SnapshotterConfig, error) { diff --git a/config/global.go b/config/global.go index 77d5c74e07..9a9ee2ac1d 100644 --- a/config/global.go +++ b/config/global.go @@ -69,6 +69,14 @@ func GetPrefetchRoot() string { return globalConfig.PrefetchRoot } +func GetPrefetchEndpoint() string { + return globalConfig.origin.PrefetchControllerConfig.PrefetchConfig +} + +func IsPrefetchEnabled() bool { + return globalConfig.origin.PrefetchControllerConfig.Enable +} + func GetMirrorsConfigDir() string { return globalConfig.MirrorsConfig.Dir } diff --git a/misc/snapshotter/config.toml b/misc/snapshotter/config.toml index 29a2aafe8b..d5d625b358 100644 --- a/misc/snapshotter/config.toml +++ b/misc/snapshotter/config.toml @@ -9,6 +9,10 @@ daemon_mode = "dedicated" # Whether snapshotter should try to clean up resources when it is closed cleanup_on_close = false +[prefetch] +enable = false +get_prefetch_endpoint = "http://localhost:1323/api/v1/prefetch/download" + [system] # Snapshotter's debug and trace HTTP server interface enable = true diff --git a/pkg/daemon/config.go b/pkg/daemon/config.go index fb2fda575f..a6da7b2186 100644 --- a/pkg/daemon/config.go +++ b/pkg/daemon/config.go @@ -36,7 +36,7 @@ func WithPrefetchDir(dir, imageID string) NewDaemonOpt { return func(d *Daemon) error { s := filepath.Join(dir, d.ID()) prefetchDir, err := prefetch.GetPrefetchList(s, imageID) - if err != nil && !errors.Is(err, prefetch.ErrUds) { + if err != nil { return errors.Wrapf(err, "failed to get prefetchList for image %s in path %s", imageID, s) } if prefetchDir != "" { diff --git a/pkg/prefetch/prefetch.go b/pkg/prefetch/prefetch.go index 0dee246562..738d93bfd7 100644 --- a/pkg/prefetch/prefetch.go +++ b/pkg/prefetch/prefetch.go @@ -7,11 +7,9 @@ package prefetch import ( - "context" "encoding/json" "fmt" "io" - "net" "net/http" "os" "path/filepath" @@ -19,60 +17,46 @@ import ( "github.com/containerd/containerd/log" "github.com/pkg/errors" + + "github.com/containerd/nydus-snapshotter/config" ) type prefetchlist struct { FilePaths []string `json:"files"` } -const ( - endpointPrefetch = "/api/v1/imagename" - udsSocket = "/run/optimizer/prefetch.sock" -) - -var ErrUds = errors.New("failed to connect unix domain socket") - func GetPrefetchList(prefetchDir, imageRepo string) (string, error) { - url := fmt.Sprintf("http://unix%s", endpointPrefetch) - - req, err := http.NewRequest(http.MethodPost, url, strings.NewReader(imageRepo)) - if err != nil { - return "", err - } + if config.IsPrefetchEnabled() { + url := config.GetPrefetchEndpoint() + getURL := fmt.Sprintf("%s?imageName=%s", url, imageRepo) - client := &http.Client{ - Transport: &http.Transport{ - DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { - return net.Dial("unix", udsSocket) - }, - }, - } - resp, err := client.Do(req) - if err != nil { - log.L.Infof("failed to connect unix domain socket. Skipping prefetch for image: %s\n", imageRepo) - return "", ErrUds - } - defer resp.Body.Close() + resp, err := http.Get(getURL) + if err != nil { + return "", err + } + defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("failed to send data, status code: %v", resp.StatusCode) - } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNotFound { + return "", fmt.Errorf("get from server returned a non-OK status code: %d, HTTP Status Error", resp.StatusCode) + } - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } - if strings.Contains(string(body), "CacheItem not found") { - log.L.Infof("Cache item not found for image: %s\n", imageRepo) - return "", nil - } + if strings.Contains(string(body), "CacheItem not found") { + log.L.Infof("Cache item not found for image: %s\n", imageRepo) + return "", nil + } - prefetchfilePath, err := storePrefetchList(prefetchDir, body) - if err != nil { - return "", err + prefetchfilePath, err := storePrefetchList(prefetchDir, body) + if err != nil { + return "", err + } + return prefetchfilePath, nil } - return prefetchfilePath, nil + return "", nil } func storePrefetchList(prefetchDir string, list []byte) (string, error) {