From af7cd2c6e3abe6c85f594cb1049fed8dc3225d5d Mon Sep 17 00:00:00 2001 From: Changwei Ge Date: Mon, 12 Dec 2022 09:24:21 +0800 Subject: [PATCH 1/2] maintenance api server's default socket should be under root dir We should put all snapshotter's stuff under root dir which can be customized. Signed-off-by: Changwei Ge --- cmd/containerd-nydus-grpc/pkg/command/flags.go | 11 +++++------ config/config.go | 4 ++-- snapshot/snapshot.go | 4 ++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/cmd/containerd-nydus-grpc/pkg/command/flags.go b/cmd/containerd-nydus-grpc/pkg/command/flags.go index ae03220310..923502d828 100644 --- a/cmd/containerd-nydus-grpc/pkg/command/flags.go +++ b/cmd/containerd-nydus-grpc/pkg/command/flags.go @@ -15,7 +15,6 @@ const ( defaultAddress = "/run/containerd-nydus/containerd-nydus-grpc.sock" defaultLogLevel = logrus.InfoLevel defaultRootDir = "/var/lib/containerd-nydus" - defaultAPISocket = "/var/lib/containerd-nydus/api.sock" defaultGCPeriod = "24h" defaultPublicKey = "/signing/nydus-image-signing-public.key" DefaultDaemonMode string = "multiple" @@ -51,7 +50,7 @@ type Args struct { EnableKubeconfigKeychain bool `toml:"enable_kubeconfig_keychain"` RecoverPolicy string `toml:"recover_policy"` PrintVersion bool `toml:"-"` - APISocket string + EnableSystemController bool } type Flags struct { @@ -214,11 +213,11 @@ func buildFlags(args *Args) []cli.Flag { Usage: "whether to validate integrity of image bootstrap", Destination: &args.ValidateSignature, }, - &cli.StringFlag{ - Name: "api-socket", - Value: defaultAPISocket, + &cli.BoolFlag{ + Name: "enable-system-controller", Usage: "(experimental) unix domain socket path to serve HTTP-based system management", - Destination: &args.APISocket, + Destination: &args.EnableSystemController, + Value: true, }, &cli.StringFlag{ Name: "kubeconfig-path", diff --git a/config/config.go b/config/config.go index a476686df0..42cd684813 100644 --- a/config/config.go +++ b/config/config.go @@ -110,7 +110,7 @@ type Config struct { RotateLogMaxAge int `toml:"log_rotate_max_age"` RotateLogLocalTime bool `toml:"log_rotate_local_time"` RotateLogCompress bool `toml:"log_rotate_compress"` - APISocket string `toml:"api_socket"` + EnableSystemController bool `toml:"enable_system_controller"` RecoverPolicy string `toml:"recover_policy"` } @@ -205,7 +205,7 @@ func SetStartupParameter(startupFlag *command.Args, cfg *Config) error { cfg.GCPeriod = d cfg.Address = startupFlag.Address - cfg.APISocket = startupFlag.APISocket + cfg.EnableSystemController = startupFlag.EnableSystemController cfg.CleanupOnClose = startupFlag.CleanupOnClose cfg.ConvertVpcRegistry = startupFlag.ConvertVpcRegistry cfg.DisableCacheManager = startupFlag.DisableCacheManager diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 7e820492cb..4ec23a7e27 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -117,8 +117,8 @@ func NewSnapshotter(ctx context.Context, cfg *config.Config) (snapshots.Snapshot }() } - if cfg.APISocket != "" { - systemController, err := system.NewSystemController(manager, cfg.APISocket) + if cfg.EnableSystemController { + systemController, err := system.NewSystemController(manager, path.Join(cfg.RootDir, "system.sock")) if err != nil { return nil, errors.Wrap(err, "create system controller") } From 9a5c0484a433fec4a559d5c0d953039ad078494b Mon Sep 17 00:00:00 2001 From: Changwei Ge Date: Wed, 14 Dec 2022 10:06:54 +0800 Subject: [PATCH 2/2] redirect nydusd and rafs instance configuration to new location For shared daemon mode, it has diffent location to store those configurations Signed-off-by: Changwei Ge --- pkg/store/database_compat.go | 81 ++++++++++++++++++++++++++++++------ 1 file changed, 68 insertions(+), 13 deletions(-) diff --git a/pkg/store/database_compat.go b/pkg/store/database_compat.go index 0690f6802f..4abcfafd70 100644 --- a/pkg/store/database_compat.go +++ b/pkg/store/database_compat.go @@ -9,8 +9,12 @@ package store import ( "context" "encoding/json" + "io" + "os" "path" + "path/filepath" + "github.com/containerd/containerd/log" "github.com/containerd/nydus-snapshotter/pkg/daemon" "github.com/containerd/nydus-snapshotter/pkg/errdefs" "github.com/pkg/errors" @@ -56,7 +60,34 @@ func (db *Database) WalkCompatDaemons(ctx context.Context, handler func(cd *Comp }) } +// Snapshotter v0.3.0 and lower store nydusd and rafs instance configurations in the different folders. +func RedirectInstanceConfig(new, old string) error { + oldConfig, err := os.Open(old) + if err != nil { + return err + } + defer oldConfig.Close() + + err = os.MkdirAll(filepath.Dir(new), 0700) + if err != nil { + return err + } + newConfig, err := os.Create(new) + if err != nil { + return err + } + defer newConfig.Close() + + _, err = io.Copy(newConfig, oldConfig) + if err != nil { + return err + } + + return nil +} + func (db *Database) tryTranslateRecords() error { + log.L.Info("Trying to translate bucket records...") daemons := make([]*CompatDaemon, 0) err := db.WalkCompatDaemons(context.TODO(), func(cd *CompatDaemon) error { @@ -69,30 +100,54 @@ func (db *Database) tryTranslateRecords() error { } var sharedMode = false + var configDir string // Scan all the daemons if it is started as shared mode last time for _, d := range daemons { if d.ID == SharedNydusDaemonID { sharedMode = true + } else if configDir == "" { + configDir = d.ConfigDir } } for _, d := range daemons { var mp string var newDaemon *daemon.Daemon - if sharedMode && d.ID == SharedNydusDaemonID { - newDaemon = &daemon.Daemon{ - States: daemon.States{ - ID: d.ID, - ProcessID: d.Pid, - APISocket: path.Join(d.SnapshotDir, "api.sock"), - FsDriver: d.FsDriver, - Mountpoint: *d.RootMountPoint, - LogDir: d.LogDir, - LogLevel: d.LogLevel, - // Shared daemon does not need config file when start - ConfigDir: "", - }} + if sharedMode { + if d.ID == SharedNydusDaemonID { + + oldConfig := path.Join(configDir, "config.json") + newConfig := filepath.Join(filepath.Dir(configDir), SharedNydusDaemonID, "config.json") + + newDaemon = &daemon.Daemon{ + States: daemon.States{ + ID: d.ID, + ProcessID: d.Pid, + APISocket: path.Join(d.SnapshotDir, "api.sock"), + FsDriver: d.FsDriver, + Mountpoint: *d.RootMountPoint, + LogDir: d.LogDir, + LogLevel: d.LogLevel, + // Shared daemon does not need config file when start + ConfigDir: filepath.Dir(newConfig), + }} + + if err := RedirectInstanceConfig(newConfig, oldConfig); err != nil { + log.L.WithError(err).Warnf("Redirect configuration from %s to %s", oldConfig, newConfig) + } + + } else { + // Redirect rafs instance configuration files. We have to do it here to + // prevent scattering compatibility code anywhere. + oldConfig := path.Join(d.ConfigDir, "config.json") + newConfig := path.Join(filepath.Dir(d.ConfigDir), SharedNydusDaemonID, + d.SnapshotID, "config.json") + log.L.Infof("Redirect configuration to %s", newConfig) + if err := RedirectInstanceConfig(newConfig, oldConfig); err != nil { + log.L.WithError(err).Warnf("Redirect configuration from %s to %s", oldConfig, newConfig) + } + } } else if !sharedMode { mp = *d.CustomMountPoint newDaemon = &daemon.Daemon{