From c1cbcfe5a9f124cdb2064a3f42469d4eee41f67c Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Tue, 1 Aug 2023 13:26:35 +0800 Subject: [PATCH] daemon: store auth to keyring and send it to nydusd via env Signed-off-by: Bin Tang --- config/daemonconfig/daemonconfig.go | 14 ++++++++++-- pkg/daemon/daemon.go | 33 +++++++++++++++++++++++------ pkg/filesystem/fs.go | 9 +++++--- pkg/manager/daemon_adaptor.go | 22 +++++++++++++++++++ pkg/manager/manager.go | 7 +++++- 5 files changed, 72 insertions(+), 13 deletions(-) diff --git a/config/daemonconfig/daemonconfig.go b/config/daemonconfig/daemonconfig.go index cfee3d54d9..05bf88ef65 100644 --- a/config/daemonconfig/daemonconfig.go +++ b/config/daemonconfig/daemonconfig.go @@ -12,6 +12,7 @@ import ( "os" "github.com/pkg/errors" + "golang.org/x/sys/unix" "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/pkg/auth" @@ -140,7 +141,7 @@ func DumpConfigString(c interface{}) (string, error) { // Achieve a daemon configuration from template or snapshotter's configuration func SupplementDaemonConfig(c DaemonConfig, imageID, snapshotID string, - vpcRegistry bool, labels map[string]string, params map[string]string) error { + vpcRegistry bool, labels map[string]string, params map[string]string, fn func(string, *auth.PassKeyChain) error) error { image, err := registry.ParseImage(imageID) if err != nil { @@ -168,7 +169,16 @@ func SupplementDaemonConfig(c DaemonConfig, imageID, snapshotID string, // when repository is public. keyChain := auth.GetRegistryKeyChain(registryHost, imageID, labels) c.Supplement(registryHost, image.Repo, snapshotID, params) - c.FillAuth(keyChain) + if config.IsKeyringEnabled() && fn != nil { + if err := fn(registryHost, keyChain); err != nil { + if errors.Is(err, unix.EINVAL) { + c.FillAuth(keyChain) + } + return err + } + } else { + c.FillAuth(keyChain) + } // Localfs and OSS backends don't need any update, // just use the provided config in template diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index fae78616ee..9a825742cb 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -17,11 +17,13 @@ import ( "time" "github.com/pkg/errors" + "golang.org/x/sys/unix" "github.com/containerd/containerd/log" "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/config/daemonconfig" + "github.com/containerd/nydus-snapshotter/pkg/auth" "github.com/containerd/nydus-snapshotter/pkg/daemon/types" "github.com/containerd/nydus-snapshotter/pkg/errdefs" "github.com/containerd/nydus-snapshotter/pkg/supervisor" @@ -82,6 +84,7 @@ type Daemon struct { Version types.BuildTimeInfo ref int32 + // Cache the nydusd daemon state to avoid frequently querying nydusd by API. state types.DaemonState } @@ -222,23 +225,23 @@ func (d *Daemon) IsSharedDaemon() bool { return d.HostMountpoint() == config.GetRootMountpoint() } -func (d *Daemon) SharedMount(rafs *Rafs) error { +func (d *Daemon) SharedMount(rafs *Rafs, authCache *auth.Cache) error { defer d.SendStates() switch d.States.FsDriver { case config.FsDriverFscache: - if err := d.sharedErofsMount(rafs); err != nil { + if err := d.sharedErofsMount(rafs, authCache); err != nil { return errors.Wrapf(err, "mount erofs") } return nil case config.FsDriverFusedev: - return d.sharedFusedevMount(rafs) + return d.sharedFusedevMount(rafs, authCache) default: return errors.Errorf("unsupported fs driver %s", d.States.FsDriver) } } -func (d *Daemon) sharedFusedevMount(rafs *Rafs) error { +func (d *Daemon) sharedFusedevMount(rafs *Rafs, authCache *auth.Cache) error { client, err := d.GetClient() if err != nil { return errors.Wrapf(err, "mount instance %s", rafs.SnapshotID) @@ -255,6 +258,14 @@ func (d *Daemon) sharedFusedevMount(rafs *Rafs) error { d.ConfigFile(rafs.SnapshotID)) } + if config.IsKeyringEnabled() { + keyChain, err := authCache.GetKeyChain(rafs.ImageID) + if err != nil && !errors.Is(err, unix.EINVAL) { + return err + } + c.FillAuth(keyChain) + } + cfg, err := c.DumpString() if err != nil { return errors.Wrap(err, "dump instance configuration") @@ -268,7 +279,7 @@ func (d *Daemon) sharedFusedevMount(rafs *Rafs) error { return nil } -func (d *Daemon) sharedErofsMount(rafs *Rafs) error { +func (d *Daemon) sharedErofsMount(rafs *Rafs, authCache *auth.Cache) error { client, err := d.GetClient() if err != nil { return errors.Wrapf(err, "bind blob %s", d.ID()) @@ -285,6 +296,14 @@ func (d *Daemon) sharedErofsMount(rafs *Rafs) error { return err } + if config.IsKeyringEnabled() { + keyChain, err := authCache.GetKeyChain(rafs.ImageID) + if err != nil && !errors.Is(err, unix.EINVAL) { + return err + } + c.FillAuth(keyChain) + } + cfgStr, err := c.DumpString() if err != nil { return err @@ -624,7 +643,7 @@ func (d *Daemon) CloneInstances(src *Daemon) { } // Daemon must be started and reach RUNNING state before call this method -func (d *Daemon) RecoveredMountInstances() error { +func (d *Daemon) RecoveredMountInstances(authCache *auth.Cache) error { if d.IsSharedDaemon() { d.Instances.Lock() defer d.Instances.Unlock() @@ -641,7 +660,7 @@ func (d *Daemon) RecoveredMountInstances() error { for _, i := range instances { if d.HostMountpoint() != i.GetMountpoint() { log.L.Infof("Recovered mount instance %s", i.SnapshotID) - if err := d.SharedMount(i); err != nil { + if err := d.SharedMount(i, authCache); err != nil { return err } } diff --git a/pkg/filesystem/fs.go b/pkg/filesystem/fs.go index 85046841a9..c349aa13f9 100644 --- a/pkg/filesystem/fs.go +++ b/pkg/filesystem/fs.go @@ -25,6 +25,7 @@ import ( snpkg "github.com/containerd/containerd/pkg/snapshotters" "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/config/daemonconfig" + "github.com/containerd/nydus-snapshotter/pkg/auth" "github.com/containerd/nydus-snapshotter/pkg/cache" "github.com/containerd/nydus-snapshotter/pkg/daemon" "github.com/containerd/nydus-snapshotter/pkg/daemon/types" @@ -132,7 +133,7 @@ func NewFileSystem(ctx context.Context, opt ...NewFSOpt) (*Filesystem, error) { if err := d.WaitUntilState(types.DaemonStateRunning); err != nil { return nil, errors.Wrapf(err, "wait for daemon %s", d.ID()) } - if err := d.RecoveredMountInstances(); err != nil { + if err := d.RecoveredMountInstances(fsManager.AuthCache); err != nil { return nil, errors.Wrapf(err, "recover mounts for daemon %s", d.ID()) } fs.TryRetainSharedDaemon(d) @@ -298,7 +299,9 @@ func (fs *Filesystem) Mount(snapshotID string, labels map[string]string) (err er daemonconfig.CacheDir: cacheDir, } cfg := deepcopy.Copy(fsManager.DaemonConfig).(daemonconfig.DaemonConfig) - err = daemonconfig.SupplementDaemonConfig(cfg, imageID, snapshotID, false, labels, params) + err = daemonconfig.SupplementDaemonConfig(cfg, imageID, snapshotID, false, labels, params, func(imageHost string, keyChain *auth.PassKeyChain) error { + return fsManager.AuthCache.UpdateAuth(imageHost, keyChain.ToBase64()) + }) if err != nil { return errors.Wrap(err, "supplement configuration") } @@ -490,7 +493,7 @@ func (fs *Filesystem) mountRemote(fsManager *manager.Manager, useSharedDaemon bo } else { r.SetMountpoint(path.Join(r.GetSnapshotDir(), "mnt")) } - if err := d.SharedMount(r); err != nil { + if err := d.SharedMount(r, fsManager.AuthCache); err != nil { return errors.Wrapf(err, "failed to mount") } } else { diff --git a/pkg/manager/daemon_adaptor.go b/pkg/manager/daemon_adaptor.go index 62ce6e2d04..60071c8b13 100644 --- a/pkg/manager/daemon_adaptor.go +++ b/pkg/manager/daemon_adaptor.go @@ -7,12 +7,14 @@ package manager import ( + "fmt" "os" "os/exec" "strings" "time" "github.com/pkg/errors" + "golang.org/x/sys/unix" "github.com/containerd/containerd/log" @@ -25,6 +27,10 @@ import ( metrics "github.com/containerd/nydus-snapshotter/pkg/metrics/tool" ) +const ( + imagePullAuthEnvName = "IMAGE_PULL_AUTH" +) + // Fork the nydusd daemon with the process PID decided func (m *Manager) StartDaemon(d *daemon.Daemon) error { cmd, err := m.BuildDaemonCommand(d, "", false) @@ -192,6 +198,22 @@ func (m *Manager) BuildDaemonCommand(d *daemon.Daemon, bin string, upgrade bool) cmd := exec.Command(nydusdPath, args...) + if config.IsKeyringEnabled() && !d.IsSharedDaemon() { + if d.Instances.Len() > 1 { + return nil, errors.New("nydusd is not running in shared mode but the instance length is greater than 1") + } + + imageID := d.Instances.Head().ImageID + keyChain, err := m.AuthCache.GetKeyChain(imageID) + if err != nil && !errors.Is(err, unix.EINVAL) { + return nil, err + } + + if keyChain != nil { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", imagePullAuthEnvName, keyChain.ToBase64())) + } + } + // nydusd standard output and standard error rather than its logs are // always redirected to snapshotter's respectively cmd.Stdout = os.Stdout diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 2ace2fb296..258cf63bfe 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -20,6 +20,7 @@ import ( "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/config/daemonconfig" + "github.com/containerd/nydus-snapshotter/pkg/auth" "github.com/containerd/nydus-snapshotter/pkg/cgroup" "github.com/containerd/nydus-snapshotter/pkg/daemon" "github.com/containerd/nydus-snapshotter/pkg/daemon/types" @@ -139,6 +140,9 @@ type Manager struct { // Cgroup manager for nydusd CgroupMgr *cgroup.Manager + // Cache for registry authorization + AuthCache *auth.Cache + // In order to validate daemon fs driver is consistent with the latest snapshotter boot FsDriver string @@ -222,7 +226,7 @@ func (m *Manager) doDaemonRestart(d *daemon.Daemon) { break } - if err := d.SharedMount(r); err != nil { + if err := d.SharedMount(r, m.AuthCache); err != nil { log.L.Warnf("Failed to mount rafs instance, %v", err) } } @@ -284,6 +288,7 @@ func NewManager(opt Opt) (*Manager, error) { SupervisorSet: supervisorSet, DaemonConfig: opt.DaemonConfig, CgroupMgr: opt.CgroupMgr, + AuthCache: auth.NewCache(), FsDriver: opt.FsDriver, }