diff --git a/config/daemonconfig/daemonconfig.go b/config/daemonconfig/daemonconfig.go index 8cd6a5465b..d55fa66c00 100644 --- a/config/daemonconfig/daemonconfig.go +++ b/config/daemonconfig/daemonconfig.go @@ -141,7 +141,7 @@ func DumpConfigString(c interface{}) (string, error) { // Achieve a daemon configuration from template or snapshotter's configuration func SupplementDaemonConfig(c DaemonConfig, imageID, snapshotID string, - vpcRegistry bool, labels map[string]string, params map[string]string) error { + vpcRegistry bool, labels map[string]string, params map[string]string, fn func(string, *auth.PassKeyChain)) error { image, err := registry.ParseImage(imageID) if err != nil { @@ -169,7 +169,11 @@ func SupplementDaemonConfig(c DaemonConfig, imageID, snapshotID string, // when repository is public. keyChain := auth.GetRegistryKeyChain(registryHost, imageID, labels) c.Supplement(registryHost, image.Repo, snapshotID, params) - c.FillAuth(keyChain) + if config.IsKeyringEnabled() && fn != nil { + fn(registryHost, keyChain) + } else { + c.FillAuth(keyChain) + } // Localfs and OSS backends don't need any update, // just use the provided config in template diff --git a/pkg/daemon/command/command.go b/pkg/daemon/command/command.go index eca408297d..7b14dc121a 100644 --- a/pkg/daemon/command/command.go +++ b/pkg/daemon/command/command.go @@ -99,6 +99,10 @@ func BuildCommand(opts []Opt) ([]string, error) { return args, nil } +func (dc *DaemonCommand) GetConfigPath() string { + return dc.Config +} + func WithMode(m string) Opt { return func(cmd *DaemonCommand) { cmd.Mode = m diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index fae78616ee..e44a4c2339 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -17,16 +17,19 @@ import ( "time" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/containerd/containerd/log" "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/config/daemonconfig" + "github.com/containerd/nydus-snapshotter/pkg/auth" "github.com/containerd/nydus-snapshotter/pkg/daemon/types" "github.com/containerd/nydus-snapshotter/pkg/errdefs" "github.com/containerd/nydus-snapshotter/pkg/supervisor" "github.com/containerd/nydus-snapshotter/pkg/utils/erofs" "github.com/containerd/nydus-snapshotter/pkg/utils/mount" + "github.com/containerd/nydus-snapshotter/pkg/utils/registry" "github.com/containerd/nydus-snapshotter/pkg/utils/retry" ) @@ -66,6 +69,7 @@ type Daemon struct { // fusedev shared mode: zero, one or more RAFS instances // fscache shared mode: zero, one or more RAFS instances Instances rafsSet + // authCache *lru.Cache // Protect nydusd http client cmu sync.Mutex @@ -82,6 +86,7 @@ type Daemon struct { Version types.BuildTimeInfo ref int32 + // Cache the nydusd daemon state to avoid frequently querying nydusd by API. state types.DaemonState } @@ -222,7 +227,7 @@ func (d *Daemon) IsSharedDaemon() bool { return d.HostMountpoint() == config.GetRootMountpoint() } -func (d *Daemon) SharedMount(rafs *Rafs) error { +func (d *Daemon) SharedMount(rafs *Rafs, authCache *auth.Cache) error { defer d.SendStates() switch d.States.FsDriver { @@ -232,13 +237,13 @@ func (d *Daemon) SharedMount(rafs *Rafs) error { } return nil case config.FsDriverFusedev: - return d.sharedFusedevMount(rafs) + return d.sharedFusedevMount(rafs, authCache) default: return errors.Errorf("unsupported fs driver %s", d.States.FsDriver) } } -func (d *Daemon) sharedFusedevMount(rafs *Rafs) error { +func (d *Daemon) sharedFusedevMount(rafs *Rafs, authCache *auth.Cache) error { client, err := d.GetClient() if err != nil { return errors.Wrapf(err, "mount instance %s", rafs.SnapshotID) @@ -255,6 +260,25 @@ func (d *Daemon) sharedFusedevMount(rafs *Rafs) error { d.ConfigFile(rafs.SnapshotID)) } + if config.IsKeyringEnabled() { + image, err := registry.ParseImage(rafs.ImageID) + if err != nil { + return errors.Wrapf(err, "parse image %s", rafs.ImageID) + } + + logrus.Debugf("get key for %s", image.Host) + cachedAuth, err := authCache.GetAuth(image.Host) + if err != nil { + return err + } + + keyChain, err := auth.FromBase64(cachedAuth) + if err != nil { + return err + } + c.FillAuth(&keyChain) + } + cfg, err := c.DumpString() if err != nil { return errors.Wrap(err, "dump instance configuration") @@ -624,7 +648,7 @@ func (d *Daemon) CloneInstances(src *Daemon) { } // Daemon must be started and reach RUNNING state before call this method -func (d *Daemon) RecoveredMountInstances() error { +func (d *Daemon) RecoveredMountInstances(authCache *auth.Cache) error { if d.IsSharedDaemon() { d.Instances.Lock() defer d.Instances.Unlock() @@ -641,7 +665,7 @@ func (d *Daemon) RecoveredMountInstances() error { for _, i := range instances { if d.HostMountpoint() != i.GetMountpoint() { log.L.Infof("Recovered mount instance %s", i.SnapshotID) - if err := d.SharedMount(i); err != nil { + if err := d.SharedMount(i, authCache); err != nil { return err } } @@ -657,6 +681,7 @@ func NewDaemon(opt ...NewDaemonOpt) (*Daemon, error) { d.States.ID = newID() d.States.DaemonMode = config.DaemonModeDedicated d.Instances = rafsSet{instances: make(map[string]*Rafs)} + // d.authCache = lru.New(32) for _, o := range opt { err := o(d) diff --git a/pkg/filesystem/fs.go b/pkg/filesystem/fs.go index 85046841a9..f91da00501 100644 --- a/pkg/filesystem/fs.go +++ b/pkg/filesystem/fs.go @@ -18,6 +18,7 @@ import ( "github.com/mohae/deepcopy" "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/containerd/containerd/log" "github.com/containerd/containerd/snapshots" @@ -25,6 +26,7 @@ import ( snpkg "github.com/containerd/containerd/pkg/snapshotters" "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/config/daemonconfig" + "github.com/containerd/nydus-snapshotter/pkg/auth" "github.com/containerd/nydus-snapshotter/pkg/cache" "github.com/containerd/nydus-snapshotter/pkg/daemon" "github.com/containerd/nydus-snapshotter/pkg/daemon/types" @@ -132,7 +134,7 @@ func NewFileSystem(ctx context.Context, opt ...NewFSOpt) (*Filesystem, error) { if err := d.WaitUntilState(types.DaemonStateRunning); err != nil { return nil, errors.Wrapf(err, "wait for daemon %s", d.ID()) } - if err := d.RecoveredMountInstances(); err != nil { + if err := d.RecoveredMountInstances(fsManager.AuthCache); err != nil { return nil, errors.Wrapf(err, "recover mounts for daemon %s", d.ID()) } fs.TryRetainSharedDaemon(d) @@ -298,7 +300,14 @@ func (fs *Filesystem) Mount(snapshotID string, labels map[string]string) (err er daemonconfig.CacheDir: cacheDir, } cfg := deepcopy.Copy(fsManager.DaemonConfig).(daemonconfig.DaemonConfig) - err = daemonconfig.SupplementDaemonConfig(cfg, imageID, snapshotID, false, labels, params) + var updateErr error + err = daemonconfig.SupplementDaemonConfig(cfg, imageID, snapshotID, false, labels, params, func(imageHost string, keyChain *auth.PassKeyChain) { + logrus.Debugf("add key for %s", imageHost) + updateErr = fsManager.AuthCache.UpdateAuth(imageHost, keyChain.ToBase64()) + }) + if updateErr != nil { + return updateErr + } if err != nil { return errors.Wrap(err, "supplement configuration") } @@ -490,7 +499,7 @@ func (fs *Filesystem) mountRemote(fsManager *manager.Manager, useSharedDaemon bo } else { r.SetMountpoint(path.Join(r.GetSnapshotDir(), "mnt")) } - if err := d.SharedMount(r); err != nil { + if err := d.SharedMount(r, fsManager.AuthCache); err != nil { return errors.Wrapf(err, "failed to mount") } } else { diff --git a/pkg/manager/daemon_adaptor.go b/pkg/manager/daemon_adaptor.go index 62ce6e2d04..f287e9574b 100644 --- a/pkg/manager/daemon_adaptor.go +++ b/pkg/manager/daemon_adaptor.go @@ -7,12 +7,14 @@ package manager import ( + "fmt" "os" "os/exec" "strings" "time" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/containerd/containerd/log" @@ -23,6 +25,7 @@ import ( "github.com/containerd/nydus-snapshotter/pkg/errdefs" "github.com/containerd/nydus-snapshotter/pkg/metrics/collector" metrics "github.com/containerd/nydus-snapshotter/pkg/metrics/tool" + "github.com/containerd/nydus-snapshotter/pkg/utils/registry" ) // Fork the nydusd daemon with the process PID decided @@ -192,6 +195,25 @@ func (m *Manager) BuildDaemonCommand(d *daemon.Daemon, bin string, upgrade bool) cmd := exec.Command(nydusdPath, args...) + if config.IsKeyringEnabled() && !d.IsSharedDaemon() { + if d.Instances.Len() > 1 { + return nil, errors.New("nydusd is not running in shared mode but the instance length is large than 1") + } + + imageID := d.Instances.Head().ImageID + image, err := registry.ParseImage(imageID) + if err != nil { + return nil, errors.Wrapf(err, "parse image %s", imageID) + } + + logrus.Debugf("get key for %s", image.Host) + auth, err := m.AuthCache.GetAuth(image.Host) + if err != nil { + return nil, err + } + cmd.Env = append(cmd.Env, fmt.Sprintf("IMAGE_PULL_AUTH=%s", auth)) + } + // nydusd standard output and standard error rather than its logs are // always redirected to snapshotter's respectively cmd.Stdout = os.Stdout diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 2ace2fb296..258cf63bfe 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -20,6 +20,7 @@ import ( "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/config/daemonconfig" + "github.com/containerd/nydus-snapshotter/pkg/auth" "github.com/containerd/nydus-snapshotter/pkg/cgroup" "github.com/containerd/nydus-snapshotter/pkg/daemon" "github.com/containerd/nydus-snapshotter/pkg/daemon/types" @@ -139,6 +140,9 @@ type Manager struct { // Cgroup manager for nydusd CgroupMgr *cgroup.Manager + // Cache for registry authorization + AuthCache *auth.Cache + // In order to validate daemon fs driver is consistent with the latest snapshotter boot FsDriver string @@ -222,7 +226,7 @@ func (m *Manager) doDaemonRestart(d *daemon.Daemon) { break } - if err := d.SharedMount(r); err != nil { + if err := d.SharedMount(r, m.AuthCache); err != nil { log.L.Warnf("Failed to mount rafs instance, %v", err) } } @@ -284,6 +288,7 @@ func NewManager(opt Opt) (*Manager, error) { SupervisorSet: supervisorSet, DaemonConfig: opt.DaemonConfig, CgroupMgr: opt.CgroupMgr, + AuthCache: auth.NewCache(), FsDriver: opt.FsDriver, }