daemon: store auth to keyring and send it to nydusd via env

Signed-off-by: Bin Tang <[email protected]>

config/daemonconfig/daemonconfig.go

@@ -12,6 +12,7 @@ import (
 	"os"
 
 	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
 
 	"github.com/containerd/nydus-snapshotter/config"
 	"github.com/containerd/nydus-snapshotter/pkg/auth"
@@ -140,7 +141,7 @@ func DumpConfigString(c interface{}) (string, error) {
 
 // Achieve a daemon configuration from template or snapshotter's configuration
 func SupplementDaemonConfig(c DaemonConfig, imageID, snapshotID string,
-	vpcRegistry bool, labels map[string]string, params map[string]string) error {
+	vpcRegistry bool, labels map[string]string, params map[string]string, fn func(string, *auth.PassKeyChain) error) error {
 
 	image, err := registry.ParseImage(imageID)
 	if err != nil {
@@ -168,7 +169,16 @@ func SupplementDaemonConfig(c DaemonConfig, imageID, snapshotID string,
 		// when repository is public.
 		keyChain := auth.GetRegistryKeyChain(registryHost, imageID, labels)
 		c.Supplement(registryHost, image.Repo, snapshotID, params)
-		c.FillAuth(keyChain)
+		if config.IsKeyringEnabled() && fn != nil {
+			if err := fn(registryHost, keyChain); err != nil {
+				if errors.Is(err, unix.EINVAL) {
+					c.FillAuth(keyChain)
+				}
+				return err
+			}
+		} else {
+			c.FillAuth(keyChain)
+		}
 
 	// Localfs and OSS backends don't need any update,
 	// just use the provided config in template

pkg/daemon/daemon.go

@@ -17,11 +17,13 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
 
 	"github.com/containerd/containerd/log"
 
 	"github.com/containerd/nydus-snapshotter/config"
 	"github.com/containerd/nydus-snapshotter/config/daemonconfig"
+	"github.com/containerd/nydus-snapshotter/pkg/auth"
 	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
 	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
 	"github.com/containerd/nydus-snapshotter/pkg/supervisor"
@@ -82,6 +84,7 @@ type Daemon struct {
 	Version               types.BuildTimeInfo
 
 	ref int32
+
 	// Cache the nydusd daemon state to avoid frequently querying nydusd by API.
 	state types.DaemonState
 }
@@ -222,23 +225,23 @@ func (d *Daemon) IsSharedDaemon() bool {
 	return d.HostMountpoint() == config.GetRootMountpoint()
 }
 
-func (d *Daemon) SharedMount(rafs *Rafs) error {
+func (d *Daemon) SharedMount(rafs *Rafs, authCache *auth.Cache) error {
 	defer d.SendStates()
 
 	switch d.States.FsDriver {
 	case config.FsDriverFscache:
-		if err := d.sharedErofsMount(rafs); err != nil {
+		if err := d.sharedErofsMount(rafs, authCache); err != nil {
 			return errors.Wrapf(err, "mount erofs")
 		}
 		return nil
 	case config.FsDriverFusedev:
-		return d.sharedFusedevMount(rafs)
+		return d.sharedFusedevMount(rafs, authCache)
 	default:
 		return errors.Errorf("unsupported fs driver %s", d.States.FsDriver)
 	}
 }
 
-func (d *Daemon) sharedFusedevMount(rafs *Rafs) error {
+func (d *Daemon) sharedFusedevMount(rafs *Rafs, authCache *auth.Cache) error {
 	client, err := d.GetClient()
 	if err != nil {
 		return errors.Wrapf(err, "mount instance %s", rafs.SnapshotID)
@@ -255,6 +258,14 @@ func (d *Daemon) sharedFusedevMount(rafs *Rafs) error {
 			d.ConfigFile(rafs.SnapshotID))
 	}
 
+	if config.IsKeyringEnabled() {
+		keyChain, err := authCache.GetKeyChain(rafs.ImageID)
+		if err != nil && !errors.Is(err, unix.EINVAL) {
+			return err
+		}
+		c.FillAuth(keyChain)
+	}
+
 	cfg, err := c.DumpString()
 	if err != nil {
 		return errors.Wrap(err, "dump instance configuration")
@@ -268,7 +279,7 @@ func (d *Daemon) sharedFusedevMount(rafs *Rafs) error {
 	return nil
 }
 
-func (d *Daemon) sharedErofsMount(rafs *Rafs) error {
+func (d *Daemon) sharedErofsMount(rafs *Rafs, authCache *auth.Cache) error {
 	client, err := d.GetClient()
 	if err != nil {
 		return errors.Wrapf(err, "bind blob %s", d.ID())
@@ -285,6 +296,14 @@ func (d *Daemon) sharedErofsMount(rafs *Rafs) error {
 		return err
 	}
 
+	if config.IsKeyringEnabled() {
+		keyChain, err := authCache.GetKeyChain(rafs.ImageID)
+		if err != nil && !errors.Is(err, unix.EINVAL) {
+			return err
+		}
+		c.FillAuth(keyChain)
+	}
+
 	cfgStr, err := c.DumpString()
 	if err != nil {
 		return err
@@ -624,7 +643,7 @@ func (d *Daemon) CloneInstances(src *Daemon) {
 }
 
 // Daemon must be started and reach RUNNING state before call this method
-func (d *Daemon) RecoveredMountInstances() error {
+func (d *Daemon) RecoveredMountInstances(authCache *auth.Cache) error {
 	if d.IsSharedDaemon() {
 		d.Instances.Lock()
 		defer d.Instances.Unlock()
@@ -641,7 +660,7 @@ func (d *Daemon) RecoveredMountInstances() error {
 		for _, i := range instances {
 			if d.HostMountpoint() != i.GetMountpoint() {
 				log.L.Infof("Recovered mount instance %s", i.SnapshotID)
-				if err := d.SharedMount(i); err != nil {
+				if err := d.SharedMount(i, authCache); err != nil {
 					return err
 				}
 			}

pkg/filesystem/fs.go

@@ -25,6 +25,7 @@ import (
 	snpkg "github.com/containerd/containerd/pkg/snapshotters"
 	"github.com/containerd/nydus-snapshotter/config"
 	"github.com/containerd/nydus-snapshotter/config/daemonconfig"
+	"github.com/containerd/nydus-snapshotter/pkg/auth"
 	"github.com/containerd/nydus-snapshotter/pkg/cache"
 	"github.com/containerd/nydus-snapshotter/pkg/daemon"
 	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
@@ -132,7 +133,7 @@ func NewFileSystem(ctx context.Context, opt ...NewFSOpt) (*Filesystem, error) {
 		if err := d.WaitUntilState(types.DaemonStateRunning); err != nil {
 			return nil, errors.Wrapf(err, "wait for daemon %s", d.ID())
 		}
-		if err := d.RecoveredMountInstances(); err != nil {
+		if err := d.RecoveredMountInstances(fsManager.AuthCache); err != nil {
 			return nil, errors.Wrapf(err, "recover mounts for daemon %s", d.ID())
 		}
 		fs.TryRetainSharedDaemon(d)
@@ -298,7 +299,9 @@ func (fs *Filesystem) Mount(snapshotID string, labels map[string]string) (err er
 			daemonconfig.CacheDir:  cacheDir,
 		}
 		cfg := deepcopy.Copy(fsManager.DaemonConfig).(daemonconfig.DaemonConfig)
-		err = daemonconfig.SupplementDaemonConfig(cfg, imageID, snapshotID, false, labels, params)
+		err = daemonconfig.SupplementDaemonConfig(cfg, imageID, snapshotID, false, labels, params, func(imageHost string, keyChain *auth.PassKeyChain) error {
+			return fsManager.AuthCache.UpdateAuth(imageHost, keyChain.ToBase64())
+		})
 		if err != nil {
 			return errors.Wrap(err, "supplement configuration")
 		}
@@ -490,7 +493,7 @@ func (fs *Filesystem) mountRemote(fsManager *manager.Manager, useSharedDaemon bo
 		} else {
 			r.SetMountpoint(path.Join(r.GetSnapshotDir(), "mnt"))
 		}
-		if err := d.SharedMount(r); err != nil {
+		if err := d.SharedMount(r, fsManager.AuthCache); err != nil {
 			return errors.Wrapf(err, "failed to mount")
 		}
 	} else {

pkg/manager/daemon_adaptor.go

@@ -7,12 +7,14 @@
 package manager
 
 import (
+	"fmt"
 	"os"
 	"os/exec"
 	"strings"
 	"time"
 
 	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
 
 	"github.com/containerd/containerd/log"
 
@@ -25,6 +27,10 @@ import (
 	metrics "github.com/containerd/nydus-snapshotter/pkg/metrics/tool"
 )
 
+const (
+	imagePullAuthEnvName = "IMAGE_PULL_AUTH"
+)
+
 // Fork the nydusd daemon with the process PID decided
 func (m *Manager) StartDaemon(d *daemon.Daemon) error {
 	cmd, err := m.BuildDaemonCommand(d, "", false)
@@ -192,6 +198,22 @@ func (m *Manager) BuildDaemonCommand(d *daemon.Daemon, bin string, upgrade bool)
 
 	cmd := exec.Command(nydusdPath, args...)
 
+	if config.IsKeyringEnabled() && !d.IsSharedDaemon() {
+		if d.Instances.Len() > 1 {
+			return nil, errors.New("nydusd is not running in shared mode but the instance length is greater than 1")
+		}
+
+		imageID := d.Instances.Head().ImageID
+		keyChain, err := m.AuthCache.GetKeyChain(imageID)
+		if err != nil && !errors.Is(err, unix.EINVAL) {
+			return nil, err
+		}
+
+		if keyChain != nil {
+			cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", imagePullAuthEnvName, keyChain.ToBase64()))
+		}
+	}
+
 	// nydusd standard output and standard error rather than its logs are
 	// always redirected to snapshotter's respectively
 	cmd.Stdout = os.Stdout

pkg/manager/manager.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/containerd/nydus-snapshotter/config"
 	"github.com/containerd/nydus-snapshotter/config/daemonconfig"
+	"github.com/containerd/nydus-snapshotter/pkg/auth"
 	"github.com/containerd/nydus-snapshotter/pkg/cgroup"
 	"github.com/containerd/nydus-snapshotter/pkg/daemon"
 	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
@@ -139,6 +140,9 @@ type Manager struct {
 	// Cgroup manager for nydusd
 	CgroupMgr *cgroup.Manager
 
+	// Cache for registry authorization
+	AuthCache *auth.Cache
+
 	// In order to validate daemon fs driver is consistent with the latest snapshotter boot
 	FsDriver string
 
@@ -222,7 +226,7 @@ func (m *Manager) doDaemonRestart(d *daemon.Daemon) {
 			break
 		}
 
-		if err := d.SharedMount(r); err != nil {
+		if err := d.SharedMount(r, m.AuthCache); err != nil {
 			log.L.Warnf("Failed to mount rafs instance, %v", err)
 		}
 	}
@@ -284,6 +288,7 @@ func NewManager(opt Opt) (*Manager, error) {
 		SupervisorSet:    supervisorSet,
 		DaemonConfig:     opt.DaemonConfig,
 		CgroupMgr:        opt.CgroupMgr,
+		AuthCache:        auth.NewCache(),
 		FsDriver:         opt.FsDriver,
 	}