From 3b89f95a24a89ba05791c1b9cc95b2dbd25af51a Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Tue, 5 Sep 2023 16:36:34 +0800 Subject: [PATCH 1/8] snapshot: introduce data structures to support Kata virtual volumes Introduce data structures to support Kata virtual volumes, which is a superset of Nydus `ExtraOption`. Signed-off-by: ChengyuZhu6 Signed-off-by: Jiang Liu --- snapshot/mount_option.go | 177 +++++++++++++++++++++++ snapshot/mount_option_test.go | 260 ++++++++++++++++++++++++++++++++++ 2 files changed, 437 insertions(+) create mode 100644 snapshot/mount_option_test.go diff --git a/snapshot/mount_option.go b/snapshot/mount_option.go index f9db61c2da..da6bb9b37a 100644 --- a/snapshot/mount_option.go +++ b/snapshot/mount_option.go @@ -9,9 +9,11 @@ package snapshot import ( "context" "encoding/base64" + "encoding/hex" "encoding/json" "fmt" "os" + "strings" "github.com/containerd/containerd/log" "github.com/containerd/containerd/mount" @@ -97,3 +99,178 @@ func (o *snapshotter) remoteMountWithExtraOptions(ctx context.Context, s storage }, }, nil } + +// Consts and data structures for Kata Virtual Volume +const ( + minBlockSize = 1 << 9 + maxBlockSize = 1 << 19 +) + +const ( + KataVirtualVolumeOptionName = "io.katacontainers.volume" + KataVirtualVolumeDirectBlockType = "direct_block" + KataVirtualVolumeImageRawBlockType = "image_raw_block" + KataVirtualVolumeLayerRawBlockType = "layer_raw_block" + KataVirtualVolumeImageNydusBlockType = "image_nydus_block" + KataVirtualVolumeLayerNydusBlockType = "layer_nydus_block" + KataVirtualVolumeImageNydusFsType = "image_nydus_fs" + KataVirtualVolumeLayerNydusFsType = "layer_nydus_fs" + KataVirtualVolumeImageGuestPullType = "image_guest_pull" +) + +// DmVerityInfo contains configuration information for DmVerity device. +type DmVerityInfo struct { + HashType string `json:"hashtype"` + Hash string `json:"hash"` + BlockNum uint64 `json:"blocknum"` + Blocksize uint64 `json:"blocksize"` + Hashsize uint64 `json:"hashsize"` + Offset uint64 `json:"offset"` +} + +func (d *DmVerityInfo) Validate() error { + err := d.validateHashType() + if err != nil { + return err + } + + if d.BlockNum == 0 || d.BlockNum > uint64(^uint32(0)) { + return fmt.Errorf("Zero block count for DmVerity device %s", d.Hash) + } + + if !validateBlockSize(d.Blocksize) || !validateBlockSize(d.Hashsize) { + return fmt.Errorf("Unsupported verity block size: data_block_size = %d, hash_block_size = %d", d.Blocksize, d.Hashsize) + } + + if d.Offset%d.Hashsize != 0 || d.Offset < d.Blocksize*d.BlockNum { + return fmt.Errorf("Invalid hashvalue offset %d for DmVerity device %s", d.Offset, d.Hash) + } + + return nil +} + +func (d *DmVerityInfo) validateHashType() error { + switch strings.ToLower(d.HashType) { + case "sha256": + return d.validateHash(64, "sha256") + case "sha1": + return d.validateHash(40, "sha1") + default: + return fmt.Errorf("Unsupported hash algorithm %s for DmVerity device %s", d.HashType, d.Hash) + } +} + +func (d *DmVerityInfo) validateHash(expectedLen int, hashType string) error { + _, err := hex.DecodeString(d.Hash) + if len(d.Hash) != expectedLen || err != nil { + return fmt.Errorf("Invalid hash value %s:%s for DmVerity device with %s", hashType, d.Hash, hashType) + } + return nil +} + +func validateBlockSize(blockSize uint64) bool { + return minBlockSize <= blockSize && blockSize <= maxBlockSize +} + +func ParseDmVerityInfo(option string) (*DmVerityInfo, error) { + no := &DmVerityInfo{} + if err := json.Unmarshal([]byte(option), no); err != nil { + return nil, errors.Wrapf(err, "DmVerityInfo json unmarshal err") + } + if err := no.Validate(); err != nil { + return nil, fmt.Errorf("DmVerityInfo is not correct, %+v; error = %+v", no, err) + } + return no, nil +} + +// DirectAssignedVolume contains meta information for a directly assigned volume. +type DirectAssignedVolume struct { + Metadata map[string]string `json:"metadata"` +} + +func (d *DirectAssignedVolume) Validate() bool { + return d.Metadata != nil +} + +// ImagePullVolume contains meta information for pulling an image inside the guest. +type ImagePullVolume struct { + Metadata map[string]string `json:"metadata"` +} + +func (i *ImagePullVolume) Validate() bool { + return i.Metadata != nil +} + +// NydusImageVolume contains Nydus image volume information. +type NydusImageVolume struct { + Config string `json:"config"` + SnapshotDir string `json:"snapshot_dir"` +} + +func (n *NydusImageVolume) Validate() bool { + return len(n.Config) > 0 || len(n.SnapshotDir) > 0 +} + +// KataVirtualVolume encapsulates information for extra mount options and direct volumes. +type KataVirtualVolume struct { + VolumeType string `json:"volume_type"` + Source string `json:"source,omitempty"` + FSType string `json:"fs_type,omitempty"` + Options []string `json:"options,omitempty"` + DirectVolume *DirectAssignedVolume `json:"direct_volume,omitempty"` + ImagePull *ImagePullVolume `json:"image_pull,omitempty"` + NydusImage *NydusImageVolume `json:"nydus_image,omitempty"` + DmVerity *DmVerityInfo `json:"dm_verity,omitempty"` +} + +func (k *KataVirtualVolume) Validate() bool { + switch k.VolumeType { + case KataVirtualVolumeDirectBlockType: + if k.Source != "" && k.DirectVolume != nil && k.DirectVolume.Validate() { + return true + } + case KataVirtualVolumeImageRawBlockType, KataVirtualVolumeLayerRawBlockType: + if k.Source != "" && (k.DmVerity == nil || k.DmVerity.Validate() == nil) { + return true + } + case KataVirtualVolumeImageNydusBlockType, KataVirtualVolumeLayerNydusBlockType, KataVirtualVolumeImageNydusFsType, KataVirtualVolumeLayerNydusFsType: + if k.Source != "" && k.NydusImage != nil && k.NydusImage.Validate() { + return true + } + case KataVirtualVolumeImageGuestPullType: + if k.ImagePull != nil && k.ImagePull.Validate() { + return true + } + } + + return false +} + +func ParseKataVirtualVolume(option []byte) (*KataVirtualVolume, error) { + no := &KataVirtualVolume{} + if err := json.Unmarshal(option, no); err != nil { + return nil, errors.Wrapf(err, "KataVirtualVolume json unmarshal err") + } + if !no.Validate() { + return nil, fmt.Errorf("KataVirtualVolume is not correct, %+v", no) + } + + return no, nil +} + +func ParseKataVirtualVolumeFromBase64(option string) (*KataVirtualVolume, error) { + opt, err := base64.StdEncoding.DecodeString(option) + if err != nil { + return nil, errors.Wrap(err, "KataVirtualVolume base64 decoding err") + } + return ParseKataVirtualVolume(opt) +} + +func EncodeKataVirtualVolumeToBase64(volume KataVirtualVolume) (string, error) { + validKataVirtualVolumeJSON, err := json.Marshal(volume) + if err != nil { + return "", errors.Wrapf(err, "marshal KataVirtualVolume object") + } + option := base64.StdEncoding.EncodeToString(validKataVirtualVolumeJSON) + return option, nil +} diff --git a/snapshot/mount_option_test.go b/snapshot/mount_option_test.go new file mode 100644 index 0000000000..1142aa39a7 --- /dev/null +++ b/snapshot/mount_option_test.go @@ -0,0 +1,260 @@ +package snapshot + +import ( + "encoding/base64" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDmVerityInfoValidation(t *testing.T) { + TestData := []DmVerityInfo{ + { + HashType: "md5", // "md5" is not a supported hash algorithm + Blocksize: 512, + Hashsize: 512, + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 3000, // Invalid block size, not a power of 2. + Hashsize: 512, + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 0, // Invalid block size, less than 512. + Hashsize: 512, + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 524800, // Invalid block size, greater than 524288. + Hashsize: 512, + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 512, + Hashsize: 3000, // Invalid hash block size, not a power of 2. + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 512, + Hashsize: 0, // Invalid hash block size, less than 512. + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 512, + Hashsize: 524800, // Invalid hash block size, greater than 524288. + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 512, + Hashsize: 512, + BlockNum: 0, // Invalid BlockNum, it must be greater than 0. + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 512, + Hashsize: 512, + BlockNum: 16384, + Offset: 0, // Invalid offset, it must be greater than 0. + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 512, + Hashsize: 512, + BlockNum: 16384, + Offset: 8193, // Invalid offset, it must be aligned to 512. + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + { + HashType: "sha256", + Blocksize: 512, + Hashsize: 512, + BlockNum: 16384, + Offset: 8388608 - 4096, // Invalid offset, it must be equal to blocksize * BlockNum. + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + }, + } + + for _, d := range TestData { + assert.Error(t, d.Validate()) + } + TestCorrectData := DmVerityInfo{ + HashType: "sha256", + Blocksize: 512, + Hashsize: 512, + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + } + assert.NoError(t, TestCorrectData.Validate()) +} + +func TestDirectAssignedVolumeValidation(t *testing.T) { + validDirectVolume := DirectAssignedVolume{ + Metadata: map[string]string{"key": "value"}, + } + assert.True(t, validDirectVolume.Validate()) + + invalidDirectVolume := DirectAssignedVolume{ + Metadata: nil, + } + assert.False(t, invalidDirectVolume.Validate()) +} + +func TestImagePullVolumeValidation(t *testing.T) { + validImagePull := ImagePullVolume{ + Metadata: map[string]string{"key": "value"}, + } + assert.True(t, validImagePull.Validate()) + + invalidImagePull := ImagePullVolume{ + Metadata: nil, + } + assert.False(t, invalidImagePull.Validate()) +} + +func TestNydusImageVolumeValidation(t *testing.T) { + validNydusImage := NydusImageVolume{ + Config: "config_value", + SnapshotDir: "", + } + assert.True(t, validNydusImage.Validate()) + + invalidNydusImage := NydusImageVolume{ + Config: "", + SnapshotDir: "", + } + assert.False(t, invalidNydusImage.Validate()) +} + +func TestKataVirtualVolumeValidation(t *testing.T) { + validKataVirtualVolume := KataVirtualVolume{ + VolumeType: "direct_block", + Source: "/dev/sdb", + FSType: "ext4", + Options: []string{"rw"}, + DirectVolume: &DirectAssignedVolume{ + Metadata: map[string]string{"key": "value"}, + }, + // Initialize other fields + } + assert.True(t, validKataVirtualVolume.Validate()) + + invalidKataVirtualVolume := KataVirtualVolume{ + VolumeType: "direct_block", + Source: "/dev/sdb", + FSType: "", + Options: nil, + DirectVolume: &DirectAssignedVolume{ + Metadata: nil, + }, + // Initialize other fields + } + assert.False(t, invalidKataVirtualVolume.Validate()) +} +func TestParseDmVerityInfo(t *testing.T) { + // Create a mock valid KataVirtualVolume + validDmVerityInfo := DmVerityInfo{ + HashType: "sha256", + Blocksize: 512, + Hashsize: 512, + BlockNum: 16384, + Offset: 8388608, + Hash: "9de18652fe74edfb9b805aaed72ae2aa48f94333f1ba5c452ac33b1c39325174", + } + validKataVirtualVolumeJSON, _ := json.Marshal(validDmVerityInfo) + + t.Run("Valid Option", func(t *testing.T) { + volume, err := ParseDmVerityInfo(string(validKataVirtualVolumeJSON)) + assert.NoError(t, err) + assert.NotNil(t, volume) + assert.NoError(t, volume.Validate()) + }) + + t.Run("Invalid JSON Option", func(t *testing.T) { + volume, err := ParseDmVerityInfo("invalid_json") + assert.Error(t, err) + assert.Nil(t, volume) + }) + +} +func TestParseKataVirtualVolume(t *testing.T) { + // Create a mock valid KataVirtualVolume + validKataVirtualVolume := KataVirtualVolume{ + VolumeType: "direct_block", + Source: "/dev/sdb", + FSType: "ext4", + Options: []string{"rw"}, + DirectVolume: &DirectAssignedVolume{ + Metadata: map[string]string{"key": "value"}, + }, + // Initialize other fields + } + validOption, err := EncodeKataVirtualVolumeToBase64(validKataVirtualVolume) + assert.Nil(t, err) + + t.Run("Valid Option", func(t *testing.T) { + volume, err := ParseKataVirtualVolumeFromBase64(validOption) + + assert.NoError(t, err) + assert.NotNil(t, volume) + assert.True(t, volume.Validate()) + }) + + t.Run("Invalid JSON Option", func(t *testing.T) { + invalidJSONOption := base64.StdEncoding.EncodeToString([]byte("invalid_json")) + volume, err := ParseKataVirtualVolumeFromBase64(invalidJSONOption) + + assert.Error(t, err) + assert.Nil(t, volume) + }) + + invalidBase64Option := "invalid_base64" + t.Run("Invalid Base64 Option", func(t *testing.T) { + volume, err := ParseKataVirtualVolumeFromBase64(invalidBase64Option) + + assert.Error(t, err) + assert.Nil(t, volume) + }) + + t.Run("Invalid Fields", func(t *testing.T) { + // Create a mock invalid KataVirtualVolume + validKataVirtualVolume = KataVirtualVolume{ + VolumeType: "direct_block", + Source: "/dev/sdb", + FSType: "ext4", + Options: []string{"rw"}, + // Initialize other fields + } + invalidFieldOption, err := EncodeKataVirtualVolumeToBase64(validKataVirtualVolume) + assert.Nil(t, err) + volume, err := ParseKataVirtualVolumeFromBase64(invalidFieldOption) + assert.Error(t, err) + assert.Nil(t, volume) + }) +} From 7a5725041b5732a1801a4438837fbf515d82a8c7 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Tue, 5 Sep 2023 23:12:04 +0800 Subject: [PATCH 2/8] filesystem: enhance error handling for Mount() Enhance error handling for Mount() by rollback operations on errors. Signed-off-by: Jiang Liu --- pkg/filesystem/fs.go | 56 ++++++++++++++++++++++++-------------------- snapshot/process.go | 2 +- snapshot/snapshot.go | 4 ++-- 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/pkg/filesystem/fs.go b/pkg/filesystem/fs.go index 83e7fcd923..0c998cc97f 100644 --- a/pkg/filesystem/fs.go +++ b/pkg/filesystem/fs.go @@ -228,12 +228,18 @@ func (fs *Filesystem) WaitUntilReady(snapshotID string) error { // Mount will be called when containerd snapshotter prepare remote snapshotter // this method will fork nydus daemon and manage it in the internal store, and indexed by snapshotID // It must set up all necessary resources during Mount procedure and revoke any step if necessary. -func (fs *Filesystem) Mount(snapshotID string, labels map[string]string, s *storage.Snapshot) (err error) { +func (fs *Filesystem) Mount(ctx context.Context, snapshotID string, labels map[string]string, s *storage.Snapshot) (err error) { // Do not create RAFS instance in case of nodev. if !fs.DaemonBacked() { return nil } + rafs := racache.RafsGlobalCache.Get(snapshotID) + if rafs != nil { + // Instance already exists, how could this happen? Can containerd handle this case? + return nil + } + fsDriver := config.GetFsDriver() if label.IsTarfsDataLayer(labels) { fsDriver = config.FsDriverBlockdev @@ -253,13 +259,7 @@ func (fs *Filesystem) Mount(snapshotID string, labels map[string]string, s *stor } } - r := racache.RafsGlobalCache.Get(snapshotID) - if r != nil { - // Instance already exists, how could this happen? Can containerd handle this case? - return nil - } - - rafs, err := racache.NewRafs(snapshotID, imageID, fsDriver) + rafs, err = racache.NewRafs(snapshotID, imageID, fsDriver) if err != nil { return errors.Wrapf(err, "create rafs instance %s", snapshotID) } @@ -297,8 +297,6 @@ func (fs *Filesystem) Mount(snapshotID string, labels map[string]string, s *stor if err != nil && !errdefs.IsAlreadyExists(err) { return err } - - // TODO: reclaim resources on error } // Nydusd uses cache manager's directory to store blob caches. So cache @@ -352,61 +350,66 @@ func (fs *Filesystem) Mount(snapshotID string, labels map[string]string, s *stor case config.FsDriverFscache: err = fs.mountRemote(fsManager, useSharedDaemon, d, rafs) if err != nil { - return errors.Wrapf(err, "mount file system by daemon %s, snapshot %s", d.ID(), snapshotID) + err = errors.Wrapf(err, "mount file system by daemon %s, snapshot %s", d.ID(), snapshotID) } case config.FsDriverFusedev: err = fs.mountRemote(fsManager, useSharedDaemon, d, rafs) if err != nil { - return errors.Wrapf(err, "mount file system by daemon %s, snapshot %s", d.ID(), snapshotID) + err = errors.Wrapf(err, "mount file system by daemon %s, snapshot %s", d.ID(), snapshotID) } case config.FsDriverBlockdev: err = fs.tarfsMgr.MountTarErofs(snapshotID, s, rafs) if err != nil { - return errors.Wrapf(err, "mount tarfs for snapshot %s", snapshotID) + err = errors.Wrapf(err, "mount tarfs for snapshot %s", snapshotID) } default: - return errors.Errorf("unknown filesystem driver %s for snapshot %s", fsDriver, snapshotID) + err = errors.Errorf("unknown filesystem driver %s for snapshot %s", fsDriver, snapshotID) } // Persist it after associate instance after all the states are calculated. - if err := fsManager.AddRafsInstance(rafs); err != nil { - return errors.Wrapf(err, "create instance %s", snapshotID) + if err == nil { + if err := fsManager.AddRafsInstance(rafs); err != nil { + return errors.Wrapf(err, "create instance %s", snapshotID) + } + } + + if err != nil { + _ = fs.Umount(ctx, snapshotID) + return err } return nil } func (fs *Filesystem) Umount(ctx context.Context, snapshotID string) error { - instance := racache.RafsGlobalCache.Get(snapshotID) - if instance == nil { + rafs := racache.RafsGlobalCache.Get(snapshotID) + if rafs == nil { log.L.Debugf("no RAFS filesystem instance associated with snapshot %s", snapshotID) return nil } - fsDriver := instance.GetFsDriver() + fsDriver := rafs.GetFsDriver() if fsDriver == config.FsDriverNodev { return nil } fsManager, err := fs.getManager(fsDriver) if err != nil { - return errors.Wrapf(err, "get manager for filesystem instance %s", instance.DaemonID) + return errors.Wrapf(err, "get manager for filesystem instance %s", rafs.DaemonID) } switch fsDriver { - case config.FsDriverFscache: - fallthrough - case config.FsDriverFusedev: - daemon, err := fs.getDaemonByRafs(instance) + case config.FsDriverFscache, config.FsDriverFusedev: + daemon, err := fs.getDaemonByRafs(rafs) if err != nil { log.L.Debugf("snapshot %s has no associated nydusd", snapshotID) - return errors.Wrapf(err, "get daemon with ID %s for snapshot %s", instance.DaemonID, snapshotID) + return errors.Wrapf(err, "get daemon with ID %s for snapshot %s", rafs.DaemonID, snapshotID) } daemon.RemoveRafsInstance(snapshotID) if err := fsManager.RemoveRafsInstance(snapshotID); err != nil { return errors.Wrapf(err, "remove snapshot %s", snapshotID) } - if err := daemon.UmountRafsInstance(instance); err != nil { + if err := daemon.UmountRafsInstance(rafs); err != nil { return errors.Wrapf(err, "umount instance %s", snapshotID) } // Once daemon's reference reaches 0, destroy the whole daemon @@ -425,6 +428,7 @@ func (fs *Filesystem) Umount(ctx context.Context, snapshotID string) error { default: return errors.Errorf("unknown filesystem driver %s for snapshot %s", fsDriver, snapshotID) } + return nil } diff --git a/snapshot/process.go b/snapshot/process.go index cbfaa46214..8e002b0ee1 100644 --- a/snapshot/process.go +++ b/snapshot/process.go @@ -42,7 +42,7 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry, remoteHandler := func(id string, labels map[string]string) func() (bool, []mount.Mount, error) { return func() (bool, []mount.Mount, error) { logger.Debugf("Prepare remote snapshot %s", id) - if err := sn.fs.Mount(id, labels, &s); err != nil { + if err := sn.fs.Mount(ctx, id, labels, &s); err != nil { return false, nil, err } diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index c464d94d3b..9034620375 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -482,7 +482,7 @@ func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snap // Nydusd might not be running. We should run nydusd to reflect the rootfs. if err = o.fs.WaitUntilReady(pID); err != nil { if errors.Is(err, errdefs.ErrNotFound) { - if err := o.fs.Mount(pID, pInfo.Labels, nil); err != nil { + if err := o.fs.Mount(ctx, pID, pInfo.Labels, nil); err != nil { return nil, errors.Wrapf(err, "mount rafs, instance id %s", pID) } @@ -522,7 +522,7 @@ func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snap } } } - if err := o.fs.Mount(pID, pInfo.Labels, &s); err != nil { + if err := o.fs.Mount(ctx, pID, pInfo.Labels, &s); err != nil { return nil, errors.Wrapf(err, "mount tarfs, snapshot id %s", pID) } needRemoteMounts = true From 7d24dee6d2c5aa4815a2ef8aeff3b141ebf9b1bf Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Tue, 5 Sep 2023 23:16:33 +0800 Subject: [PATCH 3/8] tarfs: append Kata volume information to mount option Append Kata volume information to mount option - change value of annotation `containerd.io/snapshot/nydus-tarfs` from bool to tarfs export mode. - save tarfs meta information in Rafs.Annotation. - add "io.katacontainers.volume=xxx" to Mount.Options Signed-off-by: Jiang Liu --- config/config.go | 2 + config/global.go | 4 ++ misc/snapshotter/config.toml | 4 ++ pkg/filesystem/fs.go | 2 +- pkg/filesystem/tarfs_adaptor.go | 7 +++ pkg/label/label.go | 4 +- pkg/tarfs/tarfs.go | 52 +++++++++++++----- snapshot/mount_option.go | 96 +++++++++++++++++++++++++++++++++ snapshot/snapshot.go | 6 ++- 9 files changed, 159 insertions(+), 18 deletions(-) diff --git a/config/config.go b/config/config.go index 932bafe3a8..6875c55013 100644 --- a/config/config.go +++ b/config/config.go @@ -114,6 +114,7 @@ type Experimental struct { type TarfsConfig struct { EnableTarfs bool `toml:"enable_tarfs"` + MountTarfsOnHost bool `toml:"mount_tarfs_on_host"` TarfsHint bool `toml:"tarfs_hint"` MaxConcurrentProc int `toml:"max_concurrent_proc"` ExportMode string `toml:"export_mode"` @@ -156,6 +157,7 @@ type ImageConfig struct { // requests from containerd type SnapshotConfig struct { EnableNydusOverlayFS bool `toml:"enable_nydus_overlayfs"` + EnableKataVolume bool `toml:"enable_kata_volume"` SyncRemove bool `toml:"sync_remove"` } diff --git a/config/global.go b/config/global.go index a50de69f7b..31be05df56 100644 --- a/config/global.go +++ b/config/global.go @@ -121,6 +121,10 @@ const ( TarfsImageBlockWithVerity string = "image_block_with_verity" ) +func GetTarfsMountOnHost() bool { + return globalConfig.origin.Experimental.TarfsConfig.MountTarfsOnHost +} + func GetTarfsExportEnabled() bool { switch globalConfig.origin.Experimental.TarfsConfig.ExportMode { case TarfsLayerVerityOnly, TarfsLayerBlockDevice, TarfsLayerBlockWithVerity: diff --git a/misc/snapshotter/config.toml b/misc/snapshotter/config.toml index dbdf7ba4f6..f83aa9e464 100644 --- a/misc/snapshotter/config.toml +++ b/misc/snapshotter/config.toml @@ -83,6 +83,8 @@ enable_cri_keychain = false [snapshot] # Let containerd use nydus-overlayfs mount helper enable_nydus_overlayfs = false +# Insert Kata Virtual Volume option to `Mount.Options` +enable_kata_volume = false # Whether to remove resources when a snapshot is removed sync_remove = false @@ -109,6 +111,8 @@ enable_referrer_detect = false # - The EROFS filesystem driver since Linux 6.4 # - Nydus Image Service release v2.3 enable_tarfs = false +# Mount rafs on host by loopdev and EROFS +mount_tarfs_on_host = false # Only enable nydus tarfs mode for images with `tarfs hint` label when true tarfs_hint = false # Maximum of concurrence to converting OCIv1 images to tarfs, 0 means default diff --git a/pkg/filesystem/fs.go b/pkg/filesystem/fs.go index 0c998cc97f..3723dd4a7d 100644 --- a/pkg/filesystem/fs.go +++ b/pkg/filesystem/fs.go @@ -358,7 +358,7 @@ func (fs *Filesystem) Mount(ctx context.Context, snapshotID string, labels map[s err = errors.Wrapf(err, "mount file system by daemon %s, snapshot %s", d.ID(), snapshotID) } case config.FsDriverBlockdev: - err = fs.tarfsMgr.MountTarErofs(snapshotID, s, rafs) + err = fs.tarfsMgr.MountTarErofs(snapshotID, s, labels, rafs) if err != nil { err = errors.Wrapf(err, "mount tarfs for snapshot %s", snapshotID) } diff --git a/pkg/filesystem/tarfs_adaptor.go b/pkg/filesystem/tarfs_adaptor.go index 09b3aeaab1..fe7a9e4bdb 100755 --- a/pkg/filesystem/tarfs_adaptor.go +++ b/pkg/filesystem/tarfs_adaptor.go @@ -71,3 +71,10 @@ func (fs *Filesystem) ExportBlockData(s storage.Snapshot, perLayer bool, labels storageLocater func(string) string) ([]string, error) { return fs.tarfsMgr.ExportBlockData(s, perLayer, labels, storageLocater) } + +func (fs *Filesystem) GetTarfsImageDiskFilePath(id string) (string, error) { + if fs.tarfsMgr == nil { + return "", errors.New("tarfs mode is not enabled") + } + return fs.tarfsMgr.ImageDiskFilePath(id), nil +} diff --git a/pkg/label/label.go b/pkg/label/label.go index 948abe6da9..c6b065c0c0 100644 --- a/pkg/label/label.go +++ b/pkg/label/label.go @@ -42,9 +42,9 @@ const ( NydusImagePullUsername = "containerd.io/snapshot/pullusername" // A bool flag to enable integrity verification of meta data blob NydusSignature = "containerd.io/snapshot/nydus-signature" - // Information for image block device + // Dm-verity information for image block device NydusImageBlockInfo = "containerd.io/snapshot/nydus-image-block" - // Information for layer block device + // Dm-verity information for layer block device NydusLayerBlockInfo = "containerd.io/snapshot/nydus-layer-block" // A bool flag to mark the blob as a estargz data blob, set by the snapshotter. diff --git a/pkg/tarfs/tarfs.go b/pkg/tarfs/tarfs.go index ce6d677b8c..ed1831d279 100755 --- a/pkg/tarfs/tarfs.go +++ b/pkg/tarfs/tarfs.go @@ -497,7 +497,7 @@ func (t *Manager) ExportBlockData(s storage.Snapshot, perLayer bool, labels map[ var metaFileName, diskFileName string if wholeImage { metaFileName = t.imageMetaFilePath(storageLocater(snapshotID)) - diskFileName = t.imageDiskFilePath(st.blobID) + diskFileName = t.ImageDiskFilePath(st.blobID) } else { metaFileName = t.layerMetaFilePath(storageLocater(snapshotID)) diskFileName = t.layerDiskFilePath(st.blobID) @@ -531,6 +531,7 @@ func (t *Manager) ExportBlockData(s storage.Snapshot, perLayer bool, labels map[ } log.L.Debugf("nydus image export command, stdout: %s, stderr: %s", &outb, &errb) + blockInfo := "" if withVerity { pattern := "dm-verity options: --no-superblock --format=1 -s \"\" --hash=sha256 --data-block-size=512 --hash-block-size=4096 --data-blocks %d --hash-offset %d %s\n" var dataBlobks, hashOffset uint64 @@ -538,17 +539,16 @@ func (t *Manager) ExportBlockData(s storage.Snapshot, perLayer bool, labels map[ if count, err := fmt.Sscanf(outb.String(), pattern, &dataBlobks, &hashOffset, &rootHash); err != nil || count != 3 { return updateFields, errors.Errorf("failed to parse dm-verity options from nydus image output: %s", outb.String()) } - - blockInfo := strconv.FormatUint(dataBlobks, 10) + "," + strconv.FormatUint(hashOffset, 10) + "," + "sha256:" + rootHash - if wholeImage { - labels[label.NydusImageBlockInfo] = blockInfo - updateFields = append(updateFields, "labels."+label.NydusImageBlockInfo) - } else { - labels[label.NydusLayerBlockInfo] = blockInfo - updateFields = append(updateFields, "labels."+label.NydusLayerBlockInfo) - } - log.L.Warnf("export block labels %v", labels) + blockInfo = strconv.FormatUint(dataBlobks, 10) + "," + strconv.FormatUint(hashOffset, 10) + "," + "sha256:" + rootHash } + if wholeImage { + labels[label.NydusImageBlockInfo] = blockInfo + updateFields = append(updateFields, "labels."+label.NydusImageBlockInfo) + } else { + labels[label.NydusLayerBlockInfo] = blockInfo + updateFields = append(updateFields, "labels."+label.NydusLayerBlockInfo) + } + log.L.Debugf("export block labels %v", labels) err = os.Rename(diskFileNameTmp, diskFileName) if err != nil { @@ -558,12 +558,20 @@ func (t *Manager) ExportBlockData(s storage.Snapshot, perLayer bool, labels map[ return updateFields, nil } -func (t *Manager) MountTarErofs(snapshotID string, s *storage.Snapshot, rafs *rafs.Rafs) error { +func (t *Manager) MountTarErofs(snapshotID string, s *storage.Snapshot, labels map[string]string, rafs *rafs.Rafs) error { if s == nil { return errors.New("snapshot object for MountTarErofs() is nil") } + // Copy meta info from snapshot to rafs + t.copyTarfsAnnotations(labels, rafs) + upperDirPath := path.Join(rafs.GetSnapshotDir(), "fs") + if !config.GetTarfsMountOnHost() { + rafs.SetMountpoint(upperDirPath) + return nil + } + mergedBootstrap := t.imageMetaFilePath(upperDirPath) blobInfo, err := t.getImageBlobInfo(mergedBootstrap) if err != nil { @@ -654,8 +662,9 @@ func (t *Manager) UmountTarErofs(snapshotID string) error { if err != nil { return errors.Wrapf(err, "umount erofs tarfs %s", st.erofsMountPoint) } + st.erofsMountPoint = "" } - st.erofsMountPoint = "" + return nil } @@ -671,6 +680,7 @@ func (t *Manager) DetachLayer(snapshotID string) error { st.mutex.Unlock() return errors.Wrapf(err, "umount erofs tarfs %s", st.erofsMountPoint) } + st.erofsMountPoint = "" } if st.metaLoopdev != nil { @@ -788,6 +798,20 @@ func (t *Manager) GetConcurrentLimiter(ref string) *semaphore.Weighted { return limiter } +func (t *Manager) copyTarfsAnnotations(labels map[string]string, rafs *rafs.Rafs) { + keys := []string{ + label.NydusTarfsLayer, + label.NydusImageBlockInfo, + label.NydusLayerBlockInfo, + } + + for _, k := range keys { + if v, ok := labels[k]; ok { + rafs.AddAnnotation(k, v) + } + } +} + func (t *Manager) layerTarFilePath(blobID string) string { return filepath.Join(t.cacheDirPath, blobID) } @@ -796,7 +820,7 @@ func (t *Manager) layerDiskFilePath(blobID string) string { return filepath.Join(t.cacheDirPath, blobID+"."+TarfsLayerDiskName) } -func (t *Manager) imageDiskFilePath(blobID string) string { +func (t *Manager) ImageDiskFilePath(blobID string) string { return filepath.Join(t.cacheDirPath, blobID+"."+TarfsImageDiskName) } diff --git a/snapshot/mount_option.go b/snapshot/mount_option.go index da6bb9b37a..39b01c40d1 100644 --- a/snapshot/mount_option.go +++ b/snapshot/mount_option.go @@ -19,6 +19,7 @@ import ( "github.com/containerd/containerd/mount" "github.com/containerd/containerd/snapshots/storage" "github.com/containerd/nydus-snapshotter/config/daemonconfig" + "github.com/containerd/nydus-snapshotter/pkg/label" "github.com/containerd/nydus-snapshotter/pkg/layout" "github.com/containerd/nydus-snapshotter/pkg/rafs" "github.com/pkg/errors" @@ -100,6 +101,101 @@ func (o *snapshotter) remoteMountWithExtraOptions(ctx context.Context, s storage }, nil } +func (o *snapshotter) mountWithKataVolume(ctx context.Context, s storage.Snapshot, id string, + overlayOptions []string) ([]mount.Mount, error) { + hasVolume := false + rafs := rafs.RafsGlobalCache.Get(id) + if rafs == nil { + return []mount.Mount{}, errors.Errorf("failed to find RAFS instance for snapshot %s", id) + } + + // Insert Kata volume for tarfs + if label.IsTarfsDataLayer(rafs.Annotations) { + options, err := o.mountWithTarfsVolume(*rafs, id) + if err != nil { + return []mount.Mount{}, errors.Wrapf(err, "create kata volume for tarfs") + } + if len(options) > 0 { + overlayOptions = append(overlayOptions, options...) + hasVolume = true + } + } + + if hasVolume { + log.G(ctx).Debugf("fuse.nydus-overlayfs mount options %v", overlayOptions) + return []mount.Mount{ + { + Type: "fuse.nydus-overlayfs", + Source: "overlay", + Options: overlayOptions, + }, + }, nil + } + + return overlayMount(overlayOptions), nil +} + +func (o *snapshotter) mountWithTarfsVolume(rafs rafs.Rafs, id string) ([]string, error) { + var volume *KataVirtualVolume + + if info, ok := rafs.Annotations[label.NydusImageBlockInfo]; ok { + path, err := o.fs.GetTarfsImageDiskFilePath(id) + if err != nil { + return []string{}, errors.Wrapf(err, "get tarfs image disk file path") + } + volume = &KataVirtualVolume{ + VolumeType: KataVirtualVolumeImageRawBlockType, + Source: path, + FSType: "erofs", + Options: []string{"ro"}, + } + if len(info) > 0 { + dmverity, err := parseTarfsDmVerityInfo(info) + if err != nil { + return []string{}, err + } + volume.DmVerity = &dmverity + } + } + + if volume != nil { + if !volume.Validate() { + return []string{}, errors.Errorf("got invalid kata volume, %v", volume) + } + info, err := EncodeKataVirtualVolumeToBase64(*volume) + if err != nil { + return []string{}, errors.Errorf("failed to encoding Kata Volume info %v", volume) + } + opt := fmt.Sprintf("%s=%s", KataVirtualVolumeOptionName, info) + return []string{opt}, nil + } + + return []string{}, nil +} + +func parseTarfsDmVerityInfo(info string) (DmVerityInfo, error) { + var dataBlocks, hashOffset uint64 + var rootHash string + + pattern := "%d,%d,sha256:%s" + if count, err := fmt.Sscanf(info, pattern, &dataBlocks, &hashOffset, &rootHash); err == nil && count == 3 { + di := DmVerityInfo{ + HashType: "sha256", + Hash: rootHash, + BlockNum: dataBlocks, + Blocksize: 512, + Hashsize: 4096, + Offset: hashOffset, + } + if err := di.Validate(); err != nil { + return DmVerityInfo{}, errors.Wrap(err, "validate dm-verity information") + } + return di, nil + } + + return DmVerityInfo{}, errors.Errorf("invalid dm-verity information: %s", info) +} + // Consts and data structures for Kata Virtual Volume const ( minBlockSize = 1 << 9 diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 9034620375..e6dd5e6f06 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -55,6 +55,7 @@ type snapshotter struct { fs *filesystem.Filesystem cgroupManager *cgroup.Manager enableNydusOverlayFS bool + enableKataVolume bool syncRemove bool cleanupOnClose bool } @@ -283,6 +284,7 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho fs: nydusFs, cgroupManager: cgroupMgr, enableNydusOverlayFS: cfg.SnapshotsConfig.EnableNydusOverlayFS, + enableKataVolume: cfg.SnapshotsConfig.EnableKataVolume, cleanupOnClose: cfg.CleanupOnClose, }, nil } @@ -846,11 +848,13 @@ func (o *snapshotter) mountRemote(ctx context.Context, labels map[string]string, overlayOptions = append(overlayOptions, lowerDirOption) log.G(ctx).Infof("remote mount options %v", overlayOptions) + if o.enableKataVolume { + return o.mountWithKataVolume(ctx, s, id, overlayOptions) + } // Add `extraoption` if NydusOverlayFS is enable or daemonMode is `None` if o.enableNydusOverlayFS || config.GetDaemonMode() == config.DaemonModeNone { return o.remoteMountWithExtraOptions(ctx, s, id, overlayOptions) } - return overlayMount(overlayOptions), nil } From 6683fb0baf4c113d2250b8034c967a63b7b6e620 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Wed, 6 Sep 2023 13:57:21 +0800 Subject: [PATCH 4/8] tarfs: add a sample configuration file for tarfs Create a minimal configuration file for tarfs - treat tarfs as DaemonMode == "none" - create rafs instance for DaemonMode == "none" Signed-off-by: Jiang Liu --- config/config.go | 4 +- config/global.go | 4 ++ .../snapshotter/config-coco-host-sharing.toml | 40 +++++++++++++++++++ pkg/filesystem/fs.go | 35 +++++----------- pkg/filesystem/tarfs_adaptor.go | 4 ++ pkg/label/label.go | 10 ++--- pkg/manager/manager.go | 4 +- pkg/tarfs/tarfs.go | 11 +++-- snapshot/mount_option.go | 11 +++-- snapshot/process.go | 1 - snapshot/snapshot.go | 32 +++++++++------ 11 files changed, 101 insertions(+), 55 deletions(-) create mode 100644 misc/snapshotter/config-coco-host-sharing.toml diff --git a/config/config.go b/config/config.go index 6875c55013..57b3a85ead 100644 --- a/config/config.go +++ b/config/config.go @@ -184,6 +184,7 @@ type AuthConfig struct { type RemoteConfig struct { AuthConfig AuthConfig `toml:"auth"` ConvertVpcRegistry bool `toml:"convert_vpc_registry"` + SkipSSLVerify bool `toml:"skip_ssl_verify"` MirrorsConfig MirrorsConfig `toml:"mirrors_config"` } @@ -274,7 +275,8 @@ func ValidateConfig(c *SnapshotterConfig) error { return errors.New("empty root directory") } - if c.DaemonConfig.FsDriver != FsDriverFscache && c.DaemonConfig.FsDriver != FsDriverFusedev { + if c.DaemonConfig.FsDriver != FsDriverFscache && c.DaemonConfig.FsDriver != FsDriverFusedev && + c.DaemonConfig.FsDriver != FsDriverBlockdev && c.DaemonConfig.FsDriver != FsDriverNodev { return errors.Errorf("invalid filesystem driver %q", c.DaemonConfig.FsDriver) } if _, err := ParseRecoverPolicy(c.DaemonConfig.RecoverPolicy); err != nil { diff --git a/config/global.go b/config/global.go index 31be05df56..d866c4dce6 100644 --- a/config/global.go +++ b/config/global.go @@ -112,6 +112,10 @@ func GetDaemonProfileCPUDuration() int64 { return globalConfig.origin.SystemControllerConfig.DebugConfig.ProfileDuration } +func GetSkipSSLVerify() bool { + return globalConfig.origin.RemoteConfig.SkipSSLVerify +} + const ( TarfsLayerVerityOnly string = "layer_verity_only" TarfsImageVerityOnly string = "image_verity_only" diff --git a/misc/snapshotter/config-coco-host-sharing.toml b/misc/snapshotter/config-coco-host-sharing.toml new file mode 100644 index 0000000000..9d8313cc37 --- /dev/null +++ b/misc/snapshotter/config-coco-host-sharing.toml @@ -0,0 +1,40 @@ + +version = 1 +# Snapshotter's own home directory where it stores and creates necessary resources +root = "/var/lib/containerd-nydus" +# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket +address = "/run/containerd-nydus/containerd-nydus-grpc.sock" +# No nydusd daemon needed +daemon_mode = "none" + +[daemon] +# Use `blockdev` for tarfs +fs_driver = "blockdev" +# Path to nydus-image binary +nydusimage_path = "/usr/local/bin/nydus-image" + +[remote] +skip_ssl_verify = true + +[snapshot] +# Insert Kata volume information to `Mount.Options` +enable_kata_volume = true + +[experimental.tarfs] +# Whether to enable nydus tarfs mode. Tarfs is supported by: +# - The EROFS filesystem driver since Linux 6.4 +# - Nydus Image Service release v2.3 +enable_tarfs = true + +# Mount rafs on host by loopdev and EROFS +mount_tarfs_on_host = false + +# Mode to export tarfs images: +# - "none" or "": do not export tarfs +# - "layer_verity_only": only generate disk verity information for a layer blob +# - "image_verity_only": only generate disk verity information for all blobs of an image +# - "layer_block": generate a raw block disk image with tarfs for a layer +# - "image_block": generate a raw block disk image with tarfs for an image +# - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info +# - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info +export_mode = "image_block_with_verity" \ No newline at end of file diff --git a/pkg/filesystem/fs.go b/pkg/filesystem/fs.go index 3723dd4a7d..a1beae63b7 100644 --- a/pkg/filesystem/fs.go +++ b/pkg/filesystem/fs.go @@ -199,13 +199,12 @@ func (fs *Filesystem) TryStopSharedDaemon() { // WaitUntilReady wait until daemon ready by snapshotID, it will wait until nydus domain socket established // and the status of nydusd daemon must be ready func (fs *Filesystem) WaitUntilReady(snapshotID string) error { - // If NoneDaemon mode, there's no need to wait for daemon ready - if !fs.DaemonBacked() { - return nil - } - rafs := racache.RafsGlobalCache.Get(snapshotID) if rafs == nil { + // If NoneDaemon mode, there's no need to wait for daemon ready + if config.GetDaemonMode() == config.DaemonModeNone { + return nil + } return errors.Wrapf(errdefs.ErrNotFound, "no instance %s", snapshotID) } @@ -229,11 +228,6 @@ func (fs *Filesystem) WaitUntilReady(snapshotID string) error { // this method will fork nydus daemon and manage it in the internal store, and indexed by snapshotID // It must set up all necessary resources during Mount procedure and revoke any step if necessary. func (fs *Filesystem) Mount(ctx context.Context, snapshotID string, labels map[string]string, s *storage.Snapshot) (err error) { - // Do not create RAFS instance in case of nodev. - if !fs.DaemonBacked() { - return nil - } - rafs := racache.RafsGlobalCache.Get(snapshotID) if rafs != nil { // Instance already exists, how could this happen? Can containerd handle this case? @@ -309,7 +303,7 @@ func (fs *Filesystem) Mount(ctx context.Context, snapshotID string, labels map[s daemonconfig.WorkDir: workDir, daemonconfig.CacheDir: cacheDir, } - cfg := deepcopy.Copy(fsManager.DaemonConfig).(daemonconfig.DaemonConfig) + cfg := deepcopy.Copy(*fsManager.DaemonConfig).(daemonconfig.DaemonConfig) err = daemonconfig.SupplementDaemonConfig(cfg, imageID, snapshotID, false, labels, params) if err != nil { return errors.Wrap(err, "supplement configuration") @@ -362,6 +356,8 @@ func (fs *Filesystem) Mount(ctx context.Context, snapshotID string, labels map[s if err != nil { err = errors.Wrapf(err, "mount tarfs for snapshot %s", snapshotID) } + case config.FsDriverNodev: + // Nothing to do default: err = errors.Errorf("unknown filesystem driver %s for snapshot %s", fsDriver, snapshotID) } @@ -492,14 +488,6 @@ func (fs *Filesystem) Teardown(ctx context.Context) error { } func (fs *Filesystem) MountPoint(snapshotID string) (string, error) { - if !fs.DaemonBacked() { - // For NoneDaemon mode, return a dummy mountpoint which is very likely not - // existed on host. NoneDaemon mode does not start nydusd, so NO fuse mount is - // ever performed. Only mount option carries meaningful info to containerd and - // finally passes to shim. - return fs.rootMountpoint, nil - } - rafs := racache.RafsGlobalCache.Get(snapshotID) if rafs != nil { return rafs.GetMountpoint(), nil @@ -510,6 +498,9 @@ func (fs *Filesystem) MountPoint(snapshotID string) (string, error) { func (fs *Filesystem) BootstrapFile(id string) (string, error) { rafs := racache.RafsGlobalCache.Get(id) + if rafs == nil { + return "", errors.Errorf("no RAFS instance for %s", id) + } return rafs.BootstrapFile() } @@ -593,7 +584,7 @@ func (fs *Filesystem) initSharedDaemon(fsManager *manager.Manager) (err error) { // Shared nydusd daemon does not need configuration to start process but // it is loaded when requesting mount api // Dump the configuration file since it is reloaded when recovering the nydusd - d.Config = fsManager.DaemonConfig + d.Config = *fsManager.DaemonConfig err = d.Config.DumpFile(d.ConfigFile("")) if err != nil && !errors.Is(err, errdefs.ErrAlreadyExists) { return errors.Wrapf(err, "dump configuration file %s", d.ConfigFile("")) @@ -651,10 +642,6 @@ func (fs *Filesystem) createDaemon(fsManager *manager.Manager, daemonMode config return d, nil } -func (fs *Filesystem) DaemonBacked() bool { - return config.GetDaemonMode() != config.DaemonModeNone -} - func (fs *Filesystem) getManager(fsDriver string) (*manager.Manager, error) { switch fsDriver { case config.FsDriverBlockdev: diff --git a/pkg/filesystem/tarfs_adaptor.go b/pkg/filesystem/tarfs_adaptor.go index fe7a9e4bdb..3e47295b38 100755 --- a/pkg/filesystem/tarfs_adaptor.go +++ b/pkg/filesystem/tarfs_adaptor.go @@ -12,6 +12,7 @@ import ( "github.com/containerd/containerd/log" snpkg "github.com/containerd/containerd/pkg/snapshotters" "github.com/containerd/containerd/snapshots/storage" + "github.com/containerd/nydus-snapshotter/pkg/label" "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) @@ -56,6 +57,9 @@ func (fs *Filesystem) PrepareTarfsLayer(ctx context.Context, labels map[string]s limiter.Release(1) } + layerBlobID := layerDigest.Hex() + labels[label.NydusTarfsLayer] = layerBlobID + return nil } diff --git a/pkg/label/label.go b/pkg/label/label.go index c6b065c0c0..30e204992f 100644 --- a/pkg/label/label.go +++ b/pkg/label/label.go @@ -34,18 +34,18 @@ const ( NydusMetaLayer = "containerd.io/snapshot/nydus-bootstrap" // The referenced blob sha256 in format of `sha256:xxx`, set by image builders. NydusRefLayer = "containerd.io/snapshot/nydus-ref" - // A bool flag to mark the layer as a nydus tarfs, set by the snapshotter + // The blobID of associated layer, also marking the layer as a nydus tarfs, set by the snapshotter NydusTarfsLayer = "containerd.io/snapshot/nydus-tarfs" + // Dm-verity information for image block device + NydusImageBlockInfo = "containerd.io/snapshot/nydus-image-block" + // Dm-verity information for layer block device + NydusLayerBlockInfo = "containerd.io/snapshot/nydus-layer-block" // Annotation containing secret to pull images from registry, set by the snapshotter. NydusImagePullSecret = "containerd.io/snapshot/pullsecret" // Annotation containing username to pull images from registry, set by the snapshotter. NydusImagePullUsername = "containerd.io/snapshot/pullusername" // A bool flag to enable integrity verification of meta data blob NydusSignature = "containerd.io/snapshot/nydus-signature" - // Dm-verity information for image block device - NydusImageBlockInfo = "containerd.io/snapshot/nydus-image-block" - // Dm-verity information for layer block device - NydusLayerBlockInfo = "containerd.io/snapshot/nydus-layer-block" // A bool flag to mark the blob as a estargz data blob, set by the snapshotter. StargzLayer = "containerd.io/snapshot/stargz" diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index eae219ce56..27b6fad7d6 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -42,7 +42,7 @@ type Manager struct { // The `daemonCache` is cache for nydusd daemons stored in `store`. // You should update `store` first before modifying cached state. daemonCache *DaemonCache - DaemonConfig daemonconfig.DaemonConfig // Daemon configuration template. + DaemonConfig *daemonconfig.DaemonConfig // Daemon configuration template. CgroupMgr *cgroup.Manager monitor LivenessMonitor LivenessNotifier chan deathEvent // TODO: Close me @@ -54,7 +54,7 @@ type Manager struct { type Opt struct { CacheDir string CgroupMgr *cgroup.Manager - DaemonConfig daemonconfig.DaemonConfig + DaemonConfig *daemonconfig.DaemonConfig Database *store.Database FsDriver string NydusdBinaryPath string diff --git a/pkg/tarfs/tarfs.go b/pkg/tarfs/tarfs.go index ed1831d279..b2f135e2cc 100755 --- a/pkg/tarfs/tarfs.go +++ b/pkg/tarfs/tarfs.go @@ -494,13 +494,18 @@ func (t *Manager) ExportBlockData(s storage.Snapshot, perLayer bool, labels map[ return updateFields, errors.Errorf("tarfs snapshot %s is not ready, %d", snapshotID, st.status) } + blobID, ok := labels[label.NydusTarfsLayer] + if !ok { + return updateFields, errors.Errorf("Missing Nydus tarfs layer annotation for snapshot %s", s.ID) + } + var metaFileName, diskFileName string if wholeImage { metaFileName = t.imageMetaFilePath(storageLocater(snapshotID)) - diskFileName = t.ImageDiskFilePath(st.blobID) + diskFileName = t.ImageDiskFilePath(blobID) } else { metaFileName = t.layerMetaFilePath(storageLocater(snapshotID)) - diskFileName = t.layerDiskFilePath(st.blobID) + diskFileName = t.layerDiskFilePath(blobID) } // Do not regenerate if the disk image already exists. @@ -520,7 +525,7 @@ func (t *Manager) ExportBlockData(s storage.Snapshot, perLayer bool, labels map[ if withVerity { options = append(options, "--verity") } - log.L.Warnf("nydus image command %v", options) + log.L.Debugf("nydus image command %v", options) cmd := exec.Command(t.nydusImagePath, options...) var errb, outb bytes.Buffer cmd.Stderr = &errb diff --git a/snapshot/mount_option.go b/snapshot/mount_option.go index 39b01c40d1..94f69954ce 100644 --- a/snapshot/mount_option.go +++ b/snapshot/mount_option.go @@ -101,8 +101,7 @@ func (o *snapshotter) remoteMountWithExtraOptions(ctx context.Context, s storage }, nil } -func (o *snapshotter) mountWithKataVolume(ctx context.Context, s storage.Snapshot, id string, - overlayOptions []string) ([]mount.Mount, error) { +func (o *snapshotter) mountWithKataVolume(ctx context.Context, id string, overlayOptions []string) ([]mount.Mount, error) { hasVolume := false rafs := rafs.RafsGlobalCache.Get(id) if rafs == nil { @@ -110,8 +109,8 @@ func (o *snapshotter) mountWithKataVolume(ctx context.Context, s storage.Snapsho } // Insert Kata volume for tarfs - if label.IsTarfsDataLayer(rafs.Annotations) { - options, err := o.mountWithTarfsVolume(*rafs, id) + if blobID, ok := rafs.Annotations[label.NydusTarfsLayer]; ok { + options, err := o.mountWithTarfsVolume(*rafs, blobID) if err != nil { return []mount.Mount{}, errors.Wrapf(err, "create kata volume for tarfs") } @@ -135,11 +134,11 @@ func (o *snapshotter) mountWithKataVolume(ctx context.Context, s storage.Snapsho return overlayMount(overlayOptions), nil } -func (o *snapshotter) mountWithTarfsVolume(rafs rafs.Rafs, id string) ([]string, error) { +func (o *snapshotter) mountWithTarfsVolume(rafs rafs.Rafs, blobID string) ([]string, error) { var volume *KataVirtualVolume if info, ok := rafs.Annotations[label.NydusImageBlockInfo]; ok { - path, err := o.fs.GetTarfsImageDiskFilePath(id) + path, err := o.fs.GetTarfsImageDiskFilePath(blobID) if err != nil { return []string{}, errors.Wrapf(err, "get tarfs image disk file path") } diff --git a/snapshot/process.go b/snapshot/process.go index 8e002b0ee1..fe24bdb170 100644 --- a/snapshot/process.go +++ b/snapshot/process.go @@ -101,7 +101,6 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry, return nil, "", errors.Wrap(err, "export layer as tarfs block device") } } - labels[label.NydusTarfsLayer] = "true" handler = skipHandler } } diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index e6dd5e6f06..8e33509a42 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -66,11 +66,6 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho return nil, errors.Wrap(err, "initialize image verifier") } - daemonConfig, err := daemonconfig.NewDaemonConfig(config.GetFsDriver(), cfg.DaemonConfig.NydusdConfigPath) - if err != nil { - return nil, errors.Wrap(err, "load daemon configuration") - } - db, err := store.NewDatabase(cfg.Root) if err != nil { return nil, errors.Wrap(err, "create database") @@ -98,6 +93,21 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho } } + var skipSSLVerify bool + var daemonConfig *daemonconfig.DaemonConfig + fsDriver := config.GetFsDriver() + if fsDriver == config.FsDriverFscache || fsDriver == config.FsDriverFusedev { + config, err := daemonconfig.NewDaemonConfig(config.GetFsDriver(), cfg.DaemonConfig.NydusdConfigPath) + if err != nil { + return nil, errors.Wrap(err, "load daemon configuration") + } + daemonConfig = &config + _, backendConfig := config.StorageBackend() + skipSSLVerify = backendConfig.SkipVerify + } else { + skipSSLVerify = config.GetSkipSSLVerify() + } + var blockdevManager *mgr.Manager if cfg.Experimental.TarfsConfig.EnableTarfs { blockdevManager, err = mgr.NewManager(mgr.Opt{ @@ -107,7 +117,7 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho RootDir: cfg.Root, RecoverPolicy: rp, FsDriver: config.FsDriverBlockdev, - DaemonConfig: daemonConfig, + DaemonConfig: nil, CgroupMgr: cgroupMgr, }) if err != nil { @@ -199,16 +209,12 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho } if cfg.Experimental.EnableReferrerDetect { - // FIXME: get the insecure option from nydusd config. - _, backendConfig := daemonConfig.StorageBackend() - referrerMgr := referrer.NewManager(backendConfig.SkipVerify) + referrerMgr := referrer.NewManager(skipSSLVerify) opts = append(opts, filesystem.WithReferrerManager(referrerMgr)) } if cfg.Experimental.TarfsConfig.EnableTarfs { - // FIXME: get the insecure option from nydusd config. - _, backendConfig := daemonConfig.StorageBackend() - tarfsMgr := tarfs.NewManager(backendConfig.SkipVerify, cfg.Experimental.TarfsConfig.TarfsHint, + tarfsMgr := tarfs.NewManager(skipSSLVerify, cfg.Experimental.TarfsConfig.TarfsHint, cacheConfig.CacheDir, cfg.DaemonConfig.NydusImagePath, int64(cfg.Experimental.TarfsConfig.MaxConcurrentProc)) opts = append(opts, filesystem.WithTarfsManager(tarfsMgr)) @@ -849,7 +855,7 @@ func (o *snapshotter) mountRemote(ctx context.Context, labels map[string]string, log.G(ctx).Infof("remote mount options %v", overlayOptions) if o.enableKataVolume { - return o.mountWithKataVolume(ctx, s, id, overlayOptions) + return o.mountWithKataVolume(ctx, id, overlayOptions) } // Add `extraoption` if NydusOverlayFS is enable or daemonMode is `None` if o.enableNydusOverlayFS || config.GetDaemonMode() == config.DaemonModeNone { From 57f088de29c00ed73ef3b3d7bc173178c5d82e34 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Fri, 8 Sep 2023 10:58:44 +0800 Subject: [PATCH 5/8] filesystem: use hashmap to manage registered managers Simplify code by using hashmap to manage registered managers. Signed-off-by: Jiang Liu --- pkg/filesystem/config.go | 19 +++---- pkg/filesystem/fs.go | 112 ++++++++++++++++----------------------- pkg/metrics/serve.go | 6 +-- snapshot/snapshot.go | 35 ++++-------- 4 files changed, 64 insertions(+), 108 deletions(-) diff --git a/pkg/filesystem/config.go b/pkg/filesystem/config.go index 4f2fb668bd..53386f6a9c 100644 --- a/pkg/filesystem/config.go +++ b/pkg/filesystem/config.go @@ -8,7 +8,6 @@ package filesystem import ( - "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/pkg/cache" "github.com/containerd/nydus-snapshotter/pkg/manager" "github.com/containerd/nydus-snapshotter/pkg/referrer" @@ -27,20 +26,14 @@ func WithNydusImageBinaryPath(p string) NewFSOpt { } } -func WithManager(pm *manager.Manager) NewFSOpt { +func WithManagers(managers []*manager.Manager) NewFSOpt { return func(fs *Filesystem) error { - if pm != nil { - switch pm.FsDriver { - case config.FsDriverBlockdev: - fs.blockdevManager = pm - case config.FsDriverFscache: - fs.fscacheManager = pm - case config.FsDriverFusedev: - fs.fusedevManager = pm - } - fs.enabledManagers = append(fs.enabledManagers, pm) + if fs.enabledManagers == nil { + fs.enabledManagers = map[string]*manager.Manager{} + } + for _, pm := range managers { + fs.enabledManagers[pm.FsDriver] = pm } - return nil } } diff --git a/pkg/filesystem/fs.go b/pkg/filesystem/fs.go index a1beae63b7..403bbc3073 100644 --- a/pkg/filesystem/fs.go +++ b/pkg/filesystem/fs.go @@ -40,15 +40,10 @@ import ( "github.com/containerd/nydus-snapshotter/pkg/tarfs" ) -// TODO: refact `enabledManagers` and `xxxManager` into `ManagerCoordinator` type Filesystem struct { fusedevSharedDaemon *daemon.Daemon fscacheSharedDaemon *daemon.Daemon - blockdevManager *manager.Manager - fusedevManager *manager.Manager - fscacheManager *manager.Manager - nodevManager *manager.Manager - enabledManagers []*manager.Manager + enabledManagers map[string]*manager.Manager cacheMgr *cache.Manager referrerMgr *referrer.Manager stargzResolver *stargz.Resolver @@ -104,25 +99,25 @@ func NewFileSystem(ctx context.Context, opt ...NewFSOpt) (*Filesystem, error) { // a new nydusd for it. // TODO: We still need to consider shared daemon the time sequence of initializing daemon, // start daemon commit its state to DB and retrieving its state. - if fs.fscacheManager == nil { - if hasFscacheSharedDaemon { - return nil, errors.Errorf("shared fscache daemon is present, but manager is missing") - } - } else if !hasFscacheSharedDaemon && fs.fscacheSharedDaemon == nil { - log.L.Infof("initializing shared nydus daemon for fscache") - if err := fs.initSharedDaemon(fs.fscacheManager); err != nil { - return nil, errors.Wrap(err, "start shared nydusd daemon for fscache") + if fscacheManager, ok := fs.enabledManagers[config.FsDriverFscache]; ok { + if !hasFscacheSharedDaemon && fs.fscacheSharedDaemon == nil { + log.L.Infof("initializing shared nydus daemon for fscache") + if err := fs.initSharedDaemon(fscacheManager); err != nil { + return nil, errors.Wrap(err, "start shared nydusd daemon for fscache") + } } + } else if hasFscacheSharedDaemon { + return nil, errors.Errorf("shared fscache daemon is present, but manager is missing") } - if fs.fusedevManager == nil { - if hasFusedevSharedDaemon { - return nil, errors.Errorf("shared fusedev daemon is present, but manager is missing") - } - } else if config.IsFusedevSharedModeEnabled() && !hasFusedevSharedDaemon && fs.fusedevSharedDaemon == nil { - log.L.Infof("initializing shared nydus daemon for fusedev") - if err := fs.initSharedDaemon(fs.fusedevManager); err != nil { - return nil, errors.Wrap(err, "start shared nydusd daemon for fusedev") + if fusedevManager, ok := fs.enabledManagers[config.FsDriverFusedev]; ok { + if config.IsFusedevSharedModeEnabled() && !hasFusedevSharedDaemon && fs.fusedevSharedDaemon == nil { + log.L.Infof("initializing shared nydus daemon for fusedev") + if err := fs.initSharedDaemon(fusedevManager); err != nil { + return nil, errors.Wrap(err, "start shared nydusd daemon for fusedev") + } } + } else if hasFusedevSharedDaemon { + return nil, errors.Errorf("shared fusedev daemon is present, but manager is missing") } // Try to bring all persisted and stopped nydusd up and remount Rafs @@ -178,19 +173,23 @@ func (fs *Filesystem) TryRetainSharedDaemon(d *daemon.Daemon) { func (fs *Filesystem) TryStopSharedDaemon() { if fs.fusedevSharedDaemon != nil { if fs.fusedevSharedDaemon.GetRef() == 1 { - if err := fs.fusedevManager.DestroyDaemon(fs.fusedevSharedDaemon); err != nil { - log.L.WithError(err).Errorf("Terminate shared daemon %s failed", fs.fusedevSharedDaemon.ID()) - } else { - fs.fusedevSharedDaemon = nil + if fusedevManager, ok := fs.enabledManagers[config.FsDriverFusedev]; ok { + if err := fusedevManager.DestroyDaemon(fs.fusedevSharedDaemon); err != nil { + log.L.WithError(err).Errorf("Terminate shared daemon %s failed", fs.fusedevSharedDaemon.ID()) + } else { + fs.fusedevSharedDaemon = nil + } } } } if fs.fscacheSharedDaemon != nil { if fs.fscacheSharedDaemon.GetRef() == 1 { - if err := fs.fscacheManager.DestroyDaemon(fs.fscacheSharedDaemon); err != nil { - log.L.WithError(err).Errorf("Terminate shared daemon %s failed", fs.fscacheSharedDaemon.ID()) - } else { - fs.fscacheSharedDaemon = nil + if fscacheManager, ok := fs.enabledManagers[config.FsDriverFscache]; ok { + if err := fscacheManager.DestroyDaemon(fs.fscacheSharedDaemon); err != nil { + log.L.WithError(err).Errorf("Terminate shared daemon %s failed", fs.fscacheSharedDaemon.ID()) + } else { + fs.fscacheSharedDaemon = nil + } } } } @@ -450,17 +449,19 @@ func (fs *Filesystem) RemoveCache(blobDigest string) error { } blobID := digest.Hex() - if fs.fscacheManager != nil { - c, err := fs.fscacheSharedDaemon.GetClient() - if err != nil { - return err - } - // delete fscache blob cache file - // TODO: skip error for blob not existing - if err := c.UnbindBlob("", blobID); err != nil { - return err + if fscacheManager, ok := fs.enabledManagers[config.FsDriverFscache]; ok { + if fscacheManager != nil { + c, err := fs.fscacheSharedDaemon.GetClient() + if err != nil { + return err + } + // delete fscache blob cache file + // TODO: skip error for blob not existing + if err := c.UnbindBlob("", blobID); err != nil { + return err + } + return nil } - return nil } return fs.cacheMgr.RemoveBlobCache(blobID) @@ -643,23 +644,8 @@ func (fs *Filesystem) createDaemon(fsManager *manager.Manager, daemonMode config } func (fs *Filesystem) getManager(fsDriver string) (*manager.Manager, error) { - switch fsDriver { - case config.FsDriverBlockdev: - if fs.blockdevManager != nil { - return fs.blockdevManager, nil - } - case config.FsDriverFscache: - if fs.fscacheManager != nil { - return fs.fscacheManager, nil - } - case config.FsDriverFusedev: - if fs.fusedevManager != nil { - return fs.fusedevManager, nil - } - case config.FsDriverNodev: - if fs.nodevManager != nil { - return fs.nodevManager, nil - } + if fsManager, ok := fs.enabledManagers[fsDriver]; ok { + return fsManager, nil } return nil, errors.Errorf("no manager for filesystem driver %s", fsDriver) @@ -682,15 +668,9 @@ func (fs *Filesystem) getSharedDaemon(fsDriver string) (*daemon.Daemon, error) { func (fs *Filesystem) getDaemonByRafs(rafs *racache.Rafs) (*daemon.Daemon, error) { switch rafs.GetFsDriver() { - case config.FsDriverFscache: - if fs.fscacheManager != nil { - if d := fs.fscacheManager.GetByDaemonID(rafs.DaemonID); d != nil { - return d, nil - } - } - case config.FsDriverFusedev: - if fs.fusedevManager != nil { - if d := fs.fusedevManager.GetByDaemonID(rafs.DaemonID); d != nil { + case config.FsDriverFscache, config.FsDriverFusedev: + if fsManager, ok := fs.enabledManagers[rafs.GetFsDriver()]; ok { + if d := fsManager.GetByDaemonID(rafs.DaemonID); d != nil { return d, nil } } diff --git a/pkg/metrics/serve.go b/pkg/metrics/serve.go index 87eb66d440..f2a1b8010e 100644 --- a/pkg/metrics/serve.go +++ b/pkg/metrics/serve.go @@ -34,11 +34,9 @@ type Server struct { inflightCollector *collector.InflightMetricsVecCollector } -func WithProcessManager(pm *manager.Manager) ServerOpt { +func WithProcessManagers(managers []*manager.Manager) ServerOpt { return func(s *Server) error { - if pm != nil { - s.managers = append(s.managers, pm) - } + s.managers = append(s.managers, managers...) return nil } } diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 8e33509a42..65b55521c9 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -108,9 +108,9 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho skipSSLVerify = config.GetSkipSSLVerify() } - var blockdevManager *mgr.Manager + fsManagers := []*mgr.Manager{} if cfg.Experimental.TarfsConfig.EnableTarfs { - blockdevManager, err = mgr.NewManager(mgr.Opt{ + blockdevManager, err := mgr.NewManager(mgr.Opt{ NydusdBinaryPath: "", Database: db, CacheDir: cfg.CacheManagerConfig.CacheDir, @@ -123,11 +123,11 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho if err != nil { return nil, errors.Wrap(err, "create blockdevice manager") } + fsManagers = append(fsManagers, blockdevManager) } - var fscacheManager *mgr.Manager if config.GetFsDriver() == config.FsDriverFscache { - mgr, err := mgr.NewManager(mgr.Opt{ + fscacheManager, err := mgr.NewManager(mgr.Opt{ NydusdBinaryPath: cfg.DaemonConfig.NydusdPath, Database: db, CacheDir: cfg.CacheManagerConfig.CacheDir, @@ -140,12 +140,11 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho if err != nil { return nil, errors.Wrap(err, "create fscache manager") } - fscacheManager = mgr + fsManagers = append(fsManagers, fscacheManager) } - var fusedevManager *mgr.Manager if config.GetFsDriver() == config.FsDriverFusedev { - mgr, err := mgr.NewManager(mgr.Opt{ + fusedevManager, err := mgr.NewManager(mgr.Opt{ NydusdBinaryPath: cfg.DaemonConfig.NydusdPath, Database: db, CacheDir: cfg.CacheManagerConfig.CacheDir, @@ -158,14 +157,12 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho if err != nil { return nil, errors.Wrap(err, "create fusedev manager") } - fusedevManager = mgr + fsManagers = append(fsManagers, fusedevManager) } metricServer, err := metrics.NewServer( ctx, - metrics.WithProcessManager(blockdevManager), - metrics.WithProcessManager(fscacheManager), - metrics.WithProcessManager(fusedevManager), + metrics.WithProcessManagers(fsManagers), ) if err != nil { return nil, errors.Wrap(err, "create metrics server") @@ -186,9 +183,7 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho } opts := []filesystem.NewFSOpt{ - filesystem.WithManager(blockdevManager), - filesystem.WithManager(fscacheManager), - filesystem.WithManager(fusedevManager), + filesystem.WithManagers(fsManagers), filesystem.WithNydusImageBinaryPath(cfg.DaemonConfig.NydusdPath), filesystem.WithVerifier(verifier), filesystem.WithRootMountpoint(config.GetRootMountpoint()), @@ -226,17 +221,7 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho } if config.IsSystemControllerEnabled() { - managers := []*mgr.Manager{} - if blockdevManager != nil { - managers = append(managers, blockdevManager) - } - if fscacheManager != nil { - managers = append(managers, fscacheManager) - } - if fusedevManager != nil { - managers = append(managers, fusedevManager) - } - systemController, err := system.NewSystemController(nydusFs, managers, config.SystemControllerAddress()) + systemController, err := system.NewSystemController(nydusFs, fsManagers, config.SystemControllerAddress()) if err != nil { return nil, errors.Wrap(err, "create system controller") } From 794ff0933574952d43308dd818b2dac8035541ca Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Wed, 6 Sep 2023 16:50:02 +0800 Subject: [PATCH 6/8] snapshot: add proxy mode to support guest pulling for CoCo Add proxy mode to support guest pulling for Confidential Containers, which doesn't download any image layer content on host but relay the image pulling request to Kata runtime/agent. It generates a fake rootfs with an extra mount option for Kata virtual volume. Signed-off-by: Jiang Liu --- config/config.go | 4 +- internal/constant/values.go | 2 + .../config-coco-guest-pulling.toml | 15 ++++ pkg/filesystem/fs.go | 10 +++ pkg/label/label.go | 7 ++ snapshot/mount_option.go | 44 ++++++++++- snapshot/process.go | 73 ++++++++++--------- snapshot/snapshot.go | 25 ++++++- 8 files changed, 137 insertions(+), 43 deletions(-) create mode 100644 misc/snapshotter/config-coco-guest-pulling.toml diff --git a/config/config.go b/config/config.go index 57b3a85ead..c3f4c4f797 100644 --- a/config/config.go +++ b/config/config.go @@ -104,6 +104,7 @@ const ( FsDriverFusedev string = constant.FsDriverFusedev FsDriverFscache string = constant.FsDriverFscache FsDriverNodev string = constant.FsDriverNodev + FsDriverProxy string = constant.FsDriverProxy ) type Experimental struct { @@ -276,7 +277,8 @@ func ValidateConfig(c *SnapshotterConfig) error { } if c.DaemonConfig.FsDriver != FsDriverFscache && c.DaemonConfig.FsDriver != FsDriverFusedev && - c.DaemonConfig.FsDriver != FsDriverBlockdev && c.DaemonConfig.FsDriver != FsDriverNodev { + c.DaemonConfig.FsDriver != FsDriverBlockdev && c.DaemonConfig.FsDriver != FsDriverNodev && + c.DaemonConfig.FsDriver != FsDriverProxy { return errors.Errorf("invalid filesystem driver %q", c.DaemonConfig.FsDriver) } if _, err := ParseRecoverPolicy(c.DaemonConfig.RecoverPolicy); err != nil { diff --git a/internal/constant/values.go b/internal/constant/values.go index e35306258a..9ad7a99488 100644 --- a/internal/constant/values.go +++ b/internal/constant/values.go @@ -25,6 +25,8 @@ const ( FsDriverFscache string = "fscache" // Only prepare/supply meta/data blobs, do not mount RAFS filesystem. FsDriverNodev string = "nodev" + // Relay layer content download operation to other agents. + FsDriverProxy string = "proxy" ) const ( diff --git a/misc/snapshotter/config-coco-guest-pulling.toml b/misc/snapshotter/config-coco-guest-pulling.toml new file mode 100644 index 0000000000..0cc77b0bd4 --- /dev/null +++ b/misc/snapshotter/config-coco-guest-pulling.toml @@ -0,0 +1,15 @@ +version = 1 + +# Snapshotter's own home directory where it stores and creates necessary resources +root = "/var/lib/containerd-nydus" + +# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket +address = "/run/containerd-nydus/containerd-nydus-grpc.sock" + +[daemon] +# Enable proxy mode +fs_driver = "proxy" + +[snapshot] +# Insert Kata volume information to `Mount.Options` +enable_kata_volume = true diff --git a/pkg/filesystem/fs.go b/pkg/filesystem/fs.go index 403bbc3073..4a95c4a3c6 100644 --- a/pkg/filesystem/fs.go +++ b/pkg/filesystem/fs.go @@ -357,6 +357,14 @@ func (fs *Filesystem) Mount(ctx context.Context, snapshotID string, labels map[s } case config.FsDriverNodev: // Nothing to do + case config.FsDriverProxy: + if label.IsNydusProxyMode(labels) { + if v, ok := labels[label.CRILayerDigest]; ok { + rafs.AddAnnotation(label.CRILayerDigest, v) + } + rafs.AddAnnotation(label.NydusProxyMode, "true") + rafs.SetMountpoint(path.Join(rafs.GetSnapshotDir(), "fs")) + } default: err = errors.Errorf("unknown filesystem driver %s for snapshot %s", fsDriver, snapshotID) } @@ -420,6 +428,8 @@ func (fs *Filesystem) Umount(ctx context.Context, snapshotID string) error { if err := fsManager.RemoveRafsInstance(snapshotID); err != nil { return errors.Wrapf(err, "remove snapshot %s", snapshotID) } + case config.FsDriverNodev, config.FsDriverProxy: + // Nothing to do default: return errors.Errorf("unknown filesystem driver %s for snapshot %s", fsDriver, snapshotID) } diff --git a/pkg/label/label.go b/pkg/label/label.go index 30e204992f..f5392771d4 100644 --- a/pkg/label/label.go +++ b/pkg/label/label.go @@ -44,6 +44,8 @@ const ( NydusImagePullSecret = "containerd.io/snapshot/pullsecret" // Annotation containing username to pull images from registry, set by the snapshotter. NydusImagePullUsername = "containerd.io/snapshot/pullusername" + // Proxy image pull actions to other agents. + NydusProxyMode = "containerd.io/snapshot/nydus-proxy-mode" // A bool flag to enable integrity verification of meta data blob NydusSignature = "containerd.io/snapshot/nydus-signature" @@ -75,6 +77,11 @@ func IsTarfsDataLayer(labels map[string]string) bool { return ok } +func IsNydusProxyMode(labels map[string]string) bool { + _, ok := labels[NydusProxyMode] + return ok +} + func HasTarfsHint(labels map[string]string) bool { _, ok := labels[TarfsHint] return ok diff --git a/snapshot/mount_option.go b/snapshot/mount_option.go index 94f69954ce..657a6e271f 100644 --- a/snapshot/mount_option.go +++ b/snapshot/mount_option.go @@ -108,6 +108,18 @@ func (o *snapshotter) mountWithKataVolume(ctx context.Context, id string, overla return []mount.Mount{}, errors.Errorf("failed to find RAFS instance for snapshot %s", id) } + // Insert Kata volume for proxy + if label.IsNydusProxyMode(rafs.Annotations) { + options, err := o.mountWithProxyVolume(*rafs) + if err != nil { + return []mount.Mount{}, errors.Wrapf(err, "create kata volume for proxy") + } + if len(options) > 0 { + overlayOptions = append(overlayOptions, options...) + hasVolume = true + } + } + // Insert Kata volume for tarfs if blobID, ok := rafs.Annotations[label.NydusTarfsLayer]; ok { options, err := o.mountWithTarfsVolume(*rafs, blobID) @@ -122,18 +134,45 @@ func (o *snapshotter) mountWithKataVolume(ctx context.Context, id string, overla if hasVolume { log.G(ctx).Debugf("fuse.nydus-overlayfs mount options %v", overlayOptions) - return []mount.Mount{ + mounts := []mount.Mount{ { Type: "fuse.nydus-overlayfs", Source: "overlay", Options: overlayOptions, }, - }, nil + } + return mounts, nil } return overlayMount(overlayOptions), nil } +func (o *snapshotter) mountWithProxyVolume(rafs rafs.Rafs) ([]string, error) { + options := []string{} + for k, v := range rafs.Annotations { + options = append(options, fmt.Sprintf("%s=%s", k, v)) + } + + volume := &KataVirtualVolume{ + VolumeType: KataVirtualVolumeImageGuestPullType, + Source: "", + FSType: "", + Options: options, + ImagePull: &ImagePullVolume{Metadata: rafs.Annotations}, + } + if !volume.Validate() { + return []string{}, errors.Errorf("got invalid kata volume, %v", volume) + } + + info, err := EncodeKataVirtualVolumeToBase64(*volume) + if err != nil { + return []string{}, errors.Errorf("failed to encoding Kata Volume info %v", volume) + } + opt := fmt.Sprintf("%s=%s", KataVirtualVolumeOptionName, info) + + return []string{opt}, nil +} + func (o *snapshotter) mountWithTarfsVolume(rafs rafs.Rafs, blobID string) ([]string, error) { var volume *KataVirtualVolume @@ -366,6 +405,7 @@ func EncodeKataVirtualVolumeToBase64(volume KataVirtualVolume) (string, error) { if err != nil { return "", errors.Wrapf(err, "marshal KataVirtualVolume object") } + log.L.Infof("Mount info with kata volume %s", validKataVirtualVolumeJSON) option := base64.StdEncoding.EncodeToString(validKataVirtualVolumeJSON) return option, nil } diff --git a/snapshot/process.go b/snapshot/process.go index fe24bdb170..9e832bff84 100644 --- a/snapshot/process.go +++ b/snapshot/process.go @@ -63,6 +63,14 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry, if isRoLayer { // Containerd won't consume mount slice for below snapshots switch { + case config.GetFsDriver() == config.FsDriverProxy: + logger.Debugf("proxy image pull request to other agents") + if ref := labels[label.CRILayerDigest]; len(ref) > 0 { + labels[label.NydusProxyMode] = "true" + handler = skipHandler + } else { + return nil, "", errors.Errorf("missing CRI reference annotation for snaposhot %s", s.ID) + } case label.IsNydusMetaLayer(labels): logger.Debugf("found nydus meta layer") handler = defaultHandler @@ -109,11 +117,19 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry, // Container writable layer comes into this branch. // It should not be committed during this Prepare() operation. + pID, pInfo, _, pErr := snapshot.GetSnapshotInfo(ctx, sn.ms, parent) + if pErr == nil && label.IsNydusProxyMode(pInfo.Labels) { + logger.Infof("Prepare active snapshot %s in proxy mode", key) + handler = remoteHandler(pID, pInfo.Labels) + } + // Hope to find bootstrap layer and prepares to start nydusd // TODO: Trying find nydus meta layer will slow down setting up rootfs to OCI images - if id, info, err := sn.findMetaLayer(ctx, key); err == nil { - logger.Infof("Prepare active Nydus snapshot %s", key) - handler = remoteHandler(id, info.Labels) + if handler == nil { + if id, info, err := sn.findMetaLayer(ctx, key); err == nil { + logger.Infof("Prepare active Nydus snapshot %s", key) + handler = remoteHandler(id, info.Labels) + } } if handler == nil && sn.fs.ReferrerDetectEnabled() { @@ -127,46 +143,31 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry, } } - if handler == nil && sn.fs.StargzEnabled() { - // `pInfo` must be the uppermost parent layer - id, pInfo, _, err := snapshot.GetSnapshotInfo(ctx, sn.ms, parent) - if err != nil { - return nil, "", errors.Wrap(err, "get parent snapshot info") - } - - if sn.fs.StargzLayer(pInfo.Labels) { - if err := sn.fs.MergeStargzMetaLayer(ctx, s); err != nil { - return nil, "", errors.Wrap(err, "merge stargz meta layers") - } - handler = remoteHandler(id, pInfo.Labels) - logger.Infof("Generated estargz merged meta for %s", key) + if handler == nil && pErr == nil && sn.fs.StargzEnabled() && sn.fs.StargzLayer(pInfo.Labels) { + if err := sn.fs.MergeStargzMetaLayer(ctx, s); err != nil { + return nil, "", errors.Wrap(err, "merge stargz meta layers") } + handler = remoteHandler(pID, pInfo.Labels) + logger.Infof("Generated estargz merged meta for %s", key) } - if handler == nil && sn.fs.TarfsEnabled() { + if handler == nil && pErr == nil && sn.fs.TarfsEnabled() && label.IsTarfsDataLayer(pInfo.Labels) { // Merge and mount tarfs on the uppermost parent layer. - id, pInfo, _, err := snapshot.GetSnapshotInfo(ctx, sn.ms, parent) - switch { - case err != nil: - logger.Warnf("Tarfs enabled but can't get parent of snapshot %s", s.ID) - case !label.IsTarfsDataLayer(pInfo.Labels): - logger.Debugf("Tarfs enabled but Parent (%s) of snapshot %s is not a tarfs layer", id, s.ID) - default: - // TODO may need to check all parrent layers, in case share layers with other images - // which have already been prepared by overlay snapshotter - - err := sn.fs.MergeTarfsLayers(s, func(id string) string { return sn.upperPath(id) }) + // TODO may need to check all parrent layers, in case share layers with other images + // which have already been prepared by overlay snapshotter + + err := sn.fs.MergeTarfsLayers(s, func(id string) string { return sn.upperPath(id) }) + if err != nil { + return nil, "", errors.Wrap(err, "merge tarfs layers") + } + if config.GetTarfsExportEnabled() { + _, err = sn.fs.ExportBlockData(s, false, labels, func(id string) string { return sn.upperPath(id) }) if err != nil { - return nil, "", errors.Wrap(err, "merge tarfs layers") - } - if config.GetTarfsExportEnabled() { - _, err = sn.fs.ExportBlockData(s, false, labels, func(id string) string { return sn.upperPath(id) }) - if err != nil { - return nil, "", errors.Wrap(err, "export image as tarfs block device") - } + return nil, "", errors.Wrap(err, "export image as tarfs block device") } - handler = remoteHandler(id, pInfo.Labels) } + logger.Infof("Prepare active Nydus snapshot %s in tarfs mode", key) + handler = remoteHandler(pID, pInfo.Labels) } } diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 65b55521c9..e1f8192cc5 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -160,6 +160,23 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho fsManagers = append(fsManagers, fusedevManager) } + if config.GetFsDriver() == config.FsDriverProxy { + proxyManager, err := mgr.NewManager(mgr.Opt{ + NydusdBinaryPath: "", + Database: db, + CacheDir: cfg.CacheManagerConfig.CacheDir, + RootDir: cfg.Root, + RecoverPolicy: rp, + FsDriver: config.FsDriverProxy, + DaemonConfig: nil, + CgroupMgr: cgroupMgr, + }) + if err != nil { + return nil, errors.Wrap(err, "create proxy manager") + } + fsManagers = append(fsManagers, proxyManager) + } + metricServer, err := metrics.NewServer( ctx, metrics.WithProcessManagers(fsManagers), @@ -378,21 +395,21 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er needRemoteMounts = true metaSnapshotID = id } - } else if label.IsTarfsDataLayer(info.Labels) { + } else if (o.fs.TarfsEnabled() && label.IsTarfsDataLayer(info.Labels)) || label.IsNydusProxyMode(info.Labels) { needRemoteMounts = true metaSnapshotID = id } case snapshots.KindActive: if info.Parent != "" { pKey := info.Parent - if pID, info, _, err := snapshot.GetSnapshotInfo(ctx, o.ms, pKey); err == nil { - if label.IsNydusMetaLayer(info.Labels) { + if pID, pInfo, _, err := snapshot.GetSnapshotInfo(ctx, o.ms, pKey); err == nil { + if label.IsNydusMetaLayer(pInfo.Labels) { if err = o.fs.WaitUntilReady(pID); err != nil { return nil, errors.Wrapf(err, "mounts: snapshot %s is not ready, err: %v", pID, err) } needRemoteMounts = true metaSnapshotID = pID - } else if o.fs.TarfsEnabled() && label.IsTarfsDataLayer(info.Labels) { + } else if (o.fs.TarfsEnabled() && label.IsTarfsDataLayer(pInfo.Labels)) || label.IsNydusProxyMode(pInfo.Labels) { needRemoteMounts = true metaSnapshotID = pID } From fedb996240e9584f27083ed5db9aca5ac86e02bd Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Fri, 8 Sep 2023 17:36:20 +0800 Subject: [PATCH 7/8] tarfs: fix several bugs related to tarfs 1. chooseProcessor() forgot to update tarfs annotations for parent snapshot during preparing rw snapshot. 2. initialize `Labels` map in function createSnapshot() to avoid accessing nil pointer. 3. enhance error hanlding of tarfs::Manager::PrepareLayer(). Signed-off-by: Jiang Liu --- pkg/tarfs/tarfs.go | 110 +++++++++++++++++++++---------------------- snapshot/process.go | 15 ++---- snapshot/snapshot.go | 66 ++++++++++++++++---------- 3 files changed, 102 insertions(+), 89 deletions(-) diff --git a/pkg/tarfs/tarfs.go b/pkg/tarfs/tarfs.go index b2f135e2cc..543dc5ceda 100755 --- a/pkg/tarfs/tarfs.go +++ b/pkg/tarfs/tarfs.go @@ -306,63 +306,89 @@ func (t *Manager) getImageBlobInfo(metaFilePath string) (string, error) { } // download & uncompress an oci/docker blob, and then generate the tarfs bootstrap -func (t *Manager) blobProcess(ctx context.Context, snapshotID, ref string, manifestDigest, layerDigest digest.Digest, - layerBlobID, upperDirPath string) error { +func (t *Manager) blobProcess(ctx context.Context, wg *sync.WaitGroup, snapshotID, ref string, + manifestDigest, layerDigest digest.Digest, upperDirPath string) error { + layerBlobID := layerDigest.Hex() + epilog := func(err error, msg string) { + st, err1 := t.getSnapshotStatus(snapshotID, true) + if err1 != nil { + // return errors.Errorf("can not found status object for snapshot %s after prepare", snapshotID) + err1 = errors.Wrapf(err1, "can not found status object for snapshot %s after prepare", snapshotID) + log.L.WithError(err1).Errorf("async prepare tarfs layer for snapshot ID %s", snapshotID) + return + } + defer st.mutex.Unlock() + + st.blobID = layerBlobID + st.blobTarFilePath = t.layerTarFilePath(layerBlobID) + if err != nil { + log.L.WithError(err).Errorf(msg) + st.status = TarfsStatusFailed + } else { + st.status = TarfsStatusReady + } + log.L.Infof(msg) + } + keyChain, err := auth.GetKeyChainByRef(ref, nil) if err != nil { + epilog(err, "create key chain for connection") return err } remote := remote.New(keyChain, t.insecure) + rc, _, err := t.getBlobStream(ctx, remote, ref, layerDigest) + if err != nil && remote.RetryWithPlainHTTP(ref, err) { + rc, _, err = t.getBlobStream(ctx, remote, ref, layerDigest) + } + if err != nil { + epilog(err, "get blob stream for layer") + return errors.Wrapf(err, "get blob stream by digest") + } - handle := func() error { - rc, _, err := t.getBlobStream(ctx, remote, ref, layerDigest) - if err != nil { - return err - } + go func() { + defer wg.Done() defer rc.Close() + ds, err := compression.DecompressStream(rc) if err != nil { - return errors.Wrap(err, "unpack layer blob stream for tarfs") + epilog(err, "unpack layer blob stream for tarfs") + return } defer ds.Close() if t.validateDiffID { diffID, err := t.getBlobDiffID(ctx, remote, ref, manifestDigest, layerDigest) if err != nil { - return errors.Wrap(err, "get image layer diffID") + epilog(err, "get layer diffID") + return } digester := digest.Canonical.Digester() dr := io.TeeReader(ds, digester.Hash()) err = t.generateBootstrap(dr, snapshotID, layerBlobID, upperDirPath) - if err != nil && !errdefs.IsAlreadyExists(err) { - return errors.Wrap(err, "generate tarfs data from image layer blob") - } - if err == nil { - if digester.Digest() != diffID { - return errors.Errorf("image layer diffID %s for tarfs does not match", diffID) - } - log.L.Infof("tarfs data for layer %s is ready, digest %s", snapshotID, digester.Digest()) + switch { + case err != nil && !errdefs.IsAlreadyExists(err): + epilog(err, "generate tarfs from image layer blob") + case err == nil && digester.Digest() != diffID: + epilog(err, "image layer diffID does not match") + default: + msg := fmt.Sprintf("nydus tarfs for snapshot %s is ready, digest %s", snapshotID, digester.Digest()) + epilog(nil, msg) } } else { err = t.generateBootstrap(ds, snapshotID, layerBlobID, upperDirPath) if err != nil && !errdefs.IsAlreadyExists(err) { - return errors.Wrap(err, "generate tarfs data from image layer blob") + epilog(err, "generate tarfs data from image layer blob") + } else { + msg := fmt.Sprintf("nydus tarfs for snapshot %s is ready", snapshotID) + epilog(nil, msg) } - log.L.Infof("tarfs data for layer %s is ready", snapshotID) } - return nil - } - - err = handle() - if err != nil && remote.RetryWithPlainHTTP(ref, err) { - err = handle() - } + }() return err } -func (t *Manager) PrepareLayer(snapshotID, ref string, manifestDigest, layerDigest digest.Digest, - upperDirPath string) error { +func (t *Manager) PrepareLayer(snapshotID, ref string, manifestDigest, layerDigest digest.Digest, upperDirPath string) error { t.mutex.Lock() if _, ok := t.snapshotMap[snapshotID]; ok { t.mutex.Unlock() @@ -379,33 +405,7 @@ func (t *Manager) PrepareLayer(snapshotID, ref string, manifestDigest, layerDige } t.mutex.Unlock() - go func() { - defer wg.Done() - - layerBlobID := layerDigest.Hex() - err := t.blobProcess(ctx, snapshotID, ref, manifestDigest, layerDigest, layerBlobID, upperDirPath) - - st, err1 := t.getSnapshotStatus(snapshotID, true) - if err1 != nil { - // return errors.Errorf("can not found status object for snapshot %s after prepare", snapshotID) - err1 = errors.Wrapf(err1, "can not found status object for snapshot %s after prepare", snapshotID) - log.L.WithError(err1).Errorf("async prepare tarfs layer of snapshot ID %s", snapshotID) - return - } - defer st.mutex.Unlock() - - st.blobID = layerBlobID - st.blobTarFilePath = t.layerTarFilePath(layerBlobID) - if err != nil { - log.L.WithError(err).Errorf("failed to convert OCI image to tarfs") - st.status = TarfsStatusFailed - } else { - st.status = TarfsStatusReady - } - log.L.Debugf("finish converting snapshot %s to tarfs, status %d", snapshotID, st.status) - }() - - return nil + return t.blobProcess(ctx, wg, snapshotID, ref, manifestDigest, layerDigest, upperDirPath) } func (t *Manager) MergeLayers(s storage.Snapshot, storageLocater func(string) string) error { diff --git a/snapshot/process.go b/snapshot/process.go index 9e832bff84..4c012fbb26 100644 --- a/snapshot/process.go +++ b/snapshot/process.go @@ -98,11 +98,11 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry, } if handler == nil && sn.fs.TarfsEnabled() { + logger.Debugf("convert OCIv1 layer to tarfs") err := sn.fs.PrepareTarfsLayer(ctx, labels, s.ID, sn.upperPath(s.ID)) if err != nil { logger.Warnf("snapshot ID %s can't be converted into tarfs, fallback to containerd, err: %v", s.ID, err) } else { - logger.Debugf("convert OCIv1 layer to tarfs") if config.GetTarfsExportEnabled() { _, err = sn.fs.ExportBlockData(s, true, labels, func(id string) string { return sn.upperPath(id) }) if err != nil { @@ -156,17 +156,12 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry, // TODO may need to check all parrent layers, in case share layers with other images // which have already been prepared by overlay snapshotter - err := sn.fs.MergeTarfsLayers(s, func(id string) string { return sn.upperPath(id) }) + logger.Infof("Prepare active snapshot %s in Nydus tarfs mode", key) + err = sn.mergeTarfs(ctx, s, pID, pInfo) if err != nil { - return nil, "", errors.Wrap(err, "merge tarfs layers") + return nil, "", errors.Wrapf(err, "merge tarfs layers for snapshot %s", pID) } - if config.GetTarfsExportEnabled() { - _, err = sn.fs.ExportBlockData(s, false, labels, func(id string) string { return sn.upperPath(id) }) - if err != nil { - return nil, "", errors.Wrap(err, "export image as tarfs block device") - } - } - logger.Infof("Prepare active Nydus snapshot %s in tarfs mode", key) + logger.Infof("Prepared active snapshot %s in Nydus tarfs mode", key) handler = remoteHandler(pID, pInfo.Labels) } } diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index e1f8192cc5..71507032d6 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -441,7 +441,8 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er } func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { - log.L.Debugf("[Prepare] snapshot with key %s, parent %s", key, parent) + log.L.Infof("[Prepare] snapshot with key %s parent %s", key, parent) + if timer := collector.NewSnapshotMetricsTimer(collector.SnapshotMethodPrepare); timer != nil { defer timer.ObserveDuration() } @@ -476,7 +477,7 @@ func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...s // 1. View on the topmost layer of nydus images or zran images // 2. View on the any layer of nydus images or zran images func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { - log.L.Debugf("[View] snapshot with key %s, parent %s", key, parent) + log.L.Infof("[View] snapshot with key %s parent %s", key, parent) pID, pInfo, _, err := snapshot.GetSnapshotInfo(ctx, o.ms, parent) if err != nil { @@ -517,34 +518,22 @@ func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snap } if o.fs.TarfsEnabled() && label.IsTarfsDataLayer(pInfo.Labels) { - if err := o.fs.MergeTarfsLayers(s, func(id string) string { return o.upperPath(id) }); err != nil { - return nil, errors.Wrapf(err, "tarfs merge fail %s", pID) - } - if config.GetTarfsExportEnabled() { - updateFields, err := o.fs.ExportBlockData(s, false, pInfo.Labels, func(id string) string { return o.upperPath(id) }) - if err != nil { - return nil, errors.Wrap(err, "export tarfs as block image") - } - if len(updateFields) > 0 { - _, err = o.Update(ctx, pInfo, updateFields...) - if err != nil { - return nil, errors.Wrapf(err, "update snapshot label information") - } - } + log.L.Infof("Prepare view snapshot %s in Nydus tarfs mode", pID) + err = o.mergeTarfs(ctx, s, pID, pInfo) + if err != nil { + return nil, errors.Wrapf(err, "merge tarfs layers for snapshot %s", pID) } if err := o.fs.Mount(ctx, pID, pInfo.Labels, &s); err != nil { return nil, errors.Wrapf(err, "mount tarfs, snapshot id %s", pID) } + log.L.Infof("Prepared view snapshot %s in Nydus tarfs mode", pID) needRemoteMounts = true metaSnapshotID = pID } - log.L.Infof("[View] snapshot with key %s parent %s", key, parent) - if needRemoteMounts { return o.mountRemote(ctx, base.Labels, s, metaSnapshotID) } - return o.mountNative(ctx, base.Labels, s) } @@ -593,6 +582,7 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap } func (o *snapshotter) Remove(ctx context.Context, key string) error { + log.L.Debugf("[Remove] snapshot with key %s", key) if timer := collector.NewSnapshotMetricsTimer(collector.SnapshotMethodRemove); timer != nil { defer timer.ObserveDuration() } @@ -614,13 +604,16 @@ func (o *snapshotter) Remove(ctx context.Context, key string) error { return errors.Wrapf(err, "get snapshot %s", key) } - // For example: remove snapshot with key sha256:c33c40022c8f333e7f199cd094bd56758bc479ceabf1e490bb75497bf47c2ebf - log.L.Debugf("[Remove] snapshot with key %s snapshot id %s", key, id) - - if label.IsNydusMetaLayer(info.Labels) { + switch { + case label.IsNydusMetaLayer(info.Labels): log.L.Infof("[Remove] nydus meta snapshot with key %s snapshot id %s", key, id) - } else if label.IsTarfsDataLayer(info.Labels) { + case label.IsNydusDataLayer(info.Labels): + log.L.Infof("[Remove] nydus data snapshot with key %s snapshot id %s", key, id) + case label.IsTarfsDataLayer(info.Labels): log.L.Infof("[Remove] nydus tarfs snapshot with key %s snapshot id %s", key, id) + default: + // For example: remove snapshot with key sha256:c33c40022c8f333e7f199cd094bd56758bc479ceabf1e490bb75497bf47c2ebf + log.L.Infof("[Remove] snapshot with key %s snapshot id %s", key, id) } if info.Kind == snapshots.KindCommitted { @@ -676,6 +669,8 @@ func (o *snapshotter) Walk(ctx context.Context, fn snapshots.WalkFunc, fs ...str } func (o *snapshotter) Close() error { + log.L.Info("[Close] shutdown snapshotter") + if o.cleanupOnClose { err := o.fs.Teardown(context.Background()) if err != nil { @@ -745,6 +740,9 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k return &base, storage.Snapshot{}, err } } + if base.Labels == nil { + base.Labels = map[string]string{} + } var td, path string defer func() { @@ -798,6 +796,26 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k return &base, s, nil } +func (o *snapshotter) mergeTarfs(ctx context.Context, s storage.Snapshot, pID string, pInfo snapshots.Info) error { + if err := o.fs.MergeTarfsLayers(s, func(id string) string { return o.upperPath(id) }); err != nil { + return errors.Wrapf(err, "tarfs merge fail %s", pID) + } + if config.GetTarfsExportEnabled() { + updateFields, err := o.fs.ExportBlockData(s, false, pInfo.Labels, func(id string) string { return o.upperPath(id) }) + if err != nil { + return errors.Wrap(err, "export tarfs as block image") + } + if len(updateFields) > 0 { + _, err = o.Update(ctx, pInfo, updateFields...) + if err != nil { + return errors.Wrapf(err, "update snapshot label information") + } + } + } + + return nil +} + func bindMount(source, roFlag string) []mount.Mount { return []mount.Mount{ { From 0da2966cc00643eae3999da19c9b8a7fdffd1f38 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Sat, 9 Sep 2023 00:51:35 +0800 Subject: [PATCH 8/8] snapshot: fix a bug in generating mount for view snapshot When generating a view snapshot with only one parent, it will generate bind mount instead of overlayfs mount, and missing extra options, such as `KataVirtualVolume` or `extraoptions`. Fix it by using overlayfs for view snapshot with only one parent. Signed-off-by: Jiang Liu --- snapshot/mount_option.go | 2 +- snapshot/snapshot.go | 19 ++++++++----------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/snapshot/mount_option.go b/snapshot/mount_option.go index 657a6e271f..0e161629a6 100644 --- a/snapshot/mount_option.go +++ b/snapshot/mount_option.go @@ -405,7 +405,7 @@ func EncodeKataVirtualVolumeToBase64(volume KataVirtualVolume) (string, error) { if err != nil { return "", errors.Wrapf(err, "marshal KataVirtualVolume object") } - log.L.Infof("Mount info with kata volume %s", validKataVirtualVolumeJSON) + log.L.Infof("encode kata volume %s", validKataVirtualVolumeJSON) option := base64.StdEncoding.EncodeToString(validKataVirtualVolumeJSON) return option, nil } diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 71507032d6..47a1bc7ae2 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -843,16 +843,8 @@ func overlayMount(options []string) []mount.Mount { // `s` and `id` can represent a different layer, it's useful when View an image func (o *snapshotter) mountRemote(ctx context.Context, labels map[string]string, s storage.Snapshot, id string) ([]mount.Mount, error) { var overlayOptions []string - if s.Kind == snapshots.KindActive { - overlayOptions = append(overlayOptions, - fmt.Sprintf("workdir=%s", o.workPath(s.ID)), - fmt.Sprintf("upperdir=%s", o.upperPath(s.ID)), - ) - if _, ok := labels[label.OverlayfsVolatileOpt]; ok { - overlayOptions = append(overlayOptions, "volatile") - } - } else if len(s.ParentIDs) == 1 { - return bindMount(o.upperPath(s.ParentIDs[0]), "ro"), nil + if _, ok := labels[label.OverlayfsVolatileOpt]; ok { + overlayOptions = append(overlayOptions, "volatile") } lowerPaths := make([]string, 0, 8) @@ -862,7 +854,12 @@ func (o *snapshotter) mountRemote(ctx context.Context, labels map[string]string, } lowerPaths = append(lowerPaths, lowerPathNydus) - if s.Kind == snapshots.KindView { + if s.Kind == snapshots.KindActive { + overlayOptions = append(overlayOptions, + fmt.Sprintf("workdir=%s", o.workPath(s.ID)), + fmt.Sprintf("upperdir=%s", o.upperPath(s.ID)), + ) + } else if s.Kind == snapshots.KindView { lowerPathNormal, err := o.lowerPath(s.ID) if err != nil { return nil, errors.Wrapf(err, "failed to locate overlay lowerdir for view snapshot")