Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feats: support encrypting nydus data blobs #498

Merged
merged 1 commit into from
Jul 13, 2023
Merged

feats: support encrypting nydus data blobs #498

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pkg/converter/constant.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ const (
LayerAnnotationNydusBlobSize = "containerd.io/snapshot/nydus-blob-size"
LayerAnnotationNydusBootstrap = "containerd.io/snapshot/nydus-bootstrap"
LayerAnnotationNydusSourceChainID = "containerd.io/snapshot/nydus-source-chainid"
LayerAnnotationNydusEncryptedBlob = "containerd.io/snapshot/nydus-encrypted-blob"

LayerAnnotationNydusReferenceBlobIDs = "containerd.io/snapshot/nydus-reference-blob-ids"

Expand Down
14 changes: 14 additions & 0 deletions pkg/converter/convert_unix.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,9 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
if opt.BatchSize != "" && opt.BatchSize != "0" {
requiredFeatures.Add(tool.FeatureBatchSize)
}
if opt.Encrypt {
requiredFeatures.Add(tool.FeatureEncrypt)
}

detectedFeatures, err := tool.DetectFeatures(builderPath, requiredFeatures, tool.GetHelp)
if err != nil {
Expand Down Expand Up @@ -407,6 +410,7 @@ func packFromDirectory(ctx context.Context, dest io.Writer, opt PackOption, buil
ChunkSize: opt.ChunkSize,
Compressor: opt.Compressor,
Timeout: opt.Timeout,
Encrypt: opt.Encrypt,

Features: opt.features,
})
Expand Down Expand Up @@ -510,6 +514,7 @@ func packFromTar(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteC
BatchSize: opt.BatchSize,
Compressor: opt.Compressor,
Timeout: opt.Timeout,
Encrypt: opt.Encrypt,

Features: opt.features,
})
Expand Down Expand Up @@ -852,6 +857,10 @@ func LayerConvertFunc(opt PackOption) converter.ConvertFunc {
newDesc.Annotations[label.NydusRefLayer] = desc.Digest.String()
}

if opt.Encrypt {
newDesc.Annotations[LayerAnnotationNydusEncryptedBlob] = "true"
}

if opt.Backend != nil {
taoohong marked this conversation as resolved.
Show resolved Hide resolved
if err := opt.Backend.Push(ctx, cs, newDesc); err != nil {
return nil, errors.Wrap(err, "push to storage backend")
Expand Down Expand Up @@ -1134,6 +1143,11 @@ func MergeLayers(ctx context.Context, cs content.Store, descs []ocispec.Descript
if opt.OCIRef {
blobDesc.Annotations[label.NydusRefLayer] = layers[idx].OriginalDigest.String()
}

if len(opt.EncryptRecipients) != 0 {
blobDesc.Annotations[LayerAnnotationNydusEncryptedBlob] = "true"
}

blobDescs = append(blobDescs, blobDesc)
}

Expand Down
4 changes: 4 additions & 0 deletions pkg/converter/tool/builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
AlignedChunk bool
ChunkSize string
BatchSize string
Encrypt bool
Timeout *time.Duration

Features Features
Expand Down Expand Up @@ -136,6 +137,9 @@
if option.Features.Contains(FeatureBatchSize) {
args = append(args, "--batch-size", option.BatchSize)
}
if option.Encrypt {
args = append(args, "--encrypt")
}

Check warning on line 142 in pkg/converter/tool/builder.go

View check run for this annotation

Codecov / codecov/patch

pkg/converter/tool/builder.go#L140-L142 

Added lines #L140 - L142 were not covered by tests
args = append(args, option.SourcePath)

return args
Expand Down
3 changes: 3 additions & 0 deletions pkg/converter/tool/feature.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ const (
// into a big batch chunk, which can reduce the the size of the image
// and accelerate the runtime file loading.
FeatureBatchSize Feature = "--batch-size"
// The option `--encrypt` enables converting directories, tar files
// or OCI images into encrypted nydus blob.
FeatureEncrypt Feature = "--encrypt"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we detect if the required feature encrypt is supported by nydus-image? Seems we only define a constant here.

	requiredFeatures := tool.NewFeatures(tool.FeatureTar2Rafs)
	if opt.BatchSize != "" && opt.BatchSize != "0" {
		requiredFeatures.Add(tool.FeatureBatchSize)
	}

	detectedFeatures, err := tool.DetectFeatures(builderPath, required features, tool.GetHelp)
	if err != nil {
		return nil, err

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

)

var requiredFeatures Features
Expand Down
132 changes: 130 additions & 2 deletions pkg/converter/tool/feature_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,71 @@ func TestDetectFeature(t *testing.T) {
Print help information
`),
},
{
name: "'--encrypt' is supported in v2.2.0-261-g22ad0e2c",
feature: FeatureEncrypt,
expect: true,
helpMsg: []byte(`
Create RAFS filesystems from directories, tar files or OCI images

Usage: nydus-image create [OPTIONS] <SOURCE>

Arguments:
<SOURCE> source from which to build the RAFS filesystem

Options:
-L, --log-file <log-file>
Log file path
-t, --type <type>
Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index]
-B, --bootstrap <bootstrap>
File path to save the generated RAFS metadata blob
-l, --log-level <log-level>
Log level: [default: info] [possible values: trace, debug, info, warn, error]
-D, --blob-dir <blob-dir>
Directory path to save generated RAFS metadata and data blobs
-b, --blob <blob>
File path to save the generated RAFS data blob
--blob-inline-meta
Inline RAFS metadata and blob metadata into the data blob
--blob-id <blob-id>
OSS object id for the generated RAFS data blob
--blob-data-size <blob-data-size>
Set data blob size for 'estargztoc-ref' conversion
--chunk-size <chunk-size>
Set the size of data chunks, must be power of two and between 0x1000-0x1000000:
--batch-size <batch-size>
Set the batch size to merge small chunks, must be power of two, between 0x1000-0x1000000 or be zero: [default: 0]
--compressor <compressor>
Algorithm to compress data chunks: [default: zstd] [possible values: none, lz4_block, zstd]
--digester <digester>
Algorithm to digest data chunks: [default: blake3] [possible values: blake3, sha256]
-C, --config <config>
Configuration file for storage backend, cache and RAFS FUSE filesystem.
-v, --fs-version <fs-version>
Set RAFS format version number: [default: 6] [possible values: 5, 6]
--features <features>
Enable/disable features [possible values: blob-toc]
--chunk-dict <chunk-dict>
File path of chunk dictionary for data deduplication
--parent-bootstrap <parent-bootstrap>
File path of the parent/referenced RAFS metadata blob (optional)
--aligned-chunk
Align uncompressed data chunks to 4K, only for RAFS V5
--repeatable
Generate reproducible RAFS metadata
--whiteout-spec <whiteout-spec>
Set the type of whiteout specification: [default: oci] [possible values: oci, overlayfs, none]
--prefetch-policy <prefetch-policy>
Set data prefetch policy [default: none] [possible values: fs, blob, none]
-J, --output-json <output-json>
File path to save operation result in JSON format
-E, --encrypt
Encrypt the generated RAFS metadata and data blobs
-h, --help
Print help information
`),
},

{
name: "'--type tar-rafs' is not supported in v2.1.4",
Expand Down Expand Up @@ -552,6 +617,69 @@ func TestDetectFeature(t *testing.T) {
<SOURCE> source path
`),
},
{
name: "'--encrypt' is not supported in v2.2.0",
feature: FeatureEncrypt,
expect: false,
helpMsg: []byte(`
Create RAFS filesystems from directories, tar files or OCI images

Usage: nydus-image create [OPTIONS] <SOURCE>

Arguments:
<SOURCE> source from which to build the RAFS filesystem

Options:
-L, --log-file <log-file>
Log file path
-t, --type <type>
Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index]
-B, --bootstrap <bootstrap>
File path to save the generated RAFS metadata blob
-l, --log-level <log-level>
Log level: [default: info] [possible values: trace, debug, info, warn, error]
-D, --blob-dir <blob-dir>
Directory path to save generated RAFS metadata and data blobs
-b, --blob <blob>
File path to save the generated RAFS data blob
--blob-inline-meta
Inline RAFS metadata and blob metadata into the data blob
--blob-id <blob-id>
OSS object id for the generated RAFS data blob
--blob-data-size <blob-data-size>
Set data blob size for 'estargztoc-ref' conversion
--chunk-size <chunk-size>
Set the size of data chunks, must be power of two and between 0x1000-0x1000000:
--batch-size <batch-size>
Set the batch size to merge small chunks, must be power of two, between 0x1000-0x1000000 or be zero: [default: 0]
--compressor <compressor>
Algorithm to compress data chunks: [default: zstd] [possible values: none, lz4_block, zstd]
--digester <digester>
Algorithm to digest data chunks: [default: blake3] [possible values: blake3, sha256]
-C, --config <config>
Configuration file for storage backend, cache and RAFS FUSE filesystem.
-v, --fs-version <fs-version>
Set RAFS format version number: [default: 6] [possible values: 5, 6]
--features <features>
Enable/disable features [possible values: blob-toc]
--chunk-dict <chunk-dict>
File path of chunk dictionary for data deduplication
--parent-bootstrap <parent-bootstrap>
File path of the parent/referenced RAFS metadata blob (optional)
--aligned-chunk
Align uncompressed data chunks to 4K, only for RAFS V5
--repeatable
Generate reproducible RAFS metadata
--whiteout-spec <whiteout-spec>
Set the type of whiteout specification: [default: oci] [possible values: oci, overlayfs, none]
--prefetch-policy <prefetch-policy>
Set data prefetch policy [default: none] [possible values: fs, blob, none]
-J, --output-json <output-json>
File path to save operation result in JSON format
-h, --help
Print help information
`),
},
{
name: "detectFeature should support empty input",
feature: "",
Expand Down Expand Up @@ -597,7 +725,7 @@ func TestDetectFeatures(t *testing.T) {
expectErr: false,
},
{
name: "should not support '--batch-size' and '--type tar-rafs' in v2.1.4",
name: "should not support '--encrypt', '--batch-size' or '--type tar-rafs' in v2.1.4",
resetGlobal: true,
disableTar2Rafs: true,
helpText: []byte(`
Expand Down Expand Up @@ -662,7 +790,7 @@ func TestDetectFeatures(t *testing.T) {
ARGS:
<SOURCE>... source path to build the nydus image from
`),
required: Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}},
required: Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}, FeatureEncrypt: {}},
detected: Features{},
expectErr: false,
},
Expand Down
2 changes: 2 additions & 0 deletions pkg/converter/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ type PackOption struct {
Backend Backend
// Timeout cancels execution once exceed the specified time.
Timeout *time.Duration
// Whether the generated Nydus blobs should be encrypted.
Encrypt bool

// Features keeps a feature list supported by newer version of builder,
// It is detected automatically, so don't export it.
Expand Down