Skip to content

Commit

Permalink
feats: support encrypting nydus data blobs
Browse files Browse the repository at this point in the history 
Support encrypting nydus data blobs.

Signed-off-by: taohong <[email protected]>
  • Loading branch information
@taoohong
taoohong committed Jul 13, 2023
1 parent 3338db6 commit 3915fce
Show file tree
Hide file tree
Showing 6 changed files with 154 additions and 2 deletions.
1 change: 1 addition & 0 deletions pkg/converter/constant.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ const (
LayerAnnotationNydusBlobSize = "containerd.io/snapshot/nydus-blob-size"
LayerAnnotationNydusBootstrap = "containerd.io/snapshot/nydus-bootstrap"
LayerAnnotationNydusSourceChainID = "containerd.io/snapshot/nydus-source-chainid"
LayerAnnotationNydusEncryptedBlob = "containerd.io/snapshot/nydus-encrypted-blob"

LayerAnnotationNydusReferenceBlobIDs = "containerd.io/snapshot/nydus-reference-blob-ids"

Expand Down
14 changes: 14 additions & 0 deletions pkg/converter/convert_unix.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,9 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
if opt.BatchSize != "" && opt.BatchSize != "0" {
requiredFeatures.Add(tool.FeatureBatchSize)
}
if opt.Encrypt {
requiredFeatures.Add(tool.FeatureEncrypt)
}

detectedFeatures, err := tool.DetectFeatures(builderPath, requiredFeatures, tool.GetHelp)
if err != nil {
Expand Down Expand Up @@ -407,6 +410,7 @@ func packFromDirectory(ctx context.Context, dest io.Writer, opt PackOption, buil
ChunkSize: opt.ChunkSize,
Compressor: opt.Compressor,
Timeout: opt.Timeout,
Encrypt: opt.Encrypt,

Features: opt.features,
})
Expand Down Expand Up @@ -510,6 +514,7 @@ func packFromTar(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteC
BatchSize: opt.BatchSize,
Compressor: opt.Compressor,
Timeout: opt.Timeout,
Encrypt: opt.Encrypt,

Features: opt.features,
})
Expand Down Expand Up @@ -852,6 +857,10 @@ func LayerConvertFunc(opt PackOption) converter.ConvertFunc {
newDesc.Annotations[label.NydusRefLayer] = desc.Digest.String()
}

if opt.Encrypt {
newDesc.Annotations[LayerAnnotationNydusEncryptedBlob] = "true"
}

if opt.Backend != nil {
if err := opt.Backend.Push(ctx, cs, newDesc); err != nil {
return nil, errors.Wrap(err, "push to storage backend")
Expand Down Expand Up @@ -1134,6 +1143,11 @@ func MergeLayers(ctx context.Context, cs content.Store, descs []ocispec.Descript
if opt.OCIRef {
blobDesc.Annotations[label.NydusRefLayer] = layers[idx].OriginalDigest.String()
}

if len(opt.EncryptRecipients) != 0 {
blobDesc.Annotations[LayerAnnotationNydusEncryptedBlob] = "true"
}

blobDescs = append(blobDescs, blobDesc)
}

Expand Down
4 changes: 4 additions & 0 deletions pkg/converter/tool/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ type PackOption struct {
AlignedChunk bool
ChunkSize string
BatchSize string
Encrypt bool
Timeout *time.Duration

Features Features
Expand Down Expand Up @@ -136,6 +137,9 @@ func buildPackArgs(option PackOption) []string {
if option.Features.Contains(FeatureBatchSize) {
args = append(args, "--batch-size", option.BatchSize)
}
if option.Encrypt {
args = append(args, "--encrypt")
}

Check warning on line 142 in pkg/converter/tool/builder.go

View check run for this annotation

Codecov / codecov/patch

pkg/converter/tool/builder.go#L140-L142 

Added lines #L140 - L142 were not covered by tests
args = append(args, option.SourcePath)

return args
Expand Down
3 changes: 3 additions & 0 deletions pkg/converter/tool/feature.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ const (
// into a big batch chunk, which can reduce the the size of the image
// and accelerate the runtime file loading.
FeatureBatchSize Feature = "--batch-size"
// The option `--encrypt` enables converting directories, tar files
// or OCI images into encrypted nydus blob.
FeatureEncrypt Feature = "--encrypt"
)

var requiredFeatures Features
Expand Down
132 changes: 130 additions & 2 deletions pkg/converter/tool/feature_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,71 @@ func TestDetectFeature(t *testing.T) {
Print help information
`),
},
{
name: "'--encrypt' is supported in v2.2.0-261-g22ad0e2c",
feature: FeatureEncrypt,
expect: true,
helpMsg: []byte(`
Create RAFS filesystems from directories, tar files or OCI images
Usage: nydus-image create [OPTIONS] <SOURCE>
Arguments:
<SOURCE> source from which to build the RAFS filesystem
Options:
-L, --log-file <log-file>
Log file path
-t, --type <type>
Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index]
-B, --bootstrap <bootstrap>
File path to save the generated RAFS metadata blob
-l, --log-level <log-level>
Log level: [default: info] [possible values: trace, debug, info, warn, error]
-D, --blob-dir <blob-dir>
Directory path to save generated RAFS metadata and data blobs
-b, --blob <blob>
File path to save the generated RAFS data blob
--blob-inline-meta
Inline RAFS metadata and blob metadata into the data blob
--blob-id <blob-id>
OSS object id for the generated RAFS data blob
--blob-data-size <blob-data-size>
Set data blob size for 'estargztoc-ref' conversion
--chunk-size <chunk-size>
Set the size of data chunks, must be power of two and between 0x1000-0x1000000:
--batch-size <batch-size>
Set the batch size to merge small chunks, must be power of two, between 0x1000-0x1000000 or be zero: [default: 0]
--compressor <compressor>
Algorithm to compress data chunks: [default: zstd] [possible values: none, lz4_block, zstd]
--digester <digester>
Algorithm to digest data chunks: [default: blake3] [possible values: blake3, sha256]
-C, --config <config>
Configuration file for storage backend, cache and RAFS FUSE filesystem.
-v, --fs-version <fs-version>
Set RAFS format version number: [default: 6] [possible values: 5, 6]
--features <features>
Enable/disable features [possible values: blob-toc]
--chunk-dict <chunk-dict>
File path of chunk dictionary for data deduplication
--parent-bootstrap <parent-bootstrap>
File path of the parent/referenced RAFS metadata blob (optional)
--aligned-chunk
Align uncompressed data chunks to 4K, only for RAFS V5
--repeatable
Generate reproducible RAFS metadata
--whiteout-spec <whiteout-spec>
Set the type of whiteout specification: [default: oci] [possible values: oci, overlayfs, none]
--prefetch-policy <prefetch-policy>
Set data prefetch policy [default: none] [possible values: fs, blob, none]
-J, --output-json <output-json>
File path to save operation result in JSON format
-E, --encrypt
Encrypt the generated RAFS metadata and data blobs
-h, --help
Print help information
`),
},

{
name: "'--type tar-rafs' is not supported in v2.1.4",
Expand Down Expand Up @@ -552,6 +617,69 @@ func TestDetectFeature(t *testing.T) {
<SOURCE> source path
`),
},
{
name: "'--encrypt' is not supported in v2.2.0",
feature: FeatureEncrypt,
expect: false,
helpMsg: []byte(`
Create RAFS filesystems from directories, tar files or OCI images
Usage: nydus-image create [OPTIONS] <SOURCE>
Arguments:
<SOURCE> source from which to build the RAFS filesystem
Options:
-L, --log-file <log-file>
Log file path
-t, --type <type>
Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index]
-B, --bootstrap <bootstrap>
File path to save the generated RAFS metadata blob
-l, --log-level <log-level>
Log level: [default: info] [possible values: trace, debug, info, warn, error]
-D, --blob-dir <blob-dir>
Directory path to save generated RAFS metadata and data blobs
-b, --blob <blob>
File path to save the generated RAFS data blob
--blob-inline-meta
Inline RAFS metadata and blob metadata into the data blob
--blob-id <blob-id>
OSS object id for the generated RAFS data blob
--blob-data-size <blob-data-size>
Set data blob size for 'estargztoc-ref' conversion
--chunk-size <chunk-size>
Set the size of data chunks, must be power of two and between 0x1000-0x1000000:
--batch-size <batch-size>
Set the batch size to merge small chunks, must be power of two, between 0x1000-0x1000000 or be zero: [default: 0]
--compressor <compressor>
Algorithm to compress data chunks: [default: zstd] [possible values: none, lz4_block, zstd]
--digester <digester>
Algorithm to digest data chunks: [default: blake3] [possible values: blake3, sha256]
-C, --config <config>
Configuration file for storage backend, cache and RAFS FUSE filesystem.
-v, --fs-version <fs-version>
Set RAFS format version number: [default: 6] [possible values: 5, 6]
--features <features>
Enable/disable features [possible values: blob-toc]
--chunk-dict <chunk-dict>
File path of chunk dictionary for data deduplication
--parent-bootstrap <parent-bootstrap>
File path of the parent/referenced RAFS metadata blob (optional)
--aligned-chunk
Align uncompressed data chunks to 4K, only for RAFS V5
--repeatable
Generate reproducible RAFS metadata
--whiteout-spec <whiteout-spec>
Set the type of whiteout specification: [default: oci] [possible values: oci, overlayfs, none]
--prefetch-policy <prefetch-policy>
Set data prefetch policy [default: none] [possible values: fs, blob, none]
-J, --output-json <output-json>
File path to save operation result in JSON format
-h, --help
Print help information
`),
},
{
name: "detectFeature should support empty input",
feature: "",
Expand Down Expand Up @@ -597,7 +725,7 @@ func TestDetectFeatures(t *testing.T) {
expectErr: false,
},
{
name: "should not support '--batch-size' and '--type tar-rafs' in v2.1.4",
name: "should not support '--encrypt', '--batch-size' or '--type tar-rafs' in v2.1.4",
resetGlobal: true,
disableTar2Rafs: true,
helpText: []byte(`
Expand Down Expand Up @@ -662,7 +790,7 @@ func TestDetectFeatures(t *testing.T) {
ARGS:
<SOURCE>... source path to build the nydus image from
`),
required: Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}},
required: Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}, FeatureEncrypt: {}},
detected: Features{},
expectErr: false,
},
Expand Down
2 changes: 2 additions & 0 deletions pkg/converter/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ type PackOption struct {
Backend Backend
// Timeout cancels execution once exceed the specified time.
Timeout *time.Duration
// Whether the generated Nydus blobs should be encrypted.
Encrypt bool

// Features keeps a feature list supported by newer version of builder,
// It is detected automatically, so don't export it.
Expand Down

0 comments on commit 3915fce

Please sign in to comment.