Skip to content

Commit

Permalink
Merge pull request #117 from AkihiroSuda/dev
Browse files Browse the repository at this point in the history 
Add scale factor for maxTarBlobSize etc.
  • Loading branch information
@AkihiroSuda
AkihiroSuda committed Aug 16, 2024
2 parents 46e0283 + 4471fcb commit b63c89d
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 18 deletions.
6 changes: 6 additions & 0 deletions cmd/diffoci/commands/diff/diff.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ func NewCommand() *cobra.Command {
flags.String("report-file", "", "Create a report file to the specified path (EXPERIMENTAL)")
flags.String("report-dir", "", "Create a detailed report in the specified directory")
flags.String("pull", imagegetter.PullMissing, "Pull mode (always|missing|never)")
flags.Float64("max-scale", 1.0, "Scale factor for maximum values (e.g., maxTarBlobSize = 4GiB)")
return cmd
}

Expand Down Expand Up @@ -153,6 +154,11 @@ func action(cmd *cobra.Command, args []string) error {
options.EventHandler = diff.VerboseEventHandler
}

options.MaxScale, err = flags.GetFloat64("max-scale")
if err != nil {
return err
}

pullMode, err := flags.GetString("pull")
if err != nil {
return err
Expand Down
41 changes: 23 additions & 18 deletions pkg/diff/diff.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ type Options struct {
EventHandler
ReportFile string
ReportDir string
MaxScale float64
}

func (o *Options) digestMayChange() bool {
Expand Down Expand Up @@ -101,6 +102,9 @@ func Diff(ctx context.Context, cs content.Provider, descs [2]ocispec.Descriptor,
}
reportFiles = append(reportFiles, filepath.Join(o.ReportDir, ReportDirReportJSON))
}
if o.MaxScale == 0.0 {
o.MaxScale = 1.0
}
d := differ{
cs: cs,
platMC: platMC,
Expand Down Expand Up @@ -404,7 +408,7 @@ func (d *differ) diffAnnotationsField(ctx context.Context, node *EventTreeNode,
func (d *differ) diffIndex(ctx context.Context, node *EventTreeNode, in [2]EventInput) error {
for i := 0; i < 2; i++ {
var err error
in[i].Index, err = readBlobWithType[ocispec.Index](ctx, d.cs, *in[i].Descriptor)
in[i].Index, err = readBlobWithType[ocispec.Index](ctx, d.cs, *in[i].Descriptor, d.o.MaxScale)
if err != nil {
return fmt.Errorf("failed to read index (%v): %w", in[i].Descriptor, err) // critical, not joined
}
Expand All @@ -431,7 +435,7 @@ func (d *differ) diffIndex(ctx context.Context, node *EventTreeNode, in [2]Event
if err := d.diffDescriptorSliceField(ctx, node, in, EventTypeIndexBlobMismatch, [2][]ocispec.Descriptor{
in[0].Index.Manifests,
in[1].Index.Manifests,
}, "Manifests", maxManifests,
}, "Manifests", int(maxManifests*d.o.MaxScale),
func(desc ocispec.Descriptor) (tolerable bool, vErr error) {
if !images.IsManifestType(desc.MediaType) {
return false, fmt.Errorf("expected a manifest type, got %q", desc.MediaType)
Expand Down Expand Up @@ -469,7 +473,7 @@ func (d *differ) diffManifest(ctx context.Context, node *EventTreeNode, in [2]Ev
}
for i := 0; i < 2; i++ {
var err error
in[i].Manifest, err = readBlobWithType[ocispec.Manifest](ctx, d.cs, *in[i].Descriptor)
in[i].Manifest, err = readBlobWithType[ocispec.Manifest](ctx, d.cs, *in[i].Descriptor, d.o.MaxScale)
if err != nil {
return fmt.Errorf("failed to read manifest (%v): %w", in[i].Descriptor, err)
}
Expand Down Expand Up @@ -503,7 +507,7 @@ func (d *differ) diffManifest(ctx context.Context, node *EventTreeNode, in [2]Ev
if err := d.diffDescriptorSliceField(ctx, node, in, EventTypeManifestBlobMismatch, [2][]ocispec.Descriptor{
in[0].Manifest.Layers,
in[1].Manifest.Layers,
}, "Layers", maxLayers,
}, "Layers", int(maxLayers*d.o.MaxScale),
func(desc ocispec.Descriptor) (tolerable bool, vErr error) {
if !images.IsLayerType(desc.MediaType) {
return false, fmt.Errorf("expected a layer type, got %q", desc.MediaType)
Expand Down Expand Up @@ -541,7 +545,7 @@ func (d *differ) diffManifest(ctx context.Context, node *EventTreeNode, in [2]Ev
func (d *differ) diffConfig(ctx context.Context, node *EventTreeNode, in [2]EventInput) error {
for i := 0; i < 2; i++ {
var err error
in[i].Config, err = readBlobWithType[ocispec.Image](ctx, d.cs, *in[i].Descriptor)
in[i].Config, err = readBlobWithType[ocispec.Image](ctx, d.cs, *in[i].Descriptor, d.o.MaxScale)
if err != nil {
return fmt.Errorf("failed to read config (%v): %w", in[i].Descriptor, err)
}
Expand Down Expand Up @@ -622,7 +626,7 @@ func (d *differ) diffConfig(ctx context.Context, node *EventTreeNode, in [2]Even
}

func (d *differ) diffLayersWithSquashing(ctx context.Context, node *EventTreeNode, in [2]EventInput) error {
tr0, trCloser0, err := openTarReaderWithSquashing(ctx, d.cs, in[0].Manifest.Layers)
tr0, trCloser0, err := openTarReaderWithSquashing(ctx, d.cs, in[0].Manifest.Layers, d.o.MaxScale)
if err != nil {
return err
}
Expand All @@ -632,7 +636,7 @@ func (d *differ) diffLayersWithSquashing(ctx context.Context, node *EventTreeNod
}
}()

tr1, trCloser1, err := openTarReaderWithSquashing(ctx, d.cs, in[1].Manifest.Layers)
tr1, trCloser1, err := openTarReaderWithSquashing(ctx, d.cs, in[1].Manifest.Layers, d.o.MaxScale)
if err != nil {
return err
}
Expand All @@ -645,7 +649,7 @@ func (d *differ) diffLayersWithSquashing(ctx context.Context, node *EventTreeNod
}

func (d *differ) diffLayer(ctx context.Context, node *EventTreeNode, in [2]EventInput) error {
tr0, trCloser0, err := openTarReader(ctx, d.cs, *in[0].Descriptor)
tr0, trCloser0, err := openTarReader(ctx, d.cs, *in[0].Descriptor, d.o.MaxScale)
if err != nil {
return err
}
Expand All @@ -655,7 +659,7 @@ func (d *differ) diffLayer(ctx context.Context, node *EventTreeNode, in [2]Event
}
}()

tr1, trCloser1, err := openTarReader(ctx, d.cs, *in[1].Descriptor)
tr1, trCloser1, err := openTarReader(ctx, d.cs, *in[1].Descriptor, d.o.MaxScale)
if err != nil {
return err
}
Expand Down Expand Up @@ -894,9 +898,9 @@ func (d *differ) diffTarEntry(ctx context.Context, node *EventTreeNode, in [2]Ev
return dirsToBeRemovedIfEmpty, errors.Join(errs...)
}

func openTarReader(ctx context.Context, cs content.Provider, desc ocispec.Descriptor) (tr tarReader, closer func() error, err error) {
if desc.Size > int64(maxTarBlobSize) {
return nil, nil, fmt.Errorf("too large tar blob (%d > %d bytes)", desc.Size, int64(maxTarBlobSize))
func openTarReader(ctx context.Context, cs content.Provider, desc ocispec.Descriptor, maxScale float64) (tr tarReader, closer func() error, err error) {
if desc.Size > int64(maxTarBlobSize*maxScale) {
return nil, nil, fmt.Errorf("too large tar blob (%d > %d bytes)", desc.Size, int64(maxTarBlobSize*maxScale))
}
ra, err := cs.ReaderAt(ctx, desc)
if err != nil {
Expand All @@ -908,16 +912,16 @@ func openTarReader(ctx context.Context, cs content.Provider, desc ocispec.Descri
ra.Close()
return nil, nil, err
}
lr := io.LimitReader(dr, maxTarStreamSize)
lr := io.LimitReader(dr, int64(maxTarStreamSize*maxScale))
return tar.NewReader(lr), ra.Close, nil
}

func openTarReaderWithSquashing(ctx context.Context, cs content.Provider, descs []ocispec.Descriptor) (tr tarReader, closer func() error, err error) {
func openTarReaderWithSquashing(ctx context.Context, cs content.Provider, descs []ocispec.Descriptor, maxScale float64) (tr tarReader, closer func() error, err error) {
tarReaders := make([]tarReader, len(descs))
closers := make([]func() error, len(descs))
for i := 0; i < len(descs); i++ {
var err error
tarReaders[i], closers[i], err = openTarReader(ctx, cs, descs[i])
tarReaders[i], closers[i], err = openTarReader(ctx, cs, descs[i], maxScale)
if err != nil {
return nil, nil, err
}
Expand Down Expand Up @@ -963,9 +967,9 @@ begin:

func readBlobWithType[T interface {
ocispec.Index | ocispec.Manifest | ocispec.Image
}](ctx context.Context, cs content.Provider, desc ocispec.Descriptor) (*T, error) {
if desc.Size > maxJSONBlobSize {
return nil, fmt.Errorf("too large JSON blob (%d > %d bytes)", desc.Size, maxJSONBlobSize)
}](ctx context.Context, cs content.Provider, desc ocispec.Descriptor, maxScale float64) (*T, error) {
if desc.Size > int64(maxJSONBlobSize*maxScale) {
return nil, fmt.Errorf("too large JSON blob (%d > %d bytes)", desc.Size, int64(maxJSONBlobSize*maxScale))
}
b, err := content.ReadBlob(ctx, cs, desc)
if err != nil {
Expand Down Expand Up @@ -1039,6 +1043,7 @@ const (
EventTypeTarEntryMismatch = EventType("TarEntryMismatch")
)

// MaxScale option is multiplied to these constants
const (
maxManifests = 4096
maxLayers = 4096
Expand Down

0 comments on commit b63c89d

Please sign in to comment.