Skip to content

Commit

Permalink
refactor: upgrade to distribution-spec v1.1.0-rc4 (#690)
Browse files Browse the repository at this point in the history
1. If the `OCI-Subject` header is not set in the response of manifest
push, do not ping the Referrers API (see `repository.go`)
2. Update tests accordingly
3. Update the spec version in comments

Resolve: #685
Signed-off-by: Lixia (Sylvia) Lei <[email protected]>
  • Loading branch information
Wwwsylvia authored Jan 23, 2024
1 parent 64fedf4 commit 66ea3df
Show file tree
Hide file tree
Showing 11 changed files with 52 additions and 373 deletions.
6 changes: 3 additions & 3 deletions content/oci/oci.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ limitations under the License.
*/

// Package oci provides access to an OCI content store.
// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc5/image-layout.md
// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc6/image-layout.md
package oci

import (
Expand Down Expand Up @@ -43,7 +43,7 @@ import (

// Store implements `oras.Target`, and represents a content store
// based on file system with the OCI-Image layout.
// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc5/image-layout.md
// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc6/image-layout.md
type Store struct {
// AutoSaveIndex controls if the OCI store will automatically save the index
// file when needed.
Expand Down Expand Up @@ -228,7 +228,7 @@ func (s *Store) delete(ctx context.Context, target ocispec.Descriptor) ([]ocispe

// Tag tags a descriptor with a reference string.
// reference should be a valid tag (e.g. "latest").
// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc5/image-layout.md#indexjson-file
// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc6/image-layout.md#indexjson-file
func (s *Store) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error {
s.sync.RLock()
defer s.sync.RUnlock()
Expand Down
4 changes: 2 additions & 2 deletions registry/reference.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,13 @@ var (
//
// References:
// - https://github.com/distribution/distribution/blob/v2.7.1/reference/regexp.go#L53
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#pulling-manifests
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#pulling-manifests
repositoryRegexp = regexp.MustCompile(`^[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*)*$`)

// tagRegexp checks the tag name.
// The docker and OCI spec have the same regular expression.
//
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#pulling-manifests
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#pulling-manifests
tagRegexp = regexp.MustCompile(`^[\w][\w.-]{0,127}$`)
)

Expand Down
6 changes: 3 additions & 3 deletions registry/remote/errcode/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import (
)

// References:
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#error-codes
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#error-codes
// - https://docs.docker.com/registry/spec/api/#errors-2
const (
ErrorCodeBlobUnknown = "BLOB_UNKNOWN"
Expand All @@ -45,7 +45,7 @@ const (
// Error represents a response inner error returned by the remote
// registry.
// References:
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#error-codes
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#error-codes
// - https://docs.docker.com/registry/spec/api/#errors-2
type Error struct {
Code string `json:"code"`
Expand Down Expand Up @@ -73,7 +73,7 @@ func (e Error) Error() string {
// Errors represents a list of response inner errors returned by the remote
// server.
// References:
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#error-codes
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#error-codes
// - https://docs.docker.com/registry/spec/api/#errors-2
type Errors []Error

Expand Down
1 change: 1 addition & 0 deletions registry/remote/example_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,7 @@ func TestMain(m *testing.M) {
case p == fmt.Sprintf("/v2/%s/manifests/%s", exampleRepositoryName, ManifestDigest) && m == "PUT":
w.WriteHeader(http.StatusCreated)
case p == fmt.Sprintf("/v2/%s/manifests/%s", exampleRepositoryName, ReferenceManifestDigest) && m == "PUT":
w.Header().Set("OCI-Subject", "sha256:a3f9d449466b9b7194c3a76ca4890d792e11eb4e62e59aa8b4c3cce0a56f129d")
w.WriteHeader(http.StatusCreated)
case p == fmt.Sprintf("/v2/%s/manifests/%s", exampleRepositoryName, exampleSignatureManifestDescriptor.Digest) && m == "GET":
w.Header().Set("Content-Type", spec.MediaTypeArtifactManifest)
Expand Down
2 changes: 1 addition & 1 deletion registry/remote/referrers.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func (e *ReferrersError) IsReferrersIndexDelete() bool {

// buildReferrersTag builds the referrers tag for the given manifest descriptor.
// Format: <algorithm>-<digest>
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#unavailable-referrers-api
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#unavailable-referrers-api
func buildReferrersTag(desc ocispec.Descriptor) string {
alg := desc.Digest.Algorithm().String()
encoded := desc.Digest.Encoded()
Expand Down
2 changes: 1 addition & 1 deletion registry/remote/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ func (r *Registry) do(req *http.Request) (*http.Response, error) {
//
// References:
// - https://docs.docker.com/registry/spec/api/#base
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#api
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#api
func (r *Registry) Ping(ctx context.Context) error {
url := buildRegistryBaseURL(r.PlainHTTP, r.Reference)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
Expand Down
67 changes: 35 additions & 32 deletions registry/remote/repository.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,15 +53,15 @@ const (
//
// References:
// - https://docs.docker.com/registry/spec/api/#digest-header
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#pull
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#pull
headerDockerContentDigest = "Docker-Content-Digest"

// headerOCIFiltersApplied is the "OCI-Filters-Applied" header.
// If present on the response, it contains a comma-separated list of the
// applied filters.
//
// Reference:
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#listing-referrers
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#listing-referrers
headerOCIFiltersApplied = "OCI-Filters-Applied"

// headerOCISubject is the "OCI-Subject" header.
Expand All @@ -74,7 +74,7 @@ const (
// referrers.
//
// References:
// - Latest spec: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#listing-referrers
// - Latest spec: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#listing-referrers
// - Compatible spec: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#listing-referrers
const filterTypeArtifactType = "artifactType"

Expand Down Expand Up @@ -118,7 +118,7 @@ type Repository struct {
// ReferrerListPageSize specifies the page size when invoking the Referrers
// API.
// If zero, the page size is determined by the remote registry.
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#listing-referrers
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#listing-referrers
ReferrerListPageSize int

// MaxMetadataBytes specifies a limit on how many response bytes are allowed
Expand All @@ -133,16 +133,16 @@ type Repository struct {
// is successfully uploaded.
// - If true, the old referrers index is kept.
// By default, it is disabled (set to false). See also:
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#referrers-tag-schema
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#pushing-manifests-with-subject
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#deleting-manifests
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#referrers-tag-schema
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#pushing-manifests-with-subject
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#deleting-manifests
SkipReferrersGC bool

// HandleWarning handles the warning returned by the remote server.
// Callers SHOULD deduplicate warnings from multiple associated responses.
//
// References:
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#warnings
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#warnings
// - https://www.rfc-editor.org/rfc/rfc7234#section-5.5
HandleWarning func(warning Warning)

Expand Down Expand Up @@ -212,9 +212,9 @@ func (r *Repository) clone() *Repository {
// SetReferrersCapability returns ErrReferrersCapabilityAlreadySet if the
// Referrers API capability has been already set.
// - When the capability is set to true, the Referrers() function will always
// request the Referrers API. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#listing-referrers
// request the Referrers API. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#listing-referrers
// - When the capability is set to false, the Referrers() function will always
// request the Referrers Tag. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#referrers-tag-schema
// request the Referrers Tag. Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#referrers-tag-schema
// - When the capability is not set, the Referrers() function will automatically
// determine which API to use.
func (r *Repository) SetReferrersCapability(capable bool) error {
Expand Down Expand Up @@ -388,7 +388,7 @@ func (r *Repository) ParseReference(reference string) (registry.Reference, error
// of the Tags list.
//
// References:
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#content-discovery
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#content-discovery
// - https://docs.docker.com/registry/spec/api/#tags
func (r *Repository) Tags(ctx context.Context, last string, fn func(tags []string) error) error {
ctx = auth.AppendRepositoryScope(ctx, r.Reference, auth.ActionPull)
Expand Down Expand Up @@ -447,7 +447,7 @@ func (r *Repository) tags(ctx context.Context, last string, fn func(tags []strin
// Predecessors returns the descriptors of image or artifact manifests directly
// referencing the given manifest descriptor.
// Predecessors internally leverages Referrers.
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#listing-referrers
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#listing-referrers
func (r *Repository) Predecessors(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
var res []ocispec.Descriptor
if err := r.Referrers(ctx, desc, "", func(referrers []ocispec.Descriptor) error {
Expand All @@ -466,7 +466,7 @@ func (r *Repository) Predecessors(ctx context.Context, desc ocispec.Descriptor)
// If artifactType is not empty, only referrers of the same artifact type are
// fed to fn.
//
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#listing-referrers
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#listing-referrers
func (r *Repository) Referrers(ctx context.Context, desc ocispec.Descriptor, artifactType string, fn func(referrers []ocispec.Descriptor) error) error {
state := r.loadReferrersState()
if state == referrersStateUnsupported {
Expand Down Expand Up @@ -565,7 +565,7 @@ func (r *Repository) referrersPageByAPI(ctx context.Context, artifactType string
referrers := index.Manifests
if artifactType != "" {
// check both filters header and filters annotations for compatibility
// latest spec for filters header: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#listing-referrers
// latest spec for filters header: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#listing-referrers
// older spec for filters annotations: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#listing-referrers
filtersHeader := resp.Header.Get(headerOCIFiltersApplied)
filtersAnnotation := index.Annotations[spec.AnnotationReferrersFiltersApplied]
Expand All @@ -587,7 +587,7 @@ func (r *Repository) referrersPageByAPI(ctx context.Context, artifactType string
// referencing the given manifest descriptor by requesting referrers tag.
// fn is called for the referrers result. If artifactType is not empty,
// only referrers of the same artifact type are fed to fn.
// reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#backwards-compatibility
// reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#backwards-compatibility
func (r *Repository) referrersByTagSchema(ctx context.Context, desc ocispec.Descriptor, artifactType string, fn func(referrers []ocispec.Descriptor) error) error {
referrersTag := buildReferrersTag(desc)
_, referrers, err := r.referrersFromIndex(ctx, referrersTag)
Expand Down Expand Up @@ -801,7 +801,7 @@ func (s *blobStore) Mount(ctx context.Context, desc ocispec.Descriptor, fromRepo
// push it. If the caller has provided a getContent function, we
// can use that, otherwise pull the content from the source repository.
//
// [spec]: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#mounting-a-blob-from-another-repository
// [spec]: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#mounting-a-blob-from-another-repository

var r io.ReadCloser
if getContent != nil {
Expand Down Expand Up @@ -836,7 +836,7 @@ func (s *blobStore) sibling(otherRepoName string) *blobStore {
// References:
// - https://docs.docker.com/registry/spec/api/#pushing-an-image
// - https://docs.docker.com/registry/spec/api/#initiate-blob-upload
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#pushing-a-blob-monolithically
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#pushing-a-blob-monolithically
func (s *blobStore) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) error {
// start an upload
// pushing usually requires both pull and push actions.
Expand Down Expand Up @@ -1146,7 +1146,7 @@ func (s *manifestStore) deleteWithIndexing(ctx context.Context, target ocispec.D
// on manifest delete.
//
// References:
// - Latest spec: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#deleting-manifests
// - Latest spec: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#deleting-manifests
// - Compatible spec: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#deleting-manifests
func (s *manifestStore) indexReferrersForDelete(ctx context.Context, desc ocispec.Descriptor, manifestJSON []byte) error {
var manifest struct {
Expand Down Expand Up @@ -1332,13 +1332,19 @@ func (s *manifestStore) push(ctx context.Context, expected ocispec.Descriptor, c

// checkOCISubjectHeader checks the "OCI-Subject" header in the response and
// sets referrers capability accordingly.
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#pushing-manifests-with-subject
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#pushing-manifests-with-subject
func (s *manifestStore) checkOCISubjectHeader(resp *http.Response) {
// Referrers capability is not set to false when the subject header is not
// present, as the server may still conform to an older version of the spec
// If the "OCI-Subject" header is set, it indicates that the registry
// supports the Referrers API and has processed the subject of the manifest.
if subjectHeader := resp.Header.Get(headerOCISubject); subjectHeader != "" {
s.repo.SetReferrersCapability(true)
}

// If the "OCI-Subject" header is NOT set, it means that either the manifest
// has no subject OR the referrers API is NOT supported by the registry.
//
// Since we don't know whether the pushed manifest has a subject or not,
// we do not set the referrers capability to false at here.
}

// pushWithIndexing pushes the manifest content matching the expected descriptor,
Expand All @@ -1363,6 +1369,8 @@ func (s *manifestStore) pushWithIndexing(ctx context.Context, expected ocispec.D
}
// check referrers API availability again after push
if state := s.repo.loadReferrersState(); state == referrersStateSupported {
// the subject has been processed the registry, no client-side
// indexing needed
return nil
}
return s.indexReferrersForPush(ctx, expected, manifestJSON)
Expand All @@ -1375,7 +1383,7 @@ func (s *manifestStore) pushWithIndexing(ctx context.Context, expected ocispec.D
// on manifest push.
//
// References:
// - Latest spec: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#pushing-manifests-with-subject
// - Latest spec: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#pushing-manifests-with-subject
// - Compatible spec: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#pushing-manifests-with-subject
func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec.Descriptor, manifestJSON []byte) error {
var subject ocispec.Descriptor
Expand Down Expand Up @@ -1423,22 +1431,17 @@ func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec.
return nil
}

ok, err := s.repo.pingReferrers(ctx)
if err != nil {
return err
}
if ok {
// referrers API is available, no client-side indexing needed
return nil
}
// if the manifest has a subject but the remote registry does not process it,
// it means that the Referrers API is not supported by the registry.
s.repo.SetReferrersCapability(false)
return s.updateReferrersIndex(ctx, subject, referrerChange{desc, referrerOperationAdd})
}

// updateReferrersIndex updates the referrers index for desc referencing subject
// on manifest push and manifest delete.
// References:
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#pushing-manifests-with-subject
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc3/spec.md#deleting-manifests
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#pushing-manifests-with-subject
// - https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc4/spec.md#deleting-manifests
func (s *manifestStore) updateReferrersIndex(ctx context.Context, subject ocispec.Descriptor, change referrerChange) (err error) {
referrersTag := buildReferrersTag(subject)

Expand Down
Loading

0 comments on commit 66ea3df

Please sign in to comment.