feat: add oci pull capabilities

go.mod

@@ -59,6 +59,7 @@ require (
 	golang.org/x/tools v0.25.0
 	google.golang.org/grpc v1.67.1
 	helm.sh/helm/v3 v3.16.1
+	oras.land/oras-go/v2 v2.5.0
 )
 
 // Kubernetes

go.sum

@@ -882,6 +882,8 @@ k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1
 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo=
 oras.land/oras-go v1.2.5/go.mod h1:PuAwRShRZCsZb7g8Ar3jKKQR/2A/qN+pkYxIOd/FAoo=
+oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c=
+oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
 sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q=

internal/oci/oci.go

@@ -0,0 +1,168 @@
+/*
+Copyright 2024 k0s authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"oras.land/oras-go/v2"
+	"oras.land/oras-go/v2/content/file"
+	"oras.land/oras-go/v2/registry"
+	"oras.land/oras-go/v2/registry/remote"
+	"oras.land/oras-go/v2/registry/remote/auth"
+	"oras.land/oras-go/v2/registry/remote/credentials"
+)
+
+// Download downloads the OCI artifact present at the given registry URL.
+// Usage example:
+//
+// artifact := "docker.io/company/k0s:latest"
+// fp, _ := os.CreateTemp("", "k0s-oci-artifact-*")
+// err := oci.Download(ctx, artifact, fp)
+//
+// This function expects only one artifact to be present, if none is found this
+// returns an error. The artifact is downloaded in a temporary location before
+// being copied to the target.
+func Download(ctx context.Context, url string, target io.Writer, options ...DownloadOption) (err error) {
+	opts := downloadOptions{}
+	for _, opt := range options {
+		opt(&opts)
+	}
+
+	creds, err := opts.auth.CredentialStore(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to create credential store: %w", err)
+	}
+
+	imgref, err := registry.ParseReference(url)
+	if err != nil {
+		return fmt.Errorf("failed to parse artifact reference: %w", err)
+	}
+
+	tmpdir, err := os.MkdirTemp("", "k0s-oci-artifact-*")
+	if err != nil {
+		return fmt.Errorf("failed to create temp dir: %w", err)
+	}
+	defer func() { _ = os.RemoveAll(tmpdir) }()
+
+	repo, err := remote.NewRepository(url)
+	if err != nil {
+		return fmt.Errorf("failed to create repository: %w", err)
+	}
+
+	fs, err := file.New(tmpdir)
+	if err != nil {
+		return fmt.Errorf("failed to create file store: %w", err)
+	}
+	defer fs.Close()
+
+	transp := http.DefaultTransport.(*http.Transport).Clone()
+	if opts.insecureSkipTLSVerify {
+		transp.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+
+	repo.Client = &auth.Client{
+		Client:     &http.Client{Transport: transp},
+		Credential: creds.Get,
+	}
+
+	tag := imgref.Reference
+	if _, err := oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions); err != nil {
+		return fmt.Errorf("failed to fetch artifact: %w", err)
+	}
+
+	files, err := os.ReadDir(tmpdir)
+	if err != nil {
+		return fmt.Errorf("failed to read temp dir: %w", err)
+	}
+
+	// we always expect only one single file to be downloaded.
+	if len(files) == 0 {
+		return fmt.Errorf("no artifacts found")
+	} else if len(files) > 1 {
+		return fmt.Errorf("multiple artifacts found")
+	}
+
+	fpath := filepath.Join(tmpdir, files[0].Name())
+	fp, err := os.Open(fpath)
+	if err != nil {
+		return fmt.Errorf("failed to open file: %w", err)
+	}
+	defer fp.Close()
+
+	if _, err := io.Copy(target, fp); err != nil {
+		return fmt.Errorf("failed to copy file: %w", err)
+	}
+
+	return nil
+}
+
+// downloadOptions holds the options used when downloading OCI artifacts.
+type downloadOptions struct {
+	insecureSkipTLSVerify bool
+	auth                  DockerConfig
+}
+
+// DownloadOption is a function that sets an option for the OCI download.
+type DownloadOption func(*downloadOptions)
+
+// WithInsecureSkipTLSVerify sets the insecureSkipTLSVerify option to true.
+func WithInsecureSkipTLSVerify() DownloadOption {
+	return func(opts *downloadOptions) {
+		opts.insecureSkipTLSVerify = true
+	}
+}
+
+// WithDockerAuth sets the Docker config to be used when authenticating to
+// the registry.
+func WithDockerAuth(auth DockerConfig) DownloadOption {
+	return func(opts *downloadOptions) {
+		opts.auth = auth
+	}
+}
+
+// DockerConfigEntry holds an entry in the '.dockerconfigjson' file.
+type DockerConfigEntry struct {
+	Username string `json:"username"`
+	Password string `json:"password"`
+}
+
+// DockerConfig represents the content of the '.dockerconfigjson' file.
+type DockerConfig struct {
+	Auths map[string]DockerConfigEntry `json:"auths"`
+}
+
+// CredentialStore turns the Docker configuration into a credential store and
+// returns it.
+func (d DockerConfig) CredentialStore(ctx context.Context) (credentials.Store, error) {
+	creds := credentials.NewMemoryStore()
+	for addr, entry := range d.Auths {
+		if err := creds.Put(ctx, addr, auth.Credential{
+			Username: entry.Username,
+			Password: entry.Password,
+		}); err != nil {
+			return nil, fmt.Errorf("failed to add credential: %w", err)
+		}
+	}
+	return creds, nil
+}

internal/oci/oci_test.go

@@ -0,0 +1,173 @@
+/*
+Copyright 2024 k0s authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci_test
+
+import (
+	"bytes"
+	"context"
+	"embed"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"path"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/k0sproject/k0s/internal/oci"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sigs.k8s.io/yaml"
+)
+
+//go:embed testdata/*
+var testData embed.FS
+
+// we define our tests as yaml files inside the testdata directory. this
+// function parses them and returns a map of the tests.
+func parseTestsYAML[T any](t *testing.T) map[string]T {
+	entries, err := testData.ReadDir("testdata")
+	require.NoError(t, err)
+	tests := make(map[string]T, 0)
+	for _, entry := range entries {
+		fpath := path.Join("testdata", entry.Name())
+		data, err := testData.ReadFile(fpath)
+		require.NoError(t, err)
+
+		var onetest T
+		err = yaml.Unmarshal(data, &onetest)
+		require.NoError(t, err)
+
+		tests[fpath] = onetest
+	}
+	return tests
+}
+
+// testFile represents a single test file inside the testdata directory.
+type testFile struct {
+	Manifest      string            `json:"manifest"`
+	Expected      string            `json:"expected"`
+	Error         string            `json:"error"`
+	Authenticated bool              `json:"authenticated"`
+	AuthUser      string            `json:"authUser"`
+	AuthPass      string            `json:"authPass"`
+	Artifacts     map[string]string `json:"artifacts"`
+}
+
+func TestDownload(t *testing.T) {
+	for tname, tt := range parseTestsYAML[testFile](t) {
+		t.Run(tname, func(t *testing.T) {
+			addr := startOCIMockServer(t, tname, tt)
+
+			opts := []oci.DownloadOption{oci.WithInsecureSkipTLSVerify()}
+			if tt.Authenticated {
+				entry := oci.DockerConfigEntry{tt.AuthUser, tt.AuthPass}
+				opts = append(opts, oci.WithDockerAuth(
+					oci.DockerConfig{
+						Auths: map[string]oci.DockerConfigEntry{
+							addr: entry,
+						},
+					},
+				))
+			}
+
+			buf := bytes.NewBuffer(nil)
+			url := fmt.Sprintf("%s/repository/artifact:latest", addr)
+			err := oci.Download(context.TODO(), url, buf, opts...)
+			if tt.Expected != "" {
+				require.NoError(t, err)
+				require.Empty(t, tt.Error)
+				require.Equal(t, tt.Expected, buf.String())
+				return
+			}
+			require.NotEmpty(t, tt.Error)
+			require.ErrorContains(t, err, tt.Error)
+		})
+	}
+}
+
+// startOCIMockServer starts a mock server that will respond to the given test.
+// this mimics the behavior of the real OCI registry. This function returns the
+// address of the server.
+func startOCIMockServer(t *testing.T, tname string, test testFile) string {
+	var serverAddr string
+	server := httptest.NewTLSServer(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+			// this is a request to authenticate.
+			if strings.Contains(r.URL.Path, "/token") {
+				user, pass, _ := r.BasicAuth()
+				if user != "user" || pass != "pass" {
+					w.WriteHeader(http.StatusUnauthorized)
+					return
+				}
+				res := map[string]string{"token": tname}
+				marshalled, err := json.Marshal(res)
+				require.NoError(t, err)
+				_, _ = w.Write(marshalled)
+				return
+			}
+
+			// verify if the request should be authenticated or
+			// not. if it has already been authenticated then just
+			// moves on. the token returned is the test name.
+			tokenhdr, authenticated := r.Header["Authorization"]
+			if !authenticated && test.Authenticated {
+				header := fmt.Sprintf(`Bearer realm="https://%s/token"`, serverAddr)
+				w.Header().Add("WWW-Authenticate", header)
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+
+			// verify if the token provided by the client matches
+			// the expected token.
+			if test.Authenticated {
+				require.Len(t, tokenhdr, 1)
+				require.Contains(t, tokenhdr[0], tname)
+			}
+
+			// serve the manifest.
+			if strings.Contains(r.URL.Path, "/manifests/") {
+				w.Header().Add("Content-Type", "application/vnd.oci.image.manifest.v1+json")
+				_, _ = w.Write([]byte(test.Manifest))
+				return
+			}
+
+			// serve a layer or the config blob.
+			if strings.Contains(r.URL.Path, "/blobs/") {
+				for sha, content := range test.Artifacts {
+					if !strings.Contains(r.URL.Path, sha) {
+						continue
+					}
+					w.Header().Add("Content-Length", strconv.Itoa(len(content)))
+					_, _ = w.Write([]byte(content))
+					return
+				}
+			}
+
+			assert.Failf(t, "unexpected request", "%s", r.URL.Path)
+			w.WriteHeader(http.StatusNotFound)
+		}),
+	)
+
+	u, err := url.Parse(server.URL)
+	require.NoError(t, err)
+	serverAddr = u.Host
+	return serverAddr
+}

internal/oci/testdata/authenticated.yaml

@@ -0,0 +1,31 @@
+manifest: |
+  {
+    "schemaVersion": 2,
+    "mediaType": "application/vnd.oci.image.manifest.v1+json",
+    "artifactType": "application/vnd.unknown.artifact.v1",
+    "config": {
+      "mediaType": "application/vnd.oci.empty.v1+json",
+      "digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
+      "size": 2
+    },
+    "layers": [
+      {
+        "mediaType": "application/vnd.oci.image.layer.v1.tar",
+        "digest": "sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
+        "size": 4,
+        "annotations": {
+          "org.opencontainers.image.title": "file1"
+        }
+      }
+    ],
+    "annotations": {
+      "org.opencontainers.image.created": "2024-10-03T14:32:57Z"
+    }
+  }
+artifacts:
+  44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a: "{}"
+  9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08: "test"
+expected: test
+authenticated: true
+authUser: user
+authPass: pass