Skip to content

Commit

Permalink
pulp: implement artifact pulling
Browse files Browse the repository at this point in the history
  • Loading branch information
lzap authored and loadtheaccumulator committed Aug 1, 2024
1 parent 5418c86 commit 4761228
Show file tree
Hide file tree
Showing 10 changed files with 11,328 additions and 5,866 deletions.
23 changes: 19 additions & 4 deletions cmd/pulpcli/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,22 @@ func domainList(ctx context.Context, c *pulp.PulpService) {
// nolint: gosec
var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))

func fixtureCreate(ctx context.Context, c *pulp.PulpService, orgID, tarFilename string) {
func fixtureCreate(ctx context.Context, c *pulp.PulpService, orgID, _ string) {
resourceName := fmt.Sprintf("test-%d", rnd.Int())

artifact, err := c.ArtifactsCreatePipe(ctx, tarFilename)
frepo, err := c.FileRepositoriesEnsure(ctx)
if err != nil {
panic(err)
}
fmt.Println("Artifact uploaded", *artifact.PulpHref)
fmt.Println("File repo found or created", frepo)
fmt.Println("--------------------------------")

artifact, version, err := c.FileRepositoriesImport(ctx, frepo, "https://home.zapletalovi.com/small_commit1.tar")
if err != nil {
panic(err)
}

fmt.Println("Artifact uploaded", artifact)
fmt.Println("--------------------------------")

repo, err := c.RepositoriesCreate(ctx, resourceName)
Expand All @@ -59,13 +67,20 @@ func fixtureCreate(ctx context.Context, c *pulp.PulpService, orgID, tarFilename
fmt.Println("Distribution created", *dist.PulpHref)
fmt.Println("--------------------------------")

repoImported, err := c.RepositoriesImport(ctx, pulp.ScanUUID(repo.PulpHref), "repo", *artifact.PulpHref)
repoImported, err := c.RepositoriesImport(ctx, pulp.ScanUUID(repo.PulpHref), "repo", artifact)
if err != nil {
panic(err)
}
fmt.Println("Repository imported", *repoImported.PulpHref)
fmt.Println("--------------------------------")

err = c.FileRepositoriesVersionDelete(ctx, pulp.ScanUUID(&version), pulp.ScanRepoFileVersion(&version))
if err != nil {
panic(err)
}
fmt.Println("Artifact version deleted", version)
fmt.Println("--------------------------------")

fmt.Printf("curl -L --proxy http://squid.xxxx.redhat.com:3128 --cert /etc/pki/consumer/cert.pem --key /etc/pki/consumer/key.pem https://cert.console.stage.redhat.com/api/pulp-content/%s/%s/\n", c.Domain(), resourceName)
fmt.Printf("curl -L --proxy http://squid.xxxx.redhat.com:3128 -u edge-content-dev:XXXX https://pulp.stage.xxxx.net/api/pulp-content/%s/%s/\n", c.Domain(), resourceName)
fmt.Println("--------------------------------")
Expand Down
6 changes: 5 additions & 1 deletion pkg/clients/pulp/distributions.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,14 @@ func (ps *PulpService) DistributionsCreate(ctx context.Context, name, basePath,
return nil, fmt.Errorf("unexpected response: %d, body: %s", resp.StatusCode(), string(resp.Body))
}

href, err := ps.WaitForTask(ctx, resp.JSON202.Task)
hrefs, err := ps.WaitForTask(ctx, resp.JSON202.Task)
if err != nil {
return nil, err
}
if len(hrefs) != 1 {
return nil, fmt.Errorf("unexpected number of created resources: %d", len(hrefs))
}
href := hrefs[0]

result, err := ps.DistributionsRead(ctx, ScanUUID(&href))
if err != nil {
Expand Down
192 changes: 192 additions & 0 deletions pkg/clients/pulp/file_repositories.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,192 @@
package pulp

import (
"context"
"fmt"
"io"
"strings"

"github.com/google/uuid"
"github.com/redhatinsights/edge-api/pkg/ptr"
)

const FileRepositoryName = "edge-tar-uploads"

// FileRepositoriesCreate creates a new file repository.
func (ps *PulpService) FileRepositoriesCreate(ctx context.Context, name string) (*FileFileRepositoryResponse, error) {
body := FileFileRepository{
Name: name,
Description: ptr.To("Temporary repository for file uploads"),
RetainRepoVersions: ptr.To(int64(1)),
}
resp, err := ps.cwr.RepositoriesFileFileCreateWithResponse(ctx, ps.dom, body, addAuthenticationHeader)

if err != nil {
return nil, err
}

if resp.JSON201 == nil {
return nil, fmt.Errorf("unexpected response: %d, body: %s", resp.StatusCode(), string(resp.Body))
}

return resp.JSON201, nil
}

// FileRepositoriesImport imports a file into a file repository. Returns the artifact href and a version
// artifact href which can be used to delete the artifact.
func (ps *PulpService) FileRepositoriesImport(ctx context.Context, repoHref, url string) (string, string, error) {
body := ContentFileFilesCreateFormdataRequestBody{
Repository: &repoHref,
FileUrl: &url, // Pulp 3.56.1 or higher needed for this flag
RelativePath: uuid.NewString(),
}
resp, err := ps.cwr.ContentFileFilesCreateWithFormdataBody(ctx, ps.dom, body, addAuthenticationHeader)
defer func() { _ = resp.Body.Close() }()

if err != nil {
return "", "", err
}

if resp.StatusCode != 202 {
body, _ := io.ReadAll(resp.Body)
return "", "", fmt.Errorf("unexpected response: %d, body: %s", resp.StatusCode, body)
}

cfcr, err := ParseContentFileFilesCreateResponse(resp)
if err != nil {
return "", "", err
}

hrefs, err := ps.WaitForTask(ctx, cfcr.JSON202.Task)
if err != nil {
return "", "", err
}

artifactHref := ""
versionHref := ""
for _, h := range hrefs {
// pulp returns several resources, we are interested in the file resource
if strings.Contains(h, "content/file/files") {
file, err := ps.FileRepositoriesArtifactRead(ctx, ScanUUID(ptr.To(h)))
if err != nil {
return "", "", err
}
artifactHref = *file.Artifact
}
if strings.Contains(h, "repositories/file/file") {
versionHref = h
}
}

return artifactHref, versionHref, nil
}

// FileRepositoriesArtifactRead reads a file artifact from a file repository.
func (ps *PulpService) FileRepositoriesArtifactRead(ctx context.Context, id uuid.UUID) (*FileFileContentResponse, error) {
req := ContentFileFilesReadParams{}
resp, err := ps.cwr.ContentFileFilesReadWithResponse(ctx, ps.dom, id, &req, addAuthenticationHeader)

if err != nil {
return nil, err
}

if resp.JSON200 == nil {
return nil, fmt.Errorf("unexpected response: %d, body: %s", resp.StatusCode(), string(resp.Body))
}

return resp.JSON200, nil
}

// FileRepositoriesVersionDelete deletes a file version from a file repository.
func (ps *PulpService) FileRepositoriesVersionDelete(ctx context.Context, id uuid.UUID, version int64) error {
resp, err := ps.cwr.RepositoriesFileFileVersionsDelete(ctx, ps.dom, id, version, addAuthenticationHeader)

if err != nil {
return err
}
resp.Body.Close()

if resp.StatusCode != 202 {
return fmt.Errorf("unexpected response: %d", resp.StatusCode)
}

return nil
}

// FileRepositoriesRead reads a file repository.
func (ps *PulpService) FileRepositoriesRead(ctx context.Context, id uuid.UUID) (*FileFileRepositoryResponse, error) {
req := RepositoriesFileFileReadParams{}
resp, err := ps.cwr.RepositoriesFileFileReadWithResponse(ctx, ps.dom, id, &req, addAuthenticationHeader)

if err != nil {
return nil, err
}

if resp.JSON200 == nil {
return nil, fmt.Errorf("unexpected response: %d, body: %s", resp.StatusCode(), string(resp.Body))
}

return resp.JSON200, nil
}

// FileRepositoriesDelete deletes a file repository.
func (ps *PulpService) FileRepositoriesDelete(ctx context.Context, id uuid.UUID) error {
resp, err := ps.cwr.RepositoriesFileFileDelete(ctx, ps.dom, id, addAuthenticationHeader)

if err != nil {
return err
}
resp.Body.Close()

if resp.StatusCode != 204 {
return fmt.Errorf("unexpected response: %d", resp.StatusCode)
}

return nil
}

// FileRepositoriesList lists file repositories.
func (ps *PulpService) FileRepositoriesList(ctx context.Context, nameFilter string) ([]FileFileRepositoryResponse, error) {
req := RepositoriesFileFileListParams{
Limit: &DefaultPageSize,
}
if nameFilter != "" {
req.Name = &nameFilter
}

resp, err := ps.cwr.RepositoriesFileFileListWithResponse(ctx, ps.dom, &req, addAuthenticationHeader)

if err != nil {
return nil, err
}

if resp.JSON200 == nil {
return nil, fmt.Errorf("unexpected response: %d, body: %s", resp.StatusCode(), string(resp.Body))
}

if resp.JSON200.Count > DefaultPageSize {
return nil, fmt.Errorf("default page size too small: %d", resp.JSON200.Count)
}

return resp.JSON200.Results, nil
}

// FileRepositoriesEnsure ensures a file repository exists.
func (ps *PulpService) FileRepositoriesEnsure(ctx context.Context) (string, error) {
var repo *FileFileRepositoryResponse
repos, err := ps.FileRepositoriesList(ctx, FileRepositoryName)
if err != nil {
return "", err
}

if len(repos) != 1 {
repo, err = ps.FileRepositoriesCreate(ctx, FileRepositoryName)
if err != nil {
return "", err
}
} else {
repo = &repos[0]
}

return *repo.PulpHref, nil
}
23 changes: 23 additions & 0 deletions pkg/clients/pulp/href.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package pulp

import (
"regexp"
"strconv"

"github.com/google/uuid"
)
Expand All @@ -26,3 +27,25 @@ func ScanUUID(href *string) uuid.UUID {
}
return u
}

// /api/pulp/edge-integration-test-2/api/v3/repositories/file/file/01910e45-ceb3-7213-bed8-0727e76d0d12/versions/1/
var repoVerRegexp = regexp.MustCompile("versions/([0-9]+)")

func ScanRepoFileVersion(href *string) int64 {
if href == nil {
return 0
}

str := repoVerRegexp.FindStringSubmatch(*href)

if len(str) != 2 {
return 0
}

result, err := strconv.Atoi(str[1])
if err != nil {
return 0
}

return int64(result)
}
42 changes: 42 additions & 0 deletions pkg/clients/pulp/href_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,45 @@ func TestScanUUID(t *testing.T) {
}
})
}

func TestScanRepoFileVersion(t *testing.T) {

t.Run("scanRepoFileVersion", func(t *testing.T) {
var tests = []struct {
name string
href string
expected int64
}{
{
name: "valid version",
href: "/api/pulp/edge-integration-test-2/api/v3/repositories/file/file/01910e45-ceb3-7213-bed8-0727e76d0d12/versions/1/",
expected: 1,
},
{
name: "valid version",
href: "/api/pulp/edge-integration-test-2/api/v3/repositories/file/file/01910e45-ceb3-7213-bed8-0727e76d0d12/versions/2/",
expected: 2,
},
{
name: "empty string",
href: "",
expected: 0,
},
{
name: "invalid version",
href: "this is not a version",
expected: 0,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// linter G601 workaround
actual := ScanRepoFileVersion(ptr.To(tt.href))
if actual != tt.expected {
t.Errorf("scanRepoFileVersion(%s): expected %v, actual %v", tt.href, tt.expected, actual)
}
})
}
})
}
Loading

0 comments on commit 4761228

Please sign in to comment.