Replay pipeline using cli exec by downloading metadata

cli/exec/exec.go

@@ -129,7 +129,13 @@ func runExec(ctx context.Context, c *cli.Command, file, repoPath string) error {
 }
 
 func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, axis matrix.Axis) error {
-	metadata := metadataFromContext(ctx, c, axis)
+	metadata, err := metadataFromContext(ctx, c, axis)
+	if err != nil {
+		return err
+	} else if metadata == nil {
+		return fmt.Errorf("metadata is nil")
+	}
+
 	environ := metadata.Environ()
 	var secrets []compiler.Secret
 	for key, val := range metadata.Workflow.Matrix {
@@ -234,7 +240,7 @@ func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, ax
 			c.String("netrc-password"),
 			c.String("netrc-machine"),
 		),
-		compiler.WithMetadata(metadata),
+		compiler.WithMetadata(*metadata),
 		compiler.WithSecret(secrets...),
 		compiler.WithEnviron(pipelineEnv),
 	).Compile(conf)

cli/exec/flags.go

@@ -32,6 +32,11 @@ var flags = []cli.Flag{
 		Name:    "repo-path",
 		Usage:   "path to local repository",
 	},
+	&cli.StringFlag{
+		Sources: cli.EnvVars("WOODPECKER_METADATA_FILE"),
+		Name:    "metadata-file",
+		Usage:   "path to metadata file who has stored the pipeline environment to emulate (can be downloaded from an existing pipeline), infos can be overwritten.",
+	},
 	&cli.DurationFlag{
 		Sources: cli.EnvVars("WOODPECKER_TIMEOUT"),
 		Name:    "timeout",

cli/exec/metadata.go

@@ -16,6 +16,8 @@ package exec
 
 import (
 	"context"
+	"encoding/json"
+	"os"
 	"runtime"
 	"strings"
 
@@ -27,94 +29,107 @@ import (
 )
 
 // return the metadata from the cli context.
-func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) metadata.Metadata {
+func metadataFromContext(_ context.Context, c *cli.Command, axis matrix.Axis) (*metadata.Metadata, error) {
+	m := &metadata.Metadata{}
+
+	if c.IsSet("metadata-file") {
+		metadataFile, err := os.Open(c.String("metadata-file"))
+		if err != nil {
+			return nil, err
+		}
+		defer metadataFile.Close()
+
+		if err := json.NewDecoder(metadataFile).Decode(m); err != nil {
+			return nil, err
+		}
+	}
+
 	platform := c.String("system-platform")
 	if platform == "" {
 		platform = runtime.GOOS + "/" + runtime.GOARCH
 	}
 
-	fullRepoName := c.String("repo-name")
-	repoOwner := ""
-	repoName := ""
-	if idx := strings.LastIndex(fullRepoName, "/"); idx != -1 {
-		repoOwner = fullRepoName[:idx]
-		repoName = fullRepoName[idx+1:]
-	}
+	metadataFileAndOverrideOrDefault(c, "repo-name", func(fullRepoName string) {
+		if idx := strings.LastIndex(fullRepoName, "/"); idx != -1 {
+			m.Repo.Owner = fullRepoName[:idx]
+			m.Repo.Name = fullRepoName[idx+1:]
+		}
+	}, c.String)
+
+	// Repo
+	metadataFileAndOverrideOrDefault(c, "repo-remote-id", func(s string) { m.Repo.RemoteID = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "repo-url", func(s string) { m.Repo.ForgeURL = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "repo-clone-url", func(s string) { m.Repo.CloneURL = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "repo-clone-ssh-url", func(s string) { m.Repo.CloneSSHURL = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "repo-private", func(b bool) { m.Repo.Private = b }, c.Bool)
+	metadataFileAndOverrideOrDefault(c, "repo-trusted", func(b bool) { m.Repo.Trusted = b }, c.Bool)
+
+	// Current Pipeline
+	metadataFileAndOverrideOrDefault(c, "pipeline-number", func(i int64) { m.Curr.Number = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "pipeline-parent", func(i int64) { m.Curr.Parent = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "pipeline-created", func(i int64) { m.Curr.Created = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "pipeline-started", func(i int64) { m.Curr.Started = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "pipeline-finished", func(i int64) { m.Curr.Finished = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "pipeline-status", func(s string) { m.Curr.Status = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "pipeline-event", func(s string) { m.Curr.Event = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "pipeline-url", func(s string) { m.Curr.ForgeURL = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "pipeline-deploy-to", func(s string) { m.Curr.DeployTo = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "pipeline-deploy-task", func(s string) { m.Curr.DeployTask = s }, c.String)
+
+	// Current Pipeline Commit
+	metadataFileAndOverrideOrDefault(c, "commit-sha", func(s string) { m.Curr.Commit.Sha = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "commit-ref", func(s string) { m.Curr.Commit.Ref = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "commit-refspec", func(s string) { m.Curr.Commit.Refspec = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "commit-branch", func(s string) { m.Curr.Commit.Branch = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "commit-message", func(s string) { m.Curr.Commit.Message = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "commit-author-name", func(s string) { m.Curr.Commit.Author.Name = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "commit-author-email", func(s string) { m.Curr.Commit.Author.Email = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "commit-author-avatar", func(s string) { m.Curr.Commit.Author.Avatar = s }, c.String)
+
+	// Previous Pipeline
+	metadataFileAndOverrideOrDefault(c, "prev-pipeline-number", func(i int64) { m.Prev.Number = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "prev-pipeline-created", func(i int64) { m.Prev.Created = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "prev-pipeline-started", func(i int64) { m.Prev.Started = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "prev-pipeline-finished", func(i int64) { m.Prev.Finished = i }, c.Int)
+	metadataFileAndOverrideOrDefault(c, "prev-pipeline-status", func(s string) { m.Prev.Status = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-pipeline-event", func(s string) { m.Prev.Event = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-pipeline-url", func(s string) { m.Prev.ForgeURL = s }, c.String)
+
+	// Previous Pipeline Commit
+	metadataFileAndOverrideOrDefault(c, "prev-commit-sha", func(s string) { m.Prev.Commit.Sha = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-commit-ref", func(s string) { m.Prev.Commit.Ref = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-commit-refspec", func(s string) { m.Prev.Commit.Refspec = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-commit-branch", func(s string) { m.Prev.Commit.Branch = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-commit-message", func(s string) { m.Prev.Commit.Message = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-commit-author-name", func(s string) { m.Prev.Commit.Author.Name = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-commit-author-email", func(s string) { m.Prev.Commit.Author.Email = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "prev-commit-author-avatar", func(s string) { m.Prev.Commit.Author.Avatar = s }, c.String)
+
+	// Workflow
+	metadataFileAndOverrideOrDefault(c, "workflow-name", func(s string) { m.Workflow.Name = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "workflow-number", func(i int64) { m.Workflow.Number = int(i) }, c.Int)
+	m.Workflow.Matrix = axis
+
+	// Step
+	metadataFileAndOverrideOrDefault(c, "step-name", func(s string) { m.Step.Name = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "step-number", func(i int64) { m.Step.Number = int(i) }, c.Int)
+
+	// System
+	metadataFileAndOverrideOrDefault(c, "system-name", func(s string) { m.Sys.Name = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "system-url", func(s string) { m.Sys.URL = s }, c.String)
+	m.Sys.Platform = platform
+	m.Sys.Version = version.Version
+
+	// Forge
+	metadataFileAndOverrideOrDefault(c, "forge-type", func(s string) { m.Forge.Type = s }, c.String)
+	metadataFileAndOverrideOrDefault(c, "forge-url", func(s string) { m.Forge.URL = s }, c.String)
+
+	return m, nil
+}
 
-	return metadata.Metadata{
-		Repo: metadata.Repo{
-			Name:        repoName,
-			Owner:       repoOwner,
-			RemoteID:    c.String("repo-remote-id"),
-			ForgeURL:    c.String("repo-url"),
-			CloneURL:    c.String("repo-clone-url"),
-			CloneSSHURL: c.String("repo-clone-ssh-url"),
-			Private:     c.Bool("repo-private"),
-			Trusted:     c.Bool("repo-trusted"),
-		},
-		Curr: metadata.Pipeline{
-			Number:     c.Int("pipeline-number"),
-			Parent:     c.Int("pipeline-parent"),
-			Created:    c.Int("pipeline-created"),
-			Started:    c.Int("pipeline-started"),
-			Finished:   c.Int("pipeline-finished"),
-			Status:     c.String("pipeline-status"),
-			Event:      c.String("pipeline-event"),
-			ForgeURL:   c.String("pipeline-url"),
-			DeployTo:   c.String("pipeline-deploy-to"),
-			DeployTask: c.String("pipeline-deploy-task"),
-			Commit: metadata.Commit{
-				Sha:     c.String("commit-sha"),
-				Ref:     c.String("commit-ref"),
-				Refspec: c.String("commit-refspec"),
-				Branch:  c.String("commit-branch"),
-				Message: c.String("commit-message"),
-				Author: metadata.Author{
-					Name:   c.String("commit-author-name"),
-					Email:  c.String("commit-author-email"),
-					Avatar: c.String("commit-author-avatar"),
-				},
-			},
-		},
-		Prev: metadata.Pipeline{
-			Number:   c.Int("prev-pipeline-number"),
-			Created:  c.Int("prev-pipeline-created"),
-			Started:  c.Int("prev-pipeline-started"),
-			Finished: c.Int("prev-pipeline-finished"),
-			Status:   c.String("prev-pipeline-status"),
-			Event:    c.String("prev-pipeline-event"),
-			ForgeURL: c.String("prev-pipeline-url"),
-			Commit: metadata.Commit{
-				Sha:     c.String("prev-commit-sha"),
-				Ref:     c.String("prev-commit-ref"),
-				Refspec: c.String("prev-commit-refspec"),
-				Branch:  c.String("prev-commit-branch"),
-				Message: c.String("prev-commit-message"),
-				Author: metadata.Author{
-					Name:   c.String("prev-commit-author-name"),
-					Email:  c.String("prev-commit-author-email"),
-					Avatar: c.String("prev-commit-author-avatar"),
-				},
-			},
-		},
-		Workflow: metadata.Workflow{
-			Name:   c.String("workflow-name"),
-			Number: int(c.Int("workflow-number")),
-			Matrix: axis,
-		},
-		Step: metadata.Step{
-			Name:   c.String("step-name"),
-			Number: int(c.Int("step-number")),
-		},
-		Sys: metadata.System{
-			Name:     c.String("system-name"),
-			URL:      c.String("system-url"),
-			Platform: platform,
-			Version:  version.Version,
-		},
-		Forge: metadata.Forge{
-			Type: c.String("forge-type"),
-			URL:  c.String("forge-url"),
-		},
+// metadataFileAndOverrideOrDefault will either use the flag default or if metadata file is set only overload if explicit set.
+func metadataFileAndOverrideOrDefault[T any](c *cli.Command, flag string, setter func(T), getter func(string) T) {
+	if !c.IsSet("metadata-file") || c.IsSet(flag) {
+		setter(getter(flag))
 	}
 }

cli/exec/metadata_test.go

@@ -0,0 +1,140 @@
+// Copyright 2024 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exec
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v3"
+
+	"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
+	"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/matrix"
+)
+
+func TestMetadataFromContext(t *testing.T) {
+	sampleMetadata := &metadata.Metadata{
+		Repo: metadata.Repo{Owner: "test-user", Name: "test-repo"},
+		Curr: metadata.Pipeline{Number: 5},
+	}
+
+	runCommand := func(flags []cli.Flag, fn func(c *cli.Command)) {
+		c := &cli.Command{
+			Flags: flags,
+			Action: func(_ context.Context, c *cli.Command) error {
+				fn(c)
+				return nil
+			},
+		}
+		assert.NoError(t, c.Run(context.Background(), []string{"woodpecker-cli"}))
+	}
+
+	t.Run("LoadFromFile", func(t *testing.T) {
+		tempFileName := createTempFile(t, sampleMetadata)
+
+		flags := []cli.Flag{
+			&cli.StringFlag{Name: "metadata-file"},
+		}
+
+		runCommand(flags, func(c *cli.Command) {
+			_ = c.Set("metadata-file", tempFileName)
+
+			m, err := metadataFromContext(context.Background(), c, nil)
+			require.NoError(t, err)
+			assert.Equal(t, "test-repo", m.Repo.Name)
+			assert.Equal(t, int64(5), m.Curr.Number)
+		})
+	})
+
+	t.Run("OverrideFromFlags", func(t *testing.T) {
+		tempFileName := createTempFile(t, sampleMetadata)
+
+		flags := []cli.Flag{
+			&cli.StringFlag{Name: "metadata-file"},
+			&cli.StringFlag{Name: "repo-name"},
+			&cli.IntFlag{Name: "pipeline-number"},
+		}
+
+		runCommand(flags, func(c *cli.Command) {
+			_ = c.Set("metadata-file", tempFileName)
+			_ = c.Set("repo-name", "aUser/override-repo")
+			_ = c.Set("pipeline-number", "10")
+
+			m, err := metadataFromContext(context.Background(), c, nil)
+			require.NoError(t, err)
+			assert.Equal(t, "override-repo", m.Repo.Name)
+			assert.Equal(t, int64(10), m.Curr.Number)
+		})
+	})
+
+	t.Run("InvalidFile", func(t *testing.T) {
+		tempFile, err := os.CreateTemp("", "invalid.json")
+		require.NoError(t, err)
+		t.Cleanup(func() { os.Remove(tempFile.Name()) })
+
+		_, err = tempFile.Write([]byte("invalid json"))
+		require.NoError(t, err)
+
+		flags := []cli.Flag{
+			&cli.StringFlag{Name: "metadata-file"},
+		}
+
+		runCommand(flags, func(c *cli.Command) {
+			_ = c.Set("metadata-file", tempFile.Name())
+
+			_, err = metadataFromContext(context.Background(), c, nil)
+			assert.Error(t, err)
+		})
+	})
+
+	t.Run("DefaultValues", func(t *testing.T) {
+		flags := []cli.Flag{
+			&cli.StringFlag{Name: "repo-name", Value: "test/default-repo"},
+			&cli.IntFlag{Name: "pipeline-number", Value: 1},
+		}
+
+		runCommand(flags, func(c *cli.Command) {
+			m, err := metadataFromContext(context.Background(), c, nil)
+			require.NoError(t, err)
+			assert.Equal(t, "test", m.Repo.Owner)
+			assert.Equal(t, "default-repo", m.Repo.Name)
+			assert.Equal(t, int64(1), m.Curr.Number)
+		})
+	})
+
+	t.Run("MatrixAxis", func(t *testing.T) {
+		runCommand([]cli.Flag{}, func(c *cli.Command) {
+			axis := matrix.Axis{"go": "1.16", "os": "linux"}
+			m, err := metadataFromContext(context.Background(), c, axis)
+			require.NoError(t, err)
+			assert.EqualValues(t, map[string]string{"go": "1.16", "os": "linux"}, m.Workflow.Matrix)
+		})
+	})
+}
+
+func createTempFile(t *testing.T, content interface{}) string {
+	t.Helper()
+	tempFile, err := os.CreateTemp("", "metadata.json")
+	require.NoError(t, err)
+	t.Cleanup(func() { os.Remove(tempFile.Name()) })
+
+	err = json.NewEncoder(tempFile).Encode(content)
+	require.NoError(t, err)
+	return tempFile.Name()
+}