Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Integration test - Fetcher must collect SHAs #3073

Merged
merged 3 commits into from
Jul 17, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions pkg/testing/fetcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
"github.com/hashicorp/go-multierror"
)

const hashExt = ".sha512"

var (
// ErrUnsupportedPlatform returned when the operating system and architecture combination is not supported.
ErrUnsupportedPlatform = errors.New("platform is not supported")
Expand Down Expand Up @@ -131,7 +133,7 @@

fi := f.FileInfo()
mode := fi.Mode()
abs := filepath.Join(extractDir, f.Name)

Check failure on line 136 in pkg/testing/fetcher.go

View workflow job for this annotation

GitHub Actions / lint (ubuntu-latest)

G305: File traversal when extracting zip/tar archive (gosec)
switch {
case mode.IsRegular():
// just to be sure, it should already be created by Dir type
Expand All @@ -144,7 +146,7 @@
return fmt.Errorf("failed creating file %s: %w", abs, err)
}

_, err = io.Copy(wf, tr)

Check failure on line 149 in pkg/testing/fetcher.go

View workflow job for this annotation

GitHub Actions / lint (ubuntu-latest)

G110: Potential DoS vulnerability via decompression bomb (gosec)
if closeErr := wf.Close(); closeErr != nil && err == nil {
err = closeErr
}
Expand Down Expand Up @@ -191,7 +193,7 @@

fi := f.FileInfo()
mode := fi.Mode()
abs := filepath.Join(extractDir, f.Name)

Check failure on line 196 in pkg/testing/fetcher.go

View workflow job for this annotation

GitHub Actions / lint (ubuntu-latest)

G305: File traversal when extracting zip/tar archive (gosec)
switch {
case mode.IsRegular():
// just to be sure, it should already be created by Dir type
Expand All @@ -209,7 +211,7 @@
}
}()

if _, err = io.Copy(f, rc); err != nil {

Check failure on line 214 in pkg/testing/fetcher.go

View workflow job for this annotation

GitHub Actions / lint (ubuntu-latest)

G110: Potential DoS vulnerability via decompression bomb (gosec)
return fmt.Errorf("error writing file %s: %w", abs, err)
}
case mode.IsDir():
Expand Down
7 changes: 7 additions & 0 deletions pkg/testing/fetcher_artifact.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,13 @@ func (r *artifactResult) Fetch(ctx context.Context, l Logger, dir string) error
if err != nil {
return fmt.Errorf("failed to download %s: %w", r.src, err)
}

// fetch package hash
err = DownloadPackage(ctx, l, r.doer, r.src+hashExt, filepath.Join(dir, r.path+hashExt))
if err != nil {
return fmt.Errorf("failed to download %s: %w", r.src, err)
}

return nil
}

Expand Down
18 changes: 17 additions & 1 deletion pkg/testing/fetcher_artifact_test.go

Large diffs are not rendered by default.

29 changes: 22 additions & 7 deletions pkg/testing/fetcher_local.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,27 +85,42 @@ func (r *localFetcherResult) Name() string {
// Fetch performs the actual fetch into the provided directory.
func (r *localFetcherResult) Fetch(_ context.Context, _ Logger, dir string) error {
fullPath := filepath.Join(r.src, r.path)
path := filepath.Join(dir, r.path)

err := copyFile(fullPath, path)
if err != nil {
return fmt.Errorf("error copying file: %w", err)
}

reader, err := os.Open(fullPath)
// fetch artifact hash
err = copyFile(fullPath+hashExt, path+hashExt)
if err != nil {
return fmt.Errorf("failed to open file %s: %w", fullPath, err)
return fmt.Errorf("error copying file: %w", err)
}

return nil
}

func copyFile(src, dst string) error {
reader, err := os.Open(src)
if err != nil {
return fmt.Errorf("failed to open file %s: %w", src, err)
}
defer reader.Close()

path := filepath.Join(dir, r.path)
w, err := os.Create(path)
w, err := os.Create(dst)
if err != nil {
return fmt.Errorf("failed to create file %s: %w", path, err)
return fmt.Errorf("failed to create file %s: %w", dst, err)
}
defer w.Close()

_, err = io.Copy(w, reader)
if err != nil {
return fmt.Errorf("failed to write file %s: %w", path, err)
return fmt.Errorf("failed to write file %s: %w", dst, err)
}
err = w.Sync()
if err != nil {
return fmt.Errorf("failed to sync file %s: %w", path, err)
return fmt.Errorf("failed to sync file %s: %w", dst, err)
}
return nil
}
53 changes: 34 additions & 19 deletions pkg/testing/fetcher_local_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,41 +25,52 @@ func TestLocalFetcher(t *testing.T) {
// t.Skip()
baseVersion := "8.7.0"
snapshotContent := []byte("snapshot contents")
snapshotContentHash := []byte("snapshot contents hash")
noSnapshotContent := []byte("not snapshot contents")
noSnapshotContentHash := []byte("not snapshot contents hash")

testdata := t.TempDir()
suffix, err := GetPackageSuffix(runtime.GOOS, runtime.GOARCH)
require.NoError(t, err)

snapshotPath := fmt.Sprintf("elastic-agent-%s-SNAPSHOT-%s", baseVersion, suffix)
notSnapshotPath := fmt.Sprintf("elastic-agent-%s-%s", baseVersion, suffix)
require.NoError(t, os.WriteFile(filepath.Join(testdata, snapshotPath), snapshotContent, 0644))
snapshotPathHash := fmt.Sprintf("elastic-agent-%s-SNAPSHOT-%s%s", baseVersion, suffix, hashExt)
require.NoError(t, os.WriteFile(filepath.Join(testdata, snapshotPathHash), snapshotContentHash, 0644))
notSnapshotPath := fmt.Sprintf("elastic-agent-%s-%s", baseVersion, suffix)
require.NoError(t, os.WriteFile(filepath.Join(testdata, notSnapshotPath), noSnapshotContent, 0644))
notSnapshotPathHash := fmt.Sprintf("elastic-agent-%s-%s%s", baseVersion, suffix, hashExt)
require.NoError(t, os.WriteFile(filepath.Join(testdata, notSnapshotPathHash), noSnapshotContentHash, 0644))

tcs := []struct {
name string
version string
opts []localFetcherOpt
want []byte
name string
version string
opts []localFetcherOpt
want []byte
wantHash []byte
}{
{
name: "IgnoreSnapshot",
version: baseVersion,
want: noSnapshotContent,
name: "IgnoreSnapshot",
version: baseVersion,
want: noSnapshotContent,
wantHash: noSnapshotContentHash,
}, {
name: "SnapshotOnly",
version: baseVersion,
opts: []localFetcherOpt{WithLocalSnapshotOnly()},
want: snapshotContent,
name: "SnapshotOnly",
version: baseVersion,
opts: []localFetcherOpt{WithLocalSnapshotOnly()},
want: snapshotContent,
wantHash: snapshotContentHash,
}, {
name: "version with snapshot",
version: baseVersion + "-SNAPSHOT",
want: snapshotContent,
name: "version with snapshot",
version: baseVersion + "-SNAPSHOT",
want: snapshotContent,
wantHash: snapshotContentHash,
}, {
name: "version with snapshot and SnapshotOnly",
version: baseVersion + "-SNAPSHOT",
opts: []localFetcherOpt{WithLocalSnapshotOnly()},
want: snapshotContent,
name: "version with snapshot and SnapshotOnly",
version: baseVersion + "-SNAPSHOT",
opts: []localFetcherOpt{WithLocalSnapshotOnly()},
want: snapshotContent,
wantHash: snapshotContentHash,
},
}

Expand All @@ -77,5 +88,9 @@ func TestLocalFetcher(t *testing.T) {
require.NoError(t, err)

assert.Equal(t, string(tc.want), string(content))
contentHash, err := os.ReadFile(filepath.Join(tmp, got.Name()+hashExt))
require.NoError(t, err)

assert.Equal(t, string(tc.wantHash), string(contentHash))
}
}
30 changes: 16 additions & 14 deletions pkg/testing/runner/debian.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
type DebianRunner struct{}

// Prepare the test
func (DebianRunner) Prepare(ctx context.Context, sshClient *ssh.Client, logger Logger, arch string, goVersion string, repoArchive string, buildPath string) error {
func (DebianRunner) Prepare(ctx context.Context, sshClient *ssh.Client, logger Logger, arch string, goVersion string, repoArchive string, buildPaths []string) error {
// prepare build-essential and unzip
//
// apt-get update and install are so terrible that we have to place this in a loop, because in some cases the
Expand Down Expand Up @@ -106,19 +106,21 @@ func (DebianRunner) Prepare(ctx context.Context, sshClient *ssh.Client, logger L
}

// place the build for the agent on the host
logger.Logf("Copying agent build %s", filepath.Base(buildPath))
err = sshSCP(sshClient, buildPath, filepath.Base(buildPath))
if err != nil {
return fmt.Errorf("failed to SCP build %s: %w", filepath.Base(buildPath), err)
}
insideAgentDir := filepath.Join("agent", buildPath)
stdOut, errOut, err = sshRunCommand(ctx, sshClient, "mkdir", []string{"-p", filepath.Dir(insideAgentDir)}, nil)
if err != nil {
return fmt.Errorf("failed to create %s directory: %w (stdout: %s, stderr: %s)", filepath.Dir(insideAgentDir), err, stdOut, errOut)
}
stdOut, errOut, err = sshRunCommand(ctx, sshClient, "mv", []string{filepath.Base(buildPath), insideAgentDir}, nil)
if err != nil {
return fmt.Errorf("failed to move %s to %s: %w (stdout: %s, stderr: %s)", filepath.Base(buildPath), insideAgentDir, err, stdOut, errOut)
for _, buildPath := range buildPaths {
logger.Logf("Copying agent build %s", filepath.Base(buildPath))
err = sshSCP(sshClient, buildPath, filepath.Base(buildPath))
if err != nil {
return fmt.Errorf("failed to SCP build %s: %w", filepath.Base(buildPath), err)
}
insideAgentDir := filepath.Join("agent", buildPath)
stdOut, errOut, err = sshRunCommand(ctx, sshClient, "mkdir", []string{"-p", filepath.Dir(insideAgentDir)}, nil)
if err != nil {
return fmt.Errorf("failed to create %s directory: %w (stdout: %s, stderr: %s)", filepath.Dir(insideAgentDir), err, stdOut, errOut)
}
stdOut, errOut, err = sshRunCommand(ctx, sshClient, "mv", []string{filepath.Base(buildPath), insideAgentDir}, nil)
if err != nil {
return fmt.Errorf("failed to move %s to %s: %w (stdout: %s, stderr: %s)", filepath.Base(buildPath), insideAgentDir, err, stdOut, errOut)
}
}

return nil
Expand Down
17 changes: 11 additions & 6 deletions pkg/testing/runner/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ type OSRunnerResult struct {
// OSRunner provides an interface to run the tests on the OS.
type OSRunner interface {
// Prepare prepares the runner to actual run on the host.
Prepare(ctx context.Context, c *ssh.Client, logger Logger, arch string, goVersion string, repoArchive string, buildPath string) error
Prepare(ctx context.Context, c *ssh.Client, logger Logger, arch string, goVersion string, repoArchive string, buildPaths []string) error
// Run runs the actual tests and provides the result.
Run(ctx context.Context, verbose bool, c *ssh.Client, logger Logger, agentVersion string, prefix string, batch define.Batch, env map[string]string) (OSRunnerResult, error)
}
Expand Down Expand Up @@ -303,9 +303,11 @@ func (r *Runner) validate() error {
var requiredFiles []string
for _, b := range r.batches {
if !b.Skip {
buildPath := r.getBuildPath(b)
if !slices.Contains(requiredFiles, buildPath) {
requiredFiles = append(requiredFiles, buildPath)
buildPaths := r.getBuildPath(b)
for _, buildPath := range buildPaths {
if !slices.Contains(requiredFiles, buildPath) {
requiredFiles = append(requiredFiles, buildPath)
}
}
}
}
Expand All @@ -325,7 +327,7 @@ func (r *Runner) validate() error {
}

// getBuildPath returns the path of the build required for the test.
func (r *Runner) getBuildPath(b LayoutBatch) string {
func (r *Runner) getBuildPath(b LayoutBatch) []string {
arch := b.LayoutOS.OS.Arch
if arch == define.AMD64 {
arch = "x86_64"
Expand All @@ -334,7 +336,10 @@ func (r *Runner) getBuildPath(b LayoutBatch) string {
if b.LayoutOS.OS.Type == define.Windows {
ext = "zip"
}
return filepath.Join(r.cfg.BuildDir, fmt.Sprintf("elastic-agent-%s-%s-%s.%s", r.cfg.AgentVersion, b.LayoutOS.OS.Type, arch, ext))
hashExt := ".sha512"
packageName := filepath.Join(r.cfg.BuildDir, fmt.Sprintf("elastic-agent-%s-%s-%s.%s", r.cfg.AgentVersion, b.LayoutOS.OS.Type, arch, ext))
packageHashName := packageName + hashExt
return []string{packageName, packageHashName}
}

// prepare prepares for the runner to run.
Expand Down
Loading