[8.9](backport #2834 and #3164) Enhance mock fleet-server and add --proxy-url integration test and disable tests

.github/workflows/build.yml

@@ -11,10 +11,12 @@ jobs:
         with:
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
 
+      - name: Extract Go version from .go-version
+        run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version-file: 'go.mod'
+          go-version: "${{ env.GO_VERSION }}"
           cache: true
 
       - name: Go cache

Vagrantfile

@@ -149,7 +149,7 @@ Vagrant.configure("2") do |config|
         vb.gui = false
         vb.customize ["modifyvm", :id, "--vram", "128"]
         # Customize the amount of memory on the VM:
-        vb.memory = "2048"
+        vb.memory = "4096"
       end
 
       nodeconfig.vm.provision "shell", inline: <<-SHELL

docs/test-framework-dev-guide.md

@@ -67,6 +67,20 @@ between, and it can be very specific or not very specific.
 > **_NOTE:_**  This only filters down the tests based on the platform. It will not execute a tests on a platform unless
 > the test defines as supporting it.
 
+### Manually running the tests
+
+If you want to run the tests manually, skipping the test runner, set the
+`TEST_DEFINE_PREFIX` environment variable to any value and run your tests normally
+with `go test`. E.g.:
+
+```shell
+TEST_DEFINE_PREFIX=gambiarra go test -v -tags integration -run TestProxyURL ./testing/integration/
+```
+
+Tests with external dependencies might need more environment variables to be set
+when running them manually, such as `ELASTICSEARCH_HOST`, `ELASTICSEARCH_USERNAME`,
+`ELASTICSEARCH_PASSWORD`, `KIBANA_HOST`, `KIBANA_USERNAME`, and `KIBANA_PASSWORD`.
+
 #### Passing additional go test flags
 
 When running the tests we can pass additional go test flag using the env variable `GOTEST_FLAGS`.

magefile.go

@@ -1290,7 +1290,7 @@ func majorMinor() string {
 	return ""
 }
 
-// cleans up the integration testing leftovers
+// Clean cleans up the integration testing leftovers
 func (Integration) Clean() error {
 	_ = os.RemoveAll(".agent-testing")
 
@@ -1314,13 +1314,13 @@ func (Integration) Clean() error {
 	return nil
 }
 
-// checks that integration tests are using define.Require
+// Check checks that integration tests are using define.Require
 func (Integration) Check() error {
 	fmt.Println(">> check: Checking for define.Require in integration tests") // nolint:forbidigo // it's ok to use fmt.println in mage
 	return define.ValidateDir("testing/integration")
 }
 
-// runs only the integration tests that support local mode
+// Local runs only the integration tests that support local mode
 // it takes as argument the test name to run or all if we want to run them all.
 func (Integration) Local(ctx context.Context, testName string) error {
 	if shouldBuildAgent() {
@@ -1349,7 +1349,7 @@ func (Integration) Local(ctx context.Context, testName string) error {
 	return devtools.GoTest(ctx, params)
 }
 
-// authenticates users who run it to various IaaS CSPs and ESS
+// Auth authenticates users who run it to various IaaS CSPs and ESS
 func (Integration) Auth(ctx context.Context) error {
 	if err := authGCP(ctx); err != nil {
 		return fmt.Errorf("unable to authenticate to GCP: %w", err)
@@ -1368,27 +1368,27 @@ func (Integration) Auth(ctx context.Context) error {
 	return nil
 }
 
-// run integration tests on remote hosts
+// Test runs integration tests on remote hosts
 func (Integration) Test(ctx context.Context) error {
 	return integRunner(ctx, false, "")
 }
 
-// run integration tests on a matrix of all supported remote hosts
+// Matrix runs integration tests on a matrix of all supported remote hosts
 func (Integration) Matrix(ctx context.Context) error {
 	return integRunner(ctx, true, "")
 }
 
-// run single integration test on remote host
+// Single runs single integration test on remote host
 func (Integration) Single(ctx context.Context, testName string) error {
 	return integRunner(ctx, false, testName)
 }
 
-// don't call locally (called on remote host to prepare it for testing)
+// PrepareOnRemote shouldn't be called locally (called on remote host to prepare it for testing)
 func (Integration) PrepareOnRemote() {
 	mg.Deps(mage.InstallGoTestTools)
 }
 
-// don't call locally (called on remote host to perform testing)
+// TestOnRemote shouldn't be called locally (called on remote host to perform testing)
 func (Integration) TestOnRemote(ctx context.Context) error {
 	mg.Deps(Build.TestBinaries)
 	version := os.Getenv("AGENT_VERSION")

pkg/testing/define/define.go

@@ -24,6 +24,7 @@ import (
 
 	atesting "github.com/elastic/elastic-agent/pkg/testing"
 	"github.com/elastic/elastic-agent/pkg/utils"
+	semver "github.com/elastic/elastic-agent/pkg/version"
 	"github.com/elastic/elastic-agent/version"
 )
 
@@ -77,7 +78,18 @@ func NewFixture(t *testing.T, version string, opts ...atesting.FixtureOpt) (*ate
 		buildsDir = filepath.Join(projectDir, "build", "distributions")
 	}
 
-	f := atesting.LocalFetcher(buildsDir)
+	ver, err := semver.ParseVersion(version)
+	if err != nil {
+		return nil, fmt.Errorf("%q is an invalid agent version: %w", version, err)
+	}
+
+	var f atesting.Fetcher
+	if ver.IsSnapshot() {
+		f = atesting.LocalFetcher(buildsDir, atesting.WithLocalSnapshotOnly())
+	} else {
+		f = atesting.LocalFetcher(buildsDir)
+	}
+
 	opts = append(opts, atesting.WithFetcher(f), atesting.WithLogOutput())
 	return atesting.NewFixture(t, version, opts...)
 }

pkg/testing/fixture.go

@@ -6,6 +6,7 @@ package testing
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -18,14 +19,15 @@ import (
 	"testing"
 	"time"
 
-	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
-
+	"github.com/hashicorp/go-multierror"
 	"github.com/otiai10/copy"
 	"gopkg.in/yaml.v2"
 
+	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
 	"github.com/elastic/elastic-agent/pkg/component"
 	"github.com/elastic/elastic-agent/pkg/control"
 	"github.com/elastic/elastic-agent/pkg/control/v2/client"
+	"github.com/elastic/elastic-agent/pkg/control/v2/cproto"
 	"github.com/elastic/elastic-agent/pkg/core/process"
 )
 
@@ -360,6 +362,91 @@ func (f *Fixture) Exec(ctx context.Context, args []string, opts ...process.CmdOp
 	return cmd.CombinedOutput()
 }
 
+type ExecErr struct {
+	err    error
+	Output []byte
+}
+
+func (e *ExecErr) Error() string {
+	return e.err.Error()
+}
+
+func (e *ExecErr) String() string {
+	return fmt.Sprintf("error: %v, output: %s", e.err, e.Output)
+}
+
+func (e *ExecErr) As(target any) bool {
+	switch target.(type) {
+	case *ExecErr:
+		target = e
+		return true
+	case ExecErr:
+		target = *e
+		return true
+	default:
+		return errors.As(e.err, &target)
+	}
+}
+
+func (e *ExecErr) Unwrap() error {
+	return e.err
+}
+
+// ExecStatus executes the status subcommand on the prepared Elastic Agent binary.
+// It returns the parsed output and the error from the execution. Keep in mind
+// the agent exits with status 1 if it's unhealthy, but it still outputs the
+// status successfully. Therefore, a not empty AgentStatusOutput is valid
+// regardless of the error. An empty AgentStatusOutput and non nil error
+// means the output could not be parsed.
+// It should work with any 8.6+ agent
+func (f *Fixture) ExecStatus(ctx context.Context, opts ...process.CmdOption) (AgentStatusOutput, error) {
+	out, err := f.Exec(ctx, []string{"status", "--output", "json"}, opts...)
+	status := AgentStatusOutput{}
+	if uerr := json.Unmarshal(out, &status); uerr != nil {
+		return AgentStatusOutput{}, multierror.Append(
+			fmt.Errorf("could not unmarshal agent status output: %w", uerr),
+			&ExecErr{
+				err:    err,
+				Output: out,
+			})
+	}
+
+	return status, err
+}
+
+// ExecInspect executes to inspect subcommand on the prepared Elastic Agent binary.
+// It returns the parsed output and the error from the execution or an empty
+// AgentInspectOutput and the unmarshalling error if it cannot unmarshal the
+// output.
+// It should work with any 8.6+ agent
+func (f *Fixture) ExecInspect(ctx context.Context, opts ...process.CmdOption) (AgentInspectOutput, error) {
+	out, err := f.Exec(ctx, []string{"inspect"}, opts...)
+	inspect := AgentInspectOutput{}
+	if uerr := yaml.Unmarshal(out, &inspect); uerr != nil {
+		return AgentInspectOutput{},
+			multierror.Append(
+				fmt.Errorf("could not unmarshal agent inspect output: %w", uerr),
+				&ExecErr{
+					err:    err,
+					Output: out,
+				})
+	}
+
+	return inspect, err
+}
+
+// IsHealthy returns if the prepared Elastic Agent reports itself as healthy.
+// It returns false, err if it cannot determine the state of the agent.
+// It should work with any 8.6+ agent
+func (f *Fixture) IsHealthy(ctx context.Context, opts ...process.CmdOption) (bool, error) {
+	status, err := f.ExecStatus(ctx, opts...)
+	if err != nil {
+		return false, fmt.Errorf("agent status returned and error: %w", err)
+	}
+
+	return status.State == int(cproto.State_HEALTHY), nil
+}
+
 func (f *Fixture) ensurePrepared(ctx context.Context) error {
 	if f.workDir == "" {
 		return f.Prepare(ctx)
@@ -685,3 +772,126 @@ func performConfigure(ctx context.Context, c client.Client, cfg string, timeout
 	}
 	return nil
 }
+
+type AgentStatusOutput struct {
+	Info struct {
+		ID        string `json:"id"`
+		Version   string `json:"version"`
+		Commit    string `json:"commit"`
+		BuildTime string `json:"build_time"`
+		Snapshot  bool   `json:"snapshot"`
+	} `json:"info"`
+	State      int    `json:"state"`
+	Message    string `json:"message"`
+	Components []struct {
+		ID      string `json:"id"`
+		Name    string `json:"name"`
+		State   int    `json:"state"`
+		Message string `json:"message"`
+		Units   []struct {
+			UnitID   string `json:"unit_id"`
+			UnitType int    `json:"unit_type"`
+			State    int    `json:"state"`
+			Message  string `json:"message"`
+			Payload  struct {
+				OsqueryVersion string `json:"osquery_version"`
+			} `json:"payload"`
+		} `json:"units"`
+		VersionInfo struct {
+			Name    string `json:"name"`
+			Version string `json:"version"`
+			Meta    struct {
+				BuildTime string `json:"build_time"`
+				Commit    string `json:"commit"`
+			} `json:"meta"`
+		} `json:"version_info,omitempty"`
+	} `json:"components"`
+	FleetState   int    `json:"FleetState"`
+	FleetMessage string `json:"FleetMessage"`
+}
+
+type AgentInspectOutput struct {
+	Agent struct {
+		Download struct {
+			SourceURI string `yaml:"sourceURI"`
+		} `yaml:"download"`
+		Features interface{} `yaml:"features"`
+		Headers  interface{} `yaml:"headers"`
+		ID       string      `yaml:"id"`
+		Logging  struct {
+			Level string `yaml:"level"`
+		} `yaml:"logging"`
+		Monitoring struct {
+			Enabled bool `yaml:"enabled"`
+			HTTP    struct {
+				Buffer  interface{} `yaml:"buffer"`
+				Enabled bool        `yaml:"enabled"`
+				Host    string      `yaml:"host"`
+				Port    int         `yaml:"port"`
+			} `yaml:"http"`
+			Logs      bool   `yaml:"logs"`
+			Metrics   bool   `yaml:"metrics"`
+			Namespace string `yaml:"namespace"`
+			UseOutput string `yaml:"use_output"`
+		} `yaml:"monitoring"`
+		Protection struct {
+			Enabled            bool   `yaml:"enabled"`
+			SigningKey         string `yaml:"signing_key"`
+			UninstallTokenHash string `yaml:"uninstall_token_hash"`
+		} `yaml:"protection"`
+	} `yaml:"agent"`
+	Fleet struct {
+		AccessAPIKey string `yaml:"access_api_key"`
+		Agent        struct {
+			ID string `yaml:"id"`
+		} `yaml:"agent"`
+		Enabled   bool     `yaml:"enabled"`
+		Host      string   `yaml:"host"`
+		Hosts     []string `yaml:"hosts"`
+		Protocol  string   `yaml:"protocol"`
+		ProxyURL  string   `yaml:"proxy_url"`
+		Reporting struct {
+			CheckFrequencySec int `yaml:"check_frequency_sec"`
+			Threshold         int `yaml:"threshold"`
+		} `yaml:"reporting"`
+		Ssl struct {
+			Renegotiation    string `yaml:"renegotiation"`
+			VerificationMode string `yaml:"verification_mode"`
+		} `yaml:"ssl"`
+		Timeout string `yaml:"timeout"`
+	} `yaml:"fleet"`
+	Host struct {
+		ID string `yaml:"id"`
+	} `yaml:"host"`
+	ID      string      `yaml:"id"`
+	Inputs  interface{} `yaml:"inputs"`
+	Outputs struct {
+		Default struct {
+			APIKey string   `yaml:"api_key"`
+			Hosts  []string `yaml:"hosts"`
+			Type   string   `yaml:"type"`
+		} `yaml:"default"`
+	} `yaml:"outputs"`
+	Path struct {
+		Config string `yaml:"config"`
+		Data   string `yaml:"data"`
+		Home   string `yaml:"home"`
+		Logs   string `yaml:"logs"`
+	} `yaml:"path"`
+	Revision int `yaml:"revision"`
+	Runtime  struct {
+		Arch   string `yaml:"arch"`
+		Os     string `yaml:"os"`
+		Osinfo struct {
+			Family  string `yaml:"family"`
+			Major   int    `yaml:"major"`
+			Minor   int    `yaml:"minor"`
+			Patch   int    `yaml:"patch"`
+			Type    string `yaml:"type"`
+			Version string `yaml:"version"`
+		} `yaml:"osinfo"`
+	} `yaml:"runtime"`
+	Signed struct {
+		Data string `yaml:"data"`
+	} `yaml:"signed"`
+}