refactor testing/integration and pkg/testing (#3378)

* pkg/testing/tools
  - split the 'tools' into smaller packages with well defined scope: 'estools', 'fleettools' and 'check'
  - adjust method names for consistency

* testing/integration:
  - make diagnostics to require to be run on a remote host
  - add more context to errors
  - reorder functions on endpoint_security_test so tests are the first functions and when the test fails because the 'Endpoint' directory isn't removed, it now prints the content of the directory

pkg/component/runtime/manager_test.go

@@ -1371,52 +1371,59 @@ func TestManager_FakeInput_GoodUnitToBad(t *testing.T) {
 				t.Logf("component state changed: %+v", state)
 				if state.State == client.UnitStateFailed {
 					subErrCh <- fmt.Errorf("component failed: %s", state.Message)
+					return
+				}
+
+				unit, ok := state.Units[ComponentUnitKey{UnitType: client.UnitTypeInput, UnitID: "good-input"}]
+				if !ok {
+					subErrCh <- errors.New("unit missing: good-input")
+					return
+				}
+				if unitGood {
+					switch unit.State {
+					case client.UnitStateFailed:
+						subErrCh <- fmt.Errorf("unit failed: %s", unit.Message)
+						return
+					case client.UnitStateHealthy:
+						// good unit it; now make it bad
+						t.Logf("marking good-input as having a hard-error for config")
+						updatedComp := comp
+						updatedComp.Units = make([]component.Unit, len(comp.Units))
+						copy(updatedComp.Units, comp.Units)
+						updatedComp.Units[1] = component.Unit{
+							ID:   "good-input",
+							Type: client.UnitTypeInput,
+							Err:  errors.New("hard-error for config"),
+						}
+						unitGood = false
+						err := m.Update(component.Model{Components: []component.Component{updatedComp}})
+						if err != nil {
+							subErrCh <- err
+						}
+					case client.UnitStateStarting:
+						// acceptable
+					default:
+						// unknown state that should not have occurred
+						subErrCh <- fmt.Errorf("unit reported unexpected state: %v", unit.State)
+					}
 				} else {
-					unit, ok := state.Units[ComponentUnitKey{UnitType: client.UnitTypeInput, UnitID: "good-input"}]
-					if ok {
-						if unitGood {
-							if unit.State == client.UnitStateFailed {
-								subErrCh <- fmt.Errorf("unit failed: %s", unit.Message)
-							} else if unit.State == client.UnitStateHealthy {
-								// good unit it; now make it bad
-								t.Logf("marking good-input as having a hard-error for config")
-								updatedComp := comp
-								updatedComp.Units = make([]component.Unit, len(comp.Units))
-								copy(updatedComp.Units, comp.Units)
-								updatedComp.Units[1] = component.Unit{
-									ID:   "good-input",
-									Type: client.UnitTypeInput,
-									Err:  errors.New("hard-error for config"),
-								}
-								unitGood = false
-								err := m.Update(component.Model{Components: []component.Component{updatedComp}})
-								if err != nil {
-									subErrCh <- err
-								}
-							} else if unit.State == client.UnitStateStarting {
-								// acceptable
-							} else {
-								// unknown state that should not have occurred
-								subErrCh <- fmt.Errorf("unit reported unexpected state: %v", unit.State)
-							}
-						} else {
-							if unit.State == client.UnitStateFailed {
-								// went to failed; stop whole component
-								err := m.Update(component.Model{Components: []component.Component{}})
-								if err != nil {
-									subErrCh <- err
-								}
-							} else if unit.State == client.UnitStateStopped {
-								// unit was stopped
-								subErrCh <- nil
-							} else {
-								subErrCh <- errors.New("good-input unit should be either failed or stopped")
-							}
+					switch unit.State {
+					case client.UnitStateFailed:
+						// went to failed; stop whole component
+						err := m.Update(component.Model{Components: []component.Component{}})
+						if err != nil {
+							subErrCh <- err
 						}
-					} else {
-						subErrCh <- errors.New("unit missing: good-input")
+					case client.UnitStateStopped:
+						// unit was stopped
+						subErrCh <- nil
+					default:
+						subErrCh <- fmt.Errorf(
+							"good-input unit should be either failed or stopped, but it's '%s'",
+							unit.State)
 					}
 				}
+
 			}
 		}
 	}()

pkg/testing/fixture_install.go

@@ -84,8 +84,12 @@ func (i InstallOpts) toCmdArgs() []string {
 	return args
 }
 
-// Install installs the prepared Elastic Agent binary and returns:
-//   - the combined output of stdout and stderr
+// Install installs the prepared Elastic Agent binary and registers a t.Cleanup
+// function to uninstall the agent if it hasn't been uninstalled. It also takes
+// care of collecting a diagnostics when AGENT_COLLECT_DIAG=true or the test
+// has failed.
+// It returns:
+//   - the combined output of Install command stdout and stderr
 //   - an error if any.
 func (f *Fixture) Install(ctx context.Context, installOpts *InstallOpts, opts ...process.CmdOption) ([]byte, error) {
 	f.t.Logf("[test %s] Inside fixture install function", f.t.Name())
@@ -134,12 +138,12 @@ func (f *Fixture) Install(ctx context.Context, installOpts *InstallOpts, opts ..
 
 		// environment variable AGENT_KEEP_INSTALLED=true will skip the uninstall
 		// useful to debug the issue with the Elastic Agent
-		if f.t.Failed() && keepInstalled() {
+		if f.t.Failed() && keepInstalledFlag() {
 			f.t.Logf("skipping uninstall; test failed and AGENT_KEEP_INSTALLED=true")
 			return
 		}
 
-		if keepInstalled() {
+		if keepInstalledFlag() {
 			f.t.Logf("ignoring AGENT_KEEP_INSTALLED=true as test succeeded, " +
 				"keeping the agent installed will jeopardise other tests")
 		}
@@ -327,7 +331,7 @@ func collectDiagFlag() bool {
 	return v
 }
 
-func keepInstalled() bool {
+func keepInstalledFlag() bool {
 	// failure reports false (ignore error)
 	v, _ := strconv.ParseBool(os.Getenv("AGENT_KEEP_INSTALLED"))
 	return v

pkg/testing/tools/artifacts_api.go

@@ -135,7 +135,7 @@ type ArtifactAPIClient struct {
 	url string
 }
 
-// Creates a new Artifact API client
+// NewArtifactAPIClient creates a new Artifact API client
 func NewArtifactAPIClient(opts ...ArtifactAPIClientOpt) *ArtifactAPIClient {
 	c := &ArtifactAPIClient{
 		url: defaultArtifactAPIURL,
@@ -204,7 +204,7 @@ func (aac ArtifactAPIClient) GetBuildDetails(ctx context.Context, version string
 	return checkResponseAndUnmarshal[BuildDetails](resp)
 }
 
-func (aac *ArtifactAPIClient) composeURL(relativePath string) (string, error) {
+func (aac ArtifactAPIClient) composeURL(relativePath string) (string, error) {
 	joinedURL, err := url.JoinPath(aac.url, relativePath)
 	if err != nil {
 		return "", fmt.Errorf("composing URL with %q %q: %w", aac.url, relativePath, err)
@@ -213,7 +213,7 @@ func (aac *ArtifactAPIClient) composeURL(relativePath string) (string, error) {
 	return joinedURL, nil
 }
 
-func (aac *ArtifactAPIClient) createAndPerformRequest(ctx context.Context, URL string) (*http.Response, error) {
+func (aac ArtifactAPIClient) createAndPerformRequest(ctx context.Context, URL string) (*http.Response, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, URL, nil)
 	if err != nil {
 		err = fmt.Errorf("composing request: %w", err)

pkg/testing/tools/check/check.go

@@ -11,14 +11,16 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
+	"github.com/elastic/elastic-agent-libs/kibana"
 	"github.com/elastic/elastic-agent/pkg/control/v2/cproto"
 	integrationtest "github.com/elastic/elastic-agent/pkg/testing"
+	"github.com/elastic/elastic-agent/pkg/testing/tools/fleettools"
 )
 
 // ConnectedToFleet checks if the agent defined in the fixture is connected to
 // Fleet Server. It uses assert.Eventually and if it fails the last error will
 // be printed. It returns if the agent is connected to Fleet Server or not.
-func ConnectedToFleet(t *testing.T, fixture *integrationtest.Fixture) bool {
+func ConnectedToFleet(t *testing.T, fixture *integrationtest.Fixture, timeout time.Duration) bool {
 	t.Helper()
 
 	var err error
@@ -28,7 +30,7 @@ func ConnectedToFleet(t *testing.T, fixture *integrationtest.Fixture) bool {
 		return agentStatus.FleetState == int(cproto.State_HEALTHY)
 	}
 
-	connected := assert.Eventually(t, assertFn, 5*time.Minute, 5*time.Second,
+	connected := assert.Eventually(t, assertFn, timeout, 5*time.Second,
 		"want fleet state %s, got %s. agent status: %v",
 		cproto.State_HEALTHY, cproto.State(agentStatus.FleetState), agentStatus)
 
@@ -39,3 +41,26 @@ func ConnectedToFleet(t *testing.T, fixture *integrationtest.Fixture) bool {
 
 	return connected
 }
+
+// FleetAgentStatus returns a niladic function that returns true if the agent
+// has reached expectedStatus; false otherwise. The returned function is intended
+// for use with assert.Eventually or require.Eventually.
+func FleetAgentStatus(t *testing.T,
+	client *kibana.Client,
+	policyID,
+	expectedStatus string) func() bool {
+	return func() bool {
+		currentStatus, err := fleettools.GetAgentStatus(client, policyID)
+		if err != nil {
+			t.Errorf("unable to determine agent status: %s", err.Error())
+			return false
+		}
+
+		if currentStatus == expectedStatus {
+			return true
+		}
+
+		t.Logf("Agent fleet status: %s", currentStatus)
+		return false
+	}
+}

pkg/testing/tools/elasticsearch.go → pkg/testing/tools/estools/elasticsearch.go

@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package tools
+package estools
 
 import (
 	"bytes"

pkg/testing/tools/agents.go → pkg/testing/tools/fleettools/fleet.go

@@ -2,21 +2,15 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package tools
+package fleettools
 
 import (
 	"context"
 	"errors"
 	"fmt"
 	"os"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/elastic-agent-libs/kibana"
-	"github.com/elastic/elastic-agent/pkg/control/v2/client"
-	"github.com/elastic/elastic-agent/pkg/control/v2/cproto"
 )
 
 // GetAgentByPolicyIDAndHostnameFromList get an agent by the local_metadata.host.name property, reading from the agents list
@@ -47,6 +41,14 @@ func GetAgentByPolicyIDAndHostnameFromList(client *kibana.Client, policyID, host
 	return hostnameAgents[0], nil
 }
 
+func GetAgentIDByHostname(client *kibana.Client, policyID, hostname string) (string, error) {
+	agent, err := GetAgentByPolicyIDAndHostnameFromList(client, policyID, hostname)
+	if err != nil {
+		return "", err
+	}
+	return agent.Agent.ID, nil
+}
+
 func GetAgentStatus(client *kibana.Client, policyID string) (string, error) {
 	hostname, err := os.Hostname()
 	if err != nil {
@@ -97,15 +99,8 @@ func UnEnrollAgent(client *kibana.Client, policyID string) error {
 	return nil
 }
 
-func GetAgentIDByHostname(client *kibana.Client, policyID, hostname string) (string, error) {
-	agent, err := GetAgentByPolicyIDAndHostnameFromList(client, policyID, hostname)
-	if err != nil {
-		return "", err
-	}
-	return agent.Agent.ID, nil
-}
-
 func UpgradeAgent(client *kibana.Client, policyID, version string, force bool) error {
+	// TODO: fix me: this does not work if FQDN is enabled
 	hostname, err := os.Hostname()
 	if err != nil {
 		return err
@@ -128,7 +123,7 @@ func UpgradeAgent(client *kibana.Client, policyID, version string, force bool) e
 	return nil
 }
 
-func GetDefaultFleetServerURL(client *kibana.Client) (string, error) {
+func DefaultURL(client *kibana.Client) (string, error) {
 	req := kibana.ListFleetServerHostsRequest{}
 	resp, err := client.ListFleetServerHosts(context.Background(), req)
 	if err != nil {
@@ -144,23 +139,5 @@ func GetDefaultFleetServerURL(client *kibana.Client) (string, error) {
 		}
 	}
 
-	return "", errors.New("unable to determine default fleet server host")
-}
-
-func WaitForAgent(ctx context.Context, t *testing.T, c client.Client) {
-	require.Eventually(t, func() bool {
-		err := c.Connect(ctx)
-		if err != nil {
-			t.Logf("connecting client to agent: %v", err)
-			return false
-		}
-		defer c.Disconnect()
-		state, err := c.State(ctx)
-		if err != nil {
-			t.Logf("error getting the agent state: %v", err)
-			return false
-		}
-		t.Logf("agent state: %+v", state)
-		return state.State == cproto.State_HEALTHY
-	}, 2*time.Minute, 10*time.Second, "Agent never became healthy")
+	return "", errors.New("unable to determine default fleet server URL")
 }