refactor TestEnrollAndLog to use the agent status for health check

* Instead of relying in the absence of error logs, now it uses the agent and components status to ensure all is running as expected. The final state is what maters as there are transients, recoverable errors that might happens and are no reason for the test to fail.
* Rename the test and its file to better reflect what it's testing
* Remove testify.Suite as it wan't really used.
* add integration tag
* create testing/tools/check package and move ConnectedToFleet to it

pkg/testing/tools/check/check.go

@@ -0,0 +1,41 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package check
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/elastic-agent/pkg/control/v2/cproto"
+	integrationtest "github.com/elastic/elastic-agent/pkg/testing"
+)
+
+// ConnectedToFleet checks if the agent defined in the fixture is connected to
+// Fleet Server. It uses assert.Eventually and if it fails the last error will
+// be printed. It returns if the agent is connected to Fleet Server or not.
+func ConnectedToFleet(t *testing.T, fixture *integrationtest.Fixture) bool {
+	t.Helper()
+
+	var err error
+	var agentStatus integrationtest.AgentStatusOutput
+	assertFn := func() bool {
+		agentStatus, err = fixture.ExecStatus(context.Background())
+		return agentStatus.FleetState == int(cproto.State_HEALTHY)
+	}
+
+	connected := assert.Eventually(t, assertFn, 5*time.Minute, 5*time.Second,
+		"want fleet state %s, got %s. agent status: %v",
+		cproto.State_HEALTHY, cproto.State(agentStatus.FleetState), agentStatus)
+
+	if !connected && err != nil {
+		t.Logf("agent isn't connected to fleet-server: last error from agent status command: %v",
+			err)
+	}
+
+	return connected
+}

testing/integration/enroll_test.go → testing/integration/monitoring_logs_test.go

@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-// //go:build integration
+//go:build integration
 
 package integration
 
@@ -15,49 +15,31 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/stretchr/testify/suite"
 
 	"github.com/elastic/elastic-agent-libs/kibana"
+	"github.com/elastic/elastic-agent/pkg/control/v2/client"
 	atesting "github.com/elastic/elastic-agent/pkg/testing"
 	"github.com/elastic/elastic-agent/pkg/testing/define"
 	"github.com/elastic/elastic-agent/pkg/testing/tools"
+	"github.com/elastic/elastic-agent/pkg/testing/tools/check"
 )
 
-func TestEnrollAndLog(t *testing.T) {
+func TestMonitoringLogsShipped(t *testing.T) {
 	info := define.Require(t, define.Requirements{
-		OS: []define.OS{
-			{Type: define.Linux},
-		},
+		OS:    []define.OS{{Type: define.Linux}},
 		Stack: &define.Stack{},
 		Local: false,
 		Sudo:  true,
 	})
+	ctx := context.Background()
+
 	t.Logf("got namespace: %s", info.Namespace)
 	t.Skip("Test is flaky; see https://github.com/elastic/elastic-agent/issues/3081")
-	suite.Run(t, &EnrollRunner{requirementsInfo: info})
-}
-
-type EnrollRunner struct {
-	suite.Suite
-	requirementsInfo *define.Info
-	agentFixture     *atesting.Fixture
-}
-
-func (runner *EnrollRunner) SetupSuite() {
-	runner.T().Logf("In SetupSuite")
-	agentFixture, err := define.NewFixture(runner.T(), define.Version())
-	runner.agentFixture = agentFixture
-	require.NoError(runner.T(), err)
-}
-
-func (runner *EnrollRunner) SetupTest() {}
 
-func (runner *EnrollRunner) TestEnroll() {
-	t := runner.T()
-	info := runner.requirementsInfo
-
-	kibClient := info.KibanaClient
+	agentFixture, err := define.NewFixture(t, define.Version())
+	require.NoError(t, err)
 
 	t.Log("Enrolling agent in Fleet with a test policy")
 	createPolicyReq := kibana.AgentPolicy{
@@ -75,32 +57,34 @@ func (runner *EnrollRunner) TestEnroll() {
 			},
 		},
 	}
+
 	// Stage 1: Install
 	// As part of the cleanup process, we'll uninstall the agent
 	installOpts := atesting.InstallOpts{
 		NonInteractive: true,
 		Force:          true,
 	}
-	policy, err := tools.InstallAgentWithPolicy(t, context.Background(), installOpts, runner.agentFixture, kibClient, createPolicyReq)
+	policy, err := tools.InstallAgentWithPolicy(t, ctx,
+		installOpts, agentFixture, info.KibanaClient, createPolicyReq)
 	require.NoError(t, err)
 	t.Logf("created policy: %s", policy.ID)
 
-	t.Cleanup(func() {
-		// After: unenroll
-		t.Logf("Cleanup: unenrolling agent")
-		err = tools.UnEnrollAgent(info.KibanaClient)
-		require.NoError(t, err)
-	})
+	// t.Cleanup(func() {
+	// 	// After: unenroll
+	// 	t.Logf("Cleanup: unenrolling agent")
+	// 	err = tools.UnEnrollAgent(info.KibanaClient)
+	// 	require.NoError(t, err)
+	// })
 
-	t.Logf("sleeping for one minute...")
-	time.Sleep(time.Second * 60)
+	check.ConnectedToFleet(t, agentFixture)
 
-	// Stage 2: check indicies
+	// Stage 2: check indices
 	// This is mostly for debugging
 	resp, err := tools.GetAllindicies(info.ESClient)
 	require.NoError(t, err)
 	for _, run := range resp {
-		t.Logf("%s: %d/%d deleted: %d\n", run.Index, run.DocsCount, run.StoreSizeBytes, run.DocsDeleted)
+		t.Logf("%s: %d/%d deleted: %d\n",
+			run.Index, run.DocsCount, run.StoreSizeBytes, run.DocsDeleted)
 	}
 
 	// Stage 3: Make sure metricbeat logs are populated
@@ -111,21 +95,22 @@ func (runner *EnrollRunner) TestEnroll() {
 	require.NotZero(t, len(docs.Hits.Hits))
 	t.Logf("metricbeat: Got %d documents", len(docs.Hits.Hits))
 
-	// Stage 4: make sure we have no errors
-	t.Log("Making sure there are no error logs")
-	docs = findESDocs(t, func() (tools.Documents, error) {
-		return tools.CheckForErrorsInLogs(info.ESClient, info.Namespace, []string{})
-	})
-	t.Logf("errors: Got %d documents", len(docs.Hits.Hits))
-	for _, doc := range docs.Hits.Hits {
-		t.Logf("%#v", doc.Source)
+	// Stage 4: make sure all components are health
+	t.Log("Making sure all components are healthy")
+	status, err := agentFixture.ExecStatus(ctx)
+	require.NoError(t, err,
+		"could not get agent status to verify all components are healthy")
+	for _, c := range status.Components {
+		assert.Equalf(t, client.Healthy, client.State(c.State),
+			"component %s: want %s, got %s",
+			c.Name, client.Healthy, client.State(c.State))
 	}
-	require.Empty(t, docs.Hits.Hits)
 
 	// Stage 5: Make sure we have message confirming central management is running
 	t.Log("Making sure we have message confirming central management is running")
 	docs = findESDocs(t, func() (tools.Documents, error) {
-		return tools.FindMatchingLogLines(info.ESClient, info.Namespace, "Parsed configuration and determined agent is managed by Fleet")
+		return tools.FindMatchingLogLines(info.ESClient, info.Namespace,
+			"Parsed configuration and determined agent is managed by Fleet")
 	})
 	require.NotZero(t, len(docs.Hits.Hits))
 

testing/integration/proxy_url_test.go

@@ -17,9 +17,9 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/elastic/elastic-agent/pkg/control/v2/cproto"
 	integrationtest "github.com/elastic/elastic-agent/pkg/testing"
 	"github.com/elastic/elastic-agent/pkg/testing/define"
+	"github.com/elastic/elastic-agent/pkg/testing/tools/check"
 	"github.com/elastic/elastic-agent/testing/fleetservertest"
 	"github.com/elastic/elastic-agent/testing/proxytest"
 	"github.com/elastic/elastic-agent/version"
@@ -130,7 +130,7 @@ func TestProxyURL_EnrollProxyAndNoProxyInThePolicy(t *testing.T) {
 		require.NoError(t, err, "failed to install agent")
 	}
 
-	p.assertConnectedFleet(t)
+	check.ConnectedToFleet(t, p.fixture)
 }
 
 func TestProxyURL_EnrollProxyAndEmptyProxyInThePolicy(t *testing.T) {
@@ -175,7 +175,7 @@ func TestProxyURL_EnrollProxyAndEmptyProxyInThePolicy(t *testing.T) {
 		require.NoError(t, err, "failed to install agent")
 	}
 
-	p.assertConnectedFleet(t)
+	check.ConnectedToFleet(t, p.fixture)
 }
 
 func TestProxyURL_ProxyInThePolicyTakesPrecedence(t *testing.T) {
@@ -220,7 +220,7 @@ func TestProxyURL_ProxyInThePolicyTakesPrecedence(t *testing.T) {
 		require.NoError(t, err, "failed to install agent")
 	}
 
-	p.assertConnectedFleet(t)
+	check.ConnectedToFleet(t, p.fixture)
 
 	// ensure the agent is communicating through the proxy set in the policy
 	want := fleetservertest.NewPathCheckin(p.policyData.AgentID)
@@ -282,7 +282,7 @@ func TestProxyURL_NoEnrollProxyAndProxyInThePolicy(t *testing.T) {
 		require.NoError(t, err, "failed to install agent")
 	}
 
-	p.assertConnectedFleet(t)
+	check.ConnectedToFleet(t, p.fixture)
 
 	// ensure the agent is communicating through the new proxy
 	if !assert.Eventually(t, func() bool {
@@ -343,7 +343,7 @@ func TestProxyURL_RemoveProxyFromThePolicy(t *testing.T) {
 	}
 
 	// assert the agent is actually connected to fleet.
-	p.assertConnectedFleet(t)
+	check.ConnectedToFleet(t, p.fixture)
 
 	// ensure the agent is communicating through the proxy set in the policy
 	if !assert.Eventually(t, func() bool {
@@ -389,24 +389,7 @@ func TestProxyURL_RemoveProxyFromThePolicy(t *testing.T) {
 	assert.Equal(t, inspect.Fleet.ProxyURL, want)
 
 	// assert, again, the agent is actually connected to fleet.
-	p.assertConnectedFleet(t)
-}
-
-func (p *ProxyURL) assertConnectedFleet(t *testing.T) {
-	t.Helper()
-
-	var err error
-	var agentStatus integrationtest.AgentStatusOutput
-	if !assert.Eventually(t, func() bool {
-		agentStatus, err = p.fixture.ExecStatus(context.Background())
-		return agentStatus.FleetState == int(cproto.State_HEALTHY)
-	}, 5*time.Minute, 5*time.Second,
-		"want fleet state %s, got %s. agent status: %v",
-		cproto.State_HEALTHY, cproto.State(agentStatus.FleetState), agentStatus) {
-		if err != nil {
-			t.Logf("[assertConnectedFleet] last error from agent status command: %v", err)
-		}
-	}
+	check.ConnectedToFleet(t, p.fixture)
 }
 
 func (p *ProxyURL) setupFleet(t *testing.T, fleetHost string) {