Add serverless beats tests (#3658)

* cleaning up

* final bit of cleanup

* fix magefile, cleanup docs

* clean up errors, make linter happy

* fix headers

* fix fields in runner config

* add dashboard checks

* clean up, refactor

* clean up

* tinker with env vars

* fix defaults in fixture

* check binary name in test setup

* allow ilm override in tests

* fix filebeat tests, add cleanup

* tinker with dashboards

* fix ilm tests

* use API keys for auth

* add additional integration tests

* remove beats-specific code

* hack in serverless tests

* tinker with tests

* change env var naming

* actually use correct provisioner name

* tinker with buildkite again

* fix things after refactor

* fix buildkite

* fix my bash scripts

* my bash is a tad rusty

* tinker with script hooks

* not sure what ci role I broke

* clean up es handlers

* deal with recent refactor

* fix my broken refactor

* change url, see what happens

* tinker with tests more

* swap pipelines, see what happens

* break apart beat runners

* create test

* add more tests

* override tests in progress

* finish overwrite tests

* still adding tests

* cleanup

* fix env var names

* fix filebeat setup, test names

* fixing up filebeat

* use templates, fix ES query

* tinker with buildkite

* still tinkering

* still trying to get it to work

* still trying to get it to work

* add script, see what happens

* tinkering with script

* tinkering with script

* trying to diagnose buildkite

* still fighting with buildkite

* still trying to get it to work

* set workspace

* change workspace

* change workspace, again

* change workspace, again

* trying to debug buildkite

* add debug statements

* run mage twice

* I swear I'll figure this out

* potential script install issue?

* use different condition for if block

* fix potential wrong flag in setup scripts

* change -p flag

* fix workspace path, add tests,remove debug code

* use proper env vars

* fix mage call, directories

* set working directories

* now trying to get auth working

* change name to make setup happy

* disable cleanup

* re-enable cleanup

* run one test at a time

* refactor, run filebeat

* move function def

* fix if blocks

* fix test run, add support for auditbeat and packetbeat

* add packetbeat support, run auditbeat tests

* add serverless buildkite logic

* add more documentation

* fix up tests

.buildkite/pipeline.yml

@@ -228,6 +228,18 @@ steps:
       provider: "gcp"
       machineType: "n1-standard-8"
 
+  - label: "Serverless Beats Tests"
+    key: "serverless-beats-integration-tests"
+    command: ".buildkite/scripts/steps/beats_tests.sh"
+    if: "build.env('CRON') == 'yes'"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+    retry:
+      manual:
+        allowed: true
+
+
   - wait: ~
     continue_on_failure: true
   - label: "Processing test results"

.buildkite/scripts/common.sh

@@ -53,7 +53,7 @@ getOSOptions() {
 # Wrapper function for executing mage
 mage() {
     go version
-    if ! [ -x "$(type -p mage | sed 's/mage is //g')" ];
+    if ! [ -x "$(type -P mage | sed 's/mage is //g')" ];
     then
         echo "installing mage ${SETUP_MAGE_VERSION}"
         make mage
@@ -68,7 +68,7 @@ mage() {
 # Wrapper function for executing go
 go(){
     # Search for the go in the Path
-    if ! [ -x "$(type -p go | sed 's/go is //g')" ];
+    if ! [ -x "$(type -P go | sed 's/go is //g')" ];
     then
         getOSOptions
         echo "installing golang "${GO_VERSION}" for "${AGENT_OS_NAME}/${AGENT_OS_ARCH}" "

.buildkite/scripts/steps/beats_tests.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+#=========================
+# NOTE: This entire script is a temporary hack until we have buildkite set up on the beats repo.
+# until then, we need some kind of serverless integration tests, hence this script, which just clones the beats repo,
+# and runs the serverless integration suite against different beats
+# After buildkite is set up on beats, this file/PR should be reverted.
+#==========================
+
+source .buildkite/scripts/common.sh
+STACK_PROVISIONER="${1:-"serverless"}"
+
+run_test_for_beat(){
+    local beat_name=$1
+
+    #build
+    export WORKSPACE="build/beats/x-pack/${beat_name}"
+    SNAPSHOT=true PLATFORMS=linux/amd64 PACKAGES=tar.gz,zip mage package
+
+    #run
+    export AGENT_BUILD_DIR="build/beats/x-pack/${beat_name}/build/distributions"
+    export WORKSPACE=$(pwd)
+
+    set +e
+    TEST_INTEG_CLEAN_ON_EXIT=true TEST_PLATFORMS="linux/amd64" STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage integration:testBeatServerless $beat_name
+    TESTS_EXIT_STATUS=$?
+    set -e
+
+    return $TESTS_EXIT_STATUS
+}
+#run mage before setup, since this will install go and mage
+#the setup scripts will do a few things that assume we're running out of elastic-agent and will break things for beats, so run before we do actual setup
+mage -l
+
+mkdir -p build
+cd build
+
+git clone --filter=tree:0 [email protected]:elastic/beats.git
+cd ..
+
+# export WORKSPACE=beats/x-pack/metricbeat
+
+# SNAPSHOT=true PLATFORMS=linux/amd64,windows/amd64 PACKAGES=tar.gz,zip mage package
+
+
+# cd ..
+
+# export AGENT_BUILD_DIR=build/beats/x-pack/metricbeat/build/distributions
+# export WORKSPACE=$(pwd)
+
+# set +e
+# TEST_INTEG_CLEAN_ON_EXIT=true TEST_PLATFORMS="linux/amd64" STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage integration:testBeatServerless metricbeat
+# TESTS_EXIT_STATUS=$?
+# set -e
+
+# exit $TESTS_EXIT_STATUS
+
+echo "testing metricbeat..."
+run_test_for_beat metricbeat
+
+
+
+echo "testing filebeat..."
+run_test_for_beat filebeat
+
+
+
+echo "testing auditbeat..."
+run_test_for_beat auditbeat

magefile.go

@@ -1567,6 +1567,33 @@ func (Integration) PrepareOnRemote() {
 	mg.Deps(mage.InstallGoTestTools)
 }
 
+// Run beat serverless tests
+func (Integration) TestBeatServerless(ctx context.Context, beatname string) error {
+	beatBuildPath := filepath.Join("..", "beats", "x-pack", beatname, "build", "distributions")
+	if os.Getenv("AGENT_BUILD_DIR") == "" {
+		err := os.Setenv("AGENT_BUILD_DIR", beatBuildPath)
+		if err != nil {
+			return fmt.Errorf("error setting build dir: %s", err)
+		}
+	}
+
+	// a bit of bypass logic; run as serverless by default
+	if os.Getenv("STACK_PROVISIONER") == "" {
+		err := os.Setenv("STACK_PROVISIONER", "serverless")
+		if err != nil {
+			return fmt.Errorf("error setting serverless stack var: %w", err)
+		}
+	} else if os.Getenv("STACK_PROVISIONER") == "stateful" {
+		fmt.Printf(">>> Warning: running TestBeatServerless as stateful\n")
+	}
+
+	err := os.Setenv("TEST_BINARY_NAME", beatname)
+	if err != nil {
+		return fmt.Errorf("error setting binary name: %w", err)
+	}
+	return integRunner(ctx, false, "TestBeatsServerless")
+}
+
 // TestOnRemote shouldn't be called locally (called on remote host to perform testing)
 func (Integration) TestOnRemote(ctx context.Context) error {
 	mg.Deps(Build.TestBinaries)

pkg/testing/tools/estools/elasticsearch.go

@@ -212,7 +212,7 @@ func GetLatestDocumentMatchingQuery(ctx context.Context, client elastictransport
 	queryRaw := map[string]interface{}{
 		"query": query,
 		"sort": map[string]interface{}{
-			"timestamp": "desc",
+			"@timestamp": "desc",
 		},
 		"size": 1,
 	}