run tests in parallel

.github/workflows/integration-tests.yml

@@ -21,5 +21,8 @@ jobs:
       run: echo "OTEL_EXPORTER_ENDPOINT=http://172.17.0.1:8080" >> $GITHUB_ENV
     - name: Run tests
       run: |
+        top -b -n 10 -d 5 > top_output.txt &
+        TOP_PID=$!
         cd integration-tests/tests
-        ./run-integration-tests.sh
+        ./run-integration-tests.sh
+        kill $TOP_PID

integration-tests/common/common.go

@@ -11,8 +11,8 @@ type Unmarshaler interface {
 	Unmarshal([]byte) error
 }
 
-const DefaultRetryInterval = time.Second * 5
-const DefaultTimeout = time.Minute * 2
+const DefaultRetryInterval = 100 * time.Millisecond
+const DefaultTimeout = time.Minute
 
 func FetchDataFromURL(url string, target Unmarshaler) error {
 	resp, err := http.Get(url)

integration-tests/tests/docker-compose.yaml → integration-tests/docker-compose.yaml

@@ -4,7 +4,7 @@ services:
   mimir:
     image: grafana/mimir:2.9.0
     volumes:
-      - ../configs/mimir:/etc/mimir-config
+      - ./configs/mimir:/etc/mimir-config
     entrypoint:
       - /bin/mimir
       - -config.file=/etc/mimir-config/mimir.yaml
@@ -22,7 +22,7 @@ services:
     restart: always
     command: ["--config=/etc/otel-collector-contrib.yaml", ""]
     volumes:
-      - ../configs/otel-collector-contrib/otel-collector-contrib.yaml:/etc/otel-collector-contrib.yaml
+      - ./configs/otel-collector-contrib/otel-collector-contrib.yaml:/etc/otel-collector-contrib.yaml
     ports:
       - "4317:4317"   # OTLP gRPC receiver
       - "4318:4318"   # OTLP HTTP exporter
@@ -32,7 +32,7 @@ services:
   demo-client:
     build:
       dockerfile: Dockerfile
-      context: ../configs/otel-gen-client
+      context: ./configs/otel-gen-client
     restart: always
     environment:
       - OTEL_EXPORTER_OTLP_ENDPOINT=otel-collector:4317
@@ -43,7 +43,7 @@ services:
   demo-server:
     build:
       dockerfile: Dockerfile
-      context: ../configs/otel-gen-server
+      context: ./configs/otel-gen-server
     restart: always
     environment:
       - OTEL_EXPORTER_OTLP_ENDPOINT=otel-collector:4317

integration-tests/run-integration-tests.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+set -e
+
+fail_flag_file="/tmp/test_fail_flag"
+rm -f "$fail_flag_file"
+tmp_dir="temp_logs"
+
+cleanup() {
+    if [ -d "$tmp_dir" ]; then
+        for tmp_log in "$tmp_dir"/*; do
+            echo "$tmp_log"
+            if grep -q "FAIL" "$tmp_log"; then
+                echo "Failure detected in $tmp_log:"
+                cat "$tmp_log"
+            fi
+            rm -f "$tmp_log"
+        done
+        rmdir "$tmp_dir"
+    fi
+    docker-compose down
+}
+
+success() {
+    echo "All integration tests passed!"
+    exit 0
+}
+
+#make -C .. agent-flow
+AGENT_BINARY_PATH="../../../build/grafana-agent-flow"
+
+docker-compose up -d
+
+mkdir -p "$tmp_dir"
+
+counter=0
+
+# Run tests in parallel
+while read -r test_dir; do
+    (
+        pushd "$test_dir"
+        dir_name=$(basename "$test_dir")
+        agent_logfile="../../${tmp_dir}/${dir_name}_agent.log"
+        test_logfile="../../${tmp_dir}/${dir_name}_test.log"
+        "$AGENT_BINARY_PATH" run config.river > "$agent_logfile" 2>&1 &
+        AGENT_PID=$!
+        if ! go test >> "$test_logfile" 2>&1; then
+            echo "FAIL" >> "$test_logfile"
+            touch "$fail_flag_file"
+        fi
+        # Concatenate the log files into one if desired.
+        cat "$agent_logfile" >> "$test_logfile"
+        rm "$agent_logfile"
+
+        rm -rf data-agent
+        kill $AGENT_PID || true
+        popd
+    ) &
+
+    # Increment the counter
+    counter=$((counter+1))
+
+    # If 5 tests are running, wait for them to finish
+    if [ "$counter" -eq 5 ]; then
+        wait
+        counter=0
+    fi
+done < <(find ./tests -maxdepth 1 -type d ! -path ./tests)
+
+wait
+
+pwd
+
+cleanup
+
+if [ -f "$fail_flag_file" ]; then
+    rm "$fail_flag_file"
+    exit 1
+else
+    success
+fi

integration-tests/tests/scrap-prom-metrics/config.river

@@ -3,15 +3,15 @@ prometheus.scrape "scrap_prom_metrics" {
     {"__address__" = "localhost:9001"},
   ]
   forward_to = [prometheus.remote_write.scrap_prom_metrics.receiver]
-  scrape_interval = "10s"
-  scrape_timeout = "5s"
+  scrape_interval = "100ms"
+  scrape_timeout = "50ms"
 }
 
 prometheus.remote_write "scrap_prom_metrics" {
   endpoint {
     url = "http://localhost:9009/api/v1/push"
     metadata_config {
-        send_interval = "1s"
+        send_interval = "200ms"
     }
     queue_config {
         max_samples_per_send = 100