RUM-6596 Make sure the legacy integration test fail the CI job if fai…

…ling

.gitlab-ci.yml

@@ -53,10 +53,7 @@ stages:
     - set +e
     - exit_code=0
     - $ANDROID_HOME/emulator/emulator -avd "$EMULATOR_NAME" -grpc-use-jwt -no-snapstorage -no-audio -no-window -no-boot-anim -verbose -qemu -machine virt &
-    - GRADLE_OPTS="-Xmx3072m" ./gradlew :instrumented:integration:assembleDebug :instrumented:integration:assembleDebugAndroidTest --stacktrace --no-daemon $( (( $ANDROID_API <= 23 )) && echo "-Puse-api21-java-backport -Puse-desugaring" )
-    - $ANDROID_HOME/platform-tools/adb install -t -d $( (( $ANDROID_API >= 23 )) && echo "-g" ) -r instrumented/integration/build/outputs/apk/androidTest/debug/integration-debug-androidTest.apk
-    - $ANDROID_HOME/platform-tools/adb install -t -d $( (( $ANDROID_API >= 23 )) && echo "-g" ) -r instrumented/integration/build/outputs/apk/debug/integration-debug.apk
-    - $ANDROID_HOME/platform-tools/adb shell am instrument -w com.datadog.android.sdk.integration.test/androidx.test.runner.AndroidJUnitRunner || exit_code=$?
+    - GRADLE_OPTS="-Xmx3072m" ./gradlew :instrumented:integration:connectedDebugAndroidTest --stacktrace --no-daemon $( (( $ANDROID_API <= 23 )) && echo "-Puse-api21-java-backport -Puse-desugaring" ) || exit_code=$?
     - $ANDROID_HOME/platform-tools/adb emu kill
     - if [[ "$exit_code" -ne 0 ]]; then exit 1; fi
     - exit 0

instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/ConsentPendingGrantedFragmentTrackingTest.kt

@@ -6,6 +6,7 @@
 
 package com.datadog.android.sdk.integration.rum
 
+import android.os.Build
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.filters.LargeTest
 import androidx.test.platform.app.InstrumentationRegistry
@@ -31,6 +32,10 @@ internal class ConsentPendingGrantedFragmentTrackingTest : FragmentTrackingTest(
 
     @Test
     fun verifyViewEventsOnSwipe() {
+        if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) {
+            // We skip this test on Android 28 and below because Espresso doesn't work well with ViewPager
+            return
+        }
         val expectedEvents = runInstrumentationScenario(mockServerRule)
 
         // update the tracking consent

instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/FragmentTrackingTest.kt

@@ -45,7 +45,6 @@ internal abstract class FragmentTrackingTest :
         val fragmentAViewUrl = currentFragmentViewUrl(activity)
         // ignore view event for view start, it will be reduced
 
-        // view stopped
         expectedEvents.add(
             ExpectedViewEvent(
                 fragmentAViewUrl,
@@ -55,13 +54,15 @@ internal abstract class FragmentTrackingTest :
         )
 
         // swipe to change the fragment
+        instrumentation.waitForIdleSync()
         onView(ViewMatchers.withId(R.id.btn_next)).perform(ViewActions.click())
         instrumentation.waitForIdleSync()
-        Thread.sleep(200) // give time to the view id to update
+        Thread.sleep(200)
         val fragmentBViewUrl = currentFragmentViewUrl(activity)
         mockServerRule.activity.supportFragmentManager.fragments
         // ignore view event for updating the time, it will be reduced
         // view stopped
+        waitForPendingRUMEvents()
         expectedEvents.add(
             ExpectedViewEvent(
                 fragmentBViewUrl,
@@ -73,8 +74,8 @@ internal abstract class FragmentTrackingTest :
         // swipe to close the view
         onView(ViewMatchers.withId(R.id.btn_last)).perform(ViewActions.click())
         instrumentation.waitForIdleSync()
-        Thread.sleep(200) // give time to the view id to update
-
+        Thread.sleep(200)
+        waitForPendingRUMEvents()
         // for updating the time
         expectedEvents.add(
             ExpectedViewEvent(
@@ -84,15 +85,6 @@ internal abstract class FragmentTrackingTest :
             )
         )
 
-        // view stopped
-//        expectedEvents.add(
-//            ExpectedViewEvent(
-//                fragmentAViewUrl,
-//                3,
-//                currentFragmentExtras(activity)
-//            )
-//        )
-
         instrumentation.runOnMainSync {
             instrumentation.callActivityOnStop(mockServerRule.activity)
         }

instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/RumTest.kt

@@ -17,6 +17,7 @@ import com.datadog.android.sdk.utils.isRumUrl
 import com.google.gson.JsonObject
 import org.assertj.core.api.Assertions.assertThat
 import java.lang.Long.max
+import java.util.LinkedList
 import java.util.concurrent.TimeUnit
 
 internal abstract class RumTest<R : Activity, T : MockServerActivityTestRule<R>> {
@@ -28,8 +29,10 @@ internal abstract class RumTest<R : Activity, T : MockServerActivityTestRule<R>>
         handledRequests: List<HandledRequest>,
         expectedEvents: List<ExpectedEvent>
     ) {
-        val sentGestureEvents = mutableListOf<JsonObject>()
-        val sentLaunchEvents = mutableListOf<JsonObject>()
+        val sentViewEvents = LinkedList<JsonObject>()
+        val sentActionEvents = LinkedList<JsonObject>()
+        val sentResourceEvents = LinkedList<JsonObject>()
+        val sentLaunchEvents = LinkedList<JsonObject>()
         handledRequests
             .filter { it.url?.isRumUrl() ?: false }
             .forEach { request ->
@@ -44,25 +47,39 @@ internal abstract class RumTest<R : Activity, T : MockServerActivityTestRule<R>>
                         .forEach {
                             if (it.isEventRelatedToApplicationLaunch) {
                                 sentLaunchEvents += it
-                            } else {
-                                sentGestureEvents += it
+                            } else if (it.isViewEvent) {
+                                sentViewEvents += it
+                            } else if (it.isActionEvent) {
+                                sentActionEvents += it
+                            } else if (it.isResourceEvent) {
+                                sentResourceEvents += it
                             }
                         }
                 }
             }
-        // Because launch events can be weirdly order dependent, consider them separately
         val launchEventPredicate = { event: ExpectedEvent ->
             event is ExpectedApplicationLaunchViewEvent || event is ExpectedApplicationStartActionEvent
         }
         val expectedLaunchEvents = expectedEvents.filter(launchEventPredicate)
-        sentLaunchEvents
-            .reduceViewEvents()
-            .verifyEventMatches(expectedLaunchEvents)
-
-        val otherExpectedEvents = expectedEvents.filterNot(launchEventPredicate)
-        sentGestureEvents
-            .reduceViewEvents()
-            .verifyEventMatches(otherExpectedEvents)
+        val expectedViewEvents = expectedEvents.filterIsInstance<ExpectedViewEvent>()
+        val expectedActionEvents = expectedEvents.filterIsInstance<ExpectedGestureEvent>()
+        val expectedResourceEvents = expectedEvents.filterIsInstance<ExpectedResourceEvent>()
+        if (expectedLaunchEvents.isNotEmpty()) {
+            sentLaunchEvents
+                .reduceViewEvents()
+                .verifyEventMatches(expectedLaunchEvents)
+        }
+        if (expectedViewEvents.isNotEmpty()) {
+            sentViewEvents
+                .reduceViewEvents()
+                .verifyViewEventsMatches(expectedViewEvents)
+        }
+        if (expectedActionEvents.isNotEmpty()) {
+            sentActionEvents.verifyEventMatches(expectedActionEvents)
+        }
+        if (expectedResourceEvents.isNotEmpty()) {
+            sentResourceEvents.verifyEventMatches(expectedResourceEvents)
+        }
     }
 
     protected fun verifyNoRumPayloadSent(
@@ -88,6 +105,12 @@ internal abstract class RumTest<R : Activity, T : MockServerActivityTestRule<R>>
     private val JsonObject.isViewEvent
         get() = get("type")?.asString == "view"
 
+    private val JsonObject.isActionEvent
+        get() = get("type")?.asString == "action"
+
+    private val JsonObject.isResourceEvent
+        get() = get("type")?.asString == "resource"
+
     private val JsonObject.isTelemetryEvent
         get() = get("type")?.asString == "telemetry"
 
@@ -119,6 +142,10 @@ internal abstract class RumTest<R : Activity, T : MockServerActivityTestRule<R>>
         }
     }
 
+    private fun List<JsonObject>.dropActionEvents(): List<JsonObject> {
+        return filterNot { it.isActionEvent }
+    }
+
     companion object {
         internal val FINAL_WAIT_MS = TimeUnit.SECONDS.toMillis(60)
     }

instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/RumTestUtils.kt

@@ -36,11 +36,9 @@ internal fun List<JsonObject>.verifyEventMatches(
                 this.joinToString("\n") { "\t>> $it" }
         )
         .isEqualTo(expected.size)
-
     this.forEachIndexed { index, event ->
         when (val expectedEvent = expected[index]) {
             is ExpectedApplicationLaunchViewEvent -> event.verifyEventMatches(expectedEvent)
-            is ExpectedViewEvent -> event.verifyEventMatches(expectedEvent)
             is ExpectedGestureEvent -> event.verifyEventMatches(expectedEvent)
             is ExpectedApplicationStartActionEvent -> event.verifyEventMatches(expectedEvent)
             is ExpectedResourceEvent -> event.verifyEventMatches(expectedEvent)
@@ -52,6 +50,43 @@ internal fun List<JsonObject>.verifyEventMatches(
     }
 }
 
+internal fun List<JsonObject>.verifyViewEventsMatches(
+    expected: List<ExpectedViewEvent>
+) {
+    Assertions.assertThat(this.size)
+        .withFailMessage(
+            "We were expecting ${expected.size} rum " +
+                "view events instead they were ${this.size}: \n" +
+                " -- EXPECTED -- \n" +
+                expected.joinToString("\n") { "\t>> $it" } +
+                "\n -- ACTUAL -- \n" +
+                this.joinToString("\n") { "\t>> $it" }
+        )
+        .isEqualTo(expected.size)
+    // in case of views because they are reduced by their document version and they are not going to follow
+    // the exact order of the expected events, we need to match them by their context and view id
+    expected.forEach { expectedEvent ->
+        val matchingEvent = this.find { actualEvent ->
+            val viewId = actualEvent
+                .getAsJsonObject("view")
+                .getAsJsonPrimitive("id").asString
+            val applicationId = actualEvent
+                .getAsJsonObject("application")
+                .getAsJsonPrimitive("id").asString
+            val sessionId = actualEvent
+                .getAsJsonObject("session")
+                .getAsJsonPrimitive("id").asString
+            expectedEvent.rumContext.viewId == viewId &&
+                expectedEvent.rumContext.applicationId == applicationId &&
+                expectedEvent.rumContext.sessionId == sessionId
+        }
+        checkNotNull(matchingEvent) {
+            "No matching event found for $expectedEvent"
+        }
+        matchingEvent.verifyEventMatches(expectedEvent)
+    }
+}
+
 private fun JsonObject.verifyEventMatches(event: ExpectedApplicationLaunchViewEvent) {
     assertThat(this)
         .hasField("application") {
@@ -111,10 +146,15 @@ private fun JsonObject.verifyEventMatches(event: ExpectedViewEvent) {
         .hasField("view") {
             hasField("url", event.viewUrl)
         }
-        .hasField("_dd") {
-            hasField("document_version", event.docVersion)
-        }
-
+    this.getAsJsonObject("_dd").apply {
+        val documentVersion = getAsJsonPrimitive("document_version").asInt
+        Assertions.assertThat(documentVersion)
+            .withFailMessage(
+                "Expected document version for view with url: ${event.viewUrl} " +
+                    "to be greater than or equal to ${event.docVersion} but instead was $documentVersion"
+            )
+            .isGreaterThanOrEqualTo(event.docVersion)
+    }
     assertThat(this).containsAttributes(event.extraAttributes)
     val viewArguments = event.viewArguments
         .map { "$VIEW_ARGUMENTS_PREFIX${it.key}" to it.value }
@@ -167,14 +207,28 @@ private fun JsonObject.verifyEventMatches(event: ExpectedErrorEvent) {
 }
 
 private fun JsonObject.verifyRootMatches(event: ExpectedEvent) {
-    assertThat(this)
-        .hasField("application") {
-            hasField("id", event.rumContext.applicationId)
-        }
-        .hasField("session") {
-            hasField("id", event.rumContext.sessionId)
-        }
-        .hasField("view") {
-            hasField("id", event.rumContext.viewId)
-        }
+    val applicationId = getAsJsonObject("application")
+        .getAsJsonPrimitive("id").asString
+    val sessionId = getAsJsonObject("session")
+        .getAsJsonPrimitive("id").asString
+    val viewId = getAsJsonObject("view")
+        .getAsJsonPrimitive("id").asString
+    Assertions.assertThat(applicationId)
+        .withFailMessage(
+            "Expected event \n $this \n to have same application " +
+                "id as \n $event \n but instead was $applicationId"
+        )
+        .isEqualTo(event.rumContext.applicationId)
+    Assertions.assertThat(sessionId)
+        .withFailMessage(
+            "Expected event \n $this \n to have same session " +
+                "id as \n $event \n but instead was $sessionId"
+        )
+        .isEqualTo(event.rumContext.sessionId)
+    Assertions.assertThat(viewId)
+        .withFailMessage(
+            "Expected event \n $this \n to have same view " +
+                "id as \n $event \n but instead was $viewId"
+        )
+        .isEqualTo(event.rumContext.viewId)
 }