From ae4faf813d2e039aa29725da6893b317b96b81c2 Mon Sep 17 00:00:00 2001 From: Marius Constantin Date: Thu, 17 Oct 2024 16:09:34 +0200 Subject: [PATCH] RUM-6596 Make sure the legacy integration test fail the CI job if failing --- .gitlab-ci.yml | 5 +- ...nsentPendingGrantedFragmentTrackingTest.kt | 5 ++ .../integration/rum/FragmentTrackingTest.kt | 18 ++-- .../android/sdk/integration/rum/RumTest.kt | 53 +++++++++--- .../sdk/integration/rum/RumTestUtils.kt | 86 +++++++++++++++---- 5 files changed, 121 insertions(+), 46 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b722fee63c..0e34351bbe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -53,10 +53,7 @@ stages: - set +e - exit_code=0 - $ANDROID_HOME/emulator/emulator -avd "$EMULATOR_NAME" -grpc-use-jwt -no-snapstorage -no-audio -no-window -no-boot-anim -verbose -qemu -machine virt & - - GRADLE_OPTS="-Xmx3072m" ./gradlew :instrumented:integration:assembleDebug :instrumented:integration:assembleDebugAndroidTest --stacktrace --no-daemon $( (( $ANDROID_API <= 23 )) && echo "-Puse-api21-java-backport -Puse-desugaring" ) - - $ANDROID_HOME/platform-tools/adb install -t -d $( (( $ANDROID_API >= 23 )) && echo "-g" ) -r instrumented/integration/build/outputs/apk/androidTest/debug/integration-debug-androidTest.apk - - $ANDROID_HOME/platform-tools/adb install -t -d $( (( $ANDROID_API >= 23 )) && echo "-g" ) -r instrumented/integration/build/outputs/apk/debug/integration-debug.apk - - $ANDROID_HOME/platform-tools/adb shell am instrument -w com.datadog.android.sdk.integration.test/androidx.test.runner.AndroidJUnitRunner || exit_code=$? + - GRADLE_OPTS="-Xmx3072m" ./gradlew :instrumented:integration:connectedDebugAndroidTest --stacktrace --no-daemon $( (( $ANDROID_API <= 23 )) && echo "-Puse-api21-java-backport -Puse-desugaring" ) || exit_code=$? - $ANDROID_HOME/platform-tools/adb emu kill - if [[ "$exit_code" -ne 0 ]]; then exit 1; fi - exit 0 diff --git a/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/ConsentPendingGrantedFragmentTrackingTest.kt b/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/ConsentPendingGrantedFragmentTrackingTest.kt index a463c749ea..eedafbb10a 100644 --- a/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/ConsentPendingGrantedFragmentTrackingTest.kt +++ b/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/ConsentPendingGrantedFragmentTrackingTest.kt @@ -6,6 +6,7 @@ package com.datadog.android.sdk.integration.rum +import android.os.Build import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.filters.LargeTest import androidx.test.platform.app.InstrumentationRegistry @@ -31,6 +32,10 @@ internal class ConsentPendingGrantedFragmentTrackingTest : FragmentTrackingTest( @Test fun verifyViewEventsOnSwipe() { + if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) { + // We skip this test on Android 28 and below because Espresso doesn't work well with ViewPager + return + } val expectedEvents = runInstrumentationScenario(mockServerRule) // update the tracking consent diff --git a/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/FragmentTrackingTest.kt b/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/FragmentTrackingTest.kt index f59645a96f..81ce239a06 100644 --- a/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/FragmentTrackingTest.kt +++ b/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/FragmentTrackingTest.kt @@ -45,7 +45,6 @@ internal abstract class FragmentTrackingTest : val fragmentAViewUrl = currentFragmentViewUrl(activity) // ignore view event for view start, it will be reduced - // view stopped expectedEvents.add( ExpectedViewEvent( fragmentAViewUrl, @@ -55,13 +54,15 @@ internal abstract class FragmentTrackingTest : ) // swipe to change the fragment + instrumentation.waitForIdleSync() onView(ViewMatchers.withId(R.id.btn_next)).perform(ViewActions.click()) instrumentation.waitForIdleSync() - Thread.sleep(200) // give time to the view id to update + Thread.sleep(200) val fragmentBViewUrl = currentFragmentViewUrl(activity) mockServerRule.activity.supportFragmentManager.fragments // ignore view event for updating the time, it will be reduced // view stopped + waitForPendingRUMEvents() expectedEvents.add( ExpectedViewEvent( fragmentBViewUrl, @@ -73,8 +74,8 @@ internal abstract class FragmentTrackingTest : // swipe to close the view onView(ViewMatchers.withId(R.id.btn_last)).perform(ViewActions.click()) instrumentation.waitForIdleSync() - Thread.sleep(200) // give time to the view id to update - + Thread.sleep(200) + waitForPendingRUMEvents() // for updating the time expectedEvents.add( ExpectedViewEvent( @@ -84,15 +85,6 @@ internal abstract class FragmentTrackingTest : ) ) - // view stopped -// expectedEvents.add( -// ExpectedViewEvent( -// fragmentAViewUrl, -// 3, -// currentFragmentExtras(activity) -// ) -// ) - instrumentation.runOnMainSync { instrumentation.callActivityOnStop(mockServerRule.activity) } diff --git a/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/RumTest.kt b/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/RumTest.kt index 6f66776556..0de79910fa 100644 --- a/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/RumTest.kt +++ b/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/RumTest.kt @@ -17,6 +17,7 @@ import com.datadog.android.sdk.utils.isRumUrl import com.google.gson.JsonObject import org.assertj.core.api.Assertions.assertThat import java.lang.Long.max +import java.util.LinkedList import java.util.concurrent.TimeUnit internal abstract class RumTest> { @@ -28,8 +29,10 @@ internal abstract class RumTest> handledRequests: List, expectedEvents: List ) { - val sentGestureEvents = mutableListOf() - val sentLaunchEvents = mutableListOf() + val sentViewEvents = LinkedList() + val sentActionEvents = LinkedList() + val sentResourceEvents = LinkedList() + val sentLaunchEvents = LinkedList() handledRequests .filter { it.url?.isRumUrl() ?: false } .forEach { request -> @@ -44,25 +47,39 @@ internal abstract class RumTest> .forEach { if (it.isEventRelatedToApplicationLaunch) { sentLaunchEvents += it - } else { - sentGestureEvents += it + } else if (it.isViewEvent) { + sentViewEvents += it + } else if (it.isActionEvent) { + sentActionEvents += it + } else if (it.isResourceEvent) { + sentResourceEvents += it } } } } - // Because launch events can be weirdly order dependent, consider them separately val launchEventPredicate = { event: ExpectedEvent -> event is ExpectedApplicationLaunchViewEvent || event is ExpectedApplicationStartActionEvent } val expectedLaunchEvents = expectedEvents.filter(launchEventPredicate) - sentLaunchEvents - .reduceViewEvents() - .verifyEventMatches(expectedLaunchEvents) - - val otherExpectedEvents = expectedEvents.filterNot(launchEventPredicate) - sentGestureEvents - .reduceViewEvents() - .verifyEventMatches(otherExpectedEvents) + val expectedViewEvents = expectedEvents.filterIsInstance() + val expectedActionEvents = expectedEvents.filterIsInstance() + val expectedResourceEvents = expectedEvents.filterIsInstance() + if (expectedLaunchEvents.isNotEmpty()) { + sentLaunchEvents + .reduceViewEvents() + .verifyEventMatches(expectedLaunchEvents) + } + if (expectedViewEvents.isNotEmpty()) { + sentViewEvents + .reduceViewEvents() + .verifyViewEventsMatches(expectedViewEvents) + } + if (expectedActionEvents.isNotEmpty()) { + sentActionEvents.verifyEventMatches(expectedActionEvents) + } + if (expectedResourceEvents.isNotEmpty()) { + sentResourceEvents.verifyEventMatches(expectedResourceEvents) + } } protected fun verifyNoRumPayloadSent( @@ -88,6 +105,12 @@ internal abstract class RumTest> private val JsonObject.isViewEvent get() = get("type")?.asString == "view" + private val JsonObject.isActionEvent + get() = get("type")?.asString == "action" + + private val JsonObject.isResourceEvent + get() = get("type")?.asString == "resource" + private val JsonObject.isTelemetryEvent get() = get("type")?.asString == "telemetry" @@ -119,6 +142,10 @@ internal abstract class RumTest> } } + private fun List.dropActionEvents(): List { + return filterNot { it.isActionEvent } + } + companion object { internal val FINAL_WAIT_MS = TimeUnit.SECONDS.toMillis(60) } diff --git a/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/RumTestUtils.kt b/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/RumTestUtils.kt index ce6858b215..ef215c082b 100644 --- a/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/RumTestUtils.kt +++ b/instrumented/integration/src/androidTest/kotlin/com/datadog/android/sdk/integration/rum/RumTestUtils.kt @@ -36,11 +36,9 @@ internal fun List.verifyEventMatches( this.joinToString("\n") { "\t>> $it" } ) .isEqualTo(expected.size) - this.forEachIndexed { index, event -> when (val expectedEvent = expected[index]) { is ExpectedApplicationLaunchViewEvent -> event.verifyEventMatches(expectedEvent) - is ExpectedViewEvent -> event.verifyEventMatches(expectedEvent) is ExpectedGestureEvent -> event.verifyEventMatches(expectedEvent) is ExpectedApplicationStartActionEvent -> event.verifyEventMatches(expectedEvent) is ExpectedResourceEvent -> event.verifyEventMatches(expectedEvent) @@ -52,6 +50,43 @@ internal fun List.verifyEventMatches( } } +internal fun List.verifyViewEventsMatches( + expected: List +) { + Assertions.assertThat(this.size) + .withFailMessage( + "We were expecting ${expected.size} rum " + + "view events instead they were ${this.size}: \n" + + " -- EXPECTED -- \n" + + expected.joinToString("\n") { "\t>> $it" } + + "\n -- ACTUAL -- \n" + + this.joinToString("\n") { "\t>> $it" } + ) + .isEqualTo(expected.size) + // in case of views because they are reduced by their document version and they are not going to follow + // the exact order of the expected events, we need to match them by their context and view id + expected.forEach { expectedEvent -> + val matchingEvent = this.find { actualEvent -> + val viewId = actualEvent + .getAsJsonObject("view") + .getAsJsonPrimitive("id").asString + val applicationId = actualEvent + .getAsJsonObject("application") + .getAsJsonPrimitive("id").asString + val sessionId = actualEvent + .getAsJsonObject("session") + .getAsJsonPrimitive("id").asString + expectedEvent.rumContext.viewId == viewId && + expectedEvent.rumContext.applicationId == applicationId && + expectedEvent.rumContext.sessionId == sessionId + } + checkNotNull(matchingEvent) { + "No matching event found for $expectedEvent" + } + matchingEvent.verifyEventMatches(expectedEvent) + } +} + private fun JsonObject.verifyEventMatches(event: ExpectedApplicationLaunchViewEvent) { assertThat(this) .hasField("application") { @@ -111,10 +146,15 @@ private fun JsonObject.verifyEventMatches(event: ExpectedViewEvent) { .hasField("view") { hasField("url", event.viewUrl) } - .hasField("_dd") { - hasField("document_version", event.docVersion) - } - + this.getAsJsonObject("_dd").apply { + val documentVersion = getAsJsonPrimitive("document_version").asInt + Assertions.assertThat(documentVersion) + .withFailMessage( + "Expected document version for view with url: ${event.viewUrl} " + + "to be greater than or equal to ${event.docVersion} but instead was $documentVersion" + ) + .isGreaterThanOrEqualTo(event.docVersion) + } assertThat(this).containsAttributes(event.extraAttributes) val viewArguments = event.viewArguments .map { "$VIEW_ARGUMENTS_PREFIX${it.key}" to it.value } @@ -167,14 +207,28 @@ private fun JsonObject.verifyEventMatches(event: ExpectedErrorEvent) { } private fun JsonObject.verifyRootMatches(event: ExpectedEvent) { - assertThat(this) - .hasField("application") { - hasField("id", event.rumContext.applicationId) - } - .hasField("session") { - hasField("id", event.rumContext.sessionId) - } - .hasField("view") { - hasField("id", event.rumContext.viewId) - } + val applicationId = getAsJsonObject("application") + .getAsJsonPrimitive("id").asString + val sessionId = getAsJsonObject("session") + .getAsJsonPrimitive("id").asString + val viewId = getAsJsonObject("view") + .getAsJsonPrimitive("id").asString + Assertions.assertThat(applicationId) + .withFailMessage( + "Expected event \n $this \n to have same application " + + "id as \n $event \n but instead was $applicationId" + ) + .isEqualTo(event.rumContext.applicationId) + Assertions.assertThat(sessionId) + .withFailMessage( + "Expected event \n $this \n to have same session " + + "id as \n $event \n but instead was $sessionId" + ) + .isEqualTo(event.rumContext.sessionId) + Assertions.assertThat(viewId) + .withFailMessage( + "Expected event \n $this \n to have same view " + + "id as \n $event \n but instead was $viewId" + ) + .isEqualTo(event.rumContext.viewId) }