Merge branch 'develop' into yl/it/merge-develop

.gitlab-ci.yml

@@ -4,7 +4,7 @@ include:
 # SETUP
 
 variables:
-  CURRENT_CI_IMAGE: "12"
+  CURRENT_CI_IMAGE: "13"
   CI_IMAGE_DOCKER: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/dd-sdk-android:$CURRENT_CI_IMAGE
   GIT_DEPTH: 5
 
@@ -53,10 +53,7 @@ stages:
     - set +e
     - exit_code=0
     - $ANDROID_HOME/emulator/emulator -avd "$EMULATOR_NAME" -grpc-use-jwt -no-snapstorage -no-audio -no-window -no-boot-anim -verbose -qemu -machine virt &
-    - GRADLE_OPTS="-Xmx3072m" ./gradlew :instrumented:integration:assembleDebug :instrumented:integration:assembleDebugAndroidTest --stacktrace --no-daemon $( (( $ANDROID_API <= 23 )) && echo "-Puse-api21-java-backport -Puse-desugaring" )
-    - $ANDROID_HOME/platform-tools/adb install -t -d $( (( $ANDROID_API >= 23 )) && echo "-g" ) -r instrumented/integration/build/outputs/apk/androidTest/debug/integration-debug-androidTest.apk
-    - $ANDROID_HOME/platform-tools/adb install -t -d $( (( $ANDROID_API >= 23 )) && echo "-g" ) -r instrumented/integration/build/outputs/apk/debug/integration-debug.apk
-    - $ANDROID_HOME/platform-tools/adb shell am instrument -w com.datadog.android.sdk.integration.test/androidx.test.runner.AndroidJUnitRunner || exit_code=$?
+    - GRADLE_OPTS="-Xmx3072m" ./gradlew :instrumented:integration:connectedDebugAndroidTest --stacktrace --no-daemon $( (( $ANDROID_API <= 23 )) && echo "-Puse-api21-java-backport -Puse-desugaring" ) || exit_code=$?
     - $ANDROID_HOME/platform-tools/adb emu kill
     - if [[ "$exit_code" -ne 0 ]]; then exit 1; fi
     - exit 0
@@ -370,6 +367,22 @@ test-pyramid:legacy-integration-instrumented-median-api:
     - !reference [.snippets, install-android-sdk]
     - !reference [.snippets, run-legacy-integration-instrumented]
 
+test-pyramid:detekt-api-coverage:
+  tags: [ "arch:amd64" ]
+  image: $CI_IMAGE_DOCKER
+  stage: test-pyramid
+  timeout: 1h
+  script:
+    - mkdir -p ./config/
+    - aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.gradle-properties --with-decryption --query "Parameter.Value" --out text >> ./gradle.properties
+    - GRADLE_OPTS="-Xmx4096M" ./gradlew assembleLibrariesDebug --stacktrace --no-daemon
+    - GRADLE_OPTS="-Xmx4096M" ./gradlew printSdkDebugRuntimeClasspath --stacktrace --no-daemon
+    - GRADLE_OPTS="-Xmx4096M" ./gradlew :tools:detekt:jar --stacktrace --no-daemon
+    - curl -sSLO https://github.com/detekt/detekt/releases/download/v1.23.4/detekt-cli-1.23.4-all.jar
+    - java -jar detekt-cli-1.23.4-all.jar --config detekt_test_pyramid.yml --plugins tools/detekt/build/libs/detekt.jar -ex "**/*.kts" --jvm-target 11 -cp $(cat sdk_classpath)
+    # For now we just print the uncovered apis, eventually we will fail if it's not empty
+    - grep -v -f apiUsage.log apiSurface.log
+
 test-pyramid:publish-e2e-synthetics:
   tags: [ "arch:amd64" ]
   image: $CI_IMAGE_DOCKER
@@ -424,7 +437,6 @@ test-pyramid:publish-webview-synthetics:
     paths:
       - sample/kotlin/build/outputs/apk/us1/release/kotlin-us1-release.apk
 
-
 test-pyramid:publish-staging-synthetics:
   tags: [ "arch:amd64" ]
   image: $CI_IMAGE_DOCKER

Dockerfile.gitlab

@@ -31,7 +31,7 @@ ENV ANDROID_BUILD_TOOLS 35.0.0
 ENV ANDROID_SDK_TOOLS 11076708
 ENV NDK_VERSION 25.1.8937393
 ENV CMAKE_VERSION 3.22.1
-ENV DD_TRACER_VERSION 1.39.0
+ENV DD_TRACER_VERSION 1.41.0
 # requires build with BuildKit to be available https://docs.docker.com/build/building/variables/#multi-platform-build-arguments
 ARG TARGETARCH
 ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-${TARGETARCH}

buildSrc/src/main/kotlin/com/datadog/gradle/config/TestPyramidConfig.kt

@@ -19,7 +19,6 @@ fun Project.registerSubModuleAggregationTask(
 ) {
     tasks.register(taskName) {
         project.subprojects.forEach { subProject ->
-            println("SubProject ${subProject.name} / ${subProject.path}")
             if (!exceptions.contains(subProject.name) &&
                 subProject.name.startsWith(subModuleNamePrefix) &&
                 subProject.path.startsWith(subModulePathPrefix)

dd-sdk-android-core/api/apiSurface

@@ -290,7 +290,6 @@ fun Long.toHexString(): String
 fun java.math.BigInteger.toHexString(): String
 fun Thread.State.asString(): String
 fun Array<StackTraceElement>.loggableStackTrace(): String
-fun Throwable.loggableStackTrace(): String
 enum com.datadog.android.core.metrics.MethodCallSamplingRate
   constructor(Float)
   - ALL

dd-sdk-android-core/api/dd-sdk-android-core.api

@@ -783,10 +783,6 @@ public final class com/datadog/android/core/internal/utils/ThreadExtKt {
 	public static final fun loggableStackTrace ([Ljava/lang/StackTraceElement;)Ljava/lang/String;
 }
 
-public final class com/datadog/android/core/internal/utils/ThrowableExtKt {
-	public static final fun loggableStackTrace (Ljava/lang/Throwable;)Ljava/lang/String;
-}
-
 public final class com/datadog/android/core/metrics/MethodCallSamplingRate : java/lang/Enum {
 	public static final field ALL Lcom/datadog/android/core/metrics/MethodCallSamplingRate;
 	public static final field HIGH Lcom/datadog/android/core/metrics/MethodCallSamplingRate;

dd-sdk-android-core/src/main/kotlin/com/datadog/android/Datadog.kt

@@ -20,8 +20,8 @@ import com.datadog.android.core.internal.HashGenerator
 import com.datadog.android.core.internal.NoOpInternalSdkCore
 import com.datadog.android.core.internal.SdkCoreRegistry
 import com.datadog.android.core.internal.Sha256HashGenerator
-import com.datadog.android.core.internal.utils.loggableStackTrace
 import com.datadog.android.core.internal.utils.unboundInternalLogger
+import com.datadog.android.internal.utils.loggableStackTrace
 import com.datadog.android.lint.InternalApi
 import com.datadog.android.privacy.TrackingConsent
 import java.util.Locale

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/SdkFeature.kt

@@ -351,7 +351,9 @@ internal class SdkFeature(
             fileMover = FileMover(internalLogger),
             internalLogger = internalLogger,
             filePersistenceConfig = filePersistenceConfig,
-            metricsDispatcher = metricsDispatcher
+            metricsDispatcher = metricsDispatcher,
+            coreFeature.trackingConsentProvider,
+            featureName
         )
     }
 

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/metrics/BatchMetricsDispatcher.kt

@@ -16,8 +16,6 @@ import com.datadog.android.core.internal.persistence.file.existsSafe
 import com.datadog.android.core.internal.persistence.file.lengthSafe
 import com.datadog.android.core.internal.time.TimeProvider
 import com.datadog.android.core.metrics.MethodCallSamplingRate
-import com.datadog.android.core.sampling.RateBasedSampler
-import com.datadog.android.core.sampling.Sampler
 import com.datadog.android.privacy.TrackingConsent
 import java.io.File
 import java.util.Locale
@@ -28,8 +26,7 @@ internal class BatchMetricsDispatcher(
     private val uploadConfiguration: DataUploadConfiguration?,
     private val filePersistenceConfig: FilePersistenceConfig,
     private val internalLogger: InternalLogger,
-    private val dateTimeProvider: TimeProvider,
-    private val sampler: Sampler = RateBasedSampler(METRICS_DISPATCHER_DEFAULT_SAMPLING_RATE)
+    private val dateTimeProvider: TimeProvider
 
 ) : MetricsDispatcher, ProcessLifecycleMonitor.Callback {
 
@@ -39,7 +36,7 @@ internal class BatchMetricsDispatcher(
     // region MetricsDispatcher
 
     override fun sendBatchDeletedMetric(batchFile: File, removalReason: RemovalReason) {
-        if (!removalReason.includeInMetrics() || trackName == null || !sampler.sample()) {
+        if (!removalReason.includeInMetrics() || trackName == null) {
             return
         }
         resolveBatchDeletedMetricAttributes(batchFile, removalReason)?.let {
@@ -52,7 +49,7 @@ internal class BatchMetricsDispatcher(
     }
 
     override fun sendBatchClosedMetric(batchFile: File, batchMetadata: BatchClosedMetadata) {
-        if (trackName == null || !sampler.sample() || !batchFile.existsSafe(internalLogger)) {
+        if (trackName == null || !batchFile.existsSafe(internalLogger)) {
             return
         }
         resolveBatchClosedMetricAttributes(batchFile, batchMetadata)?.let {
@@ -190,8 +187,6 @@ internal class BatchMetricsDispatcher(
         internal const val SR_TRACK_NAME = "sr"
         internal const val SR_RESOURCES_TRACK_NAME = "sr-resources"
 
-        private const val METRICS_DISPATCHER_DEFAULT_SAMPLING_RATE = 1.5f
-
         internal const val WRONG_FILE_NAME_MESSAGE_FORMAT =
             "Unable to parse the file name as a timestamp: %s"
 

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/persistence/AbstractStorage.kt

@@ -30,7 +30,7 @@ internal class AbstractStorage(
     private val executorService: ExecutorService,
     private val internalLogger: InternalLogger,
     internal val storageConfiguration: FeatureStorageConfiguration,
-    consentProvider: ConsentProvider
+    private val consentProvider: ConsentProvider
 ) : Storage, TrackingConsentProviderCallback {
 
     private val grantedPersistenceStrategy: PersistenceStrategy by lazy {
@@ -64,13 +64,8 @@ internal class AbstractStorage(
         forceNewBatch: Boolean,
         callback: (EventBatchWriter) -> Unit
     ) {
-        val strategy = when (datadogContext.trackingConsent) {
-            TrackingConsent.GRANTED -> grantedPersistenceStrategy
-            TrackingConsent.PENDING -> pendingPersistenceStrategy
-            TrackingConsent.NOT_GRANTED -> notGrantedPersistenceStrategy
-        }
-
         executorService.submitSafe("Data write", internalLogger) {
+            val strategy = resolvePersistenceStrategy()
             val writer = object : EventBatchWriter {
                 @WorkerThread
                 override fun currentMetadata(): ByteArray? {
@@ -86,6 +81,14 @@ internal class AbstractStorage(
         }
     }
 
+    @WorkerThread
+    private fun resolvePersistenceStrategy() =
+        when (consentProvider.getConsent()) {
+            TrackingConsent.GRANTED -> grantedPersistenceStrategy
+            TrackingConsent.PENDING -> pendingPersistenceStrategy
+            TrackingConsent.NOT_GRANTED -> notGrantedPersistenceStrategy
+        }
+
     @WorkerThread
     override fun readNextBatch(): BatchData? {
         return grantedPersistenceStrategy.lockAndReadNext()?.let {

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/persistence/ConsentAwareStorage.kt

@@ -19,6 +19,7 @@ import com.datadog.android.core.internal.persistence.file.FilePersistenceConfig
 import com.datadog.android.core.internal.persistence.file.FileReaderWriter
 import com.datadog.android.core.internal.persistence.file.batch.BatchFileReaderWriter
 import com.datadog.android.core.internal.persistence.file.existsSafe
+import com.datadog.android.core.internal.privacy.ConsentProvider
 import com.datadog.android.core.internal.utils.submitSafe
 import com.datadog.android.core.metrics.MethodCallSamplingRate
 import com.datadog.android.core.metrics.TelemetryMetricType
@@ -36,7 +37,9 @@ internal class ConsentAwareStorage(
     private val fileMover: FileMover,
     private val internalLogger: InternalLogger,
     internal val filePersistenceConfig: FilePersistenceConfig,
-    private val metricsDispatcher: MetricsDispatcher
+    private val metricsDispatcher: MetricsDispatcher,
+    private val consentProvider: ConsentProvider,
+    private val featureName: String
 ) : Storage {
 
     /**
@@ -53,28 +56,27 @@ internal class ConsentAwareStorage(
         forceNewBatch: Boolean,
         callback: (EventBatchWriter) -> Unit
     ) {
-        val orchestrator = when (datadogContext.trackingConsent) {
-            TrackingConsent.GRANTED -> grantedOrchestrator
-            TrackingConsent.PENDING -> pendingOrchestrator
-            TrackingConsent.NOT_GRANTED -> null
-        }
-
         val metric = internalLogger.startPerformanceMeasure(
             callerClass = ConsentAwareStorage::class.java.name,
             metric = TelemetryMetricType.MethodCalled,
             samplingRate = MethodCallSamplingRate.RARE.rate,
-            operationName = "writeCurrentBatch[${orchestrator?.getRootDirName()}]"
+            operationName = "writeCurrentBatch[$featureName]"
         )
-
         executorService.submitSafe("Data write", internalLogger) {
+            val orchestrator = resolveOrchestrator()
+            if (orchestrator == null) {
+                callback.invoke(NoOpEventBatchWriter())
+                metric?.stopAndSend(false)
+                return@submitSafe
+            }
             synchronized(writeLock) {
-                val batchFile = orchestrator?.getWritableFile(forceNewBatch)
+                val batchFile = orchestrator.getWritableFile(forceNewBatch)
                 val metadataFile = if (batchFile != null) {
                     orchestrator.getMetadataFile(batchFile)
                 } else {
                     null
                 }
-                val writer = if (orchestrator == null || batchFile == null) {
+                val writer = if (batchFile == null) {
                     NoOpEventBatchWriter()
                 } else {
                     FileEventBatchWriter(
@@ -153,6 +155,16 @@ internal class ConsentAwareStorage(
         }
     }
 
+    @WorkerThread
+    private fun resolveOrchestrator(): FileOrchestrator? {
+        val consent = consentProvider.getConsent()
+        return when (consent) {
+            TrackingConsent.GRANTED -> grantedOrchestrator
+            TrackingConsent.PENDING -> pendingOrchestrator
+            TrackingConsent.NOT_GRANTED -> null
+        }
+    }
+
     @WorkerThread
     private fun deleteBatch(batch: Batch, reason: RemovalReason) {
         deleteBatch(batch.file, batch.metaFile, reason)

dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/internal/persistence/file/advanced/ConsentAwareFileOrchestrator.kt

@@ -27,6 +27,7 @@ internal open class ConsentAwareFileOrchestrator(
     internal val internalLogger: InternalLogger
 ) : FileOrchestrator, TrackingConsentProviderCallback {
 
+    @Volatile
     private lateinit var delegateOrchestrator: FileOrchestrator
 
     init {

dd-sdk-android-core/src/main/kotlin/com/datadog/android/error/internal/DatadogExceptionHandler.kt

@@ -18,6 +18,7 @@ import com.datadog.android.core.internal.thread.waitToIdle
 import com.datadog.android.core.internal.utils.asString
 import com.datadog.android.core.internal.utils.loggableStackTrace
 import com.datadog.android.core.internal.utils.triggerUploadWorker
+import com.datadog.android.internal.utils.loggableStackTrace
 import java.lang.ref.WeakReference
 import java.util.concurrent.ThreadPoolExecutor
 

dd-sdk-android-core/src/test/kotlin/com/datadog/android/DatadogTest.kt

@@ -19,6 +19,7 @@ import com.datadog.android.core.internal.NoOpInternalSdkCore
 import com.datadog.android.core.internal.SdkCoreRegistry
 import com.datadog.android.core.internal.Sha256HashGenerator
 import com.datadog.android.core.internal.utils.loggableStackTrace
+import com.datadog.android.internal.utils.loggableStackTrace
 import com.datadog.android.privacy.TrackingConsent
 import com.datadog.android.utils.config.ApplicationContextTestConfiguration
 import com.datadog.android.utils.config.InternalLoggerTestConfiguration