Skip to content

Commit

Permalink
RUM-6197: Add ImageSemanticsNodeMapper to support image role for SR
Browse files Browse the repository at this point in the history
  • Loading branch information
ambushwork committed Oct 14, 2024
1 parent a0a5cf8 commit 830ef53
Show file tree
Hide file tree
Showing 17 changed files with 605 additions and 53 deletions.
1 change: 1 addition & 0 deletions detekt_custom.yml
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ datadog:
- "android.database.sqlite.SQLiteDatabase.endTransaction():java.lang.IllegalStateException"
- "android.database.sqlite.SQLiteDatabase.setTransactionSuccessful():java.lang.IllegalStateException"
- "android.graphics.Bitmap.compress(android.graphics.Bitmap.CompressFormat, kotlin.Int, java.io.OutputStream):java.lang.NullPointerException,java.lang.IllegalArgumentException"
- "android.graphics.Bitmap.copy(android.graphics.Bitmap.Config, kotlin.Boolean):java.lang.IllegalArgumentException"
- "android.graphics.Bitmap.createBitmap(android.util.DisplayMetrics?, kotlin.Int, kotlin.Int, android.graphics.Bitmap.Config):java.lang.IllegalArgumentException"
- "android.graphics.Bitmap.createScaledBitmap(android.graphics.Bitmap, kotlin.Int, kotlin.Int, kotlin.Boolean):java.lang.IllegalArgumentException"
- "android.graphics.Canvas.constructor(android.graphics.Bitmap):java.lang.IllegalStateException"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,14 @@ package com.datadog.android.sessionreplay.compose.internal.data

import androidx.compose.ui.unit.Density
import com.datadog.android.sessionreplay.SessionReplayPrivacy
import com.datadog.android.sessionreplay.utils.ImageWireframeHelper

internal data class UiContext(
val parentContentColor: String?,
val density: Float,
val privacy: SessionReplayPrivacy,
val isInUserInputLayout: Boolean = false
val isInUserInputLayout: Boolean = false,
val imageWireframeHelper: ImageWireframeHelper
) {
val composeDensity: Density
get() = Density(density)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import com.datadog.android.sessionreplay.recorder.mapper.BaseWireframeMapper
import com.datadog.android.sessionreplay.utils.AsyncJobStatusCallback
import com.datadog.android.sessionreplay.utils.ColorStringFormatter
import com.datadog.android.sessionreplay.utils.DrawableToColorMapper
import com.datadog.android.sessionreplay.utils.ImageWireframeHelper
import com.datadog.android.sessionreplay.utils.ViewBoundsResolver
import com.datadog.android.sessionreplay.utils.ViewIdentifierResolver

Expand Down Expand Up @@ -71,6 +72,7 @@ internal class ComposeWireframeMapper(
composer = composer,
density = density,
privacy = privacy,
imageWireframeHelper = mappingContext.imageWireframeHelper,
internalLogger = internalLogger
)
wireframes
Expand Down Expand Up @@ -102,6 +104,7 @@ internal class ComposeWireframeMapper(
composer: Composer,
density: Float,
privacy: SessionReplayPrivacy,
imageWireframeHelper: ImageWireframeHelper,
internalLogger: InternalLogger
): List<MobileSegment.Wireframe> {
val wireframes = mutableListOf<MobileSegment.Wireframe>()
Expand All @@ -113,7 +116,8 @@ internal class ComposeWireframeMapper(
parentUiContext = UiContext(
parentContentColor = null,
density = density,
privacy = privacy
privacy = privacy,
imageWireframeHelper = imageWireframeHelper
),
internalLogger = internalLogger
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import com.datadog.android.sessionreplay.compose.internal.data.ComposeWireframe
import com.datadog.android.sessionreplay.compose.internal.data.UiContext
import com.datadog.android.sessionreplay.compose.internal.utils.SemanticsUtils
import com.datadog.android.sessionreplay.model.MobileSegment
import com.datadog.android.sessionreplay.utils.AsyncJobStatusCallback
import com.datadog.android.sessionreplay.utils.ColorStringFormatter
import com.datadog.android.sessionreplay.utils.GlobalBounds

Expand All @@ -20,7 +21,11 @@ internal class ButtonSemanticsNodeMapper(
private val semanticsUtils: SemanticsUtils = SemanticsUtils()
) : AbstractSemanticsNodeMapper(colorStringFormatter) {

override fun map(semanticsNode: SemanticsNode, parentContext: UiContext): ComposeWireframe {
override fun map(
semanticsNode: SemanticsNode,
parentContext: UiContext,
asyncJobStatusCallback: AsyncJobStatusCallback
): ComposeWireframe {
val density = semanticsNode.layoutInfo.density
val bounds = resolveBound(semanticsNode)
val buttonStyle = resolveSemanticsButtonStyle(semanticsNode, bounds, density)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
/*
* Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
* This product includes software developed at Datadog (https://www.datadoghq.com/).
* Copyright 2016-Present Datadog, Inc.
*/

package com.datadog.android.sessionreplay.compose.internal.mappers.semantics

import android.content.Context
import android.graphics.Bitmap
import android.view.ViewGroup
import androidx.compose.ui.graphics.painter.BitmapPainter
import androidx.compose.ui.graphics.painter.Painter
import androidx.compose.ui.graphics.vector.VectorPainter
import androidx.compose.ui.semantics.SemanticsNode
import com.datadog.android.sessionreplay.ImagePrivacy
import com.datadog.android.sessionreplay.compose.internal.data.ComposeWireframe
import com.datadog.android.sessionreplay.compose.internal.data.UiContext
import com.datadog.android.sessionreplay.compose.internal.reflection.ComposeReflection
import com.datadog.android.sessionreplay.compose.internal.reflection.ComposeReflection.BitmapField
import com.datadog.android.sessionreplay.compose.internal.reflection.ComposeReflection.ContentPainterModifierClass
import com.datadog.android.sessionreplay.compose.internal.reflection.ComposeReflection.ImageField
import com.datadog.android.sessionreplay.compose.internal.reflection.ComposeReflection.LayoutNodeOwnerField
import com.datadog.android.sessionreplay.compose.internal.reflection.ComposeReflection.PainterElementClass
import com.datadog.android.sessionreplay.compose.internal.reflection.ComposeReflection.PainterField
import com.datadog.android.sessionreplay.compose.internal.reflection.ComposeReflection.PainterFieldOfAsyncImagePainter
import com.datadog.android.sessionreplay.compose.internal.reflection.ComposeReflection.PainterFieldOfContentPainter
import com.datadog.android.sessionreplay.compose.internal.reflection.getSafe
import com.datadog.android.sessionreplay.utils.AsyncJobStatusCallback
import com.datadog.android.sessionreplay.utils.ColorStringFormatter

internal class ImageSemanticsNodeMapper(
colorStringFormatter: ColorStringFormatter
) : AbstractSemanticsNodeMapper(colorStringFormatter) {

private var applicationContext: Context? = null

override fun map(
semanticsNode: SemanticsNode,
parentContext: UiContext,
asyncJobStatusCallback: AsyncJobStatusCallback
): ComposeWireframe? {
val bounds = resolveBound(semanticsNode)
val bitmapInfo = resolveSemanticsPainter(semanticsNode)
if (applicationContext == null) {
applicationContext = resolveApplicationContext(semanticsNode)
}
val imageWireframe = applicationContext?.let { context ->
if (bitmapInfo != null) {
parentContext.imageWireframeHelper.createImageWireframe(
id = semanticsNode.id.toLong(),
globalBounds = bounds,
applicationContext = context,
bitmap = bitmapInfo.bitmap,
density = parentContext.density,
isContextualImage = bitmapInfo.isContextualImage,
imagePrivacy = ImagePrivacy.MASK_NONE,
asyncJobStatusCallback = asyncJobStatusCallback,
clipping = null,
shapeStyle = null,
border = null
)
} else {
null
}
}
return imageWireframe?.let {
ComposeWireframe(
imageWireframe,
null
)
}
}

private fun resolveSemanticsPainter(
semanticsNode: SemanticsNode
): BitmapInfo? {
var isContextualImage = false
var painter = tryParseLocalImagePainter(semanticsNode)
if (painter == null) {
painter = tryParseAsyncImagePainter(semanticsNode)
if (painter != null) {
isContextualImage = true
}
}
// TODO RUM-6535: support more painters.
val bitmap = when (painter) {
is BitmapPainter -> tryParseBitmapPainterToBitmap(painter)
is VectorPainter -> tryParseVectorPainterToBitmap(painter)
else -> {
null
}
}

val newBitmap = bitmap?.let {
@Suppress("UnsafeThirdPartyFunctionCall") // isMutable is always false
it.copy(it.config, false)
}
return newBitmap?.let {
BitmapInfo(it, isContextualImage)
}
}

private fun resolveApplicationContext(semanticsNode: SemanticsNode): Context? {
val owner = LayoutNodeOwnerField?.getSafe(semanticsNode.layoutInfo) as? ViewGroup
return owner?.context
}

private fun tryParseVectorPainterToBitmap(vectorPainter: VectorPainter): Bitmap? {
val vector = ComposeReflection.VectorField?.getSafe(vectorPainter)
val cacheDrawScope = ComposeReflection.CacheDrawScopeField?.getSafe(vector)
val mCachedImage = ComposeReflection.CachedImageField?.getSafe(cacheDrawScope)
return BitmapField?.getSafe(mCachedImage) as? Bitmap
}

private fun tryParseBitmapPainterToBitmap(bitmapPainter: BitmapPainter): Bitmap? {
val image = ImageField?.getSafe(bitmapPainter)
return BitmapField?.getSafe(image) as? Bitmap
}

private fun tryParseLocalImagePainter(semanticsNode: SemanticsNode): Painter? {
val modifier = semanticsNode.layoutInfo.getModifierInfo().firstOrNull {
PainterElementClass?.isInstance(it.modifier) == true
}?.modifier
return PainterField?.getSafe(modifier) as? Painter
}

private fun tryParseAsyncImagePainter(semanticsNode: SemanticsNode): Painter? {
val modifier = semanticsNode.layoutInfo.getModifierInfo().firstOrNull {
ContentPainterModifierClass?.isInstance(it.modifier) == true
}?.modifier
val asyncPainter = PainterFieldOfContentPainter?.getSafe(modifier)
return PainterFieldOfAsyncImagePainter?.getSafe(asyncPainter) as? Painter
}

private data class BitmapInfo(
val bitmap: Bitmap,
val isContextualImage: Boolean
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@ package com.datadog.android.sessionreplay.compose.internal.mappers.semantics
import androidx.compose.ui.semantics.SemanticsNode
import com.datadog.android.sessionreplay.compose.internal.data.ComposeWireframe
import com.datadog.android.sessionreplay.compose.internal.data.UiContext
import com.datadog.android.sessionreplay.utils.AsyncJobStatusCallback

internal interface SemanticsNodeMapper {

fun map(
semanticsNode: SemanticsNode,
parentContext: UiContext
parentContext: UiContext,
asyncJobStatusCallback: AsyncJobStatusCallback
): ComposeWireframe?
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ internal class SemanticsWireframeMapper(
private val semanticsUtils: SemanticsUtils = SemanticsUtils(),
private val semanticsNodeMapper: Map<Role, SemanticsNodeMapper> = mapOf(
// TODO RUM-6189 Add Mappers for each Semantics Role
Role.Button to ButtonSemanticsNodeMapper(colorStringFormatter)
Role.Button to ButtonSemanticsNodeMapper(colorStringFormatter),
Role.Image to ImageSemanticsNodeMapper(colorStringFormatter)
),
// Text doesn't have a role in semantics, so it should be a fallback mapper.
private val textSemanticsNodeMapper: TextSemanticsNodeMapper = TextSemanticsNodeMapper(colorStringFormatter)
Expand All @@ -51,7 +52,7 @@ internal class SemanticsWireframeMapper(
val density = mappingContext.systemInformation.screenDensity.let { if (it == 0.0f) 1.0f else it }
val privacy = mappingContext.privacy
return semanticsUtils.findRootSemanticsNode(view)?.let { node ->
createComposeWireframes(node, density, privacy)
createComposeWireframes(node, density, mappingContext, privacy, asyncJobStatusCallback)
} ?: emptyList()
}

Expand All @@ -65,7 +66,9 @@ internal class SemanticsWireframeMapper(
private fun createComposeWireframes(
semanticsNode: SemanticsNode,
density: Float,
privacy: SessionReplayPrivacy
mappingContext: MappingContext,
privacy: SessionReplayPrivacy,
asyncJobStatusCallback: AsyncJobStatusCallback
): List<MobileSegment.Wireframe> {
val wireframes = mutableListOf<MobileSegment.Wireframe>()
createComposerWireframes(
Expand All @@ -74,24 +77,30 @@ internal class SemanticsWireframeMapper(
parentUiContext = UiContext(
parentContentColor = null,
density = density,
privacy = privacy
)
privacy = privacy,
imageWireframeHelper = mappingContext.imageWireframeHelper
),
asyncJobStatusCallback = asyncJobStatusCallback
)
return wireframes
}

private fun createComposerWireframes(
semanticsNode: SemanticsNode,
wireframes: MutableList<MobileSegment.Wireframe>,
parentUiContext: UiContext
parentUiContext: UiContext,
asyncJobStatusCallback: AsyncJobStatusCallback
) {
getSemanticsNodeMapper(semanticsNode)
.map(semanticsNode, parentUiContext)?.wireframe?.let {
wireframes.add(it)
}
getSemanticsNodeMapper(semanticsNode).map(
semanticsNode = semanticsNode,
parentContext = parentUiContext,
asyncJobStatusCallback = asyncJobStatusCallback
)?.wireframe?.let {
wireframes.add(it)
}
val children = semanticsNode.children
children.forEach {
createComposerWireframes(it, wireframes, parentUiContext)
createComposerWireframes(it, wireframes, parentUiContext, asyncJobStatusCallback)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,16 @@ import com.datadog.android.sessionreplay.compose.internal.data.UiContext
import com.datadog.android.sessionreplay.compose.internal.reflection.ComposeReflection
import com.datadog.android.sessionreplay.compose.internal.reflection.getSafe
import com.datadog.android.sessionreplay.model.MobileSegment
import com.datadog.android.sessionreplay.utils.AsyncJobStatusCallback
import com.datadog.android.sessionreplay.utils.ColorStringFormatter

internal class TextSemanticsNodeMapper(colorStringFormatter: ColorStringFormatter) :
AbstractSemanticsNodeMapper(colorStringFormatter) {
override fun map(semanticsNode: SemanticsNode, parentContext: UiContext): ComposeWireframe {
override fun map(
semanticsNode: SemanticsNode,
parentContext: UiContext,
asyncJobStatusCallback: AsyncJobStatusCallback
): ComposeWireframe {
val text = resolveText(semanticsNode.config)
val textStyle = resolveTextStyle(semanticsNode, parentContext) ?: defaultTextStyle
val bounds = resolveBound(semanticsNode)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ internal object ComposeReflection {

val OwnerField = WrappedCompositionClass?.getDeclaredFieldSafe("owner")

val LayoutNodeClass = getClassSafe("androidx.compose.ui.node.LayoutNode")
val LayoutNodeOwnerField = LayoutNodeClass?.getDeclaredFieldSafe("owner")
val AndroidComposeViewClass = getClassSafe("androidx.compose.ui.platform.AndroidComposeView")
val SemanticsOwner = AndroidComposeViewClass?.getDeclaredFieldSafe("semanticsOwner")

Expand All @@ -44,6 +46,30 @@ internal object ComposeReflection {
val BackgroundElementClass = getClassSafe("androidx.compose.foundation.BackgroundElement")
val ColorField = BackgroundElementClass?.getDeclaredFieldSafe("color")
val ShapeField = BackgroundElementClass?.getDeclaredFieldSafe("shape")

val PainterElementClass = getClassSafe("androidx.compose.ui.draw.PainterElement")
val PainterField = PainterElementClass?.getDeclaredFieldSafe("painter")

val VectorPainterClass = getClassSafe("androidx.compose.ui.graphics.vector.VectorPainter")
val VectorField = VectorPainterClass?.getDeclaredFieldSafe("vector")

val BitmapPainterClass = getClassSafe("androidx.compose.ui.graphics.painter.BitmapPainter")
val ImageField = BitmapPainterClass?.getDeclaredFieldSafe("image")

val VectorComponent = getClassSafe("androidx.compose.ui.graphics.vector.VectorComponent")
val CacheDrawScopeField = VectorComponent?.getDeclaredFieldSafe("cacheDrawScope")

val DrawCacheClass = getClassSafe("androidx.compose.ui.graphics.vector.DrawCache")
val CachedImageField = DrawCacheClass?.getDeclaredFieldSafe("mCachedImage")

val AndroidImageBitmapClass = getClassSafe("androidx.compose.ui.graphics.AndroidImageBitmap")
val BitmapField = AndroidImageBitmapClass?.getDeclaredFieldSafe("bitmap")

val ContentPainterModifierClass = getClassSafe("coil.compose.ContentPainterModifier")
val PainterFieldOfContentPainter = ContentPainterModifierClass?.getDeclaredFieldSafe("painter")

val AsyncImagePainterClass = getClassSafe("coil.compose.AsyncImagePainter")
val PainterFieldOfAsyncImagePainter = AsyncImagePainterClass?.getDeclaredFieldSafe("_painter")
}

internal fun Field.accessible(): Field {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import androidx.compose.ui.unit.Density
import com.datadog.android.sessionreplay.compose.internal.data.ComposeWireframe
import com.datadog.android.sessionreplay.compose.internal.data.UiContext
import com.datadog.android.sessionreplay.compose.test.elmyr.SessionReplayComposeForgeConfigurator
import com.datadog.android.sessionreplay.utils.AsyncJobStatusCallback
import com.datadog.android.sessionreplay.utils.ColorStringFormatter
import com.datadog.android.sessionreplay.utils.GlobalBounds
import fr.xgouchet.elmyr.Forge
Expand Down Expand Up @@ -131,7 +132,11 @@ internal class StubAbstractSemanticsNodeMapper(

var mappedWireframe: ComposeWireframe? = null

override fun map(semanticsNode: SemanticsNode, parentContext: UiContext): ComposeWireframe? {
override fun map(
semanticsNode: SemanticsNode,
parentContext: UiContext,
asyncJobStatusCallback: AsyncJobStatusCallback
): ComposeWireframe? {
return null
}

Expand Down
1 change: 1 addition & 0 deletions features/dd-sdk-android-session-replay/api/apiSurface
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ interface com.datadog.android.sessionreplay.utils.DrawableToColorMapper
data class com.datadog.android.sessionreplay.utils.GlobalBounds
constructor(Long, Long, Long, Long)
interface com.datadog.android.sessionreplay.utils.ImageWireframeHelper
fun createImageWireframe(Long, GlobalBounds, android.content.Context, android.graphics.Bitmap, Float, Boolean, com.datadog.android.sessionreplay.ImagePrivacy, AsyncJobStatusCallback, com.datadog.android.sessionreplay.model.MobileSegment.WireframeClip? = null, com.datadog.android.sessionreplay.model.MobileSegment.ShapeStyle? = null, com.datadog.android.sessionreplay.model.MobileSegment.ShapeBorder? = null): com.datadog.android.sessionreplay.model.MobileSegment.Wireframe?
fun createImageWireframe(android.view.View, com.datadog.android.sessionreplay.ImagePrivacy, Int, Long, Long, Int, Int, Boolean, android.graphics.drawable.Drawable, AsyncJobStatusCallback, com.datadog.android.sessionreplay.model.MobileSegment.WireframeClip? = null, com.datadog.android.sessionreplay.model.MobileSegment.ShapeStyle? = null, com.datadog.android.sessionreplay.model.MobileSegment.ShapeBorder? = null, String? = DRAWABLE_CHILD_NAME): com.datadog.android.sessionreplay.model.MobileSegment.Wireframe?
fun createCompoundDrawableWireframes(android.widget.TextView, com.datadog.android.sessionreplay.recorder.MappingContext, Int, AsyncJobStatusCallback): MutableList<com.datadog.android.sessionreplay.model.MobileSegment.Wireframe>
companion object
Expand Down
Loading

0 comments on commit 830ef53

Please sign in to comment.