Merge pull request #127 from ooni/feature/121

Evaluate measurement keys to get isFailed and isAnomaly. Improve result and measurement states.
ooni · Sep 25, 2024 · 7bba0fb · 7bba0fb
2 parents d2d9531 + 80baf32
commit 7bba0fb
Show file tree

Hide file tree

Showing 26 changed files with 551 additions and 171 deletions.
diff --git a/composeApp/src/commonMain/composeResources/drawable/ic_measurement_anomaly.xml b/composeApp/src/commonMain/composeResources/drawable/ic_measurement_anomaly.xml
@@ -0,0 +1,15 @@
+<vector
+    xmlns:android="http://schemas.android.com/apk/res/android"
+    android:name="vector"
+    android:width="24dp"
+    android:height="24dp"
+    android:viewportWidth="19"
+    android:viewportHeight="19">
+    <group android:name="group">
+        <path
+            android:name="path"
+            android:pathData="M 11.008 14.317 C 11.216 14.513 11.382 14.75 11.495 15.013 C 11.608 15.276 11.665 15.559 11.663 15.845 C 11.668 16.131 11.612 16.416 11.499 16.679 C 11.386 16.943 11.219 17.179 11.008 17.373 C 10.601 17.774 10.051 17.998 9.48 17.997 C 9.197 18 8.915 17.947 8.653 17.84 C 8.39 17.733 8.152 17.574 7.952 17.373 C 7.675 17.109 7.475 16.776 7.373 16.407 C 7.272 16.039 7.272 15.65 7.373 15.282 C 7.475 14.913 7.675 14.58 7.952 14.316 C 8.356 13.906 8.905 13.671 9.48 13.661 C 9.766 13.659 10.05 13.717 10.313 13.829 C 10.575 13.942 10.812 14.108 11.008 14.317 Z M 11.597 1.997 L 11.067 11.728 L 7.89 11.728 L 7.359 1.997 Z"
+            android:fillColor="#f08c00"
+            android:strokeWidth="1"/>
+    </group>
+</vector>
diff --git a/composeApp/src/commonMain/composeResources/drawable/ic_measurement_failed.xml b/composeApp/src/commonMain/composeResources/drawable/ic_measurement_failed.xml
@@ -0,0 +1,13 @@
+<vector
+    xmlns:android="http://schemas.android.com/apk/res/android"
+    android:name="vector"
+    android:width="24dp"
+    android:height="24dp"
+    android:viewportWidth="32"
+    android:viewportHeight="32">
+    <path
+        android:name="path"
+        android:pathData="M 16.291 3 C 12.691 3 10.073 4.418 8 6.818 L 11.345 9.4 C 12.691 7.836 14.218 7.109 15.782 7.109 C 17.527 7.109 18.582 7.945 18.582 9.436 C 18.582 12.636 12.691 12.164 12.691 18.418 L 12.691 19.691 L 17.454 19.691 L 17.454 18.6 C 17.454 14.345 24 14.854 24 8.964 C 24 5.836 21.418 3 16.291 3 Z M 15.127 23.109 C 13.382 23.109 12.036 24.527 12.036 26.2 C 12.036 27.909 13.382 29.327 15.127 29.327 C 16.873 29.327 18.255 27.909 18.255 26.2 C 18.255 24.527 16.873 23.109 15.127 23.109 Z"
+        android:fillColor="#adb5bd"
+        android:strokeWidth="1"/>
+</vector>
diff --git a/composeApp/src/commonMain/composeResources/drawable/ic_measurement_ok.xml b/composeApp/src/commonMain/composeResources/drawable/ic_measurement_ok.xml
@@ -0,0 +1,14 @@
+<vector xmlns:android="http://schemas.android.com/apk/res/android"
+    android:name="vector"
+    android:width="24dp"
+    android:height="24dp"
+    android:viewportWidth="19"
+    android:viewportHeight="19">
+    <group android:name="group">
+        <path
+            android:name="path"
+            android:fillColor="#2f9e44"
+            android:pathData="M 1.264 9.71 L 3.944 6.854 L 7.325 10.454 L 14.564 2.747 L 17.264 5.622 L 7.34 16.183 Z"
+            android:strokeWidth="1" />
+    </group>
+</vector>
diff --git a/composeApp/src/commonMain/composeResources/values/strings-common.xml b/composeApp/src/commonMain/composeResources/values/strings-common.xml
@@ -249,4 +249,7 @@
     <string name="task_origin_all">All Sources</string>
     <string name="vpn">VPN</string>
     <string name="disable_vpn_instructions">Go to Settings > General > VPN and disconnect from your VPN.</string>
+    <string name="measurement_failed">Failed</string>
+    <string name="measurement_ok">OK</string>
+    <string name="measurement_anomaly">Anomaly</string>
 </resources>
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/models/MeasurementResult.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/models/MeasurementResult.kt
@@ -25,8 +25,6 @@ data class MeasurementResult(
     val reportId: String? = null,
     @SerialName("input")
     val input: String? = null,
-    /*
-     Field `test_keys` is ignored because we're not planning on storing the measurement results
-     as structured data.
-     */
+    @SerialName("test_keys")
+    val testKeys: TestKeys? = null,
 )
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskEventResult.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TaskEventResult.kt
@@ -4,61 +4,45 @@ import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 
 @Serializable
-class TaskEventResult {
+data class TaskEventResult(
     @SerialName("key")
-    var key: String? = null
-
+    var key: String? = null,
     @SerialName("value")
-    var value: Value? = null
-
+    var value: Value? = null,
+) {
     @Serializable
-    class Value {
+    data class Value(
         @SerialName("key")
-        var key: Double = 0.0
-
+        var key: Double = 0.0,
         @SerialName("log_level")
-        var logLevel: String? = null
-
+        var logLevel: String? = null,
         @SerialName("message")
-        var message: String? = null
-
+        var message: String? = null,
         @SerialName("percentage")
-        var percentage: Double = 0.0
-
+        var percentage: Double = 0.0,
         @SerialName("json_str")
-        var jsonStr: String? = null
-
+        var jsonStr: String? = null,
         @SerialName("idx")
-        var idx: Int = 0
-
+        var idx: Int = 0,
         @SerialName("report_id")
-        var reportId: String? = null
-
+        var reportId: String? = null,
         @SerialName("probe_ip")
-        var probeIp: String? = null
-
+        var probeIp: String? = null,
         @SerialName("probe_asn")
-        var probeAsn: String? = null
-
+        var probeAsn: String? = null,
         @SerialName("probe_cc")
-        var probeCc: String? = null
-
+        var probeCc: String? = null,
         @SerialName("probe_network_name")
-        var probeNetworkName: String? = null
-
+        var probeNetworkName: String? = null,
         @SerialName("downloaded_kb")
-        var downloadedKb: Double = 0.0
-
+        var downloadedKb: Double = 0.0,
         @SerialName("uploaded_kb")
-        var uploadedKb: Double = 0.0
-
+        var uploadedKb: Double = 0.0,
         @SerialName("input")
-        var input: String? = null
-
+        var input: String? = null,
         @SerialName("failure")
-        var failure: String? = null
-
+        var failure: String? = null,
         @SerialName("orig_key")
-        var origKey: String? = null
-    }
+        var origKey: String? = null,
+    )
 }
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TestKeys.kt b/composeApp/src/commonMain/kotlin/org/ooni/engine/models/TestKeys.kt
@@ -0,0 +1,128 @@
+package org.ooni.engine.models
+
+import kotlinx.serialization.KSerializer
+import kotlinx.serialization.PolymorphicSerializer
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.SerializationException
+import kotlinx.serialization.encoding.Decoder
+import kotlinx.serialization.encoding.Encoder
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonDecoder
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.booleanOrNull
+import kotlinx.serialization.json.decodeFromJsonElement
+
+@Serializable
+data class TestKeys(
+    @SerialName("blocking") val blocking: String? = null,
+    @SerialName("accessible") val accessible: String? = null,
+    @SerialName("sent") val sent: List<String>? = null,
+    @SerialName("received") val received: List<String>? = null,
+    @SerialName("failure") val failure: String? = null,
+    @SerialName("whatsapp_endpoints_status") val whatsappEndpointsStatus: String? = null,
+    @SerialName("whatsapp_web_status") val whatsappWebStatus: String? = null,
+    @SerialName("registration_server_status") val registrationServerStatus: String? = null,
+    @SerialName("facebook_tcp_blocking") val facebookTcpBlocking: Boolean? = null,
+    @SerialName("facebook_dns_blocking") val facebookDnsBlocking: Boolean? = null,
+    @SerialName("telegram_http_blocking") val telegramHttpBlocking: Boolean? = null,
+    @SerialName("telegram_tcp_blocking") val telegramTcpBlocking: Boolean? = null,
+    @SerialName("telegram_web_status") val telegramWebStatus: String? = null,
+    @SerialName("signal_backend_status") val signalBackendStatus: String? = null,
+    @SerialName("signal_backend_failure") val signalBackendFailure: String? = null,
+    @SerialName("protocol") val protocol: Int? = null,
+    @SerialName("simple") val simple: Simple? = null,
+    @SerialName("summary") val summary: Summary? = null,
+    @SerialName("server") val server: Server? = null,
+    @SerialName("tampering") val tampering: Tampering? = null,
+    // Psiphon
+    @SerialName("bootstrap_time") val bootstrapTime: Double? = null,
+    // Tor
+    @SerialName("dir_port_total") val dirPortTotal: Long? = null,
+    @SerialName("dir_port_accessible") val dirPortAccessible: Long? = null,
+    @SerialName("obfs4_total") val obfs4Total: Long? = null,
+    @SerialName("obfs4_accessible") val obfs4Accessible: Long? = null,
+    @SerialName("or_port_dirauth_total") val orPortDirauthTotal: Long? = null,
+    @SerialName("or_port_dirauth_accessible") val orPortDirauthAccessible: Long? = null,
+    @SerialName("or_port_total") val orPortTotal: Long? = null,
+    @SerialName("or_port_accessible") val orPortAccessible: Long? = null,
+) {
+    @Serializable
+    data class Summary(
+        @SerialName("upload") val upload: Double? = null,
+        @SerialName("download") val download: Double? = null,
+        @SerialName("ping") val ping: Double? = null,
+        @SerialName("max_rtt") val maxRtt: Double? = null,
+        @SerialName("avg_rtt") val avgRtt: Double? = null,
+        @SerialName("min_rtt") val minRtt: Double? = null,
+        @SerialName("mss") val mss: Double? = null,
+        @SerialName("retransmit_rate") val retransmitRate: Double? = null,
+    )
+
+    @Serializable
+    data class Server(
+        @SerialName("hostname") val hostname: String? = null,
+        @SerialName("site") val site: String? = null,
+    )
+
+    // DASHSummary and NDT5Summary
+    @Serializable
+    class Simple(
+        @SerialName("median_bitrate") val medianBitrate: Double? = null,
+        @SerialName("min_playout_delay") val minPlayoutDelay: Double? = null,
+    )
+
+    @Serializable(with = TamperingSerializer::class)
+    data class Tampering(
+        val value: Boolean = false,
+    )
+
+    @Serializable
+    data class TamperingKeys(
+        @SerialName("header_field_name")
+        val headerFieldName: Boolean = false,
+        @SerialName("header_field_number")
+        val headerFieldNumber: Boolean = false,
+        @SerialName("header_field_value")
+        val headerFieldValue: Boolean = false,
+        @SerialName("header_name_capitalization")
+        val headerNameCapitalization: Boolean = false,
+        @SerialName("request_line_capitalization")
+        val requestLineCapitalization: Boolean = false,
+        @SerialName("total")
+        val total: Boolean = false,
+    ) {
+        val value
+            get() = headerFieldName || headerFieldNumber || headerFieldValue ||
+                headerNameCapitalization || requestLineCapitalization || total
+    }
+
+    companion object {
+        const val BLOCKED_VALUE = "BLOCKED"
+    }
+}
+
+object TamperingSerializer : KSerializer<TestKeys.Tampering> {
+    override val descriptor = PolymorphicSerializer(TestKeys.Tampering::class).descriptor
+
+    override fun deserialize(decoder: Decoder): TestKeys.Tampering =
+        when (val element = (decoder as JsonDecoder).decodeJsonElement()) {
+            is JsonPrimitive ->
+                TestKeys.Tampering(element.booleanOrNull == true)
+
+            is JsonObject -> {
+                val keys = Json.decodeFromJsonElement<TestKeys.TamperingKeys>(element)
+                TestKeys.Tampering(keys.value)
+            }
+
+            else -> throw SerializationException("Could not deserialize TestKeys.Tampering")
+        }
+
+    override fun serialize(
+        encoder: Encoder,
+        value: TestKeys.Tampering,
+    ) {
+        encoder.encodeBoolean(value.value)
+    }
+}
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/App.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/App.kt
@@ -85,6 +85,9 @@ fun App(
         dependencies.bootstrapTestDescriptors()
         dependencies.bootstrapPreferences()
     }
+    LaunchedEffect(Unit) {
+        dependencies.finishInProgressData()
+    }
     LaunchedEffect(Unit) {
         dependencies.observeAndConfigureAutoRun()
     }

diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/MeasurementKeysResult.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/MeasurementKeysResult.kt
@@ -0,0 +1,6 @@
+package org.ooni.probe.data.models
+
+data class MeasurementKeysResult(
+    val isFailed: Boolean = false,
+    val isAnomaly: Boolean = false,
+)
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/MeasurementModel.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/MeasurementModel.kt
@@ -42,7 +42,10 @@ data class MeasurementModel(
         get() = id?.let { "Measurement/${id.value}_${test.name}.json".toPath() }
 
     val isMissingUpload
-        get() = isDone && (!isUploaded || reportId == null)
+        get() = !isUploaded || reportId == null
+
+    val isDoneAndMissingUpload
+        get() = isDone && isMissingUpload
 
     companion object {
         fun logFilePath(

diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/ResultItem.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/data/models/ResultItem.kt
@@ -8,7 +8,8 @@ data class ResultItem(
     val network: NetworkModel?,
     val measurements: List<MeasurementWithUrl>,
 ) {
-    val anyMeasurementMissingUpload = measurements.any { it.measurement.isMissingUpload }
+    val anyMeasurementMissingUpload =
+        result.isDone && measurements.any { it.measurement.isDoneAndMissingUpload }
 
     val totalRuntime get() = measurements.sumOf { it.measurement.runtime ?: 0.0 }.seconds
 }
diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/data/repositories/MeasurementRepository.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/data/repositories/MeasurementRepository.kt
@@ -47,6 +47,13 @@ class MeasurementRepository(
             .mapToList(backgroundDispatcher)
             .map { list -> list.mapNotNull { it.toModel() } }
 
+    fun selectByResultRunId(descriptorId: InstalledTestDescriptorModel.Id) =
+        database.measurementQueries
+            .selectByResultRunId(descriptorId.value)
+            .asFlow()
+            .mapToList(backgroundDispatcher)
+            .map { list -> list.mapNotNull { it.toModel() } }
+
     suspend fun createOrUpdate(model: MeasurementModel): MeasurementModel.Id =
         withContext(backgroundDispatcher) {
             database.transactionWithResult {
@@ -82,13 +89,6 @@ class MeasurementRepository(
         }
     }
 
-    fun selectByResultRunId(descriptorId: InstalledTestDescriptorModel.Id) =
-        database.measurementQueries
-            .selectByResultRunId(descriptorId.value)
-            .asFlow()
-            .mapToList(backgroundDispatcher)
-            .map { list -> list.mapNotNull { it.toModel() } }
-
     private fun Measurement.toModel(): MeasurementModel? {
         return MeasurementModel(
             id = MeasurementModel.Id(id),

diff --git a/composeApp/src/commonMain/kotlin/org/ooni/probe/data/repositories/ResultRepository.kt b/composeApp/src/commonMain/kotlin/org/ooni/probe/data/repositories/ResultRepository.kt
@@ -5,6 +5,7 @@ import app.cash.sqldelight.coroutines.mapToList
 import app.cash.sqldelight.coroutines.mapToOneOrNull
 import kotlinx.coroutines.CoroutineDispatcher
 import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.first
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.withContext
 import org.ooni.engine.models.TaskOrigin
@@ -85,11 +86,30 @@ class ResultRepository(
             }
         }
 
+    suspend fun getByIdAndUpdate(
+        id: ResultModel.Id,
+        update: (ResultModel) -> ResultModel,
+    ) = withContext(backgroundDispatcher) {
+        getById(id).first()?.first?.let { result ->
+            createOrUpdate(update(result))
+        }
+    }
+
     suspend fun markAsViewed(resultId: ResultModel.Id) =
         withContext(backgroundDispatcher) {
             database.resultQueries.markAsViewed(resultId.value)
         }
 
+    suspend fun markAsDone(resultId: ResultModel.Id) =
+        withContext(backgroundDispatcher) {
+            database.resultQueries.markAsDone(resultId.value)
+        }
+
+    suspend fun markAllAsDone() =
+        withContext(backgroundDispatcher) {
+            database.resultQueries.markAllAsDone()
+        }
+
     suspend fun deleteByRunId(resultId: InstalledTestDescriptorModel.Id) =
         withContext(backgroundDispatcher) {
             database.resultQueries.deleteByRunId(resultId.value)