Merge branch 'molly-7.17'

app/build.gradle.kts

@@ -18,8 +18,8 @@ apply {
   from("fix-profm.gradle")
 }
 
-val canonicalVersionCode = 1451
-val canonicalVersionName = "7.15.4"
+val canonicalVersionCode = 1461
+val canonicalVersionName = "7.17.4"
 val currentHotfixVersion = 0
 val maxHotfixVersions = 100
 val mollyRevision = 1
@@ -45,6 +45,7 @@ val signalBuildToolsVersion: String by rootProject.extra
 val signalCompileSdkVersion: String by rootProject.extra
 val signalTargetSdkVersion: Int by rootProject.extra
 val signalMinSdkVersion: Int by rootProject.extra
+val signalNdkVersion: String by rootProject.extra
 val signalJavaVersion: JavaVersion by rootProject.extra
 val signalKotlinJvmTarget: String by rootProject.extra
 
@@ -79,6 +80,7 @@ android {
 
   buildToolsVersion = signalBuildToolsVersion
   compileSdkVersion = signalCompileSdkVersion
+  ndkVersion = signalNdkVersion
 
   flavorDimensions += listOf("environment", "license", "distribution")
   useLibrary("org.apache.http.legacy")
@@ -88,6 +90,7 @@ android {
 
   kotlinOptions {
     jvmTarget = signalKotlinJvmTarget
+    freeCompilerArgs = listOf("-Xjvm-default=all")
   }
 
   signingConfigs {

app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest.kt

@@ -1,22 +1,39 @@
 package org.thoughtcrime.securesms.database
 
+import android.content.Context
 import android.net.Uri
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.filters.FlakyTest
+import androidx.test.platform.app.InstrumentationRegistry
+import org.junit.Assert.assertArrayEquals
 import org.junit.Assert.assertEquals
 import org.junit.Assert.assertNotEquals
 import org.junit.Before
 import org.junit.Ignore
 import org.junit.Test
 import org.junit.runner.RunWith
+import org.signal.core.util.copyTo
+import org.signal.core.util.readFully
+import org.signal.core.util.stream.NullOutputStream
+import org.thoughtcrime.securesms.attachments.Attachment
 import org.thoughtcrime.securesms.attachments.AttachmentId
+import org.thoughtcrime.securesms.attachments.PointerAttachment
 import org.thoughtcrime.securesms.attachments.UriAttachment
 import org.thoughtcrime.securesms.mms.MediaStream
 import org.thoughtcrime.securesms.mms.SentMediaQuality
 import org.thoughtcrime.securesms.providers.BlobProvider
 import org.thoughtcrime.securesms.testing.assertIs
 import org.thoughtcrime.securesms.testing.assertIsNot
 import org.thoughtcrime.securesms.util.MediaUtil
+import org.thoughtcrime.securesms.util.Util
+import org.whispersystems.signalservice.api.crypto.AttachmentCipherInputStream
+import org.whispersystems.signalservice.api.crypto.AttachmentCipherOutputStream
+import org.whispersystems.signalservice.api.crypto.NoCipherOutputStream
+import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
+import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
+import org.whispersystems.signalservice.internal.crypto.PaddingInputStream
+import java.io.ByteArrayOutputStream
+import java.io.File
 import java.util.Optional
 
 @RunWith(AndroidJUnit4::class)
@@ -163,6 +180,91 @@ class AttachmentTableTest {
     highInfo.file.exists() assertIs true
   }
 
+  @Test
+  fun finalizeAttachmentAfterDownload_fixDigestOnNonZeroPadding() {
+    // Insert attachment metadata for badly-padded attachment
+    val plaintext = byteArrayOf(1, 2, 3, 4)
+    val key = Util.getSecretBytes(64)
+    val iv = Util.getSecretBytes(16)
+
+    val badlyPaddedPlaintext = PaddingInputStream(plaintext.inputStream(), plaintext.size.toLong()).readFully().also { it[it.size - 1] = 0x42 }
+    val badlyPaddedCiphertext = encryptPrePaddedBytes(badlyPaddedPlaintext, key, iv)
+    val badlyPaddedDigest = getDigest(badlyPaddedCiphertext)
+
+    val cipherFile = getTempFile()
+    cipherFile.writeBytes(badlyPaddedCiphertext)
+
+    val mmsId = -1L
+    val attachmentId = SignalDatabase.attachments.insertAttachmentsForMessage(mmsId, listOf(createAttachmentPointer(key, badlyPaddedDigest, plaintext.size)), emptyList()).values.first()
+
+    // Give data to attachment table
+    val cipherInputStream = AttachmentCipherInputStream.createForAttachment(cipherFile, plaintext.size.toLong(), key, badlyPaddedDigest, null, 4, false)
+    SignalDatabase.attachments.finalizeAttachmentAfterDownload(mmsId, attachmentId, cipherInputStream, iv)
+
+    // Verify the digest has been updated to the properly padded one
+    val properlyPaddedPlaintext = PaddingInputStream(plaintext.inputStream(), plaintext.size.toLong()).readFully()
+    val properlyPaddedCiphertext = encryptPrePaddedBytes(properlyPaddedPlaintext, key, iv)
+    val properlyPaddedDigest = getDigest(properlyPaddedCiphertext)
+
+    val newDigest = SignalDatabase.attachments.getAttachment(attachmentId)!!.remoteDigest!!
+
+    assertArrayEquals(properlyPaddedDigest, newDigest)
+  }
+
+  @Test
+  fun finalizeAttachmentAfterDownload_leaveDigestAloneForAllZeroPadding() {
+    // Insert attachment metadata for properly-padded attachment
+    val plaintext = byteArrayOf(1, 2, 3, 4)
+    val key = Util.getSecretBytes(64)
+    val iv = Util.getSecretBytes(16)
+
+    val paddedPlaintext = PaddingInputStream(plaintext.inputStream(), plaintext.size.toLong()).readFully()
+    val ciphertext = encryptPrePaddedBytes(paddedPlaintext, key, iv)
+    val digest = getDigest(ciphertext)
+
+    val cipherFile = getTempFile()
+    cipherFile.writeBytes(ciphertext)
+
+    val mmsId = -1L
+    val attachmentId = SignalDatabase.attachments.insertAttachmentsForMessage(mmsId, listOf(createAttachmentPointer(key, digest, plaintext.size)), emptyList()).values.first()
+
+    // Give data to attachment table
+    val cipherInputStream = AttachmentCipherInputStream.createForAttachment(cipherFile, plaintext.size.toLong(), key, digest, null, 4, false)
+    SignalDatabase.attachments.finalizeAttachmentAfterDownload(mmsId, attachmentId, cipherInputStream, iv)
+
+    // Verify the digest hasn't changed
+    val newDigest = SignalDatabase.attachments.getAttachment(attachmentId)!!.remoteDigest!!
+    assertArrayEquals(digest, newDigest)
+  }
+
+  private fun createAttachmentPointer(key: ByteArray, digest: ByteArray, size: Int): Attachment {
+    return PointerAttachment.forPointer(
+      pointer = Optional.of(
+        SignalServiceAttachmentPointer(
+          cdnNumber = 3,
+          remoteId = SignalServiceAttachmentRemoteId.V4("asdf"),
+          contentType = MediaUtil.IMAGE_JPEG,
+          key = key,
+          size = Optional.of(size),
+          preview = Optional.empty(),
+          width = 2,
+          height = 2,
+          digest = Optional.of(digest),
+          incrementalDigest = Optional.empty(),
+          incrementalMacChunkSize = 0,
+          fileName = Optional.of("file.jpg"),
+          voiceNote = false,
+          isBorderless = false,
+          isGif = false,
+          caption = Optional.empty(),
+          blurHash = Optional.empty(),
+          uploadTimestamp = 0,
+          uuid = null
+        )
+      )
+    ).get()
+  }
+
   private fun createAttachment(id: Long, uri: Uri, transformProperties: AttachmentTable.TransformProperties): UriAttachment {
     return UriAttachmentBuilder.build(
       id,
@@ -179,4 +281,24 @@ class AttachmentTableTest {
   private fun createMediaStream(byteArray: ByteArray): MediaStream {
     return MediaStream(byteArray.inputStream(), MediaUtil.IMAGE_JPEG, 2, 2)
   }
+
+  private fun getDigest(ciphertext: ByteArray): ByteArray {
+    val digestStream = NoCipherOutputStream(NullOutputStream)
+    ciphertext.inputStream().copyTo(digestStream)
+    return digestStream.transmittedDigest
+  }
+
+  private fun encryptPrePaddedBytes(plaintext: ByteArray, key: ByteArray, iv: ByteArray): ByteArray {
+    val outputStream = ByteArrayOutputStream()
+    val cipherStream = AttachmentCipherOutputStream(key, iv, outputStream)
+    plaintext.inputStream().copyTo(cipherStream)
+
+    return outputStream.toByteArray()
+  }
+
+  private fun getTempFile(): File {
+    val dir = InstrumentationRegistry.getInstrumentation().targetContext.getDir("temp", Context.MODE_PRIVATE)
+    dir.mkdir()
+    return File.createTempFile("transfer", ".mms", dir)
+  }
 }

app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest_deduping.kt

@@ -15,7 +15,6 @@ import org.signal.core.util.Base64
 import org.signal.core.util.update
 import org.thoughtcrime.securesms.attachments.AttachmentId
 import org.thoughtcrime.securesms.attachments.Cdn
-import org.thoughtcrime.securesms.attachments.PointerAttachment
 import org.thoughtcrime.securesms.backup.v2.BackupRepository.getMediaName
 import org.thoughtcrime.securesms.database.AttachmentTable.TransformProperties
 import org.thoughtcrime.securesms.keyvalue.SignalStore
@@ -27,7 +26,9 @@ import org.thoughtcrime.securesms.providers.BlobProvider
 import org.thoughtcrime.securesms.recipients.Recipient
 import org.thoughtcrime.securesms.util.MediaUtil
 import org.thoughtcrime.securesms.util.Util
+import org.whispersystems.signalservice.api.attachment.AttachmentUploadResult
 import org.whispersystems.signalservice.api.backup.MediaId
+import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentRemoteId
 import org.whispersystems.signalservice.api.push.ServiceId
 import java.io.File
 import java.util.UUID
@@ -110,15 +111,6 @@ class AttachmentTableTest_deduping {
       assertDataFilesAreDifferent(id1, id2)
       assertDataHashStartMatches(id1, id2)
     }
-
-    // Non-matching mp4 fast start
-    test {
-      val id1 = insertWithData(DATA_A, TransformProperties(mp4FastStart = true))
-      val id2 = insertWithData(DATA_A, TransformProperties(mp4FastStart = false))
-
-      assertDataFilesAreDifferent(id1, id2)
-      assertDataHashStartMatches(id1, id2)
-    }
   }
 
   /**
@@ -661,7 +653,7 @@ class AttachmentTableTest_deduping {
     }
 
     fun upload(attachmentId: AttachmentId, uploadTimestamp: Long = System.currentTimeMillis()) {
-      SignalDatabase.attachments.finalizeAttachmentAfterUpload(attachmentId, createPointerAttachment(attachmentId, uploadTimestamp), uploadTimestamp)
+      SignalDatabase.attachments.finalizeAttachmentAfterUpload(attachmentId, createUploadResult(attachmentId, uploadTimestamp))
 
       val attachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
       SignalDatabase.attachments.setArchiveData(
@@ -763,6 +755,7 @@ class AttachmentTableTest_deduping {
 
       assertEquals(lhsAttachment.remoteLocation, rhsAttachment.remoteLocation)
       assertEquals(lhsAttachment.remoteKey, rhsAttachment.remoteKey)
+      assertArrayEquals(lhsAttachment.remoteIv, rhsAttachment.remoteIv)
       assertArrayEquals(lhsAttachment.remoteDigest, rhsAttachment.remoteDigest)
       assertArrayEquals(lhsAttachment.incrementalDigest, rhsAttachment.incrementalDigest)
       assertEquals(lhsAttachment.incrementalMacChunkSize, rhsAttachment.incrementalMacChunkSize)
@@ -796,36 +789,19 @@ class AttachmentTableTest_deduping {
       return MediaStream(this.inputStream(), MediaUtil.IMAGE_JPEG, 2, 2)
     }
 
-    private fun createPointerAttachment(attachmentId: AttachmentId, uploadTimestamp: Long = System.currentTimeMillis()): PointerAttachment {
-      val location = "somewhere-${Random.nextLong()}"
-      val key = "somekey-${Random.nextLong()}"
-      val digest = Random.nextBytes(32)
-      val incrementalDigest = Random.nextBytes(16)
-
+    private fun createUploadResult(attachmentId: AttachmentId, uploadTimestamp: Long = System.currentTimeMillis()): AttachmentUploadResult {
       val databaseAttachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
 
-      return PointerAttachment(
-        "image/jpeg",
-        AttachmentTable.TRANSFER_PROGRESS_DONE,
-        databaseAttachment.size, // size
-        null,
-        Cdn.CDN_3, // cdnNumber
-        location,
-        key,
-        digest,
-        incrementalDigest,
-        5, // incrementalMacChunkSize
-        null,
-        databaseAttachment.voiceNote,
-        databaseAttachment.borderless,
-        databaseAttachment.videoGif,
-        databaseAttachment.width,
-        databaseAttachment.height,
-        uploadTimestamp,
-        databaseAttachment.caption,
-        databaseAttachment.stickerLocator,
-        databaseAttachment.blurHash,
-        databaseAttachment.uuid
+      return AttachmentUploadResult(
+        remoteId = SignalServiceAttachmentRemoteId.V4("somewhere-${Random.nextLong()}"),
+        cdnNumber = Cdn.CDN_3.cdnNumber,
+        key = databaseAttachment.remoteKey?.let { Base64.decode(it) } ?: Util.getSecretBytes(64),
+        iv = databaseAttachment.remoteIv ?: Util.getSecretBytes(16),
+        digest = Random.nextBytes(32),
+        incrementalDigest = Random.nextBytes(16),
+        incrementalDigestChunkSize = 5,
+        uploadTimestamp = uploadTimestamp,
+        dataSize = databaseAttachment.size
       )
     }
   }

app/src/androidTest/java/org/thoughtcrime/securesms/database/CallLinkTableTest.kt

@@ -86,7 +86,8 @@ class CallLinkTableTest {
           linkKeyBytes = roomId,
           adminPassBytes = null
         ),
-        state = SignalCallLinkState()
+        state = SignalCallLinkState(),
+        deletionTimestamp = 0L
       )
     )
 

app/src/androidTest/java/org/thoughtcrime/securesms/database/SQLiteDatabaseTest.kt

@@ -8,7 +8,6 @@ import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
 import org.signal.core.util.concurrent.SignalExecutors
-import org.thoughtcrime.securesms.testing.SignalFlakyTest
 import org.thoughtcrime.securesms.testing.SignalFlakyTestRule
 import java.util.concurrent.CountDownLatch
 import java.util.concurrent.atomic.AtomicBoolean
@@ -187,8 +186,8 @@ class SQLiteDatabaseTest {
     assertTrue(hasRun2.get())
   }
 
-  @SignalFlakyTest
-  @Test
+//  @SignalFlakyTest
+//  @Test
   fun runPostSuccessfulTransaction_runsAfterMainTransactionInNestedTransaction() {
     val hasRun1 = AtomicBoolean(false)
     val hasRun2 = AtomicBoolean(false)

app/src/androidTest/java/org/thoughtcrime/securesms/dependencies/InstrumentationApplicationDependencyProvider.kt

@@ -35,6 +35,7 @@ import org.whispersystems.signalservice.internal.configuration.SignalServiceConf
 import org.whispersystems.signalservice.internal.configuration.SignalServiceUrl
 import org.whispersystems.signalservice.internal.configuration.SignalStorageUrl
 import org.whispersystems.signalservice.internal.configuration.SignalSvr2Url
+import org.whispersystems.signalservice.internal.push.PushServiceSocket
 
 /**
  * Dependency provider used for instrumentation tests (aka androidTests).
@@ -113,10 +114,10 @@ class InstrumentationApplicationDependencyProvider(val application: Application,
   override fun provideSignalServiceMessageSender(
     signalWebSocket: SignalWebSocket,
     protocolStore: SignalServiceDataStore,
-    signalServiceConfiguration: SignalServiceConfiguration
+    pushServiceSocket: PushServiceSocket
   ): SignalServiceMessageSender {
     if (signalServiceMessageSender == null) {
-      signalServiceMessageSender = spyk(objToCopy = default.provideSignalServiceMessageSender(signalWebSocket, protocolStore, signalServiceConfiguration))
+      signalServiceMessageSender = spyk(objToCopy = default.provideSignalServiceMessageSender(signalWebSocket, protocolStore, pushServiceSocket))
     }
     return signalServiceMessageSender!!
   }

app/src/androidTest/java/org/thoughtcrime/securesms/messages/EditMessageSyncProcessorTest.kt

@@ -77,7 +77,7 @@ class EditMessageSyncProcessorTest {
             .build()
         ).build()
       ).build()
-      SignalDatabase.recipients.setExpireMessages(toRecipient.id, content.dataMessage?.expireTimer ?: 0)
+      SignalDatabase.recipients.setExpireMessages(toRecipient.id, content.dataMessage?.expireTimer ?: 0, content.dataMessage?.expireTimerVersion ?: 1)
       val syncTextMessage = TestMessage(
         envelope = MessageContentFuzzer.envelope(originalTimestamp),
         content = syncContent,
@@ -112,7 +112,7 @@ class EditMessageSyncProcessorTest {
 
       testResult.runSync(listOf(syncTextMessage, syncEditMessage))
 
-      SignalDatabase.recipients.setExpireMessages(toRecipient.id, (content.dataMessage?.expireTimer ?: 0) / 1000)
+      SignalDatabase.recipients.setExpireMessages(toRecipient.id, (content.dataMessage?.expireTimer ?: 0) / 1000, content.dataMessage?.expireTimerVersion ?: 1)
       val originalTextMessage = OutgoingMessage(
         threadRecipient = toRecipient,
         sentTimeMillis = originalTimestamp,