App backup format v2 with compression and deduplication

.github/scripts/run_tests.sh

@@ -10,10 +10,8 @@ echo "Installing Seedvault app..."
 ./gradlew --stacktrace :app:installDebugAndroidTest
 sleep 60
 
-D2D_BACKUP_TEST=$1
-
 large_test_exit_code=0
-./gradlew --stacktrace -Pinstrumented_test_size=large -Pd2d_backup_test="$D2D_BACKUP_TEST" :app:connectedAndroidTest || large_test_exit_code=$?
+./gradlew --stacktrace -Pinstrumented_test_size=large :app:connectedAndroidTest || large_test_exit_code=$?
 
 adb pull /sdcard/seedvault_test_results
 

.github/workflows/test.yml

@@ -20,7 +20,6 @@ jobs:
       matrix:
         android_target: [ 34 ]
         emulator_type: [ aosp_atd ]
-        d2d_backup_test: [ true, false ]
     steps:
       - name: Checkout Code
         uses: actions/checkout@v3
@@ -53,7 +52,7 @@ jobs:
             disable-animations: true
             script: |
               ./app/development/scripts/provision_emulator.sh "test" "system-images;android-${{ matrix.android_target }};${{ matrix.emulator_type }};x86_64"
-              ./.github/scripts/run_tests.sh ${{ matrix.d2d_backup_test }}
+              ./.github/scripts/run_tests.sh
 
       - name: Upload test results
         if: always()

Android.bp

@@ -8,12 +8,23 @@ android_app {
     srcs: [
         "app/src/main/java/**/*.kt",
         "app/src/main/java/**/*.java",
+        "app/src/main/proto/*.proto",
+        // as of Android 15, there is no way to pass --kotlin_out to aprotoc compiler
+        "app/build/generated/source/proto/debug/kotlin/com/stevesoltys/seedvault/proto/*.kt",
     ],
     resource_dirs: [
         "app/src/main/res",
     ],
+    asset_dirs: [
+        "app/src/main/assets"
+    ],
+    proto: {
+        type: "lite",
+        local_include_dirs: ["app/src/main/proto"],
+    },
     static_libs: [
         "kotlin-stdlib-jdk8",
+        "libprotobuf-java-lite",
         "androidx.core_core-ktx",
         "androidx.fragment_fragment-ktx",
         "androidx.activity_activity-ktx",
@@ -26,6 +37,13 @@ android_app {
         "com.google.android.material_material",
         "kotlinx-coroutines-android",
         "kotlinx-coroutines-core",
+        "seedvault-lib-kotlin-logging-jvm",
+        // app backup related libs
+        "seedvault-lib-protobuf-kotlin-lite",
+        "seedvault-logback-android",
+        "seedvault-lib-chunker",
+        "seedvault-lib-zstd-jni",
+        "okio-lib",
         // our own gradle module libs
         "seedvault-lib-core",
         "seedvault-lib-storage",
@@ -34,10 +52,8 @@ android_app {
         "seedvault-lib-koin-android",
         // bip39
         "seedvault-lib-kotlin-bip39",
-        // WebDAV
-        "seedvault-lib-dav4jvm",
-        "seedvault-lib-okhttp",
     ],
+    use_embedded_native_libs: true,
     manifest: "app/src/main/AndroidManifest.xml",
 
     platform_apis: true,

app/build.gradle.kts

@@ -3,12 +3,14 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+import com.google.protobuf.gradle.id
 import org.gradle.api.tasks.testing.logging.TestExceptionFormat
 import java.io.ByteArrayOutputStream
 
 plugins {
     alias(libs.plugins.android.application)
     alias(libs.plugins.jetbrains.kotlin.android)
+    alias(libs.plugins.google.protobuf)
 }
 
 val gitDescribe = {
@@ -37,9 +39,6 @@ android {
 
             testInstrumentationRunnerArguments["size"] = testSize
         }
-
-        val d2dBackupTest = project.findProperty("d2d_backup_test")?.toString() ?: "true"
-        testInstrumentationRunnerArguments["d2d_backup_test"] = d2dBackupTest
     }
 
     signingConfigs {
@@ -93,6 +92,30 @@ android {
         }
     }
 
+    protobuf {
+        protoc {
+            artifact = if ("aarch64" == System.getProperty("os.arch")) {
+                // mac m1
+                "com.google.protobuf:protoc:${libs.versions.protobuf.get()}:osx-x86_64"
+            } else {
+                // other
+                "com.google.protobuf:protoc:${libs.versions.protobuf.get()}"
+            }
+        }
+        generateProtoTasks {
+            all().forEach { task ->
+                task.plugins {
+                    id("java") {
+                        option("lite")
+                    }
+                    id("kotlin") {
+                        option("lite")
+                    }
+                }
+            }
+        }
+    }
+
     lint {
         abortOnError = true
 
@@ -132,7 +155,10 @@ dependencies {
     implementation(libs.androidx.work.runtime.ktx)
     implementation(libs.google.material)
 
+    implementation(libs.google.protobuf.javalite)
     implementation(libs.google.tink.android)
+    implementation(libs.kotlin.logging)
+    implementation(libs.squareup.okio)
 
     /**
      * Storage Dependencies
@@ -151,9 +177,13 @@ dependencies {
     implementation(fileTree("${rootProject.rootDir}/libs/koin-android").include("*.jar"))
     implementation(fileTree("${rootProject.rootDir}/libs/koin-android").include("*.aar"))
 
-    implementation(fileTree("${rootProject.rootDir}/libs").include("kotlin-bip39-jvm-1.0.6.jar"))
-
-    implementation(fileTree("${rootProject.rootDir}/libs/dav4jvm").include("*.jar"))
+    implementation(
+        fileTree("${rootProject.rootDir}/libs").include("protobuf-kotlin-lite-3.21.12.jar")
+    )
+    implementation(fileTree("${rootProject.rootDir}/libs").include("seedvault-chunker-0.1.jar"))
+    implementation(fileTree("${rootProject.rootDir}/libs").include("zstd-jni-1.5.6-5.aar"))
+    implementation(fileTree("${rootProject.rootDir}/libs").include("kotlin-bip39-jvm-1.0.8.jar"))
+    implementation(fileTree("${rootProject.rootDir}/libs").include("logback-android-3.0.0.aar"))
 
     /**
      * Test Dependencies (do not concern the AOSP build)
@@ -163,6 +193,7 @@ dependencies {
     // anything less than 'implementation' fails tests run with gradlew
     testImplementation(aospLibs)
     testImplementation("androidx.test.ext:junit:1.1.5")
+    testImplementation("org.slf4j:slf4j-simple:2.0.3")
     testImplementation("org.robolectric:robolectric:4.12.2")
     testImplementation("org.hamcrest:hamcrest:2.2")
     testImplementation("org.junit.jupiter:junit-jupiter-api:${libs.versions.junit5.get()}")
@@ -173,6 +204,7 @@ dependencies {
     )
     testImplementation("app.cash.turbine:turbine:1.0.0")
     testImplementation("org.bitcoinj:bitcoinj-core:0.16.2")
+    testImplementation("com.github.luben:zstd-jni:1.5.6-5")
     testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${libs.versions.junit5.get()}")
     testRuntimeOnly("org.junit.vintage:junit-vintage-engine:${libs.versions.junit5.get()}")