Custom Audio input (#528)

* Custom audio handling

* Screen share audio example

* Screen Audio Capturer in sdk

* spotless

* Spotless

* fix extraneous logging

* changeset

* Update webrtc-sdk to 125.6422.06.1

* Test compile fixes

.changeset/brave-cows-confess.md

@@ -0,0 +1,5 @@
+---
+"client-sdk-android": minor
+---
+
+Implement custom audio mixing into audio track

.changeset/modern-phones-stare.md

@@ -0,0 +1,5 @@
+---
+"client-sdk-android": minor
+---
+
+Update to webrtc-sdk 125.6422.06.1

.changeset/twelve-ways-itch.md

@@ -0,0 +1,5 @@
+---
+"client-sdk-android": minor
+---
+
+Implement screen share audio capturer

examples/screenshare-audio/.gitignore

@@ -0,0 +1 @@
+/build

examples/screenshare-audio/build.gradle

@@ -0,0 +1,71 @@
+plugins {
+    id 'com.android.application'
+    id 'org.jetbrains.kotlin.android'
+}
+
+android {
+    namespace 'io.livekit.android.example.screenshareaudio'
+    compileSdk 34
+
+    defaultConfig {
+        applicationId "io.livekit.android.example.screenshareaudio"
+        minSdk 29
+        targetSdk 34
+        versionCode 1
+        versionName "1.0"
+
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+        vectorDrawables {
+            useSupportLibrary true
+        }
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+        }
+    }
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+    kotlinOptions {
+        jvmTarget = '1.8'
+    }
+    buildFeatures {
+        compose true
+    }
+    composeOptions {
+        kotlinCompilerExtensionVersion compose_compiler_version
+    }
+    packaging {
+        resources {
+            excludes += '/META-INF/{AL2.0,LGPL2.1}'
+        }
+    }
+}
+
+dependencies {
+
+    // If building the sample app outside the context of this repo, replace the following with:
+    // api "io.livekit:livekit-android:<version>"
+    api project(":livekit-android-sdk")
+    api project(":sample-app-common")
+    implementation libs.androidx.core.ktx
+    implementation libs.androidx.lifecycle.runtime.ktx
+    implementation libs.androidx.activity.compose
+    implementation platform(libs.compose.bom)
+    implementation libs.androidx.ui
+    implementation libs.androidx.ui.graphics
+    implementation libs.androidx.ui.tooling.preview
+    implementation libs.androidx.material3
+    implementation libs.timber
+    testImplementation libs.junit
+    androidTestImplementation libs.androidx.test.junit
+    androidTestImplementation libs.espresso
+    androidTestImplementation platform(libs.compose.bom)
+    androidTestImplementation libs.androidx.ui.test.junit4
+    debugImplementation libs.androidx.ui.tooling
+    debugImplementation libs.androidx.ui.test.manifest
+}

examples/screenshare-audio/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

examples/screenshare-audio/src/androidTest/java/io/livekit/android/example/screenshareaudio/ExampleInstrumentedTest.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2024 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.example.screenshareaudio
+
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
+import org.junit.Assert.*
+import org.junit.Test
+import org.junit.runner.RunWith
+
+/**
+ * Instrumented test, which will execute on an Android device.
+ *
+ * See [testing documentation](http://d.android.com/tools/testing).
+ */
+@RunWith(AndroidJUnit4::class)
+class ExampleInstrumentedTest {
+    @Test
+    fun useAppContext() {
+        // Context of the app under test.
+        val appContext = InstrumentationRegistry.getInstrumentation().targetContext
+        assertEquals("io.livekit.android.example.screenshareaudio", appContext.packageName)
+    }
+}

examples/screenshare-audio/src/main/AndroidManifest.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+    <application
+        android:allowBackup="true"
+        android:icon="@mipmap/ic_launcher"
+        android:label="@string/app_name"
+        android:roundIcon="@mipmap/ic_launcher_round"
+        android:supportsRtl="true"
+        android:theme="@style/Theme.Livekitandroid"
+        android:usesCleartextTraffic="true">
+        <activity
+            android:name=".MainActivity"
+            android:exported="true"
+            android:label="@string/app_name"
+            android:theme="@style/Theme.Livekitandroid">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+
+</manifest>

examples/screenshare-audio/src/main/java/io/livekit/android/example/screenshareaudio/MainActivity.kt

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2024 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.example.screenshareaudio
+
+import android.media.projection.MediaProjectionManager
+import android.os.Bundle
+import androidx.activity.ComponentActivity
+import androidx.activity.compose.setContent
+import androidx.activity.result.contract.ActivityResultContracts
+import androidx.activity.viewModels
+import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.material3.Button
+import androidx.compose.material3.MaterialTheme
+import androidx.compose.material3.Surface
+import androidx.compose.material3.Text
+import androidx.compose.runtime.getValue
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.remember
+import androidx.compose.runtime.setValue
+import androidx.compose.ui.Modifier
+import io.livekit.android.LiveKit
+import io.livekit.android.example.screenshareaudio.ui.theme.LivekitandroidTheme
+import io.livekit.android.util.LoggingLevel
+
+class MainActivity : ComponentActivity() {
+
+    private val viewModel: MainViewModel by viewModels<MainViewModel>()
+    private val screenCaptureIntentLauncher =
+        registerForActivityResult(
+            ActivityResultContracts.StartActivityForResult(),
+        ) { result ->
+            val resultCode = result.resultCode
+            val data = result.data
+            if (resultCode != RESULT_OK || data == null) {
+                return@registerForActivityResult
+            }
+            viewModel.startScreenCapture(data)
+        }
+
+    private fun requestMediaProjection() {
+        val mediaProjectionManager =
+            getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
+        screenCaptureIntentLauncher.launch(mediaProjectionManager.createScreenCaptureIntent())
+    }
+
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+        LiveKit.loggingLevel = LoggingLevel.INFO
+        viewModel
+        setContent {
+            LivekitandroidTheme {
+                // A surface container using the 'background' color from the theme
+                Surface(modifier = Modifier.fillMaxSize(), color = MaterialTheme.colorScheme.background) {
+                    var enableScreenCapture by remember {
+                        mutableStateOf(true)
+                    }
+                    Button(
+                        onClick = {
+                            if (enableScreenCapture) {
+                                requestMediaProjection()
+                            } else {
+                                viewModel.stopScreenCapture()
+                            }
+                            enableScreenCapture = !enableScreenCapture
+                        },
+                    ) {
+                        val text = if (enableScreenCapture) {
+                            "enable"
+                        } else {
+                            "disable"
+                        }
+                        Text(text = text)
+                    }
+                }
+            }
+        }
+    }
+}

examples/screenshare-audio/src/main/java/io/livekit/android/example/screenshareaudio/MainViewModel.kt

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2024 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.example.screenshareaudio
+
+import android.Manifest
+import android.app.Application
+import android.content.Intent
+import android.content.pm.PackageManager
+import android.os.Build
+import androidx.annotation.RequiresApi
+import androidx.core.app.ActivityCompat
+import androidx.lifecycle.AndroidViewModel
+import androidx.lifecycle.viewModelScope
+import io.livekit.android.LiveKit
+import io.livekit.android.audio.ScreenAudioCapturer
+import io.livekit.android.room.track.LocalAudioTrack
+import io.livekit.android.room.track.LocalVideoTrack
+import io.livekit.android.room.track.Track
+import io.livekit.android.sample.service.ForegroundService
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
+
+val url = "wss://example.com"
+val token = ""
+
+@RequiresApi(Build.VERSION_CODES.Q)
+class MainViewModel(application: Application) : AndroidViewModel(application) {
+
+    val room = LiveKit.create(application)
+    var audioCapturer: ScreenAudioCapturer? = null
+
+    init {
+        viewModelScope.launch(Dispatchers.IO) {
+            room.connect(url, token)
+        }
+
+        // Start a foreground service to keep the call from being interrupted if the
+        // app goes into the background.
+        val foregroundServiceIntent = Intent(application, ForegroundService::class.java)
+        application.startForegroundService(foregroundServiceIntent)
+    }
+
+    fun startScreenCapture(data: Intent) {
+        viewModelScope.launch(Dispatchers.IO) {
+            if (ActivityCompat.checkSelfPermission(getApplication(), Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) {
+                return@launch
+            }
+            room.localParticipant.setScreenShareEnabled(true, data)
+            room.localParticipant.setMicrophoneEnabled(true)
+            val screenCaptureTrack = room.localParticipant.getTrackPublication(Track.Source.SCREEN_SHARE)?.track as? LocalVideoTrack ?: return@launch
+            val audioTrack = room.localParticipant.getTrackPublication(Track.Source.MICROPHONE)?.track as? LocalAudioTrack ?: return@launch
+
+            audioCapturer = ScreenAudioCapturer.createFromScreenShareTrack(screenCaptureTrack) ?: return@launch
+            audioCapturer?.gain = 0.1f // Lower the volume so that mic can still be heard clearly.
+            audioTrack.setAudioBufferCallback(audioCapturer!!)
+        }
+    }
+
+    fun stopScreenCapture() {
+        viewModelScope.launch(Dispatchers.IO) {
+            (room.localParticipant.getTrackPublication(Track.Source.MICROPHONE)?.track as? LocalAudioTrack)
+                ?.setAudioBufferCallback(null)
+            room.localParticipant.setMicrophoneEnabled(false)
+            room.localParticipant.setScreenShareEnabled(false)
+            audioCapturer?.releaseAudioResources()
+        }
+    }
+}

examples/screenshare-audio/src/main/java/io/livekit/android/example/screenshareaudio/ui/theme/Color.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2024 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.example.screenshareaudio.ui.theme
+
+import androidx.compose.ui.graphics.Color
+
+val Purple80 = Color(0xFFD0BCFF)
+val PurpleGrey80 = Color(0xFFCCC2DC)
+val Pink80 = Color(0xFFEFB8C8)
+
+val Purple40 = Color(0xFF6650a4)
+val PurpleGrey40 = Color(0xFF625b71)
+val Pink40 = Color(0xFF7D5260)