refactor: low latency audio processing outside of JNI

2026-04-14 23:38:19 +08:00
parent c002096a1e
commit 08b996e98c
11 changed files with 233 additions and 25 deletions
--- a/app/build.gradle.kts
+++ b/app/build.gradle.kts
@@ -40,8 +40,8 @@ android {
        applicationId = "io.github.miuzarte.scrcpyforandroid"
        minSdk = 26
        targetSdk = 36
-        versionCode = 12
-        versionName = "0.1.6"
+        versionCode = 13
+        versionName = "0.1.7"

        externalNativeBuild {
            cmake {
--- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/MainActivity.kt
+++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/MainActivity.kt
@@ -1,6 +1,5 @@
 package io.github.miuzarte.scrcpyforandroid

-import android.annotation.SuppressLint
 import android.os.Bundle
 import androidx.activity.compose.setContent
 import androidx.activity.enableEdgeToEdge
--- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/ScrcpyAudioPlayer.kt
+++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/ScrcpyAudioPlayer.kt
@@ -2,16 +2,19 @@ package io.github.miuzarte.scrcpyforandroid.nativecore

 // Go reader note: Audio output helper for scrcpy stream: decodes/plays PCM or codec audio frames.

+import android.content.Context
 import android.media.AudioAttributes
 import android.media.AudioFormat
 import android.media.AudioManager
 import android.media.AudioTrack
 import android.media.MediaCodec
 import android.media.MediaFormat
+import android.os.Build
 import android.util.Log
 import io.github.miuzarte.scrcpyforandroid.scrcpy.Shared.Codec
 import java.nio.ByteBuffer
 import java.nio.ByteOrder
+import kotlin.math.max

 /**
 * Decodes and plays scrcpy audio stream (OPUS / AAC / FLAC / RAW PCM).
@@ -19,11 +22,21 @@ import java.nio.ByteOrder
 * All [feedPacket] calls are expected from a single background thread.
 * [release] may be called from any thread.
 */
-class ScrcpyAudioPlayer(private val codecId: Int) {
+class ScrcpyAudioPlayer(
+    context: Context,
+    private val codecId: Int,
+    private val lowLatency: Boolean,
+) {

    private var mediaCodec: MediaCodec? = null
    private var audioTrack: AudioTrack? = null
    private val bufferInfo = MediaCodec.BufferInfo()
+    private val audioManager = context.applicationContext
+        .getSystemService(Context.AUDIO_SERVICE) as? AudioManager
+    private var reusablePcmBuffer = ByteArray(0)
+
+    @Volatile
+    private var audioThreadPriorityApplied = false

    @Volatile
    private var prepared = false
@@ -34,6 +47,7 @@ class ScrcpyAudioPlayer(private val codecId: Int) {

    fun feedPacket(data: ByteArray, ptsUs: Long, isConfig: Boolean) {
        if (released) return
+        applyAudioThreadPriorityIfNeeded()

        if (isConfig) {
            Log.i(
@@ -57,7 +71,13 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
        }

        if (codecId == Codec.RAW.id) {
-            ensureRawAudioTrack()?.write(data, 0, data.size, AudioTrack.WRITE_NON_BLOCKING)
+            val track = ensureRawAudioTrack() ?: return
+            track.write(
+                data,
+                0,
+                data.size,
+                AudioTrack.WRITE_NON_BLOCKING,
+            )
            return
        }

@@ -155,20 +175,64 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
        val minBuf = AudioTrack.getMinBufferSize(
            SAMPLE_RATE, AudioFormat.CHANNEL_OUT_STEREO, AudioFormat.ENCODING_PCM_16BIT,
        ).coerceAtLeast(1)
-        return AudioTrack(
-            AudioAttributes.Builder()
-                .setUsage(AudioAttributes.USAGE_MEDIA)
-                .setContentType(AudioAttributes.CONTENT_TYPE_MOVIE)
-                .build(),
-            AudioFormat.Builder()
-                .setEncoding(AudioFormat.ENCODING_PCM_16BIT)
-                .setSampleRate(SAMPLE_RATE)
-                .setChannelMask(AudioFormat.CHANNEL_OUT_STEREO)
-                .build(),
-            (minBuf * 4).coerceAtLeast(65536),
-            AudioTrack.MODE_STREAM,
-            AudioManager.AUDIO_SESSION_ID_GENERATE,
-        )
+        val framesPerBurst = audioManager
+            ?.getProperty(AudioManager.PROPERTY_OUTPUT_FRAMES_PER_BUFFER)
+            ?.toIntOrNull()
+            ?.takeIf { it > 0 }
+            ?: DEFAULT_FRAMES_PER_BURST
+        val nativeSampleRate = audioManager
+            ?.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE)
+            ?.toIntOrNull()
+            ?.takeIf { it > 0 }
+            ?: SAMPLE_RATE
+
+        val bufferSize = if (lowLatency) {
+            val targetBuffer = framesPerBurst * 2 * BYTES_PER_FRAME_PCM16_STEREO
+            max(minBuf, targetBuffer)
+        } else {
+            (minBuf * 4).coerceAtLeast(DEFAULT_BUFFER_SIZE_BYTES)
+        }
+
+        val attributesBuilder = AudioAttributes.Builder()
+            .setUsage(
+                if (lowLatency) AudioAttributes.USAGE_GAME
+                else AudioAttributes.USAGE_MEDIA
+            )
+            .setContentType(
+                if (lowLatency) AudioAttributes.CONTENT_TYPE_SONIFICATION
+                else AudioAttributes.CONTENT_TYPE_MOVIE
+            )
+        if (lowLatency) {
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+                attributesBuilder.setAllowedCapturePolicy(AudioAttributes.ALLOW_CAPTURE_BY_NONE)
+            }
+        }
+
+        val trackBuilder = AudioTrack.Builder()
+            .setAudioAttributes(attributesBuilder.build())
+            .setAudioFormat(
+                AudioFormat.Builder()
+                    .setEncoding(AudioFormat.ENCODING_PCM_16BIT)
+                    .setSampleRate(SAMPLE_RATE)
+                    .setChannelMask(AudioFormat.CHANNEL_OUT_STEREO)
+                    .build()
+            )
+            .setBufferSizeInBytes(bufferSize)
+            .setTransferMode(AudioTrack.MODE_STREAM)
+            .setSessionId(AudioManager.AUDIO_SESSION_ID_GENERATE)
+        if (lowLatency)
+            trackBuilder.setPerformanceMode(AudioTrack.PERFORMANCE_MODE_LOW_LATENCY)
+
+        val track = trackBuilder.build()
+
+        if (lowLatency) {
+            Log.i(
+                TAG,
+                "low-latency audio requested: nativeSampleRate=$nativeSampleRate streamSampleRate=$SAMPLE_RATE " +
+                        "framesPerBurst=$framesPerBurst bufferSize=$bufferSize performanceMode=${track.performanceMode}"
+            )
+        }
+        return track
    }

    /**
@@ -183,10 +247,15 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
            val outBuf = codec.getOutputBuffer(idx) ?: break
            val size = bufferInfo.size
            if (size > 0) {
-                val pcm = ByteArray(size)
+                ensurePcmBufferCapacity(size)
                outBuf.position(bufferInfo.offset)
-                outBuf.get(pcm)
-                track.write(pcm, 0, size, AudioTrack.WRITE_NON_BLOCKING)
+                outBuf.get(reusablePcmBuffer, 0, size)
+                track.write(
+                    reusablePcmBuffer,
+                    0,
+                    size,
+                    AudioTrack.WRITE_NON_BLOCKING,
+                )
            }
            codec.releaseOutputBuffer(idx, false)
            idx = codec.dequeueOutputBuffer(bufferInfo, 0L)
@@ -208,6 +277,22 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
        audioTrack = null
    }

+    private fun applyAudioThreadPriorityIfNeeded() {
+        if (!lowLatency || audioThreadPriorityApplied) return
+        runCatching {
+            android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO)
+            audioThreadPriorityApplied = true
+        }.onFailure {
+            Log.w(TAG, "set audio thread priority failed", it)
+        }
+    }
+
+    private fun ensurePcmBufferCapacity(requiredSize: Int) {
+        if (reusablePcmBuffer.size < requiredSize) {
+            reusablePcmBuffer = ByteArray(requiredSize)
+        }
+    }
+
    private fun longBuffer(value: Long): ByteBuffer =
        ByteBuffer.allocate(8).order(ByteOrder.nativeOrder()).apply { putLong(value); flip() }

@@ -217,5 +302,8 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
        private const val CHANNELS = 2
        private const val CODEC_TIMEOUT_US = 10_000L
        private const val OPUS_SEEK_PREROLL_NS = 80_000_000L // 80 ms
+        private const val DEFAULT_BUFFER_SIZE_BYTES = 65536
+        private const val DEFAULT_FRAMES_PER_BURST = 128
+        private const val BYTES_PER_FRAME_PCM16_STEREO = 4
    }
 }
--- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/pages/MainScreen.kt
+++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/pages/MainScreen.kt
@@ -222,17 +222,20 @@ fun MainScreen() {
        .ifBlank { Scrcpy.DEFAULT_SERVER_VERSION }
    val serverRemotePath = asBundle.serverRemotePath
        .ifBlank { AppSettings.SERVER_REMOTE_PATH.defaultValue }
+    val lowLatency = asBundle.lowLatency
    val scrcpy = remember(
        appContext,
        customServerUri,
        customServerVersion,
        serverRemotePath,
+        lowLatency,
    ) {
        Scrcpy(
            appContext = appContext,
            customServerUri = customServerUri,
            serverVersion = customServerVersion,
            serverRemotePath = serverRemotePath,
+            lowLatency = lowLatency,
        ).also {
            AppRuntime.scrcpy = it
        }
--- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/pages/SettingsScreen.kt
+++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/pages/SettingsScreen.kt
@@ -37,6 +37,7 @@ import io.github.miuzarte.scrcpyforandroid.scaffolds.LazyColumn
 import io.github.miuzarte.scrcpyforandroid.scaffolds.SuperSlider
 import io.github.miuzarte.scrcpyforandroid.scaffolds.SuperTextField
 import io.github.miuzarte.scrcpyforandroid.scrcpy.Scrcpy
+import io.github.miuzarte.scrcpyforandroid.services.AppRuntime
 import io.github.miuzarte.scrcpyforandroid.services.AppUpdateChecker
 import io.github.miuzarte.scrcpyforandroid.services.LocalSnackbarController
 import io.github.miuzarte.scrcpyforandroid.storage.AppSettings
@@ -126,6 +127,7 @@ fun SettingsPage(
    val snackbar = LocalSnackbarController.current
    val navigator = LocalRootNavigator.current
    val serverPicker = LocalServerPicker.current
+    val isScrcpyStreaming = AppRuntime.scrcpy?.isStarted() == true

    val asBundleShared by appSettings.bundleState.collectAsState()
    val asBundleSharedLatest by rememberUpdatedState(asBundleShared)
@@ -264,6 +266,18 @@ fun SettingsPage(
        item {
            SectionSmallTitle("投屏")
            Card {
+                SwitchPreference(
+                    title = "低延迟音频（实验性）",
+                    summary = "启用后将尝试使用低延迟音频路径" +
+                            "\n推荐配合 RAW PCM 编解码" +
+                            "\n修改后建议划卡重启应用",
+                    checked = asBundle.lowLatency,
+                    onCheckedChange = {
+                        if (!isScrcpyStreaming)
+                            asBundle = asBundle.copy(lowLatency = it)
+                    },
+                    enabled = !isScrcpyStreaming,
+                )
                SwitchPreference(
                    title = "启用调试信息",
                    summary = "在全屏界面悬浮显示分辨率、帧率和触点信息",
--- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/scaffolds/SuperTextField.kt
+++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/scaffolds/SuperTextField.kt
@@ -18,7 +18,6 @@ import androidx.compose.ui.graphics.Color
 import androidx.compose.ui.graphics.SolidColor
 import androidx.compose.ui.text.TextStyle
 import androidx.compose.ui.text.input.VisualTransformation
-import androidx.compose.ui.unit.Dp
 import androidx.compose.ui.unit.DpSize
 import androidx.compose.ui.unit.dp
 import top.yukonga.miuix.kmp.basic.TextField
--- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/scrcpy/Scrcpy.kt
+++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/scrcpy/Scrcpy.kt
@@ -58,6 +58,7 @@ class Scrcpy(
    private val customServerUri: String? = null,
    private val serverVersion: String = DEFAULT_SERVER_VERSION,
    private val serverRemotePath: String = DEFAULT_REMOTE_PATH,
+    private val lowLatency: Boolean = false,
 ) {

    private val session = Session()
@@ -160,7 +161,7 @@ class Scrcpy(
                        info.audioCodecId.toUInt().toString(16)
                    }"
                )
-                val player = ScrcpyAudioPlayer(info.audioCodecId)
+                val player = ScrcpyAudioPlayer(appContext, info.audioCodecId, lowLatency)
                audioPlayer = player
                session.attachAudioConsumer { packet ->
                    player.feedPacket(packet.data, packet.ptsUs, packet.isConfig)
--- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/storage/AppSettings.kt
+++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/storage/AppSettings.kt
@@ -38,6 +38,10 @@ class AppSettings(context: Context) : Settings(context, "AppSettings") {
            booleanPreferencesKey("smooth_corner"),
            false,
        )
+        val LOW_LATENCY = Pair(
+            booleanPreferencesKey("low_latency"),
+            false,
+        )
        val FULLSCREEN_DEBUG_INFO = Pair(
            booleanPreferencesKey("fullscreen_debug_info"),
            false
@@ -122,6 +126,7 @@ class AppSettings(context: Context) : Settings(context, "AppSettings") {
    val smoothCorner by setting(SMOOTH_CORNER)

    // Scrcpy Settings
+    val lowLatency by setting(LOW_LATENCY)
    val fullscreenDebugInfo by setting(FULLSCREEN_DEBUG_INFO)
    val showFullscreenVirtualButtons by setting(SHOW_FULLSCREEN_VIRTUAL_BUTTONS)
    val showFullscreenFloatingButton by setting(SHOW_FULLSCREEN_FLOATING_BUTTON)
@@ -153,6 +158,7 @@ class AppSettings(context: Context) : Settings(context, "AppSettings") {
        val floatingBottomBar: Boolean,
        val floatingBottomBarBlur: Boolean,
        val smoothCorner: Boolean,
+        val lowLatency: Boolean,
        val fullscreenDebugInfo: Boolean,
        val showFullscreenVirtualButtons: Boolean,
        val showFullscreenFloatingButton: Boolean,
@@ -181,6 +187,7 @@ class AppSettings(context: Context) : Settings(context, "AppSettings") {
        bundleField(FLOATING_BOTTOM_BAR) { bundle: Bundle -> bundle.floatingBottomBar },
        bundleField(FLOATING_BOTTOM_BAR_BLUR) { bundle: Bundle -> bundle.floatingBottomBarBlur },
        bundleField(SMOOTH_CORNER) { bundle: Bundle -> bundle.smoothCorner },
+        bundleField(LOW_LATENCY) { bundle: Bundle -> bundle.lowLatency },
        bundleField(FULLSCREEN_DEBUG_INFO) { bundle: Bundle -> bundle.fullscreenDebugInfo },
        bundleField(SHOW_FULLSCREEN_VIRTUAL_BUTTONS) { bundle: Bundle -> bundle.showFullscreenVirtualButtons },
        bundleField(SHOW_FULLSCREEN_FLOATING_BUTTON) { bundle: Bundle -> bundle.showFullscreenFloatingButton },
@@ -210,6 +217,7 @@ class AppSettings(context: Context) : Settings(context, "AppSettings") {
        floatingBottomBar = preferences.read(FLOATING_BOTTOM_BAR),
        floatingBottomBarBlur = preferences.read(FLOATING_BOTTOM_BAR_BLUR),
        smoothCorner = preferences.read(SMOOTH_CORNER),
+        lowLatency = preferences.read(LOW_LATENCY),
        fullscreenDebugInfo = preferences.read(FULLSCREEN_DEBUG_INFO),
        showFullscreenVirtualButtons = preferences.read(SHOW_FULLSCREEN_VIRTUAL_BUTTONS),
        showFullscreenFloatingButton = preferences.read(SHOW_FULLSCREEN_FLOATING_BUTTON),