refactor: low latency audio processing outside of JNI

2026-04-14 23:38:19 +08:00
parent c002096a1e
commit 08b996e98c
11 changed files with 233 additions and 25 deletions
--- a/README.md
+++ b/README.md
@@ -23,7 +23,11 @@

 ## Features

+- 低延迟音频链路（默认未启用）
+  - 受控设备播放 `USAGE_MEDIA` 流时（[namidaco/namida](https://github.com/namidaco/namida)），两设备的音频延迟只差半拍（没有具体测量能力）
+  - 受控设备播放 `USAGE_GAME` 流时（明日方舟 Bilibili 服），仍存在 100~200ms 的有感延迟
 - 带生物认证的锁屏密码自动填充 (入口位于虚拟按钮中)
+- 多配置切换，设备绑定配置，连接后直接进入全屏
 - 可替换 scrcpy-server
 - 利用 mDNS 服务实现自动连接启用无线调试的设备、自动发现等待配对设备的IP与端口
 - 自动横竖屏切换（算吗
--- a/TODO.md
+++ b/TODO.md
@@ -16,6 +16,5 @@

 顺序无关

- [音频延迟](https://developer.android.com/ndk/guides/audio/audio-latency)
 - 横屏布局
 - I18N
--- a/app/build.gradle.kts
+++ b/app/build.gradle.kts
@@ -40,8 +40,8 @@ android {
        applicationId = "io.github.miuzarte.scrcpyforandroid"
        minSdk = 26
        targetSdk = 36
-        versionCode = 12
-        versionName = "0.1.6"
+        versionCode = 13
+        versionName = "0.1.7"

        externalNativeBuild {
            cmake {
--- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/MainActivity.kt
+++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/MainActivity.kt
@@ -1,6 +1,5 @@
 package io.github.miuzarte.scrcpyforandroid

-import android.annotation.SuppressLint
 import android.os.Bundle
 import androidx.activity.compose.setContent
 import androidx.activity.enableEdgeToEdge
--- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/ScrcpyAudioPlayer.kt
+++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/ScrcpyAudioPlayer.kt
@@ -2,16 +2,19 @@ package io.github.miuzarte.scrcpyforandroid.nativecore

 // Go reader note: Audio output helper for scrcpy stream: decodes/plays PCM or codec audio frames.

+import android.content.Context
 import android.media.AudioAttributes
 import android.media.AudioFormat
 import android.media.AudioManager
 import android.media.AudioTrack
 import android.media.MediaCodec
 import android.media.MediaFormat
+import android.os.Build
 import android.util.Log
 import io.github.miuzarte.scrcpyforandroid.scrcpy.Shared.Codec
 import java.nio.ByteBuffer
 import java.nio.ByteOrder
+import kotlin.math.max

 /**
 * Decodes and plays scrcpy audio stream (OPUS / AAC / FLAC / RAW PCM).
@@ -19,11 +22,21 @@ import java.nio.ByteOrder
 * All [feedPacket] calls are expected from a single background thread.
 * [release] may be called from any thread.
 */
-class ScrcpyAudioPlayer(private val codecId: Int) {
+class ScrcpyAudioPlayer(
+    context: Context,
+    private val codecId: Int,
+    private val lowLatency: Boolean,
+) {

    private var mediaCodec: MediaCodec? = null
    private var audioTrack: AudioTrack? = null
    private val bufferInfo = MediaCodec.BufferInfo()
+    private val audioManager = context.applicationContext
+        .getSystemService(Context.AUDIO_SERVICE) as? AudioManager
+    private var reusablePcmBuffer = ByteArray(0)
+
+    @Volatile
+    private var audioThreadPriorityApplied = false

    @Volatile
    private var prepared = false
@@ -34,6 +47,7 @@ class ScrcpyAudioPlayer(private val codecId: Int) {

    fun feedPacket(data: ByteArray, ptsUs: Long, isConfig: Boolean) {
        if (released) return
+        applyAudioThreadPriorityIfNeeded()

        if (isConfig) {
            Log.i(
@@ -57,7 +71,13 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
        }

        if (codecId == Codec.RAW.id) {
-            ensureRawAudioTrack()?.write(data, 0, data.size, AudioTrack.WRITE_NON_BLOCKING)
+            val track = ensureRawAudioTrack() ?: return
+            track.write(
+                data,
+                0,
+                data.size,
+                AudioTrack.WRITE_NON_BLOCKING,
+            )
            return
        }

@@ -155,20 +175,64 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
        val minBuf = AudioTrack.getMinBufferSize(
            SAMPLE_RATE, AudioFormat.CHANNEL_OUT_STEREO, AudioFormat.ENCODING_PCM_16BIT,
        ).coerceAtLeast(1)
-        return AudioTrack(
-            AudioAttributes.Builder()
-                .setUsage(AudioAttributes.USAGE_MEDIA)
-                .setContentType(AudioAttributes.CONTENT_TYPE_MOVIE)
-                .build(),
+        val framesPerBurst = audioManager
+            ?.getProperty(AudioManager.PROPERTY_OUTPUT_FRAMES_PER_BUFFER)
+            ?.toIntOrNull()
+            ?.takeIf { it > 0 }
+            ?: DEFAULT_FRAMES_PER_BURST
+        val nativeSampleRate = audioManager
+            ?.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE)
+            ?.toIntOrNull()
+            ?.takeIf { it > 0 }
+            ?: SAMPLE_RATE
+
+        val bufferSize = if (lowLatency) {
+            val targetBuffer = framesPerBurst * 2 * BYTES_PER_FRAME_PCM16_STEREO
+            max(minBuf, targetBuffer)
+        } else {
+            (minBuf * 4).coerceAtLeast(DEFAULT_BUFFER_SIZE_BYTES)
+        }
+
+        val attributesBuilder = AudioAttributes.Builder()
+            .setUsage(
+                if (lowLatency) AudioAttributes.USAGE_GAME
+                else AudioAttributes.USAGE_MEDIA
+            )
+            .setContentType(
+                if (lowLatency) AudioAttributes.CONTENT_TYPE_SONIFICATION
+                else AudioAttributes.CONTENT_TYPE_MOVIE
+            )
+        if (lowLatency) {
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+                attributesBuilder.setAllowedCapturePolicy(AudioAttributes.ALLOW_CAPTURE_BY_NONE)
+            }
+        }
+
+        val trackBuilder = AudioTrack.Builder()
+            .setAudioAttributes(attributesBuilder.build())
+            .setAudioFormat(
                AudioFormat.Builder()
                    .setEncoding(AudioFormat.ENCODING_PCM_16BIT)
                    .setSampleRate(SAMPLE_RATE)
                    .setChannelMask(AudioFormat.CHANNEL_OUT_STEREO)
-                .build(),
-            (minBuf * 4).coerceAtLeast(65536),
-            AudioTrack.MODE_STREAM,
-            AudioManager.AUDIO_SESSION_ID_GENERATE,
+                    .build()
            )
+            .setBufferSizeInBytes(bufferSize)
+            .setTransferMode(AudioTrack.MODE_STREAM)
+            .setSessionId(AudioManager.AUDIO_SESSION_ID_GENERATE)
+        if (lowLatency)
+            trackBuilder.setPerformanceMode(AudioTrack.PERFORMANCE_MODE_LOW_LATENCY)
+
+        val track = trackBuilder.build()
+
+        if (lowLatency) {
+            Log.i(
+                TAG,
+                "low-latency audio requested: nativeSampleRate=$nativeSampleRate streamSampleRate=$SAMPLE_RATE " +
+                        "framesPerBurst=$framesPerBurst bufferSize=$bufferSize performanceMode=${track.performanceMode}"
+            )
+        }
+        return track
    }

    /**
@@ -183,10 +247,15 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
            val outBuf = codec.getOutputBuffer(idx) ?: break
            val size = bufferInfo.size
            if (size > 0) {
-                val pcm = ByteArray(size)
+                ensurePcmBufferCapacity(size)
                outBuf.position(bufferInfo.offset)
-                outBuf.get(pcm)
-                track.write(pcm, 0, size, AudioTrack.WRITE_NON_BLOCKING)
+                outBuf.get(reusablePcmBuffer, 0, size)
+                track.write(
+                    reusablePcmBuffer,
+                    0,
+                    size,
+                    AudioTrack.WRITE_NON_BLOCKING,
+                )
            }
            codec.releaseOutputBuffer(idx, false)
            idx = codec.dequeueOutputBuffer(bufferInfo, 0L)
@@ -208,6 +277,22 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
        audioTrack = null
    }

+    private fun applyAudioThreadPriorityIfNeeded() {
+        if (!lowLatency || audioThreadPriorityApplied) return
+        runCatching {
+            android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO)
+            audioThreadPriorityApplied = true
+        }.onFailure {
+            Log.w(TAG, "set audio thread priority failed", it)
+        }
+    }
+
+    private fun ensurePcmBufferCapacity(requiredSize: Int) {
+        if (reusablePcmBuffer.size < requiredSize) {
+            reusablePcmBuffer = ByteArray(requiredSize)
+        }
+    }
+
    private fun longBuffer(value: Long): ByteBuffer =
        ByteBuffer.allocate(8).order(ByteOrder.nativeOrder()).apply { putLong(value); flip() }

@@ -217,5 +302,8 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
        private const val CHANNELS = 2
        private const val CODEC_TIMEOUT_US = 10_000L
        private const val OPUS_SEEK_PREROLL_NS = 80_000_000L // 80 ms
+        private const val DEFAULT_BUFFER_SIZE_BYTES = 65536
+        private const val DEFAULT_FRAMES_PER_BURST = 128
+        private const val BYTES_PER_FRAME_PCM16_STEREO = 4
    }
 }
--- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/pages/MainScreen.kt
+++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/pages/MainScreen.kt
@@ -222,17 +222,20 @@ fun MainScreen() {
        .ifBlank { Scrcpy.DEFAULT_SERVER_VERSION }
    val serverRemotePath = asBundle.serverRemotePath
        .ifBlank { AppSettings.SERVER_REMOTE_PATH.defaultValue }
+    val lowLatency = asBundle.lowLatency
    val scrcpy = remember(
        appContext,
        customServerUri,
        customServerVersion,
        serverRemotePath,
+        lowLatency,
    ) {
        Scrcpy(
            appContext = appContext,
            customServerUri = customServerUri,
            serverVersion = customServerVersion,
            serverRemotePath = serverRemotePath,
+            lowLatency = lowLatency,
        ).also {
            AppRuntime.scrcpy = it
        }
--- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/pages/SettingsScreen.kt
+++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/pages/SettingsScreen.kt
@@ -37,6 +37,7 @@ import io.github.miuzarte.scrcpyforandroid.scaffolds.LazyColumn
 import io.github.miuzarte.scrcpyforandroid.scaffolds.SuperSlider
 import io.github.miuzarte.scrcpyforandroid.scaffolds.SuperTextField
 import io.github.miuzarte.scrcpyforandroid.scrcpy.Scrcpy
+import io.github.miuzarte.scrcpyforandroid.services.AppRuntime
 import io.github.miuzarte.scrcpyforandroid.services.AppUpdateChecker
 import io.github.miuzarte.scrcpyforandroid.services.LocalSnackbarController
 import io.github.miuzarte.scrcpyforandroid.storage.AppSettings
@@ -126,6 +127,7 @@ fun SettingsPage(
    val snackbar = LocalSnackbarController.current
    val navigator = LocalRootNavigator.current
    val serverPicker = LocalServerPicker.current
+    val isScrcpyStreaming = AppRuntime.scrcpy?.isStarted() == true

    val asBundleShared by appSettings.bundleState.collectAsState()
    val asBundleSharedLatest by rememberUpdatedState(asBundleShared)
@@ -264,6 +266,18 @@ fun SettingsPage(
        item {
            SectionSmallTitle("投屏")
            Card {
+                SwitchPreference(
+                    title = "低延迟音频（实验性）",
+                    summary = "启用后将尝试使用低延迟音频路径" +
+                            "\n推荐配合 RAW PCM 编解码" +
+                            "\n修改后建议划卡重启应用",
+                    checked = asBundle.lowLatency,
+                    onCheckedChange = {
+                        if (!isScrcpyStreaming)
+                            asBundle = asBundle.copy(lowLatency = it)
+                    },
+                    enabled = !isScrcpyStreaming,
+                )
                SwitchPreference(
                    title = "启用调试信息",
                    summary = "在全屏界面悬浮显示分辨率、帧率和触点信息",
--- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/scaffolds/SuperTextField.kt
+++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/scaffolds/SuperTextField.kt
@@ -18,7 +18,6 @@ import androidx.compose.ui.graphics.Color
 import androidx.compose.ui.graphics.SolidColor
 import androidx.compose.ui.text.TextStyle
 import androidx.compose.ui.text.input.VisualTransformation
-import androidx.compose.ui.unit.Dp
 import androidx.compose.ui.unit.DpSize
 import androidx.compose.ui.unit.dp
 import top.yukonga.miuix.kmp.basic.TextField
--- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/scrcpy/Scrcpy.kt
+++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/scrcpy/Scrcpy.kt
@@ -58,6 +58,7 @@ class Scrcpy(
    private val customServerUri: String? = null,
    private val serverVersion: String = DEFAULT_SERVER_VERSION,
    private val serverRemotePath: String = DEFAULT_REMOTE_PATH,
+    private val lowLatency: Boolean = false,
 ) {

    private val session = Session()
@@ -160,7 +161,7 @@ class Scrcpy(
                        info.audioCodecId.toUInt().toString(16)
                    }"
                )
-                val player = ScrcpyAudioPlayer(info.audioCodecId)
+                val player = ScrcpyAudioPlayer(appContext, info.audioCodecId, lowLatency)
                audioPlayer = player
                session.attachAudioConsumer { packet ->
                    player.feedPacket(packet.data, packet.ptsUs, packet.isConfig)
--- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/storage/AppSettings.kt
+++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/storage/AppSettings.kt
@@ -38,6 +38,10 @@ class AppSettings(context: Context) : Settings(context, "AppSettings") {
            booleanPreferencesKey("smooth_corner"),
            false,
        )
+        val LOW_LATENCY = Pair(
+            booleanPreferencesKey("low_latency"),
+            false,
+        )
        val FULLSCREEN_DEBUG_INFO = Pair(
            booleanPreferencesKey("fullscreen_debug_info"),
            false
@@ -122,6 +126,7 @@ class AppSettings(context: Context) : Settings(context, "AppSettings") {
    val smoothCorner by setting(SMOOTH_CORNER)

    // Scrcpy Settings
+    val lowLatency by setting(LOW_LATENCY)
    val fullscreenDebugInfo by setting(FULLSCREEN_DEBUG_INFO)
    val showFullscreenVirtualButtons by setting(SHOW_FULLSCREEN_VIRTUAL_BUTTONS)
    val showFullscreenFloatingButton by setting(SHOW_FULLSCREEN_FLOATING_BUTTON)
@@ -153,6 +158,7 @@ class AppSettings(context: Context) : Settings(context, "AppSettings") {
        val floatingBottomBar: Boolean,
        val floatingBottomBarBlur: Boolean,
        val smoothCorner: Boolean,
+        val lowLatency: Boolean,
        val fullscreenDebugInfo: Boolean,
        val showFullscreenVirtualButtons: Boolean,
        val showFullscreenFloatingButton: Boolean,
@@ -181,6 +187,7 @@ class AppSettings(context: Context) : Settings(context, "AppSettings") {
        bundleField(FLOATING_BOTTOM_BAR) { bundle: Bundle -> bundle.floatingBottomBar },
        bundleField(FLOATING_BOTTOM_BAR_BLUR) { bundle: Bundle -> bundle.floatingBottomBarBlur },
        bundleField(SMOOTH_CORNER) { bundle: Bundle -> bundle.smoothCorner },
+        bundleField(LOW_LATENCY) { bundle: Bundle -> bundle.lowLatency },
        bundleField(FULLSCREEN_DEBUG_INFO) { bundle: Bundle -> bundle.fullscreenDebugInfo },
        bundleField(SHOW_FULLSCREEN_VIRTUAL_BUTTONS) { bundle: Bundle -> bundle.showFullscreenVirtualButtons },
        bundleField(SHOW_FULLSCREEN_FLOATING_BUTTON) { bundle: Bundle -> bundle.showFullscreenFloatingButton },
@@ -210,6 +217,7 @@ class AppSettings(context: Context) : Settings(context, "AppSettings") {
        floatingBottomBar = preferences.read(FLOATING_BOTTOM_BAR),
        floatingBottomBarBlur = preferences.read(FLOATING_BOTTOM_BAR_BLUR),
        smoothCorner = preferences.read(SMOOTH_CORNER),
+        lowLatency = preferences.read(LOW_LATENCY),
        fullscreenDebugInfo = preferences.read(FULLSCREEN_DEBUG_INFO),
        showFullscreenVirtualButtons = preferences.read(SHOW_FULLSCREEN_VIRTUAL_BUTTONS),
        showFullscreenFloatingButton = preferences.read(SHOW_FULLSCREEN_FLOATING_BUTTON),
--- a/doc/scheme/低延迟音频_kotlin_side.md
+++ b/doc/scheme/低延迟音频_kotlin_side.md
@@ -0,0 +1,93 @@
+# 低延迟音频_kotlin_side
+
+## 使用“低延迟模式”标志
+
+```kotlin
+val audioTrack = AudioTrack.Builder()
+    .setAudioAttributes(AudioAttributes.Builder()
+        .setUsage(AudioAttributes.USAGE_GAME) // 游戏用途延迟最低
+        .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION)
+        // 请求低延迟硬件标志
+        .setFlags(AudioAttributes.FLAG_LOW_LATENCY) 
+        .build())
+    .setAudioFormat(AudioFormat.Builder()
+        .setEncoding(AudioFormat.ENCODING_PCM_16BIT) // TODO: 查看 scrcpy 官方仓库了解实际 PCM 参数 `B:\Git\scrcpy`
+        .setSampleRate(48000) // 匹配硬件原生采样率
+        .setChannelMask(AudioFormat.CHANNEL_OUT_STEREO)
+        .build())
+    // 性能模式设置为 LOW_LATENCY
+    .setBufferSizeInBytes(minBufferSize)
+    .setTransferMode(AudioTrack.MODE_STREAM)
+    .setPerformanceMode(AudioTrack.PERFORMANCE_MODE_LOW_LATENCY) 
+    .build()
+
+// audioTrack.performanceMode 检查结果
+```
+
+## 匹配“原生采样率”与“缓冲区步长”
+
+```kotlin
+val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
+
+// 获取原生采样率 (通常是 48000)
+val sampleRate = audioManager.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE)?.toInt() ?: 48000
+
+// 获取硬件每秒处理的帧数步长 (Frames Per Burst)
+val framesPerBurst = audioManager.getProperty(AudioManager.PROPERTY_OUTPUT_FRAMES_PER_BUFFER)?.toInt() ?: 128
+
+val minBuffer = AudioTrack.getMinBufferSize(
+    sampleRate,
+    AudioFormat.CHANNEL_OUT_STEREO,
+    AudioFormat.ENCODING_PCM_16BIT
+)
+
+// 16bit 2 bytes
+// stereo 2 channels
+// 双缓冲 2 个 burst
+val targetBuffer = framesPerBurst * 2 * 2 * 2
+
+val bufferSize = max(minBuffer, targetBuffer)
+```
+
+## 使用 WRITE_NON_BLOCKING // 已用
+
+## audioTrack 启动预填充静音音频 // 没必要做
+
+## 查看 scrcpy-server 回传的 PCM 具体参数 // 先搁置
+
+如果音频源支持，直接使用 AudioFormat.ENCODING_PCM_FLOAT
+
+## 获取并应用系统“最优缓冲区大小”
+
+```kotlin
+// 获取硬件每一跳的帧数 (例如 128, 256 或 512)
+val framesPerBurst = audioManager.getProperty(AudioManager.PROPERTY_OUTPUT_FRAMES_PER_BUFFER).toInt()
+// 缓冲区设置为 2 倍的 Burst 帧数（双缓冲），这是理论上的延迟平衡点
+val optimizedBufferSize = framesPerBurst * 2 * (if (isStereo) 2 else 1) * (if (isFloat) 4 else 2)
+```
+
+## 绕过音频效果器
+
+```kotlin
+.setAudioAttributes(AudioAttributes.Builder()
+    .setUsage(AudioAttributes.USAGE_GAME)
+    .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION)
+    .setFlags(AudioAttributes.FLAG_LOW_LATENCY)
+    // 告诉系统不要过各种 DSP 效果器
+    .setAllowedCapturePolicy(AudioAttributes.ALLOW_CAPTURE_BY_NONE) 
+    .build())
+```
+
+## 音频线程优先级
+
+```kotlin
+// 在负责读取 Socket 并写入 AudioTrack 的线程内执行
+// 单独线程 + 无锁队列，跟网络线程分离
+android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO)
+```
+
+## 缓冲区复用减少 GC
+
+```kotlin
+val buffer = ByteArray(FIXED_SIZE)
+```