refactor: low latency audio processing outside of JNI

README.md

@@ -23,7 +23,11 @@
 
 ## Features
 
+- 低延迟音频链路（默认未启用）
+  - 受控设备播放 `USAGE_MEDIA` 流时（[namidaco/namida](https://github.com/namidaco/namida)），两设备的音频延迟只差半拍（没有具体测量能力）
+  - 受控设备播放 `USAGE_GAME` 流时（明日方舟 Bilibili 服），仍存在 100~200ms 的有感延迟
 - 带生物认证的锁屏密码自动填充 (入口位于虚拟按钮中)
+- 多配置切换，设备绑定配置，连接后直接进入全屏
 - 可替换 scrcpy-server
 - 利用 mDNS 服务实现自动连接启用无线调试的设备、自动发现等待配对设备的IP与端口
 - 自动横竖屏切换（算吗

TODO.md

@@ -16,6 +16,5 @@
 
 顺序无关
 
-- [音频延迟](https://developer.android.com/ndk/guides/audio/audio-latency)
 - 横屏布局
 - I18N

app/build.gradle.kts

@@ -40,8 +40,8 @@ android {
         applicationId = "io.github.miuzarte.scrcpyforandroid"
         minSdk = 26
         targetSdk = 36
-        versionCode = 12
+        versionCode = 13
-        versionName = "0.1.6"
+        versionName = "0.1.7"
 
         externalNativeBuild {
             cmake {

app/src/main/java/io/github/miuzarte/scrcpyforandroid/MainActivity.kt

@@ -1,6 +1,5 @@
 package io.github.miuzarte.scrcpyforandroid
 
-import android.annotation.SuppressLint
 import android.os.Bundle
 import androidx.activity.compose.setContent
 import androidx.activity.enableEdgeToEdge

app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/ScrcpyAudioPlayer.kt

@@ -2,16 +2,19 @@ package io.github.miuzarte.scrcpyforandroid.nativecore
 
 // Go reader note: Audio output helper for scrcpy stream: decodes/plays PCM or codec audio frames.
 
+import android.content.Context
 import android.media.AudioAttributes
 import android.media.AudioFormat
 import android.media.AudioManager
 import android.media.AudioTrack
 import android.media.MediaCodec
 import android.media.MediaFormat
+import android.os.Build
 import android.util.Log
 import io.github.miuzarte.scrcpyforandroid.scrcpy.Shared.Codec
 import java.nio.ByteBuffer
 import java.nio.ByteOrder
+import kotlin.math.max
 
 /**
  * Decodes and plays scrcpy audio stream (OPUS / AAC / FLAC / RAW PCM).
@@ -19,11 +22,21 @@ import java.nio.ByteOrder
  * All [feedPacket] calls are expected from a single background thread.
  * [release] may be called from any thread.
  */
-class ScrcpyAudioPlayer(private val codecId: Int) {
+class ScrcpyAudioPlayer(
+    context: Context,
+    private val codecId: Int,
+    private val lowLatency: Boolean,
+) {
 
     private var mediaCodec: MediaCodec? = null
     private var audioTrack: AudioTrack? = null
     private val bufferInfo = MediaCodec.BufferInfo()
+    private val audioManager = context.applicationContext
+        .getSystemService(Context.AUDIO_SERVICE) as? AudioManager
+    private var reusablePcmBuffer = ByteArray(0)
+
+    @Volatile
+    private var audioThreadPriorityApplied = false
 
     @Volatile
     private var prepared = false
@@ -34,6 +47,7 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
 
     fun feedPacket(data: ByteArray, ptsUs: Long, isConfig: Boolean) {
         if (released) return
+        applyAudioThreadPriorityIfNeeded()
 
         if (isConfig) {
             Log.i(
@@ -57,7 +71,13 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
         }
 
         if (codecId == Codec.RAW.id) {
-            ensureRawAudioTrack()?.write(data, 0, data.size, AudioTrack.WRITE_NON_BLOCKING)
+            val track = ensureRawAudioTrack() ?: return
+            track.write(
+                data,
+                0,
+                data.size,
+                AudioTrack.WRITE_NON_BLOCKING,
+            )
             return
         }
 
@@ -155,20 +175,64 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
         val minBuf = AudioTrack.getMinBufferSize(
             SAMPLE_RATE, AudioFormat.CHANNEL_OUT_STEREO, AudioFormat.ENCODING_PCM_16BIT,
         ).coerceAtLeast(1)
-        return AudioTrack(
+        val framesPerBurst = audioManager
-            AudioAttributes.Builder()
+            ?.getProperty(AudioManager.PROPERTY_OUTPUT_FRAMES_PER_BUFFER)
-                .setUsage(AudioAttributes.USAGE_MEDIA)
+            ?.toIntOrNull()
-                .setContentType(AudioAttributes.CONTENT_TYPE_MOVIE)
+            ?.takeIf { it > 0 }
-                .build(),
+            ?: DEFAULT_FRAMES_PER_BURST
-            AudioFormat.Builder()
+        val nativeSampleRate = audioManager
-                .setEncoding(AudioFormat.ENCODING_PCM_16BIT)
+            ?.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE)
-                .setSampleRate(SAMPLE_RATE)
+            ?.toIntOrNull()
-                .setChannelMask(AudioFormat.CHANNEL_OUT_STEREO)
+            ?.takeIf { it > 0 }
-                .build(),
+            ?: SAMPLE_RATE
-            (minBuf * 4).coerceAtLeast(65536),
+
-            AudioTrack.MODE_STREAM,
+        val bufferSize = if (lowLatency) {
-            AudioManager.AUDIO_SESSION_ID_GENERATE,
+            val targetBuffer = framesPerBurst * 2 * BYTES_PER_FRAME_PCM16_STEREO
-        )
+            max(minBuf, targetBuffer)
+        } else {
+            (minBuf * 4).coerceAtLeast(DEFAULT_BUFFER_SIZE_BYTES)
+        }
+
+        val attributesBuilder = AudioAttributes.Builder()
+            .setUsage(
+                if (lowLatency) AudioAttributes.USAGE_GAME
+                else AudioAttributes.USAGE_MEDIA
+            )
+            .setContentType(
+                if (lowLatency) AudioAttributes.CONTENT_TYPE_SONIFICATION
+                else AudioAttributes.CONTENT_TYPE_MOVIE
+            )
+        if (lowLatency) {
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+                attributesBuilder.setAllowedCapturePolicy(AudioAttributes.ALLOW_CAPTURE_BY_NONE)
+            }
+        }
+
+        val trackBuilder = AudioTrack.Builder()
+            .setAudioAttributes(attributesBuilder.build())
+            .setAudioFormat(
+                AudioFormat.Builder()
+                    .setEncoding(AudioFormat.ENCODING_PCM_16BIT)
+                    .setSampleRate(SAMPLE_RATE)
+                    .setChannelMask(AudioFormat.CHANNEL_OUT_STEREO)
+                    .build()
+            )
+            .setBufferSizeInBytes(bufferSize)
+            .setTransferMode(AudioTrack.MODE_STREAM)
+            .setSessionId(AudioManager.AUDIO_SESSION_ID_GENERATE)
+        if (lowLatency)
+            trackBuilder.setPerformanceMode(AudioTrack.PERFORMANCE_MODE_LOW_LATENCY)
+
+        val track = trackBuilder.build()
+
+        if (lowLatency) {
+            Log.i(
+                TAG,
+                "low-latency audio requested: nativeSampleRate=$nativeSampleRate streamSampleRate=$SAMPLE_RATE " +
+                        "framesPerBurst=$framesPerBurst bufferSize=$bufferSize performanceMode=${track.performanceMode}"
+            )
+        }
+        return track
     }
 
     /**
@@ -183,10 +247,15 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
             val outBuf = codec.getOutputBuffer(idx) ?: break
             val size = bufferInfo.size
             if (size > 0) {
-                val pcm = ByteArray(size)
+                ensurePcmBufferCapacity(size)
                 outBuf.position(bufferInfo.offset)
-                outBuf.get(pcm)
+                outBuf.get(reusablePcmBuffer, 0, size)
-                track.write(pcm, 0, size, AudioTrack.WRITE_NON_BLOCKING)
+                track.write(
+                    reusablePcmBuffer,
+                    0,
+                    size,
+                    AudioTrack.WRITE_NON_BLOCKING,
+                )
             }
             codec.releaseOutputBuffer(idx, false)
             idx = codec.dequeueOutputBuffer(bufferInfo, 0L)
@@ -208,6 +277,22 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
         audioTrack = null
     }
 
+    private fun applyAudioThreadPriorityIfNeeded() {
+        if (!lowLatency || audioThreadPriorityApplied) return
+        runCatching {
+            android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO)
+            audioThreadPriorityApplied = true
+        }.onFailure {
+            Log.w(TAG, "set audio thread priority failed", it)
+        }
+    }
+
+    private fun ensurePcmBufferCapacity(requiredSize: Int) {
+        if (reusablePcmBuffer.size < requiredSize) {
+            reusablePcmBuffer = ByteArray(requiredSize)
+        }
+    }
+
     private fun longBuffer(value: Long): ByteBuffer =
         ByteBuffer.allocate(8).order(ByteOrder.nativeOrder()).apply { putLong(value); flip() }
 
@@ -217,5 +302,8 @@ class ScrcpyAudioPlayer(private val codecId: Int) {
         private const val CHANNELS = 2
         private const val CODEC_TIMEOUT_US = 10_000L
         private const val OPUS_SEEK_PREROLL_NS = 80_000_000L // 80 ms
+        private const val DEFAULT_BUFFER_SIZE_BYTES = 65536
+        private const val DEFAULT_FRAMES_PER_BURST = 128
+        private const val BYTES_PER_FRAME_PCM16_STEREO = 4
     }
 }

app/src/main/java/io/github/miuzarte/scrcpyforandroid/pages/MainScreen.kt

@@ -222,17 +222,20 @@ fun MainScreen() {
         .ifBlank { Scrcpy.DEFAULT_SERVER_VERSION }
     val serverRemotePath = asBundle.serverRemotePath
         .ifBlank { AppSettings.SERVER_REMOTE_PATH.defaultValue }
+    val lowLatency = asBundle.lowLatency
     val scrcpy = remember(
         appContext,
         customServerUri,
         customServerVersion,
         serverRemotePath,
+        lowLatency,
     ) {
         Scrcpy(
             appContext = appContext,
             customServerUri = customServerUri,
             serverVersion = customServerVersion,
             serverRemotePath = serverRemotePath,
+            lowLatency = lowLatency,
         ).also {
             AppRuntime.scrcpy = it
         }

app/src/main/java/io/github/miuzarte/scrcpyforandroid/pages/SettingsScreen.kt

@@ -37,6 +37,7 @@ import io.github.miuzarte.scrcpyforandroid.scaffolds.LazyColumn
 import io.github.miuzarte.scrcpyforandroid.scaffolds.SuperSlider
 import io.github.miuzarte.scrcpyforandroid.scaffolds.SuperTextField
 import io.github.miuzarte.scrcpyforandroid.scrcpy.Scrcpy
+import io.github.miuzarte.scrcpyforandroid.services.AppRuntime
 import io.github.miuzarte.scrcpyforandroid.services.AppUpdateChecker
 import io.github.miuzarte.scrcpyforandroid.services.LocalSnackbarController
 import io.github.miuzarte.scrcpyforandroid.storage.AppSettings
@@ -126,6 +127,7 @@ fun SettingsPage(
     val snackbar = LocalSnackbarController.current
     val navigator = LocalRootNavigator.current
     val serverPicker = LocalServerPicker.current
+    val isScrcpyStreaming = AppRuntime.scrcpy?.isStarted() == true
 
     val asBundleShared by appSettings.bundleState.collectAsState()
     val asBundleSharedLatest by rememberUpdatedState(asBundleShared)
@@ -264,6 +266,18 @@ fun SettingsPage(
         item {
             SectionSmallTitle("投屏")
             Card {
+                SwitchPreference(
+                    title = "低延迟音频（实验性）",
+                    summary = "启用后将尝试使用低延迟音频路径" +
+                            "\n推荐配合 RAW PCM 编解码" +
+                            "\n修改后建议划卡重启应用",
+                    checked = asBundle.lowLatency,
+                    onCheckedChange = {
+                        if (!isScrcpyStreaming)
+                            asBundle = asBundle.copy(lowLatency = it)
+                    },
+                    enabled = !isScrcpyStreaming,
+                )
                 SwitchPreference(
                     title = "启用调试信息",
                     summary = "在全屏界面悬浮显示分辨率、帧率和触点信息",

app/src/main/java/io/github/miuzarte/scrcpyforandroid/scaffolds/SuperTextField.kt

@@ -18,7 +18,6 @@ import androidx.compose.ui.graphics.Color
 import androidx.compose.ui.graphics.SolidColor
 import androidx.compose.ui.text.TextStyle
 import androidx.compose.ui.text.input.VisualTransformation
-import androidx.compose.ui.unit.Dp
 import androidx.compose.ui.unit.DpSize
 import androidx.compose.ui.unit.dp
 import top.yukonga.miuix.kmp.basic.TextField

app/src/main/java/io/github/miuzarte/scrcpyforandroid/scrcpy/Scrcpy.kt

@@ -58,6 +58,7 @@ class Scrcpy(
     private val customServerUri: String? = null,
     private val serverVersion: String = DEFAULT_SERVER_VERSION,
     private val serverRemotePath: String = DEFAULT_REMOTE_PATH,
+    private val lowLatency: Boolean = false,
 ) {
 
     private val session = Session()
@@ -160,7 +161,7 @@ class Scrcpy(
                         info.audioCodecId.toUInt().toString(16)
                     }"
                 )
-                val player = ScrcpyAudioPlayer(info.audioCodecId)
+                val player = ScrcpyAudioPlayer(appContext, info.audioCodecId, lowLatency)
                 audioPlayer = player
                 session.attachAudioConsumer { packet ->
                     player.feedPacket(packet.data, packet.ptsUs, packet.isConfig)

app/src/main/java/io/github/miuzarte/scrcpyforandroid/storage/AppSettings.kt

@@ -38,6 +38,10 @@ class AppSettings(context: Context) : Settings(context, "AppSettings") {
             booleanPreferencesKey("smooth_corner"),
             false,
         )
+        val LOW_LATENCY = Pair(
+            booleanPreferencesKey("low_latency"),
+            false,
+        )
         val FULLSCREEN_DEBUG_INFO = Pair(
             booleanPreferencesKey("fullscreen_debug_info"),
             false
@@ -122,6 +126,7 @@ class AppSettings(context: Context) : Settings(context, "AppSettings") {
     val smoothCorner by setting(SMOOTH_CORNER)
 
     // Scrcpy Settings
+    val lowLatency by setting(LOW_LATENCY)
     val fullscreenDebugInfo by setting(FULLSCREEN_DEBUG_INFO)
     val showFullscreenVirtualButtons by setting(SHOW_FULLSCREEN_VIRTUAL_BUTTONS)
     val showFullscreenFloatingButton by setting(SHOW_FULLSCREEN_FLOATING_BUTTON)
@@ -153,6 +158,7 @@ class AppSettings(context: Context) : Settings(context, "AppSettings") {
         val floatingBottomBar: Boolean,
         val floatingBottomBarBlur: Boolean,
         val smoothCorner: Boolean,
+        val lowLatency: Boolean,
         val fullscreenDebugInfo: Boolean,
         val showFullscreenVirtualButtons: Boolean,
         val showFullscreenFloatingButton: Boolean,
@@ -181,6 +187,7 @@ class AppSettings(context: Context) : Settings(context, "AppSettings") {
         bundleField(FLOATING_BOTTOM_BAR) { bundle: Bundle -> bundle.floatingBottomBar },
         bundleField(FLOATING_BOTTOM_BAR_BLUR) { bundle: Bundle -> bundle.floatingBottomBarBlur },
         bundleField(SMOOTH_CORNER) { bundle: Bundle -> bundle.smoothCorner },
+        bundleField(LOW_LATENCY) { bundle: Bundle -> bundle.lowLatency },
         bundleField(FULLSCREEN_DEBUG_INFO) { bundle: Bundle -> bundle.fullscreenDebugInfo },
         bundleField(SHOW_FULLSCREEN_VIRTUAL_BUTTONS) { bundle: Bundle -> bundle.showFullscreenVirtualButtons },
         bundleField(SHOW_FULLSCREEN_FLOATING_BUTTON) { bundle: Bundle -> bundle.showFullscreenFloatingButton },
@@ -210,6 +217,7 @@ class AppSettings(context: Context) : Settings(context, "AppSettings") {
         floatingBottomBar = preferences.read(FLOATING_BOTTOM_BAR),
         floatingBottomBarBlur = preferences.read(FLOATING_BOTTOM_BAR_BLUR),
         smoothCorner = preferences.read(SMOOTH_CORNER),
+        lowLatency = preferences.read(LOW_LATENCY),
         fullscreenDebugInfo = preferences.read(FULLSCREEN_DEBUG_INFO),
         showFullscreenVirtualButtons = preferences.read(SHOW_FULLSCREEN_VIRTUAL_BUTTONS),
         showFullscreenFloatingButton = preferences.read(SHOW_FULLSCREEN_FLOATING_BUTTON),

doc/scheme/低延迟音频_kotlin_side.md

@@ -0,0 +1,93 @@
+# 低延迟音频_kotlin_side
+
+## 使用“低延迟模式”标志
+
+```kotlin
+val audioTrack = AudioTrack.Builder()
+    .setAudioAttributes(AudioAttributes.Builder()
+        .setUsage(AudioAttributes.USAGE_GAME) // 游戏用途延迟最低
+        .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION)
+        // 请求低延迟硬件标志
+        .setFlags(AudioAttributes.FLAG_LOW_LATENCY) 
+        .build())
+    .setAudioFormat(AudioFormat.Builder()
+        .setEncoding(AudioFormat.ENCODING_PCM_16BIT) // TODO: 查看 scrcpy 官方仓库了解实际 PCM 参数 `B:\Git\scrcpy`
+        .setSampleRate(48000) // 匹配硬件原生采样率
+        .setChannelMask(AudioFormat.CHANNEL_OUT_STEREO)
+        .build())
+    // 性能模式设置为 LOW_LATENCY
+    .setBufferSizeInBytes(minBufferSize)
+    .setTransferMode(AudioTrack.MODE_STREAM)
+    .setPerformanceMode(AudioTrack.PERFORMANCE_MODE_LOW_LATENCY) 
+    .build()
+
+// audioTrack.performanceMode 检查结果
+```
+
+## 匹配“原生采样率”与“缓冲区步长”
+
+```kotlin
+val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
+
+// 获取原生采样率 (通常是 48000)
+val sampleRate = audioManager.getProperty(AudioManager.PROPERTY_OUTPUT_SAMPLE_RATE)?.toInt() ?: 48000
+
+// 获取硬件每秒处理的帧数步长 (Frames Per Burst)
+val framesPerBurst = audioManager.getProperty(AudioManager.PROPERTY_OUTPUT_FRAMES_PER_BUFFER)?.toInt() ?: 128
+
+val minBuffer = AudioTrack.getMinBufferSize(
+    sampleRate,
+    AudioFormat.CHANNEL_OUT_STEREO,
+    AudioFormat.ENCODING_PCM_16BIT
+)
+
+// 16bit 2 bytes
+// stereo 2 channels
+// 双缓冲 2 个 burst
+val targetBuffer = framesPerBurst * 2 * 2 * 2
+
+val bufferSize = max(minBuffer, targetBuffer)
+```
+
+## 使用 WRITE_NON_BLOCKING // 已用
+
+## audioTrack 启动预填充静音音频 // 没必要做
+
+## 查看 scrcpy-server 回传的 PCM 具体参数 // 先搁置
+
+如果音频源支持，直接使用 AudioFormat.ENCODING_PCM_FLOAT
+
+## 获取并应用系统“最优缓冲区大小”
+
+```kotlin
+// 获取硬件每一跳的帧数 (例如 128, 256 或 512)
+val framesPerBurst = audioManager.getProperty(AudioManager.PROPERTY_OUTPUT_FRAMES_PER_BUFFER).toInt()
+// 缓冲区设置为 2 倍的 Burst 帧数（双缓冲），这是理论上的延迟平衡点
+val optimizedBufferSize = framesPerBurst * 2 * (if (isStereo) 2 else 1) * (if (isFloat) 4 else 2)
+```
+
+## 绕过音频效果器
+
+```kotlin
+.setAudioAttributes(AudioAttributes.Builder()
+    .setUsage(AudioAttributes.USAGE_GAME)
+    .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION)
+    .setFlags(AudioAttributes.FLAG_LOW_LATENCY)
+    // 告诉系统不要过各种 DSP 效果器
+    .setAllowedCapturePolicy(AudioAttributes.ALLOW_CAPTURE_BY_NONE) 
+    .build())
+```
+
+## 音频线程优先级
+
+```kotlin
+// 在负责读取 Socket 并写入 AudioTrack 的线程内执行
+// 单独线程 + 无锁队列，跟网络线程分离
+android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO)
+```
+
+## 缓冲区复用减少 GC
+
+```kotlin
+val buffer = ByteArray(FIXED_SIZE)
+```