diff --git a/.gitignore b/.gitignore index 65fd28b..989dea9 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,5 @@ abandoned_assets/ # scrcpy-server binary (downloaded by Gradle during build) app/src/main/assets/bin/scrcpy-server-v* + +docs/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6df5e25..c9b7c4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Change Log +## 0.4.5 + +- 重构: 改了好多不知道怎么写，总之修复了一些已知问题 + +## 0.4.4 + +- 修复: 画中画初始化空指针 + ## 0.4.3 - 改进: 对控制通道增加流控选项 diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 09d2790..880bbe2 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -60,7 +60,7 @@ android { minSdk = 26 targetSdk = 37 versionCode = 35 - versionName = "0.4.4" + versionName = "0.4.5_pre1" externalNativeBuild { cmake { diff --git a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/NativeCoreFacade.kt b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/NativeCoreFacade.kt index ff268cb..9c827dd 100644 --- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/NativeCoreFacade.kt +++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/NativeCoreFacade.kt @@ -1,58 +1,48 @@ package io.github.miuzarte.scrcpyforandroid -import android.os.Handler -import android.os.Looper import android.util.Log import android.view.Surface -import io.github.miuzarte.scrcpyforandroid.nativecore.AnnexBDecoder import io.github.miuzarte.scrcpyforandroid.nativecore.PersistentVideoRenderer +import io.github.miuzarte.scrcpyforandroid.nativecore.VideoDecoderController import io.github.miuzarte.scrcpyforandroid.scrcpy.Scrcpy -import io.github.miuzarte.scrcpyforandroid.scrcpy.Shared.Codec import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import java.util.ArrayDeque -import java.util.concurrent.CopyOnWriteArraySet /** - * Facade that centralizes video rendering. + * Facade that centralizes video rendering and decoder management. * - * Provides helpers for: - * - Surface/Decoder management for video rendering - * - Video size and FPS monitoring + * Acts as a thin front-end over [VideoDecoderController] (decoder lifecycle, bootstrap + * cache, error detection) and [PersistentVideoRenderer] (EGL / surface management). + * + * The facade owns the session lifecycle mutex and coordinates surface attach/detach + * with decoder creation. It also publishes video size / FPS to listeners. */ object NativeCoreFacade { private val sessionLifecycleMutex = Mutex() private val lifecycleScope = CoroutineScope(SupervisorJob() + Dispatchers.Main) private val renderer = PersistentVideoRenderer() + private val controller = VideoDecoderController(renderer) + + @Volatile private var activeSurfaceId: Int? = null - private var decoder: AnnexBDecoder? = null - private val videoSizeListeners = CopyOnWriteArraySet<(Int, Int) -> Unit>() - private val videoFpsListeners = CopyOnWriteArraySet<(Float) -> Unit>() - private val mainHandler = Handler(Looper.getMainLooper()) - private val bootstrapLock = Any() - private val bootstrapPackets = ArrayDeque() @Volatile private var recordingSurfaceAttached = false - @Volatile - private var latestConfigPacket: CachedPacket? = null - private var packetCount: Long = 0 - - @Volatile - private var currentSessionInfo: Scrcpy.Session.SessionInfo? = null - // Reference to Scrcpy for reading currentSessionState (set by onScrcpySessionStarted) @Volatile private var scrcpyRef: Scrcpy? = null + @Volatile + private var packetCount: Long = 0 + suspend fun close() { sessionLifecycleMutex.withLock { - releaseAllDecoders() + controller.releaseAll() renderer.release() } } @@ -71,30 +61,20 @@ object NativeCoreFacade { return } val newId = System.identityHashCode(surface) - if (activeSurfaceId == newId && decoder != null) { + if (activeSurfaceId == newId && controller.isDecoderUsable()) { return } Log.i(TAG, "attachVideoSurface(): surfaceId=$newId oldSurfaceId=$activeSurfaceId") activeSurfaceId = newId - renderer.attachDisplaySurface(surface) - val session = currentSessionInfo ?: return - val currentDecoder = decoder - if (currentDecoder != null) { + controller.attachDisplaySurface(surface) + val session = controller.getCurrentSessionInfo() ?: return + if (controller.isDecoderUsable()) { Log.i(TAG, "attachVideoSurface(): try switch decoder output to persistent surface") - val switched = currentDecoder.switchOutputSurface(renderer.getDecoderSurface()) - Log.i(TAG, "attachVideoSurface(): switchOutputSurface success=$switched") - if (switched) { + if (controller.trySwitchDecoderSurface()) { return } } - if (session.width <= 0 || session.height <= 0) { - Log.i( - TAG, - "attachVideoSurface(): defer decoder, session size not yet known (${session.width}x${session.height})", - ) - return - } - createOrReplaceDecoder(session) + controller.ensureDecoder(session) } } @@ -123,12 +103,7 @@ object NativeCoreFacade { "detachVideoSurface(): surfaceId=$requestId releaseDecoder=$releaseDecoder", ) activeSurfaceId = null - renderer.detachDisplaySurface(surface, releaseSurface = false) - if (releaseDecoder) { - Log.i(TAG, "detachVideoSurface(): releasing decoder with destroyed surface") - decoder?.release() - decoder = null - } + controller.detachDisplaySurface(surface, releaseDecoder) } } @@ -140,10 +115,10 @@ object NativeCoreFacade { ) { sessionLifecycleMutex.withLock { recordingSurfaceAttached = true - renderer.attachRecordSurface(surface, width, height, onFrameRendered) - val session = currentSessionInfo - if (session != null && decoder == null) { - createOrReplaceDecoder(session) + controller.attachRecordSurface(surface, width, height, onFrameRendered) + val session = controller.getCurrentSessionInfo() + if (session != null && !controller.isDecoderUsable()) { + controller.ensureDecoder(session) } } } @@ -154,18 +129,17 @@ object NativeCoreFacade { ) { sessionLifecycleMutex.withLock { recordingSurfaceAttached = false - renderer.detachRecordSurface(surface, releaseSurface) + controller.detachRecordSurface(surface, releaseSurface) if (activeSurfaceId == null) { - decoder?.release() - decoder = null + controller.releaseAll() } } } - fun addVideoSizeListener(listener: (Int, Int) -> Unit) = videoSizeListeners.add(listener) - fun removeVideoSizeListener(listener: (Int, Int) -> Unit) = videoSizeListeners.remove(listener) - fun addVideoFpsListener(listener: (Float) -> Unit) = videoFpsListeners.add(listener) - fun removeVideoFpsListener(listener: (Float) -> Unit) = videoFpsListeners.remove(listener) + fun addVideoSizeListener(listener: (Int, Int) -> Unit) = controller.addVideoSizeListener(listener) + fun removeVideoSizeListener(listener: (Int, Int) -> Unit) = controller.removeVideoSizeListener(listener) + fun addVideoFpsListener(listener: (Float) -> Unit) = controller.addVideoFpsListener(listener) + fun removeVideoFpsListener(listener: (Float) -> Unit) = controller.removeVideoFpsListener(listener) /** * Called by Scrcpy.kt when a session starts. @@ -177,41 +151,20 @@ object NativeCoreFacade { scrcpy: Scrcpy, ) = sessionLifecycleMutex.withLock { scrcpyRef = scrcpy - currentSessionInfo = session - releaseAllDecoders() - synchronized(bootstrapLock) { - bootstrapPackets.clear() - latestConfigPacket = null - } + controller.releaseAll() + controller.resetBootstrap() if (activeSurfaceId != null || recordingSurfaceAttached) { // v4.0: width/height come from first video session packet, not from initial metadata if (session.width > 0 && session.height > 0) { Log.i(TAG, "onScrcpySessionStarted(): bind decoder to persistent surface") - createOrReplaceDecoder(session) + controller.ensureDecoder(session) } else { Log.i(TAG, "onScrcpySessionStarted(): defer decoder until first video session packet (v4.0)") } } packetCount = 0 sessionMgr.attachVideoConsumer { packet -> - cacheBootstrapPacket(packet) - packetCount += 1 - if (packetCount == 1L || packetCount % 120L == 0L) { - Log.i( - TAG, - "videoFeed(): packets=$packetCount key=${packet.isKeyFrame} cfg=${packet.isConfig} decoder=${decoder != null}", - ) - } - - val dec = decoder ?: return@attachVideoConsumer - runCatching { - dec.feedAnnexB( - packet.data, - packet.ptsUs, - packet.isKeyFrame, - packet.isConfig, - ) - } + cacheAndFeed(packet) } } @@ -225,23 +178,7 @@ object NativeCoreFacade { val scrcpy = scrcpyRef ?: return@withLock val info = scrcpy.currentSessionState.value ?: return@withLock if (info.width <= 0 || info.height <= 0) return@withLock - - if (decoder == null) { - // Initial creation (v4.0: deferred until first session packet) - Log.i(TAG, "onVideoSizeChanged(): create decoder ${info.width}x${info.height}") - currentSessionInfo = info - createOrReplaceDecoder(info) - } else if (currentSessionInfo != null && - (info.width != currentSessionInfo!!.width || info.height != currentSessionInfo!!.height) - ) { - // Flex display: rebuild decoder on size change - Log.i( - TAG, - "onVideoSizeChanged(): rebuild decoder ${currentSessionInfo!!.width}x${currentSessionInfo!!.height} → ${info.width}x${info.height}", - ) - currentSessionInfo = info - createOrReplaceDecoder(info) - } + controller.rebuildDecoderForSize(info) } } } @@ -251,150 +188,24 @@ object NativeCoreFacade { * Cleans up decoders and resets state. */ suspend fun onScrcpySessionStopped() = sessionLifecycleMutex.withLock { - releaseAllDecoders() - synchronized(bootstrapLock) { - bootstrapPackets.clear() - latestConfigPacket = null - } + controller.releaseAll() scrcpyRef = null - currentSessionInfo = null recordingSurfaceAttached = false } + private fun cacheAndFeed(packet: Scrcpy.Session.VideoPacket) { + packetCount += 1 + if (packetCount == 1L || packetCount % 120L == 0L) { + Log.i( + TAG, + "videoFeed(): packets=$packetCount key=${packet.isKeyFrame} cfg=${packet.isConfig} usable=${controller.isDecoderUsable()}", + ) + } + val usable = controller.feed(packet) + if (!usable) { + Log.e(TAG, "videoFeed(): decoder became unusable after feed, packets=$packetCount") + } + } + private const val TAG = "NativeCoreFacade" - private const val MAX_BOOTSTRAP_PACKETS = 90 - - private data class CachedPacket( - val data: ByteArray, - val ptsUs: Long, - val isConfig: Boolean, - val isKeyFrame: Boolean, - ) { - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (javaClass != other?.javaClass) return false - - other as CachedPacket - - if (ptsUs != other.ptsUs) return false - if (isConfig != other.isConfig) return false - if (isKeyFrame != other.isKeyFrame) return false - if (!data.contentEquals(other.data)) return false - - return true - } - - override fun hashCode(): Int { - var result = ptsUs.hashCode() - result = 31 * result + isConfig.hashCode() - result = 31 * result + isKeyFrame.hashCode() - result = 31 * result + data.contentHashCode() - return result - } - } - - /** - * Create or replace the active decoder bound to [surface] for [session]. - * - * - Chooses MIME type from `session.codec` and constructs an [AnnexBDecoder]. - * - The decoder's `onOutputSizeChanged` callback publishes size changes to - * registered listeners on the main thread. - * - Newly created decoders are fed with any cached bootstrap packets to allow - * faster playback startup. - */ - private fun createOrReplaceDecoder(session: Scrcpy.Session.SessionInfo) { - val surface = renderer.getDecoderSurface() - decoder?.release() - decoder = null - Log.i( - TAG, - "createOrReplaceDecoder(): " + - "codec=${session.codec?.string ?: "null"}, " + - "size=${session.width}x${session.height}, " + - "persistent=true", - ) - val newDecoder = AnnexBDecoder( - width = session.width, - height = session.height, - outputSurface = surface, - mimeType = when (session.codec) { - Codec.H264 -> "video/avc" - Codec.H265 -> "video/hevc" - Codec.AV1 -> "video/av01" - else -> "video/avc" - }, - onOutputSizeChanged = { width, height -> - val current = currentSessionInfo - if (current == null || (current.width == width && current.height == height)) { - return@AnnexBDecoder - } - Log.i( - TAG, - "videoSizeChanged(): ${current.width}x${current.height} -> ${width}x${height}", - ) - currentSessionInfo = current.copy(width = width, height = height) - mainHandler.post { - videoSizeListeners.forEach { listener -> - runCatching { listener(width, height) } - } - } - }, - onFpsUpdated = { fps -> - mainHandler.post { - videoFpsListeners.forEach { listener -> - runCatching { listener(fps) } - } - } - }, - ) - decoder = newDecoder - replayBootstrapPackets(newDecoder) - } - - private fun replayBootstrapPackets(decoder: AnnexBDecoder) { - val snapshot = synchronized(bootstrapLock) { bootstrapPackets.toList() } - if (snapshot.isEmpty()) { - return - } - Log.i(TAG, "replayBootstrapPackets(): count=${snapshot.size}") - snapshot.forEach { packet -> - runCatching { - decoder.feedAnnexB(packet.data, packet.ptsUs, packet.isKeyFrame, packet.isConfig) - } - } - } - - private fun cacheBootstrapPacket(packet: Scrcpy.Session.VideoPacket) { - val cached = CachedPacket( - data = packet.data.copyOf(), - ptsUs = packet.ptsUs, - isConfig = packet.isConfig, - isKeyFrame = packet.isKeyFrame, - ) - synchronized(bootstrapLock) { - if (cached.isConfig) { - latestConfigPacket = cached - bootstrapPackets.clear() - bootstrapPackets.addLast(cached) - return - } - - if (cached.isKeyFrame) { - bootstrapPackets.clear() - latestConfigPacket?.let { bootstrapPackets.addLast(it) } - bootstrapPackets.addLast(cached) - return - } - - while (bootstrapPackets.size >= MAX_BOOTSTRAP_PACKETS) { - bootstrapPackets.removeFirst() - } - bootstrapPackets.addLast(cached) - } - } - - private fun releaseAllDecoders() { - runCatching { decoder?.release() } - decoder = null - } } diff --git a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/AnnexBDecoder.kt b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/AnnexBDecoder.kt index 5182ce6..b6c4796 100644 --- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/AnnexBDecoder.kt +++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/AnnexBDecoder.kt @@ -41,9 +41,21 @@ class AnnexBDecoder( @Volatile private var released = false + /** + * Set when the codec reports an unrecoverable error (either during configure/start + * or at runtime via [MediaCodec.CodecException] / [IllegalStateException]). + * + * Once non-null, [isUsable] returns false and subsequent [feedAnnexB] calls are no-ops. + * The controller ([VideoDecoderController]) checks this to trigger session restart. + */ + @Volatile + internal var codecError: Throwable? = null + private set + init { if (!outputSurface.isValid) { - throw IllegalStateException("Cannot initialize decoder: output surface is not valid") + throw DecoderException(mimeType, width, height, + IllegalStateException("Output surface is not valid")) } val format = MediaFormat.createVideoFormat(mimeType, width, height) if (sps != null) { @@ -52,8 +64,14 @@ class AnnexBDecoder( if (pps != null) { format.setByteBuffer("csd-1", java.nio.ByteBuffer.wrap(pps)) } - codec.configure(format, outputSurface, null, 0) - codec.start() + try { + codec.configure(format, outputSurface, null, 0) + codec.start() + } catch (e: Exception) { + runCatching { codec.release() } + codecError = e + throw DecoderException(mimeType, width, height, e) + } } /** @@ -68,7 +86,7 @@ class AnnexBDecoder( */ @Synchronized fun feedAnnexB(data: ByteArray, ptsUs: Long, isKeyFrame: Boolean, isConfig: Boolean = false) { - if (released) { + if (released || codecError != null) { return } runCatching { @@ -107,9 +125,13 @@ class AnnexBDecoder( } drainOutput() }.onFailure { - Log.w( + // MediaCodec.CodecException and IllegalStateException indicate the codec is in an + // unrecoverable error state (e.g. MTK OMX fatal error after surface conflicts). + // Record the error so isUsable() returns false and the controller can react. + codecError = it + Log.e( TAG, - "feed failed: mime=$decoderMime size=${data.size} key=$isKeyFrame cfg=$isConfig", + "feed failed (codec marked unusable): mime=$decoderMime size=${data.size} key=$isKeyFrame cfg=$isConfig", it, ) } @@ -145,6 +167,12 @@ class AnnexBDecoder( runCatching { codec.release() } } + /** + * Returns true when the decoder is still alive and the codec has not reported + * an unrecoverable error. + */ + fun isUsable(): Boolean = !released && codecError == null + private fun drainOutput() { while (true) { val outIndex = codec.dequeueOutputBuffer(bufferInfo, OUTPUT_TIMEOUT_US) diff --git a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/DecoderException.kt b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/DecoderException.kt new file mode 100644 index 0000000..1ed08cd --- /dev/null +++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/DecoderException.kt @@ -0,0 +1,17 @@ +package io.github.miuzarte.scrcpyforandroid.nativecore + +/** + * Thrown when a [MediaCodec] decoder fails to initialize or enters an unrecoverable error state. + * + * Carries the MIME type and target dimensions so callers (e.g. [VideoDecoderController]) + * can decide whether to downgrade the video size and restart the session. + */ +class DecoderException( + val mime: String, + val width: Int, + val height: Int, + cause: Throwable? = null, +) : RuntimeException( + "Decoder failed for mime=$mime size=${width}x${height}", + cause, +) diff --git a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/PersistentVideoRenderer.kt b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/PersistentVideoRenderer.kt index a560b9f..e6a45e1 100644 --- a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/PersistentVideoRenderer.kt +++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/PersistentVideoRenderer.kt @@ -62,6 +62,75 @@ class PersistentVideoRenderer { } } + /** + * Atomically tear down and recreate the decoder-side surface (SurfaceTexture + OES texture + * + Surface), returning a fresh [Surface] for a new MediaCodec to render into. + * + * This is needed because some OMX decoders (notably MTK) fail when a new codec is configured + * against a surface that still has pending buffers from a previously released codec. + * By recreating the underlying SurfaceTexture + texture, the new codec gets a clean buffer + * queue with no leftover state. + * + * - Must be called after the old decoder has been released, so no producer is writing to + * the old surface at the same time. + * - Runs on the renderer HandlerThread (via [handler]) so it is serialized with [drawFrame]. + * - EGL context, display/record EGL surfaces, and the shader program are preserved; only + * `oesTextureId` / `decoderSurfaceTexture` / `decoderSurface` are recreated. + */ + fun recreateDecoderSurface(): Surface { + ensureInitialized() + val latch = java.util.concurrent.CountDownLatch(1) + var result: Surface? = null + handler.post { + try { + if (released) { + error("renderer already released") + } + recreateDecoderSurfaceLocked() + result = decoderSurface + } finally { + latch.countDown() + } + } + latch.await() + return result ?: error("failed to recreate decoder surface") + } + + private fun recreateDecoderSurfaceLocked() { + Log.i(tag, "recreateDecoderSurfaceLocked(): rebuilding decoder surface") + + // Tear down old decoder-side resources. drawFrame() won't run concurrently + // because we're on the same HandlerThread. + runCatching { decoderSurface?.release() } + decoderSurface = null + runCatching { decoderSurfaceTexture?.release() } + decoderSurfaceTexture = null + if (oesTextureId != 0) { + GLES20.glDeleteTextures(1, intArrayOf(oesTextureId), 0) + oesTextureId = 0 + } + + // Recreate from scratch (mirrors initializeLocked lines 243-259) + oesTextureId = createExternalTexture() + decoderSurfaceTexture = SurfaceTexture(oesTextureId).apply { + setOnFrameAvailableListener( + { + val n = frameAvailableCount.incrementAndGet() + if (n == 1L || n % 120L == 0L) { + Log.i( + tag, + "onFrameAvailable(): available=$n consumed=${frameConsumedCount.get()} rendered=${frameRenderedCount.get()} display=${displaySurfaceId != null}", + ) + } + drawFrame() + }, + handler, + ) + } + decoderSurface = Surface(decoderSurfaceTexture) + Log.i(tag, "recreateDecoderSurfaceLocked(): decoder surface rebuilt, texture=$oesTextureId") + } + fun attachDisplaySurface(surface: Surface) { ensureInitialized() val newId = System.identityHashCode(surface) diff --git a/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/VideoDecoderController.kt b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/VideoDecoderController.kt new file mode 100644 index 0000000..ea026ff --- /dev/null +++ b/app/src/main/java/io/github/miuzarte/scrcpyforandroid/nativecore/VideoDecoderController.kt @@ -0,0 +1,422 @@ +package io.github.miuzarte.scrcpyforandroid.nativecore + +import android.os.Handler +import android.os.Looper +import android.util.Log +import android.view.Surface +import io.github.miuzarte.scrcpyforandroid.scrcpy.Scrcpy +import io.github.miuzarte.scrcpyforandroid.scrcpy.Shared.Codec +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext +import java.util.ArrayDeque +import java.util.concurrent.CopyOnWriteArraySet + +/** + * Owns the video [AnnexBDecoder] lifecycle and all state surrounding it: + * + * - Creating / rebuilding / releasing the decoder bound to the persistent renderer surface. + * - Caching and replaying bootstrap packets (config + keyframe + following frames) so a + * freshly created decoder can resume playback quickly. + * - Detecting decoder errors (both construction-time [DecoderException] and runtime + * codec errors) and exposing them via [isDecoderUsable] / the return value of [feed] + * for the facade to react. + * - Publishing video size / FPS changes to registered listeners on the main thread. + * + * Threading: + * - All public methods are safe to call from any thread; mutations are guarded by + * [controllerLock] for synchronous methods and by the facade's session mutex for + * suspend methods. + * - The decoder itself is single-threaded (AnnexBDecoder is @Synchronized). + * + * This class does **not** decide session restart policy; it reports errors and lets + * [NativeCoreFacade] decide whether to restart the scrcpy session. + */ +internal class VideoDecoderController( + private val renderer: PersistentVideoRenderer, +) { + private val mainHandler = Handler(Looper.getMainLooper()) + + private val controllerLock = Any() + private var decoder: AnnexBDecoder? = null + + @Volatile + private var currentSessionInfo: Scrcpy.Session.SessionInfo? = null + + private val videoSizeListeners = CopyOnWriteArraySet<(Int, Int) -> Unit>() + private val videoFpsListeners = CopyOnWriteArraySet<(Float) -> Unit>() + + private val bootstrapLock = Any() + private val bootstrapPackets = ArrayDeque() + + @Volatile + private var latestConfigPacket: CachedPacket? = null + + /** + * The session info currently associated with the decoder (or the most recently released + * one). Returns null if no session has ever been bound. + * + * The facade uses this instead of maintaining its own copy, so there is a single source + * of truth kept in sync by [ensureDecoder] / [rebuildDecoderForSize] / the + * `onOutputSizeChanged` callback. + */ + fun getCurrentSessionInfo(): Scrcpy.Session.SessionInfo? = currentSessionInfo + + // ---------- Listeners ---------- + + fun addVideoSizeListener(listener: (Int, Int) -> Unit) = videoSizeListeners.add(listener) + fun removeVideoSizeListener(listener: (Int, Int) -> Unit) = videoSizeListeners.remove(listener) + fun addVideoFpsListener(listener: (Float) -> Unit) = videoFpsListeners.add(listener) + fun removeVideoFpsListener(listener: (Float) -> Unit) = videoFpsListeners.remove(listener) + + // ---------- Session lifecycle ---------- + + /** + * Tear down everything: release the decoder and drop bootstrap state. + * Called by the facade when a scrcpy session stops or when the controller is closed. + */ + fun releaseAll() { + synchronized(controllerLock) { + runCatching { decoder?.release() } + decoder = null + currentSessionInfo = null + } + synchronized(bootstrapLock) { + bootstrapPackets.clear() + latestConfigPacket = null + } + } + + /** + * Reset bootstrap state without touching the decoder (used on session start before a new + * decoder is created). + */ + fun resetBootstrap() { + synchronized(bootstrapLock) { + bootstrapPackets.clear() + latestConfigPacket = null + } + } + + // ---------- Surface management ---------- + + /** + * Try to switch the active decoder's output surface to the renderer's current decoder + * surface. Returns true if the switch succeeded (or was unnecessary), false if the + * caller needs to recreate the decoder. + */ + fun trySwitchDecoderSurface(): Boolean { + synchronized(controllerLock) { + val current = decoder ?: return false + val switched = current.switchOutputSurface(renderer.getDecoderSurface()) + Log.i(TAG, "trySwitchDecoderSurface(): success=$switched") + return switched + } + } + + /** + * Attach the display surface to the renderer. Does not touch the decoder directly. + */ + fun attachDisplaySurface(surface: Surface) { + renderer.attachDisplaySurface(surface) + } + + /** + * Detach the display surface. When [releaseDecoder] is true the decoder is also released + * because the backing surface is being destroyed. + */ + fun detachDisplaySurface(surface: Surface?, releaseDecoder: Boolean) { + renderer.detachDisplaySurface(surface, releaseSurface = false) + if (releaseDecoder) { + synchronized(controllerLock) { + Log.i(TAG, "detachDisplaySurface(): releasing decoder with destroyed surface") + runCatching { decoder?.release() } + decoder = null + } + } + } + + /** + * Attach a recording surface to the renderer. If no decoder exists yet but session info + * is known, a new decoder is created. + */ + fun attachRecordSurface( + surface: Surface, + width: Int, + height: Int, + onFrameRendered: ((Long) -> Unit)?, + ) { + renderer.attachRecordSurface(surface, width, height, onFrameRendered) + } + + fun detachRecordSurface(surface: Surface?, releaseSurface: Boolean) { + renderer.detachRecordSurface(surface, releaseSurface) + } + + // ---------- Decoder creation / rebuild ---------- + + /** + * Create a decoder for [session] if none exists. Should be called when a display or + * recording surface is attached and the session size is known. + * + * Returns true if a decoder exists after the call (either pre-existing or newly created). + */ + fun ensureDecoder(session: Scrcpy.Session.SessionInfo): Boolean { + synchronized(controllerLock) { + val current = decoder + if (current != null && current.isUsable()) return true + if (session.width <= 0 || session.height <= 0) { + Log.i( + TAG, + "ensureDecoder(): defer decoder, session size not yet known (${session.width}x${session.height})", + ) + return false + } + // If a damaged decoder exists, release it first. + if (current != null) { + Log.i(TAG, "ensureDecoder(): releasing damaged decoder before rebuild") + runCatching { current.release() } + decoder = null + } + currentSessionInfo = session + createOrReplaceDecoderLocked(session, recreateSurface = false) + return decoder != null + } + } + + /** + * Rebuild the decoder when the video size changes (e.g. flex-display rotation). + * + * - Recreates the persistent decoder surface to avoid OMX buffer conflicts on chips + * like MTK that fail when a new codec is configured against a surface with leftover + * buffers from a previously released codec. + * - Caches the new session info so future size-change checks can compare. + * + * Returns true if the decoder was rebuilt, false if skipped (same size, no decoder, etc.). + */ + suspend fun rebuildDecoderForSize(session: Scrcpy.Session.SessionInfo): Boolean { + val current = synchronized(controllerLock) { currentSessionInfo } + if (current != null && current.width == session.width && current.height == session.height) { + return false + } + + val needRecreateSurface: Boolean + synchronized(controllerLock) { + if (decoder == null) { + // Initial creation (deferred until first session packet). + Log.i(TAG, "rebuildDecoderForSize(): create decoder ${session.width}x${session.height}") + currentSessionInfo = session + createOrReplaceDecoderLocked(session, recreateSurface = false) + return decoder != null + } + // Release the old decoder now so OMX stops writing to the surface, then + // drop the lock for the blocking delay + surface recreation below. + Log.i( + TAG, + "rebuildDecoderForSize(): rebuild decoder ${current?.width}x${current?.height} -> ${session.width}x${session.height}", + ) + currentSessionInfo = session + runCatching { decoder?.release() } + decoder = null + needRecreateSurface = true + } + + // Give OMX time to return buffers to the old native window before we tear down + // the SurfaceTexture, then recreate the surface. Both block, so run on IO. + // + // No lock is held here: the old decoder is already released and `decoder` is null, + // so feed() will no-op. The facade's sessionLifecycleMutex ensures no other + // lifecycle operation runs concurrently. + withContext(Dispatchers.IO) { + delay(50) + runCatching { renderer.recreateDecoderSurface() } + } + + synchronized(controllerLock) { + createOrReplaceDecoderLocked(session, recreateSurface = false) + return decoder != null + } + } + + /** + * Core method: construct a new [AnnexBDecoder] and replay bootstrap packets into it. + * + * Caller must have already released the old decoder (if any) and, when needed, recreated + * the persistent surface. Must be called while holding [controllerLock]. + */ + private fun createOrReplaceDecoderLocked( + session: Scrcpy.Session.SessionInfo, + recreateSurface: Boolean, + ) { + if (recreateSurface) { + // Should not happen in the current call path — surface recreation is handled + // by rebuildDecoderForSize outside the lock. Keep the parameter for clarity. + error("recreateSurface must be handled by caller") + } + + val surface = renderer.getDecoderSurface() + val mime = when (session.codec) { + Codec.H264 -> "video/avc" + Codec.H265 -> "video/hevc" + Codec.AV1 -> "video/av01" + else -> "video/avc" + } + Log.i( + TAG, + "createOrReplaceDecoder(): codec=${session.codec?.string ?: "null"}, " + + "size=${session.width}x${session.height}, " + + "recreateSurface=$recreateSurface, " + + "persistent=true", + ) + val newDecoder = try { + AnnexBDecoder( + width = session.width, + height = session.height, + outputSurface = surface, + mimeType = mime, + onOutputSizeChanged = { width, height -> + val current = currentSessionInfo + if (current == null || (current.width == width && current.height == height)) { + return@AnnexBDecoder + } + Log.i( + TAG, + "videoSizeChanged(): ${current.width}x${current.height} -> ${width}x${height}", + ) + currentSessionInfo = current.copy(width = width, height = height) + mainHandler.post { + videoSizeListeners.forEach { listener -> + runCatching { listener(width, height) } + } + } + }, + onFpsUpdated = { fps -> + mainHandler.post { + videoFpsListeners.forEach { listener -> + runCatching { listener(fps) } + } + } + }, + ) + } catch (e: DecoderException) { + Log.e(TAG, "createOrReplaceDecoder(): decoder init failed for ${e.mime} ${e.width}x${e.height}", e) + // Surface already exists; leave decoder null. The facade will see isUsable()==false + // and can trigger downgrade/restart. + return + } + decoder = newDecoder + replayBootstrapPackets(newDecoder) + } + + // ---------- Packet feeding ---------- + + /** + * Cache a bootstrap packet and feed it to the active decoder if one exists. + * + * Returns true if the decoder is still usable after feeding, false if the decoder + * entered an error state (the facade should restart the session in that case). + */ + fun feed(packet: Scrcpy.Session.VideoPacket): Boolean { + cacheBootstrapPacket(packet) + val dec = synchronized(controllerLock) { decoder } ?: return true + runCatching { + dec.feedAnnexB( + packet.data, + packet.ptsUs, + packet.isKeyFrame, + packet.isConfig, + ) + } + return dec.isUsable() + } + + /** + * Whether the current decoder (if any) is still usable. + */ + fun isDecoderUsable(): Boolean { + val dec = synchronized(controllerLock) { decoder } ?: return false + return dec.isUsable() + } + + // ---------- Bootstrap packet cache ---------- + + private fun replayBootstrapPackets(decoder: AnnexBDecoder) { + val snapshot = synchronized(bootstrapLock) { bootstrapPackets.toList() } + if (snapshot.isEmpty()) { + return + } + Log.i(TAG, "replayBootstrapPackets(): count=${snapshot.size}") + snapshot.forEachIndexed { index, packet -> + if (!decoder.isUsable()) { + Log.w(TAG, "replayBootstrapPackets(): decoder unusable, aborting replay at $index") + return + } + runCatching { + decoder.feedAnnexB(packet.data, packet.ptsUs, packet.isKeyFrame, packet.isConfig) + } + } + } + + private fun cacheBootstrapPacket(packet: Scrcpy.Session.VideoPacket) { + val cached = CachedPacket( + data = packet.data.copyOf(), + ptsUs = packet.ptsUs, + isConfig = packet.isConfig, + isKeyFrame = packet.isKeyFrame, + ) + synchronized(bootstrapLock) { + if (cached.isConfig) { + latestConfigPacket = cached + bootstrapPackets.clear() + bootstrapPackets.addLast(cached) + return + } + + if (cached.isKeyFrame) { + bootstrapPackets.clear() + latestConfigPacket?.let { bootstrapPackets.addLast(it) } + bootstrapPackets.addLast(cached) + return + } + + while (bootstrapPackets.size >= MAX_BOOTSTRAP_PACKETS) { + bootstrapPackets.removeFirst() + } + bootstrapPackets.addLast(cached) + } + } + + private data class CachedPacket( + val data: ByteArray, + val ptsUs: Long, + val isConfig: Boolean, + val isKeyFrame: Boolean, + ) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as CachedPacket + + if (ptsUs != other.ptsUs) return false + if (isConfig != other.isConfig) return false + if (isKeyFrame != other.isKeyFrame) return false + if (!data.contentEquals(other.data)) return false + + return true + } + + override fun hashCode(): Int { + var result = ptsUs.hashCode() + result = 31 * result + isConfig.hashCode() + result = 31 * result + isKeyFrame.hashCode() + result = 31 * result + data.contentHashCode() + return result + } + } + + companion object { + private const val TAG = "VideoDecoderController" + private const val MAX_BOOTSTRAP_PACKETS = 90 + } +}