Integrate with libass-android to support SSA/ASS subtitles in ExoPlayer

app/build.gradle.kts

@@ -152,6 +152,12 @@ android {
             isUniversalApk = true
         }
     }
+    packaging {
+        jniLibs {
+            // Work around because libass-android & wholphin-mpv both (incorrectly) package libc++_shared.so
+            pickFirsts += "lib/*/libc++_shared.so"
+        }
+    }
 
     sourceSets {
         getByName("main") {
@@ -250,6 +256,7 @@ dependencies {
     implementation(libs.androidx.media3.exoplayer.dash)
     implementation(libs.androidx.media3.ui)
     implementation(libs.androidx.media3.ui.compose)
+    implementation(libs.ass.media)
 
     implementation(libs.coil.core)
     implementation(libs.coil.compose)

app/src/main/java/com/github/damontecres/wholphin/preferences/AppPreference.kt

@@ -430,16 +430,18 @@ sealed interface AppPreference<Pref, T> {
                 summaryOn = R.string.enabled,
                 summaryOff = R.string.disabled,
             )
-        val DirectPlayAss =
-            AppSwitchPreference<AppPreferences>(
-                title = R.string.direct_play_ass,
-                defaultValue = true,
-                getter = { it.playbackPreferences.overrides.directPlayAss },
+        val AssSubtitleMode =
+            AppChoicePreference<AppPreferences, AssPlaybackMode>(
+                title = R.string.app_theme,
+                defaultValue = AssPlaybackMode.ASS_LIBASS,
+                getter = { it.playbackPreferences.overrides.assPlaybackMode },
                 setter = { prefs, value ->
-                    prefs.updatePlaybackOverrides { directPlayAss = value }
+                    prefs.updatePlaybackOverrides { assPlaybackMode = value }
                 },
-                summaryOn = R.string.enabled,
-                summaryOff = R.string.disabled,
+                displayValues = R.array.ass_subtitle_modes,
+                subtitles = R.array.ass_subtitle_modes_summary,
+                indexToValue = { AssPlaybackMode.forNumber(it) },
+                valueToIndex = { if (it != AssPlaybackMode.UNRECOGNIZED) it.number else AssPlaybackMode.ASS_LIBASS.number },
             )
         val DirectPlayPgs =
             AppSwitchPreference<AppPreferences>(
@@ -1097,7 +1099,7 @@ private val ExoPlayerSettings =
         AppPreference.FfmpegPreference,
         AppPreference.DownMixStereo,
         AppPreference.Ac3Supported,
-        AppPreference.DirectPlayAss,
+        AppPreference.AssSubtitleMode,
         AppPreference.DirectPlayPgs,
         AppPreference.DirectPlayDoviProfile7,
         AppPreference.DecodeAv1,

app/src/main/java/com/github/damontecres/wholphin/preferences/AppPreferencesSerializer.kt

@@ -61,10 +61,12 @@ class AppPreferencesSerializer
                                         .apply {
                                             ac3Supported = AppPreference.Ac3Supported.defaultValue
                                             downmixStereo = AppPreference.DownMixStereo.defaultValue
-                                            directPlayAss = AppPreference.DirectPlayAss.defaultValue
+//                                            directPlayAss = AppPreference.DirectPlayAss.defaultValue
                                             directPlayPgs = AppPreference.DirectPlayPgs.defaultValue
                                             mediaExtensionsEnabled =
                                                 AppPreference.FfmpegPreference.defaultValue
+                                            assPlaybackMode =
+                                                AppPreference.AssSubtitleMode.defaultValue
                                         }.build()
 
                                 mpvOptions =

app/src/main/java/com/github/damontecres/wholphin/services/AppUpgradeHandler.kt

@@ -141,7 +141,7 @@ class AppUpgradeHandler
                     it.updatePlaybackOverrides {
                         ac3Supported = AppPreference.Ac3Supported.defaultValue
                         downmixStereo = AppPreference.DownMixStereo.defaultValue
-                        directPlayAss = AppPreference.DirectPlayAss.defaultValue
+//                        directPlayAss = AppPreference.DirectPlayAss.defaultValue
                         directPlayPgs = AppPreference.DirectPlayPgs.defaultValue
                     }
                 }

app/src/main/java/com/github/damontecres/wholphin/services/DeviceProfileService.kt

@@ -1,6 +1,7 @@
 package com.github.damontecres.wholphin.services
 
 import android.content.Context
+import com.github.damontecres.wholphin.preferences.AssPlaybackMode
 import com.github.damontecres.wholphin.preferences.PlaybackPreferences
 import com.github.damontecres.wholphin.util.profile.MediaCodecCapabilitiesTest
 import com.github.damontecres.wholphin.util.profile.createDeviceProfile
@@ -43,7 +44,7 @@ class DeviceProfileService
                             maxBitrate = prefs.maxBitrate.toInt(),
                             isAC3Enabled = prefs.overrides.ac3Supported,
                             downMixAudio = prefs.overrides.downmixStereo,
-                            assDirectPlay = prefs.overrides.directPlayAss,
+                            assPlaybackMode = prefs.overrides.assPlaybackMode,
                             pgsDirectPlay = prefs.overrides.directPlayPgs,
                             dolbyVisionELDirectPlay = prefs.overrides.directPlayDolbyVisionEL,
                             decodeAv1 = prefs.overrides.decodeAv1,
@@ -58,7 +59,7 @@ class DeviceProfileService
                                 maxBitrate = newConfig.maxBitrate,
                                 isAC3Enabled = newConfig.isAC3Enabled,
                                 downMixAudio = newConfig.downMixAudio,
-                                assDirectPlay = newConfig.assDirectPlay,
+                                assDirectPlay = newConfig.assPlaybackMode != AssPlaybackMode.ASS_TRANSCODE,
                                 pgsDirectPlay = newConfig.pgsDirectPlay,
                                 dolbyVisionELDirectPlay = newConfig.dolbyVisionELDirectPlay,
                                 decodeAv1 = prefs.overrides.decodeAv1,
@@ -77,7 +78,7 @@ data class DeviceProfileConfiguration(
     val maxBitrate: Int,
     val isAC3Enabled: Boolean,
     val downMixAudio: Boolean,
-    val assDirectPlay: Boolean,
+    val assPlaybackMode: AssPlaybackMode,
     val pgsDirectPlay: Boolean,
     val dolbyVisionELDirectPlay: Boolean,
     val decodeAv1: Boolean,

app/src/main/java/com/github/damontecres/wholphin/services/PlayerFactory.kt

@@ -10,22 +10,29 @@ import androidx.datastore.core.DataStore
 import androidx.media3.common.C
 import androidx.media3.common.Player
 import androidx.media3.common.util.UnstableApi
+import androidx.media3.datasource.DefaultDataSource
 import androidx.media3.exoplayer.DefaultRenderersFactory
 import androidx.media3.exoplayer.ExoPlayer
 import androidx.media3.exoplayer.Renderer
+import androidx.media3.exoplayer.RenderersFactory
 import androidx.media3.exoplayer.mediacodec.MediaCodecSelector
+import androidx.media3.exoplayer.source.DefaultMediaSourceFactory
 import androidx.media3.exoplayer.video.MediaCodecVideoRenderer
 import androidx.media3.exoplayer.video.VideoRendererEventListener
-import com.github.damontecres.wholphin.preferences.AppPreference
+import androidx.media3.extractor.DefaultExtractorsFactory
 import com.github.damontecres.wholphin.preferences.AppPreferences
+import com.github.damontecres.wholphin.preferences.AssPlaybackMode
 import com.github.damontecres.wholphin.preferences.MediaExtensionStatus
 import com.github.damontecres.wholphin.preferences.PlaybackPreferences
 import com.github.damontecres.wholphin.preferences.PlayerBackend
 import com.github.damontecres.wholphin.util.mpv.MpvPlayer
 import dagger.hilt.android.qualifiers.ApplicationContext
+import io.github.peerless2012.ass.media.AssHandler
+import io.github.peerless2012.ass.media.factory.AssRenderersFactory
+import io.github.peerless2012.ass.media.kt.withAssMkvSupport
+import io.github.peerless2012.ass.media.parser.AssSubtitleParserFactory
+import io.github.peerless2012.ass.media.type.AssRenderType
 import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.flow.firstOrNull
-import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 import timber.log.Timber
 import java.lang.reflect.Constructor
@@ -46,71 +53,17 @@ class PlayerFactory
         var currentPlayer: Player? = null
             private set
 
-        fun createVideoPlayer(): Player {
-            if (currentPlayer?.isReleased == false) {
-                Timber.w("Player was not released before trying to create a new one!")
-                currentPlayer?.release()
-            }
-
-            val prefs = runBlocking { appPreferences.data.firstOrNull()?.playbackPreferences }
-            val backend = prefs?.playerBackend ?: AppPreference.PlayerBackendPref.defaultValue
-            val newPlayer =
-                when (backend) {
-                    PlayerBackend.PREFER_MPV,
-                    PlayerBackend.MPV,
-                    -> {
-                        val enableHardwareDecoding =
-                            prefs?.mpvOptions?.enableHardwareDecoding
-                                ?: AppPreference.MpvHardwareDecoding.defaultValue
-                        val useGpuNext =
-                            prefs?.mpvOptions?.useGpuNext
-                                ?: AppPreference.MpvGpuNext.defaultValue
-                        MpvPlayer(context, enableHardwareDecoding, useGpuNext)
-                            .apply {
-                                playWhenReady = true
-                            }
-                    }
-
-                    PlayerBackend.EXO_PLAYER,
-                    PlayerBackend.UNRECOGNIZED,
-                    -> {
-                        val extensions = prefs?.overrides?.mediaExtensionsEnabled
-                        val decodeAv1 = prefs?.overrides?.decodeAv1 == true
-                        Timber.v("extensions=$extensions")
-                        val rendererMode =
-                            when (extensions) {
-                                MediaExtensionStatus.MES_FALLBACK -> DefaultRenderersFactory.EXTENSION_RENDERER_MODE_ON
-                                MediaExtensionStatus.MES_PREFERRED -> DefaultRenderersFactory.EXTENSION_RENDERER_MODE_PREFER
-                                MediaExtensionStatus.MES_DISABLED -> DefaultRenderersFactory.EXTENSION_RENDERER_MODE_OFF
-                                else -> DefaultRenderersFactory.EXTENSION_RENDERER_MODE_ON
-                            }
-                        ExoPlayer
-                            .Builder(context)
-                            .setRenderersFactory(
-                                WholphinRenderersFactory(context, decodeAv1)
-                                    .setEnableDecoderFallback(true)
-                                    .setExtensionRendererMode(rendererMode),
-                            ).build()
-                            .apply {
-                                playWhenReady = true
-                            }
-                    }
-                }
-            currentPlayer = newPlayer
-            return newPlayer
-        }
-
         suspend fun createVideoPlayer(
             backend: PlayerBackend,
             prefs: PlaybackPreferences,
-        ): Player {
+        ): PlayerCreation {
             withContext(Dispatchers.Main) {
                 if (currentPlayer?.isReleased == false) {
                     Timber.w("Player was not released before trying to create a new one!")
                     currentPlayer?.release()
                 }
             }
-
+            var assHandler: AssHandler? = null
             val newPlayer =
                 when (backend) {
                     PlayerBackend.PREFER_MPV,
@@ -125,26 +78,57 @@ class PlayerFactory
                     PlayerBackend.UNRECOGNIZED,
                     -> {
                         val extensions = prefs.overrides.mediaExtensionsEnabled
+                        val useLibAss =
+                            prefs.overrides.assPlaybackMode == AssPlaybackMode.ASS_LIBASS
                         val decodeAv1 = prefs.overrides.decodeAv1
-                        Timber.v("extensions=$extensions")
+                        Timber.v(
+                            "extensions=%s, assPlaybackMode=%s",
+                            extensions,
+                            prefs.overrides.assPlaybackMode,
+                        )
                         val rendererMode =
                             when (extensions) {
                                 MediaExtensionStatus.MES_FALLBACK -> DefaultRenderersFactory.EXTENSION_RENDERER_MODE_ON
                                 MediaExtensionStatus.MES_PREFERRED -> DefaultRenderersFactory.EXTENSION_RENDERER_MODE_PREFER
                                 MediaExtensionStatus.MES_DISABLED -> DefaultRenderersFactory.EXTENSION_RENDERER_MODE_OFF
                                 else -> DefaultRenderersFactory.EXTENSION_RENDERER_MODE_ON
                             }
+                        val dataSourceFactory = DefaultDataSource.Factory(context)
+                        val extractorsFactory = DefaultExtractorsFactory()
+                        var renderersFactory: RenderersFactory =
+                            WholphinRenderersFactory(context, decodeAv1)
+                                .setEnableDecoderFallback(true)
+                                .setExtensionRendererMode(rendererMode)
+                        val mediaSourceFactory =
+                            if (useLibAss) {
+                                assHandler = AssHandler(AssRenderType.OVERLAY_OPEN_GL)
+                                val assSubtitleParserFactory = AssSubtitleParserFactory(assHandler)
+                                renderersFactory = AssRenderersFactory(assHandler, renderersFactory)
+                                DefaultMediaSourceFactory(
+                                    dataSourceFactory,
+                                    extractorsFactory.withAssMkvSupport(
+                                        assSubtitleParserFactory,
+                                        assHandler,
+                                    ),
+                                ).setSubtitleParserFactory(assSubtitleParserFactory)
+                            } else {
+                                DefaultMediaSourceFactory(
+                                    dataSourceFactory,
+                                    extractorsFactory,
+                                )
+                            }
                         ExoPlayer
                             .Builder(context)
-                            .setRenderersFactory(
-                                WholphinRenderersFactory(context, decodeAv1)
-                                    .setEnableDecoderFallback(true)
-                                    .setExtensionRendererMode(rendererMode),
-                            ).build()
+                            .setMediaSourceFactory(mediaSourceFactory)
+                            .setRenderersFactory(renderersFactory)
+                            .build()
+                            .apply {
+                                assHandler?.init(this)
+                            }
                     }
                 }
             currentPlayer = newPlayer
-            return newPlayer
+            return PlayerCreation(newPlayer, assHandler)
         }
     }
 
@@ -157,6 +141,11 @@ val Player.isReleased: Boolean
         }
     }
 
+data class PlayerCreation(
+    val player: Player,
+    val assHandler: AssHandler? = null,
+)
+
 // Code is adapted from https://github.com/androidx/media/blob/release/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java#L436
 class WholphinRenderersFactory(
     context: Context,