Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added extended headset controls (headset button click handling) #1218

Open
wants to merge 11 commits into
base: master
Choose a base branch
from
Open

Added extended headset controls (headset button click handling) #1218

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
7ba1ebf
Clickhandler works partially, I need support on the rest
sandreas May 22, 2024
bfb571a
Clickhandler works in all tested scenarios
sandreas May 23, 2024
595ac55
Merge branch 'advplyr:master' into master
sandreas May 23, 2024
b0c5588
removed useless currentPlayerDispatcher property
sandreas May 23, 2024
25c1cec
Implemented fastForward, adjusted timings
sandreas May 24, 2024
56cd667
reverted unneccessary formatting changes, added experimental headset …
sandreas May 24, 2024
dc8d0cf
added enableExperimentalHeadsetControl as a setting
sandreas May 26, 2024
c24db68
Renamed "experimental" to "extended", fixed issues with extended head…
sandreas May 27, 2024
1ee23ad
Fixed translations, reduced seekBufferTime for fastForward and rewind
sandreas May 27, 2024
d1cd7bc
Moved headset settings in an extra category
sandreas May 28, 2024
4f653a7
Update android/app/src/main/java/com/audiobookshelf/app/player/MediaS…
sandreas Jun 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions android/app/src/main/java/com/audiobookshelf/app/data/DeviceClasses.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,8 @@ data class DeviceSettings(
var sleepTimerLength: Long, // Time in milliseconds
var disableSleepTimerFadeOut: Boolean,
var disableSleepTimerResetFeedback: Boolean,
var languageCode: String
var languageCode: String,
val enableExtendedHeadsetControls: Boolean
) {
companion object {
// Static method to get default device settings
Expand All @@ -147,7 +148,8 @@ data class DeviceSettings(
autoSleepTimerAutoRewindTime = 300000L, // 5 minutes
disableSleepTimerFadeOut = false,
disableSleepTimerResetFeedback = false,
languageCode = "en-us"
languageCode = "en-us",
enableExtendedHeadsetControls = false,
)
}
}
Expand Down
97 changes: 95 additions & 2 deletions android/app/src/main/java/com/audiobookshelf/app/player/MediaSessionCallback.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,35 @@ package com.audiobookshelf.app.player

import android.annotation.SuppressLint
import android.content.Intent
import android.os.*
import android.os.Build
import android.os.Bundle
import android.os.Handler
import android.os.Looper
import android.os.Message
import android.support.v4.media.session.MediaSessionCompat
import android.util.Log
import android.view.KeyEvent
import com.audiobookshelf.app.data.DeviceSettings
import com.audiobookshelf.app.data.LibraryItemWrapper
import com.audiobookshelf.app.data.PodcastEpisode
import com.audiobookshelf.app.device.DeviceManager
import java.util.*
import java.util.Timer
import kotlin.concurrent.schedule

class MediaSessionCallback(var playerNotificationService:PlayerNotificationService) : MediaSessionCompat.Callback() {
var tag = "MediaSessionCallback"
private val deviceSettings
get() = DeviceManager.deviceData.deviceSettings ?: DeviceSettings.default()

private var mediaButtonClickCount: Int = 0
private var mediaButtonClickTimeout: Long = 1000 //ms

private var clickTimer: Timer = Timer()
private var clickTimerId: Long = System.currentTimeMillis()
private var clickCount: Int = 0
private var clickPressed: Boolean = false
private var clickTimerScheduled: Boolean = false

override fun onPrepare() {
Log.d(tag, "ON PREPARE MEDIA SESSION COMPAT")
playerNotificationService.mediaManager.getFirstItem()?.let { li ->
Expand Down Expand Up @@ -157,6 +170,12 @@ class MediaSessionCallback(var playerNotificationService:PlayerNotificationServi
@Suppress("DEPRECATION")
intent.getParcelableExtra(Intent.EXTRA_KEY_EVENT)
}
if(deviceSettings.enableExtendedHeadsetControls) {
Log.d(tag, "extended headset control: enabled")
return debounceKeyEvent(keyEvent)
} else {
Log.d(tag, "extended headset control: disabled")
}

Log.d(tag, "handleCallMediaButton keyEvent = $keyEvent | action ${keyEvent?.action}")

Expand Down Expand Up @@ -258,6 +277,80 @@ class MediaSessionCallback(var playerNotificationService:PlayerNotificationServi
}



private fun debounceKeyEvent(keyEvent: KeyEvent?): Boolean {
// how does this work:
// - every keyDown and keyUp triggers a scheduled handler
// - another keyDown or keyUp cancels the scheduled handler and re-triggers it with new values
// - the handler takes clickCount:int and clickPressed:bool (if held down)
// - keyCodes increase the number of clicks (PlayPause+=1, Next+=2, Prev+=3)
// - depending on the number of clicks, the playerNotificationService handles the configured action
// problems:
// - the logs show pretty accurate click / hold detection, but it does not really translate well in the player
// - since the trigger is scheduled, it does run in a different thread
// - this leads to strange behaviour - probably easy to fix, but I'm no kotlin native (Coroutines)
// - probably after some actions the thread of the player is no longer accessible...
if (keyEvent?.action == KeyEvent.ACTION_UP) {
clickPressed = false
// Log.d(tag, "=== KeyEvent.ACTION_UP")

} else if (keyEvent?.action == KeyEvent.ACTION_DOWN) {
// Log.d(tag, "=== KeyEvent.ACTION_DOWN")

if(clickPressed) {
return false
}
clickPressed = true

when (keyEvent.keyCode) {
KeyEvent.KEYCODE_HEADSETHOOK,
KeyEvent.KEYCODE_MEDIA_PLAY,
KeyEvent.KEYCODE_MEDIA_PAUSE,
KeyEvent.KEYCODE_MEDIA_PLAY_PAUSE -> {
clickCount++
Log.d(tag, "=== handleCallMediaButton: Headset Hook/Play/ Pause, clickCount=$clickCount")
}

KeyEvent.KEYCODE_MEDIA_NEXT -> {
clickCount += 2
Log.d(tag, "=== handleCallMediaButton: Media Next, clickCount=$clickCount")
}

KeyEvent.KEYCODE_MEDIA_PREVIOUS -> {
clickCount += 3
Log.d(tag, "=== handleCallMediaButton: Media Previous, clickCount=$clickCount")
}
Comment on lines +314 to +322
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not understanding the logic here. How do we know that KEYCODE_MEDIA_NEXT/PREVIOUS are clicks and not a separate next/prev button press?

Copy link
Author

@sandreas sandreas Jun 10, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@advplyr
Android is handling specific shortcuts not emitting exact events, but overriding them by combined events:

  • One click: Play/Pause
  • Double click: Next
  • Triple click: Previous
  • Long press: Voice over - not a media event, but an extra permission AND handler on Andr. > 11, see here or here

This cannot be prevented, because it is a system / kernel behaviour.

So double-clicking does NOT emit 2 raw click events, but 1 single (automatically rewritten) event KeyEvent.KEYCODE_MEDIA_NEXT.

I handle KeyEvent.KEYCODE_MEDIA_NEXT as if 2 single clicks had happened and increase the click count by 2. Same for the triple-click, which happens to be emitted as KeyEvent.KEYCODE_MEDIA_PREVIOUS by Android instead of 3 single keydown/keyup event-combos.

This results in a more flexible API, since you can configure behaviour based on click count now and are not required to work around pre-defined behaviour. You can also see this in the how does it work comment, the mentioned problems there are kind of fixed / unfixable - I tuned the timings as good as possible, but I need feedback from other device owners.

Best way would be to try it out (by using a headset) monitoring the logs. If you don't own the required hardware to test it yourself, let me know, maybe we can arrange something.

This code won't fully work on Android Versions < 7.1. There is no possibility to detect `KEY_DOWN` events, because Android internally waits till `KEY_UP` doing nothing and then emits `KEY_DOWN` and `KEY_UP` together with no delay in between. So the scenario of holding down the key, that should emit something like:
  • Press down: KEY_DOWN
  • Wait: // wait 1248ms
  • Release: KEY_UP

on Android 7.0 devices is

  • Press down: // nothing
  • Wait: // wait 1248ms
  • Release: KEY_DOWN, KEY_UP

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My understanding is that these events are emitted by the bluetooth device and so depending on the bluetooth device it might emit a KEYCODE_MEDIA_NEXT on 2 clicks if it is setup for that.
Or you might have a bluetooth device that has separate buttons next to play/pause that is for KEYCODE_MEDIA_NEXT.

Did you find any documentation on these KeyEvents that say when/how they are triggered?
I think this needs to be tested on a bluetooth device that has NEXT/PREV media buttons. I'll see if I have one. I think many bluetooth in vehicles have this.

Copy link
Author

@sandreas sandreas Jun 11, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My understanding is that these events are emitted by the bluetooth device and so depending on the bluetooth device it might emit a KEYCODE_MEDIA_NEXT on 2 clicks if it is setup for that.

@advplyr
Ah, now I understand. You mean a device like this - a car bluetooth receiver with multiple buttons (e.g. for play/pause, next and prev).

Well, first things first: The PR is NON-DESTRUCTIVE, so that there are no behaviour changes, as long as the according setting (extended headset controls) is disabled, which it is by default. So no fear of breaking anything, that works fine at the moment.

I own such a car bluetooth recevier with buttons for play/pause, next and previous and tested it. It shows the following behaviour (which I would have expected):

  • play/pause
    • single click = play/pause
    • double click = next
    • triple click = prev
    • 4 clicks or more (fast execution) = nothing
    • multiple clicks (slow execution) = multiple play / pause events
  • next
    • single click = next
    • 2 clicks or more (fast execution) = nothing (translated to 4 clicks or more)
    • multiple clicks (slow execution) = multiple next events
  • prev
    • single click = prev
    • 2 clicks or more (fast execution) = nothing (translated to 6 clicks or more)
    • multiple clicks (slow execution) = multiple prev events

So I think it works like expected. Even better, on bluetooth devices with A SINGLE button, you can now use it for next and prev.

The only unexpected behaviour might be that multiple fast next clicks won't do anything, as long as the timer is not hit - you have to click slower, but since this feature is disabled by default, I don't see this as a huge problem.


KeyEvent.KEYCODE_MEDIA_STOP -> {
Log.d(tag, "=== handleCallMediaButton: Media Stop, clickCount=$clickCount")
playerNotificationService.closePlayback()
clickTimer.cancel()
return true
} else -> {
Log.d(tag, "=== KeyCode:${keyEvent.keyCode}, clickCount=$clickCount")
return false
}
}
}

if(clickTimerScheduled) {
Log.d(tag, "=== clickTimer cancelled ($clickTimerId): clicks=$clickCount, hold=$clickPressed =========")
clickTimer.cancel()
clickTimer = Timer()
}

clickTimer.schedule(650) {
Log.d(tag, "=== clickTimer executed ($clickTimerId): clicks=$clickCount, hold=$clickPressed =========")
playerNotificationService.handleClicks(clickCount, clickPressed)
sandreas marked this conversation as resolved.
Show resolved Hide resolved
clickCount = 0
clickTimerScheduled = false
}
clickTimerScheduled = true
Log.d(tag, "=== clickTimer scheduled ($clickTimerId): clicks=$clickCount, hold=$clickPressed =========")
return true
}

@Suppress("DEPRECATION")
private val mediaBtnHandler : Handler = @SuppressLint("HandlerLeak")
object : Handler(){
override fun handleMessage(msg: Message) {
Expand Down
182 changes: 172 additions & 10 deletions android/app/src/main/java/com/audiobookshelf/app/player/PlayerNotificationService.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,28 @@
package com.audiobookshelf.app.player

import android.annotation.SuppressLint
import android.app.*
import android.app.Notification
import android.app.NotificationChannel
import android.app.NotificationManager
import android.app.PendingIntent
import android.app.Service
import android.content.Context
import android.content.Intent
import android.graphics.Bitmap
import android.graphics.Color
import android.graphics.ImageDecoder
import android.hardware.Sensor
import android.hardware.SensorManager
import android.net.*
import android.os.*
import android.net.ConnectivityManager
import android.net.Network
import android.net.NetworkCapabilities
import android.net.NetworkRequest
import android.os.Binder
import android.os.Build
import android.os.Bundle
import android.os.Handler
import android.os.IBinder
import android.os.Looper
import android.provider.MediaStore
import android.provider.Settings
import android.support.v4.media.MediaBrowserCompat
Expand All @@ -25,17 +37,31 @@ import androidx.core.app.NotificationCompat
import androidx.core.content.ContextCompat
import androidx.media.MediaBrowserServiceCompat
import androidx.media.utils.MediaConstants
import com.audiobookshelf.app.BuildConfig
import com.audiobookshelf.app.R
import com.audiobookshelf.app.data.*
import com.audiobookshelf.app.data.BookChapter
import com.audiobookshelf.app.data.DeviceInfo
import com.audiobookshelf.app.data.DeviceSettings
import com.audiobookshelf.app.data.LibraryItem
import com.audiobookshelf.app.data.LocalMediaProgress
import com.audiobookshelf.app.data.MediaItemHistory
import com.audiobookshelf.app.data.MediaProgressWrapper
import com.audiobookshelf.app.data.PlayItemRequestPayload
import com.audiobookshelf.app.data.PlaybackMetadata
import com.audiobookshelf.app.data.PlaybackSession
import com.audiobookshelf.app.data.PlayerState
import com.audiobookshelf.app.data.Podcast
import com.audiobookshelf.app.device.DeviceManager
import com.audiobookshelf.app.managers.DbManager
import com.audiobookshelf.app.managers.SleepTimerManager
import com.audiobookshelf.app.media.MediaManager
import com.audiobookshelf.app.media.MediaProgressSyncer
import com.audiobookshelf.app.server.ApiHandler
import com.audiobookshelf.app.BuildConfig
import com.google.android.exoplayer2.*
import com.google.android.exoplayer2.C
import com.google.android.exoplayer2.DefaultLoadControl
import com.google.android.exoplayer2.ExoPlayer
import com.google.android.exoplayer2.LoadControl
import com.google.android.exoplayer2.Player
import com.google.android.exoplayer2.audio.AudioAttributes
import com.google.android.exoplayer2.ext.mediasession.MediaSessionConnector
import com.google.android.exoplayer2.ext.mediasession.MediaSessionConnector.CustomActionProvider
Expand All @@ -46,16 +72,26 @@ import com.google.android.exoplayer2.source.MediaSource
import com.google.android.exoplayer2.source.ProgressiveMediaSource
import com.google.android.exoplayer2.source.hls.HlsMediaSource
import com.google.android.exoplayer2.ui.PlayerNotificationManager
import com.google.android.exoplayer2.upstream.*
import java.util.*
import com.google.android.exoplayer2.upstream.DefaultDataSource
import com.google.android.exoplayer2.upstream.DefaultHttpDataSource
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import java.util.Timer
import java.util.TimerTask
import kotlin.concurrent.schedule
import kotlin.coroutines.CoroutineContext


const val SLEEP_TIMER_WAKE_UP_EXPIRATION = 120000L // 2m
const val PLAYER_CAST = "cast-player"
const val PLAYER_EXO = "exo-player"

class PlayerNotificationService : MediaBrowserServiceCompat() {
class PlayerNotificationService : CoroutineScope, MediaBrowserServiceCompat() {
override val coroutineContext: CoroutineContext
get() = Dispatchers.Main + Job()

companion object {
var isStarted = false
Expand Down Expand Up @@ -886,7 +922,133 @@ class PlayerNotificationService : MediaBrowserServiceCompat() {
}
}

fun closePlayback(calledOnError:Boolean? = false) {
private var lastStatePlaying: Boolean = false

private var seekPlayBufferTime: Long = 450
/*
fun fastForward() {
val lock = Object()
launch {
withContext(coroutineContext) {
while (currentPlayer.currentPosition < currentPlayer.duration) {
seekPlayer(5L * 1000 - seekPlayBufferTime)
play()
lock.wait(seekPlayBufferTime)
}
}
}
}

*/
/*
fun rewind() {
val locker = ReentrantLock()
launch {
withContext(coroutineContext) {
while (currentPlayer.currentPosition > 0) {
seekPlayer(-5L * 1000 + seekPlayBufferTime)
play()
}
}
}
}
*/

private var stopSeeking: Boolean = false

fun handleClicks(clicks: Int, clickPressed: Boolean) {
stopSeeking = true
launch {
// the handlers should be configurlateinitable, defaults:
// hold -> jumpBackward
// click -> play / pause
// click, hold -> fast forward
// click, click -> next (chapter or track)
// click, click, hold -> rewind
// click, click, click -> previous (chapter or track)

withContext(coroutineContext) {
Log.d(tag, "=== handleClicks: count=$clicks,hold=$clickPressed")

if (clickPressed) {
lastStatePlaying = currentPlayer.isPlaying
when (clicks) {
1 -> {
jumpBackward()
}
2 -> {
Log.d(tag, "=== fastForward init, stopSeeking=$stopSeeking")

stopSeeking = false
val mainHandler = Handler(Looper.getMainLooper())
mainHandler.post(object : Runnable {
override fun run() {
Log.d(tag, "=== fastForward run, stopSeeking=$stopSeeking")
seekForward(10000 - seekPlayBufferTime)
play()
if(!stopSeeking) {
Log.d(tag, "=== fastForward recursion")
mainHandler.postDelayed(this, seekPlayBufferTime)
}
}
})
}

3 -> {
stopSeeking = false
val mainHandler = Handler(Looper.getMainLooper())
mainHandler.post(object : Runnable {
override fun run() {
seekBackward(10000 + seekPlayBufferTime)
play()
if(!stopSeeking) {
mainHandler.postDelayed(this, seekPlayBufferTime)
}
}
})
}
}
} else {
when (clicks) {
0 -> {
// switch from fastForward / rewind back to last playing state
if (lastStatePlaying) {
play()
} else {
pause()
}
}

1 -> {
playPause()
/*
if (currentPlayer.isPlaying) {
pause()
} else {
play()
}

*/
}

2 -> {
// todo: implement "next chapter"
// skipToNext()
seekForward(300000)
}

3 -> {
// todo: implement "previous chapter"
// skipToPrevious()
seekBackward(300000)
}
}
}
}
}
}

fun closePlayback(calledOnError: Boolean? = false) {
Log.d(tag, "closePlayback")
val isLocal = mediaProgressSyncer.currentIsLocal
val currentSessionId = mediaProgressSyncer.currentSessionId
Expand Down
Loading