Skip to content

Commit d44d81e

Browse files
Florian14Florian14
authored
Merge pull request #7419 from vector-im/feature/fre/voice_broadcast_live_listening
Voice broadcast - live listening
2 parents ed0d255 + 0a9f2bf commit d44d81e

File tree

12 files changed

+266
-52
lines changed

12 files changed

+266
-52
lines changed

‎changelog.d/7419.wip

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[Voice Broadcast] Live listening support

‎matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/events/model/Event.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -401,7 +401,7 @@ fun Event.getRelationContent(): RelationDefaultContent? {
401401
when (getClearType()) {
402402
EventType.STICKER -> getClearContent().toModel<MessageStickerContent>()?.relatesTo
403403
in EventType.BEACON_LOCATION_DATA -> getClearContent().toModel<MessageBeaconLocationDataContent>()?.relatesTo
404-
else -> null
404+
else -> getClearContent()?.get("m.relates_to")?.toContent().toModel()
405405
}
406406
}
407407
}

‎matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/timeline/TimelineEventDataSource.kt

+9-1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ import io.realm.Sort
2222
import org.matrix.android.sdk.api.session.events.model.getRelationContent
2323
import org.matrix.android.sdk.api.session.events.model.isImageMessage
2424
import org.matrix.android.sdk.api.session.events.model.isVideoMessage
25+
import org.matrix.android.sdk.api.session.events.model.toModel
26+
import org.matrix.android.sdk.api.session.room.model.message.MessageContent
2527
import org.matrix.android.sdk.api.session.room.timeline.TimelineEvent
2628
import org.matrix.android.sdk.api.util.Optional
2729
import org.matrix.android.sdk.internal.database.RealmSessionProvider
@@ -74,7 +76,13 @@ internal class TimelineEventDataSource @Inject constructor(
7476
.distinct(TimelineEventEntityFields.EVENT_ID)
7577
.findAll()
7678
.mapNotNull {
77-
timelineEventMapper.map(it).takeIf { it.root.getRelationContent()?.takeIf { it.type == eventType && it.eventId == eventId } != null }
79+
timelineEventMapper.map(it)
80+
.takeIf {
81+
val isEventRelatedTo = it.root.getRelationContent()?.takeIf { it.type == eventType && it.eventId == eventId } != null
82+
val isContentRelatedTo = it.root.getClearContent()?.toModel<MessageContent>()
83+
?.relatesTo?.takeIf { it.type == eventType && it.eventId == eventId } != null
84+
isEventRelatedTo || isContentRelatedTo
85+
}
7886
}
7987
}
8088
}

‎vector/src/main/java/im/vector/app/features/voicebroadcast/VoiceBroadcastHelper.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ class VoiceBroadcastHelper @Inject constructor(
4040

4141
suspend fun stopVoiceBroadcast(roomId: String) = stopVoiceBroadcastUseCase.execute(roomId)
4242

43-
fun playOrResumePlayback(roomId: String, eventId: String) = voiceBroadcastPlayer.play(roomId, eventId)
43+
fun playOrResumePlayback(roomId: String, eventId: String) = voiceBroadcastPlayer.playOrResume(roomId, eventId)
4444

4545
fun pausePlayback() = voiceBroadcastPlayer.pause()
4646

‎vector/src/main/java/im/vector/app/features/voicebroadcast/VoiceBroadcastPlayer.kt

+193-47
Original file line numberDiff line numberDiff line change
@@ -18,105 +18,194 @@ package im.vector.app.features.voicebroadcast
1818

1919
import android.media.AudioAttributes
2020
import android.media.MediaPlayer
21+
import im.vector.app.core.di.ActiveSessionHolder
2122
import im.vector.app.features.home.room.detail.timeline.helper.AudioMessagePlaybackTracker
22-
import im.vector.app.features.home.room.detail.timeline.helper.AudioMessagePlaybackTracker.Listener.State
2323
import im.vector.app.features.voice.VoiceFailure
24+
import im.vector.app.features.voicebroadcast.model.VoiceBroadcastState
25+
import im.vector.app.features.voicebroadcast.model.asVoiceBroadcastEvent
26+
import im.vector.app.features.voicebroadcast.usecase.GetVoiceBroadcastUseCase
2427
import kotlinx.coroutines.CoroutineScope
2528
import kotlinx.coroutines.Dispatchers
29+
import kotlinx.coroutines.Job
30+
import kotlinx.coroutines.SupervisorJob
2631
import kotlinx.coroutines.launch
27-
import org.matrix.android.sdk.api.extensions.orFalse
28-
import org.matrix.android.sdk.api.session.Session
2932
import org.matrix.android.sdk.api.session.events.model.RelationType
3033
import org.matrix.android.sdk.api.session.events.model.getRelationContent
3134
import org.matrix.android.sdk.api.session.getRoom
3235
import org.matrix.android.sdk.api.session.room.Room
3336
import org.matrix.android.sdk.api.session.room.model.message.MessageAudioContent
3437
import org.matrix.android.sdk.api.session.room.model.message.MessageAudioEvent
3538
import org.matrix.android.sdk.api.session.room.model.message.asMessageAudioEvent
39+
import org.matrix.android.sdk.api.session.room.timeline.Timeline
40+
import org.matrix.android.sdk.api.session.room.timeline.TimelineEvent
41+
import org.matrix.android.sdk.api.session.room.timeline.TimelineSettings
3642
import timber.log.Timber
3743
import javax.inject.Inject
3844
import javax.inject.Singleton
3945

4046
@Singleton
4147
class VoiceBroadcastPlayer @Inject constructor(
42-
private val session: Session,
48+
private val sessionHolder: ActiveSessionHolder,
4349
private val playbackTracker: AudioMessagePlaybackTracker,
50+
private val getVoiceBroadcastUseCase: GetVoiceBroadcastUseCase,
4451
) {
52+
private val session
53+
get() = sessionHolder.getActiveSession()
4554

46-
private val mediaPlayerScope = CoroutineScope(Dispatchers.IO)
55+
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
56+
private var voiceBroadcastStateJob: Job? = null
57+
private var currentTimeline: Timeline? = null
58+
set(value) {
59+
field?.removeAllListeners()
60+
field?.dispose()
61+
field = value
62+
}
63+
64+
private val mediaPlayerListener = MediaPlayerListener()
65+
private var timelineListener: TimelineListener? = null
4766

4867
private var currentMediaPlayer: MediaPlayer? = null
49-
private var currentPlayingIndex: Int = -1
68+
private var nextMediaPlayer: MediaPlayer? = null
69+
set(value) {
70+
field = value
71+
currentMediaPlayer?.setNextMediaPlayer(value)
72+
}
73+
private var currentSequence: Int? = null
74+
5075
private var playlist = emptyList<MessageAudioEvent>()
51-
private val currentVoiceBroadcastEventId
76+
private val currentVoiceBroadcastId
5277
get() = playlist.firstOrNull()?.root?.getRelationContent()?.eventId
5378

54-
private val mediaPlayerListener = MediaPlayerListener()
55-
56-
fun play(roomId: String, eventId: String) {
57-
val room = session.getRoom(roomId) ?: error("Unknown roomId: $roomId")
79+
private var state: State = State.IDLE
80+
set(value) {
81+
Timber.w("## VoiceBroadcastPlayer state: $field -> $value")
82+
field = value
83+
}
84+
private var currentRoomId: String? = null
5885

86+
fun playOrResume(roomId: String, eventId: String) {
87+
val hasChanged = currentVoiceBroadcastId != eventId
5988
when {
60-
currentVoiceBroadcastEventId != eventId -> {
61-
stop()
62-
updatePlaylist(room, eventId)
63-
startPlayback()
64-
}
65-
playbackTracker.getPlaybackState(eventId) is State.Playing -> pause()
66-
else -> resumePlayback()
89+
hasChanged -> startPlayback(roomId, eventId)
90+
state == State.PAUSED -> resumePlayback()
91+
else -> Unit
6792
}
6893
}
6994

7095
fun pause() {
7196
currentMediaPlayer?.pause()
72-
currentVoiceBroadcastEventId?.let { playbackTracker.pausePlayback(it) }
97+
currentVoiceBroadcastId?.let { playbackTracker.pausePlayback(it) }
98+
state = State.PAUSED
7399
}
74100

75101
fun stop() {
102+
// Stop playback
76103
currentMediaPlayer?.stop()
77-
currentMediaPlayer?.release()
78-
currentMediaPlayer?.setOnInfoListener(null)
104+
currentVoiceBroadcastId?.let { playbackTracker.stopPlayback(it) }
105+
106+
// Release current player
107+
release(currentMediaPlayer)
79108
currentMediaPlayer = null
80-
currentVoiceBroadcastEventId?.let { playbackTracker.stopPlayback(it) }
109+
110+
// Release next player
111+
release(nextMediaPlayer)
112+
nextMediaPlayer = null
113+
114+
// Do not observe anymore voice broadcast state changes
115+
voiceBroadcastStateJob?.cancel()
116+
voiceBroadcastStateJob = null
117+
118+
// In case of live broadcast, stop observing new chunks
119+
currentTimeline = null
120+
timelineListener = null
121+
122+
// Update state
123+
state = State.IDLE
124+
125+
// Clear playlist
81126
playlist = emptyList()
82-
currentPlayingIndex = -1
127+
currentSequence = null
128+
currentRoomId = null
83129
}
84130

85-
private fun updatePlaylist(room: Room, eventId: String) {
86-
val timelineEvents = room.timelineService().getTimelineEventsRelatedTo(RelationType.REFERENCE, eventId)
87-
val audioEvents = timelineEvents.mapNotNull { it.root.asMessageAudioEvent() }
88-
playlist = audioEvents.sortedBy { it.getVoiceBroadcastChunk()?.sequence?.toLong() ?: it.root.originServerTs }
131+
private fun startPlayback(roomId: String, eventId: String) {
132+
val room = session.getRoom(roomId) ?: error("Unknown roomId: $roomId")
133+
currentRoomId = roomId
134+
135+
// Stop listening previous voice broadcast if any
136+
if (state != State.IDLE) stop()
137+
138+
state = State.BUFFERING
139+
140+
val voiceBroadcastState = getVoiceBroadcastUseCase.execute(roomId, eventId)?.content?.voiceBroadcastState
141+
if (voiceBroadcastState == VoiceBroadcastState.STOPPED) {
142+
// Get static playlist
143+
updatePlaylist(getExistingChunks(room, eventId))
144+
startPlayback(false)
145+
} else {
146+
playLiveVoiceBroadcast(room, eventId)
147+
}
89148
}
90149

91-
private fun startPlayback() {
92-
val content = playlist.firstOrNull()?.content ?: run { Timber.w("## VoiceBroadcastPlayer: No content to play"); return }
93-
mediaPlayerScope.launch {
150+
private fun startPlayback(isLive: Boolean) {
151+
val event = if (isLive) playlist.lastOrNull() else playlist.firstOrNull()
152+
val content = event?.content ?: run { Timber.w("## VoiceBroadcastPlayer: No content to play"); return }
153+
val sequence = event.getVoiceBroadcastChunk()?.sequence
154+
coroutineScope.launch {
94155
try {
95156
currentMediaPlayer = prepareMediaPlayer(content)
96157
currentMediaPlayer?.start()
97-
currentPlayingIndex = 0
98-
currentVoiceBroadcastEventId?.let { playbackTracker.startPlayback(it) }
99-
prepareNextFile()
158+
currentVoiceBroadcastId?.let { playbackTracker.startPlayback(it) }
159+
currentSequence = sequence
160+
state = State.PLAYING
161+
nextMediaPlayer = prepareNextMediaPlayer()
100162
} catch (failure: Throwable) {
101163
Timber.e(failure, "Unable to start playback")
102164
throw VoiceFailure.UnableToPlay(failure)
103165
}
104166
}
105167
}
106168

169+
private fun playLiveVoiceBroadcast(room: Room, eventId: String) {
170+
room.timelineService().getTimelineEvent(eventId)?.root?.asVoiceBroadcastEvent() ?: error("Cannot retrieve voice broadcast $eventId")
171+
updatePlaylist(getExistingChunks(room, eventId))
172+
startPlayback(true)
173+
observeIncomingEvents(room, eventId)
174+
}
175+
176+
private fun getExistingChunks(room: Room, eventId: String): List<MessageAudioEvent> {
177+
return room.timelineService().getTimelineEventsRelatedTo(RelationType.REFERENCE, eventId)
178+
.mapNotNull { it.root.asMessageAudioEvent() }
179+
.filter { it.isVoiceBroadcast() }
180+
}
181+
182+
private fun observeIncomingEvents(room: Room, eventId: String) {
183+
currentTimeline = room.timelineService().createTimeline(null, TimelineSettings(5)).also { timeline ->
184+
timelineListener = TimelineListener(eventId).also { timeline.addListener(it) }
185+
timeline.start()
186+
}
187+
}
188+
107189
private fun resumePlayback() {
108190
currentMediaPlayer?.start()
109-
currentVoiceBroadcastEventId?.let { playbackTracker.startPlayback(it) }
191+
currentVoiceBroadcastId?.let { playbackTracker.startPlayback(it) }
192+
state = State.PLAYING
110193
}
111194

112-
private suspend fun prepareNextFile() {
113-
val nextContent = playlist.getOrNull(currentPlayingIndex + 1)?.content
114-
if (nextContent == null) {
115-
currentMediaPlayer?.setOnCompletionListener(mediaPlayerListener)
116-
} else {
117-
val nextMediaPlayer = prepareMediaPlayer(nextContent)
118-
currentMediaPlayer?.setNextMediaPlayer(nextMediaPlayer)
119-
}
195+
private fun updatePlaylist(playlist: List<MessageAudioEvent>) {
196+
this.playlist = playlist.sortedBy { it.getVoiceBroadcastChunk()?.sequence?.toLong() ?: it.root.originServerTs }
197+
}
198+
199+
private fun getNextAudioContent(): MessageAudioContent? {
200+
val nextSequence = currentSequence?.plus(1)
201+
?: timelineListener?.let { playlist.lastOrNull()?.sequence }
202+
?: 1
203+
return playlist.find { it.getVoiceBroadcastChunk()?.sequence == nextSequence }?.content
204+
}
205+
206+
private suspend fun prepareNextMediaPlayer(): MediaPlayer? {
207+
val nextContent = getNextAudioContent() ?: return null
208+
return prepareMediaPlayer(nextContent)
120209
}
121210

122211
private suspend fun prepareMediaPlayer(messageAudioContent: MessageAudioContent): MediaPlayer {
@@ -140,28 +229,78 @@ class VoiceBroadcastPlayer @Inject constructor(
140229
setDataSource(fis.fd)
141230
setOnInfoListener(mediaPlayerListener)
142231
setOnErrorListener(mediaPlayerListener)
232+
setOnCompletionListener(mediaPlayerListener)
143233
prepare()
144234
}
145235
}
146236
}
147237

148-
inner class MediaPlayerListener : MediaPlayer.OnInfoListener, MediaPlayer.OnCompletionListener, MediaPlayer.OnErrorListener {
238+
private fun release(mp: MediaPlayer?) {
239+
mp?.apply {
240+
release()
241+
setOnInfoListener(null)
242+
setOnCompletionListener(null)
243+
setOnErrorListener(null)
244+
}
245+
}
246+
247+
private inner class TimelineListener(private val voiceBroadcastId: String) : Timeline.Listener {
248+
override fun onTimelineUpdated(snapshot: List<TimelineEvent>) {
249+
val currentSequences = playlist.map { it.sequence }
250+
val newChunks = snapshot
251+
.mapNotNull { timelineEvent ->
252+
timelineEvent.root.asMessageAudioEvent()
253+
?.takeIf { it.isVoiceBroadcast() && it.getVoiceBroadcastEventId() == voiceBroadcastId && it.sequence !in currentSequences }
254+
}
255+
if (newChunks.isEmpty()) return
256+
updatePlaylist(playlist + newChunks)
257+
258+
when (state) {
259+
State.PLAYING -> {
260+
if (nextMediaPlayer == null) {
261+
coroutineScope.launch { nextMediaPlayer = prepareNextMediaPlayer() }
262+
}
263+
}
264+
State.PAUSED -> {
265+
if (nextMediaPlayer == null) {
266+
coroutineScope.launch { nextMediaPlayer = prepareNextMediaPlayer() }
267+
}
268+
}
269+
State.BUFFERING -> {
270+
val newMediaContent = getNextAudioContent()
271+
if (newMediaContent != null) startPlayback(true)
272+
}
273+
State.IDLE -> startPlayback(true)
274+
}
275+
}
276+
}
277+
278+
private inner class MediaPlayerListener : MediaPlayer.OnInfoListener, MediaPlayer.OnCompletionListener, MediaPlayer.OnErrorListener {
149279

150280
override fun onInfo(mp: MediaPlayer, what: Int, extra: Int): Boolean {
151281
when (what) {
152282
MediaPlayer.MEDIA_INFO_STARTED_AS_NEXT -> {
283+
release(currentMediaPlayer)
153284
currentMediaPlayer = mp
154-
currentPlayingIndex++
155-
mediaPlayerScope.launch { prepareNextFile() }
285+
currentSequence = currentSequence?.plus(1)
286+
coroutineScope.launch { nextMediaPlayer = prepareNextMediaPlayer() }
156287
}
157288
}
158289
return false
159290
}
160291

161292
override fun onCompletion(mp: MediaPlayer) {
162-
// Verify that a new media has not been set in the mean time
163-
if (!currentMediaPlayer?.isPlaying.orFalse()) {
293+
if (nextMediaPlayer != null) return
294+
val roomId = currentRoomId ?: return
295+
val voiceBroadcastId = currentVoiceBroadcastId ?: return
296+
val voiceBroadcastEventContent = getVoiceBroadcastUseCase.execute(roomId, voiceBroadcastId)?.content ?: return
297+
val isLive = voiceBroadcastEventContent.voiceBroadcastState != null && voiceBroadcastEventContent.voiceBroadcastState != VoiceBroadcastState.STOPPED
298+
299+
if (!isLive && voiceBroadcastEventContent.lastChunkSequence == currentSequence) {
300+
// We'll not receive new chunks anymore so we can stop the live listening
164301
stop()
302+
} else {
303+
state = State.BUFFERING
165304
}
166305
}
167306

@@ -170,4 +309,11 @@ class VoiceBroadcastPlayer @Inject constructor(
170309
return true
171310
}
172311
}
312+
313+
enum class State {
314+
PLAYING,
315+
PAUSED,
316+
BUFFERING,
317+
IDLE
318+
}
173319
}

‎vector/src/main/java/im/vector/app/features/voicebroadcast/VoiceBroadcastRecorder.kt

+1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import java.io.File
2323
interface VoiceBroadcastRecorder : VoiceRecorder {
2424

2525
var listener: Listener?
26+
var currentSequence: Int
2627

2728
fun startRecord(roomId: String, chunkLength: Int)
2829

0 commit comments

Comments
 (0)