Skip to content

Commit c9ec26f

Browse files
jmartinespjmartinesp
authored
When transcoding a video fails, send it as a file (#4257)
- If the video can't be transcoded it will be uploaded as a file instead. - If the video already has the right format and dimensions, don't transcode it. - Update the dimensions to 720p max when enabling media compression and 1080p otherwise, matching Element X iOS.
1 parent 4a702ba commit c9ec26f

File tree

3 files changed

+277
-28
lines changed

3 files changed

+277
-28
lines changed

‎libraries/mediaupload/impl/src/main/kotlin/io/element/android/libraries/mediaupload/impl/AndroidMediaPreProcessor.kt

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -188,20 +188,29 @@ class AndroidMediaPreProcessor @Inject constructor(
188188
}
189189

190190
private suspend fun processVideo(uri: Uri, mimeType: String?, shouldBeCompressed: Boolean): MediaUploadInfo {
191-
val resultFile = videoCompressor.compress(uri, shouldBeCompressed)
192-
.onEach {
193-
// TODO handle progress
194-
}
195-
.filterIsInstance<VideoTranscodingEvent.Completed>()
196-
.first()
197-
.file
198-
val thumbnailInfo = thumbnailFactory.createVideoThumbnail(resultFile)
199-
val videoInfo = extractVideoMetadata(resultFile, mimeType, thumbnailInfo)
200-
return MediaUploadInfo.Video(
201-
file = resultFile,
202-
videoInfo = videoInfo,
203-
thumbnailFile = thumbnailInfo?.file
204-
)
191+
val resultFile = runCatching {
192+
videoCompressor.compress(uri, shouldBeCompressed)
193+
.onEach {
194+
// TODO handle progress
195+
}
196+
.filterIsInstance<VideoTranscodingEvent.Completed>()
197+
.first()
198+
.file
199+
}
200+
.getOrNull()
201+
202+
if (resultFile != null) {
203+
val thumbnailInfo = thumbnailFactory.createVideoThumbnail(resultFile)
204+
val videoInfo = extractVideoMetadata(resultFile, mimeType, thumbnailInfo)
205+
return MediaUploadInfo.Video(
206+
file = resultFile,
207+
videoInfo = videoInfo,
208+
thumbnailFile = thumbnailInfo?.file
209+
)
210+
} else {
211+
// If the video could not be compressed, just use the original one, but send it as a file
212+
return processFile(uri, MimeTypes.OctetStream)
213+
}
205214
}
206215

207216
private suspend fun processAudio(uri: Uri, mimeType: String?): MediaUploadInfo {

‎libraries/mediaupload/impl/src/main/kotlin/io/element/android/libraries/mediaupload/impl/VideoCompressor.kt

Lines changed: 101 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,39 +8,49 @@
88
package io.element.android.libraries.mediaupload.impl
99

1010
import android.content.Context
11+
import android.media.MediaMetadataRetriever
1112
import android.net.Uri
13+
import android.webkit.MimeTypeMap
1214
import com.otaliastudios.transcoder.Transcoder
1315
import com.otaliastudios.transcoder.TranscoderListener
16+
import com.otaliastudios.transcoder.internal.media.MediaFormatConstants
1417
import com.otaliastudios.transcoder.resize.AtMostResizer
1518
import com.otaliastudios.transcoder.strategy.DefaultVideoStrategy
19+
import com.otaliastudios.transcoder.strategy.PassThroughTrackStrategy
20+
import com.otaliastudios.transcoder.strategy.TrackStrategy
21+
import com.otaliastudios.transcoder.validator.WriteAlwaysValidator
1622
import io.element.android.libraries.androidutils.file.createTmpFile
23+
import io.element.android.libraries.androidutils.file.getMimeType
1724
import io.element.android.libraries.androidutils.file.safeDelete
1825
import io.element.android.libraries.di.ApplicationContext
1926
import kotlinx.coroutines.channels.awaitClose
2027
import kotlinx.coroutines.flow.callbackFlow
28+
import timber.log.Timber
2129
import java.io.File
2230
import javax.inject.Inject
2331

32+
private const val MP4_EXTENSION = "mp4"
33+
2434
class VideoCompressor @Inject constructor(
2535
@ApplicationContext private val context: Context,
2636
) {
2737
fun compress(uri: Uri, shouldBeCompressed: Boolean) = callbackFlow {
28-
val tmpFile = context.createTmpFile(extension = "mp4")
38+
val metadata = getVideoMetadata(uri)
39+
40+
val expectedExtension = MimeTypeMap.getSingleton().getExtensionFromMimeType(context.getMimeType(uri))
41+
42+
val videoStrategy = VideoStrategyFactory.create(
43+
expectedExtension = expectedExtension,
44+
metadata = metadata,
45+
shouldBeCompressed = shouldBeCompressed
46+
)
47+
48+
val tmpFile = context.createTmpFile(extension = MP4_EXTENSION)
2949
val future = Transcoder.into(tmpFile.path)
30-
.setVideoTrackStrategy(
31-
DefaultVideoStrategy.Builder()
32-
.addResizer(
33-
AtMostResizer(
34-
if (shouldBeCompressed) {
35-
720
36-
} else {
37-
1080
38-
}
39-
)
40-
)
41-
.build()
42-
)
50+
.setVideoTrackStrategy(videoStrategy)
4351
.addDataSource(context, uri)
52+
// Force the output to be written, even if no transcoding was actually needed
53+
.setValidator(WriteAlwaysValidator())
4454
.setListener(object : TranscoderListener {
4555
override fun onTranscodeProgress(progress: Double) {
4656
trySend(VideoTranscodingEvent.Progress(progress.toFloat()))
@@ -69,9 +79,86 @@ class VideoCompressor @Inject constructor(
6979
}
7080
}
7181
}
82+
83+
private fun getVideoMetadata(uri: Uri): VideoFileMetadata? {
84+
return runCatching {
85+
MediaMetadataRetriever().use {
86+
it.setDataSource(context, uri)
87+
88+
val width = it.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH)?.toIntOrNull() ?: -1
89+
val height = it.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)?.toIntOrNull() ?: -1
90+
val bitrate = it.extractMetadata(MediaMetadataRetriever.METADATA_KEY_BITRATE)?.toLongOrNull() ?: -1
91+
val framerate = it.extractMetadata(MediaMetadataRetriever.METADATA_KEY_CAPTURE_FRAMERATE)?.toIntOrNull() ?: -1
92+
93+
val (actualWidth, actualHeight) = if (width == -1 || height == -1) {
94+
// Try getting the first frame instead
95+
val bitmap = it.getFrameAtTime(0) ?: return null
96+
bitmap.width to bitmap.height
97+
} else {
98+
width to height
99+
}
100+
101+
VideoFileMetadata(
102+
width = actualWidth,
103+
height = actualHeight,
104+
bitrate = bitrate,
105+
frameRate = framerate
106+
)
107+
}
108+
}.onFailure {
109+
Timber.e(it, "Failed to get video dimensions")
110+
}.getOrNull()
111+
}
72112
}
73113

114+
internal data class VideoFileMetadata(
115+
val width: Int?,
116+
val height: Int?,
117+
val bitrate: Long?,
118+
val frameRate: Int?,
119+
)
120+
74121
sealed interface VideoTranscodingEvent {
75122
data class Progress(val value: Float) : VideoTranscodingEvent
76123
data class Completed(val file: File) : VideoTranscodingEvent
77124
}
125+
126+
internal object VideoStrategyFactory {
127+
// 720p
128+
private const val MAX_COMPRESSED_PIXEL_SIZE = 1280
129+
130+
// 1080p
131+
private const val MAX_PIXEL_SIZE = 1920
132+
133+
fun create(
134+
expectedExtension: String?,
135+
metadata: VideoFileMetadata?,
136+
shouldBeCompressed: Boolean,
137+
): TrackStrategy {
138+
val width = metadata?.width ?: Int.MAX_VALUE
139+
val height = metadata?.height ?: Int.MAX_VALUE
140+
val bitrate = metadata?.bitrate
141+
val frameRate = metadata?.frameRate
142+
143+
// We only create a resizer if needed
144+
val resizer = when {
145+
shouldBeCompressed && (width > MAX_COMPRESSED_PIXEL_SIZE || height > MAX_COMPRESSED_PIXEL_SIZE) -> AtMostResizer(MAX_COMPRESSED_PIXEL_SIZE)
146+
width > MAX_PIXEL_SIZE || height > MAX_PIXEL_SIZE -> AtMostResizer(MAX_PIXEL_SIZE)
147+
else -> null
148+
}
149+
150+
return if (resizer == null && expectedExtension == MP4_EXTENSION) {
151+
// If there's no transcoding or resizing needed for the video file, just create a new file with the same contents but no metadata
152+
PassThroughTrackStrategy()
153+
} else {
154+
DefaultVideoStrategy.Builder()
155+
.apply {
156+
resizer?.let { addResizer(it) }
157+
bitrate?.let { bitRate(it) }
158+
frameRate?.let { frameRate(it) }
159+
}
160+
.mimeType(MediaFormatConstants.MIMETYPE_VIDEO_AVC)
161+
.build()
162+
}
163+
}
164+
}

‎libraries/mediaupload/impl/src/test/kotlin/io/element/android/libraries/mediaupload/impl/VideoStrategyFactoryTest.kt

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
/*
2+
* Copyright 2025 New Vector Ltd.
3+
*
4+
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
5+
* Please see LICENSE files in the repository root for full details.
6+
*/
7+
8+
package io.element.android.libraries.mediaupload.impl
9+
10+
import com.otaliastudios.transcoder.strategy.DefaultVideoStrategy
11+
import com.otaliastudios.transcoder.strategy.PassThroughTrackStrategy
12+
import com.otaliastudios.transcoder.strategy.TrackStrategy
13+
import org.junit.Test
14+
import org.junit.runner.RunWith
15+
import org.robolectric.RobolectricTestRunner
16+
17+
@Suppress("NOTHING_TO_INLINE")
18+
@RunWith(RobolectricTestRunner::class)
19+
class VideoStrategyFactoryTest {
20+
@Test
21+
fun `if we don't have metadata the video will be transcoded just in case`() {
22+
// Given
23+
val expectedExtension = "mp4"
24+
val metadata = null
25+
val shouldBeCompressed = true
26+
27+
// When
28+
val videoStrategy = VideoStrategyFactory.create(
29+
expectedExtension = expectedExtension,
30+
metadata = metadata,
31+
shouldBeCompressed = shouldBeCompressed
32+
)
33+
34+
// Then
35+
assertIsTranscoded(videoStrategy)
36+
}
37+
38+
@Test
39+
fun `if the video should be compressed and is larger than 720p it will be transcoded`() {
40+
// Given
41+
val expectedExtension = "mp4"
42+
val metadata = VideoFileMetadata(width = 1920, height = 1080, bitrate = 1_000_000, frameRate = 50)
43+
val shouldBeCompressed = true
44+
45+
// When
46+
val videoStrategy = VideoStrategyFactory.create(
47+
expectedExtension = expectedExtension,
48+
metadata = metadata,
49+
shouldBeCompressed = shouldBeCompressed
50+
)
51+
52+
// Then
53+
assertIsTranscoded(videoStrategy)
54+
}
55+
56+
@Test
57+
fun `if the video should be compressed, has the right format and is smaller or equal to 720p it will not be transcoded`() {
58+
// Given
59+
val expectedExtension = "mp4"
60+
val metadata = VideoFileMetadata(width = 1280, height = 720, bitrate = 1_000_000, frameRate = 50)
61+
val shouldBeCompressed = true
62+
63+
// When
64+
val videoStrategy = VideoStrategyFactory.create(
65+
expectedExtension = expectedExtension,
66+
metadata = metadata,
67+
shouldBeCompressed = shouldBeCompressed
68+
)
69+
70+
// Then
71+
assertIsNotTranscoded(videoStrategy)
72+
}
73+
74+
@Test
75+
fun `if the video should not be compressed and is larger than 1080p it will be transcoded`() {
76+
// Given
77+
val expectedExtension = "mp4"
78+
val metadata = VideoFileMetadata(width = 2560, height = 1440, bitrate = 1_000_000, frameRate = 50)
79+
val shouldBeCompressed = false
80+
81+
// When
82+
val videoStrategy = VideoStrategyFactory.create(
83+
expectedExtension = expectedExtension,
84+
metadata = metadata,
85+
shouldBeCompressed = shouldBeCompressed
86+
)
87+
88+
// Then
89+
assertIsTranscoded(videoStrategy)
90+
}
91+
92+
@Test
93+
fun `if the video should not be compressed, has the right format and is smaller or equal than 1080p it will not be transcoded`() {
94+
// Given
95+
val expectedExtension = "mp4"
96+
val metadata = VideoFileMetadata(width = 1920, height = 1080, bitrate = 1_000_000, frameRate = 50)
97+
val shouldBeCompressed = false
98+
99+
// When
100+
val videoStrategy = VideoStrategyFactory.create(
101+
expectedExtension = expectedExtension,
102+
metadata = metadata,
103+
shouldBeCompressed = shouldBeCompressed
104+
)
105+
106+
// Then
107+
assertIsNotTranscoded(videoStrategy)
108+
}
109+
110+
@Test
111+
fun `if the video should not be compressed but has a wrong format it will be transcoded`() {
112+
// Given
113+
val expectedExtension = "mkv"
114+
val metadata = VideoFileMetadata(width = 320, height = 240, bitrate = 1_000_000, frameRate = 50)
115+
val shouldBeCompressed = false
116+
117+
// When
118+
val videoStrategy = VideoStrategyFactory.create(
119+
expectedExtension = expectedExtension,
120+
metadata = metadata,
121+
shouldBeCompressed = shouldBeCompressed
122+
)
123+
124+
// Then
125+
assertIsTranscoded(videoStrategy)
126+
}
127+
128+
@Test
129+
fun `if the video should be compressed and has a wrong format it will be transcoded`() {
130+
// Given
131+
val expectedExtension = "mkv"
132+
val metadata = VideoFileMetadata(width = 320, height = 240, bitrate = 1_000_000, frameRate = 50)
133+
val shouldBeCompressed = true
134+
135+
// When
136+
val videoStrategy = VideoStrategyFactory.create(
137+
expectedExtension = expectedExtension,
138+
metadata = metadata,
139+
shouldBeCompressed = shouldBeCompressed
140+
)
141+
142+
// Then
143+
assertIsTranscoded(videoStrategy)
144+
}
145+
146+
private inline fun assertIsTranscoded(videoStrategy: TrackStrategy) {
147+
assert(videoStrategy is DefaultVideoStrategy)
148+
}
149+
150+
private inline fun assertIsNotTranscoded(videoStrategy: TrackStrategy) {
151+
assert(videoStrategy is PassThroughTrackStrategy)
152+
}
153+
}

0 commit comments

Comments
 (0)