Skip to content

Commit 0500bf0

Browse files
fufesoufufesou
authored
refact: android audio input, voice call (rustdesk#8037)
Signed-off-by: fufesou <[email protected]>
1 parent d70b0cd commit 0500bf0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+610
-148
lines changed

‎flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/AudioRecordHandle.kt

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
package com.carriez.flutter_hbb
2+
3+
import ffi.FFI
4+
5+
import android.Manifest
6+
import android.content.Context
7+
import android.media.*
8+
import android.content.pm.PackageManager
9+
import android.media.projection.MediaProjection
10+
import androidx.annotation.RequiresApi
11+
import androidx.core.app.ActivityCompat
12+
import android.os.Build
13+
import android.util.Log
14+
import kotlin.concurrent.thread
15+
16+
const val AUDIO_ENCODING = AudioFormat.ENCODING_PCM_FLOAT // ENCODING_OPUS need API 30
17+
const val AUDIO_SAMPLE_RATE = 48000
18+
const val AUDIO_CHANNEL_MASK = AudioFormat.CHANNEL_IN_STEREO
19+
20+
class AudioRecordHandle(private var context: Context, private var isVideoStart: ()->Boolean, private var isAudioStart: ()->Boolean) {
21+
private val logTag = "LOG_AUDIO_RECORD_HANDLE"
22+
23+
private var audioRecorder: AudioRecord? = null
24+
private var audioReader: AudioReader? = null
25+
private var minBufferSize = 0
26+
private var audioRecordStat = false
27+
private var audioThread: Thread? = null
28+
29+
@RequiresApi(Build.VERSION_CODES.M)
30+
fun createAudioRecorder(inVoiceCall: Boolean, mediaProjection: MediaProjection?): Boolean {
31+
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
32+
return false
33+
}
34+
if (ActivityCompat.checkSelfPermission(
35+
context,
36+
Manifest.permission.RECORD_AUDIO
37+
) != PackageManager.PERMISSION_GRANTED
38+
) {
39+
Log.d(logTag, "createAudioRecorder failed, no RECORD_AUDIO permission")
40+
return false
41+
}
42+
43+
var builder = AudioRecord.Builder()
44+
.setAudioFormat(
45+
AudioFormat.Builder()
46+
.setEncoding(AUDIO_ENCODING)
47+
.setSampleRate(AUDIO_SAMPLE_RATE)
48+
.setChannelMask(AUDIO_CHANNEL_MASK).build()
49+
);
50+
if (inVoiceCall) {
51+
builder.setAudioSource(MediaRecorder.AudioSource.VOICE_COMMUNICATION)
52+
} else {
53+
mediaProjection?.let {
54+
var apcc = AudioPlaybackCaptureConfiguration.Builder(it)
55+
.addMatchingUsage(AudioAttributes.USAGE_MEDIA)
56+
.addMatchingUsage(AudioAttributes.USAGE_ALARM)
57+
.addMatchingUsage(AudioAttributes.USAGE_GAME)
58+
.addMatchingUsage(AudioAttributes.USAGE_UNKNOWN).build();
59+
builder.setAudioPlaybackCaptureConfig(apcc);
60+
} ?: let {
61+
Log.d(logTag, "createAudioRecorder failed, mediaProjection null")
62+
return false
63+
}
64+
}
65+
audioRecorder = builder.build()
66+
Log.d(logTag, "createAudioRecorder done,minBufferSize:$minBufferSize")
67+
return true
68+
}
69+
70+
@RequiresApi(Build.VERSION_CODES.M)
71+
private fun checkAudioReader() {
72+
if (audioReader != null && minBufferSize != 0) {
73+
return
74+
}
75+
// read f32 to byte , length * 4
76+
minBufferSize = 2 * 4 * AudioRecord.getMinBufferSize(
77+
AUDIO_SAMPLE_RATE,
78+
AUDIO_CHANNEL_MASK,
79+
AUDIO_ENCODING
80+
)
81+
if (minBufferSize == 0) {
82+
Log.d(logTag, "get min buffer size fail!")
83+
return
84+
}
85+
audioReader = AudioReader(minBufferSize, 4)
86+
Log.d(logTag, "init audioData len:$minBufferSize")
87+
}
88+
89+
@RequiresApi(Build.VERSION_CODES.M)
90+
fun startAudioRecorder() {
91+
checkAudioReader()
92+
if (audioReader != null && audioRecorder != null && minBufferSize != 0) {
93+
try {
94+
FFI.setFrameRawEnable("audio", true)
95+
audioRecorder!!.startRecording()
96+
audioRecordStat = true
97+
audioThread = thread {
98+
while (audioRecordStat) {
99+
audioReader!!.readSync(audioRecorder!!)?.let {
100+
FFI.onAudioFrameUpdate(it)
101+
}
102+
}
103+
// let's release here rather than onDestroy to avoid threading issue
104+
audioRecorder?.release()
105+
audioRecorder = null
106+
minBufferSize = 0
107+
FFI.setFrameRawEnable("audio", false)
108+
Log.d(logTag, "Exit audio thread")
109+
}
110+
} catch (e: Exception) {
111+
Log.d(logTag, "startAudioRecorder fail:$e")
112+
}
113+
} else {
114+
Log.d(logTag, "startAudioRecorder fail")
115+
}
116+
}
117+
118+
fun onVoiceCallStarted(mediaProjection: MediaProjection?): Boolean {
119+
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
120+
return false
121+
}
122+
if (isVideoStart() || isAudioStart()) {
123+
if (!switchToVoiceCall(mediaProjection)) {
124+
return false
125+
}
126+
} else {
127+
if (!switchToVoiceCall(mediaProjection)) {
128+
return false
129+
}
130+
}
131+
return true
132+
}
133+
134+
fun onVoiceCallClosed(mediaProjection: MediaProjection?): Boolean {
135+
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
136+
return false
137+
}
138+
if (isVideoStart()) {
139+
switchOutVoiceCall(mediaProjection)
140+
}
141+
tryReleaseAudio()
142+
return true
143+
}
144+
145+
@RequiresApi(Build.VERSION_CODES.M)
146+
fun switchToVoiceCall(mediaProjection: MediaProjection?): Boolean {
147+
audioRecorder?.let {
148+
if (it.getAudioSource() == MediaRecorder.AudioSource.VOICE_COMMUNICATION) {
149+
return true
150+
}
151+
}
152+
audioRecordStat = false
153+
audioThread?.join()
154+
audioThread = null
155+
156+
if (!createAudioRecorder(true, mediaProjection)) {
157+
Log.e(logTag, "createAudioRecorder fail")
158+
return false
159+
}
160+
startAudioRecorder()
161+
return true
162+
}
163+
164+
@RequiresApi(Build.VERSION_CODES.M)
165+
fun switchOutVoiceCall(mediaProjection: MediaProjection?): Boolean {
166+
audioRecorder?.let {
167+
if (it.getAudioSource() != MediaRecorder.AudioSource.VOICE_COMMUNICATION) {
168+
return true
169+
}
170+
}
171+
audioRecordStat = false
172+
audioThread?.join()
173+
174+
if (!createAudioRecorder(false, mediaProjection)) {
175+
Log.e(logTag, "createAudioRecorder fail")
176+
return false
177+
}
178+
startAudioRecorder()
179+
return true
180+
}
181+
182+
fun tryReleaseAudio() {
183+
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
184+
return
185+
}
186+
if (isAudioStart() || isVideoStart()) {
187+
return
188+
}
189+
audioRecordStat = false
190+
audioThread?.join()
191+
audioThread = null
192+
}
193+
194+
fun destroy() {
195+
Log.d(logTag, "destroy audio record handle")
196+
197+
audioRecordStat = false
198+
audioThread?.join()
199+
}
200+
}

‎flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ class MainActivity : FlutterActivity() {
4242
private val logTag = "mMainActivity"
4343
private var mainService: MainService? = null
4444

45+
private var isAudioStart = false
46+
private val audioRecordHandle = AudioRecordHandle(this, { false }, { isAudioStart })
47+
4548
override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
4649
super.configureFlutterEngine(flutterEngine)
4750
if (MainService.isReady) {
@@ -230,6 +233,12 @@ class MainActivity : FlutterActivity() {
230233
result.success(false)
231234
}
232235
}
236+
"on_voice_call_started" -> {
237+
onVoiceCallStarted()
238+
}
239+
"on_voice_call_closed" -> {
240+
onVoiceCallClosed()
241+
}
233242
else -> {
234243
result.error("-1", "No such method", null)
235244
}
@@ -319,4 +328,44 @@ class MainActivity : FlutterActivity() {
319328
result.put("codecs", codecArray)
320329
FFI.setCodecInfo(result.toString())
321330
}
331+
332+
private fun onVoiceCallStarted() {
333+
var ok = false
334+
mainService?.let {
335+
ok = it.onVoiceCallStarted()
336+
} ?: let {
337+
isAudioStart = true
338+
ok = audioRecordHandle.onVoiceCallStarted(null)
339+
}
340+
if (!ok) {
341+
// Rarely happens, So we just add log and msgbox here.
342+
Log.e(logTag, "onVoiceCallStarted fail")
343+
flutterMethodChannel?.invokeMethod("msgbox", mapOf(
344+
"type" to "custom-nook-nocancel-hasclose-error",
345+
"title" to "Voice call",
346+
"text" to "Failed to start voice call."))
347+
} else {
348+
Log.d(logTag, "onVoiceCallStarted success")
349+
}
350+
}
351+
352+
private fun onVoiceCallClosed() {
353+
var ok = false
354+
mainService?.let {
355+
ok = it.onVoiceCallClosed()
356+
} ?: let {
357+
isAudioStart = false
358+
ok = audioRecordHandle.onVoiceCallClosed(null)
359+
}
360+
if (!ok) {
361+
// Rarely happens, So we just add log and msgbox here.
362+
Log.e(logTag, "onVoiceCallClosed fail")
363+
flutterMethodChannel?.invokeMethod("msgbox", mapOf(
364+
"type" to "custom-nook-nocancel-hasclose-error",
365+
"title" to "Voice call",
366+
"text" to "Failed to stop voice call."))
367+
} else {
368+
Log.d(logTag, "onVoiceCallClosed success")
369+
}
370+
}
322371
}

0 commit comments

Comments
 (0)