Skip to content

Commit b9183d9

Browse files
danielpaulusdanielpaulus
authored
Add audio recording mode (#61)
* add audio command * finished audio only commands, refactor gstadapter into separate pipelinebuilders for mac and linux * gitignore ogg,wav and mp3 * fix unit test, we send messages in a different order now * bump up version to 0.5-beta
1 parent 431269c commit b9183d9

9 files changed

+320
-413
lines changed

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
.vscode
22
quicktime_video_hack
3+
*.wav
4+
*.ogg
5+
*.mp3
36
# Binaries for programs and plugins
47
main
58
*.h264

‎main.go

+80-14
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,17 @@ import (
1414
log "github.com/sirupsen/logrus"
1515
)
1616

17-
const version = "v0.3-beta"
17+
const version = "v0.5-beta"
1818

1919
func main() {
2020
usage := fmt.Sprintf(`Q.uickTime V.ideo H.ack (qvh) %s
2121
2222
Usage:
2323
qvh devices [-v]
2424
qvh activate [--udid=<udid>] [-v]
25-
qvh record <h264file> <wavfile> [-v] [--udid=<udid>]
26-
qvh gstreamer [--pipeline=<pipeline>] [--examples] [-v]
25+
qvh record <h264file> <wavfile> [--udid=<udid>] [-v]
26+
qvh audio <outfile> (--mp3 | --ogg | --wav) [--udid=<udid>] [-v]
27+
qvh gstreamer [--pipeline=<pipeline>] [--examples] [--udid=<udid>] [-v]
2728
qvh --version | version
2829
2930
@@ -35,14 +36,20 @@ Options:
3536
3637
The commands work as following:
3738
devices lists iOS devices attached to this host and tells you if video streaming was activated for them
39+
3840
activate enables the video streaming config for the device specified by --udid
41+
3942
record will start video&audio recording. Video will be saved in a raw h264 file playable by VLC.
40-
Audio will be saved in a uncompressed wav file. Run like: "qvh record /home/yourname/out.h264 /home/yourname/out.wav"
43+
Audio will be saved in a uncompressed wav file. Run like: "qvh record /home/yourname/out.h264 /home/yourname/out.wav"
44+
45+
audio Records only audio from the device. It does not change the status bar like the video recording mode does.
46+
The recorded audio will be saved in <outfile> with the selected format. Currently (--mp3 | --ogg | --wav) are supported.
47+
Adding more formats is trivial though so create an issue or a PR if you need something :-)
4148
42-
gstreamer If no additional param is provided, qvh will open a new window and push AV data to gstreamer.
43-
If "qvh gstreamer --examples" is provided, qvh will print some common gstreamer pipeline examples.
44-
If --pipeline is provided, qvh will use the provided gstreamer pipeline instead of
45-
displaying audio and video in a window.
49+
gstreamer If no additional param is provided, qvh will open a new window and push AV data to gstreamer.
50+
If "qvh gstreamer --examples" is provided, qvh will print some common gstreamer pipeline examples.
51+
If --pipeline is provided, qvh will use the provided gstreamer pipeline instead of
52+
displaying audio and video in a window.
4653
`, version)
4754
arguments, _ := docopt.ParseDoc(usage)
4855
log.SetFormatter(&log.JSONFormatter{})
@@ -73,7 +80,29 @@ The commands work as following:
7380
activate(udid)
7481
return
7582
}
76-
83+
audioCommand, _ := arguments.Bool("audio")
84+
if audioCommand {
85+
outfile, err := arguments.String("<outfile>")
86+
if err != nil {
87+
printErrJSON(err, "Missing <outfile> parameter. Please specify a valid path like '/home/me/out.h264'")
88+
return
89+
}
90+
log.Infof("Recording audio only to file: %s", outfile)
91+
mp3, _ := arguments.Bool("--mp3")
92+
ogg, _ := arguments.Bool("--ogg")
93+
wav, _ := arguments.Bool("--wav")
94+
log.Debugf("recording audio only format mp3:%t ogg: %t wav:%t to file: %s", mp3, ogg, wav, outfile)
95+
if wav {
96+
recordAudioWav(outfile, udid)
97+
return
98+
}
99+
if ogg {
100+
recordAudioGst(outfile, udid, gstadapter.OGG)
101+
return
102+
}
103+
recordAudioGst(outfile, udid, gstadapter.MP3)
104+
return
105+
}
77106
recordCommand, _ := arguments.Bool("record")
78107
if recordCommand {
79108
h264FilePath, err := arguments.String("<h264file>")
@@ -137,20 +166,57 @@ func printExamples() {
137166
fmt.Print(examples)
138167
}
139168

169+
func recordAudioGst(outfile string, udid string, audiotype string) {
170+
log.Debug("Starting Gstreamer with audio pipeline")
171+
gStreamer, err := gstadapter.NewWithAudioPipeline(outfile, audiotype)
172+
if err != nil {
173+
printErrJSON(err, "Failed creating custom pipeline")
174+
return
175+
}
176+
startWithConsumer(gStreamer, udid, true)
177+
}
178+
179+
func recordAudioWav(outfile string, udid string) {
180+
log.Debug("Starting Gstreamer with audio pipeline")
181+
wavFile, err := os.Create(outfile)
182+
if err != nil {
183+
log.Debugf("Error creating wav file:%s", err)
184+
log.Errorf("Could not open wav file '%s'", outfile)
185+
}
186+
wavFileWriter := coremedia.NewAVFileWriterAudioOnly(wavFile)
187+
188+
defer func() {
189+
stat, err := wavFile.Stat()
190+
if err != nil {
191+
log.Fatal("Could not get wav file stats", err)
192+
}
193+
err = coremedia.WriteWavHeader(int(stat.Size()), wavFile)
194+
if err != nil {
195+
log.Fatalf("Error writing wave header %s might be invalid. %s", outfile, err.Error())
196+
}
197+
err = wavFile.Close()
198+
if err != nil {
199+
log.Fatalf("Error closing wave file. '%s' might be invalid. %s", outfile, err.Error())
200+
}
201+
202+
}()
203+
startWithConsumer(wavFileWriter, udid, true)
204+
}
205+
140206
func startGStreamerWithCustomPipeline(udid string, pipelineString string) {
141207
log.Debug("Starting Gstreamer with custom pipeline")
142208
gStreamer, err := gstadapter.NewWithCustomPipeline(pipelineString)
143209
if err != nil {
144210
printErrJSON(err, "Failed creating custom pipeline")
145211
return
146212
}
147-
startWithConsumer(gStreamer, udid)
213+
startWithConsumer(gStreamer, udid, false)
148214
}
149215

150216
func startGStreamer(udid string) {
151217
log.Debug("Starting Gstreamer")
152218
gStreamer := gstadapter.New()
153-
startWithConsumer(gStreamer, udid)
219+
startWithConsumer(gStreamer, udid, false)
154220
}
155221

156222
// Just dump a list of what was discovered to the console
@@ -224,10 +290,10 @@ func record(h264FilePath string, wavFilePath string, udid string) {
224290
}
225291

226292
}()
227-
startWithConsumer(writer, udid)
293+
startWithConsumer(writer, udid, false)
228294
}
229295

230-
func startWithConsumer(consumer screencapture.CmSampleBufConsumer, udid string) {
296+
func startWithConsumer(consumer screencapture.CmSampleBufConsumer, udid string, audioOnly bool) {
231297
device, err := screencapture.FindIosDevice(udid)
232298
if err != nil {
233299
printErrJSON(err, "no device found to activate")
@@ -244,7 +310,7 @@ func startWithConsumer(consumer screencapture.CmSampleBufConsumer, udid string)
244310
stopSignal := make(chan interface{})
245311
waitForSigInt(stopSignal)
246312

247-
mp := screencapture.NewMessageProcessor(&adapter, stopSignal, consumer)
313+
mp := screencapture.NewMessageProcessor(&adapter, stopSignal, consumer, audioOnly)
248314

249315
err = adapter.StartReading(device, &mp, stopSignal)
250316
consumer.Stop()

‎screencapture/coremedia/avfilewriter.go

+9-1
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,17 @@ type AVFileWriter struct {
1313
h264FileWriter io.Writer
1414
wavFileWriter io.Writer
1515
outFilePath string
16+
audioOnly bool
1617
}
1718

1819
//NewAVFileWriter binary writes nalus in annex b format to the given writer and audio buffers into a wav file.
1920
//Note that you will have to call WriteWavHeader() on the audiofile when you are done to write a wav header and get a valid file.
2021
func NewAVFileWriter(h264FileWriter io.Writer, wavFileWriter io.Writer) AVFileWriter {
21-
return AVFileWriter{h264FileWriter: h264FileWriter, wavFileWriter: wavFileWriter}
22+
return AVFileWriter{h264FileWriter: h264FileWriter, wavFileWriter: wavFileWriter, audioOnly: false}
23+
}
24+
25+
func NewAVFileWriterAudioOnly(wavFileWriter io.Writer) AVFileWriter {
26+
return AVFileWriter{h264FileWriter: nil, wavFileWriter: wavFileWriter, audioOnly: true}
2227
}
2328

2429
//Consume writes PPS and SPS as well as sample bufs into a annex b .h264 file and audio samples into a wav file
@@ -27,6 +32,9 @@ func (avfw AVFileWriter) Consume(buf CMSampleBuffer) error {
2732
if buf.MediaType == MediaTypeSound {
2833
return avfw.consumeAudio(buf)
2934
}
35+
if avfw.audioOnly {
36+
return nil
37+
}
3038
return avfw.consumeVideo(buf)
3139
}
3240

‎screencapture/gstadapter/gst_adapter_macos.go renamed to ‎screencapture/gstadapter/gst_adapter.go

+68-56
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
// +build darwin
2-
31
package gstadapter
42

53
import (
@@ -25,14 +23,18 @@ type GstAdapter struct {
2523
const audioAppSrcTargetElementName = "audio_target"
2624
const videoAppSrcTargetElementName = "video_target"
2725

26+
const MP3 = "mp3"
27+
const OGG = "ogg"
28+
2829
//New creates a new MAC OSX compatible gstreamer pipeline that will play device video and audio
2930
//in a nice little window :-D
3031
func New() *GstAdapter {
3132
log.Info("Starting Gstreamer..")
3233
pl := gst.NewPipeline("QT_Hack_Pipeline")
3334

3435
videoAppSrc := setUpVideoPipeline(pl)
35-
audioAppSrc := setUpAudioPipeline(pl)
36+
audioAppSrc := setUpAudioPipelineBase(pl)
37+
setupLivePlayAudio(pl)
3638

3739
pl.SetState(gst.STATE_PLAYING)
3840
runGlibMainLoop()
@@ -43,6 +45,29 @@ func New() *GstAdapter {
4345
return &gsta
4446
}
4547

48+
func NewWithAudioPipeline(outfile string, audiotype string) (*GstAdapter, error) {
49+
log.Info("Starting Gstreamer..")
50+
pl := gst.NewPipeline("QT_Hack_Pipeline")
51+
52+
audioAppSrc := setUpAudioPipelineBase(pl)
53+
switch audiotype {
54+
case MP3:
55+
setupMp3(pl, outfile)
56+
case OGG:
57+
setupVorbis(pl, outfile)
58+
default:
59+
log.Fatalf("Unrecognized Audio type:%s", audiotype)
60+
}
61+
62+
pl.SetState(gst.STATE_PLAYING)
63+
runGlibMainLoop()
64+
65+
log.Info("Gstreamer is running!")
66+
gsta := GstAdapter{audioAppSrc: audioAppSrc, firstAudioSample: true}
67+
68+
return &gsta, nil
69+
}
70+
4671
//NewWithCustomPipeline will parse the given pipelineString, connect the videoAppSrc to whatever element has the name "video_target" and the audioAppSrc to "audio_target"
4772
//see also: https://gstreamer.freedesktop.org/documentation/application-development/appendix/programs.html?gi-language=c
4873
func NewWithCustomPipeline(pipelineString string) (*GstAdapter, error) {
@@ -89,13 +114,17 @@ func NewWithCustomPipeline(pipelineString string) (*GstAdapter, error) {
89114
//sending EOS will result in a broken mp4 file
90115
func (gsta GstAdapter) Stop() {
91116
log.Info("Stopping Gstreamer..")
92-
success := gsta.audioAppSrc.SendEvent(gst.Eos())
93-
if !success {
94-
log.Warn("Failed sending EOS signal for audio app source")
117+
if gsta.audioAppSrc != nil {
118+
success := gsta.audioAppSrc.SendEvent(gst.Eos())
119+
if !success {
120+
log.Warn("Failed sending EOS signal for audio app source")
121+
}
95122
}
96-
success = gsta.videoAppSrc.SendEvent(gst.Eos())
97-
if !success {
98-
log.Warn("Failed sending EOS signal for video app source")
123+
if gsta.videoAppSrc != nil {
124+
success := gsta.videoAppSrc.SendEvent(gst.Eos())
125+
if !success {
126+
log.Warn("Failed sending EOS signal for video app source")
127+
}
99128
}
100129

101130
if gsta.pipeline == nil {
@@ -126,8 +155,7 @@ func runGlibMainLoop() {
126155
glib.NewMainLoop(nil).Run()
127156
}()
128157
}
129-
130-
func setUpAudioPipeline(pl *gst.Pipeline) *gst.AppSrc {
158+
func setUpAudioPipelineBase(pl *gst.Pipeline) *gst.AppSrc {
131159
asrc := gst.NewAppSrc("my-audio-src")
132160
asrc.SetProperty("is-live", true)
133161

@@ -144,59 +172,43 @@ func setUpAudioPipeline(pl *gst.Pipeline) *gst.AppSrc {
144172
audioconvert := gst.ElementFactoryMake("audioconvert", "audioconvert_01")
145173
checkElem(audioconvert, "audioconvert_01")
146174

147-
autoaudiosink := gst.ElementFactoryMake("autoaudiosink", "autoaudiosink_01")
148-
checkElem(autoaudiosink, "autoaudiosink_01")
149-
autoaudiosink.SetProperty("sync", false)
150-
151-
pl.Add(asrc.AsElement(), queue1, wavparse, audioconvert, queue2, autoaudiosink)
175+
pl.Add(asrc.AsElement(), queue1, wavparse, audioconvert, queue2)
152176
asrc.Link(queue1)
153177
queue1.Link(wavparse)
154178
wavparse.Link(audioconvert)
155179

156180
audioconvert.Link(queue2)
157-
queue2.Link(autoaudiosink)
158181

159182
return asrc
160183
}
161-
162-
func setUpVideoPipeline(pl *gst.Pipeline) *gst.AppSrc {
163-
asrc := gst.NewAppSrc("my-video-src")
164-
asrc.SetProperty("is-live", true)
165-
166-
queue1 := gst.ElementFactoryMake("queue", "queue_11")
167-
checkElem(queue1, "queue_11")
168-
169-
h264parse := gst.ElementFactoryMake("h264parse", "h264parse_01")
170-
checkElem(h264parse, "h264parse")
171-
172-
avdecH264 := gst.ElementFactoryMake("vtdec", "vtdec_01")
173-
checkElem(avdecH264, "vtdec_01")
174-
175-
queue2 := gst.ElementFactoryMake("queue", "queue_12")
176-
checkElem(queue2, "queue_12")
177-
178-
videoconvert := gst.ElementFactoryMake("videoconvert", "videoconvert_01")
179-
checkElem(videoconvert, "videoconvert_01")
180-
181-
queue3 := gst.ElementFactoryMake("queue", "queue_13")
182-
checkElem(queue3, "queue_13")
183-
184-
sink := gst.ElementFactoryMake("autovideosink", "autovideosink_01")
185-
// setting this to true, creates extremely choppy video
186-
// I probably messed up something regarding the time stamps
187-
sink.SetProperty("sync", false)
188-
checkElem(sink, "autovideosink_01")
189-
190-
pl.Add(asrc.AsElement(), queue1, h264parse, avdecH264, queue2, videoconvert, queue3, sink)
191-
192-
asrc.Link(queue1)
193-
queue1.Link(h264parse)
194-
h264parse.Link(avdecH264)
195-
avdecH264.Link(queue2)
196-
queue2.Link(videoconvert)
197-
videoconvert.Link(queue3)
198-
queue3.Link(sink)
199-
return asrc
184+
func setupVorbis(pl *gst.Pipeline, filepath string) {
185+
//vorbisenc ! oggmux ! filesink location=alsasrc.ogg
186+
vorbisEnc := gst.ElementFactoryMake("vorbisenc", "vorbisenc_01")
187+
checkElem(vorbisEnc, "vorbisenc_01")
188+
oggMux := gst.ElementFactoryMake("oggmux", "oggmux_01")
189+
checkElem(oggMux, "oggmux_01")
190+
191+
filesink := gst.ElementFactoryMake("filesink", "filesink_01")
192+
filesink.SetProperty("location", filepath)
193+
checkElem(filesink, "filesink_01")
194+
195+
pl.Add(vorbisEnc, oggMux, filesink)
196+
197+
pl.GetByName("queue2").Link(vorbisEnc)
198+
vorbisEnc.Link(oggMux)
199+
oggMux.Link(filesink)
200+
}
201+
func setupMp3(pl *gst.Pipeline, filepath string) {
202+
// lamemp3enc ! filesink location=sine.mp3
203+
lameEnc := gst.ElementFactoryMake("lamemp3enc", "lamemp3enc_01")
204+
checkElem(lameEnc, "lamemp3enc_01")
205+
206+
filesink := gst.ElementFactoryMake("filesink", "filesink_01")
207+
filesink.SetProperty("location", filepath)
208+
checkElem(filesink, "filesink_01")
209+
pl.Add(lameEnc, filesink)
210+
pl.GetByName("queue2").Link(lameEnc)
211+
lameEnc.Link(filesink)
200212
}
201213

202214
func checkElem(e *gst.Element, name string) {

0 commit comments

Comments
 (0)