Skip to content

Commit 078bfea

Browse files
nanangizznanangizz
authored
Implement audio video synchronization (#4325)
1 parent 501ec30 commit 078bfea

File tree

24 files changed

+1560
-108
lines changed

24 files changed

+1560
-108
lines changed

‎pjmedia/build/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ export _LDFLAGS := $(APP_THIRD_PARTY_LIBS) \
5858
#
5959
export PJMEDIA_SRCDIR = ../src/pjmedia
6060
export PJMEDIA_OBJS += $(OS_OBJS) $(M_OBJS) $(CC_OBJS) $(HOST_OBJS) \
61-
alaw_ulaw.o alaw_ulaw_table.o avi_player.o \
61+
alaw_ulaw.o alaw_ulaw_table.o avi_player.o av_sync.o \
6262
bidirectional.o clock_thread.o codec.o conference.o \
6363
conf_switch.o converter.o converter_libswscale.o converter_libyuv.o \
6464
delaybuf.o echo_common.o \

‎pjmedia/build/pjmedia.vcproj

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3342,6 +3342,10 @@
33423342
RelativePath="..\src\pjmedia\audiodev.c"
33433343
>
33443344
</File>
3345+
<File
3346+
RelativePath="..\src\pjmedia\av_sync.c"
3347+
>
3348+
</File>
33453349
<File
33463350
RelativePath="..\src\pjmedia\avi_player.c"
33473351
>
@@ -7599,6 +7603,10 @@
75997603
RelativePath="..\include\pjmedia\audiodev.h"
76007604
>
76017605
</File>
7606+
<File
7607+
RelativePath="..\include\pjmedia\av_sync.h"
7608+
>
7609+
</File>
76027610
<File
76037611
RelativePath="..\include\pjmedia\avi.h"
76047612
>

‎pjmedia/build/pjmedia.vcxproj

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -610,6 +610,7 @@
610610
<ClCompile Include="..\src\pjmedia\alaw_ulaw_table.c" />
611611
<ClCompile Include="..\src\pjmedia\audiodev.c" />
612612
<ClCompile Include="..\src\pjmedia\avi_player.c" />
613+
<ClCompile Include="..\src\pjmedia\av_sync.c" />
613614
<ClCompile Include="..\src\pjmedia\bidirectional.c" />
614615
<ClCompile Include="..\src\pjmedia\clock_thread.c" />
615616
<ClCompile Include="..\src\pjmedia\codec.c" />
@@ -765,6 +766,7 @@
765766
<ClInclude Include="..\include\pjmedia\audiodev.h" />
766767
<ClInclude Include="..\include\pjmedia\avi.h" />
767768
<ClInclude Include="..\include\pjmedia\avi_stream.h" />
769+
<ClInclude Include="..\include\pjmedia\av_sync.h" />
768770
<ClInclude Include="..\include\pjmedia\bidirectional.h" />
769771
<ClInclude Include="..\include\pjmedia\circbuf.h" />
770772
<ClInclude Include="..\include\pjmedia\clock.h" />

‎pjmedia/build/pjmedia.vcxproj.filters

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,9 @@
233233
<ClCompile Include="..\src\pjmedia\echo_webrtc_aec3.cpp">
234234
<Filter>Source Files</Filter>
235235
</ClCompile>
236+
<ClCompile Include="..\src\pjmedia\av_sync.c">
237+
<Filter>Source Files</Filter>
238+
</ClCompile>
236239
</ItemGroup>
237240
<ItemGroup>
238241
<ClInclude Include="..\include\pjmedia\alaw_ulaw.h">
@@ -424,5 +427,8 @@
424427
<ClInclude Include="..\include\pjmedia\vid_conf.h">
425428
<Filter>Header Files</Filter>
426429
</ClInclude>
430+
<ClInclude Include="..\include\pjmedia\av_sync.h">
431+
<Filter>Header Files</Filter>
432+
</ClInclude>
427433
</ItemGroup>
428434
</Project>

‎pjmedia/include/pjmedia/av_sync.h

Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
/*
2+
* Copyright (C) 2025 Teluu Inc. (http://www.teluu.com)
3+
*
4+
* This program is free software; you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation; either version 2 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU General Public License
15+
* along with this program; if not, write to the Free Software
16+
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17+
*/
18+
#ifndef __PJMEDIA_AV_SYNC_H__
19+
#define __PJMEDIA_AV_SYNC_H__
20+
21+
/**
22+
* @file av_sync.h
23+
* @brief Inter-media Synchronization.
24+
*/
25+
#include <pjmedia/types.h>
26+
27+
28+
PJ_BEGIN_DECL
29+
30+
31+
/**
32+
* @defgroup PJMEDIA_AV_SYNC Inter-media Synchronization
33+
* @ingroup PJMEDIA_SESSION
34+
* @brief Synchronize presentation time of multiple media in a session.
35+
* @{
36+
*
37+
* A call session may consist of multiple media, e.g: some audio and some
38+
* video, which frequently have different delays when presented in the
39+
* receiver side. This module synchronizes all media in the same session
40+
* based on NTP timestamp & RTP timestamp info provided by the sender in
41+
* RTCP SR.
42+
*
43+
* Here are steps to use this module:
44+
* 1. Create AV sync using #pjmedia_av_sync_create().
45+
* 2. Adds all media to be synchronized using #pjmedia_av_sync_add_media().
46+
* 3. Call #pjmedia_av_sync_update_ref() each time the media receiving
47+
* an RTCP SR packet.
48+
* 4. Call #pjmedia_av_sync_update_pts() each time the media returning
49+
* a frame to be presented, e.g: via port.get_frame(). The function may
50+
* request the media to adjust its delay.
51+
* 5. Call #pjmedia_av_sync_del_media() when a media is removed from the
52+
* session.
53+
* 6. Call #pjmedia_av_sync_destroy() when the session is ended.
54+
*
55+
* The primary synchronization logic is implemented within the
56+
* #pjmedia_av_sync_update_pts() function. This function will calculate
57+
* the lag between the calling media to the earliest media and will provide
58+
* a feedback to the calling media whether it is in synchronized state,
59+
* late, or early so the media can respond accordingly.
60+
* Initially this function will try to request slower media to speed up.
61+
* If after a specific number of requests (i.e: configurable via
62+
* PJMEDIA_AVSYNC_MAX_SPEEDUP_REQ_CNT) and the lag is still beyond a tolerable
63+
* value (i.e: configurable via PJMEDIA_AVSYNC_MAX_TOLERABLE_LAG_MSEC), the
64+
* function will issue slow down request to the fastest media.
65+
*/
66+
67+
68+
/**
69+
* Inter-media synchronizer, opaque.
70+
*/
71+
typedef struct pjmedia_av_sync pjmedia_av_sync;
72+
73+
74+
/**
75+
* Media synchronization handle, opaque.
76+
*/
77+
typedef struct pjmedia_av_sync_media pjmedia_av_sync_media;
78+
79+
80+
/**
81+
* Synchronizer settings.
82+
*/
83+
typedef struct {
84+
/**
85+
* Name of the syncrhonizer
86+
*/
87+
char *name;
88+
89+
/**
90+
* Streaming mode. If set to PJ_TRUE, the delay adjustment values will
91+
* be smoothened and marked up to prevent possible delay increase on
92+
* all media.
93+
*/
94+
pj_bool_t is_streaming;
95+
96+
} pjmedia_av_sync_setting;
97+
98+
99+
/**
100+
* Media settings.
101+
*/
102+
typedef struct {
103+
/**
104+
* Name of the media
105+
*/
106+
char *name;
107+
108+
/**
109+
* Media type.
110+
*/
111+
pjmedia_type type;
112+
113+
/**
114+
* Media clock rate or sampling rate.
115+
*/
116+
unsigned clock_rate;
117+
118+
} pjmedia_av_sync_media_setting;
119+
120+
121+
/**
122+
* Get default settings for synchronizer.
123+
*
124+
* @param setting The synchronizer settings.
125+
*/
126+
PJ_DECL(void) pjmedia_av_sync_setting_default(
127+
pjmedia_av_sync_setting *setting);
128+
129+
/**
130+
* Get default settings for media.
131+
*
132+
* @param setting The media settings.
133+
*/
134+
PJ_DECL(void) pjmedia_av_sync_media_setting_default(
135+
pjmedia_av_sync_media_setting *setting);
136+
137+
/**
138+
* Create media synchronizer.
139+
*
140+
* @param pool The memory pool.
141+
* @param option The synchronizer settings.
142+
* @param av_sync The pointer to receive the media synchronizer.
143+
*
144+
* @return PJ_SUCCESS on success.
145+
*/
146+
PJ_DECL(pj_status_t) pjmedia_av_sync_create(
147+
pj_pool_t *pool,
148+
const pjmedia_av_sync_setting *setting,
149+
pjmedia_av_sync **av_sync);
150+
151+
152+
/**
153+
* Destroy media synchronizer.
154+
*
155+
* @param av_sync The media synchronizer.
156+
*/
157+
PJ_DECL(void) pjmedia_av_sync_destroy(pjmedia_av_sync *av_sync);
158+
159+
160+
/**
161+
* Reset synchronization states. Any existing media will NOT be removed,
162+
* but their states will be reset.
163+
*
164+
* @param av_sync The media synchronizer.
165+
*
166+
* @return PJ_SUCCESS on success.
167+
*/
168+
PJ_DECL(pj_status_t) pjmedia_av_sync_reset(pjmedia_av_sync *av_sync);
169+
170+
171+
/**
172+
* Add a media to synchronizer.
173+
*
174+
* @param av_sync The media synchronizer.
175+
* @param setting The media settings.
176+
* @param av_sync_media The pointer to receive the media synchronization
177+
* handle.
178+
*
179+
* @return PJ_SUCCESS on success.
180+
*/
181+
PJ_DECL(pj_status_t) pjmedia_av_sync_add_media(
182+
pjmedia_av_sync* av_sync,
183+
const pjmedia_av_sync_media_setting *setting,
184+
pjmedia_av_sync_media **av_sync_media);
185+
186+
187+
/**
188+
* Remove a media from synchronizer.
189+
*
190+
* @param av_sync The media synchronizer.
191+
* @param av_sync_media The media synchronization handle.
192+
*
193+
* @return PJ_SUCCESS on success.
194+
*/
195+
PJ_DECL(pj_status_t) pjmedia_av_sync_del_media(
196+
pjmedia_av_sync *av_sync,
197+
pjmedia_av_sync_media *av_sync_media);
198+
199+
200+
/**
201+
* Update synchronizer about the last presentation timestamp of the specified
202+
* media. Normally this function is called each time the media produces
203+
* a frame to be rendered (e.g: in port's get_frame() method). Upon returning,
204+
* the media may be requested to adjust its delay so it matches to the
205+
* earliest or the latest media, i.e: by speeding up or slowing down.
206+
*
207+
* Initially this function will try to request slower media to speed up.
208+
* If after a specific number of requests (i.e: configurable via
209+
* PJMEDIA_AVSYNC_MAX_SPEEDUP_REQ_CNT) and the lag is still beyond a tolerable
210+
* value (i.e: configurable via PJMEDIA_AVSYNC_MAX_TOLERABLE_LAG_MSEC), the
211+
* function will issue slow down request to the fastest media.
212+
*
213+
* @param av_sync_media The media synchronization handle.
214+
* @param pts The presentation timestamp.
215+
* @param adjust_delay Optional pointer to receive adjustment delay
216+
* required, in milliseconds, to make this media
217+
* synchronized to the fastest media.
218+
* Possible output values are:
219+
* 0 when no action is needed,
220+
* possitive value when increasing delay is needed,
221+
* or negative value when decreasing delay is needed.
222+
*
223+
* @return PJ_SUCCESS on success.
224+
*/
225+
PJ_DECL(pj_status_t) pjmedia_av_sync_update_pts(
226+
pjmedia_av_sync_media *av_sync_media,
227+
const pj_timestamp *pts,
228+
pj_int32_t *adjust_delay);
229+
230+
231+
/**
232+
* Update synchronizer about reference timestamps of the specified media.
233+
* Normally this function is called each time the media receives RTCP SR
234+
* packet.
235+
*
236+
* @param av_sync_media The media synchronization handle.
237+
* @param ntp The NTP timestamp info from RTCP SR.
238+
* @param ts The RTP timestamp info from RTCP SR.
239+
*
240+
* @return PJ_SUCCESS on success.
241+
*/
242+
PJ_DECL(pj_status_t) pjmedia_av_sync_update_ref(
243+
pjmedia_av_sync_media *av_sync_media,
244+
const pj_timestamp *ntp,
245+
const pj_timestamp *ts);
246+
247+
248+
/**
249+
* @}
250+
*/
251+
252+
253+
PJ_END_DECL
254+
255+
256+
#endif /* __PJMEDIA_AV_SYNC_H__ */

‎pjmedia/include/pjmedia/avi_stream.h

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,13 @@ enum pjmedia_avi_file_player_option
4545
* Tell the file player to return NULL frame when the whole
4646
* file has been played.
4747
*/
48-
PJMEDIA_AVI_FILE_NO_LOOP = 1
48+
PJMEDIA_AVI_FILE_NO_LOOP = 1,
49+
50+
/**
51+
* Set the file player to permit independent playback of audio and
52+
* video streams without synchronization.
53+
*/
54+
PJMEDIA_AVI_FILE_NO_SYNC = 2
4955
};
5056

5157
/**
@@ -64,9 +70,16 @@ typedef struct pjmedia_avi_streams pjmedia_avi_streams;
6470
* reading AVI file with uncompressed video format and
6571
* 16 bit PCM or compressed G.711 A-law/U-law audio format.
6672
*
73+
* By default, avi streams will loop the file playback and synchronize
74+
* audio and video streams. To change this behavior, use the flags parameter.
75+
*
76+
* When synchronization is enabled, the file player will wait for all
77+
* media streams to reach the end of file before rewinding the file.
78+
*
6779
* @param pool Pool to create the streams.
6880
* @param filename File name to open.
69-
* @param flags Avi streams creation flags.
81+
* @param flags Avi streams creation flags, bitmask combination of
82+
* #pjmedia_avi_file_player_option.
7083
* @param p_streams Pointer to receive the avi streams instance.
7184
*
7285
* @return PJ_SUCCESS on success.

‎pjmedia/include/pjmedia/config.h

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1765,6 +1765,32 @@
17651765
#undef PJMEDIA_VID_STREAM_CHECK_RTP_PT
17661766
#define PJMEDIA_VID_STREAM_CHECK_RTP_PT PJMEDIA_STREAM_CHECK_RTP_PT
17671767

1768+
1769+
/**
1770+
* Maximum tolerable presentation lag from the earliest to the latest media,
1771+
* in milliseconds, in inter-media synchronization. When the delay is
1772+
* higher than this setting, the media synchronizer will request the slower
1773+
* media to speed up. And if after a number of speed up requests the delay
1774+
* is still beyond this setting, the fastest media will be requested to
1775+
* slow down.
1776+
*
1777+
* Default: 45 ms
1778+
*/
1779+
#ifndef PJMEDIA_AVSYNC_MAX_TOLERABLE_LAG_MSEC
1780+
# define PJMEDIA_AVSYNC_MAX_TOLERABLE_LAG_MSEC 45
1781+
#endif
1782+
1783+
1784+
/**
1785+
* Maximum number of speed up request to synchronize presentation time,
1786+
* before a slow down request to the fastest media is issued.
1787+
*
1788+
* Default: 10
1789+
*/
1790+
#ifndef PJMEDIA_AVSYNC_MAX_SPEEDUP_REQ_CNT
1791+
# define PJMEDIA_AVSYNC_MAX_SPEEDUP_REQ_CNT 10
1792+
#endif
1793+
17681794
/**
17691795
* @}
17701796
*/

0 commit comments

Comments
 (0)