FS-7501: add auto sync of jb and fps detection

diff --git a/src/include/switch_core_media.h b/src/include/switch_core_media.h
index 2238fc14b1cd7375da7ecca5c0811e4f3849e868..37631fce70869e1ba8e559b335243ac4933bb11b 100644 (file)
--- a/src/include/switch_core_media.h
+++ b/src/include/switch_core_media.h
@@ -192,6 +192,7 @@ SWITCH_DECLARE(int) switch_core_session_check_incoming_crypto(switch_core_sessio
                                                                                                                          switch_media_type_t type, const char *crypto, int crypto_tag, switch_sdp_type_t sdp_type);
 
 
+SWITCH_DECLARE(uint32_t) switch_core_media_get_video_fps(switch_core_session_t *session);
 SWITCH_DECLARE(void) switch_core_media_set_rtp_session(switch_core_session_t *session, switch_media_type_t type, switch_rtp_t *rtp_session);
 
 SWITCH_DECLARE(const char *)switch_core_media_get_codec_string(switch_core_session_t *session);

diff --git a/src/include/switch_rtp.h b/src/include/switch_rtp.h
index 9d6d4c6942183286232c02c821889d3978d3f52a..53e26c5a8796bb55243744e2c8e61775ad38ec65 100644 (file)
--- a/src/include/switch_rtp.h
+++ b/src/include/switch_rtp.h
@@ -327,6 +327,8 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_activate_rtcp(switch_rtp_t *rtp_sessi
 
 SWITCH_DECLARE(switch_timer_t *) switch_rtp_get_media_timer(switch_rtp_t *rtp_session);
 
+SWITCH_DECLARE(switch_status_t) switch_rtp_set_video_buffer_size(switch_rtp_t *rtp_session, uint32_t frames);
+
 /*! 
   \brief Acvite a jitter buffer on an RTP session
   \param rtp_session the rtp session

diff --git a/src/include/switch_vidderbuffer.h b/src/include/switch_vidderbuffer.h
index d59ad3c46ba7ede04467fcfcc1b8c9d7f9986be5..132a7bf17745c40f000762bc92443181cce7012f 100644 (file)
--- a/src/include/switch_vidderbuffer.h
+++ b/src/include/switch_vidderbuffer.h
@@ -40,6 +40,7 @@ typedef enum {
 
 SWITCH_BEGIN_EXTERN_C
 SWITCH_DECLARE(switch_status_t) switch_vb_create(switch_vb_t **vbp, uint32_t min_frame_len, uint32_t max_frame_len, switch_memory_pool_t *pool);
+SWITCH_DECLARE(switch_status_t) switch_vb_set_frames(switch_vb_t *vb, uint32_t min_frame_len, uint32_t max_frame_len);
 SWITCH_DECLARE(switch_status_t) switch_vb_destroy(switch_vb_t **vbp);
 SWITCH_DECLARE(void) switch_vb_reset(switch_vb_t *vb);
 SWITCH_DECLARE(void) switch_vb_debug_level(switch_vb_t *vb, uint8_t level);

diff --git a/src/mod/endpoints/mod_verto/mod_verto.c b/src/mod/endpoints/mod_verto/mod_verto.c
index 928201931e1d5789a4f0404574ed5922fc7c47c3..89c9a77769e3e238faf343259ad94be1bf45de26 100644 (file)
--- a/src/mod/endpoints/mod_verto/mod_verto.c
+++ b/src/mod/endpoints/mod_verto/mod_verto.c
@@ -2263,7 +2263,7 @@ static void verto_set_media_options(verto_pvt_t *tech_pvt, verto_profile_t *prof
        tech_pvt->mparams->inbound_codec_string = switch_core_session_strdup(tech_pvt->session, profile->inbound_codec_string);
        tech_pvt->mparams->outbound_codec_string = switch_core_session_strdup(tech_pvt->session, profile->outbound_codec_string);
        
-       tech_pvt->mparams->jb_msec = "-1";
+       //tech_pvt->mparams->jb_msec = "-1";
        switch_media_handle_set_media_flag(tech_pvt->smh, SCMF_SUPPRESS_CNG);
 
        switch_media_handle_set_media_flag(tech_pvt->smh, SCMF_RENEG_ON_REINVITE);

diff --git a/src/mod/formats/mod_avformat/mod_avformat.c b/src/mod/formats/mod_avformat/mod_avformat.c
index 06d779aa92ab64ac190ef786a84a9238051d954c..46a59f90a9848158926bfb4c226801544ef3f8e7 100644 (file)
--- a/src/mod/formats/mod_avformat/mod_avformat.c
+++ b/src/mod/formats/mod_avformat/mod_avformat.c
@@ -38,6 +38,7 @@
 #include <libavutil/channel_layout.h>
 // #include <libavutil/timestamp.h>
 #include <libavresample/avresample.h>
+#define DFT_RECORD_OFFSET 350
 
 SWITCH_MODULE_LOAD_FUNCTION(mod_avformat_load);
 SWITCH_MODULE_DEFINITION(mod_avformat, mod_avformat_load, NULL, NULL);
@@ -665,7 +666,7 @@ SWITCH_STANDARD_APP(record_av_function)
                char buf[SWITCH_RECOMMENDED_BUFFER_SIZE] = {0};
                switch_size_t datalen = codec.implementation->decoded_bytes_per_packet;
                switch_size_t samples = datalen / 2 / codec.implementation->number_of_channels;
-               int offset = 1200;
+               int offset = DFT_RECORD_OFFSET;
                int fps = codec.implementation->actual_samples_per_second / samples;
                int lead_frames = (offset * fps) / 1000;
                
@@ -1139,7 +1140,7 @@ static switch_status_t av_file_open(switch_file_handle_t *handle, const char *pa
 
        memset(context, 0, sizeof(av_file_context_t));
 
-       context->offset = 1200;
+       context->offset = DFT_RECORD_OFFSET;
        if (handle->params && (tmp = switch_event_get_header(handle->params, "av_video_offset"))) {
                context->offset = atoi(tmp);
        }

diff --git a/src/switch_core_media.c b/src/switch_core_media.c
index 0efb43b04cc83dd531fdacb3b6b1e36cf539ca56..79132a8c9941a7da97a7aed8dbff0dac4ded1f16 100644 (file)
--- a/src/switch_core_media.c
+++ b/src/switch_core_media.c
@@ -208,6 +208,9 @@ struct switch_media_handle_s {
        switch_vid_params_t vid_params; 
        switch_file_handle_t *video_read_fh;
        switch_file_handle_t *video_write_fh;
+
+       uint64_t vid_frames;
+       time_t vid_started;
 };
 
 
@@ -309,6 +312,46 @@ static void _switch_core_media_pass_zrtp_hash2(switch_core_session_t *aleg_sessi
        }
 }
 
+static uint32_t round_to_step(uint32_t num, uint32_t step)
+{
+       uint32_t r;
+       uint32_t x;
+
+       if (!num) return 0;
+       
+       r = (num % step);
+       x = num - r;
+       
+       if (r > step / 2) {
+               x += step;
+       }
+       
+       return x;
+}
+SWITCH_DECLARE(uint32_t) switch_core_media_get_video_fps(switch_core_session_t *session)
+{
+       switch_media_handle_t *smh;
+       time_t now;
+
+       switch_assert(session);
+       
+       if (!(smh = session->media_handle)) {
+               return SWITCH_STATUS_FALSE;
+       }               
+
+       if (!switch_channel_test_flag(session->channel, CF_VIDEO)) {
+               return 0;
+       }
+
+       now = switch_epoch_time_now(NULL);
+
+       if (!(smh->vid_started && smh->vid_frames && smh->vid_started < now)) {
+               return 0;
+       }
+       
+       return round_to_step(smh->vid_frames / (now - smh->vid_started), 5);
+}
+
 SWITCH_DECLARE(void) switch_core_media_pass_zrtp_hash2(switch_core_session_t *aleg_session, switch_core_session_t *bleg_session)
 {
        _switch_core_media_pass_zrtp_hash2(aleg_session, bleg_session, SWITCH_MEDIA_TYPE_AUDIO);
@@ -1766,7 +1809,8 @@ SWITCH_DECLARE(void) switch_core_media_prepare_codecs(switch_core_session_t *ses
 }
 
 
-static void check_jb(switch_core_session_t *session, const char *input)
+
+static void check_jb(switch_core_session_t *session, const char *input, int32_t jb_msec, int32_t maxlen)
 {
        const char *val;
        switch_media_handle_t *smh;
@@ -1820,18 +1864,20 @@ static void check_jb(switch_core_session_t *session, const char *input)
        }
        
 
-       if ((val = switch_channel_get_variable(session->channel, "jitterbuffer_msec")) || (val = smh->mparams->jb_msec)) {
-               int jb_msec = atoi(val);
-               int maxlen = 0, max_drift = 0;
+       if (jb_msec || (val = switch_channel_get_variable(session->channel, "jitterbuffer_msec")) || (val = smh->mparams->jb_msec)) {
+               int max_drift = 0;
                char *p, *q;
 
+               if (!jb_msec) {
+                       jb_msec = atoi(val);
                                        
-               if ((p = strchr(val, ':'))) {
-                       p++;
-                       maxlen = atoi(p);
-                       if ((q = strchr(p, ':'))) {
-                               q++;
-                               max_drift = abs(atoi(q));
+                       if ((p = strchr(val, ':'))) {
+                               p++;
+                               maxlen = atoi(p);
+                               if ((q = strchr(p, ':'))) {
+                                       q++;
+                                       max_drift = abs(atoi(q));
+                               }
                        }
                }
 
@@ -1878,6 +1924,61 @@ static void check_jb(switch_core_session_t *session, const char *input)
 
 }
 
+static void check_jb_sync(switch_core_session_t *session)
+{
+       int32_t jb_sync_msec = 200;
+       uint32_t fps, frames = 0;
+       switch_media_handle_t *smh;
+       switch_rtp_engine_t *v_engine = NULL;
+       const char *var;
+
+       switch_assert(session);
+
+       if (!switch_channel_test_flag(session->channel, CF_VIDEO)) {
+               return;
+       }
+
+       if (!(smh = session->media_handle)) {
+               return;
+       }
+
+       v_engine = &smh->engines[SWITCH_MEDIA_TYPE_VIDEO];
+       
+       if ((var = switch_channel_get_variable(session->channel, "jb_sync_msec"))) {
+               int tmp;
+
+               if (!strcasecmp(var, "disabled")) {
+                       return;
+               }
+
+               tmp = atol(var);
+
+               if (tmp > -50 && tmp < 10000) {
+                       jb_sync_msec = tmp;
+               }
+       }
+
+       fps = switch_core_media_get_video_fps(session);
+               
+       if (!fps) return;
+
+       if (jb_sync_msec < 0) {
+               frames = abs(jb_sync_msec);
+               jb_sync_msec = 1000 / (fps / frames);
+       } else {
+               frames = fps / (1000 / jb_sync_msec);
+       }
+
+       switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), 
+                                         SWITCH_LOG_DEBUG, "%s Sync Audio and Video Jitterbuffer to %dms %u Video Frames FPS %u\n", 
+                                         switch_channel_get_name(session->channel),
+                                         jb_sync_msec, frames, fps);
+       
+       switch_rtp_set_video_buffer_size(v_engine->rtp_session, frames);
+       check_jb(session, NULL, jb_sync_msec, jb_sync_msec);
+}
+
+
 //?
 SWITCH_DECLARE(switch_status_t) switch_core_media_read_lock_unlock(switch_core_session_t *session, switch_media_type_t type, switch_bool_t lock)
 {
@@ -1991,6 +2092,18 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_read_frame(switch_core_session
                        goto end;
                }
 
+               if (type == SWITCH_MEDIA_TYPE_VIDEO && engine->read_frame.m) {
+                       if (!smh->vid_started) {
+                               smh->vid_started = switch_epoch_time_now(NULL);
+                       }
+                       smh->vid_frames++;
+
+                       if (smh->vid_frames == 45) {
+                               check_jb_sync(session);
+                       }
+               }
+
+
                /* re-set codec if necessary */
                if (engine->reset_codec > 0) {
                        const char *val;
@@ -2039,7 +2152,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_read_frame(switch_core_session
                                }
                        }
 
-                       check_jb(session, NULL);
+                       check_jb(session, NULL, 0, 0);
 
                        engine->check_frames = 0;
                        engine->last_ts = 0;
@@ -5919,7 +6032,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_activate_rtp(switch_core_sessi
 
                }
 
-               check_jb(session, NULL);
+               check_jb(session, NULL, 0, 0);
 
                if ((val = switch_channel_get_variable(session->channel, "rtp_timeout_sec"))) {
                        int v = atoi(val);
@@ -8467,7 +8580,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_receive_message(switch_core_se
        case SWITCH_MESSAGE_INDICATE_JITTER_BUFFER:
                {
                        if (switch_rtp_ready(a_engine->rtp_session)) {
-                               check_jb(session, msg->string_arg);
+                               check_jb(session, msg->string_arg, 0, 0);
                        }
                }
                break;

diff --git a/src/switch_rtp.c b/src/switch_rtp.c
index 40141098a5baad7549a390ce1a996fc484bad65f..28d6f145581c2a11f037298a61d61bf7393bc2bc 100644 (file)
--- a/src/switch_rtp.c
+++ b/src/switch_rtp.c
@@ -3549,9 +3549,12 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_create(switch_rtp_t **new_rtp_session
                                switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Starting video timer.\n");
                        }
 
-                       switch_vb_create(&rtp_session->vb, 5, 30, rtp_session->pool);
+                       //switch_vb_create(&rtp_session->vb, 5, 30, rtp_session->pool);
                        //switch_vb_debug_level(rtp_session->vb, 10);
-                       switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Starting video buffer.\n");
+                       
+                       
+                       
+
 
                } else {
                        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Not using a timer\n");
@@ -3834,6 +3837,23 @@ static void jb_logger(const char *file, const char *func, int line, int level, c
        va_end(ap);
 }
 
+SWITCH_DECLARE(switch_status_t) switch_rtp_set_video_buffer_size(switch_rtp_t *rtp_session, uint32_t frames)
+{
+       if (!switch_rtp_ready(rtp_session)) {
+               return SWITCH_STATUS_FALSE;
+       }
+
+       if (!rtp_session->vb) {  
+               switch_vb_create(&rtp_session->vb, frames, frames * 3, rtp_session->pool);
+       } else {
+               switch_vb_set_frames(rtp_session->vb, frames, frames * 3);
+       }
+
+       switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Setting video buffer %u Frames.\n", frames);
+       
+       return SWITCH_STATUS_SUCCESS;
+}
+
 SWITCH_DECLARE(switch_status_t) switch_rtp_debug_jitter_buffer(switch_rtp_t *rtp_session, const char *name)
 {
 

diff --git a/src/switch_vidderbuffer.c b/src/switch_vidderbuffer.c
index 1b0c1b7e4d22eaa5fbef392118685d5801c0c0bb..415f4f7cfc08724dd370550c55e8bb2722738004 100644 (file)
--- a/src/switch_vidderbuffer.c
+++ b/src/switch_vidderbuffer.c
@@ -415,6 +415,16 @@ SWITCH_DECLARE(void) switch_vb_reset(switch_vb_t *vb)
        switch_mutex_unlock(vb->mutex);
 }
 
+SWITCH_DECLARE(switch_status_t) switch_vb_set_frames(switch_vb_t *vb, uint32_t min_frame_len, uint32_t max_frame_len)
+{
+       switch_mutex_lock(vb->mutex);
+       vb->min_frame_len = vb->frame_len = min_frame_len;
+       vb->max_frame_len = max_frame_len;
+       switch_mutex_unlock(vb->mutex);
+
+       return SWITCH_STATUS_SUCCESS;
+}
+
 SWITCH_DECLARE(switch_status_t) switch_vb_create(switch_vb_t **vbp, uint32_t min_frame_len, uint32_t max_frame_len, switch_memory_pool_t *pool)
 {
        switch_vb_t *vb;