webrtc video tweaks

diff --git a/src/mod/applications/mod_conference/mod_conference.c b/src/mod/applications/mod_conference/mod_conference.c
index 3a8818a8138202f18fece31d4aea592727d67486..89850e262cf47896dc1fc14a31aa6412ba0b402d 100644 (file)
--- a/src/mod/applications/mod_conference/mod_conference.c
+++ b/src/mod/applications/mod_conference/mod_conference.c
@@ -202,7 +202,8 @@ typedef enum {
        CFLAG_AUDIO_ALWAYS = (1 << 15),
        CFLAG_ENDCONF_FORCED = (1 << 16),
        CFLAG_RFC4579 = (1 << 17),
-       CFLAG_FLOOR_CHANGE = (1 << 18)
+       CFLAG_FLOOR_CHANGE = (1 << 18),
+       CFLAG_VID_FLOOR_LOCK = (1 << 19)
 } conf_flag_t;
 
 typedef enum {
@@ -1454,12 +1455,11 @@ static switch_status_t conference_add_member(conference_obj_t *conference, confe
                switch_channel_set_variable(channel, "conference_recording", conference->record_filename);
                switch_channel_set_variable(channel, CONFERENCE_UUID_VARIABLE, conference->uuid_str);
 
-
-
                if (switch_channel_test_flag(channel, CF_VIDEO)) {
-                       if (!switch_test_flag(conference, CFLAG_VIDEO_BRIDGE)) {
+                       if (switch_test_flag(conference, CFLAG_VIDEO_BRIDGE)) {
+                               switch_channel_set_flag(channel, CF_VIDEO_ECHO);
+                       } else {
                                switch_channel_clear_flag(channel, CF_VIDEO_ECHO);
-                               switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE);
                        }
                        /* Tell the channel to request a fresh vid frame */
                        switch_core_session_refresh_video(member->session);
@@ -1577,36 +1577,47 @@ static switch_status_t conference_add_member(conference_obj_t *conference, confe
        return status;
 }
 
-static void conference_set_video_floor_holder(conference_obj_t *conference, conference_member_t *member)
+static void conference_set_video_floor_holder(conference_obj_t *conference, conference_member_t *member, switch_bool_t force)
 {
        switch_event_t *event;
+       conference_member_t *old_member = NULL;
        int old_id = 0;
 
-       if (switch_test_flag(conference, CFLAG_VIDEO_BRIDGE)) {
+       if (switch_test_flag(conference, CFLAG_VIDEO_BRIDGE) || (!force && switch_test_flag(conference, CFLAG_VID_FLOOR_LOCK))) {
                return;
        }
        
        if (conference->video_floor_holder) {
                if (conference->video_floor_holder == member) {
                        return;
-               } else {
+               } else {                        
+                       old_member = conference->video_floor_holder;
                        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Dropping video floor %s\n", 
-                                                         switch_channel_get_name(conference->video_floor_holder->channel));
-                       old_id = conference->video_floor_holder->id;
-                       switch_channel_clear_flag(conference->video_floor_holder->channel, CF_VIDEO_PASSIVE);
-                       switch_core_session_refresh_video(conference->video_floor_holder->session);
+                                                         switch_channel_get_name(old_member->channel));
                }
        }
 
-       if ((conference->video_floor_holder = member)) {
+
+       switch_mutex_lock(conference->mutex);
+       if (member) {
                switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Adding video floor %s\n", 
-                                                 switch_channel_get_name(conference->video_floor_holder->channel));
+                                                 switch_channel_get_name(member->channel));
                switch_channel_set_flag(member->channel, CF_VIDEO_PASSIVE);
-               switch_core_session_refresh_video(conference->video_floor_holder->session);
-               switch_set_flag(conference, CFLAG_FLOOR_CHANGE);
+               switch_core_session_refresh_video(member->session);
+               conference->video_floor_holder = member;
+       } else {
+               conference->video_floor_holder = NULL;
+       }
+
+       if (old_member) {
+               old_id = old_member->id;
+               switch_channel_clear_flag(old_member->channel, CF_VIDEO_PASSIVE);
        }
 
-       if (old_id > -1 && test_eflag(conference, EFLAG_FLOOR_CHANGE)) {
+       switch_set_flag(conference, CFLAG_FLOOR_CHANGE);
+       switch_mutex_unlock(conference->mutex);
+
+       if (test_eflag(conference, EFLAG_FLOOR_CHANGE)) {
                switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, CONF_EVENT_MAINT);
                conference_add_event_data(conference, event);
                switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Action", "video-floor-change");
@@ -1628,33 +1639,44 @@ static void conference_set_video_floor_holder(conference_obj_t *conference, conf
 static void conference_set_floor_holder(conference_obj_t *conference, conference_member_t *member)
 {
        switch_event_t *event;
+       conference_member_t *old_member = NULL;
        int old_id = 0;
 
+       if (!switch_test_flag(conference, CFLAG_VIDEO_BRIDGE) && 
+               ((conference->video_floor_holder && !member) || (member && switch_channel_test_flag(member->channel, CF_VIDEO)))) {
+               conference_set_video_floor_holder(conference, member, SWITCH_FALSE);
+       }
+
        if (conference->floor_holder) {
                if (conference->floor_holder == member) {
                        return;
                } else {
-                       old_id = conference->floor_holder->id;
-                       if (!conference->video_floor_holder && !switch_test_flag(conference, CFLAG_VIDEO_BRIDGE)) {
-                               switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Dropping floor %s\n", 
-                                                                 switch_channel_get_name(conference->floor_holder->channel));
-                               switch_channel_clear_flag(conference->floor_holder->channel, CF_VIDEO_PASSIVE);
-                               switch_core_session_refresh_video(conference->floor_holder->session);
-                       }
+                       old_member = conference->floor_holder;
+                       switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Dropping floor %s\n", 
+                                                         switch_channel_get_name(old_member->channel));
+
                }
        }
 
-       if ((conference->floor_holder = member)) {
-               if (!conference->video_floor_holder && !switch_test_flag(conference, CFLAG_VIDEO_BRIDGE)) {
-                       switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Adding floor %s\n", 
-                                                         switch_channel_get_name(conference->floor_holder->channel));
-                       switch_channel_set_flag(member->channel, CF_VIDEO_PASSIVE);
-                       switch_core_session_refresh_video(conference->floor_holder->session);
-               }
-               switch_set_flag(conference, CFLAG_FLOOR_CHANGE);
+       switch_mutex_lock(conference->mutex);
+       if (member) {
+               switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Adding floor %s\n", 
+                                                 switch_channel_get_name(member->channel));
+
+               conference->floor_holder = member;
+       } else {
+               conference->floor_holder = NULL;
+       }
+
+
+       if (old_member) {
+               old_id = old_member->id;
        }
 
-       if (old_id > -1 && test_eflag(conference, EFLAG_FLOOR_CHANGE)) {
+       switch_set_flag(conference, CFLAG_FLOOR_CHANGE);
+       switch_mutex_unlock(conference->mutex);
+
+       if (test_eflag(conference, EFLAG_FLOOR_CHANGE)) {
                switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, CONF_EVENT_MAINT);
                conference_add_event_data(conference, event);
                switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Action", "floor-change");
@@ -1763,7 +1785,7 @@ static switch_status_t conference_del_member(conference_obj_t *conference, confe
 
 
        if (member == member->conference->video_floor_holder) {
-               conference_set_video_floor_holder(member->conference, NULL);
+               conference_set_video_floor_holder(member->conference, NULL, SWITCH_TRUE);
        }
 
        member->conference = NULL;
@@ -1779,11 +1801,6 @@ static switch_status_t conference_del_member(conference_obj_t *conference, confe
                        }
                }
 
-               if (switch_channel_test_flag(channel, CF_VIDEO)) {
-                       switch_channel_set_flag(channel, CF_VIDEO_ECHO);
-                       switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE);
-               }
-
                conference_send_presence(conference);
                switch_channel_set_variable(channel, "conference_call_key", NULL);
 
@@ -1914,8 +1931,6 @@ static void *SWITCH_THREAD_FUNC conference_video_thread_run(switch_thread_t *thr
 
                if (conference->video_floor_holder) {
                        floor_holder = conference->video_floor_holder;
-               } else {
-                       floor_holder = conference->floor_holder;
                }
 
 
@@ -1924,7 +1939,7 @@ static void *SWITCH_THREAD_FUNC conference_video_thread_run(switch_thread_t *thr
                        goto do_continue;
                }
 
-               if (!switch_channel_test_flag(switch_core_session_get_channel(floor_holder->session), CF_VIDEO)) {
+               if (!floor_holder->session || !floor_holder->channel || !switch_channel_test_flag(floor_holder->channel, CF_VIDEO)) {
                        yield = 100000;
                        goto do_continue;
                }
@@ -2074,8 +2089,8 @@ static void *SWITCH_THREAD_FUNC conference_thread_run(switch_thread_t *thread, v
                        if (switch_test_flag(imember, MFLAG_RUNNING) && imember->session) {
                                switch_channel_t *channel = switch_core_session_get_channel(imember->session);
 
-                               if ((!floor_holder || (imember->score_iir > SCORE_IIR_SPEAKING_MAX && (floor_holder->score_iir < SCORE_IIR_SPEAKING_MIN))) &&
-                                       (!switch_test_flag(conference, CFLAG_VID_FLOOR) || switch_channel_test_flag(channel, CF_VIDEO))) {
+                               if ((!floor_holder || (imember->score_iir > SCORE_IIR_SPEAKING_MAX && (floor_holder->score_iir < SCORE_IIR_SPEAKING_MIN)))) {// &&
+                                       //(!switch_test_flag(conference, CFLAG_VID_FLOOR) || switch_channel_test_flag(channel, CF_VIDEO))) {
                                        floor_holder = imember;
                                }
                                
@@ -5282,7 +5297,8 @@ static switch_status_t conf_api_sub_clear_vid_floor(conference_obj_t *conference
        }
 
        switch_mutex_lock(conference->mutex);
-       conference_set_video_floor_holder(conference, NULL);
+       switch_clear_flag(conference, CFLAG_VID_FLOOR_LOCK);
+       //conference_set_video_floor_holder(conference, NULL);
        switch_mutex_unlock(conference->mutex);
 
        return SWITCH_STATUS_SUCCESS;
@@ -5307,19 +5323,22 @@ static switch_status_t conf_api_sub_vid_floor(conference_member_t *member, switc
                force = 1;
        }
 
-       if (!force && member->conference->video_floor_holder == member) {
-               conference_set_video_floor_holder(member->conference, NULL);
+       if (member->conference->video_floor_holder == member && switch_test_flag(member->conference, CFLAG_VID_FLOOR_LOCK)) {
+               switch_clear_flag(member->conference, CFLAG_VID_FLOOR_LOCK);
+
+               conference_set_floor_holder(member->conference, member);
                if (stream == NULL) {
-                       switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "conference %s OK floor none\n", member->conference->name);
+                       switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "conference %s OK video floor auto\n", member->conference->name);
                } else {
                        stream->write_function(stream, "OK floor none\n");
                } 
                
        } else if (force || member->conference->video_floor_holder == NULL) {
-               conference_set_video_floor_holder(member->conference, member);
+               switch_set_flag(member->conference, CFLAG_VID_FLOOR_LOCK);
+               conference_set_video_floor_holder(member->conference, member, SWITCH_TRUE);
                if (test_eflag(member->conference, EFLAG_FLOOR_CHANGE)) {
                        if (stream == NULL) {
-                               switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "conference %s OK floor %d %s\n", 
+                               switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "conference %s OK video floor %d %s\n", 
                                                                  member->conference->name, member->id, switch_channel_get_name(member->channel));
                        } else {
                                stream->write_function(stream, "OK floor %u\n", member->id);

diff --git a/src/switch_core_media.c b/src/switch_core_media.c
index 109158e229c21a3e60ed94871a64bd0b37abcf6e..989aacb361af4f02346f323dffc5e161a926c6be 100644 (file)
--- a/src/switch_core_media.c
+++ b/src/switch_core_media.c
@@ -3792,7 +3792,6 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi
 
        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "%s Video thread started\n", switch_channel_get_name(session->channel));
        switch_core_session_refresh_video(session);
-       switch_channel_set_flag(channel, CF_VIDEO_ECHO);        
 
        while (switch_channel_up_nosig(channel)) {
 

diff --git a/src/switch_core_session.c b/src/switch_core_session.c
index 8fc40f7d9dd15feba8375a9703d338c1cb1b73d2..90dbac3f5f56ddb731eadf94f72602500000a3de 100644 (file)
--- a/src/switch_core_session.c
+++ b/src/switch_core_session.c
@@ -2625,7 +2625,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_exec(switch_core_session_t *
        int scope = 0;
        char uuid_str[SWITCH_UUID_FORMATTED_LENGTH + 1];
        char *app_uuid = uuid_str;
-       
+
        if ((app_uuid_var = switch_channel_get_variable(channel, "app_uuid"))) {
                app_uuid = (char *)app_uuid_var;
                switch_channel_set_variable(channel, "app_uuid", NULL);
@@ -2727,8 +2727,19 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_exec(switch_core_session_t *
        msg.string_array_arg[1] = expanded;
        switch_core_session_receive_message(session, &msg);
 
+       if (switch_channel_test_flag(channel, CF_VIDEO)) {
+               switch_channel_set_flag(channel, CF_VIDEO_ECHO);
+               switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE);
+               switch_core_session_refresh_video(session);
+       }
+
        application_interface->application_function(session, expanded);
 
+       if (switch_channel_test_flag(channel, CF_VIDEO)) {
+               switch_channel_set_flag(channel, CF_VIDEO_ECHO);
+               switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE);
+               switch_core_session_refresh_video(session);
+       }
        if (switch_event_create(&event, SWITCH_EVENT_CHANNEL_EXECUTE_COMPLETE) == SWITCH_STATUS_SUCCESS) {
                const char *resp = switch_channel_get_variable(session->channel, SWITCH_CURRENT_APPLICATION_RESPONSE_VARIABLE);
                switch_channel_event_set_data(session->channel, event);

diff --git a/src/switch_rtp.c b/src/switch_rtp.c
index 6bdcbc5c0efb3134fb71a4de2c72454df62a86e7..7b41987e845084b9c5830b8f95be09fe37106b56 100644 (file)
--- a/src/switch_rtp.c
+++ b/src/switch_rtp.c
@@ -54,7 +54,7 @@
 #include <switch_version.h>
 #include <switch_ssl.h>
 
-#define FIR_COUNTDOWN 100
+#define FIR_COUNTDOWN 50
 
 #define READ_INC(rtp_session) switch_mutex_lock(rtp_session->read_mutex); rtp_session->reading++
 #define READ_DEC(rtp_session)  switch_mutex_unlock(rtp_session->read_mutex); rtp_session->reading--
@@ -270,6 +270,12 @@ typedef struct ts_normalize_s {
        uint32_t last_frame;
        uint32_t ts;
        uint32_t delta;
+       uint32_t delta_ct;
+       uint32_t delta_ttl;
+       uint32_t delta_avg;
+       uint32_t delta_delta;
+       double delta_percent;
+       uint8_t m;
 } ts_normalize_t;
 
 struct switch_rtp {
@@ -3349,6 +3355,10 @@ SWITCH_DECLARE(void) switch_rtp_break(switch_rtp_t *rtp_session)
                return;
        }
 
+       if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) {
+               return;
+       }
+
        switch_mutex_lock(rtp_session->flag_mutex);
        rtp_session->flags[SWITCH_RTP_FLAG_BREAK] = 1;
 
@@ -4247,15 +4257,6 @@ static switch_status_t read_rtp_packet(switch_rtp_t *rtp_session, switch_size_t
                return SWITCH_STATUS_SUCCESS;
        }
 
-
-       if (*bytes && rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) {
-               unsigned int diff = ts - rtp_session->last_read_ts;
-
-               if (abs(diff) > 10000) {
-                       switch_rtp_video_refresh(rtp_session);
-               }
-       }
-
        if (ts) {
                rtp_session->last_read_ts = ts;
        }
@@ -4645,7 +4646,7 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_
                                                        goto end;
                                                }
                                                // This is dumb
-                                               switch_rtp_video_refresh(rtp_session);
+                                               //switch_rtp_video_refresh(rtp_session);
                                                goto  rtcp;
                                        }
                                }
@@ -4712,7 +4713,7 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_
                                }
 
                                has_rtcp = 0;
-
+                               
                        } else if (rtp_session->rtcp_read_pollfd) {
                                rtcp_poll_status = switch_poll(rtp_session->rtcp_read_pollfd, 1, &rtcp_fdr, 0);
                        }
@@ -5272,7 +5273,7 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_zerocopy_read_frame(switch_rtp_t *rtp
        }
 
        if (rtp_session->fir_countdown) {
-               if (--rtp_session->fir_countdown == 0) {
+               if (--rtp_session->fir_countdown == FIR_COUNTDOWN / 2) {
                        send_fir(rtp_session);
                        //send_pli(rtp_session);
                }
@@ -5483,6 +5484,9 @@ static int rtp_common_write(switch_rtp_t *rtp_session,
 
                if (!rtp_session->ts_norm.last_ssrc || send_msg->header.ssrc != rtp_session->ts_norm.last_ssrc) {
                        if (rtp_session->ts_norm.last_ssrc) {
+                               rtp_session->ts_norm.m = 1;
+                               rtp_session->ts_norm.delta_ct = 1;
+                               rtp_session->ts_norm.delta_ttl = 0;
                                if (rtp_session->ts_norm.delta) {
                                        rtp_session->ts_norm.ts += rtp_session->ts_norm.delta;
                                }
@@ -5494,12 +5498,40 @@ static int rtp_common_write(switch_rtp_t *rtp_session,
 
                if (ntohl(send_msg->header.ts) != rtp_session->ts_norm.last_frame) {
                        rtp_session->ts_norm.delta = ntohl(send_msg->header.ts) - rtp_session->ts_norm.last_frame;
+
+                       if (rtp_session->ts_norm.delta > 0) {
+                               rtp_session->ts_norm.delta_ct++;
+                               if (rtp_session->ts_norm.delta_ct == 1000) {
+                                       rtp_session->ts_norm.delta_ct = 1;
+                                       rtp_session->ts_norm.delta_ttl = 0;
+                               }
+
+                               rtp_session->ts_norm.delta_ttl += rtp_session->ts_norm.delta;
+                               rtp_session->ts_norm.delta_avg = rtp_session->ts_norm.delta_ttl / rtp_session->ts_norm.delta_ct;
+                               rtp_session->ts_norm.delta_delta = abs(rtp_session->ts_norm.delta_avg - rtp_session->ts_norm.delta);
+                               rtp_session->ts_norm.delta_percent = (double)((double)rtp_session->ts_norm.delta / (double)rtp_session->ts_norm.delta_avg) * 100.0f;
+
+
+                               if (rtp_session->ts_norm.delta_ct > 50 && rtp_session->ts_norm.delta_percent > 125.0) {
+                                       //printf("%s diff %d %d (%.2f)\n", switch_core_session_get_name(rtp_session->session),
+                                       //rtp_session->ts_norm.delta, rtp_session->ts_norm.delta_avg, rtp_session->ts_norm.delta_percent);
+                                       switch_rtp_video_refresh(rtp_session);
+                               }
+                       }
                        rtp_session->ts_norm.ts += rtp_session->ts_norm.delta;
                }
                
                rtp_session->ts_norm.last_frame = ntohl(send_msg->header.ts);
                send_msg->header.ts = htonl(rtp_session->ts_norm.ts);
 
+               /* wait for a marked frame since we just switched streams */
+               if (rtp_session->ts_norm.m) {
+                       if (send_msg->header.m) {
+                               rtp_session->ts_norm.m = 0;
+                       } else {
+                               send = 0;
+                       }
+               }
        }
 
        send_msg->header.ssrc = htonl(rtp_session->ssrc);