From 765fff3d75243e59f82a42fb6191630dea8d3b0f Mon Sep 17 00:00:00 2001 From: Anthony Minessale Date: Wed, 12 Nov 2014 21:30:39 -0600 Subject: [PATCH] FS-7500: add support for codec control and use it to pass messages down to the codec and use it to implement keyframe reset for fir, pli and nack. Later we will expand to handle nack correctly. --- src/include/switch_core.h | 17 +++ src/include/switch_core_media.h | 11 ++ src/include/switch_loadable_module.h | 29 +++-- src/include/switch_module_interfaces.h | 2 + src/include/switch_types.h | 29 ++++- src/mod/codecs/mod_openh264/mod_openh264.cpp | 12 +- src/mod/codecs/mod_vpx/mod_vpx.c | 27 +++- src/mod/codecs/mod_yuv/mod_yuv.c | 2 +- src/switch_core_codec.c | 33 +++++ src/switch_core_media.c | 129 +++++++++++++------ src/switch_rtp.c | 32 +++-- 11 files changed, 258 insertions(+), 65 deletions(-) diff --git a/src/include/switch_core.h b/src/include/switch_core.h index 330b6c9e9c..83cc10aa4d 100644 --- a/src/include/switch_core.h +++ b/src/include/switch_core.h @@ -1634,6 +1634,23 @@ SWITCH_DECLARE(switch_status_t) switch_core_codec_encode_video(switch_codec_t *c switch_image_t *img, void *encoded_data, uint32_t *encoded_data_len, unsigned int *flag); +/*! + \brief send control data using a codec handle + \param codec the codec handle to use + \param cmd the command to send + \param ctype the type of the arguement + \param cmd_data a void pointer to the data matching the passed type + \param rtype the type of the response if any + \param ret_data a void pointer to a pointer of return data + \return SWITCH_STATUS_SUCCESS if the command was received +*/ +SWITCH_DECLARE(switch_status_t) switch_core_codec_control(switch_codec_t *codec, + switch_codec_control_command_t cmd, + switch_codec_control_type_t ctype, + void *cmd_data, + switch_codec_control_type_t *rtype, + void **ret_data); + /*! \brief Decode video data using a codec handle \param codec the codec handle to use diff --git a/src/include/switch_core_media.h b/src/include/switch_core_media.h index 6e9f3b282a..056a5abfce 100644 --- a/src/include/switch_core_media.h +++ b/src/include/switch_core_media.h @@ -294,6 +294,17 @@ SWITCH_DECLARE(const char *) switch_core_media_crypto_type2str(switch_rtp_crypto SWITCH_DECLARE(int) switch_core_media_crypto_keylen(switch_rtp_crypto_key_type_t type); SWITCH_DECLARE(char *) switch_core_media_filter_sdp(const char *sdp, const char *cmd, const char *arg); SWITCH_DECLARE(char *) switch_core_media_process_sdp_filter(const char *sdp, const char *cmd_buf, switch_core_session_t *session); + + +SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_session_t *session, + switch_media_type_t mtype, + switch_io_type_t iotype, + switch_codec_control_command_t cmd, + switch_codec_control_type_t ctype, + void *cmd_data, + switch_codec_control_type_t *rtype, + void **ret_data); + SWITCH_END_EXTERN_C #endif /* For Emacs: diff --git a/src/include/switch_loadable_module.h b/src/include/switch_loadable_module.h index 514ad45534..89549241f7 100644 --- a/src/include/switch_loadable_module.h +++ b/src/include/switch_loadable_module.h @@ -533,19 +533,21 @@ static inline void switch_core_codec_add_implementation(switch_memory_pool_t *po ///\} static inline void switch_core_codec_add_video_implementation(switch_memory_pool_t *pool, switch_codec_interface_t *codec_interface, - /*! the IANA code number */ - switch_payload_t ianacode, - /*! the IANA code name */ - const char *iananame, - /*! default fmtp to send (can be overridden by the init function) */ - char *fmtp, - switch_core_codec_init_func_t init, - /*! function to encode raw data into encoded data */ - switch_core_codec_video_encode_func_t encode, - /*! function to decode encoded data into raw data */ - switch_core_codec_video_decode_func_t decode, - /*! deinitalize a codec handle using this implementation */ - switch_core_codec_destroy_func_t destroy) + /*! the IANA code number */ + switch_payload_t ianacode, + /*! the IANA code name */ + const char *iananame, + /*! default fmtp to send (can be overridden by the init function) */ + char *fmtp, + switch_core_codec_init_func_t init, + /*! function to encode raw data into encoded data */ + switch_core_codec_video_encode_func_t encode, + /*! function to decode encoded data into raw data */ + switch_core_codec_video_decode_func_t decode, + /*! function to send control messages to the codec */ + switch_core_codec_control_func_t control, + /*! deinitalize a codec handle using this implementation */ + switch_core_codec_destroy_func_t destroy) { switch_codec_implementation_t *impl = (switch_codec_implementation_t *) switch_core_alloc(pool, sizeof(*impl)); @@ -564,6 +566,7 @@ static inline void switch_core_codec_add_video_implementation(switch_memory_pool impl->init = init; impl->encode_video = encode; impl->decode_video = decode; + impl->codec_control = control; impl->destroy = destroy; impl->codec_id = codec_interface->codec_id; impl->next = codec_interface->implementations; diff --git a/src/include/switch_module_interfaces.h b/src/include/switch_module_interfaces.h index 20f110ab5f..60298f0f66 100644 --- a/src/include/switch_module_interfaces.h +++ b/src/include/switch_module_interfaces.h @@ -679,6 +679,8 @@ struct switch_codec_implementation { switch_core_codec_video_encode_func_t encode_video; /*! function to decode video encoded data into raw data */ switch_core_codec_video_decode_func_t decode_video; + /*! function to send control messages to the codec */ + switch_core_codec_control_func_t codec_control; /*! deinitalize a codec handle using this implementation */ switch_core_codec_destroy_func_t destroy; uint32_t codec_id; diff --git a/src/include/switch_types.h b/src/include/switch_types.h index dd6a979a62..56093011c3 100644 --- a/src/include/switch_types.h +++ b/src/include/switch_types.h @@ -2168,12 +2168,33 @@ typedef switch_status_t (*switch_core_codec_decode_func_t) (switch_codec_t *code void *decoded_data, uint32_t *decoded_data_len, uint32_t *decoded_rate, unsigned int *flag); typedef switch_status_t (*switch_core_codec_video_encode_func_t) (switch_codec_t *codec, - switch_image_t *img, - void *encoded_data, uint32_t *encoded_data_len, unsigned int *flag); + switch_image_t *img, + void *encoded_data, uint32_t *encoded_data_len, unsigned int *flag); typedef switch_status_t (*switch_core_codec_video_decode_func_t) (switch_codec_t *codec, - switch_frame_t *frame, - switch_image_t **img, unsigned int *flag); + switch_frame_t *frame, + switch_image_t **img, unsigned int *flag); + +typedef enum { + SCC_VIDEO_REFRESH = 0 +} switch_codec_control_command_t; + +typedef enum { + SCCT_NONE = 0 +} switch_codec_control_type_t; + +typedef enum { + SWITCH_IO_READ, + SWITCH_IO_WRITE +} switch_io_type_t; + +typedef switch_status_t (*switch_core_codec_control_func_t) (switch_codec_t *codec, + switch_codec_control_command_t cmd, + switch_codec_control_type_t ctype, + void *cmd_data, + switch_codec_control_type_t *rtype, + void **ret_data); + typedef switch_status_t (*switch_core_codec_init_func_t) (switch_codec_t *, switch_codec_flag_t, const switch_codec_settings_t *codec_settings); typedef switch_status_t (*switch_core_codec_fmtp_parse_func_t) (const char *fmtp, switch_codec_fmtp_t *codec_fmtp); diff --git a/src/mod/codecs/mod_openh264/mod_openh264.cpp b/src/mod/codecs/mod_openh264/mod_openh264.cpp index c17f8af078..1b88dbce6c 100644 --- a/src/mod/codecs/mod_openh264/mod_openh264.cpp +++ b/src/mod/codecs/mod_openh264/mod_openh264.cpp @@ -533,6 +533,16 @@ end: return SWITCH_STATUS_SUCCESS; } +static switch_status_t switch_h264_control(switch_codec_t *codec, + switch_codec_control_command_t cmd, + switch_codec_control_type_t ctype, + void *cmd_data, + switch_codec_control_type_t *rtype, + void **ret_data) { + + return SWITCH_STATUS_SUCCESS; +} + static switch_status_t switch_h264_destroy(switch_codec_t *codec) { h264_codec_context_t *context = (h264_codec_context_t *)codec->private_info; @@ -565,7 +575,7 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_openh264_load) SWITCH_ADD_CODEC(codec_interface, "H264 Video (with Cisco OpenH264)"); switch_core_codec_add_video_implementation(pool, codec_interface, 99, "H264", NULL, - switch_h264_init, switch_h264_encode, switch_h264_decode, switch_h264_destroy); + switch_h264_init, switch_h264_encode, switch_h264_decode, switch_h264_control, switch_h264_destroy); /* indicate that the module should continue to be loaded */ return SWITCH_STATUS_SUCCESS; diff --git a/src/mod/codecs/mod_vpx/mod_vpx.c b/src/mod/codecs/mod_vpx/mod_vpx.c index 185b977bbf..0fe2b0a139 100644 --- a/src/mod/codecs/mod_vpx/mod_vpx.c +++ b/src/mod/codecs/mod_vpx/mod_vpx.c @@ -358,6 +358,7 @@ static switch_status_t switch_vpx_encode(switch_codec_t *codec, switch_image_t * if (context->need_key_frame > 0) { // force generate a key frame + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "VPX KEYFRAME REQ\n"); vpx_flags |= VPX_EFLAG_FORCE_KF; context->last_ts = switch_micro_time_now(); context->need_key_frame--; @@ -508,6 +509,30 @@ error: return SWITCH_STATUS_FALSE; } + +static switch_status_t switch_vpx_control(switch_codec_t *codec, + switch_codec_control_command_t cmd, + switch_codec_control_type_t ctype, + void *cmd_data, + switch_codec_control_type_t *rtype, + void **ret_data) +{ + + vpx_context_t *context = (vpx_context_t *)codec->private_info; + + switch(cmd) { + case SCC_VIDEO_REFRESH: + context->need_key_frame = 1; + break; + default: + break; + } + + + return SWITCH_STATUS_SUCCESS; +} + + static switch_status_t switch_vpx_destroy(switch_codec_t *codec) { vpx_context_t *context = (vpx_context_t *)codec->private_info; @@ -541,7 +566,7 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_vpx_load) *module_interface = switch_loadable_module_create_module_interface(pool, modname); SWITCH_ADD_CODEC(codec_interface, "VP8 Video"); switch_core_codec_add_video_implementation(pool, codec_interface, 99, "VP8", NULL, - switch_vpx_init, switch_vpx_encode, switch_vpx_decode, switch_vpx_destroy); + switch_vpx_init, switch_vpx_encode, switch_vpx_decode, switch_vpx_control, switch_vpx_destroy); /* indicate that the module should continue to be loaded */ return SWITCH_STATUS_SUCCESS; diff --git a/src/mod/codecs/mod_yuv/mod_yuv.c b/src/mod/codecs/mod_yuv/mod_yuv.c index 6748e61f12..a1f38a929b 100644 --- a/src/mod/codecs/mod_yuv/mod_yuv.c +++ b/src/mod/codecs/mod_yuv/mod_yuv.c @@ -85,7 +85,7 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_yuv_load) SWITCH_ADD_CODEC(codec_interface, "YUV I420 Video (raw)"); switch_core_codec_add_video_implementation(pool, codec_interface, 99, "I420", NULL, - switch_yuv_init, switch_yuv_encode, switch_yuv_decode, switch_yuv_destroy); + switch_yuv_init, switch_yuv_encode, switch_yuv_decode, NULL, switch_yuv_destroy); /* indicate that the module should continue to be loaded */ return SWITCH_STATUS_SUCCESS; diff --git a/src/switch_core_codec.c b/src/switch_core_codec.c index 0800db506c..ef776c790f 100644 --- a/src/switch_core_codec.c +++ b/src/switch_core_codec.c @@ -843,6 +843,39 @@ SWITCH_DECLARE(switch_status_t) switch_core_codec_decode_video(switch_codec_t *c return status; } + +SWITCH_DECLARE(switch_status_t) switch_core_codec_control(switch_codec_t *codec, + switch_codec_control_command_t cmd, + switch_codec_control_type_t ctype, + void *cmd_data, + switch_codec_control_type_t *rtype, + void **ret_data) +{ + + switch_status_t status = SWITCH_STATUS_FALSE; + + + switch_assert(codec != NULL); + + + if (!codec->implementation || !switch_core_codec_ready(codec)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec is not initialized!\n"); + return SWITCH_STATUS_NOT_INITALIZED; + } + + + if (codec->mutex) switch_mutex_lock(codec->mutex); + + if (codec->implementation->codec_control) { + status = codec->implementation->codec_control(codec, cmd, ctype, cmd_data, rtype, ret_data); + } + + if (codec->mutex) switch_mutex_unlock(codec->mutex); + + + return status; +} + SWITCH_DECLARE(switch_status_t) switch_core_codec_destroy(switch_codec_t *codec) { switch_mutex_t *mutex = codec->mutex; diff --git a/src/switch_core_media.c b/src/switch_core_media.c index 9dcedfbd9b..8387eaf372 100644 --- a/src/switch_core_media.c +++ b/src/switch_core_media.c @@ -152,6 +152,7 @@ typedef struct switch_rtp_engine_s { uint8_t fir; uint8_t pli; + uint8_t nack; uint8_t no_crypto; } switch_rtp_engine_t; @@ -4006,9 +4007,13 @@ SWITCH_DECLARE(uint8_t) switch_core_media_negotiate_sdp(switch_core_session_t *s v_engine->fir++; } - //if (switch_stristr("pli", attr->a_value)) { - // v_engine->pli++; - //} + if (switch_stristr("pli", attr->a_value)) { + v_engine->pli++; + } + + if (switch_stristr("nack", attr->a_value)) { + v_engine->nack++; + } smh->mparams->rtcp_video_interval_msec = "10000"; } @@ -6429,9 +6434,12 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess switch_media_handle_t *smh; ice_t *ice_out; int vp8 = 0; - int red = 0; + //int red = 0; payload_map_t *pmap; int is_outbound = switch_channel_direction(session->channel) == SWITCH_CALL_DIRECTION_OUTBOUND; + const char *vbw; + int bw = 256; + uint8_t fir = 0, nack = 0, pli = 0; switch_assert(session); @@ -7041,9 +7049,9 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess vp8 = v_engine->cur_payload_map->pt; } - if (!strcasecmp(v_engine->cur_payload_map->rm_encoding, "red")) { - red = v_engine->cur_payload_map->pt; - } + //if (!strcasecmp(v_engine->cur_payload_map->rm_encoding, "red")) { + // red = v_engine->cur_payload_map->pt; + //} rate = v_engine->cur_payload_map->rm_rate; switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=rtpmap:%d %s/%ld\n", @@ -7129,9 +7137,9 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess vp8 = ianacode; } - if (!strcasecmp(imp->iananame, "red")) { - red = ianacode; - } + //if (!strcasecmp(imp->iananame, "red")) { + // red = ianacode; + //} if (channels > 1) { switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=rtpmap:%d %s/%d/%d\n", ianacode, imp->iananame, @@ -7187,11 +7195,41 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess } - if (v_engine->fir || v_engine->pli) { - switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), - "a=rtcp-fb:* %s%s\n", v_engine->fir ? "fir " : "", v_engine->pli ? "pli" : ""); + if ((vbw = switch_channel_get_variable(smh->session->channel, "rtp_video_max_bandwidth"))) { + int v = atoi(vbw); + bw = v; + } + + if (bw > 0) { + switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "b=AS:%d\n", bw); + } + + + if (sdp_type == SDP_TYPE_REQUEST) { + fir++; + pli++; + nack++; + } + + if (vp8) { + + if (v_engine->fir || fir) { + switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), + "a=rtcp-fb:%d ccm fir\n", vp8); + } + + if (v_engine->nack || nack) { + switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), + "a=rtcp-fb:%d nack\n", vp8); + } + + if (v_engine->pli || pli) { + switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), + "a=rtcp-fb:%d nack pli\n", vp8); + } } + //switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=ssrc:%u\n", v_engine->ssrc); if (v_engine->ice_out.cands[0][0].ready) { @@ -7201,8 +7239,7 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess uint32_t c2 = (2^24)*126 + (2^8)*65535 + (2^0)*(256 - 2); uint32_t c3 = (2^24)*126 + (2^8)*65534 + (2^0)*(256 - 1); uint32_t c4 = (2^24)*126 + (2^8)*65534 + (2^0)*(256 - 2); - const char *vbw; - int bw = 256; + tmp1[10] = '\0'; tmp2[10] = '\0'; @@ -7210,27 +7247,8 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess switch_stun_random_string(tmp2, 10, "0123456789"); ice_out = &v_engine->ice_out; - - - if ((vbw = switch_channel_get_variable(smh->session->channel, "rtp_video_max_bandwidth"))) { - int v = atoi(vbw); - bw = v; - } - - if (bw > 0) { - switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "b=AS:%d\n", bw); - } - - if (vp8 && switch_channel_test_flag(session->channel, CF_WEBRTC)) { - switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), - "a=rtcp-fb:%d ccm fir\n", vp8); - } - - if (red) { - switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), - "a=rtcp-fb:%d nack\n", vp8); - } - + + switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=ssrc:%u cname:%s\n", v_engine->ssrc, smh->cname); switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=ssrc:%u msid:%s v0\n", v_engine->ssrc, smh->msid); switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=ssrc:%u mslabel:%s\n", v_engine->ssrc, smh->msid); @@ -9426,6 +9444,45 @@ SWITCH_DECLARE(char *) switch_core_media_process_sdp_filter(const char *sdp, con } + +SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_session_t *session, + switch_media_type_t mtype, + switch_io_type_t iotype, + switch_codec_control_command_t cmd, + switch_codec_control_type_t ctype, + void *cmd_data, + switch_codec_control_type_t *rtype, + void **ret_data) +{ + switch_rtp_engine_t *engine = NULL; + switch_media_handle_t *smh = NULL; + switch_codec_t *codec = NULL; + + switch_assert(session); + + if (!(smh = session->media_handle)) { + return SWITCH_STATUS_FALSE; + } + + if (!(engine = &smh->engines[mtype])) { + return SWITCH_STATUS_NOTIMPL; + } + + if (iotype == SWITCH_IO_READ) { + codec = &engine->read_codec; + } else { + codec = &engine->write_codec; + } + + if (codec) { + return switch_core_codec_control(codec, cmd, ctype, cmd_data, rtype, ret_data); + } + + return SWITCH_STATUS_FALSE; +} + + + /* For Emacs: * Local Variables: * mode:c diff --git a/src/switch_rtp.c b/src/switch_rtp.c index 0416d153d3..d7b46ac213 100644 --- a/src/switch_rtp.c +++ b/src/switch_rtp.c @@ -5652,8 +5652,28 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_ if (rtcp_status == SWITCH_STATUS_SUCCESS) { switch_rtp_reset_media_timer(rtp_session); - - if (rtp_session->flags[SWITCH_RTP_FLAG_RTCP_PASSTHRU] || rtp_session->rtcp_recv_msg_p->header.type == 206) { + + if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && (rtp_session->rtcp_recv_msg_p->header.type == 205 || //RTPFB + rtp_session->rtcp_recv_msg_p->header.type == 206)) {//PSFB + rtcp_ext_msg_t *extp = (rtcp_ext_msg_t *) rtp_session->rtcp_recv_msg_p; + + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "PICKED UP XRTCP type: %d fmt: %d\n", + rtp_session->rtcp_recv_msg_p->header.type, extp->header.fmt); + + if ((extp->header.fmt == 4) || (extp->header.fmt == 1)) { /* FIR || PLI */ + + switch_core_media_codec_control(rtp_session->session, + SWITCH_MEDIA_TYPE_VIDEO, + SWITCH_IO_WRITE, + SCC_VIDEO_REFRESH, + SCCT_NONE, + NULL, + NULL, + NULL); + } + } + + if (rtp_session->flags[SWITCH_RTP_FLAG_RTCP_PASSTHRU]) { switch_channel_t *channel = switch_core_session_get_channel(rtp_session->session); const char *uuid = switch_channel_get_partner_uuid(channel); @@ -5667,13 +5687,7 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_ other_rtp_session->rtcp_sock_output && switch_rtp_test_flag(other_rtp_session, SWITCH_RTP_FLAG_ENABLE_RTCP)) { other_rtp_session->rtcp_send_msg = rtp_session->rtcp_recv_msg; - - if (rtp_session->rtcp_recv_msg_p->header.type == 206) { - rtcp_ext_msg_t *extp = (rtcp_ext_msg_t *) rtp_session->rtcp_recv_msg_p; - extp->header.recv_ssrc = htonl(other_rtp_session->stats.rtcp.peer_ssrc); - } - - + #ifdef ENABLE_SRTP if (switch_rtp_test_flag(other_rtp_session, SWITCH_RTP_FLAG_SECURE_SEND)) { int sbytes = (int) rtcp_bytes; -- 2.47.2