fix memory leak - transcoding

Fixes coverity scan issues: 551230, 551229, 507422 and 507421

diff --git a/src/transcoding/transcode/audio.c b/src/transcoding/transcode/audio.c
index b9ee7a432f368d6d40a5470d2395a858cc3ec21c..4db5e070f54fdacc5db404c26379c0142bb6b704 100644 (file)
--- a/src/transcoding/transcode/audio.c
+++ b/src/transcoding/transcode/audio.c
@@ -85,7 +85,7 @@ _audio_context_channel_layout(TVHContext *self, AVDictionary **opts, AVChannelLa
 {
     const AVChannelLayout *channel_layouts =
         tvh_codec_profile_audio_get_channel_layouts(self->profile);
-    AVChannelLayout ilayout;
+    AVChannelLayout ilayout = {0};
     av_channel_layout_copy(&ilayout, &self->iavctx->ch_layout);
     AVChannelLayout olayout;
     av_channel_layout_default(&olayout, 0);
@@ -193,7 +193,6 @@ tvh_audio_context_open_encoder(TVHContext *self, AVDictionary **opts)
                         "audio encoder has no suitable channel layout");
         return -1;
     }
-    self->oavctx->ch_layout.nb_channels = self->oavctx->ch_layout.nb_channels;
 #else
     self->oavctx->channel_layout = _audio_context_channel_layout(self, opts);
     if (!self->oavctx->channel_layout) {

diff --git a/src/transcoding/transcode/hwaccels/vaapi.c b/src/transcoding/transcode/hwaccels/vaapi.c
index 711f1156023f6b73d0d9e367c3dac61df7bc79e0..bdb9914b96a610f8de98e4be630aa4b9cd95bfd0 100644 (file)
--- a/src/transcoding/transcode/hwaccels/vaapi.c
+++ b/src/transcoding/transcode/hwaccels/vaapi.c
@@ -637,6 +637,9 @@ vaapi_decode_setup_context(AVCodecContext *avctx)
         tvherror(LS_VAAPI, "Decode: Failed to Open VAAPI device and create an AVHWDeviceContext for device: "
                             "%s with error code: %s", 
                             ctx->hw_accel_device, av_err2str(ret));
+        // unref self
+        free(self);
+        self = NULL;
         return ret;
     }
 
@@ -645,6 +648,12 @@ vaapi_decode_setup_context(AVCodecContext *avctx)
     if (!avctx->hw_device_ctx) {
         tvherror(LS_VAAPI, "Decode: Failed to create a hardware device reference for device: %s.", 
                         ctx->hw_accel_device);
+        // unref hw_device_ref
+        av_buffer_unref(&self->hw_device_ref);
+        self->hw_device_ref = NULL;
+        // unref self
+        free(self);
+        self = NULL;
         return AVERROR(ENOMEM);
     }
     ctx->hw_accel_ictx = self;
@@ -767,6 +776,9 @@ vaapi_encode_setup_context(AVCodecContext *avctx)
     if ((ret = av_hwdevice_ctx_create(&self->hw_frame_ref, AV_HWDEVICE_TYPE_VAAPI, NULL, NULL, 0)) < 0) {
         tvherror(LS_VAAPI, "Encode: Failed to open VAAPI device and create an AVHWDeviceContext for it."
                 "Error code: %s",av_err2str(ret));
+        // unref self
+        free(self);
+        self = NULL;
         return ret;
     }
 
@@ -774,6 +786,12 @@ vaapi_encode_setup_context(AVCodecContext *avctx)
     if ((ret = set_hwframe_ctx(avctx, self->hw_frame_ref)) < 0) {
         tvherror(LS_VAAPI, "Encode: Failed to set hwframe context."
                 "Error code: %s",av_err2str(ret));
+        // unref hw_frame_ref
+        av_buffer_unref(&self->hw_frame_ref);
+        self->hw_frame_ref = NULL;
+        // unref self
+        free(self);
+        self = NULL;
         return ret;
     }
     ctx->hw_device_octx = av_buffer_ref(self->hw_frame_ref);