From 27b12a92d04d5120b9e61f16e620d496d18d6f6d Mon Sep 17 00:00:00 2001
From: Ukn Unknown <4031821+uknunknown@users.noreply.github.com>
Date: Fri, 23 May 2025 14:44:11 -0700
Subject: [PATCH] fix memory leak - transcoding

Fixes coverity scan issues: 551230, 551229, 507422 and 507421
---
 src/transcoding/transcode/audio.c          |  3 +--
 src/transcoding/transcode/hwaccels/vaapi.c | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/transcoding/transcode/audio.c b/src/transcoding/transcode/audio.c
index b9ee7a432..4db5e070f 100644
--- a/src/transcoding/transcode/audio.c
+++ b/src/transcoding/transcode/audio.c
@@ -85,7 +85,7 @@ _audio_context_channel_layout(TVHContext *self, AVDictionary **opts, AVChannelLa
 {
     const AVChannelLayout *channel_layouts =
         tvh_codec_profile_audio_get_channel_layouts(self->profile);
-    AVChannelLayout ilayout;
+    AVChannelLayout ilayout = {0};
     av_channel_layout_copy(&ilayout, &self->iavctx->ch_layout);
     AVChannelLayout olayout;
     av_channel_layout_default(&olayout, 0);
@@ -193,7 +193,6 @@ tvh_audio_context_open_encoder(TVHContext *self, AVDictionary **opts)
                         "audio encoder has no suitable channel layout");
         return -1;
     }
-    self->oavctx->ch_layout.nb_channels = self->oavctx->ch_layout.nb_channels;
 #else
     self->oavctx->channel_layout = _audio_context_channel_layout(self, opts);
     if (!self->oavctx->channel_layout) {
diff --git a/src/transcoding/transcode/hwaccels/vaapi.c b/src/transcoding/transcode/hwaccels/vaapi.c
index 711f11560..bdb9914b9 100644
--- a/src/transcoding/transcode/hwaccels/vaapi.c
+++ b/src/transcoding/transcode/hwaccels/vaapi.c
@@ -637,6 +637,9 @@ vaapi_decode_setup_context(AVCodecContext *avctx)
         tvherror(LS_VAAPI, "Decode: Failed to Open VAAPI device and create an AVHWDeviceContext for device: "
                             "%s with error code: %s", 
                             ctx->hw_accel_device, av_err2str(ret));
+        // unref self
+        free(self);
+        self = NULL;
         return ret;
     }
 
@@ -645,6 +648,12 @@ vaapi_decode_setup_context(AVCodecContext *avctx)
     if (!avctx->hw_device_ctx) {
         tvherror(LS_VAAPI, "Decode: Failed to create a hardware device reference for device: %s.", 
                         ctx->hw_accel_device);
+        // unref hw_device_ref
+        av_buffer_unref(&self->hw_device_ref);
+        self->hw_device_ref = NULL;
+        // unref self
+        free(self);
+        self = NULL;
         return AVERROR(ENOMEM);
     }
     ctx->hw_accel_ictx = self;
@@ -767,6 +776,9 @@ vaapi_encode_setup_context(AVCodecContext *avctx)
     if ((ret = av_hwdevice_ctx_create(&self->hw_frame_ref, AV_HWDEVICE_TYPE_VAAPI, NULL, NULL, 0)) < 0) {
         tvherror(LS_VAAPI, "Encode: Failed to open VAAPI device and create an AVHWDeviceContext for it."
                 "Error code: %s",av_err2str(ret));
+        // unref self
+        free(self);
+        self = NULL;
         return ret;
     }
 
@@ -774,6 +786,12 @@ vaapi_encode_setup_context(AVCodecContext *avctx)
     if ((ret = set_hwframe_ctx(avctx, self->hw_frame_ref)) < 0) {
         tvherror(LS_VAAPI, "Encode: Failed to set hwframe context."
                 "Error code: %s",av_err2str(ret));
+        // unref hw_frame_ref
+        av_buffer_unref(&self->hw_frame_ref);
+        self->hw_frame_ref = NULL;
+        // unref self
+        free(self);
+        self = NULL;
         return ret;
     }
     ctx->hw_device_octx = av_buffer_ref(self->hw_frame_ref);
-- 
2.47.2