]> git.ipfire.org Git - thirdparty/linux.git/commitdiff
media: amphion: Support dmabuf and v4l2 buffer without binding
authorMing Qian <ming.qian@oss.nxp.com>
Thu, 24 Apr 2025 10:33:24 +0000 (18:33 +0800)
committerHans Verkuil <hverkuil@xs4all.nl>
Sat, 12 Jul 2025 06:34:48 +0000 (08:34 +0200)
When using VB2_DMABUF, the relationship between dma-buf and v4l2 buffer
may not one-to-one, a single dma-buf may be queued via different
v4l2 buffers, and different dma-bufs may be queued via the same
v4l2 buffer, so it's not appropriate to use the v4l2 buffer index
as the frame store id.

We can generate a frame store id according to the dma address.
Then for a given dma-buf, the id is fixed.

Driver now manages the frame store and vb2-buffer states independently.

When a dmabuf is queued via another v4l2 buffer before the buffer is
released by firmware, need to pend it until firmware release it.

Signed-off-by: Ming Qian <ming.qian@oss.nxp.com>
Reviewed-by: Nicolas Dufresne <nicolas.dufresne@collabora.com>
Signed-off-by: Nicolas Dufresne <nicolas.dufresne@collabora.com>
Signed-off-by: Hans Verkuil <hverkuil@xs4all.nl>
drivers/media/platform/amphion/vdec.c
drivers/media/platform/amphion/vpu.h
drivers/media/platform/amphion/vpu_dbg.c
drivers/media/platform/amphion/vpu_v4l2.c

index 55067d9405c2b15461d78067ab4ed68c8c3a7074..32eef2fd1f2a90243256709d2a6a1f7d62a6816a 100644 (file)
@@ -26,6 +26,7 @@
 #include "vpu_cmds.h"
 #include "vpu_rpc.h"
 
+#define VDEC_SLOT_CNT_DFT              32
 #define VDEC_MIN_BUFFER_CAP            8
 #define VDEC_MIN_BUFFER_OUT            8
 
@@ -41,6 +42,14 @@ struct vdec_fs_info {
        u32 tag;
 };
 
+struct vdec_frame_store_t {
+       struct vpu_vb2_buffer *curr;
+       struct vpu_vb2_buffer *pend;
+       dma_addr_t addr;
+       unsigned int state;
+       u32 tag;
+};
+
 struct vdec_t {
        u32 seq_hdr_found;
        struct vpu_buffer udata;
@@ -48,7 +57,8 @@ struct vdec_t {
        struct vpu_dec_codec_info codec_info;
        enum vpu_codec_state state;
 
-       struct vpu_vb2_buffer *slots[VB2_MAX_FRAME];
+       struct vdec_frame_store_t *slots;
+       u32 slot_count;
        u32 req_frame_count;
        struct vdec_fs_info mbi;
        struct vdec_fs_info dcp;
@@ -287,6 +297,63 @@ static int vdec_ctrl_init(struct vpu_inst *inst)
        return 0;
 }
 
+static void vdec_attach_frame_store(struct vpu_inst *inst, struct vb2_buffer *vb)
+{
+       struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+       struct vpu_vb2_buffer *vpu_buf = to_vpu_vb2_buffer(vbuf);
+       struct vdec_t *vdec = inst->priv;
+       struct vdec_frame_store_t *new_slots = NULL;
+       dma_addr_t addr;
+       int i;
+
+       addr = vpu_get_vb_phy_addr(vb, 0);
+       for (i = 0; i < vdec->slot_count; i++) {
+               if (addr == vdec->slots[i].addr) {
+                       if (vdec->slots[i].curr && vdec->slots[i].curr != vpu_buf) {
+                               vpu_set_buffer_state(vbuf, VPU_BUF_STATE_CHANGED);
+                               vdec->slots[i].pend = vpu_buf;
+                       } else {
+                               vpu_set_buffer_state(vbuf, vdec->slots[i].state);
+                       }
+                       vpu_buf->fs_id = i;
+                       return;
+               }
+       }
+
+       for (i = 0; i < vdec->slot_count; i++) {
+               if (!vdec->slots[i].addr) {
+                       vdec->slots[i].addr = addr;
+                       vpu_buf->fs_id = i;
+                       return;
+               }
+       }
+
+       new_slots = krealloc_array(vdec->slots, vdec->slot_count * 2,
+                                  sizeof(*vdec->slots),
+                                  GFP_KERNEL | __GFP_ZERO);
+       if (!new_slots) {
+               vpu_set_buffer_state(vbuf, VPU_BUF_STATE_ERROR);
+               return;
+       }
+
+       vdec->slots = new_slots;
+       vdec->slot_count *= 2;
+
+       vdec->slots[i].addr = addr;
+       vpu_buf->fs_id = i;
+}
+
+static void vdec_reset_frame_store(struct vpu_inst *inst)
+{
+       struct vdec_t *vdec = inst->priv;
+
+       if (!vdec->slots || !vdec->slot_count)
+               return;
+
+       vpu_trace(inst->dev, "inst[%d] reset slots\n", inst->id);
+       memset(vdec->slots, 0, sizeof(*vdec->slots) * vdec->slot_count);
+}
+
 static void vdec_handle_resolution_change(struct vpu_inst *inst)
 {
        struct vdec_t *vdec = inst->priv;
@@ -743,11 +810,11 @@ static int vdec_frame_decoded(struct vpu_inst *inst, void *arg)
        struct vb2_v4l2_buffer *src_buf;
        int ret = 0;
 
-       if (!info || info->id >= ARRAY_SIZE(vdec->slots))
+       if (!info || info->id >= vdec->slot_count)
                return -EINVAL;
 
        vpu_inst_lock(inst);
-       vpu_buf = vdec->slots[info->id];
+       vpu_buf = vdec->slots[info->id].curr;
        if (!vpu_buf) {
                dev_err(inst->dev, "[%d] decoded invalid frame[%d]\n", inst->id, info->id);
                ret = -EINVAL;
@@ -768,11 +835,13 @@ static int vdec_frame_decoded(struct vpu_inst *inst, void *arg)
        if (vpu_get_buffer_state(vbuf) == VPU_BUF_STATE_DECODED)
                dev_info(inst->dev, "[%d] buf[%d] has been decoded\n", inst->id, info->id);
        vpu_set_buffer_state(vbuf, VPU_BUF_STATE_DECODED);
+       vdec->slots[info->id].state = VPU_BUF_STATE_DECODED;
        vdec->decoded_frame_count++;
        if (vdec->params.display_delay_enable) {
                struct vpu_format *cur_fmt;
 
                cur_fmt = vpu_get_format(inst, inst->cap_format.type);
+               vdec->slots[info->id].state = VPU_BUF_STATE_READY;
                vpu_set_buffer_state(vbuf, VPU_BUF_STATE_READY);
                for (int i = 0; i < vbuf->vb2_buf.num_planes; i++)
                        vb2_set_plane_payload(&vbuf->vb2_buf,
@@ -795,11 +864,11 @@ static struct vpu_vb2_buffer *vdec_find_buffer(struct vpu_inst *inst, u32 luma)
        struct vdec_t *vdec = inst->priv;
        int i;
 
-       for (i = 0; i < ARRAY_SIZE(vdec->slots); i++) {
-               if (!vdec->slots[i])
+       for (i = 0; i < vdec->slot_count; i++) {
+               if (!vdec->slots[i].curr)
                        continue;
-               if (luma == vdec->slots[i]->luma)
-                       return vdec->slots[i];
+               if (luma == vdec->slots[i].addr)
+                       return vdec->slots[i].curr;
        }
 
        return NULL;
@@ -833,11 +902,11 @@ static void vdec_buf_done(struct vpu_inst *inst, struct vpu_frame_info *frame)
 
        cur_fmt = vpu_get_format(inst, inst->cap_format.type);
        vbuf = &vpu_buf->m2m_buf.vb;
-       if (vbuf->vb2_buf.index != frame->id)
-               dev_err(inst->dev, "[%d] buffer id(%d, %d) mismatch\n",
-                       inst->id, vbuf->vb2_buf.index, frame->id);
+       if (vpu_buf->fs_id != frame->id)
+               dev_err(inst->dev, "[%d] buffer id(%d(%d), %d) mismatch\n",
+                       inst->id, vpu_buf->fs_id, vbuf->vb2_buf.index, frame->id);
 
-       if (vpu_get_buffer_state(vbuf) == VPU_BUF_STATE_READY && vdec->params.display_delay_enable)
+       if (vdec->params.display_delay_enable)
                return;
 
        if (vpu_get_buffer_state(vbuf) != VPU_BUF_STATE_DECODED)
@@ -850,10 +919,11 @@ static void vdec_buf_done(struct vpu_inst *inst, struct vpu_frame_info *frame)
        vbuf->sequence = vdec->sequence;
        dev_dbg(inst->dev, "[%d][OUTPUT TS]%32lld\n", inst->id, vbuf->vb2_buf.timestamp);
 
-       v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_DONE);
        vpu_inst_lock(inst);
+       vdec->slots[vpu_buf->fs_id].state = VPU_BUF_STATE_READY;
        vdec->display_frame_count++;
        vpu_inst_unlock(inst);
+       v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_DONE);
        dev_dbg(inst->dev, "[%d] decoded : %d, display : %d, sequence : %d\n",
                inst->id, vdec->decoded_frame_count, vdec->display_frame_count, vdec->sequence);
 }
@@ -1081,18 +1151,30 @@ static int vdec_response_frame(struct vpu_inst *inst, struct vb2_v4l2_buffer *vb
        if (!vbuf)
                return -EINVAL;
 
-       if (vdec->slots[vbuf->vb2_buf.index]) {
-               dev_err(inst->dev, "[%d] repeat alloc fs %d\n",
-                       inst->id, vbuf->vb2_buf.index);
+       vpu_buf = to_vpu_vb2_buffer(vbuf);
+       if (vpu_buf->fs_id < 0 || vpu_buf->fs_id >= vdec->slot_count) {
+               dev_err(inst->dev, "invalid fs %d for v4l2 buffer %d\n",
+                       vpu_buf->fs_id, vbuf->vb2_buf.index);
                return -EINVAL;
        }
 
+       if (vdec->slots[vpu_buf->fs_id].curr) {
+               if (vdec->slots[vpu_buf->fs_id].curr != vpu_buf) {
+                       vpu_set_buffer_state(vbuf, VPU_BUF_STATE_CHANGED);
+                       vdec->slots[vpu_buf->fs_id].pend = vpu_buf;
+               } else {
+                       vpu_set_buffer_state(vbuf, vdec->slots[vpu_buf->fs_id].state);
+               }
+               dev_err(inst->dev, "[%d] repeat alloc fs %d (v4l2 index %d)\n",
+                       inst->id, vpu_buf->fs_id, vbuf->vb2_buf.index);
+               return -EAGAIN;
+       }
+
        dev_dbg(inst->dev, "[%d] state = %s, alloc fs %d, tag = 0x%x\n",
                inst->id, vpu_codec_state_name(inst->state), vbuf->vb2_buf.index, vdec->seq_tag);
-       vpu_buf = to_vpu_vb2_buffer(vbuf);
 
        memset(&info, 0, sizeof(info));
-       info.id = vbuf->vb2_buf.index;
+       info.id = vpu_buf->fs_id;
        info.type = MEM_RES_FRAME;
        info.tag = vdec->seq_tag;
        info.luma_addr = vpu_get_vb_phy_addr(&vbuf->vb2_buf, 0);
@@ -1107,12 +1189,13 @@ static int vdec_response_frame(struct vpu_inst *inst, struct vb2_v4l2_buffer *vb
        if (ret)
                return ret;
 
-       vpu_buf->tag = info.tag;
        vpu_buf->luma = info.luma_addr;
        vpu_buf->chroma_u = info.chroma_addr;
        vpu_buf->chroma_v = 0;
        vpu_set_buffer_state(vbuf, VPU_BUF_STATE_INUSE);
-       vdec->slots[info.id] = vpu_buf;
+       vdec->slots[info.id].tag = info.tag;
+       vdec->slots[info.id].curr = vpu_buf;
+       vdec->slots[info.id].state = VPU_BUF_STATE_INUSE;
        vdec->req_frame_count--;
 
        return 0;
@@ -1173,25 +1256,47 @@ static void vdec_recycle_buffer(struct vpu_inst *inst, struct vb2_v4l2_buffer *v
        v4l2_m2m_buf_queue(inst->fh.m2m_ctx, vbuf);
 }
 
-static void vdec_clear_slots(struct vpu_inst *inst)
+static void vdec_release_curr_frame_store(struct vpu_inst *inst, u32 id)
 {
        struct vdec_t *vdec = inst->priv;
        struct vpu_vb2_buffer *vpu_buf;
        struct vb2_v4l2_buffer *vbuf;
+
+       if (id >= vdec->slot_count)
+               return;
+       if (!vdec->slots[id].curr)
+               return;
+
+       vpu_buf = vdec->slots[id].curr;
+       vbuf = &vpu_buf->m2m_buf.vb;
+
+       vdec_response_fs_release(inst, id, vdec->slots[id].tag);
+       if (vpu_buf->fs_id == id) {
+               if (vpu_buf->state != VPU_BUF_STATE_READY)
+                       vdec_recycle_buffer(inst, vbuf);
+               vpu_set_buffer_state(vbuf, VPU_BUF_STATE_IDLE);
+       }
+
+       vdec->slots[id].curr = NULL;
+       vdec->slots[id].state = VPU_BUF_STATE_IDLE;
+
+       if (vdec->slots[id].pend) {
+               vpu_set_buffer_state(&vdec->slots[id].pend->m2m_buf.vb, VPU_BUF_STATE_IDLE);
+               vdec->slots[id].pend = NULL;
+       }
+}
+
+static void vdec_clear_slots(struct vpu_inst *inst)
+{
+       struct vdec_t *vdec = inst->priv;
        int i;
 
-       for (i = 0; i < ARRAY_SIZE(vdec->slots); i++) {
-               if (!vdec->slots[i])
+       for (i = 0; i < vdec->slot_count; i++) {
+               if (!vdec->slots[i].curr)
                        continue;
 
-               vpu_buf = vdec->slots[i];
-               vbuf = &vpu_buf->m2m_buf.vb;
-
                vpu_trace(inst->dev, "clear slot %d\n", i);
-               vdec_response_fs_release(inst, i, vpu_buf->tag);
-               vdec_recycle_buffer(inst, vbuf);
-               vdec->slots[i]->state = VPU_BUF_STATE_IDLE;
-               vdec->slots[i] = NULL;
+               vdec_release_curr_frame_store(inst, i);
        }
 }
 
@@ -1322,39 +1427,29 @@ static void vdec_event_req_fs(struct vpu_inst *inst, struct vpu_fs_info *fs)
 static void vdec_evnet_rel_fs(struct vpu_inst *inst, struct vpu_fs_info *fs)
 {
        struct vdec_t *vdec = inst->priv;
-       struct vpu_vb2_buffer *vpu_buf;
-       struct vb2_v4l2_buffer *vbuf;
 
-       if (!fs || fs->id >= ARRAY_SIZE(vdec->slots))
+       if (!fs || fs->id >= vdec->slot_count)
                return;
        if (fs->type != MEM_RES_FRAME)
                return;
 
-       if (fs->id >= vpu_get_num_buffers(inst, inst->cap_format.type)) {
+       if (fs->id >= vdec->slot_count) {
                dev_err(inst->dev, "[%d] invalid fs(%d) to release\n", inst->id, fs->id);
                return;
        }
 
        vpu_inst_lock(inst);
-       vpu_buf = vdec->slots[fs->id];
-       vdec->slots[fs->id] = NULL;
-
-       if (!vpu_buf) {
+       if (!vdec->slots[fs->id].curr) {
                dev_dbg(inst->dev, "[%d] fs[%d] has bee released\n", inst->id, fs->id);
                goto exit;
        }
 
-       vbuf = &vpu_buf->m2m_buf.vb;
-       if (vpu_get_buffer_state(vbuf) == VPU_BUF_STATE_DECODED) {
+       if (vdec->slots[fs->id].state == VPU_BUF_STATE_DECODED) {
                dev_dbg(inst->dev, "[%d] frame skip\n", inst->id);
                vdec->sequence++;
        }
 
-       vdec_response_fs_release(inst, fs->id, vpu_buf->tag);
-       if (vpu_get_buffer_state(vbuf) != VPU_BUF_STATE_READY)
-               vdec_recycle_buffer(inst, vbuf);
-
-       vpu_set_buffer_state(vbuf, VPU_BUF_STATE_IDLE);
+       vdec_release_curr_frame_store(inst, fs->id);
        vpu_process_capture_buffer(inst);
 
 exit:
@@ -1544,6 +1639,11 @@ static void vdec_cleanup(struct vpu_inst *inst)
                return;
 
        vdec = inst->priv;
+       if (vdec) {
+               kfree(vdec->slots);
+               vdec->slots = NULL;
+               vdec->slot_count = 0;
+       }
        vfree(vdec);
        inst->priv = NULL;
        vfree(inst);
@@ -1665,11 +1765,43 @@ static int vdec_stop_session(struct vpu_inst *inst, u32 type)
        return 0;
 }
 
-static int vdec_get_debug_info(struct vpu_inst *inst, char *str, u32 size, u32 i)
+static int vdec_get_slot_debug_info(struct vpu_inst *inst, char *str, u32 size, u32 i)
 {
        struct vdec_t *vdec = inst->priv;
+       struct vpu_vb2_buffer *vpu_buf;
        int num = -1;
 
+       vpu_inst_lock(inst);
+       if (i >= vdec->slot_count || !vdec->slots[i].addr)
+               goto exit;
+
+       vpu_buf = vdec->slots[i].curr;
+
+       num = scnprintf(str, size, "slot[%2d] :", i);
+       if (vpu_buf) {
+               num += scnprintf(str + num, size - num, " %2d",
+                                vpu_buf->m2m_buf.vb.vb2_buf.index);
+               num += scnprintf(str + num, size - num, "; state = %d", vdec->slots[i].state);
+       } else {
+               num += scnprintf(str + num, size - num, " -1");
+       }
+
+       if (vdec->slots[i].pend)
+               num += scnprintf(str + num, size - num, "; %d",
+                                vdec->slots[i].pend->m2m_buf.vb.vb2_buf.index);
+
+       num += scnprintf(str + num, size - num, "\n");
+exit:
+       vpu_inst_unlock(inst);
+
+       return num;
+}
+
+static int vdec_get_debug_info(struct vpu_inst *inst, char *str, u32 size, u32 i)
+{
+       struct vdec_t *vdec = inst->priv;
+       int num;
+
        switch (i) {
        case 0:
                num = scnprintf(str, size,
@@ -1723,6 +1855,7 @@ static int vdec_get_debug_info(struct vpu_inst *inst, char *str, u32 size, u32 i
                                vdec->codec_info.vui_present);
                break;
        default:
+               num = vdec_get_slot_debug_info(inst, str, size, i - 10);
                break;
        }
 
@@ -1746,6 +1879,8 @@ static struct vpu_inst_ops vdec_inst_ops = {
        .get_debug_info = vdec_get_debug_info,
        .wait_prepare = vpu_inst_unlock,
        .wait_finish = vpu_inst_lock,
+       .attach_frame_store = vdec_attach_frame_store,
+       .reset_frame_store = vdec_reset_frame_store,
 };
 
 static void vdec_init(struct file *file)
@@ -1786,6 +1921,16 @@ static int vdec_open(struct file *file)
                return -ENOMEM;
        }
 
+       vdec->slots = kmalloc_array(VDEC_SLOT_CNT_DFT,
+                                   sizeof(*vdec->slots),
+                                   GFP_KERNEL | __GFP_ZERO);
+       if (!vdec->slots) {
+               vfree(vdec);
+               vfree(inst);
+               return -ENOMEM;
+       }
+       vdec->slot_count = VDEC_SLOT_CNT_DFT;
+
        inst->ops = &vdec_inst_ops;
        inst->formats = vdec_formats;
        inst->type = VPU_CORE_TYPE_DEC;
index 1451549c9dd206c1b30170f839e8f8c430e09e55..cac0f1a64feafc41d9f050f58004ae47f2e2ac90 100644 (file)
@@ -222,6 +222,8 @@ struct vpu_inst_ops {
        int (*get_debug_info)(struct vpu_inst *inst, char *str, u32 size, u32 i);
        void (*wait_prepare)(struct vpu_inst *inst);
        void (*wait_finish)(struct vpu_inst *inst);
+       void (*attach_frame_store)(struct vpu_inst *inst, struct vb2_buffer *vb);
+       void (*reset_frame_store)(struct vpu_inst *inst);
 };
 
 struct vpu_inst {
@@ -295,7 +297,8 @@ enum {
        VPU_BUF_STATE_DECODED,
        VPU_BUF_STATE_READY,
        VPU_BUF_STATE_SKIP,
-       VPU_BUF_STATE_ERROR
+       VPU_BUF_STATE_ERROR,
+       VPU_BUF_STATE_CHANGED
 };
 
 struct vpu_vb2_buffer {
@@ -304,8 +307,8 @@ struct vpu_vb2_buffer {
        dma_addr_t chroma_u;
        dma_addr_t chroma_v;
        unsigned int state;
-       u32 tag;
        u32 average_qp;
+       s32 fs_id;
 };
 
 void vpu_writel(struct vpu_dev *vpu, u32 reg, u32 val);
index 940e5bda5fa391c47552c275bd1266f47d57f475..497ae4e8a229b298a038061dda15d41c107729d7 100644 (file)
@@ -48,6 +48,7 @@ static char *vpu_stat_name[] = {
        [VPU_BUF_STATE_READY] = "ready",
        [VPU_BUF_STATE_SKIP] = "skip",
        [VPU_BUF_STATE_ERROR] = "error",
+       [VPU_BUF_STATE_CHANGED] = "changed",
 };
 
 static inline const char *to_vpu_stat_name(int state)
@@ -164,6 +165,7 @@ static int vpu_dbg_instance(struct seq_file *s, void *data)
        for (i = 0; i < vb2_get_num_buffers(vq); i++) {
                struct vb2_buffer *vb;
                struct vb2_v4l2_buffer *vbuf;
+               struct vpu_vb2_buffer *vpu_buf;
 
                vb = vb2_get_buffer(vq, i);
                if (!vb)
@@ -173,13 +175,24 @@ static int vpu_dbg_instance(struct seq_file *s, void *data)
                        continue;
 
                vbuf = to_vb2_v4l2_buffer(vb);
+               vpu_buf = to_vpu_vb2_buffer(vbuf);
 
                num = scnprintf(str, sizeof(str),
-                               "capture[%2d] state = %10s, %8s\n",
+                               "capture[%2d] state = %10s, %8s",
                                i, vb2_stat_name[vb->state],
                                to_vpu_stat_name(vpu_get_buffer_state(vbuf)));
                if (seq_write(s, str, num))
                        return 0;
+
+               if (vpu_buf->fs_id >= 0) {
+                       num = scnprintf(str, sizeof(str), "; fs %d", vpu_buf->fs_id);
+                       if (seq_write(s, str, num))
+                               return 0;
+               }
+
+               num = scnprintf(str, sizeof(str), "\n");
+               if (seq_write(s, str, num))
+                       return 0;
        }
 
        num = scnprintf(str, sizeof(str), "sequence = %d\n", inst->sequence);
index 45707931bc4f1fd0bce4852f97721bb7af98f44d..74668fa362e24fd34829b500e99c8455a9413fc1 100644 (file)
@@ -501,14 +501,25 @@ static int vpu_vb2_queue_setup(struct vb2_queue *vq,
                call_void_vop(inst, release);
        }
 
+       if (V4L2_TYPE_IS_CAPTURE(vq->type))
+               call_void_vop(inst, reset_frame_store);
+
        return 0;
 }
 
 static int vpu_vb2_buf_init(struct vb2_buffer *vb)
 {
        struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+       struct vpu_vb2_buffer *vpu_buf = to_vpu_vb2_buffer(vbuf);
+       struct vpu_inst *inst = vb2_get_drv_priv(vb->vb2_queue);
 
+       vpu_buf->fs_id = -1;
        vpu_set_buffer_state(vbuf, VPU_BUF_STATE_IDLE);
+
+       if (!inst->ops->attach_frame_store || V4L2_TYPE_IS_OUTPUT(vb->type))
+               return 0;
+
+       call_void_vop(inst, attach_frame_store, vb);
        return 0;
 }