1 From adc589d2a20808fb99d46a78175cd023f2040338 Mon Sep 17 00:00:00 2001
2 From: =?UTF-8?q?Lucas=20A=2E=20M=2E=20Magalh=C3=A3es?= <lucmaga@gmail.com>
3 Date: Mon, 21 Jan 2019 20:05:01 -0500
4 Subject: media: vimc: Add vimc-streamer for stream control
6 Content-Type: text/plain; charset=UTF-8
7 Content-Transfer-Encoding: 8bit
9 From: Lucas A. M. Magalhães <lucmaga@gmail.com>
11 commit adc589d2a20808fb99d46a78175cd023f2040338 upstream.
13 Add a linear pipeline logic for the stream control. It's created by
14 walking backwards on the entity graph. When the stream starts it will
15 simply loop through the pipeline calling the respective process_frame
16 function of each entity.
18 Fixes: f2fe89061d797 ("vimc: Virtual Media Controller core, capture
21 Cc: stable@vger.kernel.org # for v4.20
22 Signed-off-by: Lucas A. M. Magalhães <lucmaga@gmail.com>
23 Acked-by: Helen Koike <helen.koike@collabora.com>
24 Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
25 [hverkuil-cisco@xs4all.nl: fixed small space-after-tab issue in the patch]
26 Signed-off-by: Mauro Carvalho Chehab <mchehab+samsung@kernel.org>
27 Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
30 drivers/media/platform/vimc/Makefile | 3
31 drivers/media/platform/vimc/vimc-capture.c | 18 +-
32 drivers/media/platform/vimc/vimc-common.c | 35 -----
33 drivers/media/platform/vimc/vimc-common.h | 15 --
34 drivers/media/platform/vimc/vimc-debayer.c | 26 ---
35 drivers/media/platform/vimc/vimc-scaler.c | 28 ----
36 drivers/media/platform/vimc/vimc-sensor.c | 56 +-------
37 drivers/media/platform/vimc/vimc-streamer.c | 188 ++++++++++++++++++++++++++++
38 drivers/media/platform/vimc/vimc-streamer.h | 38 +++++
39 9 files changed, 260 insertions(+), 147 deletions(-)
41 --- a/drivers/media/platform/vimc/Makefile
42 +++ b/drivers/media/platform/vimc/Makefile
43 @@ -5,6 +5,7 @@ vimc_common-objs := vimc-common.o
44 vimc_debayer-objs := vimc-debayer.o
45 vimc_scaler-objs := vimc-scaler.o
46 vimc_sensor-objs := vimc-sensor.o
47 +vimc_streamer-objs := vimc-streamer.o
49 obj-$(CONFIG_VIDEO_VIMC) += vimc.o vimc_capture.o vimc_common.o vimc-debayer.o \
50 - vimc_scaler.o vimc_sensor.o
51 + vimc_scaler.o vimc_sensor.o vimc_streamer.o
52 --- a/drivers/media/platform/vimc/vimc-capture.c
53 +++ b/drivers/media/platform/vimc/vimc-capture.c
55 #include <media/videobuf2-vmalloc.h>
57 #include "vimc-common.h"
58 +#include "vimc-streamer.h"
60 #define VIMC_CAP_DRV_NAME "vimc-capture"
62 @@ -44,7 +45,7 @@ struct vimc_cap_device {
66 - struct media_pipeline pipe;
67 + struct vimc_stream stream;
70 static const struct v4l2_pix_format fmt_default = {
71 @@ -248,14 +249,13 @@ static int vimc_cap_start_streaming(stru
74 /* Start the media pipeline */
75 - ret = media_pipeline_start(entity, &vcap->pipe);
76 + ret = media_pipeline_start(entity, &vcap->stream.pipe);
78 vimc_cap_return_all_buffers(vcap, VB2_BUF_STATE_QUEUED);
82 - /* Enable streaming from the pipe */
83 - ret = vimc_pipeline_s_stream(&vcap->vdev.entity, 1);
84 + ret = vimc_streamer_s_stream(&vcap->stream, &vcap->ved, 1);
86 media_pipeline_stop(entity);
87 vimc_cap_return_all_buffers(vcap, VB2_BUF_STATE_QUEUED);
88 @@ -273,8 +273,7 @@ static void vimc_cap_stop_streaming(stru
90 struct vimc_cap_device *vcap = vb2_get_drv_priv(vq);
92 - /* Disable streaming from the pipe */
93 - vimc_pipeline_s_stream(&vcap->vdev.entity, 0);
94 + vimc_streamer_s_stream(&vcap->stream, &vcap->ved, 0);
96 /* Stop the media pipeline */
97 media_pipeline_stop(&vcap->vdev.entity);
98 @@ -355,8 +354,8 @@ static void vimc_cap_comp_unbind(struct
102 -static void vimc_cap_process_frame(struct vimc_ent_device *ved,
103 - struct media_pad *sink, const void *frame)
104 +static void *vimc_cap_process_frame(struct vimc_ent_device *ved,
107 struct vimc_cap_device *vcap = container_of(ved, struct vimc_cap_device,
109 @@ -370,7 +369,7 @@ static void vimc_cap_process_frame(struc
110 typeof(*vimc_buf), list);
112 spin_unlock(&vcap->qlock);
114 + return ERR_PTR(-EAGAIN);
117 /* Remove this entry from the list */
118 @@ -391,6 +390,7 @@ static void vimc_cap_process_frame(struc
119 vb2_set_plane_payload(&vimc_buf->vb2.vb2_buf, 0,
120 vcap->format.sizeimage);
121 vb2_buffer_done(&vimc_buf->vb2.vb2_buf, VB2_BUF_STATE_DONE);
125 static int vimc_cap_comp_bind(struct device *comp, struct device *master,
126 --- a/drivers/media/platform/vimc/vimc-common.c
127 +++ b/drivers/media/platform/vimc/vimc-common.c
128 @@ -207,41 +207,6 @@ const struct vimc_pix_map *vimc_pix_map_
130 EXPORT_SYMBOL_GPL(vimc_pix_map_by_pixelformat);
132 -int vimc_propagate_frame(struct media_pad *src, const void *frame)
134 - struct media_link *link;
136 - if (!(src->flags & MEDIA_PAD_FL_SOURCE))
139 - /* Send this frame to all sink pads that are direct linked */
140 - list_for_each_entry(link, &src->entity->links, list) {
141 - if (link->source == src &&
142 - (link->flags & MEDIA_LNK_FL_ENABLED)) {
143 - struct vimc_ent_device *ved = NULL;
144 - struct media_entity *entity = link->sink->entity;
146 - if (is_media_entity_v4l2_subdev(entity)) {
147 - struct v4l2_subdev *sd =
148 - container_of(entity, struct v4l2_subdev,
150 - ved = v4l2_get_subdevdata(sd);
151 - } else if (is_media_entity_v4l2_video_device(entity)) {
152 - struct video_device *vdev =
153 - container_of(entity,
154 - struct video_device,
156 - ved = video_get_drvdata(vdev);
158 - if (ved && ved->process_frame)
159 - ved->process_frame(ved, link->sink, frame);
165 -EXPORT_SYMBOL_GPL(vimc_propagate_frame);
167 /* Helper function to allocate and initialize pads */
168 struct media_pad *vimc_pads_init(u16 num_pads, const unsigned long *pads_flag)
170 --- a/drivers/media/platform/vimc/vimc-common.h
171 +++ b/drivers/media/platform/vimc/vimc-common.h
172 @@ -113,24 +113,13 @@ struct vimc_pix_map {
173 struct vimc_ent_device {
174 struct media_entity *ent;
175 struct media_pad *pads;
176 - void (*process_frame)(struct vimc_ent_device *ved,
177 - struct media_pad *sink, const void *frame);
178 + void * (*process_frame)(struct vimc_ent_device *ved,
179 + const void *frame);
180 void (*vdev_get_format)(struct vimc_ent_device *ved,
181 struct v4l2_pix_format *fmt);
185 - * vimc_propagate_frame - propagate a frame through the topology
187 - * @src: the source pad where the frame is being originated
188 - * @frame: the frame to be propagated
190 - * This function will call the process_frame callback from the vimc_ent_device
191 - * struct of the nodes directly connected to the @src pad
193 -int vimc_propagate_frame(struct media_pad *src, const void *frame);
196 * vimc_pads_init - initialize pads
198 * @num_pads: number of pads to initialize
199 --- a/drivers/media/platform/vimc/vimc-debayer.c
200 +++ b/drivers/media/platform/vimc/vimc-debayer.c
201 @@ -321,7 +321,6 @@ static void vimc_deb_set_rgb_mbus_fmt_rg
202 static int vimc_deb_s_stream(struct v4l2_subdev *sd, int enable)
204 struct vimc_deb_device *vdeb = v4l2_get_subdevdata(sd);
208 const struct vimc_pix_map *vpix;
209 @@ -351,22 +350,10 @@ static int vimc_deb_s_stream(struct v4l2
210 if (!vdeb->src_frame)
213 - /* Turn the stream on in the subdevices directly connected */
214 - ret = vimc_pipeline_s_stream(&vdeb->sd.entity, 1);
216 - vfree(vdeb->src_frame);
217 - vdeb->src_frame = NULL;
221 if (!vdeb->src_frame)
224 - /* Disable streaming from the pipe */
225 - ret = vimc_pipeline_s_stream(&vdeb->sd.entity, 0);
229 vfree(vdeb->src_frame);
230 vdeb->src_frame = NULL;
232 @@ -480,9 +467,8 @@ static void vimc_deb_calc_rgb_sink(struc
236 -static void vimc_deb_process_frame(struct vimc_ent_device *ved,
237 - struct media_pad *sink,
238 - const void *sink_frame)
239 +static void *vimc_deb_process_frame(struct vimc_ent_device *ved,
240 + const void *sink_frame)
242 struct vimc_deb_device *vdeb = container_of(ved, struct vimc_deb_device,
244 @@ -491,7 +477,7 @@ static void vimc_deb_process_frame(struc
246 /* If the stream in this node is not active, just return */
247 if (!vdeb->src_frame)
249 + return ERR_PTR(-EINVAL);
251 for (i = 0; i < vdeb->sink_fmt.height; i++)
252 for (j = 0; j < vdeb->sink_fmt.width; j++) {
253 @@ -499,12 +485,8 @@ static void vimc_deb_process_frame(struc
254 vdeb->set_rgb_src(vdeb, i, j, rgb);
257 - /* Propagate the frame through all source pads */
258 - for (i = 1; i < vdeb->sd.entity.num_pads; i++) {
259 - struct media_pad *pad = &vdeb->sd.entity.pads[i];
260 + return vdeb->src_frame;
262 - vimc_propagate_frame(pad, vdeb->src_frame);
266 static void vimc_deb_comp_unbind(struct device *comp, struct device *master,
267 --- a/drivers/media/platform/vimc/vimc-scaler.c
268 +++ b/drivers/media/platform/vimc/vimc-scaler.c
269 @@ -217,7 +217,6 @@ static const struct v4l2_subdev_pad_ops
270 static int vimc_sca_s_stream(struct v4l2_subdev *sd, int enable)
272 struct vimc_sca_device *vsca = v4l2_get_subdevdata(sd);
276 const struct vimc_pix_map *vpix;
277 @@ -245,22 +244,10 @@ static int vimc_sca_s_stream(struct v4l2
278 if (!vsca->src_frame)
281 - /* Turn the stream on in the subdevices directly connected */
282 - ret = vimc_pipeline_s_stream(&vsca->sd.entity, 1);
284 - vfree(vsca->src_frame);
285 - vsca->src_frame = NULL;
289 if (!vsca->src_frame)
292 - /* Disable streaming from the pipe */
293 - ret = vimc_pipeline_s_stream(&vsca->sd.entity, 0);
297 vfree(vsca->src_frame);
298 vsca->src_frame = NULL;
300 @@ -346,26 +333,19 @@ static void vimc_sca_fill_src_frame(cons
301 vimc_sca_scale_pix(vsca, i, j, sink_frame);
304 -static void vimc_sca_process_frame(struct vimc_ent_device *ved,
305 - struct media_pad *sink,
306 - const void *sink_frame)
307 +static void *vimc_sca_process_frame(struct vimc_ent_device *ved,
308 + const void *sink_frame)
310 struct vimc_sca_device *vsca = container_of(ved, struct vimc_sca_device,
314 /* If the stream in this node is not active, just return */
315 if (!vsca->src_frame)
317 + return ERR_PTR(-EINVAL);
319 vimc_sca_fill_src_frame(vsca, sink_frame);
321 - /* Propagate the frame through all source pads */
322 - for (i = 1; i < vsca->sd.entity.num_pads; i++) {
323 - struct media_pad *pad = &vsca->sd.entity.pads[i];
325 - vimc_propagate_frame(pad, vsca->src_frame);
327 + return vsca->src_frame;
330 static void vimc_sca_comp_unbind(struct device *comp, struct device *master,
331 --- a/drivers/media/platform/vimc/vimc-sensor.c
332 +++ b/drivers/media/platform/vimc/vimc-sensor.c
336 #include <linux/component.h>
337 -#include <linux/freezer.h>
338 -#include <linux/kthread.h>
339 #include <linux/module.h>
340 #include <linux/mod_devicetable.h>
341 #include <linux/platform_device.h>
342 @@ -201,38 +199,27 @@ static const struct v4l2_subdev_pad_ops
343 .set_fmt = vimc_sen_set_fmt,
346 -static int vimc_sen_tpg_thread(void *data)
347 +static void *vimc_sen_process_frame(struct vimc_ent_device *ved,
348 + const void *sink_frame)
350 - struct vimc_sen_device *vsen = data;
354 - set_current_state(TASK_UNINTERRUPTIBLE);
358 - if (kthread_should_stop())
361 - tpg_fill_plane_buffer(&vsen->tpg, 0, 0, vsen->frame);
363 - /* Send the frame to all source pads */
364 - for (i = 0; i < vsen->sd.entity.num_pads; i++)
365 - vimc_propagate_frame(&vsen->sd.entity.pads[i],
367 + struct vimc_sen_device *vsen = container_of(ved, struct vimc_sen_device,
369 + const struct vimc_pix_map *vpix;
370 + unsigned int frame_size;
372 - /* 60 frames per second */
373 - schedule_timeout(HZ/60);
375 + /* Calculate the frame size */
376 + vpix = vimc_pix_map_by_code(vsen->mbus_format.code);
377 + frame_size = vsen->mbus_format.width * vpix->bpp *
378 + vsen->mbus_format.height;
381 + tpg_fill_plane_buffer(&vsen->tpg, 0, 0, vsen->frame);
382 + return vsen->frame;
385 static int vimc_sen_s_stream(struct v4l2_subdev *sd, int enable)
387 struct vimc_sen_device *vsen =
388 container_of(sd, struct vimc_sen_device, sd);
392 const struct vimc_pix_map *vpix;
393 @@ -258,26 +245,8 @@ static int vimc_sen_s_stream(struct v4l2
394 /* configure the test pattern generator */
395 vimc_sen_tpg_s_format(vsen);
397 - /* Initialize the image generator thread */
398 - vsen->kthread_sen = kthread_run(vimc_sen_tpg_thread, vsen,
399 - "%s-sen", vsen->sd.v4l2_dev->name);
400 - if (IS_ERR(vsen->kthread_sen)) {
401 - dev_err(vsen->dev, "%s: kernel_thread() failed\n",
403 - vfree(vsen->frame);
404 - vsen->frame = NULL;
405 - return PTR_ERR(vsen->kthread_sen);
408 - if (!vsen->kthread_sen)
411 - /* Stop image generator */
412 - ret = kthread_stop(vsen->kthread_sen);
416 - vsen->kthread_sen = NULL;
420 @@ -393,6 +362,7 @@ static int vimc_sen_comp_bind(struct dev
424 + vsen->ved.process_frame = vimc_sen_process_frame;
425 dev_set_drvdata(comp, &vsen->ved);
429 +++ b/drivers/media/platform/vimc/vimc-streamer.c
431 +// SPDX-License-Identifier: GPL-2.0+
433 + * vimc-streamer.c Virtual Media Controller Driver
435 + * Copyright (C) 2018 Lucas A. M. Magalhães <lucmaga@gmail.com>
439 +#include <linux/init.h>
440 +#include <linux/module.h>
441 +#include <linux/freezer.h>
442 +#include <linux/kthread.h>
444 +#include "vimc-streamer.h"
447 + * vimc_get_source_entity - get the entity connected with the first sink pad
449 + * @ent: reference media_entity
451 + * Helper function that returns the media entity containing the source pad
452 + * linked with the first sink pad from the given media entity pad list.
454 +static struct media_entity *vimc_get_source_entity(struct media_entity *ent)
456 + struct media_pad *pad;
459 + for (i = 0; i < ent->num_pads; i++) {
460 + if (ent->pads[i].flags & MEDIA_PAD_FL_SOURCE)
462 + pad = media_entity_remote_pad(&ent->pads[i]);
463 + return pad ? pad->entity : NULL;
469 + * vimc_streamer_pipeline_terminate - Disable stream in all ved in stream
471 + * @stream: the pointer to the stream structure with the pipeline to be
474 + * Calls s_stream to disable the stream in each entity of the pipeline
477 +static void vimc_streamer_pipeline_terminate(struct vimc_stream *stream)
479 + struct media_entity *entity;
480 + struct v4l2_subdev *sd;
482 + while (stream->pipe_size) {
483 + stream->pipe_size--;
484 + entity = stream->ved_pipeline[stream->pipe_size]->ent;
485 + entity = vimc_get_source_entity(entity);
486 + stream->ved_pipeline[stream->pipe_size] = NULL;
488 + if (!is_media_entity_v4l2_subdev(entity))
491 + sd = media_entity_to_v4l2_subdev(entity);
492 + v4l2_subdev_call(sd, video, s_stream, 0);
497 + * vimc_streamer_pipeline_init - initializes the stream structure
499 + * @stream: the pointer to the stream structure to be initialized
500 + * @ved: the pointer to the vimc entity initializing the stream
502 + * Initializes the stream structure. Walks through the entity graph to
503 + * construct the pipeline used later on the streamer thread.
504 + * Calls s_stream to enable stream in all entities of the pipeline.
506 +static int vimc_streamer_pipeline_init(struct vimc_stream *stream,
507 + struct vimc_ent_device *ved)
509 + struct media_entity *entity;
510 + struct video_device *vdev;
511 + struct v4l2_subdev *sd;
514 + stream->pipe_size = 0;
515 + while (stream->pipe_size < VIMC_STREAMER_PIPELINE_MAX_SIZE) {
517 + vimc_streamer_pipeline_terminate(stream);
520 + stream->ved_pipeline[stream->pipe_size++] = ved;
522 + entity = vimc_get_source_entity(ved->ent);
523 + /* Check if the end of the pipeline was reached*/
527 + if (is_media_entity_v4l2_subdev(entity)) {
528 + sd = media_entity_to_v4l2_subdev(entity);
529 + ret = v4l2_subdev_call(sd, video, s_stream, 1);
530 + if (ret && ret != -ENOIOCTLCMD) {
531 + vimc_streamer_pipeline_terminate(stream);
534 + ved = v4l2_get_subdevdata(sd);
536 + vdev = container_of(entity,
537 + struct video_device,
539 + ved = video_get_drvdata(vdev);
543 + vimc_streamer_pipeline_terminate(stream);
547 +static int vimc_streamer_thread(void *data)
549 + struct vimc_stream *stream = data;
553 + set_current_state(TASK_UNINTERRUPTIBLE);
557 + if (kthread_should_stop())
560 + for (i = stream->pipe_size - 1; i >= 0; i--) {
561 + stream->frame = stream->ved_pipeline[i]->process_frame(
562 + stream->ved_pipeline[i],
564 + if (!stream->frame)
566 + if (IS_ERR(stream->frame))
570 + schedule_timeout(HZ / 60);
576 +int vimc_streamer_s_stream(struct vimc_stream *stream,
577 + struct vimc_ent_device *ved,
582 + if (!stream || !ved)
586 + if (stream->kthread)
589 + ret = vimc_streamer_pipeline_init(stream, ved);
593 + stream->kthread = kthread_run(vimc_streamer_thread, stream,
594 + "vimc-streamer thread");
596 + if (IS_ERR(stream->kthread))
597 + return PTR_ERR(stream->kthread);
600 + if (!stream->kthread)
603 + ret = kthread_stop(stream->kthread);
607 + stream->kthread = NULL;
609 + vimc_streamer_pipeline_terminate(stream);
614 +EXPORT_SYMBOL_GPL(vimc_streamer_s_stream);
616 +MODULE_DESCRIPTION("Virtual Media Controller Driver (VIMC) Streamer");
617 +MODULE_AUTHOR("Lucas A. M. Magalhães <lucmaga@gmail.com>");
618 +MODULE_LICENSE("GPL");
620 +++ b/drivers/media/platform/vimc/vimc-streamer.h
622 +/* SPDX-License-Identifier: GPL-2.0+ */
624 + * vimc-streamer.h Virtual Media Controller Driver
626 + * Copyright (C) 2018 Lucas A. M. Magalhães <lucmaga@gmail.com>
630 +#ifndef _VIMC_STREAMER_H_
631 +#define _VIMC_STREAMER_H_
633 +#include <media/media-device.h>
635 +#include "vimc-common.h"
637 +#define VIMC_STREAMER_PIPELINE_MAX_SIZE 16
639 +struct vimc_stream {
640 + struct media_pipeline pipe;
641 + struct vimc_ent_device *ved_pipeline[VIMC_STREAMER_PIPELINE_MAX_SIZE];
642 + unsigned int pipe_size;
644 + struct task_struct *kthread;
648 + * vimc_streamer_s_streamer - start/stop the stream
650 + * @stream: the pointer to the stream to start or stop
651 + * @ved: The last entity of the streamer pipeline
652 + * @enable: any non-zero number start the stream, zero stop
655 +int vimc_streamer_s_stream(struct vimc_stream *stream,
656 + struct vimc_ent_device *ved,
659 +#endif //_VIMC_STREAMER_H_