1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2013 Red Hat
4 * Author: Rob Clark <robdclark@gmail.com>
10 #include "msm_fence.h"
11 #include "msm_gpu_trace.h"
12 #include "adreno/adreno_gpu.h"
14 #include <generated/utsrelease.h>
15 #include <linux/string_helpers.h>
16 #include <linux/pm_opp.h>
17 #include <linux/devfreq.h>
18 #include <linux/devcoredump.h>
19 #include <linux/sched/task.h>
25 static int msm_devfreq_target(struct device
*dev
, unsigned long *freq
,
28 struct msm_gpu
*gpu
= platform_get_drvdata(to_platform_device(dev
));
29 struct dev_pm_opp
*opp
;
31 opp
= devfreq_recommended_opp(dev
, freq
, flags
);
36 if (gpu
->funcs
->gpu_set_freq
)
37 gpu
->funcs
->gpu_set_freq(gpu
, (u64
)*freq
);
39 clk_set_rate(gpu
->core_clk
, *freq
);
46 static int msm_devfreq_get_dev_status(struct device
*dev
,
47 struct devfreq_dev_status
*status
)
49 struct msm_gpu
*gpu
= platform_get_drvdata(to_platform_device(dev
));
52 if (gpu
->funcs
->gpu_get_freq
)
53 status
->current_frequency
= gpu
->funcs
->gpu_get_freq(gpu
);
55 status
->current_frequency
= clk_get_rate(gpu
->core_clk
);
57 status
->busy_time
= gpu
->funcs
->gpu_busy(gpu
);
60 status
->total_time
= ktime_us_delta(time
, gpu
->devfreq
.time
);
61 gpu
->devfreq
.time
= time
;
66 static int msm_devfreq_get_cur_freq(struct device
*dev
, unsigned long *freq
)
68 struct msm_gpu
*gpu
= platform_get_drvdata(to_platform_device(dev
));
70 if (gpu
->funcs
->gpu_get_freq
)
71 *freq
= gpu
->funcs
->gpu_get_freq(gpu
);
73 *freq
= clk_get_rate(gpu
->core_clk
);
78 static struct devfreq_dev_profile msm_devfreq_profile
= {
80 .target
= msm_devfreq_target
,
81 .get_dev_status
= msm_devfreq_get_dev_status
,
82 .get_cur_freq
= msm_devfreq_get_cur_freq
,
85 static void msm_devfreq_init(struct msm_gpu
*gpu
)
87 /* We need target support to do devfreq */
88 if (!gpu
->funcs
->gpu_busy
)
91 msm_devfreq_profile
.initial_freq
= gpu
->fast_rate
;
94 * Don't set the freq_table or max_state and let devfreq build the table
98 gpu
->devfreq
.devfreq
= devm_devfreq_add_device(&gpu
->pdev
->dev
,
99 &msm_devfreq_profile
, DEVFREQ_GOV_SIMPLE_ONDEMAND
,
102 if (IS_ERR(gpu
->devfreq
.devfreq
)) {
103 DRM_DEV_ERROR(&gpu
->pdev
->dev
, "Couldn't initialize GPU devfreq\n");
104 gpu
->devfreq
.devfreq
= NULL
;
107 devfreq_suspend_device(gpu
->devfreq
.devfreq
);
110 static int enable_pwrrail(struct msm_gpu
*gpu
)
112 struct drm_device
*dev
= gpu
->dev
;
116 ret
= regulator_enable(gpu
->gpu_reg
);
118 DRM_DEV_ERROR(dev
->dev
, "failed to enable 'gpu_reg': %d\n", ret
);
124 ret
= regulator_enable(gpu
->gpu_cx
);
126 DRM_DEV_ERROR(dev
->dev
, "failed to enable 'gpu_cx': %d\n", ret
);
134 static int disable_pwrrail(struct msm_gpu
*gpu
)
137 regulator_disable(gpu
->gpu_cx
);
139 regulator_disable(gpu
->gpu_reg
);
143 static int enable_clk(struct msm_gpu
*gpu
)
145 if (gpu
->core_clk
&& gpu
->fast_rate
)
146 clk_set_rate(gpu
->core_clk
, gpu
->fast_rate
);
148 /* Set the RBBM timer rate to 19.2Mhz */
149 if (gpu
->rbbmtimer_clk
)
150 clk_set_rate(gpu
->rbbmtimer_clk
, 19200000);
152 return clk_bulk_prepare_enable(gpu
->nr_clocks
, gpu
->grp_clks
);
155 static int disable_clk(struct msm_gpu
*gpu
)
157 clk_bulk_disable_unprepare(gpu
->nr_clocks
, gpu
->grp_clks
);
160 * Set the clock to a deliberately low rate. On older targets the clock
161 * speed had to be non zero to avoid problems. On newer targets this
162 * will be rounded down to zero anyway so it all works out.
165 clk_set_rate(gpu
->core_clk
, 27000000);
167 if (gpu
->rbbmtimer_clk
)
168 clk_set_rate(gpu
->rbbmtimer_clk
, 0);
173 static int enable_axi(struct msm_gpu
*gpu
)
176 clk_prepare_enable(gpu
->ebi1_clk
);
180 static int disable_axi(struct msm_gpu
*gpu
)
183 clk_disable_unprepare(gpu
->ebi1_clk
);
187 void msm_gpu_resume_devfreq(struct msm_gpu
*gpu
)
189 gpu
->devfreq
.busy_cycles
= 0;
190 gpu
->devfreq
.time
= ktime_get();
192 devfreq_resume_device(gpu
->devfreq
.devfreq
);
195 int msm_gpu_pm_resume(struct msm_gpu
*gpu
)
199 DBG("%s", gpu
->name
);
201 ret
= enable_pwrrail(gpu
);
205 ret
= enable_clk(gpu
);
209 ret
= enable_axi(gpu
);
213 msm_gpu_resume_devfreq(gpu
);
215 gpu
->needs_hw_init
= true;
220 int msm_gpu_pm_suspend(struct msm_gpu
*gpu
)
224 DBG("%s", gpu
->name
);
226 devfreq_suspend_device(gpu
->devfreq
.devfreq
);
228 ret
= disable_axi(gpu
);
232 ret
= disable_clk(gpu
);
236 ret
= disable_pwrrail(gpu
);
243 int msm_gpu_hw_init(struct msm_gpu
*gpu
)
247 WARN_ON(!mutex_is_locked(&gpu
->dev
->struct_mutex
));
249 if (!gpu
->needs_hw_init
)
252 disable_irq(gpu
->irq
);
253 ret
= gpu
->funcs
->hw_init(gpu
);
255 gpu
->needs_hw_init
= false;
256 enable_irq(gpu
->irq
);
261 #ifdef CONFIG_DEV_COREDUMP
262 static ssize_t
msm_gpu_devcoredump_read(char *buffer
, loff_t offset
,
263 size_t count
, void *data
, size_t datalen
)
265 struct msm_gpu
*gpu
= data
;
266 struct drm_print_iterator iter
;
267 struct drm_printer p
;
268 struct msm_gpu_state
*state
;
270 state
= msm_gpu_crashstate_get(gpu
);
279 p
= drm_coredump_printer(&iter
);
281 drm_printf(&p
, "---\n");
282 drm_printf(&p
, "kernel: " UTS_RELEASE
"\n");
283 drm_printf(&p
, "module: " KBUILD_MODNAME
"\n");
284 drm_printf(&p
, "time: %lld.%09ld\n",
285 state
->time
.tv_sec
, state
->time
.tv_nsec
);
287 drm_printf(&p
, "comm: %s\n", state
->comm
);
289 drm_printf(&p
, "cmdline: %s\n", state
->cmd
);
291 gpu
->funcs
->show(gpu
, state
, &p
);
293 msm_gpu_crashstate_put(gpu
);
295 return count
- iter
.remain
;
298 static void msm_gpu_devcoredump_free(void *data
)
300 struct msm_gpu
*gpu
= data
;
302 msm_gpu_crashstate_put(gpu
);
305 static void msm_gpu_crashstate_get_bo(struct msm_gpu_state
*state
,
306 struct msm_gem_object
*obj
, u64 iova
, u32 flags
)
308 struct msm_gpu_state_bo
*state_bo
= &state
->bos
[state
->nr_bos
];
310 /* Don't record write only objects */
311 state_bo
->size
= obj
->base
.size
;
312 state_bo
->iova
= iova
;
314 /* Only store data for non imported buffer objects marked for read */
315 if ((flags
& MSM_SUBMIT_BO_READ
) && !obj
->base
.import_attach
) {
318 state_bo
->data
= kvmalloc(obj
->base
.size
, GFP_KERNEL
);
322 ptr
= msm_gem_get_vaddr_active(&obj
->base
);
324 kvfree(state_bo
->data
);
325 state_bo
->data
= NULL
;
329 memcpy(state_bo
->data
, ptr
, obj
->base
.size
);
330 msm_gem_put_vaddr(&obj
->base
);
336 static void msm_gpu_crashstate_capture(struct msm_gpu
*gpu
,
337 struct msm_gem_submit
*submit
, char *comm
, char *cmd
)
339 struct msm_gpu_state
*state
;
341 /* Check if the target supports capturing crash state */
342 if (!gpu
->funcs
->gpu_state_get
)
345 /* Only save one crash state at a time */
349 state
= gpu
->funcs
->gpu_state_get(gpu
);
350 if (IS_ERR_OR_NULL(state
))
353 /* Fill in the additional crash state information */
354 state
->comm
= kstrdup(comm
, GFP_KERNEL
);
355 state
->cmd
= kstrdup(cmd
, GFP_KERNEL
);
360 /* count # of buffers to dump: */
361 for (i
= 0; i
< submit
->nr_bos
; i
++)
362 if (should_dump(submit
, i
))
364 /* always dump cmd bo's, but don't double count them: */
365 for (i
= 0; i
< submit
->nr_cmds
; i
++)
366 if (!should_dump(submit
, submit
->cmd
[i
].idx
))
369 state
->bos
= kcalloc(nr
,
370 sizeof(struct msm_gpu_state_bo
), GFP_KERNEL
);
372 for (i
= 0; i
< submit
->nr_bos
; i
++) {
373 if (should_dump(submit
, i
)) {
374 msm_gpu_crashstate_get_bo(state
, submit
->bos
[i
].obj
,
375 submit
->bos
[i
].iova
, submit
->bos
[i
].flags
);
379 for (i
= 0; state
->bos
&& i
< submit
->nr_cmds
; i
++) {
380 int idx
= submit
->cmd
[i
].idx
;
382 if (!should_dump(submit
, submit
->cmd
[i
].idx
)) {
383 msm_gpu_crashstate_get_bo(state
, submit
->bos
[idx
].obj
,
384 submit
->bos
[idx
].iova
, submit
->bos
[idx
].flags
);
389 /* Set the active crash state to be dumped on failure */
390 gpu
->crashstate
= state
;
392 /* FIXME: Release the crashstate if this errors out? */
393 dev_coredumpm(gpu
->dev
->dev
, THIS_MODULE
, gpu
, 0, GFP_KERNEL
,
394 msm_gpu_devcoredump_read
, msm_gpu_devcoredump_free
);
397 static void msm_gpu_crashstate_capture(struct msm_gpu
*gpu
,
398 struct msm_gem_submit
*submit
, char *comm
, char *cmd
)
404 * Hangcheck detection for locked gpu:
407 static void update_fences(struct msm_gpu
*gpu
, struct msm_ringbuffer
*ring
,
410 struct msm_gem_submit
*submit
;
412 list_for_each_entry(submit
, &ring
->submits
, node
) {
413 if (submit
->seqno
> fence
)
416 msm_update_fence(submit
->ring
->fctx
,
417 submit
->fence
->seqno
);
421 static struct msm_gem_submit
*
422 find_submit(struct msm_ringbuffer
*ring
, uint32_t fence
)
424 struct msm_gem_submit
*submit
;
426 WARN_ON(!mutex_is_locked(&ring
->gpu
->dev
->struct_mutex
));
428 list_for_each_entry(submit
, &ring
->submits
, node
)
429 if (submit
->seqno
== fence
)
435 static void retire_submits(struct msm_gpu
*gpu
);
437 static void recover_worker(struct work_struct
*work
)
439 struct msm_gpu
*gpu
= container_of(work
, struct msm_gpu
, recover_work
);
440 struct drm_device
*dev
= gpu
->dev
;
441 struct msm_drm_private
*priv
= dev
->dev_private
;
442 struct msm_gem_submit
*submit
;
443 struct msm_ringbuffer
*cur_ring
= gpu
->funcs
->active_ring(gpu
);
444 char *comm
= NULL
, *cmd
= NULL
;
447 mutex_lock(&dev
->struct_mutex
);
449 DRM_DEV_ERROR(dev
->dev
, "%s: hangcheck recover!\n", gpu
->name
);
451 submit
= find_submit(cur_ring
, cur_ring
->memptrs
->fence
+ 1);
453 struct task_struct
*task
;
455 /* Increment the fault counts */
456 gpu
->global_faults
++;
457 submit
->queue
->faults
++;
459 task
= get_pid_task(submit
->pid
, PIDTYPE_PID
);
461 comm
= kstrdup(task
->comm
, GFP_KERNEL
);
462 cmd
= kstrdup_quotable_cmdline(task
, GFP_KERNEL
);
463 put_task_struct(task
);
467 DRM_DEV_ERROR(dev
->dev
, "%s: offending task: %s (%s)\n",
468 gpu
->name
, comm
, cmd
);
470 msm_rd_dump_submit(priv
->hangrd
, submit
,
471 "offending task: %s (%s)", comm
, cmd
);
473 msm_rd_dump_submit(priv
->hangrd
, submit
, NULL
);
476 /* Record the crash state */
477 pm_runtime_get_sync(&gpu
->pdev
->dev
);
478 msm_gpu_crashstate_capture(gpu
, submit
, comm
, cmd
);
479 pm_runtime_put_sync(&gpu
->pdev
->dev
);
485 * Update all the rings with the latest and greatest fence.. this
486 * needs to happen after msm_rd_dump_submit() to ensure that the
487 * bo's referenced by the offending submit are still around.
489 for (i
= 0; i
< gpu
->nr_rings
; i
++) {
490 struct msm_ringbuffer
*ring
= gpu
->rb
[i
];
492 uint32_t fence
= ring
->memptrs
->fence
;
495 * For the current (faulting?) ring/submit advance the fence by
496 * one more to clear the faulting submit
498 if (ring
== cur_ring
)
501 update_fences(gpu
, ring
, fence
);
504 if (msm_gpu_active(gpu
)) {
505 /* retire completed submits, plus the one that hung: */
508 pm_runtime_get_sync(&gpu
->pdev
->dev
);
509 gpu
->funcs
->recover(gpu
);
510 pm_runtime_put_sync(&gpu
->pdev
->dev
);
513 * Replay all remaining submits starting with highest priority
516 for (i
= 0; i
< gpu
->nr_rings
; i
++) {
517 struct msm_ringbuffer
*ring
= gpu
->rb
[i
];
519 list_for_each_entry(submit
, &ring
->submits
, node
)
520 gpu
->funcs
->submit(gpu
, submit
, NULL
);
524 mutex_unlock(&dev
->struct_mutex
);
529 static void hangcheck_timer_reset(struct msm_gpu
*gpu
)
531 DBG("%s", gpu
->name
);
532 mod_timer(&gpu
->hangcheck_timer
,
533 round_jiffies_up(jiffies
+ DRM_MSM_HANGCHECK_JIFFIES
));
536 static void hangcheck_handler(struct timer_list
*t
)
538 struct msm_gpu
*gpu
= from_timer(gpu
, t
, hangcheck_timer
);
539 struct drm_device
*dev
= gpu
->dev
;
540 struct msm_drm_private
*priv
= dev
->dev_private
;
541 struct msm_ringbuffer
*ring
= gpu
->funcs
->active_ring(gpu
);
542 uint32_t fence
= ring
->memptrs
->fence
;
544 if (fence
!= ring
->hangcheck_fence
) {
545 /* some progress has been made.. ya! */
546 ring
->hangcheck_fence
= fence
;
547 } else if (fence
< ring
->seqno
) {
548 /* no progress and not done.. hung! */
549 ring
->hangcheck_fence
= fence
;
550 DRM_DEV_ERROR(dev
->dev
, "%s: hangcheck detected gpu lockup rb %d!\n",
551 gpu
->name
, ring
->id
);
552 DRM_DEV_ERROR(dev
->dev
, "%s: completed fence: %u\n",
554 DRM_DEV_ERROR(dev
->dev
, "%s: submitted fence: %u\n",
555 gpu
->name
, ring
->seqno
);
557 queue_work(priv
->wq
, &gpu
->recover_work
);
560 /* if still more pending work, reset the hangcheck timer: */
561 if (ring
->seqno
> ring
->hangcheck_fence
)
562 hangcheck_timer_reset(gpu
);
564 /* workaround for missing irq: */
565 queue_work(priv
->wq
, &gpu
->retire_work
);
569 * Performance Counters:
572 /* called under perf_lock */
573 static int update_hw_cntrs(struct msm_gpu
*gpu
, uint32_t ncntrs
, uint32_t *cntrs
)
575 uint32_t current_cntrs
[ARRAY_SIZE(gpu
->last_cntrs
)];
576 int i
, n
= min(ncntrs
, gpu
->num_perfcntrs
);
578 /* read current values: */
579 for (i
= 0; i
< gpu
->num_perfcntrs
; i
++)
580 current_cntrs
[i
] = gpu_read(gpu
, gpu
->perfcntrs
[i
].sample_reg
);
583 for (i
= 0; i
< n
; i
++)
584 cntrs
[i
] = current_cntrs
[i
] - gpu
->last_cntrs
[i
];
586 /* save current values: */
587 for (i
= 0; i
< gpu
->num_perfcntrs
; i
++)
588 gpu
->last_cntrs
[i
] = current_cntrs
[i
];
593 static void update_sw_cntrs(struct msm_gpu
*gpu
)
599 spin_lock_irqsave(&gpu
->perf_lock
, flags
);
600 if (!gpu
->perfcntr_active
)
604 elapsed
= ktime_to_us(ktime_sub(time
, gpu
->last_sample
.time
));
606 gpu
->totaltime
+= elapsed
;
607 if (gpu
->last_sample
.active
)
608 gpu
->activetime
+= elapsed
;
610 gpu
->last_sample
.active
= msm_gpu_active(gpu
);
611 gpu
->last_sample
.time
= time
;
614 spin_unlock_irqrestore(&gpu
->perf_lock
, flags
);
617 void msm_gpu_perfcntr_start(struct msm_gpu
*gpu
)
621 pm_runtime_get_sync(&gpu
->pdev
->dev
);
623 spin_lock_irqsave(&gpu
->perf_lock
, flags
);
624 /* we could dynamically enable/disable perfcntr registers too.. */
625 gpu
->last_sample
.active
= msm_gpu_active(gpu
);
626 gpu
->last_sample
.time
= ktime_get();
627 gpu
->activetime
= gpu
->totaltime
= 0;
628 gpu
->perfcntr_active
= true;
629 update_hw_cntrs(gpu
, 0, NULL
);
630 spin_unlock_irqrestore(&gpu
->perf_lock
, flags
);
633 void msm_gpu_perfcntr_stop(struct msm_gpu
*gpu
)
635 gpu
->perfcntr_active
= false;
636 pm_runtime_put_sync(&gpu
->pdev
->dev
);
639 /* returns -errno or # of cntrs sampled */
640 int msm_gpu_perfcntr_sample(struct msm_gpu
*gpu
, uint32_t *activetime
,
641 uint32_t *totaltime
, uint32_t ncntrs
, uint32_t *cntrs
)
646 spin_lock_irqsave(&gpu
->perf_lock
, flags
);
648 if (!gpu
->perfcntr_active
) {
653 *activetime
= gpu
->activetime
;
654 *totaltime
= gpu
->totaltime
;
656 gpu
->activetime
= gpu
->totaltime
= 0;
658 ret
= update_hw_cntrs(gpu
, ncntrs
, cntrs
);
661 spin_unlock_irqrestore(&gpu
->perf_lock
, flags
);
667 * Cmdstream submission/retirement:
670 static void retire_submit(struct msm_gpu
*gpu
, struct msm_ringbuffer
*ring
,
671 struct msm_gem_submit
*submit
)
673 int index
= submit
->seqno
% MSM_GPU_SUBMIT_STATS_COUNT
;
674 volatile struct msm_gpu_submit_stats
*stats
;
675 u64 elapsed
, clock
= 0;
678 stats
= &ring
->memptrs
->stats
[index
];
679 /* Convert 19.2Mhz alwayson ticks to nanoseconds for elapsed time */
680 elapsed
= (stats
->alwayson_end
- stats
->alwayson_start
) * 10000;
681 do_div(elapsed
, 192);
683 /* Calculate the clock frequency from the number of CP cycles */
685 clock
= (stats
->cpcycles_end
- stats
->cpcycles_start
) * 1000;
686 do_div(clock
, elapsed
);
689 trace_msm_gpu_submit_retired(submit
, elapsed
, clock
,
690 stats
->alwayson_start
, stats
->alwayson_end
);
692 for (i
= 0; i
< submit
->nr_bos
; i
++) {
693 struct msm_gem_object
*msm_obj
= submit
->bos
[i
].obj
;
694 /* move to inactive: */
695 msm_gem_move_to_inactive(&msm_obj
->base
);
696 msm_gem_unpin_iova(&msm_obj
->base
, submit
->aspace
);
697 drm_gem_object_put_locked(&msm_obj
->base
);
700 pm_runtime_mark_last_busy(&gpu
->pdev
->dev
);
701 pm_runtime_put_autosuspend(&gpu
->pdev
->dev
);
702 msm_gem_submit_free(submit
);
705 static void retire_submits(struct msm_gpu
*gpu
)
707 struct drm_device
*dev
= gpu
->dev
;
708 struct msm_gem_submit
*submit
, *tmp
;
711 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
713 /* Retire the commits starting with highest priority */
714 for (i
= 0; i
< gpu
->nr_rings
; i
++) {
715 struct msm_ringbuffer
*ring
= gpu
->rb
[i
];
717 list_for_each_entry_safe(submit
, tmp
, &ring
->submits
, node
) {
718 if (dma_fence_is_signaled(submit
->fence
))
719 retire_submit(gpu
, ring
, submit
);
724 static void retire_worker(struct work_struct
*work
)
726 struct msm_gpu
*gpu
= container_of(work
, struct msm_gpu
, retire_work
);
727 struct drm_device
*dev
= gpu
->dev
;
730 for (i
= 0; i
< gpu
->nr_rings
; i
++)
731 update_fences(gpu
, gpu
->rb
[i
], gpu
->rb
[i
]->memptrs
->fence
);
733 mutex_lock(&dev
->struct_mutex
);
735 mutex_unlock(&dev
->struct_mutex
);
738 /* call from irq handler to schedule work to retire bo's */
739 void msm_gpu_retire(struct msm_gpu
*gpu
)
741 struct msm_drm_private
*priv
= gpu
->dev
->dev_private
;
742 queue_work(priv
->wq
, &gpu
->retire_work
);
743 update_sw_cntrs(gpu
);
746 /* add bo's to gpu's ring, and kick gpu: */
747 void msm_gpu_submit(struct msm_gpu
*gpu
, struct msm_gem_submit
*submit
,
748 struct msm_file_private
*ctx
)
750 struct drm_device
*dev
= gpu
->dev
;
751 struct msm_drm_private
*priv
= dev
->dev_private
;
752 struct msm_ringbuffer
*ring
= submit
->ring
;
755 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
757 pm_runtime_get_sync(&gpu
->pdev
->dev
);
759 msm_gpu_hw_init(gpu
);
761 submit
->seqno
= ++ring
->seqno
;
763 list_add_tail(&submit
->node
, &ring
->submits
);
765 msm_rd_dump_submit(priv
->rd
, submit
, NULL
);
767 update_sw_cntrs(gpu
);
769 for (i
= 0; i
< submit
->nr_bos
; i
++) {
770 struct msm_gem_object
*msm_obj
= submit
->bos
[i
].obj
;
773 /* can't happen yet.. but when we add 2d support we'll have
774 * to deal w/ cross-ring synchronization:
776 WARN_ON(is_active(msm_obj
) && (msm_obj
->gpu
!= gpu
));
778 /* submit takes a reference to the bo and iova until retired: */
779 drm_gem_object_get(&msm_obj
->base
);
780 msm_gem_get_and_pin_iova(&msm_obj
->base
, submit
->aspace
, &iova
);
782 if (submit
->bos
[i
].flags
& MSM_SUBMIT_BO_WRITE
)
783 msm_gem_move_to_active(&msm_obj
->base
, gpu
, true, submit
->fence
);
784 else if (submit
->bos
[i
].flags
& MSM_SUBMIT_BO_READ
)
785 msm_gem_move_to_active(&msm_obj
->base
, gpu
, false, submit
->fence
);
788 gpu
->funcs
->submit(gpu
, submit
, ctx
);
791 hangcheck_timer_reset(gpu
);
798 static irqreturn_t
irq_handler(int irq
, void *data
)
800 struct msm_gpu
*gpu
= data
;
801 return gpu
->funcs
->irq(gpu
);
804 static int get_clocks(struct platform_device
*pdev
, struct msm_gpu
*gpu
)
806 int ret
= devm_clk_bulk_get_all(&pdev
->dev
, &gpu
->grp_clks
);
813 gpu
->nr_clocks
= ret
;
815 gpu
->core_clk
= msm_clk_bulk_get_clock(gpu
->grp_clks
,
816 gpu
->nr_clocks
, "core");
818 gpu
->rbbmtimer_clk
= msm_clk_bulk_get_clock(gpu
->grp_clks
,
819 gpu
->nr_clocks
, "rbbmtimer");
824 int msm_gpu_init(struct drm_device
*drm
, struct platform_device
*pdev
,
825 struct msm_gpu
*gpu
, const struct msm_gpu_funcs
*funcs
,
826 const char *name
, struct msm_gpu_config
*config
)
828 int i
, ret
, nr_rings
= config
->nr_rings
;
830 uint64_t memptrs_iova
;
832 if (WARN_ON(gpu
->num_perfcntrs
> ARRAY_SIZE(gpu
->last_cntrs
)))
833 gpu
->num_perfcntrs
= ARRAY_SIZE(gpu
->last_cntrs
);
839 INIT_LIST_HEAD(&gpu
->active_list
);
840 INIT_WORK(&gpu
->retire_work
, retire_worker
);
841 INIT_WORK(&gpu
->recover_work
, recover_worker
);
844 timer_setup(&gpu
->hangcheck_timer
, hangcheck_handler
, 0);
846 spin_lock_init(&gpu
->perf_lock
);
850 gpu
->mmio
= msm_ioremap(pdev
, config
->ioname
, name
);
851 if (IS_ERR(gpu
->mmio
)) {
852 ret
= PTR_ERR(gpu
->mmio
);
857 gpu
->irq
= platform_get_irq(pdev
, 0);
860 DRM_DEV_ERROR(drm
->dev
, "failed to get irq: %d\n", ret
);
864 ret
= devm_request_irq(&pdev
->dev
, gpu
->irq
, irq_handler
,
865 IRQF_TRIGGER_HIGH
, gpu
->name
, gpu
);
867 DRM_DEV_ERROR(drm
->dev
, "failed to request IRQ%u: %d\n", gpu
->irq
, ret
);
871 ret
= get_clocks(pdev
, gpu
);
875 gpu
->ebi1_clk
= msm_clk_get(pdev
, "bus");
876 DBG("ebi1_clk: %p", gpu
->ebi1_clk
);
877 if (IS_ERR(gpu
->ebi1_clk
))
878 gpu
->ebi1_clk
= NULL
;
880 /* Acquire regulators: */
881 gpu
->gpu_reg
= devm_regulator_get(&pdev
->dev
, "vdd");
882 DBG("gpu_reg: %p", gpu
->gpu_reg
);
883 if (IS_ERR(gpu
->gpu_reg
))
886 gpu
->gpu_cx
= devm_regulator_get(&pdev
->dev
, "vddcx");
887 DBG("gpu_cx: %p", gpu
->gpu_cx
);
888 if (IS_ERR(gpu
->gpu_cx
))
892 platform_set_drvdata(pdev
, gpu
);
894 msm_devfreq_init(gpu
);
897 gpu
->aspace
= gpu
->funcs
->create_address_space(gpu
, pdev
);
899 if (gpu
->aspace
== NULL
)
900 DRM_DEV_INFO(drm
->dev
, "%s: no IOMMU, fallback to VRAM carveout!\n", name
);
901 else if (IS_ERR(gpu
->aspace
)) {
902 ret
= PTR_ERR(gpu
->aspace
);
906 memptrs
= msm_gem_kernel_new(drm
,
907 sizeof(struct msm_rbmemptrs
) * nr_rings
,
908 MSM_BO_UNCACHED
, gpu
->aspace
, &gpu
->memptrs_bo
,
911 if (IS_ERR(memptrs
)) {
912 ret
= PTR_ERR(memptrs
);
913 DRM_DEV_ERROR(drm
->dev
, "could not allocate memptrs: %d\n", ret
);
917 msm_gem_object_set_name(gpu
->memptrs_bo
, "memptrs");
919 if (nr_rings
> ARRAY_SIZE(gpu
->rb
)) {
920 DRM_DEV_INFO_ONCE(drm
->dev
, "Only creating %zu ringbuffers\n",
921 ARRAY_SIZE(gpu
->rb
));
922 nr_rings
= ARRAY_SIZE(gpu
->rb
);
925 /* Create ringbuffer(s): */
926 for (i
= 0; i
< nr_rings
; i
++) {
927 gpu
->rb
[i
] = msm_ringbuffer_new(gpu
, i
, memptrs
, memptrs_iova
);
929 if (IS_ERR(gpu
->rb
[i
])) {
930 ret
= PTR_ERR(gpu
->rb
[i
]);
931 DRM_DEV_ERROR(drm
->dev
,
932 "could not create ringbuffer %d: %d\n", i
, ret
);
936 memptrs
+= sizeof(struct msm_rbmemptrs
);
937 memptrs_iova
+= sizeof(struct msm_rbmemptrs
);
940 gpu
->nr_rings
= nr_rings
;
945 for (i
= 0; i
< ARRAY_SIZE(gpu
->rb
); i
++) {
946 msm_ringbuffer_destroy(gpu
->rb
[i
]);
950 msm_gem_kernel_put(gpu
->memptrs_bo
, gpu
->aspace
, false);
952 platform_set_drvdata(pdev
, NULL
);
956 void msm_gpu_cleanup(struct msm_gpu
*gpu
)
960 DBG("%s", gpu
->name
);
962 WARN_ON(!list_empty(&gpu
->active_list
));
964 for (i
= 0; i
< ARRAY_SIZE(gpu
->rb
); i
++) {
965 msm_ringbuffer_destroy(gpu
->rb
[i
]);
969 msm_gem_kernel_put(gpu
->memptrs_bo
, gpu
->aspace
, false);
971 if (!IS_ERR_OR_NULL(gpu
->aspace
)) {
972 gpu
->aspace
->mmu
->funcs
->detach(gpu
->aspace
->mmu
);
973 msm_gem_address_space_put(gpu
->aspace
);