2 * Copyright (C) 2015 Etnaviv Project
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 as published by
6 * the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program. If not, see <http://www.gnu.org/licenses/>.
17 #include <linux/component.h>
18 #include <linux/dma-fence.h>
19 #include <linux/moduleparam.h>
20 #include <linux/of_device.h>
21 #include <linux/thermal.h>
23 #include "etnaviv_cmdbuf.h"
24 #include "etnaviv_dump.h"
25 #include "etnaviv_gpu.h"
26 #include "etnaviv_gem.h"
27 #include "etnaviv_mmu.h"
28 #include "etnaviv_perfmon.h"
29 #include "common.xml.h"
30 #include "state.xml.h"
31 #include "state_hi.xml.h"
32 #include "cmdstream.xml.h"
34 static const struct platform_device_id gpu_ids
[] = {
35 { .name
= "etnaviv-gpu,2d" },
39 static bool etnaviv_dump_core
= true;
40 module_param_named(dump_core
, etnaviv_dump_core
, bool, 0600);
46 int etnaviv_gpu_get_param(struct etnaviv_gpu
*gpu
, u32 param
, u64
*value
)
49 case ETNAVIV_PARAM_GPU_MODEL
:
50 *value
= gpu
->identity
.model
;
53 case ETNAVIV_PARAM_GPU_REVISION
:
54 *value
= gpu
->identity
.revision
;
57 case ETNAVIV_PARAM_GPU_FEATURES_0
:
58 *value
= gpu
->identity
.features
;
61 case ETNAVIV_PARAM_GPU_FEATURES_1
:
62 *value
= gpu
->identity
.minor_features0
;
65 case ETNAVIV_PARAM_GPU_FEATURES_2
:
66 *value
= gpu
->identity
.minor_features1
;
69 case ETNAVIV_PARAM_GPU_FEATURES_3
:
70 *value
= gpu
->identity
.minor_features2
;
73 case ETNAVIV_PARAM_GPU_FEATURES_4
:
74 *value
= gpu
->identity
.minor_features3
;
77 case ETNAVIV_PARAM_GPU_FEATURES_5
:
78 *value
= gpu
->identity
.minor_features4
;
81 case ETNAVIV_PARAM_GPU_FEATURES_6
:
82 *value
= gpu
->identity
.minor_features5
;
85 case ETNAVIV_PARAM_GPU_STREAM_COUNT
:
86 *value
= gpu
->identity
.stream_count
;
89 case ETNAVIV_PARAM_GPU_REGISTER_MAX
:
90 *value
= gpu
->identity
.register_max
;
93 case ETNAVIV_PARAM_GPU_THREAD_COUNT
:
94 *value
= gpu
->identity
.thread_count
;
97 case ETNAVIV_PARAM_GPU_VERTEX_CACHE_SIZE
:
98 *value
= gpu
->identity
.vertex_cache_size
;
101 case ETNAVIV_PARAM_GPU_SHADER_CORE_COUNT
:
102 *value
= gpu
->identity
.shader_core_count
;
105 case ETNAVIV_PARAM_GPU_PIXEL_PIPES
:
106 *value
= gpu
->identity
.pixel_pipes
;
109 case ETNAVIV_PARAM_GPU_VERTEX_OUTPUT_BUFFER_SIZE
:
110 *value
= gpu
->identity
.vertex_output_buffer_size
;
113 case ETNAVIV_PARAM_GPU_BUFFER_SIZE
:
114 *value
= gpu
->identity
.buffer_size
;
117 case ETNAVIV_PARAM_GPU_INSTRUCTION_COUNT
:
118 *value
= gpu
->identity
.instruction_count
;
121 case ETNAVIV_PARAM_GPU_NUM_CONSTANTS
:
122 *value
= gpu
->identity
.num_constants
;
125 case ETNAVIV_PARAM_GPU_NUM_VARYINGS
:
126 *value
= gpu
->identity
.varyings_count
;
130 DBG("%s: invalid param: %u", dev_name(gpu
->dev
), param
);
138 #define etnaviv_is_model_rev(gpu, mod, rev) \
139 ((gpu)->identity.model == chipModel_##mod && \
140 (gpu)->identity.revision == rev)
141 #define etnaviv_field(val, field) \
142 (((val) & field##__MASK) >> field##__SHIFT)
144 static void etnaviv_hw_specs(struct etnaviv_gpu
*gpu
)
146 if (gpu
->identity
.minor_features0
&
147 chipMinorFeatures0_MORE_MINOR_FEATURES
) {
149 unsigned int streams
;
151 specs
[0] = gpu_read(gpu
, VIVS_HI_CHIP_SPECS
);
152 specs
[1] = gpu_read(gpu
, VIVS_HI_CHIP_SPECS_2
);
153 specs
[2] = gpu_read(gpu
, VIVS_HI_CHIP_SPECS_3
);
154 specs
[3] = gpu_read(gpu
, VIVS_HI_CHIP_SPECS_4
);
156 gpu
->identity
.stream_count
= etnaviv_field(specs
[0],
157 VIVS_HI_CHIP_SPECS_STREAM_COUNT
);
158 gpu
->identity
.register_max
= etnaviv_field(specs
[0],
159 VIVS_HI_CHIP_SPECS_REGISTER_MAX
);
160 gpu
->identity
.thread_count
= etnaviv_field(specs
[0],
161 VIVS_HI_CHIP_SPECS_THREAD_COUNT
);
162 gpu
->identity
.vertex_cache_size
= etnaviv_field(specs
[0],
163 VIVS_HI_CHIP_SPECS_VERTEX_CACHE_SIZE
);
164 gpu
->identity
.shader_core_count
= etnaviv_field(specs
[0],
165 VIVS_HI_CHIP_SPECS_SHADER_CORE_COUNT
);
166 gpu
->identity
.pixel_pipes
= etnaviv_field(specs
[0],
167 VIVS_HI_CHIP_SPECS_PIXEL_PIPES
);
168 gpu
->identity
.vertex_output_buffer_size
=
169 etnaviv_field(specs
[0],
170 VIVS_HI_CHIP_SPECS_VERTEX_OUTPUT_BUFFER_SIZE
);
172 gpu
->identity
.buffer_size
= etnaviv_field(specs
[1],
173 VIVS_HI_CHIP_SPECS_2_BUFFER_SIZE
);
174 gpu
->identity
.instruction_count
= etnaviv_field(specs
[1],
175 VIVS_HI_CHIP_SPECS_2_INSTRUCTION_COUNT
);
176 gpu
->identity
.num_constants
= etnaviv_field(specs
[1],
177 VIVS_HI_CHIP_SPECS_2_NUM_CONSTANTS
);
179 gpu
->identity
.varyings_count
= etnaviv_field(specs
[2],
180 VIVS_HI_CHIP_SPECS_3_VARYINGS_COUNT
);
182 /* This overrides the value from older register if non-zero */
183 streams
= etnaviv_field(specs
[3],
184 VIVS_HI_CHIP_SPECS_4_STREAM_COUNT
);
186 gpu
->identity
.stream_count
= streams
;
189 /* Fill in the stream count if not specified */
190 if (gpu
->identity
.stream_count
== 0) {
191 if (gpu
->identity
.model
>= 0x1000)
192 gpu
->identity
.stream_count
= 4;
194 gpu
->identity
.stream_count
= 1;
197 /* Convert the register max value */
198 if (gpu
->identity
.register_max
)
199 gpu
->identity
.register_max
= 1 << gpu
->identity
.register_max
;
200 else if (gpu
->identity
.model
== chipModel_GC400
)
201 gpu
->identity
.register_max
= 32;
203 gpu
->identity
.register_max
= 64;
205 /* Convert thread count */
206 if (gpu
->identity
.thread_count
)
207 gpu
->identity
.thread_count
= 1 << gpu
->identity
.thread_count
;
208 else if (gpu
->identity
.model
== chipModel_GC400
)
209 gpu
->identity
.thread_count
= 64;
210 else if (gpu
->identity
.model
== chipModel_GC500
||
211 gpu
->identity
.model
== chipModel_GC530
)
212 gpu
->identity
.thread_count
= 128;
214 gpu
->identity
.thread_count
= 256;
216 if (gpu
->identity
.vertex_cache_size
== 0)
217 gpu
->identity
.vertex_cache_size
= 8;
219 if (gpu
->identity
.shader_core_count
== 0) {
220 if (gpu
->identity
.model
>= 0x1000)
221 gpu
->identity
.shader_core_count
= 2;
223 gpu
->identity
.shader_core_count
= 1;
226 if (gpu
->identity
.pixel_pipes
== 0)
227 gpu
->identity
.pixel_pipes
= 1;
229 /* Convert virtex buffer size */
230 if (gpu
->identity
.vertex_output_buffer_size
) {
231 gpu
->identity
.vertex_output_buffer_size
=
232 1 << gpu
->identity
.vertex_output_buffer_size
;
233 } else if (gpu
->identity
.model
== chipModel_GC400
) {
234 if (gpu
->identity
.revision
< 0x4000)
235 gpu
->identity
.vertex_output_buffer_size
= 512;
236 else if (gpu
->identity
.revision
< 0x4200)
237 gpu
->identity
.vertex_output_buffer_size
= 256;
239 gpu
->identity
.vertex_output_buffer_size
= 128;
241 gpu
->identity
.vertex_output_buffer_size
= 512;
244 switch (gpu
->identity
.instruction_count
) {
246 if (etnaviv_is_model_rev(gpu
, GC2000
, 0x5108) ||
247 gpu
->identity
.model
== chipModel_GC880
)
248 gpu
->identity
.instruction_count
= 512;
250 gpu
->identity
.instruction_count
= 256;
254 gpu
->identity
.instruction_count
= 1024;
258 gpu
->identity
.instruction_count
= 2048;
262 gpu
->identity
.instruction_count
= 256;
266 if (gpu
->identity
.num_constants
== 0)
267 gpu
->identity
.num_constants
= 168;
269 if (gpu
->identity
.varyings_count
== 0) {
270 if (gpu
->identity
.minor_features1
& chipMinorFeatures1_HALTI0
)
271 gpu
->identity
.varyings_count
= 12;
273 gpu
->identity
.varyings_count
= 8;
277 * For some cores, two varyings are consumed for position, so the
278 * maximum varying count needs to be reduced by one.
280 if (etnaviv_is_model_rev(gpu
, GC5000
, 0x5434) ||
281 etnaviv_is_model_rev(gpu
, GC4000
, 0x5222) ||
282 etnaviv_is_model_rev(gpu
, GC4000
, 0x5245) ||
283 etnaviv_is_model_rev(gpu
, GC4000
, 0x5208) ||
284 etnaviv_is_model_rev(gpu
, GC3000
, 0x5435) ||
285 etnaviv_is_model_rev(gpu
, GC2200
, 0x5244) ||
286 etnaviv_is_model_rev(gpu
, GC2100
, 0x5108) ||
287 etnaviv_is_model_rev(gpu
, GC2000
, 0x5108) ||
288 etnaviv_is_model_rev(gpu
, GC1500
, 0x5246) ||
289 etnaviv_is_model_rev(gpu
, GC880
, 0x5107) ||
290 etnaviv_is_model_rev(gpu
, GC880
, 0x5106))
291 gpu
->identity
.varyings_count
-= 1;
294 static void etnaviv_hw_identify(struct etnaviv_gpu
*gpu
)
298 chipIdentity
= gpu_read(gpu
, VIVS_HI_CHIP_IDENTITY
);
300 /* Special case for older graphic cores. */
301 if (etnaviv_field(chipIdentity
, VIVS_HI_CHIP_IDENTITY_FAMILY
) == 0x01) {
302 gpu
->identity
.model
= chipModel_GC500
;
303 gpu
->identity
.revision
= etnaviv_field(chipIdentity
,
304 VIVS_HI_CHIP_IDENTITY_REVISION
);
307 gpu
->identity
.model
= gpu_read(gpu
, VIVS_HI_CHIP_MODEL
);
308 gpu
->identity
.revision
= gpu_read(gpu
, VIVS_HI_CHIP_REV
);
311 * !!!! HACK ALERT !!!!
312 * Because people change device IDs without letting software
313 * know about it - here is the hack to make it all look the
314 * same. Only for GC400 family.
316 if ((gpu
->identity
.model
& 0xff00) == 0x0400 &&
317 gpu
->identity
.model
!= chipModel_GC420
) {
318 gpu
->identity
.model
= gpu
->identity
.model
& 0x0400;
321 /* Another special case */
322 if (etnaviv_is_model_rev(gpu
, GC300
, 0x2201)) {
323 u32 chipDate
= gpu_read(gpu
, VIVS_HI_CHIP_DATE
);
324 u32 chipTime
= gpu_read(gpu
, VIVS_HI_CHIP_TIME
);
326 if (chipDate
== 0x20080814 && chipTime
== 0x12051100) {
328 * This IP has an ECO; put the correct
331 gpu
->identity
.revision
= 0x1051;
336 * NXP likes to call the GPU on the i.MX6QP GC2000+, but in
337 * reality it's just a re-branded GC3000. We can identify this
338 * core by the upper half of the revision register being all 1.
339 * Fix model/rev here, so all other places can refer to this
340 * core by its real identity.
342 if (etnaviv_is_model_rev(gpu
, GC2000
, 0xffff5450)) {
343 gpu
->identity
.model
= chipModel_GC3000
;
344 gpu
->identity
.revision
&= 0xffff;
348 dev_info(gpu
->dev
, "model: GC%x, revision: %x\n",
349 gpu
->identity
.model
, gpu
->identity
.revision
);
351 gpu
->identity
.features
= gpu_read(gpu
, VIVS_HI_CHIP_FEATURE
);
353 /* Disable fast clear on GC700. */
354 if (gpu
->identity
.model
== chipModel_GC700
)
355 gpu
->identity
.features
&= ~chipFeatures_FAST_CLEAR
;
357 if ((gpu
->identity
.model
== chipModel_GC500
&&
358 gpu
->identity
.revision
< 2) ||
359 (gpu
->identity
.model
== chipModel_GC300
&&
360 gpu
->identity
.revision
< 0x2000)) {
363 * GC500 rev 1.x and GC300 rev < 2.0 doesn't have these
366 gpu
->identity
.minor_features0
= 0;
367 gpu
->identity
.minor_features1
= 0;
368 gpu
->identity
.minor_features2
= 0;
369 gpu
->identity
.minor_features3
= 0;
370 gpu
->identity
.minor_features4
= 0;
371 gpu
->identity
.minor_features5
= 0;
373 gpu
->identity
.minor_features0
=
374 gpu_read(gpu
, VIVS_HI_CHIP_MINOR_FEATURE_0
);
376 if (gpu
->identity
.minor_features0
&
377 chipMinorFeatures0_MORE_MINOR_FEATURES
) {
378 gpu
->identity
.minor_features1
=
379 gpu_read(gpu
, VIVS_HI_CHIP_MINOR_FEATURE_1
);
380 gpu
->identity
.minor_features2
=
381 gpu_read(gpu
, VIVS_HI_CHIP_MINOR_FEATURE_2
);
382 gpu
->identity
.minor_features3
=
383 gpu_read(gpu
, VIVS_HI_CHIP_MINOR_FEATURE_3
);
384 gpu
->identity
.minor_features4
=
385 gpu_read(gpu
, VIVS_HI_CHIP_MINOR_FEATURE_4
);
386 gpu
->identity
.minor_features5
=
387 gpu_read(gpu
, VIVS_HI_CHIP_MINOR_FEATURE_5
);
390 /* GC600 idle register reports zero bits where modules aren't present */
391 if (gpu
->identity
.model
== chipModel_GC600
) {
392 gpu
->idle_mask
= VIVS_HI_IDLE_STATE_TX
|
393 VIVS_HI_IDLE_STATE_RA
|
394 VIVS_HI_IDLE_STATE_SE
|
395 VIVS_HI_IDLE_STATE_PA
|
396 VIVS_HI_IDLE_STATE_SH
|
397 VIVS_HI_IDLE_STATE_PE
|
398 VIVS_HI_IDLE_STATE_DE
|
399 VIVS_HI_IDLE_STATE_FE
;
401 gpu
->idle_mask
= ~VIVS_HI_IDLE_STATE_AXI_LP
;
404 etnaviv_hw_specs(gpu
);
407 static void etnaviv_gpu_load_clock(struct etnaviv_gpu
*gpu
, u32 clock
)
409 gpu_write(gpu
, VIVS_HI_CLOCK_CONTROL
, clock
|
410 VIVS_HI_CLOCK_CONTROL_FSCALE_CMD_LOAD
);
411 gpu_write(gpu
, VIVS_HI_CLOCK_CONTROL
, clock
);
414 static void etnaviv_gpu_update_clock(struct etnaviv_gpu
*gpu
)
416 if (gpu
->identity
.minor_features2
&
417 chipMinorFeatures2_DYNAMIC_FREQUENCY_SCALING
) {
418 clk_set_rate(gpu
->clk_core
,
419 gpu
->base_rate_core
>> gpu
->freq_scale
);
420 clk_set_rate(gpu
->clk_shader
,
421 gpu
->base_rate_shader
>> gpu
->freq_scale
);
423 unsigned int fscale
= 1 << (6 - gpu
->freq_scale
);
424 u32 clock
= gpu_read(gpu
, VIVS_HI_CLOCK_CONTROL
);
426 clock
&= ~VIVS_HI_CLOCK_CONTROL_FSCALE_VAL__MASK
;
427 clock
|= VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(fscale
);
428 etnaviv_gpu_load_clock(gpu
, clock
);
432 static int etnaviv_hw_reset(struct etnaviv_gpu
*gpu
)
435 unsigned long timeout
;
438 /* We hope that the GPU resets in under one second */
439 timeout
= jiffies
+ msecs_to_jiffies(1000);
441 while (time_is_after_jiffies(timeout
)) {
443 unsigned int fscale
= 1 << (6 - gpu
->freq_scale
);
444 control
= VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(fscale
);
445 etnaviv_gpu_load_clock(gpu
, control
);
447 /* isolate the GPU. */
448 control
|= VIVS_HI_CLOCK_CONTROL_ISOLATE_GPU
;
449 gpu_write(gpu
, VIVS_HI_CLOCK_CONTROL
, control
);
451 /* set soft reset. */
452 control
|= VIVS_HI_CLOCK_CONTROL_SOFT_RESET
;
453 gpu_write(gpu
, VIVS_HI_CLOCK_CONTROL
, control
);
455 /* wait for reset. */
456 usleep_range(10, 20);
458 /* reset soft reset bit. */
459 control
&= ~VIVS_HI_CLOCK_CONTROL_SOFT_RESET
;
460 gpu_write(gpu
, VIVS_HI_CLOCK_CONTROL
, control
);
462 /* reset GPU isolation. */
463 control
&= ~VIVS_HI_CLOCK_CONTROL_ISOLATE_GPU
;
464 gpu_write(gpu
, VIVS_HI_CLOCK_CONTROL
, control
);
466 /* read idle register. */
467 idle
= gpu_read(gpu
, VIVS_HI_IDLE_STATE
);
469 /* try reseting again if FE it not idle */
470 if ((idle
& VIVS_HI_IDLE_STATE_FE
) == 0) {
471 dev_dbg(gpu
->dev
, "FE is not idle\n");
475 /* read reset register. */
476 control
= gpu_read(gpu
, VIVS_HI_CLOCK_CONTROL
);
478 /* is the GPU idle? */
479 if (((control
& VIVS_HI_CLOCK_CONTROL_IDLE_3D
) == 0) ||
480 ((control
& VIVS_HI_CLOCK_CONTROL_IDLE_2D
) == 0)) {
481 dev_dbg(gpu
->dev
, "GPU is not idle\n");
485 /* disable debug registers, as they are not normally needed */
486 control
|= VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS
;
487 gpu_write(gpu
, VIVS_HI_CLOCK_CONTROL
, control
);
494 idle
= gpu_read(gpu
, VIVS_HI_IDLE_STATE
);
495 control
= gpu_read(gpu
, VIVS_HI_CLOCK_CONTROL
);
497 dev_err(gpu
->dev
, "GPU failed to reset: FE %sidle, 3D %sidle, 2D %sidle\n",
498 idle
& VIVS_HI_IDLE_STATE_FE
? "" : "not ",
499 control
& VIVS_HI_CLOCK_CONTROL_IDLE_3D
? "" : "not ",
500 control
& VIVS_HI_CLOCK_CONTROL_IDLE_2D
? "" : "not ");
505 /* We rely on the GPU running, so program the clock */
506 etnaviv_gpu_update_clock(gpu
);
511 static void etnaviv_gpu_enable_mlcg(struct etnaviv_gpu
*gpu
)
515 /* enable clock gating */
516 ppc
= gpu_read(gpu
, VIVS_PM_POWER_CONTROLS
);
517 ppc
|= VIVS_PM_POWER_CONTROLS_ENABLE_MODULE_CLOCK_GATING
;
519 /* Disable stall module clock gating for 4.3.0.1 and 4.3.0.2 revs */
520 if (gpu
->identity
.revision
== 0x4301 ||
521 gpu
->identity
.revision
== 0x4302)
522 ppc
|= VIVS_PM_POWER_CONTROLS_DISABLE_STALL_MODULE_CLOCK_GATING
;
524 gpu_write(gpu
, VIVS_PM_POWER_CONTROLS
, ppc
);
526 pmc
= gpu_read(gpu
, VIVS_PM_MODULE_CONTROLS
);
528 /* Disable PA clock gating for GC400+ without bugfix except for GC420 */
529 if (gpu
->identity
.model
>= chipModel_GC400
&&
530 gpu
->identity
.model
!= chipModel_GC420
&&
531 !(gpu
->identity
.minor_features3
& chipMinorFeatures3_BUG_FIXES12
))
532 pmc
|= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_PA
;
535 * Disable PE clock gating on revs < 5.0.0.0 when HZ is
536 * present without a bug fix.
538 if (gpu
->identity
.revision
< 0x5000 &&
539 gpu
->identity
.minor_features0
& chipMinorFeatures0_HZ
&&
540 !(gpu
->identity
.minor_features1
&
541 chipMinorFeatures1_DISABLE_PE_GATING
))
542 pmc
|= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_PE
;
544 if (gpu
->identity
.revision
< 0x5422)
545 pmc
|= BIT(15); /* Unknown bit */
547 /* Disable TX clock gating on affected core revisions. */
548 if (etnaviv_is_model_rev(gpu
, GC4000
, 0x5222) ||
549 etnaviv_is_model_rev(gpu
, GC2000
, 0x5108))
550 pmc
|= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_TX
;
552 pmc
|= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_HZ
;
553 pmc
|= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_EZ
;
555 gpu_write(gpu
, VIVS_PM_MODULE_CONTROLS
, pmc
);
558 void etnaviv_gpu_start_fe(struct etnaviv_gpu
*gpu
, u32 address
, u16 prefetch
)
560 gpu_write(gpu
, VIVS_FE_COMMAND_ADDRESS
, address
);
561 gpu_write(gpu
, VIVS_FE_COMMAND_CONTROL
,
562 VIVS_FE_COMMAND_CONTROL_ENABLE
|
563 VIVS_FE_COMMAND_CONTROL_PREFETCH(prefetch
));
566 static void etnaviv_gpu_setup_pulse_eater(struct etnaviv_gpu
*gpu
)
569 * Base value for VIVS_PM_PULSE_EATER register on models where it
570 * cannot be read, extracted from vivante kernel driver.
572 u32 pulse_eater
= 0x01590880;
574 if (etnaviv_is_model_rev(gpu
, GC4000
, 0x5208) ||
575 etnaviv_is_model_rev(gpu
, GC4000
, 0x5222)) {
576 pulse_eater
|= BIT(23);
580 if (etnaviv_is_model_rev(gpu
, GC1000
, 0x5039) ||
581 etnaviv_is_model_rev(gpu
, GC1000
, 0x5040)) {
582 pulse_eater
&= ~BIT(16);
583 pulse_eater
|= BIT(17);
586 if ((gpu
->identity
.revision
> 0x5420) &&
587 (gpu
->identity
.features
& chipFeatures_PIPE_3D
))
589 /* Performance fix: disable internal DFS */
590 pulse_eater
= gpu_read(gpu
, VIVS_PM_PULSE_EATER
);
591 pulse_eater
|= BIT(18);
594 gpu_write(gpu
, VIVS_PM_PULSE_EATER
, pulse_eater
);
597 static void etnaviv_gpu_hw_init(struct etnaviv_gpu
*gpu
)
601 if ((etnaviv_is_model_rev(gpu
, GC320
, 0x5007) ||
602 etnaviv_is_model_rev(gpu
, GC320
, 0x5220)) &&
603 gpu_read(gpu
, VIVS_HI_CHIP_TIME
) != 0x2062400) {
606 mc_memory_debug
= gpu_read(gpu
, VIVS_MC_DEBUG_MEMORY
) & ~0xff;
608 if (gpu
->identity
.revision
== 0x5007)
609 mc_memory_debug
|= 0x0c;
611 mc_memory_debug
|= 0x08;
613 gpu_write(gpu
, VIVS_MC_DEBUG_MEMORY
, mc_memory_debug
);
616 /* enable module-level clock gating */
617 etnaviv_gpu_enable_mlcg(gpu
);
620 * Update GPU AXI cache atttribute to "cacheable, no allocate".
621 * This is necessary to prevent the iMX6 SoC locking up.
623 gpu_write(gpu
, VIVS_HI_AXI_CONFIG
,
624 VIVS_HI_AXI_CONFIG_AWCACHE(2) |
625 VIVS_HI_AXI_CONFIG_ARCACHE(2));
627 /* GC2000 rev 5108 needs a special bus config */
628 if (etnaviv_is_model_rev(gpu
, GC2000
, 0x5108)) {
629 u32 bus_config
= gpu_read(gpu
, VIVS_MC_BUS_CONFIG
);
630 bus_config
&= ~(VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG__MASK
|
631 VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG__MASK
);
632 bus_config
|= VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG(1) |
633 VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG(0);
634 gpu_write(gpu
, VIVS_MC_BUS_CONFIG
, bus_config
);
637 /* setup the pulse eater */
638 etnaviv_gpu_setup_pulse_eater(gpu
);
641 etnaviv_iommu_restore(gpu
);
643 /* Start command processor */
644 prefetch
= etnaviv_buffer_init(gpu
);
646 gpu_write(gpu
, VIVS_HI_INTR_ENBL
, ~0U);
647 etnaviv_gpu_start_fe(gpu
, etnaviv_cmdbuf_get_va(gpu
->buffer
),
651 int etnaviv_gpu_init(struct etnaviv_gpu
*gpu
)
655 ret
= pm_runtime_get_sync(gpu
->dev
);
657 dev_err(gpu
->dev
, "Failed to enable GPU power domain\n");
661 etnaviv_hw_identify(gpu
);
663 if (gpu
->identity
.model
== 0) {
664 dev_err(gpu
->dev
, "Unknown GPU model\n");
669 /* Exclude VG cores with FE2.0 */
670 if (gpu
->identity
.features
& chipFeatures_PIPE_VG
&&
671 gpu
->identity
.features
& chipFeatures_FE20
) {
672 dev_info(gpu
->dev
, "Ignoring GPU with VG and FE2.0\n");
678 * Set the GPU linear window to be at the end of the DMA window, where
679 * the CMA area is likely to reside. This ensures that we are able to
680 * map the command buffers while having the linear window overlap as
681 * much RAM as possible, so we can optimize mappings for other buffers.
683 * For 3D cores only do this if MC2.0 is present, as with MC1.0 it leads
684 * to different views of the memory on the individual engines.
686 if (!(gpu
->identity
.features
& chipFeatures_PIPE_3D
) ||
687 (gpu
->identity
.minor_features0
& chipMinorFeatures0_MC20
)) {
688 u32 dma_mask
= (u32
)dma_get_required_mask(gpu
->dev
);
689 if (dma_mask
< PHYS_OFFSET
+ SZ_2G
)
690 gpu
->memory_base
= PHYS_OFFSET
;
692 gpu
->memory_base
= dma_mask
- SZ_2G
+ 1;
693 } else if (PHYS_OFFSET
>= SZ_2G
) {
694 dev_info(gpu
->dev
, "Need to move linear window on MC1.0, disabling TS\n");
695 gpu
->memory_base
= PHYS_OFFSET
;
696 gpu
->identity
.features
&= ~chipFeatures_FAST_CLEAR
;
699 ret
= etnaviv_hw_reset(gpu
);
701 dev_err(gpu
->dev
, "GPU reset failed\n");
705 gpu
->mmu
= etnaviv_iommu_new(gpu
);
706 if (IS_ERR(gpu
->mmu
)) {
707 dev_err(gpu
->dev
, "Failed to instantiate GPU IOMMU\n");
708 ret
= PTR_ERR(gpu
->mmu
);
712 gpu
->cmdbuf_suballoc
= etnaviv_cmdbuf_suballoc_new(gpu
);
713 if (IS_ERR(gpu
->cmdbuf_suballoc
)) {
714 dev_err(gpu
->dev
, "Failed to create cmdbuf suballocator\n");
715 ret
= PTR_ERR(gpu
->cmdbuf_suballoc
);
720 gpu
->buffer
= etnaviv_cmdbuf_new(gpu
->cmdbuf_suballoc
, PAGE_SIZE
, 0, 0);
723 dev_err(gpu
->dev
, "could not create command buffer\n");
727 if (gpu
->mmu
->version
== ETNAVIV_IOMMU_V1
&&
728 etnaviv_cmdbuf_get_va(gpu
->buffer
) > 0x80000000) {
731 "command buffer outside valid memory window\n");
735 /* Setup event management */
736 spin_lock_init(&gpu
->event_spinlock
);
737 init_completion(&gpu
->event_free
);
738 bitmap_zero(gpu
->event_bitmap
, ETNA_NR_EVENTS
);
739 for (i
= 0; i
< ARRAY_SIZE(gpu
->event
); i
++)
740 complete(&gpu
->event_free
);
742 /* Now program the hardware */
743 mutex_lock(&gpu
->lock
);
744 etnaviv_gpu_hw_init(gpu
);
745 gpu
->exec_state
= -1;
746 mutex_unlock(&gpu
->lock
);
748 pm_runtime_mark_last_busy(gpu
->dev
);
749 pm_runtime_put_autosuspend(gpu
->dev
);
754 etnaviv_cmdbuf_free(gpu
->buffer
);
757 etnaviv_iommu_destroy(gpu
->mmu
);
760 pm_runtime_mark_last_busy(gpu
->dev
);
761 pm_runtime_put_autosuspend(gpu
->dev
);
766 #ifdef CONFIG_DEBUG_FS
772 static void verify_dma(struct etnaviv_gpu
*gpu
, struct dma_debug
*debug
)
776 debug
->address
[0] = gpu_read(gpu
, VIVS_FE_DMA_ADDRESS
);
777 debug
->state
[0] = gpu_read(gpu
, VIVS_FE_DMA_DEBUG_STATE
);
779 for (i
= 0; i
< 500; i
++) {
780 debug
->address
[1] = gpu_read(gpu
, VIVS_FE_DMA_ADDRESS
);
781 debug
->state
[1] = gpu_read(gpu
, VIVS_FE_DMA_DEBUG_STATE
);
783 if (debug
->address
[0] != debug
->address
[1])
786 if (debug
->state
[0] != debug
->state
[1])
791 int etnaviv_gpu_debugfs(struct etnaviv_gpu
*gpu
, struct seq_file
*m
)
793 struct dma_debug debug
;
794 u32 dma_lo
, dma_hi
, axi
, idle
;
797 seq_printf(m
, "%s Status:\n", dev_name(gpu
->dev
));
799 ret
= pm_runtime_get_sync(gpu
->dev
);
803 dma_lo
= gpu_read(gpu
, VIVS_FE_DMA_LOW
);
804 dma_hi
= gpu_read(gpu
, VIVS_FE_DMA_HIGH
);
805 axi
= gpu_read(gpu
, VIVS_HI_AXI_STATUS
);
806 idle
= gpu_read(gpu
, VIVS_HI_IDLE_STATE
);
808 verify_dma(gpu
, &debug
);
810 seq_puts(m
, "\tfeatures\n");
811 seq_printf(m
, "\t minor_features0: 0x%08x\n",
812 gpu
->identity
.minor_features0
);
813 seq_printf(m
, "\t minor_features1: 0x%08x\n",
814 gpu
->identity
.minor_features1
);
815 seq_printf(m
, "\t minor_features2: 0x%08x\n",
816 gpu
->identity
.minor_features2
);
817 seq_printf(m
, "\t minor_features3: 0x%08x\n",
818 gpu
->identity
.minor_features3
);
819 seq_printf(m
, "\t minor_features4: 0x%08x\n",
820 gpu
->identity
.minor_features4
);
821 seq_printf(m
, "\t minor_features5: 0x%08x\n",
822 gpu
->identity
.minor_features5
);
824 seq_puts(m
, "\tspecs\n");
825 seq_printf(m
, "\t stream_count: %d\n",
826 gpu
->identity
.stream_count
);
827 seq_printf(m
, "\t register_max: %d\n",
828 gpu
->identity
.register_max
);
829 seq_printf(m
, "\t thread_count: %d\n",
830 gpu
->identity
.thread_count
);
831 seq_printf(m
, "\t vertex_cache_size: %d\n",
832 gpu
->identity
.vertex_cache_size
);
833 seq_printf(m
, "\t shader_core_count: %d\n",
834 gpu
->identity
.shader_core_count
);
835 seq_printf(m
, "\t pixel_pipes: %d\n",
836 gpu
->identity
.pixel_pipes
);
837 seq_printf(m
, "\t vertex_output_buffer_size: %d\n",
838 gpu
->identity
.vertex_output_buffer_size
);
839 seq_printf(m
, "\t buffer_size: %d\n",
840 gpu
->identity
.buffer_size
);
841 seq_printf(m
, "\t instruction_count: %d\n",
842 gpu
->identity
.instruction_count
);
843 seq_printf(m
, "\t num_constants: %d\n",
844 gpu
->identity
.num_constants
);
845 seq_printf(m
, "\t varyings_count: %d\n",
846 gpu
->identity
.varyings_count
);
848 seq_printf(m
, "\taxi: 0x%08x\n", axi
);
849 seq_printf(m
, "\tidle: 0x%08x\n", idle
);
850 idle
|= ~gpu
->idle_mask
& ~VIVS_HI_IDLE_STATE_AXI_LP
;
851 if ((idle
& VIVS_HI_IDLE_STATE_FE
) == 0)
852 seq_puts(m
, "\t FE is not idle\n");
853 if ((idle
& VIVS_HI_IDLE_STATE_DE
) == 0)
854 seq_puts(m
, "\t DE is not idle\n");
855 if ((idle
& VIVS_HI_IDLE_STATE_PE
) == 0)
856 seq_puts(m
, "\t PE is not idle\n");
857 if ((idle
& VIVS_HI_IDLE_STATE_SH
) == 0)
858 seq_puts(m
, "\t SH is not idle\n");
859 if ((idle
& VIVS_HI_IDLE_STATE_PA
) == 0)
860 seq_puts(m
, "\t PA is not idle\n");
861 if ((idle
& VIVS_HI_IDLE_STATE_SE
) == 0)
862 seq_puts(m
, "\t SE is not idle\n");
863 if ((idle
& VIVS_HI_IDLE_STATE_RA
) == 0)
864 seq_puts(m
, "\t RA is not idle\n");
865 if ((idle
& VIVS_HI_IDLE_STATE_TX
) == 0)
866 seq_puts(m
, "\t TX is not idle\n");
867 if ((idle
& VIVS_HI_IDLE_STATE_VG
) == 0)
868 seq_puts(m
, "\t VG is not idle\n");
869 if ((idle
& VIVS_HI_IDLE_STATE_IM
) == 0)
870 seq_puts(m
, "\t IM is not idle\n");
871 if ((idle
& VIVS_HI_IDLE_STATE_FP
) == 0)
872 seq_puts(m
, "\t FP is not idle\n");
873 if ((idle
& VIVS_HI_IDLE_STATE_TS
) == 0)
874 seq_puts(m
, "\t TS is not idle\n");
875 if (idle
& VIVS_HI_IDLE_STATE_AXI_LP
)
876 seq_puts(m
, "\t AXI low power mode\n");
878 if (gpu
->identity
.features
& chipFeatures_DEBUG_MODE
) {
879 u32 read0
= gpu_read(gpu
, VIVS_MC_DEBUG_READ0
);
880 u32 read1
= gpu_read(gpu
, VIVS_MC_DEBUG_READ1
);
881 u32 write
= gpu_read(gpu
, VIVS_MC_DEBUG_WRITE
);
883 seq_puts(m
, "\tMC\n");
884 seq_printf(m
, "\t read0: 0x%08x\n", read0
);
885 seq_printf(m
, "\t read1: 0x%08x\n", read1
);
886 seq_printf(m
, "\t write: 0x%08x\n", write
);
889 seq_puts(m
, "\tDMA ");
891 if (debug
.address
[0] == debug
.address
[1] &&
892 debug
.state
[0] == debug
.state
[1]) {
893 seq_puts(m
, "seems to be stuck\n");
894 } else if (debug
.address
[0] == debug
.address
[1]) {
895 seq_puts(m
, "address is constant\n");
897 seq_puts(m
, "is running\n");
900 seq_printf(m
, "\t address 0: 0x%08x\n", debug
.address
[0]);
901 seq_printf(m
, "\t address 1: 0x%08x\n", debug
.address
[1]);
902 seq_printf(m
, "\t state 0: 0x%08x\n", debug
.state
[0]);
903 seq_printf(m
, "\t state 1: 0x%08x\n", debug
.state
[1]);
904 seq_printf(m
, "\t last fetch 64 bit word: 0x%08x 0x%08x\n",
909 pm_runtime_mark_last_busy(gpu
->dev
);
910 pm_runtime_put_autosuspend(gpu
->dev
);
917 * Hangcheck detection for locked gpu:
919 static void recover_worker(struct work_struct
*work
)
921 struct etnaviv_gpu
*gpu
= container_of(work
, struct etnaviv_gpu
,
926 dev_err(gpu
->dev
, "hangcheck recover!\n");
928 if (pm_runtime_get_sync(gpu
->dev
) < 0)
931 mutex_lock(&gpu
->lock
);
933 /* Only catch the first event, or when manually re-armed */
934 if (etnaviv_dump_core
) {
935 etnaviv_core_dump(gpu
);
936 etnaviv_dump_core
= false;
939 etnaviv_hw_reset(gpu
);
941 /* complete all events, the GPU won't do it after the reset */
942 spin_lock_irqsave(&gpu
->event_spinlock
, flags
);
943 for_each_set_bit_from(i
, gpu
->event_bitmap
, ETNA_NR_EVENTS
) {
944 dma_fence_signal(gpu
->event
[i
].fence
);
945 gpu
->event
[i
].fence
= NULL
;
946 complete(&gpu
->event_free
);
948 bitmap_zero(gpu
->event_bitmap
, ETNA_NR_EVENTS
);
949 spin_unlock_irqrestore(&gpu
->event_spinlock
, flags
);
950 gpu
->completed_fence
= gpu
->active_fence
;
952 etnaviv_gpu_hw_init(gpu
);
954 gpu
->exec_state
= -1;
956 mutex_unlock(&gpu
->lock
);
957 pm_runtime_mark_last_busy(gpu
->dev
);
958 pm_runtime_put_autosuspend(gpu
->dev
);
960 /* Retire the buffer objects in a work */
961 etnaviv_queue_work(gpu
->drm
, &gpu
->retire_work
);
964 static void hangcheck_timer_reset(struct etnaviv_gpu
*gpu
)
966 DBG("%s", dev_name(gpu
->dev
));
967 mod_timer(&gpu
->hangcheck_timer
,
968 round_jiffies_up(jiffies
+ DRM_ETNAVIV_HANGCHECK_JIFFIES
));
971 static void hangcheck_handler(struct timer_list
*t
)
973 struct etnaviv_gpu
*gpu
= from_timer(gpu
, t
, hangcheck_timer
);
974 u32 fence
= gpu
->completed_fence
;
975 bool progress
= false;
977 if (fence
!= gpu
->hangcheck_fence
) {
978 gpu
->hangcheck_fence
= fence
;
983 u32 dma_addr
= gpu_read(gpu
, VIVS_FE_DMA_ADDRESS
);
984 int change
= dma_addr
- gpu
->hangcheck_dma_addr
;
986 if (change
< 0 || change
> 16) {
987 gpu
->hangcheck_dma_addr
= dma_addr
;
992 if (!progress
&& fence_after(gpu
->active_fence
, fence
)) {
993 dev_err(gpu
->dev
, "hangcheck detected gpu lockup!\n");
994 dev_err(gpu
->dev
, " completed fence: %u\n", fence
);
995 dev_err(gpu
->dev
, " active fence: %u\n",
997 etnaviv_queue_work(gpu
->drm
, &gpu
->recover_work
);
1000 /* if still more pending work, reset the hangcheck timer: */
1001 if (fence_after(gpu
->active_fence
, gpu
->hangcheck_fence
))
1002 hangcheck_timer_reset(gpu
);
1005 static void hangcheck_disable(struct etnaviv_gpu
*gpu
)
1007 del_timer_sync(&gpu
->hangcheck_timer
);
1008 cancel_work_sync(&gpu
->recover_work
);
1011 /* fence object management */
1012 struct etnaviv_fence
{
1013 struct etnaviv_gpu
*gpu
;
1014 struct dma_fence base
;
1017 static inline struct etnaviv_fence
*to_etnaviv_fence(struct dma_fence
*fence
)
1019 return container_of(fence
, struct etnaviv_fence
, base
);
1022 static const char *etnaviv_fence_get_driver_name(struct dma_fence
*fence
)
1027 static const char *etnaviv_fence_get_timeline_name(struct dma_fence
*fence
)
1029 struct etnaviv_fence
*f
= to_etnaviv_fence(fence
);
1031 return dev_name(f
->gpu
->dev
);
1034 static bool etnaviv_fence_enable_signaling(struct dma_fence
*fence
)
1039 static bool etnaviv_fence_signaled(struct dma_fence
*fence
)
1041 struct etnaviv_fence
*f
= to_etnaviv_fence(fence
);
1043 return fence_completed(f
->gpu
, f
->base
.seqno
);
1046 static void etnaviv_fence_release(struct dma_fence
*fence
)
1048 struct etnaviv_fence
*f
= to_etnaviv_fence(fence
);
1050 kfree_rcu(f
, base
.rcu
);
1053 static const struct dma_fence_ops etnaviv_fence_ops
= {
1054 .get_driver_name
= etnaviv_fence_get_driver_name
,
1055 .get_timeline_name
= etnaviv_fence_get_timeline_name
,
1056 .enable_signaling
= etnaviv_fence_enable_signaling
,
1057 .signaled
= etnaviv_fence_signaled
,
1058 .wait
= dma_fence_default_wait
,
1059 .release
= etnaviv_fence_release
,
1062 static struct dma_fence
*etnaviv_gpu_fence_alloc(struct etnaviv_gpu
*gpu
)
1064 struct etnaviv_fence
*f
;
1067 * GPU lock must already be held, otherwise fence completion order might
1068 * not match the seqno order assigned here.
1070 lockdep_assert_held(&gpu
->lock
);
1072 f
= kzalloc(sizeof(*f
), GFP_KERNEL
);
1078 dma_fence_init(&f
->base
, &etnaviv_fence_ops
, &gpu
->fence_spinlock
,
1079 gpu
->fence_context
, ++gpu
->next_fence
);
1084 int etnaviv_gpu_fence_sync_obj(struct etnaviv_gem_object
*etnaviv_obj
,
1085 unsigned int context
, bool exclusive
, bool explicit)
1087 struct reservation_object
*robj
= etnaviv_obj
->resv
;
1088 struct reservation_object_list
*fobj
;
1089 struct dma_fence
*fence
;
1093 ret
= reservation_object_reserve_shared(robj
);
1102 * If we have any shared fences, then the exclusive fence
1103 * should be ignored as it will already have been signalled.
1105 fobj
= reservation_object_get_list(robj
);
1106 if (!fobj
|| fobj
->shared_count
== 0) {
1107 /* Wait on any existing exclusive fence which isn't our own */
1108 fence
= reservation_object_get_excl(robj
);
1109 if (fence
&& fence
->context
!= context
) {
1110 ret
= dma_fence_wait(fence
, true);
1116 if (!exclusive
|| !fobj
)
1119 for (i
= 0; i
< fobj
->shared_count
; i
++) {
1120 fence
= rcu_dereference_protected(fobj
->shared
[i
],
1121 reservation_object_held(robj
));
1122 if (fence
->context
!= context
) {
1123 ret
= dma_fence_wait(fence
, true);
1136 static int event_alloc(struct etnaviv_gpu
*gpu
, unsigned nr_events
,
1137 unsigned int *events
)
1139 unsigned long flags
, timeout
= msecs_to_jiffies(10 * 10000);
1140 unsigned i
, acquired
= 0;
1142 for (i
= 0; i
< nr_events
; i
++) {
1145 ret
= wait_for_completion_timeout(&gpu
->event_free
, timeout
);
1148 dev_err(gpu
->dev
, "wait_for_completion_timeout failed");
1156 spin_lock_irqsave(&gpu
->event_spinlock
, flags
);
1158 for (i
= 0; i
< nr_events
; i
++) {
1159 int event
= find_first_zero_bit(gpu
->event_bitmap
, ETNA_NR_EVENTS
);
1162 memset(&gpu
->event
[event
], 0, sizeof(struct etnaviv_event
));
1163 set_bit(event
, gpu
->event_bitmap
);
1166 spin_unlock_irqrestore(&gpu
->event_spinlock
, flags
);
1171 for (i
= 0; i
< acquired
; i
++)
1172 complete(&gpu
->event_free
);
1177 static void event_free(struct etnaviv_gpu
*gpu
, unsigned int event
)
1179 unsigned long flags
;
1181 spin_lock_irqsave(&gpu
->event_spinlock
, flags
);
1183 if (!test_bit(event
, gpu
->event_bitmap
)) {
1184 dev_warn(gpu
->dev
, "event %u is already marked as free",
1186 spin_unlock_irqrestore(&gpu
->event_spinlock
, flags
);
1188 clear_bit(event
, gpu
->event_bitmap
);
1189 spin_unlock_irqrestore(&gpu
->event_spinlock
, flags
);
1191 complete(&gpu
->event_free
);
1196 * Cmdstream submission/retirement:
1199 static void retire_worker(struct work_struct
*work
)
1201 struct etnaviv_gpu
*gpu
= container_of(work
, struct etnaviv_gpu
,
1203 u32 fence
= gpu
->completed_fence
;
1204 struct etnaviv_cmdbuf
*cmdbuf
, *tmp
;
1207 mutex_lock(&gpu
->lock
);
1208 list_for_each_entry_safe(cmdbuf
, tmp
, &gpu
->active_cmd_list
, node
) {
1209 if (!dma_fence_is_signaled(cmdbuf
->fence
))
1212 list_del(&cmdbuf
->node
);
1213 dma_fence_put(cmdbuf
->fence
);
1215 for (i
= 0; i
< cmdbuf
->nr_bos
; i
++) {
1216 struct etnaviv_vram_mapping
*mapping
= cmdbuf
->bo_map
[i
];
1217 struct etnaviv_gem_object
*etnaviv_obj
= mapping
->object
;
1219 atomic_dec(&etnaviv_obj
->gpu_active
);
1220 /* drop the refcount taken in etnaviv_gpu_submit */
1221 etnaviv_gem_mapping_unreference(mapping
);
1224 etnaviv_cmdbuf_free(cmdbuf
);
1226 * We need to balance the runtime PM count caused by
1227 * each submission. Upon submission, we increment
1228 * the runtime PM counter, and allocate one event.
1229 * So here, we put the runtime PM count for each
1232 pm_runtime_put_autosuspend(gpu
->dev
);
1235 gpu
->retired_fence
= fence
;
1237 mutex_unlock(&gpu
->lock
);
1239 wake_up_all(&gpu
->fence_event
);
1242 int etnaviv_gpu_wait_fence_interruptible(struct etnaviv_gpu
*gpu
,
1243 u32 fence
, struct timespec
*timeout
)
1247 if (fence_after(fence
, gpu
->next_fence
)) {
1248 DRM_ERROR("waiting on invalid fence: %u (of %u)\n",
1249 fence
, gpu
->next_fence
);
1254 /* No timeout was requested: just test for completion */
1255 ret
= fence_completed(gpu
, fence
) ? 0 : -EBUSY
;
1257 unsigned long remaining
= etnaviv_timeout_to_jiffies(timeout
);
1259 ret
= wait_event_interruptible_timeout(gpu
->fence_event
,
1260 fence_completed(gpu
, fence
),
1263 DBG("timeout waiting for fence: %u (retired: %u completed: %u)",
1264 fence
, gpu
->retired_fence
,
1265 gpu
->completed_fence
);
1267 } else if (ret
!= -ERESTARTSYS
) {
1276 * Wait for an object to become inactive. This, on it's own, is not race
1277 * free: the object is moved by the retire worker off the active list, and
1278 * then the iova is put. Moreover, the object could be re-submitted just
1279 * after we notice that it's become inactive.
1281 * Although the retirement happens under the gpu lock, we don't want to hold
1282 * that lock in this function while waiting.
1284 int etnaviv_gpu_wait_obj_inactive(struct etnaviv_gpu
*gpu
,
1285 struct etnaviv_gem_object
*etnaviv_obj
, struct timespec
*timeout
)
1287 unsigned long remaining
;
1291 return !is_active(etnaviv_obj
) ? 0 : -EBUSY
;
1293 remaining
= etnaviv_timeout_to_jiffies(timeout
);
1295 ret
= wait_event_interruptible_timeout(gpu
->fence_event
,
1296 !is_active(etnaviv_obj
),
1299 struct etnaviv_drm_private
*priv
= gpu
->drm
->dev_private
;
1301 /* Synchronise with the retire worker */
1302 flush_workqueue(priv
->wq
);
1304 } else if (ret
== -ERESTARTSYS
) {
1305 return -ERESTARTSYS
;
1311 int etnaviv_gpu_pm_get_sync(struct etnaviv_gpu
*gpu
)
1313 return pm_runtime_get_sync(gpu
->dev
);
1316 void etnaviv_gpu_pm_put(struct etnaviv_gpu
*gpu
)
1318 pm_runtime_mark_last_busy(gpu
->dev
);
1319 pm_runtime_put_autosuspend(gpu
->dev
);
1322 static void sync_point_perfmon_sample(struct etnaviv_gpu
*gpu
,
1323 struct etnaviv_event
*event
, unsigned int flags
)
1325 const struct etnaviv_cmdbuf
*cmdbuf
= event
->cmdbuf
;
1328 for (i
= 0; i
< cmdbuf
->nr_pmrs
; i
++) {
1329 const struct etnaviv_perfmon_request
*pmr
= cmdbuf
->pmrs
+ i
;
1331 if (pmr
->flags
== flags
)
1332 etnaviv_perfmon_process(gpu
, pmr
);
1336 static void sync_point_perfmon_sample_pre(struct etnaviv_gpu
*gpu
,
1337 struct etnaviv_event
*event
)
1341 /* disable clock gating */
1342 val
= gpu_read(gpu
, VIVS_PM_POWER_CONTROLS
);
1343 val
&= ~VIVS_PM_POWER_CONTROLS_ENABLE_MODULE_CLOCK_GATING
;
1344 gpu_write(gpu
, VIVS_PM_POWER_CONTROLS
, val
);
1346 /* enable debug register */
1347 val
= gpu_read(gpu
, VIVS_HI_CLOCK_CONTROL
);
1348 val
&= ~VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS
;
1349 gpu_write(gpu
, VIVS_HI_CLOCK_CONTROL
, val
);
1351 sync_point_perfmon_sample(gpu
, event
, ETNA_PM_PROCESS_PRE
);
1354 static void sync_point_perfmon_sample_post(struct etnaviv_gpu
*gpu
,
1355 struct etnaviv_event
*event
)
1357 const struct etnaviv_cmdbuf
*cmdbuf
= event
->cmdbuf
;
1361 sync_point_perfmon_sample(gpu
, event
, ETNA_PM_PROCESS_POST
);
1363 for (i
= 0; i
< cmdbuf
->nr_pmrs
; i
++) {
1364 const struct etnaviv_perfmon_request
*pmr
= cmdbuf
->pmrs
+ i
;
1366 *pmr
->bo_vma
= pmr
->sequence
;
1369 /* disable debug register */
1370 val
= gpu_read(gpu
, VIVS_HI_CLOCK_CONTROL
);
1371 val
|= VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS
;
1372 gpu_write(gpu
, VIVS_HI_CLOCK_CONTROL
, val
);
1374 /* enable clock gating */
1375 val
= gpu_read(gpu
, VIVS_PM_POWER_CONTROLS
);
1376 val
|= VIVS_PM_POWER_CONTROLS_ENABLE_MODULE_CLOCK_GATING
;
1377 gpu_write(gpu
, VIVS_PM_POWER_CONTROLS
, val
);
1381 /* add bo's to gpu's ring, and kick gpu: */
1382 int etnaviv_gpu_submit(struct etnaviv_gpu
*gpu
,
1383 struct etnaviv_gem_submit
*submit
, struct etnaviv_cmdbuf
*cmdbuf
)
1385 struct dma_fence
*fence
;
1386 unsigned int i
, nr_events
= 1, event
[3];
1389 ret
= etnaviv_gpu_pm_get_sync(gpu
);
1394 * if there are performance monitor requests we need to have
1395 * - a sync point to re-configure gpu and process ETNA_PM_PROCESS_PRE
1397 * - a sync point to re-configure gpu, process ETNA_PM_PROCESS_POST requests
1398 * and update the sequence number for userspace.
1400 if (cmdbuf
->nr_pmrs
)
1403 ret
= event_alloc(gpu
, nr_events
, event
);
1405 DRM_ERROR("no free events\n");
1409 mutex_lock(&gpu
->lock
);
1411 fence
= etnaviv_gpu_fence_alloc(gpu
);
1413 for (i
= 0; i
< nr_events
; i
++)
1414 event_free(gpu
, event
[i
]);
1420 gpu
->event
[event
[0]].fence
= fence
;
1421 submit
->fence
= dma_fence_get(fence
);
1422 gpu
->active_fence
= submit
->fence
->seqno
;
1424 if (gpu
->lastctx
!= cmdbuf
->ctx
) {
1425 gpu
->mmu
->need_flush
= true;
1426 gpu
->switch_context
= true;
1427 gpu
->lastctx
= cmdbuf
->ctx
;
1430 if (cmdbuf
->nr_pmrs
) {
1431 gpu
->event
[event
[1]].sync_point
= &sync_point_perfmon_sample_pre
;
1432 gpu
->event
[event
[1]].cmdbuf
= cmdbuf
;
1433 etnaviv_sync_point_queue(gpu
, event
[1]);
1436 etnaviv_buffer_queue(gpu
, event
[0], cmdbuf
);
1438 if (cmdbuf
->nr_pmrs
) {
1439 gpu
->event
[event
[2]].sync_point
= &sync_point_perfmon_sample_post
;
1440 gpu
->event
[event
[2]].cmdbuf
= cmdbuf
;
1441 etnaviv_sync_point_queue(gpu
, event
[2]);
1444 cmdbuf
->fence
= fence
;
1445 list_add_tail(&cmdbuf
->node
, &gpu
->active_cmd_list
);
1447 /* We're committed to adding this command buffer, hold a PM reference */
1448 pm_runtime_get_noresume(gpu
->dev
);
1450 for (i
= 0; i
< submit
->nr_bos
; i
++) {
1451 struct etnaviv_gem_object
*etnaviv_obj
= submit
->bos
[i
].obj
;
1453 /* Each cmdbuf takes a refcount on the mapping */
1454 etnaviv_gem_mapping_reference(submit
->bos
[i
].mapping
);
1455 cmdbuf
->bo_map
[i
] = submit
->bos
[i
].mapping
;
1456 atomic_inc(&etnaviv_obj
->gpu_active
);
1458 if (submit
->bos
[i
].flags
& ETNA_SUBMIT_BO_WRITE
)
1459 reservation_object_add_excl_fence(etnaviv_obj
->resv
,
1462 reservation_object_add_shared_fence(etnaviv_obj
->resv
,
1465 cmdbuf
->nr_bos
= submit
->nr_bos
;
1466 hangcheck_timer_reset(gpu
);
1470 mutex_unlock(&gpu
->lock
);
1473 etnaviv_gpu_pm_put(gpu
);
1478 static void sync_point_worker(struct work_struct
*work
)
1480 struct etnaviv_gpu
*gpu
= container_of(work
, struct etnaviv_gpu
,
1482 struct etnaviv_event
*event
= &gpu
->event
[gpu
->sync_point_event
];
1483 u32 addr
= gpu_read(gpu
, VIVS_FE_DMA_ADDRESS
);
1485 event
->sync_point(gpu
, event
);
1486 event_free(gpu
, gpu
->sync_point_event
);
1488 /* restart FE last to avoid GPU and IRQ racing against this worker */
1489 etnaviv_gpu_start_fe(gpu
, addr
+ 2, 2);
1495 static irqreturn_t
irq_handler(int irq
, void *data
)
1497 struct etnaviv_gpu
*gpu
= data
;
1498 irqreturn_t ret
= IRQ_NONE
;
1500 u32 intr
= gpu_read(gpu
, VIVS_HI_INTR_ACKNOWLEDGE
);
1505 pm_runtime_mark_last_busy(gpu
->dev
);
1507 dev_dbg(gpu
->dev
, "intr 0x%08x\n", intr
);
1509 if (intr
& VIVS_HI_INTR_ACKNOWLEDGE_AXI_BUS_ERROR
) {
1510 dev_err(gpu
->dev
, "AXI bus error\n");
1511 intr
&= ~VIVS_HI_INTR_ACKNOWLEDGE_AXI_BUS_ERROR
;
1514 if (intr
& VIVS_HI_INTR_ACKNOWLEDGE_MMU_EXCEPTION
) {
1517 dev_err_ratelimited(gpu
->dev
,
1518 "MMU fault status 0x%08x\n",
1519 gpu_read(gpu
, VIVS_MMUv2_STATUS
));
1520 for (i
= 0; i
< 4; i
++) {
1521 dev_err_ratelimited(gpu
->dev
,
1522 "MMU %d fault addr 0x%08x\n",
1524 VIVS_MMUv2_EXCEPTION_ADDR(i
)));
1526 intr
&= ~VIVS_HI_INTR_ACKNOWLEDGE_MMU_EXCEPTION
;
1529 while ((event
= ffs(intr
)) != 0) {
1530 struct dma_fence
*fence
;
1534 intr
&= ~(1 << event
);
1536 dev_dbg(gpu
->dev
, "event %u\n", event
);
1538 if (gpu
->event
[event
].sync_point
) {
1539 gpu
->sync_point_event
= event
;
1540 etnaviv_queue_work(gpu
->drm
, &gpu
->sync_point_work
);
1543 fence
= gpu
->event
[event
].fence
;
1547 gpu
->event
[event
].fence
= NULL
;
1548 dma_fence_signal(fence
);
1551 * Events can be processed out of order. Eg,
1552 * - allocate and queue event 0
1553 * - allocate event 1
1554 * - event 0 completes, we process it
1555 * - allocate and queue event 0
1556 * - event 1 and event 0 complete
1557 * we can end up processing event 0 first, then 1.
1559 if (fence_after(fence
->seqno
, gpu
->completed_fence
))
1560 gpu
->completed_fence
= fence
->seqno
;
1562 event_free(gpu
, event
);
1565 /* Retire the buffer objects in a work */
1566 etnaviv_queue_work(gpu
->drm
, &gpu
->retire_work
);
1574 static int etnaviv_gpu_clk_enable(struct etnaviv_gpu
*gpu
)
1579 ret
= clk_prepare_enable(gpu
->clk_bus
);
1584 if (gpu
->clk_core
) {
1585 ret
= clk_prepare_enable(gpu
->clk_core
);
1587 goto disable_clk_bus
;
1590 if (gpu
->clk_shader
) {
1591 ret
= clk_prepare_enable(gpu
->clk_shader
);
1593 goto disable_clk_core
;
1600 clk_disable_unprepare(gpu
->clk_core
);
1603 clk_disable_unprepare(gpu
->clk_bus
);
1608 static int etnaviv_gpu_clk_disable(struct etnaviv_gpu
*gpu
)
1610 if (gpu
->clk_shader
)
1611 clk_disable_unprepare(gpu
->clk_shader
);
1613 clk_disable_unprepare(gpu
->clk_core
);
1615 clk_disable_unprepare(gpu
->clk_bus
);
1620 int etnaviv_gpu_wait_idle(struct etnaviv_gpu
*gpu
, unsigned int timeout_ms
)
1622 unsigned long timeout
= jiffies
+ msecs_to_jiffies(timeout_ms
);
1625 u32 idle
= gpu_read(gpu
, VIVS_HI_IDLE_STATE
);
1627 if ((idle
& gpu
->idle_mask
) == gpu
->idle_mask
)
1630 if (time_is_before_jiffies(timeout
)) {
1632 "timed out waiting for idle: idle=0x%x\n",
1641 static int etnaviv_gpu_hw_suspend(struct etnaviv_gpu
*gpu
)
1644 /* Replace the last WAIT with END */
1645 etnaviv_buffer_end(gpu
);
1648 * We know that only the FE is busy here, this should
1649 * happen quickly (as the WAIT is only 200 cycles). If
1650 * we fail, just warn and continue.
1652 etnaviv_gpu_wait_idle(gpu
, 100);
1655 return etnaviv_gpu_clk_disable(gpu
);
1659 static int etnaviv_gpu_hw_resume(struct etnaviv_gpu
*gpu
)
1663 ret
= mutex_lock_killable(&gpu
->lock
);
1667 etnaviv_gpu_update_clock(gpu
);
1668 etnaviv_gpu_hw_init(gpu
);
1670 gpu
->switch_context
= true;
1671 gpu
->exec_state
= -1;
1673 mutex_unlock(&gpu
->lock
);
1680 etnaviv_gpu_cooling_get_max_state(struct thermal_cooling_device
*cdev
,
1681 unsigned long *state
)
1689 etnaviv_gpu_cooling_get_cur_state(struct thermal_cooling_device
*cdev
,
1690 unsigned long *state
)
1692 struct etnaviv_gpu
*gpu
= cdev
->devdata
;
1694 *state
= gpu
->freq_scale
;
1700 etnaviv_gpu_cooling_set_cur_state(struct thermal_cooling_device
*cdev
,
1701 unsigned long state
)
1703 struct etnaviv_gpu
*gpu
= cdev
->devdata
;
1705 mutex_lock(&gpu
->lock
);
1706 gpu
->freq_scale
= state
;
1707 if (!pm_runtime_suspended(gpu
->dev
))
1708 etnaviv_gpu_update_clock(gpu
);
1709 mutex_unlock(&gpu
->lock
);
1714 static struct thermal_cooling_device_ops cooling_ops
= {
1715 .get_max_state
= etnaviv_gpu_cooling_get_max_state
,
1716 .get_cur_state
= etnaviv_gpu_cooling_get_cur_state
,
1717 .set_cur_state
= etnaviv_gpu_cooling_set_cur_state
,
1720 static int etnaviv_gpu_bind(struct device
*dev
, struct device
*master
,
1723 struct drm_device
*drm
= data
;
1724 struct etnaviv_drm_private
*priv
= drm
->dev_private
;
1725 struct etnaviv_gpu
*gpu
= dev_get_drvdata(dev
);
1728 if (IS_ENABLED(CONFIG_DRM_ETNAVIV_THERMAL
)) {
1729 gpu
->cooling
= thermal_of_cooling_device_register(dev
->of_node
,
1730 (char *)dev_name(dev
), gpu
, &cooling_ops
);
1731 if (IS_ERR(gpu
->cooling
))
1732 return PTR_ERR(gpu
->cooling
);
1736 ret
= pm_runtime_get_sync(gpu
->dev
);
1738 ret
= etnaviv_gpu_clk_enable(gpu
);
1741 if (IS_ENABLED(CONFIG_DRM_ETNAVIV_THERMAL
))
1742 thermal_cooling_device_unregister(gpu
->cooling
);
1747 gpu
->fence_context
= dma_fence_context_alloc(1);
1748 spin_lock_init(&gpu
->fence_spinlock
);
1750 INIT_LIST_HEAD(&gpu
->active_cmd_list
);
1751 INIT_WORK(&gpu
->retire_work
, retire_worker
);
1752 INIT_WORK(&gpu
->sync_point_work
, sync_point_worker
);
1753 INIT_WORK(&gpu
->recover_work
, recover_worker
);
1754 init_waitqueue_head(&gpu
->fence_event
);
1756 timer_setup(&gpu
->hangcheck_timer
, hangcheck_handler
, TIMER_DEFERRABLE
);
1758 priv
->gpu
[priv
->num_gpus
++] = gpu
;
1760 pm_runtime_mark_last_busy(gpu
->dev
);
1761 pm_runtime_put_autosuspend(gpu
->dev
);
1766 static void etnaviv_gpu_unbind(struct device
*dev
, struct device
*master
,
1769 struct etnaviv_gpu
*gpu
= dev_get_drvdata(dev
);
1771 DBG("%s", dev_name(gpu
->dev
));
1773 hangcheck_disable(gpu
);
1776 pm_runtime_get_sync(gpu
->dev
);
1777 pm_runtime_put_sync_suspend(gpu
->dev
);
1779 etnaviv_gpu_hw_suspend(gpu
);
1783 etnaviv_cmdbuf_free(gpu
->buffer
);
1787 if (gpu
->cmdbuf_suballoc
) {
1788 etnaviv_cmdbuf_suballoc_destroy(gpu
->cmdbuf_suballoc
);
1789 gpu
->cmdbuf_suballoc
= NULL
;
1793 etnaviv_iommu_destroy(gpu
->mmu
);
1799 if (IS_ENABLED(CONFIG_DRM_ETNAVIV_THERMAL
))
1800 thermal_cooling_device_unregister(gpu
->cooling
);
1801 gpu
->cooling
= NULL
;
1804 static const struct component_ops gpu_ops
= {
1805 .bind
= etnaviv_gpu_bind
,
1806 .unbind
= etnaviv_gpu_unbind
,
1809 static const struct of_device_id etnaviv_gpu_match
[] = {
1811 .compatible
= "vivante,gc"
1816 static int etnaviv_gpu_platform_probe(struct platform_device
*pdev
)
1818 struct device
*dev
= &pdev
->dev
;
1819 struct etnaviv_gpu
*gpu
;
1822 gpu
= devm_kzalloc(dev
, sizeof(*gpu
), GFP_KERNEL
);
1826 gpu
->dev
= &pdev
->dev
;
1827 mutex_init(&gpu
->lock
);
1829 /* Map registers: */
1830 gpu
->mmio
= etnaviv_ioremap(pdev
, NULL
, dev_name(gpu
->dev
));
1831 if (IS_ERR(gpu
->mmio
))
1832 return PTR_ERR(gpu
->mmio
);
1834 /* Get Interrupt: */
1835 gpu
->irq
= platform_get_irq(pdev
, 0);
1837 dev_err(dev
, "failed to get irq: %d\n", gpu
->irq
);
1841 err
= devm_request_irq(&pdev
->dev
, gpu
->irq
, irq_handler
, 0,
1842 dev_name(gpu
->dev
), gpu
);
1844 dev_err(dev
, "failed to request IRQ%u: %d\n", gpu
->irq
, err
);
1849 gpu
->clk_bus
= devm_clk_get(&pdev
->dev
, "bus");
1850 DBG("clk_bus: %p", gpu
->clk_bus
);
1851 if (IS_ERR(gpu
->clk_bus
))
1852 gpu
->clk_bus
= NULL
;
1854 gpu
->clk_core
= devm_clk_get(&pdev
->dev
, "core");
1855 DBG("clk_core: %p", gpu
->clk_core
);
1856 if (IS_ERR(gpu
->clk_core
))
1857 gpu
->clk_core
= NULL
;
1858 gpu
->base_rate_core
= clk_get_rate(gpu
->clk_core
);
1860 gpu
->clk_shader
= devm_clk_get(&pdev
->dev
, "shader");
1861 DBG("clk_shader: %p", gpu
->clk_shader
);
1862 if (IS_ERR(gpu
->clk_shader
))
1863 gpu
->clk_shader
= NULL
;
1864 gpu
->base_rate_shader
= clk_get_rate(gpu
->clk_shader
);
1866 /* TODO: figure out max mapped size */
1867 dev_set_drvdata(dev
, gpu
);
1870 * We treat the device as initially suspended. The runtime PM
1871 * autosuspend delay is rather arbitary: no measurements have
1872 * yet been performed to determine an appropriate value.
1874 pm_runtime_use_autosuspend(gpu
->dev
);
1875 pm_runtime_set_autosuspend_delay(gpu
->dev
, 200);
1876 pm_runtime_enable(gpu
->dev
);
1878 err
= component_add(&pdev
->dev
, &gpu_ops
);
1880 dev_err(&pdev
->dev
, "failed to register component: %d\n", err
);
1887 static int etnaviv_gpu_platform_remove(struct platform_device
*pdev
)
1889 component_del(&pdev
->dev
, &gpu_ops
);
1890 pm_runtime_disable(&pdev
->dev
);
1895 static int etnaviv_gpu_rpm_suspend(struct device
*dev
)
1897 struct etnaviv_gpu
*gpu
= dev_get_drvdata(dev
);
1900 /* If we have outstanding fences, we're not idle */
1901 if (gpu
->completed_fence
!= gpu
->active_fence
)
1904 /* Check whether the hardware (except FE) is idle */
1905 mask
= gpu
->idle_mask
& ~VIVS_HI_IDLE_STATE_FE
;
1906 idle
= gpu_read(gpu
, VIVS_HI_IDLE_STATE
) & mask
;
1910 return etnaviv_gpu_hw_suspend(gpu
);
1913 static int etnaviv_gpu_rpm_resume(struct device
*dev
)
1915 struct etnaviv_gpu
*gpu
= dev_get_drvdata(dev
);
1918 ret
= etnaviv_gpu_clk_enable(gpu
);
1922 /* Re-initialise the basic hardware state */
1923 if (gpu
->drm
&& gpu
->buffer
) {
1924 ret
= etnaviv_gpu_hw_resume(gpu
);
1926 etnaviv_gpu_clk_disable(gpu
);
1935 static const struct dev_pm_ops etnaviv_gpu_pm_ops
= {
1936 SET_RUNTIME_PM_OPS(etnaviv_gpu_rpm_suspend
, etnaviv_gpu_rpm_resume
,
1940 struct platform_driver etnaviv_gpu_driver
= {
1942 .name
= "etnaviv-gpu",
1943 .owner
= THIS_MODULE
,
1944 .pm
= &etnaviv_gpu_pm_ops
,
1945 .of_match_table
= etnaviv_gpu_match
,
1947 .probe
= etnaviv_gpu_platform_probe
,
1948 .remove
= etnaviv_gpu_platform_remove
,
1949 .id_table
= gpu_ids
,