]> git.ipfire.org Git - people/ms/linux.git/blame - drivers/gpu/drm/radeon/cik.c
projects / people / ms / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: Report doorbell configuration to amdkfd
[people/ms/linux.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
8cc1a532
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
6f2043ce 29#include "radeon_asic.h"
8cc1a532
AD		 30#include "cikd.h"
31#include "atom.h"
841cf442 32#include "cik_blit_shaders.h"
8c68e393 33#include "radeon_ucode.h"
22c775ce 34#include "clearstate_ci.h"
02c81327
AD		 35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
277babc3 41MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
02c81327 42MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 43MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 44MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
f2c6b0f4
AD		 45
46MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
47MODULE_FIRMWARE("radeon/bonaire_me.bin");
48MODULE_FIRMWARE("radeon/bonaire_ce.bin");
49MODULE_FIRMWARE("radeon/bonaire_mec.bin");
50MODULE_FIRMWARE("radeon/bonaire_mc.bin");
51MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
52MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
53MODULE_FIRMWARE("radeon/bonaire_smc.bin");
54
d4775655
AD		 55MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
56MODULE_FIRMWARE("radeon/HAWAII_me.bin");
57MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
58MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
59MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
277babc3 60MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
d4775655
AD		 61MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
62MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
63MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
f2c6b0f4
AD		 64
65MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
66MODULE_FIRMWARE("radeon/hawaii_me.bin");
67MODULE_FIRMWARE("radeon/hawaii_ce.bin");
68MODULE_FIRMWARE("radeon/hawaii_mec.bin");
69MODULE_FIRMWARE("radeon/hawaii_mc.bin");
70MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
71MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
72MODULE_FIRMWARE("radeon/hawaii_smc.bin");
73
02c81327
AD		 74MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
75MODULE_FIRMWARE("radeon/KAVERI_me.bin");
76MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
77MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
78MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 79MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
f2c6b0f4
AD		 80
81MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
82MODULE_FIRMWARE("radeon/kaveri_me.bin");
83MODULE_FIRMWARE("radeon/kaveri_ce.bin");
84MODULE_FIRMWARE("radeon/kaveri_mec.bin");
85MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
86MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
87MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
88
02c81327
AD		 89MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
90MODULE_FIRMWARE("radeon/KABINI_me.bin");
91MODULE_FIRMWARE("radeon/KABINI_ce.bin");
92MODULE_FIRMWARE("radeon/KABINI_mec.bin");
93MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 94MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
f2c6b0f4
AD		 95
96MODULE_FIRMWARE("radeon/kabini_pfp.bin");
97MODULE_FIRMWARE("radeon/kabini_me.bin");
98MODULE_FIRMWARE("radeon/kabini_ce.bin");
99MODULE_FIRMWARE("radeon/kabini_mec.bin");
100MODULE_FIRMWARE("radeon/kabini_rlc.bin");
101MODULE_FIRMWARE("radeon/kabini_sdma.bin");
102
f73a9e83
SL		 103MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
104MODULE_FIRMWARE("radeon/MULLINS_me.bin");
105MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
106MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
107MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
108MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
02c81327 109
f2c6b0f4
AD		 110MODULE_FIRMWARE("radeon/mullins_pfp.bin");
111MODULE_FIRMWARE("radeon/mullins_me.bin");
112MODULE_FIRMWARE("radeon/mullins_ce.bin");
113MODULE_FIRMWARE("radeon/mullins_mec.bin");
114MODULE_FIRMWARE("radeon/mullins_rlc.bin");
115MODULE_FIRMWARE("radeon/mullins_sdma.bin");
116
a59781bb
AD		 117extern int r600_ih_ring_alloc(struct radeon_device *rdev);
118extern void r600_ih_ring_fini(struct radeon_device *rdev);
6f2043ce
AD		 119extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
120extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
cc066715 121extern bool evergreen_is_display_hung(struct radeon_device *rdev);
1fd11777
AD		 122extern void sumo_rlc_fini(struct radeon_device *rdev);
123extern int sumo_rlc_init(struct radeon_device *rdev);
1c49165d 124extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 125extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 126extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
65fcf668 127static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
2483b4ea
CK		 128extern int cik_sdma_resume(struct radeon_device *rdev);
129extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
130extern void cik_sdma_fini(struct radeon_device *rdev);
a1d6f97c 131extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
cc066715 132static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 133static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 134static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 135static void cik_init_pg(struct radeon_device *rdev);
136static void cik_init_cg(struct radeon_device *rdev);
fb2c7f4d
AD		 137static void cik_fini_pg(struct radeon_device *rdev);
138static void cik_fini_cg(struct radeon_device *rdev);
4214faf6
AD		 139static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
140 bool enable);
6f2043ce 141
286d9cc6
AD		 142/* get temperature in millidegrees */
143int ci_get_temp(struct radeon_device *rdev)
144{
145 u32 temp;
146 int actual_temp = 0;
147
148 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
149 CTF_TEMP_SHIFT;
150
151 if (temp & 0x200)
152 actual_temp = 255;
153 else
154 actual_temp = temp & 0x1ff;
155
156 actual_temp = actual_temp * 1000;
157
158 return actual_temp;
159}
160
161/* get temperature in millidegrees */
162int kv_get_temp(struct radeon_device *rdev)
163{
164 u32 temp;
165 int actual_temp = 0;
166
167 temp = RREG32_SMC(0xC0300E0C);
168
169 if (temp)
170 actual_temp = (temp / 8) - 49;
171 else
172 actual_temp = 0;
173
174 actual_temp = actual_temp * 1000;
175
176 return actual_temp;
177}
6f2043ce 178
6e2c3c0a
AD		 179/*
180 * Indirect registers accessor
181 */
182u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
183{
0a5b7b0b 184 unsigned long flags;
6e2c3c0a
AD		 185 u32 r;
186
0a5b7b0b 187 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 188 WREG32(PCIE_INDEX, reg);
189 (void)RREG32(PCIE_INDEX);
190 r = RREG32(PCIE_DATA);
0a5b7b0b 191 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 192 return r;
193}
194
195void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
196{
0a5b7b0b
AD		 197 unsigned long flags;
198
199 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 200 WREG32(PCIE_INDEX, reg);
201 (void)RREG32(PCIE_INDEX);
202 WREG32(PCIE_DATA, v);
203 (void)RREG32(PCIE_DATA);
0a5b7b0b 204 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 205}
206
22c775ce
AD		 207static const u32 spectre_rlc_save_restore_register_list[] =
208{
209 (0x0e00 << 16) | (0xc12c >> 2),
210 0x00000000,
211 (0x0e00 << 16) | (0xc140 >> 2),
212 0x00000000,
213 (0x0e00 << 16) | (0xc150 >> 2),
214 0x00000000,
215 (0x0e00 << 16) | (0xc15c >> 2),
216 0x00000000,
217 (0x0e00 << 16) | (0xc168 >> 2),
218 0x00000000,
219 (0x0e00 << 16) | (0xc170 >> 2),
220 0x00000000,
221 (0x0e00 << 16) | (0xc178 >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0xc204 >> 2),
224 0x00000000,
225 (0x0e00 << 16) | (0xc2b4 >> 2),
226 0x00000000,
227 (0x0e00 << 16) | (0xc2b8 >> 2),
228 0x00000000,
229 (0x0e00 << 16) | (0xc2bc >> 2),
230 0x00000000,
231 (0x0e00 << 16) | (0xc2c0 >> 2),
232 0x00000000,
233 (0x0e00 << 16) | (0x8228 >> 2),
234 0x00000000,
235 (0x0e00 << 16) | (0x829c >> 2),
236 0x00000000,
237 (0x0e00 << 16) | (0x869c >> 2),
238 0x00000000,
239 (0x0600 << 16) | (0x98f4 >> 2),
240 0x00000000,
241 (0x0e00 << 16) | (0x98f8 >> 2),
242 0x00000000,
243 (0x0e00 << 16) | (0x9900 >> 2),
244 0x00000000,
245 (0x0e00 << 16) | (0xc260 >> 2),
246 0x00000000,
247 (0x0e00 << 16) | (0x90e8 >> 2),
248 0x00000000,
249 (0x0e00 << 16) | (0x3c000 >> 2),
250 0x00000000,
251 (0x0e00 << 16) | (0x3c00c >> 2),
252 0x00000000,
253 (0x0e00 << 16) | (0x8c1c >> 2),
254 0x00000000,
255 (0x0e00 << 16) | (0x9700 >> 2),
256 0x00000000,
257 (0x0e00 << 16) | (0xcd20 >> 2),
258 0x00000000,
259 (0x4e00 << 16) | (0xcd20 >> 2),
260 0x00000000,
261 (0x5e00 << 16) | (0xcd20 >> 2),
262 0x00000000,
263 (0x6e00 << 16) | (0xcd20 >> 2),
264 0x00000000,
265 (0x7e00 << 16) | (0xcd20 >> 2),
266 0x00000000,
267 (0x8e00 << 16) | (0xcd20 >> 2),
268 0x00000000,
269 (0x9e00 << 16) | (0xcd20 >> 2),
270 0x00000000,
271 (0xae00 << 16) | (0xcd20 >> 2),
272 0x00000000,
273 (0xbe00 << 16) | (0xcd20 >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0x89bc >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0x8900 >> 2),
278 0x00000000,
279 0x3,
280 (0x0e00 << 16) | (0xc130 >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0xc134 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0xc1fc >> 2),
285 0x00000000,
286 (0x0e00 << 16) | (0xc208 >> 2),
287 0x00000000,
288 (0x0e00 << 16) | (0xc264 >> 2),
289 0x00000000,
290 (0x0e00 << 16) | (0xc268 >> 2),
291 0x00000000,
292 (0x0e00 << 16) | (0xc26c >> 2),
293 0x00000000,
294 (0x0e00 << 16) | (0xc270 >> 2),
295 0x00000000,
296 (0x0e00 << 16) | (0xc274 >> 2),
297 0x00000000,
298 (0x0e00 << 16) | (0xc278 >> 2),
299 0x00000000,
300 (0x0e00 << 16) | (0xc27c >> 2),
301 0x00000000,
302 (0x0e00 << 16) | (0xc280 >> 2),
303 0x00000000,
304 (0x0e00 << 16) | (0xc284 >> 2),
305 0x00000000,
306 (0x0e00 << 16) | (0xc288 >> 2),
307 0x00000000,
308 (0x0e00 << 16) | (0xc28c >> 2),
309 0x00000000,
310 (0x0e00 << 16) | (0xc290 >> 2),
311 0x00000000,
312 (0x0e00 << 16) | (0xc294 >> 2),
313 0x00000000,
314 (0x0e00 << 16) | (0xc298 >> 2),
315 0x00000000,
316 (0x0e00 << 16) | (0xc29c >> 2),
317 0x00000000,
318 (0x0e00 << 16) | (0xc2a0 >> 2),
319 0x00000000,
320 (0x0e00 << 16) | (0xc2a4 >> 2),
321 0x00000000,
322 (0x0e00 << 16) | (0xc2a8 >> 2),
323 0x00000000,
324 (0x0e00 << 16) | (0xc2ac >> 2),
325 0x00000000,
326 (0x0e00 << 16) | (0xc2b0 >> 2),
327 0x00000000,
328 (0x0e00 << 16) | (0x301d0 >> 2),
329 0x00000000,
330 (0x0e00 << 16) | (0x30238 >> 2),
331 0x00000000,
332 (0x0e00 << 16) | (0x30250 >> 2),
333 0x00000000,
334 (0x0e00 << 16) | (0x30254 >> 2),
335 0x00000000,
336 (0x0e00 << 16) | (0x30258 >> 2),
337 0x00000000,
338 (0x0e00 << 16) | (0x3025c >> 2),
339 0x00000000,
340 (0x4e00 << 16) | (0xc900 >> 2),
341 0x00000000,
342 (0x5e00 << 16) | (0xc900 >> 2),
343 0x00000000,
344 (0x6e00 << 16) | (0xc900 >> 2),
345 0x00000000,
346 (0x7e00 << 16) | (0xc900 >> 2),
347 0x00000000,
348 (0x8e00 << 16) | (0xc900 >> 2),
349 0x00000000,
350 (0x9e00 << 16) | (0xc900 >> 2),
351 0x00000000,
352 (0xae00 << 16) | (0xc900 >> 2),
353 0x00000000,
354 (0xbe00 << 16) | (0xc900 >> 2),
355 0x00000000,
356 (0x4e00 << 16) | (0xc904 >> 2),
357 0x00000000,
358 (0x5e00 << 16) | (0xc904 >> 2),
359 0x00000000,
360 (0x6e00 << 16) | (0xc904 >> 2),
361 0x00000000,
362 (0x7e00 << 16) | (0xc904 >> 2),
363 0x00000000,
364 (0x8e00 << 16) | (0xc904 >> 2),
365 0x00000000,
366 (0x9e00 << 16) | (0xc904 >> 2),
367 0x00000000,
368 (0xae00 << 16) | (0xc904 >> 2),
369 0x00000000,
370 (0xbe00 << 16) | (0xc904 >> 2),
371 0x00000000,
372 (0x4e00 << 16) | (0xc908 >> 2),
373 0x00000000,
374 (0x5e00 << 16) | (0xc908 >> 2),
375 0x00000000,
376 (0x6e00 << 16) | (0xc908 >> 2),
377 0x00000000,
378 (0x7e00 << 16) | (0xc908 >> 2),
379 0x00000000,
380 (0x8e00 << 16) | (0xc908 >> 2),
381 0x00000000,
382 (0x9e00 << 16) | (0xc908 >> 2),
383 0x00000000,
384 (0xae00 << 16) | (0xc908 >> 2),
385 0x00000000,
386 (0xbe00 << 16) | (0xc908 >> 2),
387 0x00000000,
388 (0x4e00 << 16) | (0xc90c >> 2),
389 0x00000000,
390 (0x5e00 << 16) | (0xc90c >> 2),
391 0x00000000,
392 (0x6e00 << 16) | (0xc90c >> 2),
393 0x00000000,
394 (0x7e00 << 16) | (0xc90c >> 2),
395 0x00000000,
396 (0x8e00 << 16) | (0xc90c >> 2),
397 0x00000000,
398 (0x9e00 << 16) | (0xc90c >> 2),
399 0x00000000,
400 (0xae00 << 16) | (0xc90c >> 2),
401 0x00000000,
402 (0xbe00 << 16) | (0xc90c >> 2),
403 0x00000000,
404 (0x4e00 << 16) | (0xc910 >> 2),
405 0x00000000,
406 (0x5e00 << 16) | (0xc910 >> 2),
407 0x00000000,
408 (0x6e00 << 16) | (0xc910 >> 2),
409 0x00000000,
410 (0x7e00 << 16) | (0xc910 >> 2),
411 0x00000000,
412 (0x8e00 << 16) | (0xc910 >> 2),
413 0x00000000,
414 (0x9e00 << 16) | (0xc910 >> 2),
415 0x00000000,
416 (0xae00 << 16) | (0xc910 >> 2),
417 0x00000000,
418 (0xbe00 << 16) | (0xc910 >> 2),
419 0x00000000,
420 (0x0e00 << 16) | (0xc99c >> 2),
421 0x00000000,
422 (0x0e00 << 16) | (0x9834 >> 2),
423 0x00000000,
424 (0x0000 << 16) | (0x30f00 >> 2),
425 0x00000000,
426 (0x0001 << 16) | (0x30f00 >> 2),
427 0x00000000,
428 (0x0000 << 16) | (0x30f04 >> 2),
429 0x00000000,
430 (0x0001 << 16) | (0x30f04 >> 2),
431 0x00000000,
432 (0x0000 << 16) | (0x30f08 >> 2),
433 0x00000000,
434 (0x0001 << 16) | (0x30f08 >> 2),
435 0x00000000,
436 (0x0000 << 16) | (0x30f0c >> 2),
437 0x00000000,
438 (0x0001 << 16) | (0x30f0c >> 2),
439 0x00000000,
440 (0x0600 << 16) | (0x9b7c >> 2),
441 0x00000000,
442 (0x0e00 << 16) | (0x8a14 >> 2),
443 0x00000000,
444 (0x0e00 << 16) | (0x8a18 >> 2),
445 0x00000000,
446 (0x0600 << 16) | (0x30a00 >> 2),
447 0x00000000,
448 (0x0e00 << 16) | (0x8bf0 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x8bcc >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x8b24 >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x30a04 >> 2),
455 0x00000000,
456 (0x0600 << 16) | (0x30a10 >> 2),
457 0x00000000,
458 (0x0600 << 16) | (0x30a14 >> 2),
459 0x00000000,
460 (0x0600 << 16) | (0x30a18 >> 2),
461 0x00000000,
462 (0x0600 << 16) | (0x30a2c >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0xc700 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0xc704 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0xc708 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0xc768 >> 2),
471 0x00000000,
472 (0x0400 << 16) | (0xc770 >> 2),
473 0x00000000,
474 (0x0400 << 16) | (0xc774 >> 2),
475 0x00000000,
476 (0x0400 << 16) | (0xc778 >> 2),
477 0x00000000,
478 (0x0400 << 16) | (0xc77c >> 2),
479 0x00000000,
480 (0x0400 << 16) | (0xc780 >> 2),
481 0x00000000,
482 (0x0400 << 16) | (0xc784 >> 2),
483 0x00000000,
484 (0x0400 << 16) | (0xc788 >> 2),
485 0x00000000,
486 (0x0400 << 16) | (0xc78c >> 2),
487 0x00000000,
488 (0x0400 << 16) | (0xc798 >> 2),
489 0x00000000,
490 (0x0400 << 16) | (0xc79c >> 2),
491 0x00000000,
492 (0x0400 << 16) | (0xc7a0 >> 2),
493 0x00000000,
494 (0x0400 << 16) | (0xc7a4 >> 2),
495 0x00000000,
496 (0x0400 << 16) | (0xc7a8 >> 2),
497 0x00000000,
498 (0x0400 << 16) | (0xc7ac >> 2),
499 0x00000000,
500 (0x0400 << 16) | (0xc7b0 >> 2),
501 0x00000000,
502 (0x0400 << 16) | (0xc7b4 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0x9100 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0x3c010 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0x92a8 >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0x92ac >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x92b4 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x92b8 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0x92bc >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x92c0 >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0x92c4 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0x92c8 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0x92cc >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0x92d0 >> 2),
527 0x00000000,
528 (0x0e00 << 16) | (0x8c00 >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0x8c04 >> 2),
531 0x00000000,
532 (0x0e00 << 16) | (0x8c20 >> 2),
533 0x00000000,
534 (0x0e00 << 16) | (0x8c38 >> 2),
535 0x00000000,
536 (0x0e00 << 16) | (0x8c3c >> 2),
537 0x00000000,
538 (0x0e00 << 16) | (0xae00 >> 2),
539 0x00000000,
540 (0x0e00 << 16) | (0x9604 >> 2),
541 0x00000000,
542 (0x0e00 << 16) | (0xac08 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0xac0c >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0xac10 >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0xac14 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0xac58 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0xac68 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0xac6c >> 2),
555 0x00000000,
556 (0x0e00 << 16) | (0xac70 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0xac74 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0xac78 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0xac7c >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0xac80 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0xac84 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0xac88 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0xac8c >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x970c >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x9714 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x9718 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x971c >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x31068 >> 2),
581 0x00000000,
582 (0x4e00 << 16) | (0x31068 >> 2),
583 0x00000000,
584 (0x5e00 << 16) | (0x31068 >> 2),
585 0x00000000,
586 (0x6e00 << 16) | (0x31068 >> 2),
587 0x00000000,
588 (0x7e00 << 16) | (0x31068 >> 2),
589 0x00000000,
590 (0x8e00 << 16) | (0x31068 >> 2),
591 0x00000000,
592 (0x9e00 << 16) | (0x31068 >> 2),
593 0x00000000,
594 (0xae00 << 16) | (0x31068 >> 2),
595 0x00000000,
596 (0xbe00 << 16) | (0x31068 >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0xcd10 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0xcd14 >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0x88b0 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0x88b4 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0x88b8 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0x88bc >> 2),
609 0x00000000,
610 (0x0400 << 16) | (0x89c0 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0x88c4 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0x88c8 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0x88d0 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0x88d4 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0x88d8 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0x8980 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x30938 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x3093c >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x30940 >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0x89a0 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x30900 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x30904 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x89b4 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x3c210 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x3c214 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x3c218 >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x8904 >> 2),
645 0x00000000,
646 0x5,
647 (0x0e00 << 16) | (0x8c28 >> 2),
648 (0x0e00 << 16) | (0x8c2c >> 2),
649 (0x0e00 << 16) | (0x8c30 >> 2),
650 (0x0e00 << 16) | (0x8c34 >> 2),
651 (0x0e00 << 16) | (0x9600 >> 2),
652};
653
654static const u32 kalindi_rlc_save_restore_register_list[] =
655{
656 (0x0e00 << 16) | (0xc12c >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0xc140 >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0xc150 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0xc15c >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0xc168 >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0xc170 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0xc204 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0xc2b4 >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0xc2b8 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0xc2bc >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0xc2c0 >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0x8228 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0x829c >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0x869c >> 2),
683 0x00000000,
684 (0x0600 << 16) | (0x98f4 >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0x98f8 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0x9900 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0xc260 >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0x90e8 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x3c000 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x3c00c >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x8c1c >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x9700 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0xcd20 >> 2),
703 0x00000000,
704 (0x4e00 << 16) | (0xcd20 >> 2),
705 0x00000000,
706 (0x5e00 << 16) | (0xcd20 >> 2),
707 0x00000000,
708 (0x6e00 << 16) | (0xcd20 >> 2),
709 0x00000000,
710 (0x7e00 << 16) | (0xcd20 >> 2),
711 0x00000000,
712 (0x0e00 << 16) | (0x89bc >> 2),
713 0x00000000,
714 (0x0e00 << 16) | (0x8900 >> 2),
715 0x00000000,
716 0x3,
717 (0x0e00 << 16) | (0xc130 >> 2),
718 0x00000000,
719 (0x0e00 << 16) | (0xc134 >> 2),
720 0x00000000,
721 (0x0e00 << 16) | (0xc1fc >> 2),
722 0x00000000,
723 (0x0e00 << 16) | (0xc208 >> 2),
724 0x00000000,
725 (0x0e00 << 16) | (0xc264 >> 2),
726 0x00000000,
727 (0x0e00 << 16) | (0xc268 >> 2),
728 0x00000000,
729 (0x0e00 << 16) | (0xc26c >> 2),
730 0x00000000,
731 (0x0e00 << 16) | (0xc270 >> 2),
732 0x00000000,
733 (0x0e00 << 16) | (0xc274 >> 2),
734 0x00000000,
735 (0x0e00 << 16) | (0xc28c >> 2),
736 0x00000000,
737 (0x0e00 << 16) | (0xc290 >> 2),
738 0x00000000,
739 (0x0e00 << 16) | (0xc294 >> 2),
740 0x00000000,
741 (0x0e00 << 16) | (0xc298 >> 2),
742 0x00000000,
743 (0x0e00 << 16) | (0xc2a0 >> 2),
744 0x00000000,
745 (0x0e00 << 16) | (0xc2a4 >> 2),
746 0x00000000,
747 (0x0e00 << 16) | (0xc2a8 >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0xc2ac >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0x301d0 >> 2),
752 0x00000000,
753 (0x0e00 << 16) | (0x30238 >> 2),
754 0x00000000,
755 (0x0e00 << 16) | (0x30250 >> 2),
756 0x00000000,
757 (0x0e00 << 16) | (0x30254 >> 2),
758 0x00000000,
759 (0x0e00 << 16) | (0x30258 >> 2),
760 0x00000000,
761 (0x0e00 << 16) | (0x3025c >> 2),
762 0x00000000,
763 (0x4e00 << 16) | (0xc900 >> 2),
764 0x00000000,
765 (0x5e00 << 16) | (0xc900 >> 2),
766 0x00000000,
767 (0x6e00 << 16) | (0xc900 >> 2),
768 0x00000000,
769 (0x7e00 << 16) | (0xc900 >> 2),
770 0x00000000,
771 (0x4e00 << 16) | (0xc904 >> 2),
772 0x00000000,
773 (0x5e00 << 16) | (0xc904 >> 2),
774 0x00000000,
775 (0x6e00 << 16) | (0xc904 >> 2),
776 0x00000000,
777 (0x7e00 << 16) | (0xc904 >> 2),
778 0x00000000,
779 (0x4e00 << 16) | (0xc908 >> 2),
780 0x00000000,
781 (0x5e00 << 16) | (0xc908 >> 2),
782 0x00000000,
783 (0x6e00 << 16) | (0xc908 >> 2),
784 0x00000000,
785 (0x7e00 << 16) | (0xc908 >> 2),
786 0x00000000,
787 (0x4e00 << 16) | (0xc90c >> 2),
788 0x00000000,
789 (0x5e00 << 16) | (0xc90c >> 2),
790 0x00000000,
791 (0x6e00 << 16) | (0xc90c >> 2),
792 0x00000000,
793 (0x7e00 << 16) | (0xc90c >> 2),
794 0x00000000,
795 (0x4e00 << 16) | (0xc910 >> 2),
796 0x00000000,
797 (0x5e00 << 16) | (0xc910 >> 2),
798 0x00000000,
799 (0x6e00 << 16) | (0xc910 >> 2),
800 0x00000000,
801 (0x7e00 << 16) | (0xc910 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0xc99c >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0x9834 >> 2),
806 0x00000000,
807 (0x0000 << 16) | (0x30f00 >> 2),
808 0x00000000,
809 (0x0000 << 16) | (0x30f04 >> 2),
810 0x00000000,
811 (0x0000 << 16) | (0x30f08 >> 2),
812 0x00000000,
813 (0x0000 << 16) | (0x30f0c >> 2),
814 0x00000000,
815 (0x0600 << 16) | (0x9b7c >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x8a14 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0x8a18 >> 2),
820 0x00000000,
821 (0x0600 << 16) | (0x30a00 >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0x8bf0 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0x8bcc >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0x8b24 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0x30a04 >> 2),
830 0x00000000,
831 (0x0600 << 16) | (0x30a10 >> 2),
832 0x00000000,
833 (0x0600 << 16) | (0x30a14 >> 2),
834 0x00000000,
835 (0x0600 << 16) | (0x30a18 >> 2),
836 0x00000000,
837 (0x0600 << 16) | (0x30a2c >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0xc700 >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0xc704 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0xc708 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0xc768 >> 2),
846 0x00000000,
847 (0x0400 << 16) | (0xc770 >> 2),
848 0x00000000,
849 (0x0400 << 16) | (0xc774 >> 2),
850 0x00000000,
851 (0x0400 << 16) | (0xc798 >> 2),
852 0x00000000,
853 (0x0400 << 16) | (0xc79c >> 2),
854 0x00000000,
855 (0x0e00 << 16) | (0x9100 >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0x3c010 >> 2),
858 0x00000000,
859 (0x0e00 << 16) | (0x8c00 >> 2),
860 0x00000000,
861 (0x0e00 << 16) | (0x8c04 >> 2),
862 0x00000000,
863 (0x0e00 << 16) | (0x8c20 >> 2),
864 0x00000000,
865 (0x0e00 << 16) | (0x8c38 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0x8c3c >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0xae00 >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0x9604 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0xac08 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0xac0c >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0xac10 >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0xac14 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0xac58 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0xac68 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0xac6c >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0xac70 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0xac74 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0xac78 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0xac7c >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0xac80 >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0xac84 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0xac88 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0xac8c >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x970c >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x9714 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x9718 >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x971c >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x31068 >> 2),
912 0x00000000,
913 (0x4e00 << 16) | (0x31068 >> 2),
914 0x00000000,
915 (0x5e00 << 16) | (0x31068 >> 2),
916 0x00000000,
917 (0x6e00 << 16) | (0x31068 >> 2),
918 0x00000000,
919 (0x7e00 << 16) | (0x31068 >> 2),
920 0x00000000,
921 (0x0e00 << 16) | (0xcd10 >> 2),
922 0x00000000,
923 (0x0e00 << 16) | (0xcd14 >> 2),
924 0x00000000,
925 (0x0e00 << 16) | (0x88b0 >> 2),
926 0x00000000,
927 (0x0e00 << 16) | (0x88b4 >> 2),
928 0x00000000,
929 (0x0e00 << 16) | (0x88b8 >> 2),
930 0x00000000,
931 (0x0e00 << 16) | (0x88bc >> 2),
932 0x00000000,
933 (0x0400 << 16) | (0x89c0 >> 2),
934 0x00000000,
935 (0x0e00 << 16) | (0x88c4 >> 2),
936 0x00000000,
937 (0x0e00 << 16) | (0x88c8 >> 2),
938 0x00000000,
939 (0x0e00 << 16) | (0x88d0 >> 2),
940 0x00000000,
941 (0x0e00 << 16) | (0x88d4 >> 2),
942 0x00000000,
943 (0x0e00 << 16) | (0x88d8 >> 2),
944 0x00000000,
945 (0x0e00 << 16) | (0x8980 >> 2),
946 0x00000000,
947 (0x0e00 << 16) | (0x30938 >> 2),
948 0x00000000,
949 (0x0e00 << 16) | (0x3093c >> 2),
950 0x00000000,
951 (0x0e00 << 16) | (0x30940 >> 2),
952 0x00000000,
953 (0x0e00 << 16) | (0x89a0 >> 2),
954 0x00000000,
955 (0x0e00 << 16) | (0x30900 >> 2),
956 0x00000000,
957 (0x0e00 << 16) | (0x30904 >> 2),
958 0x00000000,
959 (0x0e00 << 16) | (0x89b4 >> 2),
960 0x00000000,
961 (0x0e00 << 16) | (0x3e1fc >> 2),
962 0x00000000,
963 (0x0e00 << 16) | (0x3c210 >> 2),
964 0x00000000,
965 (0x0e00 << 16) | (0x3c214 >> 2),
966 0x00000000,
967 (0x0e00 << 16) | (0x3c218 >> 2),
968 0x00000000,
969 (0x0e00 << 16) | (0x8904 >> 2),
970 0x00000000,
971 0x5,
972 (0x0e00 << 16) | (0x8c28 >> 2),
973 (0x0e00 << 16) | (0x8c2c >> 2),
974 (0x0e00 << 16) | (0x8c30 >> 2),
975 (0x0e00 << 16) | (0x8c34 >> 2),
976 (0x0e00 << 16) | (0x9600 >> 2),
977};
978
0aafd313
AD		 979static const u32 bonaire_golden_spm_registers[] =
980{
981 0x30800, 0xe0ffffff, 0xe0000000
982};
983
984static const u32 bonaire_golden_common_registers[] =
985{
986 0xc770, 0xffffffff, 0x00000800,
987 0xc774, 0xffffffff, 0x00000800,
988 0xc798, 0xffffffff, 0x00007fbf,
989 0xc79c, 0xffffffff, 0x00007faf
990};
991
992static const u32 bonaire_golden_registers[] =
993{
994 0x3354, 0x00000333, 0x00000333,
995 0x3350, 0x000c0fc0, 0x00040200,
996 0x9a10, 0x00010000, 0x00058208,
997 0x3c000, 0xffff1fff, 0x00140000,
998 0x3c200, 0xfdfc0fff, 0x00000100,
999 0x3c234, 0x40000000, 0x40000200,
1000 0x9830, 0xffffffff, 0x00000000,
1001 0x9834, 0xf00fffff, 0x00000400,
1002 0x9838, 0x0002021c, 0x00020200,
1003 0xc78, 0x00000080, 0x00000000,
1004 0x5bb0, 0x000000f0, 0x00000070,
1005 0x5bc0, 0xf0311fff, 0x80300000,
1006 0x98f8, 0x73773777, 0x12010001,
1007 0x350c, 0x00810000, 0x408af000,
1008 0x7030, 0x31000111, 0x00000011,
1009 0x2f48, 0x73773777, 0x12010001,
1010 0x220c, 0x00007fb6, 0x0021a1b1,
1011 0x2210, 0x00007fb6, 0x002021b1,
1012 0x2180, 0x00007fb6, 0x00002191,
1013 0x2218, 0x00007fb6, 0x002121b1,
1014 0x221c, 0x00007fb6, 0x002021b1,
1015 0x21dc, 0x00007fb6, 0x00002191,
1016 0x21e0, 0x00007fb6, 0x00002191,
1017 0x3628, 0x0000003f, 0x0000000a,
1018 0x362c, 0x0000003f, 0x0000000a,
1019 0x2ae4, 0x00073ffe, 0x000022a2,
1020 0x240c, 0x000007ff, 0x00000000,
1021 0x8a14, 0xf000003f, 0x00000007,
1022 0x8bf0, 0x00002001, 0x00000001,
1023 0x8b24, 0xffffffff, 0x00ffffff,
1024 0x30a04, 0x0000ff0f, 0x00000000,
1025 0x28a4c, 0x07ffffff, 0x06000000,
1026 0x4d8, 0x00000fff, 0x00000100,
1027 0x3e78, 0x00000001, 0x00000002,
1028 0x9100, 0x03000000, 0x0362c688,
1029 0x8c00, 0x000000ff, 0x00000001,
1030 0xe40, 0x00001fff, 0x00001fff,
1031 0x9060, 0x0000007f, 0x00000020,
1032 0x9508, 0x00010000, 0x00010000,
1033 0xac14, 0x000003ff, 0x000000f3,
1034 0xac0c, 0xffffffff, 0x00001032
1035};
1036
1037static const u32 bonaire_mgcg_cgcg_init[] =
1038{
1039 0xc420, 0xffffffff, 0xfffffffc,
1040 0x30800, 0xffffffff, 0xe0000000,
1041 0x3c2a0, 0xffffffff, 0x00000100,
1042 0x3c208, 0xffffffff, 0x00000100,
1043 0x3c2c0, 0xffffffff, 0xc0000100,
1044 0x3c2c8, 0xffffffff, 0xc0000100,
1045 0x3c2c4, 0xffffffff, 0xc0000100,
1046 0x55e4, 0xffffffff, 0x00600100,
1047 0x3c280, 0xffffffff, 0x00000100,
1048 0x3c214, 0xffffffff, 0x06000100,
1049 0x3c220, 0xffffffff, 0x00000100,
1050 0x3c218, 0xffffffff, 0x06000100,
1051 0x3c204, 0xffffffff, 0x00000100,
1052 0x3c2e0, 0xffffffff, 0x00000100,
1053 0x3c224, 0xffffffff, 0x00000100,
1054 0x3c200, 0xffffffff, 0x00000100,
1055 0x3c230, 0xffffffff, 0x00000100,
1056 0x3c234, 0xffffffff, 0x00000100,
1057 0x3c250, 0xffffffff, 0x00000100,
1058 0x3c254, 0xffffffff, 0x00000100,
1059 0x3c258, 0xffffffff, 0x00000100,
1060 0x3c25c, 0xffffffff, 0x00000100,
1061 0x3c260, 0xffffffff, 0x00000100,
1062 0x3c27c, 0xffffffff, 0x00000100,
1063 0x3c278, 0xffffffff, 0x00000100,
1064 0x3c210, 0xffffffff, 0x06000100,
1065 0x3c290, 0xffffffff, 0x00000100,
1066 0x3c274, 0xffffffff, 0x00000100,
1067 0x3c2b4, 0xffffffff, 0x00000100,
1068 0x3c2b0, 0xffffffff, 0x00000100,
1069 0x3c270, 0xffffffff, 0x00000100,
1070 0x30800, 0xffffffff, 0xe0000000,
1071 0x3c020, 0xffffffff, 0x00010000,
1072 0x3c024, 0xffffffff, 0x00030002,
1073 0x3c028, 0xffffffff, 0x00040007,
1074 0x3c02c, 0xffffffff, 0x00060005,
1075 0x3c030, 0xffffffff, 0x00090008,
1076 0x3c034, 0xffffffff, 0x00010000,
1077 0x3c038, 0xffffffff, 0x00030002,
1078 0x3c03c, 0xffffffff, 0x00040007,
1079 0x3c040, 0xffffffff, 0x00060005,
1080 0x3c044, 0xffffffff, 0x00090008,
1081 0x3c048, 0xffffffff, 0x00010000,
1082 0x3c04c, 0xffffffff, 0x00030002,
1083 0x3c050, 0xffffffff, 0x00040007,
1084 0x3c054, 0xffffffff, 0x00060005,
1085 0x3c058, 0xffffffff, 0x00090008,
1086 0x3c05c, 0xffffffff, 0x00010000,
1087 0x3c060, 0xffffffff, 0x00030002,
1088 0x3c064, 0xffffffff, 0x00040007,
1089 0x3c068, 0xffffffff, 0x00060005,
1090 0x3c06c, 0xffffffff, 0x00090008,
1091 0x3c070, 0xffffffff, 0x00010000,
1092 0x3c074, 0xffffffff, 0x00030002,
1093 0x3c078, 0xffffffff, 0x00040007,
1094 0x3c07c, 0xffffffff, 0x00060005,
1095 0x3c080, 0xffffffff, 0x00090008,
1096 0x3c084, 0xffffffff, 0x00010000,
1097 0x3c088, 0xffffffff, 0x00030002,
1098 0x3c08c, 0xffffffff, 0x00040007,
1099 0x3c090, 0xffffffff, 0x00060005,
1100 0x3c094, 0xffffffff, 0x00090008,
1101 0x3c098, 0xffffffff, 0x00010000,
1102 0x3c09c, 0xffffffff, 0x00030002,
1103 0x3c0a0, 0xffffffff, 0x00040007,
1104 0x3c0a4, 0xffffffff, 0x00060005,
1105 0x3c0a8, 0xffffffff, 0x00090008,
1106 0x3c000, 0xffffffff, 0x96e00200,
1107 0x8708, 0xffffffff, 0x00900100,
1108 0xc424, 0xffffffff, 0x0020003f,
1109 0x38, 0xffffffff, 0x0140001c,
1110 0x3c, 0x000f0000, 0x000f0000,
1111 0x220, 0xffffffff, 0xC060000C,
1112 0x224, 0xc0000fff, 0x00000100,
1113 0xf90, 0xffffffff, 0x00000100,
1114 0xf98, 0x00000101, 0x00000000,
1115 0x20a8, 0xffffffff, 0x00000104,
1116 0x55e4, 0xff000fff, 0x00000100,
1117 0x30cc, 0xc0000fff, 0x00000104,
1118 0xc1e4, 0x00000001, 0x00000001,
1119 0xd00c, 0xff000ff0, 0x00000100,
1120 0xd80c, 0xff000ff0, 0x00000100
1121};
1122
1123static const u32 spectre_golden_spm_registers[] =
1124{
1125 0x30800, 0xe0ffffff, 0xe0000000
1126};
1127
1128static const u32 spectre_golden_common_registers[] =
1129{
1130 0xc770, 0xffffffff, 0x00000800,
1131 0xc774, 0xffffffff, 0x00000800,
1132 0xc798, 0xffffffff, 0x00007fbf,
1133 0xc79c, 0xffffffff, 0x00007faf
1134};
1135
1136static const u32 spectre_golden_registers[] =
1137{
1138 0x3c000, 0xffff1fff, 0x96940200,
1139 0x3c00c, 0xffff0001, 0xff000000,
1140 0x3c200, 0xfffc0fff, 0x00000100,
1141 0x6ed8, 0x00010101, 0x00010000,
1142 0x9834, 0xf00fffff, 0x00000400,
1143 0x9838, 0xfffffffc, 0x00020200,
1144 0x5bb0, 0x000000f0, 0x00000070,
1145 0x5bc0, 0xf0311fff, 0x80300000,
1146 0x98f8, 0x73773777, 0x12010001,
1147 0x9b7c, 0x00ff0000, 0x00fc0000,
1148 0x2f48, 0x73773777, 0x12010001,
1149 0x8a14, 0xf000003f, 0x00000007,
1150 0x8b24, 0xffffffff, 0x00ffffff,
1151 0x28350, 0x3f3f3fff, 0x00000082,
f1553174 1152 0x28354, 0x0000003f, 0x00000000,
0aafd313
AD		 1153 0x3e78, 0x00000001, 0x00000002,
1154 0x913c, 0xffff03df, 0x00000004,
1155 0xc768, 0x00000008, 0x00000008,
1156 0x8c00, 0x000008ff, 0x00000800,
1157 0x9508, 0x00010000, 0x00010000,
1158 0xac0c, 0xffffffff, 0x54763210,
1159 0x214f8, 0x01ff01ff, 0x00000002,
1160 0x21498, 0x007ff800, 0x00200000,
1161 0x2015c, 0xffffffff, 0x00000f40,
1162 0x30934, 0xffffffff, 0x00000001
1163};
1164
1165static const u32 spectre_mgcg_cgcg_init[] =
1166{
1167 0xc420, 0xffffffff, 0xfffffffc,
1168 0x30800, 0xffffffff, 0xe0000000,
1169 0x3c2a0, 0xffffffff, 0x00000100,
1170 0x3c208, 0xffffffff, 0x00000100,
1171 0x3c2c0, 0xffffffff, 0x00000100,
1172 0x3c2c8, 0xffffffff, 0x00000100,
1173 0x3c2c4, 0xffffffff, 0x00000100,
1174 0x55e4, 0xffffffff, 0x00600100,
1175 0x3c280, 0xffffffff, 0x00000100,
1176 0x3c214, 0xffffffff, 0x06000100,
1177 0x3c220, 0xffffffff, 0x00000100,
1178 0x3c218, 0xffffffff, 0x06000100,
1179 0x3c204, 0xffffffff, 0x00000100,
1180 0x3c2e0, 0xffffffff, 0x00000100,
1181 0x3c224, 0xffffffff, 0x00000100,
1182 0x3c200, 0xffffffff, 0x00000100,
1183 0x3c230, 0xffffffff, 0x00000100,
1184 0x3c234, 0xffffffff, 0x00000100,
1185 0x3c250, 0xffffffff, 0x00000100,
1186 0x3c254, 0xffffffff, 0x00000100,
1187 0x3c258, 0xffffffff, 0x00000100,
1188 0x3c25c, 0xffffffff, 0x00000100,
1189 0x3c260, 0xffffffff, 0x00000100,
1190 0x3c27c, 0xffffffff, 0x00000100,
1191 0x3c278, 0xffffffff, 0x00000100,
1192 0x3c210, 0xffffffff, 0x06000100,
1193 0x3c290, 0xffffffff, 0x00000100,
1194 0x3c274, 0xffffffff, 0x00000100,
1195 0x3c2b4, 0xffffffff, 0x00000100,
1196 0x3c2b0, 0xffffffff, 0x00000100,
1197 0x3c270, 0xffffffff, 0x00000100,
1198 0x30800, 0xffffffff, 0xe0000000,
1199 0x3c020, 0xffffffff, 0x00010000,
1200 0x3c024, 0xffffffff, 0x00030002,
1201 0x3c028, 0xffffffff, 0x00040007,
1202 0x3c02c, 0xffffffff, 0x00060005,
1203 0x3c030, 0xffffffff, 0x00090008,
1204 0x3c034, 0xffffffff, 0x00010000,
1205 0x3c038, 0xffffffff, 0x00030002,
1206 0x3c03c, 0xffffffff, 0x00040007,
1207 0x3c040, 0xffffffff, 0x00060005,
1208 0x3c044, 0xffffffff, 0x00090008,
1209 0x3c048, 0xffffffff, 0x00010000,
1210 0x3c04c, 0xffffffff, 0x00030002,
1211 0x3c050, 0xffffffff, 0x00040007,
1212 0x3c054, 0xffffffff, 0x00060005,
1213 0x3c058, 0xffffffff, 0x00090008,
1214 0x3c05c, 0xffffffff, 0x00010000,
1215 0x3c060, 0xffffffff, 0x00030002,
1216 0x3c064, 0xffffffff, 0x00040007,
1217 0x3c068, 0xffffffff, 0x00060005,
1218 0x3c06c, 0xffffffff, 0x00090008,
1219 0x3c070, 0xffffffff, 0x00010000,
1220 0x3c074, 0xffffffff, 0x00030002,
1221 0x3c078, 0xffffffff, 0x00040007,
1222 0x3c07c, 0xffffffff, 0x00060005,
1223 0x3c080, 0xffffffff, 0x00090008,
1224 0x3c084, 0xffffffff, 0x00010000,
1225 0x3c088, 0xffffffff, 0x00030002,
1226 0x3c08c, 0xffffffff, 0x00040007,
1227 0x3c090, 0xffffffff, 0x00060005,
1228 0x3c094, 0xffffffff, 0x00090008,
1229 0x3c098, 0xffffffff, 0x00010000,
1230 0x3c09c, 0xffffffff, 0x00030002,
1231 0x3c0a0, 0xffffffff, 0x00040007,
1232 0x3c0a4, 0xffffffff, 0x00060005,
1233 0x3c0a8, 0xffffffff, 0x00090008,
1234 0x3c0ac, 0xffffffff, 0x00010000,
1235 0x3c0b0, 0xffffffff, 0x00030002,
1236 0x3c0b4, 0xffffffff, 0x00040007,
1237 0x3c0b8, 0xffffffff, 0x00060005,
1238 0x3c0bc, 0xffffffff, 0x00090008,
1239 0x3c000, 0xffffffff, 0x96e00200,
1240 0x8708, 0xffffffff, 0x00900100,
1241 0xc424, 0xffffffff, 0x0020003f,
1242 0x38, 0xffffffff, 0x0140001c,
1243 0x3c, 0x000f0000, 0x000f0000,
1244 0x220, 0xffffffff, 0xC060000C,
1245 0x224, 0xc0000fff, 0x00000100,
1246 0xf90, 0xffffffff, 0x00000100,
1247 0xf98, 0x00000101, 0x00000000,
1248 0x20a8, 0xffffffff, 0x00000104,
1249 0x55e4, 0xff000fff, 0x00000100,
1250 0x30cc, 0xc0000fff, 0x00000104,
1251 0xc1e4, 0x00000001, 0x00000001,
1252 0xd00c, 0xff000ff0, 0x00000100,
1253 0xd80c, 0xff000ff0, 0x00000100
1254};
1255
1256static const u32 kalindi_golden_spm_registers[] =
1257{
1258 0x30800, 0xe0ffffff, 0xe0000000
1259};
1260
1261static const u32 kalindi_golden_common_registers[] =
1262{
1263 0xc770, 0xffffffff, 0x00000800,
1264 0xc774, 0xffffffff, 0x00000800,
1265 0xc798, 0xffffffff, 0x00007fbf,
1266 0xc79c, 0xffffffff, 0x00007faf
1267};
1268
1269static const u32 kalindi_golden_registers[] =
1270{
1271 0x3c000, 0xffffdfff, 0x6e944040,
1272 0x55e4, 0xff607fff, 0xfc000100,
1273 0x3c220, 0xff000fff, 0x00000100,
1274 0x3c224, 0xff000fff, 0x00000100,
1275 0x3c200, 0xfffc0fff, 0x00000100,
1276 0x6ed8, 0x00010101, 0x00010000,
1277 0x9830, 0xffffffff, 0x00000000,
1278 0x9834, 0xf00fffff, 0x00000400,
1279 0x5bb0, 0x000000f0, 0x00000070,
1280 0x5bc0, 0xf0311fff, 0x80300000,
1281 0x98f8, 0x73773777, 0x12010001,
1282 0x98fc, 0xffffffff, 0x00000010,
1283 0x9b7c, 0x00ff0000, 0x00fc0000,
1284 0x8030, 0x00001f0f, 0x0000100a,
1285 0x2f48, 0x73773777, 0x12010001,
1286 0x2408, 0x000fffff, 0x000c007f,
1287 0x8a14, 0xf000003f, 0x00000007,
1288 0x8b24, 0x3fff3fff, 0x00ffcfff,
1289 0x30a04, 0x0000ff0f, 0x00000000,
1290 0x28a4c, 0x07ffffff, 0x06000000,
1291 0x4d8, 0x00000fff, 0x00000100,
1292 0x3e78, 0x00000001, 0x00000002,
1293 0xc768, 0x00000008, 0x00000008,
1294 0x8c00, 0x000000ff, 0x00000003,
1295 0x214f8, 0x01ff01ff, 0x00000002,
1296 0x21498, 0x007ff800, 0x00200000,
1297 0x2015c, 0xffffffff, 0x00000f40,
1298 0x88c4, 0x001f3ae3, 0x00000082,
1299 0x88d4, 0x0000001f, 0x00000010,
1300 0x30934, 0xffffffff, 0x00000000
1301};
1302
1303static const u32 kalindi_mgcg_cgcg_init[] =
1304{
1305 0xc420, 0xffffffff, 0xfffffffc,
1306 0x30800, 0xffffffff, 0xe0000000,
1307 0x3c2a0, 0xffffffff, 0x00000100,
1308 0x3c208, 0xffffffff, 0x00000100,
1309 0x3c2c0, 0xffffffff, 0x00000100,
1310 0x3c2c8, 0xffffffff, 0x00000100,
1311 0x3c2c4, 0xffffffff, 0x00000100,
1312 0x55e4, 0xffffffff, 0x00600100,
1313 0x3c280, 0xffffffff, 0x00000100,
1314 0x3c214, 0xffffffff, 0x06000100,
1315 0x3c220, 0xffffffff, 0x00000100,
1316 0x3c218, 0xffffffff, 0x06000100,
1317 0x3c204, 0xffffffff, 0x00000100,
1318 0x3c2e0, 0xffffffff, 0x00000100,
1319 0x3c224, 0xffffffff, 0x00000100,
1320 0x3c200, 0xffffffff, 0x00000100,
1321 0x3c230, 0xffffffff, 0x00000100,
1322 0x3c234, 0xffffffff, 0x00000100,
1323 0x3c250, 0xffffffff, 0x00000100,
1324 0x3c254, 0xffffffff, 0x00000100,
1325 0x3c258, 0xffffffff, 0x00000100,
1326 0x3c25c, 0xffffffff, 0x00000100,
1327 0x3c260, 0xffffffff, 0x00000100,
1328 0x3c27c, 0xffffffff, 0x00000100,
1329 0x3c278, 0xffffffff, 0x00000100,
1330 0x3c210, 0xffffffff, 0x06000100,
1331 0x3c290, 0xffffffff, 0x00000100,
1332 0x3c274, 0xffffffff, 0x00000100,
1333 0x3c2b4, 0xffffffff, 0x00000100,
1334 0x3c2b0, 0xffffffff, 0x00000100,
1335 0x3c270, 0xffffffff, 0x00000100,
1336 0x30800, 0xffffffff, 0xe0000000,
1337 0x3c020, 0xffffffff, 0x00010000,
1338 0x3c024, 0xffffffff, 0x00030002,
1339 0x3c028, 0xffffffff, 0x00040007,
1340 0x3c02c, 0xffffffff, 0x00060005,
1341 0x3c030, 0xffffffff, 0x00090008,
1342 0x3c034, 0xffffffff, 0x00010000,
1343 0x3c038, 0xffffffff, 0x00030002,
1344 0x3c03c, 0xffffffff, 0x00040007,
1345 0x3c040, 0xffffffff, 0x00060005,
1346 0x3c044, 0xffffffff, 0x00090008,
1347 0x3c000, 0xffffffff, 0x96e00200,
1348 0x8708, 0xffffffff, 0x00900100,
1349 0xc424, 0xffffffff, 0x0020003f,
1350 0x38, 0xffffffff, 0x0140001c,
1351 0x3c, 0x000f0000, 0x000f0000,
1352 0x220, 0xffffffff, 0xC060000C,
1353 0x224, 0xc0000fff, 0x00000100,
1354 0x20a8, 0xffffffff, 0x00000104,
1355 0x55e4, 0xff000fff, 0x00000100,
1356 0x30cc, 0xc0000fff, 0x00000104,
1357 0xc1e4, 0x00000001, 0x00000001,
1358 0xd00c, 0xff000ff0, 0x00000100,
1359 0xd80c, 0xff000ff0, 0x00000100
1360};
1361
8efff337
AD		 1362static const u32 hawaii_golden_spm_registers[] =
1363{
1364 0x30800, 0xe0ffffff, 0xe0000000
1365};
1366
1367static const u32 hawaii_golden_common_registers[] =
1368{
1369 0x30800, 0xffffffff, 0xe0000000,
1370 0x28350, 0xffffffff, 0x3a00161a,
1371 0x28354, 0xffffffff, 0x0000002e,
1372 0x9a10, 0xffffffff, 0x00018208,
1373 0x98f8, 0xffffffff, 0x12011003
1374};
1375
1376static const u32 hawaii_golden_registers[] =
1377{
1378 0x3354, 0x00000333, 0x00000333,
1379 0x9a10, 0x00010000, 0x00058208,
1380 0x9830, 0xffffffff, 0x00000000,
1381 0x9834, 0xf00fffff, 0x00000400,
1382 0x9838, 0x0002021c, 0x00020200,
1383 0xc78, 0x00000080, 0x00000000,
1384 0x5bb0, 0x000000f0, 0x00000070,
1385 0x5bc0, 0xf0311fff, 0x80300000,
1386 0x350c, 0x00810000, 0x408af000,
1387 0x7030, 0x31000111, 0x00000011,
1388 0x2f48, 0x73773777, 0x12010001,
1389 0x2120, 0x0000007f, 0x0000001b,
1390 0x21dc, 0x00007fb6, 0x00002191,
1391 0x3628, 0x0000003f, 0x0000000a,
1392 0x362c, 0x0000003f, 0x0000000a,
1393 0x2ae4, 0x00073ffe, 0x000022a2,
1394 0x240c, 0x000007ff, 0x00000000,
1395 0x8bf0, 0x00002001, 0x00000001,
1396 0x8b24, 0xffffffff, 0x00ffffff,
1397 0x30a04, 0x0000ff0f, 0x00000000,
1398 0x28a4c, 0x07ffffff, 0x06000000,
1399 0x3e78, 0x00000001, 0x00000002,
1400 0xc768, 0x00000008, 0x00000008,
1401 0xc770, 0x00000f00, 0x00000800,
1402 0xc774, 0x00000f00, 0x00000800,
1403 0xc798, 0x00ffffff, 0x00ff7fbf,
1404 0xc79c, 0x00ffffff, 0x00ff7faf,
1405 0x8c00, 0x000000ff, 0x00000800,
1406 0xe40, 0x00001fff, 0x00001fff,
1407 0x9060, 0x0000007f, 0x00000020,
1408 0x9508, 0x00010000, 0x00010000,
1409 0xae00, 0x00100000, 0x000ff07c,
1410 0xac14, 0x000003ff, 0x0000000f,
1411 0xac10, 0xffffffff, 0x7564fdec,
1412 0xac0c, 0xffffffff, 0x3120b9a8,
1413 0xac08, 0x20000000, 0x0f9c0000
1414};
1415
1416static const u32 hawaii_mgcg_cgcg_init[] =
1417{
1418 0xc420, 0xffffffff, 0xfffffffd,
1419 0x30800, 0xffffffff, 0xe0000000,
1420 0x3c2a0, 0xffffffff, 0x00000100,
1421 0x3c208, 0xffffffff, 0x00000100,
1422 0x3c2c0, 0xffffffff, 0x00000100,
1423 0x3c2c8, 0xffffffff, 0x00000100,
1424 0x3c2c4, 0xffffffff, 0x00000100,
1425 0x55e4, 0xffffffff, 0x00200100,
1426 0x3c280, 0xffffffff, 0x00000100,
1427 0x3c214, 0xffffffff, 0x06000100,
1428 0x3c220, 0xffffffff, 0x00000100,
1429 0x3c218, 0xffffffff, 0x06000100,
1430 0x3c204, 0xffffffff, 0x00000100,
1431 0x3c2e0, 0xffffffff, 0x00000100,
1432 0x3c224, 0xffffffff, 0x00000100,
1433 0x3c200, 0xffffffff, 0x00000100,
1434 0x3c230, 0xffffffff, 0x00000100,
1435 0x3c234, 0xffffffff, 0x00000100,
1436 0x3c250, 0xffffffff, 0x00000100,
1437 0x3c254, 0xffffffff, 0x00000100,
1438 0x3c258, 0xffffffff, 0x00000100,
1439 0x3c25c, 0xffffffff, 0x00000100,
1440 0x3c260, 0xffffffff, 0x00000100,
1441 0x3c27c, 0xffffffff, 0x00000100,
1442 0x3c278, 0xffffffff, 0x00000100,
1443 0x3c210, 0xffffffff, 0x06000100,
1444 0x3c290, 0xffffffff, 0x00000100,
1445 0x3c274, 0xffffffff, 0x00000100,
1446 0x3c2b4, 0xffffffff, 0x00000100,
1447 0x3c2b0, 0xffffffff, 0x00000100,
1448 0x3c270, 0xffffffff, 0x00000100,
1449 0x30800, 0xffffffff, 0xe0000000,
1450 0x3c020, 0xffffffff, 0x00010000,
1451 0x3c024, 0xffffffff, 0x00030002,
1452 0x3c028, 0xffffffff, 0x00040007,
1453 0x3c02c, 0xffffffff, 0x00060005,
1454 0x3c030, 0xffffffff, 0x00090008,
1455 0x3c034, 0xffffffff, 0x00010000,
1456 0x3c038, 0xffffffff, 0x00030002,
1457 0x3c03c, 0xffffffff, 0x00040007,
1458 0x3c040, 0xffffffff, 0x00060005,
1459 0x3c044, 0xffffffff, 0x00090008,
1460 0x3c048, 0xffffffff, 0x00010000,
1461 0x3c04c, 0xffffffff, 0x00030002,
1462 0x3c050, 0xffffffff, 0x00040007,
1463 0x3c054, 0xffffffff, 0x00060005,
1464 0x3c058, 0xffffffff, 0x00090008,
1465 0x3c05c, 0xffffffff, 0x00010000,
1466 0x3c060, 0xffffffff, 0x00030002,
1467 0x3c064, 0xffffffff, 0x00040007,
1468 0x3c068, 0xffffffff, 0x00060005,
1469 0x3c06c, 0xffffffff, 0x00090008,
1470 0x3c070, 0xffffffff, 0x00010000,
1471 0x3c074, 0xffffffff, 0x00030002,
1472 0x3c078, 0xffffffff, 0x00040007,
1473 0x3c07c, 0xffffffff, 0x00060005,
1474 0x3c080, 0xffffffff, 0x00090008,
1475 0x3c084, 0xffffffff, 0x00010000,
1476 0x3c088, 0xffffffff, 0x00030002,
1477 0x3c08c, 0xffffffff, 0x00040007,
1478 0x3c090, 0xffffffff, 0x00060005,
1479 0x3c094, 0xffffffff, 0x00090008,
1480 0x3c098, 0xffffffff, 0x00010000,
1481 0x3c09c, 0xffffffff, 0x00030002,
1482 0x3c0a0, 0xffffffff, 0x00040007,
1483 0x3c0a4, 0xffffffff, 0x00060005,
1484 0x3c0a8, 0xffffffff, 0x00090008,
1485 0x3c0ac, 0xffffffff, 0x00010000,
1486 0x3c0b0, 0xffffffff, 0x00030002,
1487 0x3c0b4, 0xffffffff, 0x00040007,
1488 0x3c0b8, 0xffffffff, 0x00060005,
1489 0x3c0bc, 0xffffffff, 0x00090008,
1490 0x3c0c0, 0xffffffff, 0x00010000,
1491 0x3c0c4, 0xffffffff, 0x00030002,
1492 0x3c0c8, 0xffffffff, 0x00040007,
1493 0x3c0cc, 0xffffffff, 0x00060005,
1494 0x3c0d0, 0xffffffff, 0x00090008,
1495 0x3c0d4, 0xffffffff, 0x00010000,
1496 0x3c0d8, 0xffffffff, 0x00030002,
1497 0x3c0dc, 0xffffffff, 0x00040007,
1498 0x3c0e0, 0xffffffff, 0x00060005,
1499 0x3c0e4, 0xffffffff, 0x00090008,
1500 0x3c0e8, 0xffffffff, 0x00010000,
1501 0x3c0ec, 0xffffffff, 0x00030002,
1502 0x3c0f0, 0xffffffff, 0x00040007,
1503 0x3c0f4, 0xffffffff, 0x00060005,
1504 0x3c0f8, 0xffffffff, 0x00090008,
1505 0xc318, 0xffffffff, 0x00020200,
1506 0x3350, 0xffffffff, 0x00000200,
1507 0x15c0, 0xffffffff, 0x00000400,
1508 0x55e8, 0xffffffff, 0x00000000,
1509 0x2f50, 0xffffffff, 0x00000902,
1510 0x3c000, 0xffffffff, 0x96940200,
1511 0x8708, 0xffffffff, 0x00900100,
1512 0xc424, 0xffffffff, 0x0020003f,
1513 0x38, 0xffffffff, 0x0140001c,
1514 0x3c, 0x000f0000, 0x000f0000,
1515 0x220, 0xffffffff, 0xc060000c,
1516 0x224, 0xc0000fff, 0x00000100,
1517 0xf90, 0xffffffff, 0x00000100,
1518 0xf98, 0x00000101, 0x00000000,
1519 0x20a8, 0xffffffff, 0x00000104,
1520 0x55e4, 0xff000fff, 0x00000100,
1521 0x30cc, 0xc0000fff, 0x00000104,
1522 0xc1e4, 0x00000001, 0x00000001,
1523 0xd00c, 0xff000ff0, 0x00000100,
1524 0xd80c, 0xff000ff0, 0x00000100
1525};
1526
f73a9e83
SL		 1527static const u32 godavari_golden_registers[] =
1528{
1529 0x55e4, 0xff607fff, 0xfc000100,
1530 0x6ed8, 0x00010101, 0x00010000,
1531 0x9830, 0xffffffff, 0x00000000,
1532 0x98302, 0xf00fffff, 0x00000400,
1533 0x6130, 0xffffffff, 0x00010000,
1534 0x5bb0, 0x000000f0, 0x00000070,
1535 0x5bc0, 0xf0311fff, 0x80300000,
1536 0x98f8, 0x73773777, 0x12010001,
1537 0x98fc, 0xffffffff, 0x00000010,
1538 0x8030, 0x00001f0f, 0x0000100a,
1539 0x2f48, 0x73773777, 0x12010001,
1540 0x2408, 0x000fffff, 0x000c007f,
1541 0x8a14, 0xf000003f, 0x00000007,
1542 0x8b24, 0xffffffff, 0x00ff0fff,
1543 0x30a04, 0x0000ff0f, 0x00000000,
1544 0x28a4c, 0x07ffffff, 0x06000000,
1545 0x4d8, 0x00000fff, 0x00000100,
1546 0xd014, 0x00010000, 0x00810001,
1547 0xd814, 0x00010000, 0x00810001,
1548 0x3e78, 0x00000001, 0x00000002,
1549 0xc768, 0x00000008, 0x00000008,
1550 0xc770, 0x00000f00, 0x00000800,
1551 0xc774, 0x00000f00, 0x00000800,
1552 0xc798, 0x00ffffff, 0x00ff7fbf,
1553 0xc79c, 0x00ffffff, 0x00ff7faf,
1554 0x8c00, 0x000000ff, 0x00000001,
1555 0x214f8, 0x01ff01ff, 0x00000002,
1556 0x21498, 0x007ff800, 0x00200000,
1557 0x2015c, 0xffffffff, 0x00000f40,
1558 0x88c4, 0x001f3ae3, 0x00000082,
1559 0x88d4, 0x0000001f, 0x00000010,
1560 0x30934, 0xffffffff, 0x00000000
1561};
1562
1563
0aafd313
AD		 1564static void cik_init_golden_registers(struct radeon_device *rdev)
1565{
1566 switch (rdev->family) {
1567 case CHIP_BONAIRE:
1568 radeon_program_register_sequence(rdev,
1569 bonaire_mgcg_cgcg_init,
1570 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1571 radeon_program_register_sequence(rdev,
1572 bonaire_golden_registers,
1573 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1574 radeon_program_register_sequence(rdev,
1575 bonaire_golden_common_registers,
1576 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1577 radeon_program_register_sequence(rdev,
1578 bonaire_golden_spm_registers,
1579 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1580 break;
1581 case CHIP_KABINI:
1582 radeon_program_register_sequence(rdev,
1583 kalindi_mgcg_cgcg_init,
1584 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1585 radeon_program_register_sequence(rdev,
1586 kalindi_golden_registers,
1587 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1588 radeon_program_register_sequence(rdev,
1589 kalindi_golden_common_registers,
1590 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1591 radeon_program_register_sequence(rdev,
1592 kalindi_golden_spm_registers,
1593 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1594 break;
f73a9e83
SL		 1595 case CHIP_MULLINS:
1596 radeon_program_register_sequence(rdev,
1597 kalindi_mgcg_cgcg_init,
1598 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1599 radeon_program_register_sequence(rdev,
1600 godavari_golden_registers,
1601 (const u32)ARRAY_SIZE(godavari_golden_registers));
1602 radeon_program_register_sequence(rdev,
1603 kalindi_golden_common_registers,
1604 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1605 radeon_program_register_sequence(rdev,
1606 kalindi_golden_spm_registers,
1607 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1608 break;
0aafd313
AD		 1609 case CHIP_KAVERI:
1610 radeon_program_register_sequence(rdev,
1611 spectre_mgcg_cgcg_init,
1612 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1613 radeon_program_register_sequence(rdev,
1614 spectre_golden_registers,
1615 (const u32)ARRAY_SIZE(spectre_golden_registers));
1616 radeon_program_register_sequence(rdev,
1617 spectre_golden_common_registers,
1618 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1619 radeon_program_register_sequence(rdev,
1620 spectre_golden_spm_registers,
1621 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1622 break;
8efff337
AD		 1623 case CHIP_HAWAII:
1624 radeon_program_register_sequence(rdev,
1625 hawaii_mgcg_cgcg_init,
1626 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1627 radeon_program_register_sequence(rdev,
1628 hawaii_golden_registers,
1629 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1630 radeon_program_register_sequence(rdev,
1631 hawaii_golden_common_registers,
1632 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1633 radeon_program_register_sequence(rdev,
1634 hawaii_golden_spm_registers,
1635 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1636 break;
0aafd313
AD		 1637 default:
1638 break;
1639 }
1640}
1641
2c67912c
AD		 1642/**
1643 * cik_get_xclk - get the xclk
1644 *
1645 * @rdev: radeon_device pointer
1646 *
1647 * Returns the reference clock used by the gfx engine
1648 * (CIK).
1649 */
1650u32 cik_get_xclk(struct radeon_device *rdev)
1651{
1652 u32 reference_clock = rdev->clock.spll.reference_freq;
1653
1654 if (rdev->flags & RADEON_IS_IGP) {
1655 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1656 return reference_clock / 2;
1657 } else {
1658 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1659 return reference_clock / 4;
1660 }
1661 return reference_clock;
1662}
1663
75efdee1
AD		 1664/**
1665 * cik_mm_rdoorbell - read a doorbell dword
1666 *
1667 * @rdev: radeon_device pointer
d5754ab8 1668 * @index: doorbell index
75efdee1
AD		 1669 *
1670 * Returns the value in the doorbell aperture at the
d5754ab8 1671 * requested doorbell index (CIK).
75efdee1 1672 */
d5754ab8 1673u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
75efdee1 1674{
d5754ab8
AL		 1675 if (index < rdev->doorbell.num_doorbells) {
1676 return readl(rdev->doorbell.ptr + index);
75efdee1 1677 } else {
d5754ab8 1678 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1679 return 0;
1680 }
1681}
1682
1683/**
1684 * cik_mm_wdoorbell - write a doorbell dword
1685 *
1686 * @rdev: radeon_device pointer
d5754ab8 1687 * @index: doorbell index
75efdee1
AD		 1688 * @v: value to write
1689 *
1690 * Writes @v to the doorbell aperture at the
d5754ab8 1691 * requested doorbell index (CIK).
75efdee1 1692 */
d5754ab8 1693void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
75efdee1 1694{
d5754ab8
AL		 1695 if (index < rdev->doorbell.num_doorbells) {
1696 writel(v, rdev->doorbell.ptr + index);
75efdee1 1697 } else {
d5754ab8 1698 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1699 }
1700}
1701
bc8273fe
AD		 1702#define BONAIRE_IO_MC_REGS_SIZE 36
1703
1704static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1705{
1706 {0x00000070, 0x04400000},
1707 {0x00000071, 0x80c01803},
1708 {0x00000072, 0x00004004},
1709 {0x00000073, 0x00000100},
1710 {0x00000074, 0x00ff0000},
1711 {0x00000075, 0x34000000},
1712 {0x00000076, 0x08000014},
1713 {0x00000077, 0x00cc08ec},
1714 {0x00000078, 0x00000400},
1715 {0x00000079, 0x00000000},
1716 {0x0000007a, 0x04090000},
1717 {0x0000007c, 0x00000000},
1718 {0x0000007e, 0x4408a8e8},
1719 {0x0000007f, 0x00000304},
1720 {0x00000080, 0x00000000},
1721 {0x00000082, 0x00000001},
1722 {0x00000083, 0x00000002},
1723 {0x00000084, 0xf3e4f400},
1724 {0x00000085, 0x052024e3},
1725 {0x00000087, 0x00000000},
1726 {0x00000088, 0x01000000},
1727 {0x0000008a, 0x1c0a0000},
1728 {0x0000008b, 0xff010000},
1729 {0x0000008d, 0xffffefff},
1730 {0x0000008e, 0xfff3efff},
1731 {0x0000008f, 0xfff3efbf},
1732 {0x00000092, 0xf7ffffff},
1733 {0x00000093, 0xffffff7f},
1734 {0x00000095, 0x00101101},
1735 {0x00000096, 0x00000fff},
1736 {0x00000097, 0x00116fff},
1737 {0x00000098, 0x60010000},
1738 {0x00000099, 0x10010000},
1739 {0x0000009a, 0x00006000},
1740 {0x0000009b, 0x00001000},
1741 {0x0000009f, 0x00b48000}
1742};
1743
d4775655
AD		 1744#define HAWAII_IO_MC_REGS_SIZE 22
1745
1746static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1747{
1748 {0x0000007d, 0x40000000},
1749 {0x0000007e, 0x40180304},
1750 {0x0000007f, 0x0000ff00},
1751 {0x00000081, 0x00000000},
1752 {0x00000083, 0x00000800},
1753 {0x00000086, 0x00000000},
1754 {0x00000087, 0x00000100},
1755 {0x00000088, 0x00020100},
1756 {0x00000089, 0x00000000},
1757 {0x0000008b, 0x00040000},
1758 {0x0000008c, 0x00000100},
1759 {0x0000008e, 0xff010000},
1760 {0x00000090, 0xffffefff},
1761 {0x00000091, 0xfff3efff},
1762 {0x00000092, 0xfff3efbf},
1763 {0x00000093, 0xf7ffffff},
1764 {0x00000094, 0xffffff7f},
1765 {0x00000095, 0x00000fff},
1766 {0x00000096, 0x00116fff},
1767 {0x00000097, 0x60010000},
1768 {0x00000098, 0x10010000},
1769 {0x0000009f, 0x00c79000}
1770};
1771
1772
b556b12e
AD		 1773/**
1774 * cik_srbm_select - select specific register instances
1775 *
1776 * @rdev: radeon_device pointer
1777 * @me: selected ME (micro engine)
1778 * @pipe: pipe
1779 * @queue: queue
1780 * @vmid: VMID
1781 *
1782 * Switches the currently active registers instances. Some
1783 * registers are instanced per VMID, others are instanced per
1784 * me/pipe/queue combination.
1785 */
1786static void cik_srbm_select(struct radeon_device *rdev,
1787 u32 me, u32 pipe, u32 queue, u32 vmid)
1788{
1789 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1790 MEID(me & 0x3) |
1791 VMID(vmid & 0xf) |
1792 QUEUEID(queue & 0x7));
1793 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1794}
1795
bc8273fe
AD		 1796/* ucode loading */
1797/**
1798 * ci_mc_load_microcode - load MC ucode into the hw
1799 *
1800 * @rdev: radeon_device pointer
1801 *
1802 * Load the GDDR MC ucode into the hw (CIK).
1803 * Returns 0 on success, error on failure.
1804 */
6c7bccea 1805int ci_mc_load_microcode(struct radeon_device *rdev)
bc8273fe 1806{
f2c6b0f4
AD		 1807 const __be32 *fw_data = NULL;
1808 const __le32 *new_fw_data = NULL;
9feb3dda 1809 u32 running, blackout = 0, tmp;
f2c6b0f4
AD		 1810 u32 *io_mc_regs = NULL;
1811 const __le32 *new_io_mc_regs = NULL;
bcddee29 1812 int i, regs_size, ucode_size;
bc8273fe
AD		 1813
1814 if (!rdev->mc_fw)
1815 return -EINVAL;
1816
f2c6b0f4
AD		 1817 if (rdev->new_fw) {
1818 const struct mc_firmware_header_v1_0 *hdr =
1819 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
bcddee29 1820
f2c6b0f4
AD		 1821 radeon_ucode_print_mc_hdr(&hdr->header);
1822
1823 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1824 new_io_mc_regs = (const __le32 *)
1825 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1826 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1827 new_fw_data = (const __le32 *)
1828 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1829 } else {
1830 ucode_size = rdev->mc_fw->size / 4;
1831
1832 switch (rdev->family) {
1833 case CHIP_BONAIRE:
1834 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1835 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1836 break;
1837 case CHIP_HAWAII:
1838 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1839 regs_size = HAWAII_IO_MC_REGS_SIZE;
1840 break;
1841 default:
1842 return -EINVAL;
1843 }
1844 fw_data = (const __be32 *)rdev->mc_fw->data;
bc8273fe
AD		 1845 }
1846
1847 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1848
1849 if (running == 0) {
1850 if (running) {
1851 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1852 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1853 }
1854
1855 /* reset the engine and set to writable */
1856 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1857 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1858
1859 /* load mc io regs */
1860 for (i = 0; i < regs_size; i++) {
f2c6b0f4
AD		 1861 if (rdev->new_fw) {
1862 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1863 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1864 } else {
1865 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1866 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1867 }
bc8273fe 1868 }
9feb3dda
AD		 1869
1870 tmp = RREG32(MC_SEQ_MISC0);
1871 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1872 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1873 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1874 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1875 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1876 }
1877
bc8273fe 1878 /* load the MC ucode */
f2c6b0f4
AD		 1879 for (i = 0; i < ucode_size; i++) {
1880 if (rdev->new_fw)
1881 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1882 else
1883 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1884 }
bc8273fe
AD		 1885
1886 /* put the engine back into the active state */
1887 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1888 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1889 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1890
1891 /* wait for training to complete */
1892 for (i = 0; i < rdev->usec_timeout; i++) {
1893 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1894 break;
1895 udelay(1);
1896 }
1897 for (i = 0; i < rdev->usec_timeout; i++) {
1898 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1899 break;
1900 udelay(1);
1901 }
1902
1903 if (running)
1904 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1905 }
1906
1907 return 0;
1908}
1909
02c81327
AD		 1910/**
1911 * cik_init_microcode - load ucode images from disk
1912 *
1913 * @rdev: radeon_device pointer
1914 *
1915 * Use the firmware interface to load the ucode images into
1916 * the driver (not loaded into hw).
1917 * Returns 0 on success, error on failure.
1918 */
1919static int cik_init_microcode(struct radeon_device *rdev)
1920{
02c81327 1921 const char *chip_name;
f2c6b0f4 1922 const char *new_chip_name;
02c81327 1923 size_t pfp_req_size, me_req_size, ce_req_size,
d4775655 1924 mec_req_size, rlc_req_size, mc_req_size = 0,
277babc3 1925 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
02c81327 1926 char fw_name[30];
f2c6b0f4 1927 int new_fw = 0;
02c81327 1928 int err;
f2c6b0f4 1929 int num_fw;
02c81327
AD		 1930
1931 DRM_DEBUG("\n");
1932
02c81327
AD		 1933 switch (rdev->family) {
1934 case CHIP_BONAIRE:
1935 chip_name = "BONAIRE";
f2c6b0f4 1936 new_chip_name = "bonaire";
02c81327
AD		 1937 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1938 me_req_size = CIK_ME_UCODE_SIZE * 4;
1939 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1940 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1941 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
277babc3
AD		 1942 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1943 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
21a93e13 1944 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 1945 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
f2c6b0f4 1946 num_fw = 8;
02c81327 1947 break;
d4775655
AD		 1948 case CHIP_HAWAII:
1949 chip_name = "HAWAII";
f2c6b0f4 1950 new_chip_name = "hawaii";
d4775655
AD		 1951 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1952 me_req_size = CIK_ME_UCODE_SIZE * 4;
1953 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1954 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1955 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1956 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
277babc3 1957 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
d4775655
AD		 1958 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1959 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
f2c6b0f4 1960 num_fw = 8;
d4775655 1961 break;
02c81327
AD		 1962 case CHIP_KAVERI:
1963 chip_name = "KAVERI";
f2c6b0f4 1964 new_chip_name = "kaveri";
02c81327
AD		 1965 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1966 me_req_size = CIK_ME_UCODE_SIZE * 4;
1967 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1968 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1969 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 1970 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 1971 num_fw = 7;
02c81327
AD		 1972 break;
1973 case CHIP_KABINI:
1974 chip_name = "KABINI";
f2c6b0f4 1975 new_chip_name = "kabini";
02c81327
AD		 1976 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1977 me_req_size = CIK_ME_UCODE_SIZE * 4;
1978 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1979 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1980 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 1981 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 1982 num_fw = 6;
02c81327 1983 break;
f73a9e83
SL		 1984 case CHIP_MULLINS:
1985 chip_name = "MULLINS";
f2c6b0f4 1986 new_chip_name = "mullins";
f73a9e83
SL		 1987 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1988 me_req_size = CIK_ME_UCODE_SIZE * 4;
1989 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1990 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1991 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1992 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 1993 num_fw = 6;
f73a9e83 1994 break;
02c81327
AD		 1995 default: BUG();
1996 }
1997
f2c6b0f4 1998 DRM_INFO("Loading %s Microcode\n", new_chip_name);
02c81327 1999
f2c6b0f4 2000 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
0a168933 2001 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2002 if (err) {
2003 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2004 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2005 if (err)
2006 goto out;
2007 if (rdev->pfp_fw->size != pfp_req_size) {
2008 printk(KERN_ERR
2009 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2010 rdev->pfp_fw->size, fw_name);
2011 err = -EINVAL;
2012 goto out;
2013 }
2014 } else {
2015 err = radeon_ucode_validate(rdev->pfp_fw);
2016 if (err) {
2017 printk(KERN_ERR
2018 "cik_fw: validation failed for firmware \"%s\"\n",
2019 fw_name);
2020 goto out;
2021 } else {
2022 new_fw++;
2023 }
02c81327
AD		 2024 }
2025
f2c6b0f4 2026 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
0a168933 2027 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2028 if (err) {
2029 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2030 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2031 if (err)
2032 goto out;
2033 if (rdev->me_fw->size != me_req_size) {
2034 printk(KERN_ERR
2035 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2036 rdev->me_fw->size, fw_name);
2037 err = -EINVAL;
2038 }
2039 } else {
2040 err = radeon_ucode_validate(rdev->me_fw);
2041 if (err) {
2042 printk(KERN_ERR
2043 "cik_fw: validation failed for firmware \"%s\"\n",
2044 fw_name);
2045 goto out;
2046 } else {
2047 new_fw++;
2048 }
02c81327
AD		 2049 }
2050
f2c6b0f4 2051 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
0a168933 2052 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2053 if (err) {
2054 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2055 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2056 if (err)
2057 goto out;
2058 if (rdev->ce_fw->size != ce_req_size) {
2059 printk(KERN_ERR
2060 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2061 rdev->ce_fw->size, fw_name);
2062 err = -EINVAL;
2063 }
2064 } else {
2065 err = radeon_ucode_validate(rdev->ce_fw);
2066 if (err) {
2067 printk(KERN_ERR
2068 "cik_fw: validation failed for firmware \"%s\"\n",
2069 fw_name);
2070 goto out;
2071 } else {
2072 new_fw++;
2073 }
02c81327
AD		 2074 }
2075
f2c6b0f4 2076 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
0a168933 2077 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2078 if (err) {
2079 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2080 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2081 if (err)
2082 goto out;
2083 if (rdev->mec_fw->size != mec_req_size) {
2084 printk(KERN_ERR
2085 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2086 rdev->mec_fw->size, fw_name);
2087 err = -EINVAL;
2088 }
2089 } else {
2090 err = radeon_ucode_validate(rdev->mec_fw);
2091 if (err) {
2092 printk(KERN_ERR
2093 "cik_fw: validation failed for firmware \"%s\"\n",
2094 fw_name);
2095 goto out;
2096 } else {
2097 new_fw++;
2098 }
2099 }
2100
2101 if (rdev->family == CHIP_KAVERI) {
2102 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2103 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2104 if (err) {
2105 goto out;
2106 } else {
2107 err = radeon_ucode_validate(rdev->mec2_fw);
2108 if (err) {
2109 goto out;
2110 } else {
2111 new_fw++;
2112 }
2113 }
02c81327
AD		 2114 }
2115
f2c6b0f4 2116 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
0a168933 2117 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2118 if (err) {
2119 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2120 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2121 if (err)
2122 goto out;
2123 if (rdev->rlc_fw->size != rlc_req_size) {
2124 printk(KERN_ERR
2125 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2126 rdev->rlc_fw->size, fw_name);
2127 err = -EINVAL;
2128 }
2129 } else {
2130 err = radeon_ucode_validate(rdev->rlc_fw);
2131 if (err) {
2132 printk(KERN_ERR
2133 "cik_fw: validation failed for firmware \"%s\"\n",
2134 fw_name);
2135 goto out;
2136 } else {
2137 new_fw++;
2138 }
02c81327
AD		 2139 }
2140
f2c6b0f4 2141 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
0a168933 2142 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2143 if (err) {
2144 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2145 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2146 if (err)
2147 goto out;
2148 if (rdev->sdma_fw->size != sdma_req_size) {
2149 printk(KERN_ERR
2150 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2151 rdev->sdma_fw->size, fw_name);
2152 err = -EINVAL;
2153 }
2154 } else {
2155 err = radeon_ucode_validate(rdev->sdma_fw);
2156 if (err) {
2157 printk(KERN_ERR
2158 "cik_fw: validation failed for firmware \"%s\"\n",
2159 fw_name);
2160 goto out;
2161 } else {
2162 new_fw++;
2163 }
21a93e13
AD		 2164 }
2165
cc8dbbb4 2166 /* No SMC, MC ucode on APUs */
02c81327 2167 if (!(rdev->flags & RADEON_IS_IGP)) {
f2c6b0f4 2168 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
0a168933 2169 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
277babc3 2170 if (err) {
f2c6b0f4 2171 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
277babc3 2172 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2173 if (err) {
2174 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2175 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2176 if (err)
2177 goto out;
2178 }
2179 if ((rdev->mc_fw->size != mc_req_size) &&
2180 (rdev->mc_fw->size != mc2_req_size)){
2181 printk(KERN_ERR
2182 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2183 rdev->mc_fw->size, fw_name);
2184 err = -EINVAL;
2185 }
2186 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2187 } else {
2188 err = radeon_ucode_validate(rdev->mc_fw);
2189 if (err) {
2190 printk(KERN_ERR
2191 "cik_fw: validation failed for firmware \"%s\"\n",
2192 fw_name);
277babc3 2193 goto out;
f2c6b0f4
AD		 2194 } else {
2195 new_fw++;
2196 }
277babc3 2197 }
cc8dbbb4 2198
f2c6b0f4 2199 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
cc8dbbb4
AD		 2200 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2201 if (err) {
f2c6b0f4
AD		 2202 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2203 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2204 if (err) {
2205 printk(KERN_ERR
2206 "smc: error loading firmware \"%s\"\n",
2207 fw_name);
2208 release_firmware(rdev->smc_fw);
2209 rdev->smc_fw = NULL;
2210 err = 0;
2211 } else if (rdev->smc_fw->size != smc_req_size) {
2212 printk(KERN_ERR
2213 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2214 rdev->smc_fw->size, fw_name);
2215 err = -EINVAL;
2216 }
2217 } else {
2218 err = radeon_ucode_validate(rdev->smc_fw);
2219 if (err) {
2220 printk(KERN_ERR
2221 "cik_fw: validation failed for firmware \"%s\"\n",
2222 fw_name);
2223 goto out;
2224 } else {
2225 new_fw++;
2226 }
cc8dbbb4 2227 }
02c81327
AD		 2228 }
2229
f2c6b0f4
AD		 2230 if (new_fw == 0) {
2231 rdev->new_fw = false;
2232 } else if (new_fw < num_fw) {
2233 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2234 err = -EINVAL;
2235 } else {
2236 rdev->new_fw = true;
2237 }
2238
02c81327 2239out:
02c81327
AD		 2240 if (err) {
2241 if (err != -EINVAL)
2242 printk(KERN_ERR
2243 "cik_cp: Failed to load firmware \"%s\"\n",
2244 fw_name);
2245 release_firmware(rdev->pfp_fw);
2246 rdev->pfp_fw = NULL;
2247 release_firmware(rdev->me_fw);
2248 rdev->me_fw = NULL;
2249 release_firmware(rdev->ce_fw);
2250 rdev->ce_fw = NULL;
f2c6b0f4
AD		 2251 release_firmware(rdev->mec_fw);
2252 rdev->mec_fw = NULL;
2253 release_firmware(rdev->mec2_fw);
2254 rdev->mec2_fw = NULL;
02c81327
AD		 2255 release_firmware(rdev->rlc_fw);
2256 rdev->rlc_fw = NULL;
f2c6b0f4
AD		 2257 release_firmware(rdev->sdma_fw);
2258 rdev->sdma_fw = NULL;
02c81327
AD		 2259 release_firmware(rdev->mc_fw);
2260 rdev->mc_fw = NULL;
cc8dbbb4
AD		 2261 release_firmware(rdev->smc_fw);
2262 rdev->smc_fw = NULL;
02c81327
AD		 2263 }
2264 return err;
2265}
2266
8cc1a532
AD		 2267/*
2268 * Core functions
2269 */
2270/**
2271 * cik_tiling_mode_table_init - init the hw tiling table
2272 *
2273 * @rdev: radeon_device pointer
2274 *
2275 * Starting with SI, the tiling setup is done globally in a
2276 * set of 32 tiling modes. Rather than selecting each set of
2277 * parameters per surface as on older asics, we just select
2278 * which index in the tiling table we want to use, and the
2279 * surface uses those parameters (CIK).
2280 */
2281static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2282{
2283 const u32 num_tile_mode_states = 32;
2284 const u32 num_secondary_tile_mode_states = 16;
2285 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2286 u32 num_pipe_configs;
2287 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2288 rdev->config.cik.max_shader_engines;
2289
2290 switch (rdev->config.cik.mem_row_size_in_kb) {
2291 case 1:
2292 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2293 break;
2294 case 2:
2295 default:
2296 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2297 break;
2298 case 4:
2299 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2300 break;
2301 }
2302
2303 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2304 if (num_pipe_configs > 8)
21e438af 2305 num_pipe_configs = 16;
8cc1a532 2306
21e438af
AD		 2307 if (num_pipe_configs == 16) {
2308 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2309 switch (reg_offset) {
2310 case 0:
2311 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2312 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2313 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2314 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2315 break;
2316 case 1:
2317 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2318 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2319 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2320 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2321 break;
2322 case 2:
2323 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2324 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2325 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2326 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2327 break;
2328 case 3:
2329 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2330 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2331 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2332 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2333 break;
2334 case 4:
2335 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2336 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2337 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2338 TILE_SPLIT(split_equal_to_row_size));
2339 break;
2340 case 5:
2341 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2342 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2343 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2344 break;
2345 case 6:
2346 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2347 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2348 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2349 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2350 break;
2351 case 7:
2352 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2353 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2354 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2355 TILE_SPLIT(split_equal_to_row_size));
2356 break;
2357 case 8:
2358 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2359 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2360 break;
2361 case 9:
2362 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2363 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2364 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2365 break;
2366 case 10:
2367 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2368 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2369 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2370 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2371 break;
2372 case 11:
2373 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2374 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2375 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2376 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2377 break;
2378 case 12:
2379 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2380 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2381 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2382 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2383 break;
2384 case 13:
2385 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2386 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2387 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2388 break;
2389 case 14:
2390 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2391 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2392 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2393 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2394 break;
2395 case 16:
2396 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2397 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2398 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2399 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2400 break;
2401 case 17:
2402 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2403 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2404 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2405 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2406 break;
2407 case 27:
2408 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2409 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2410 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2411 break;
2412 case 28:
2413 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2414 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2415 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2416 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2417 break;
2418 case 29:
2419 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2420 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2421 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2422 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2423 break;
2424 case 30:
2425 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2426 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2427 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2428 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2429 break;
2430 default:
2431 gb_tile_moden = 0;
2432 break;
2433 }
2434 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2435 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2436 }
2437 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2438 switch (reg_offset) {
2439 case 0:
2440 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2441 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2442 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2443 NUM_BANKS(ADDR_SURF_16_BANK));
2444 break;
2445 case 1:
2446 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2447 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2448 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2449 NUM_BANKS(ADDR_SURF_16_BANK));
2450 break;
2451 case 2:
2452 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2453 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2454 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2455 NUM_BANKS(ADDR_SURF_16_BANK));
2456 break;
2457 case 3:
2458 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2459 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2460 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2461 NUM_BANKS(ADDR_SURF_16_BANK));
2462 break;
2463 case 4:
2464 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2465 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2466 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2467 NUM_BANKS(ADDR_SURF_8_BANK));
2468 break;
2469 case 5:
2470 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2471 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2472 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2473 NUM_BANKS(ADDR_SURF_4_BANK));
2474 break;
2475 case 6:
2476 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2477 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2478 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2479 NUM_BANKS(ADDR_SURF_2_BANK));
2480 break;
2481 case 8:
2482 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2483 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2484 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2485 NUM_BANKS(ADDR_SURF_16_BANK));
2486 break;
2487 case 9:
2488 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2489 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2490 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2491 NUM_BANKS(ADDR_SURF_16_BANK));
2492 break;
2493 case 10:
2494 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2495 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2496 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2497 NUM_BANKS(ADDR_SURF_16_BANK));
2498 break;
2499 case 11:
2500 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2501 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2502 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2503 NUM_BANKS(ADDR_SURF_8_BANK));
2504 break;
2505 case 12:
2506 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2507 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2508 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2509 NUM_BANKS(ADDR_SURF_4_BANK));
2510 break;
2511 case 13:
2512 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2513 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2514 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2515 NUM_BANKS(ADDR_SURF_2_BANK));
2516 break;
2517 case 14:
2518 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2519 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2520 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2521 NUM_BANKS(ADDR_SURF_2_BANK));
2522 break;
2523 default:
2524 gb_tile_moden = 0;
2525 break;
2526 }
1b2c4869 2527 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
21e438af
AD		 2528 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2529 }
2530 } else if (num_pipe_configs == 8) {
8cc1a532
AD		 2531 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2532 switch (reg_offset) {
2533 case 0:
2534 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2535 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2536 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2537 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2538 break;
2539 case 1:
2540 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2541 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2542 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2543 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2544 break;
2545 case 2:
2546 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2547 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2548 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2549 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2550 break;
2551 case 3:
2552 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2553 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2554 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2555 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2556 break;
2557 case 4:
2558 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2559 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2560 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2561 TILE_SPLIT(split_equal_to_row_size));
2562 break;
2563 case 5:
2564 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2565 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2566 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2567 break;
2568 case 6:
2569 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2570 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2571 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2572 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2573 break;
2574 case 7:
2575 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2576 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2577 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2578 TILE_SPLIT(split_equal_to_row_size));
2579 break;
2580 case 8:
2581 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2582 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2583 break;
2584 case 9:
2585 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2586 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2587 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2588 break;
2589 case 10:
2590 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2591 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2592 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2593 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2594 break;
2595 case 11:
2596 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2597 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2598 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2599 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2600 break;
2601 case 12:
2602 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2603 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2604 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2605 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2606 break;
2607 case 13:
2608 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2609 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2610 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2611 break;
2612 case 14:
2613 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2614 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2615 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2616 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2617 break;
2618 case 16:
2619 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2620 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2621 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2622 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2623 break;
2624 case 17:
2625 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2626 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2627 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2628 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2629 break;
2630 case 27:
2631 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2632 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2633 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2634 break;
2635 case 28:
2636 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2637 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2638 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2639 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2640 break;
2641 case 29:
2642 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2643 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2644 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2645 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2646 break;
2647 case 30:
2648 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2649 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2650 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2651 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2652 break;
2653 default:
2654 gb_tile_moden = 0;
2655 break;
2656 }
39aee490 2657 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2658 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2659 }
2660 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2661 switch (reg_offset) {
2662 case 0:
2663 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2664 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2665 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2666 NUM_BANKS(ADDR_SURF_16_BANK));
2667 break;
2668 case 1:
2669 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2670 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2671 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2672 NUM_BANKS(ADDR_SURF_16_BANK));
2673 break;
2674 case 2:
2675 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2676 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2677 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2678 NUM_BANKS(ADDR_SURF_16_BANK));
2679 break;
2680 case 3:
2681 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2682 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2683 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2684 NUM_BANKS(ADDR_SURF_16_BANK));
2685 break;
2686 case 4:
2687 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2688 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2689 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2690 NUM_BANKS(ADDR_SURF_8_BANK));
2691 break;
2692 case 5:
2693 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2694 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2695 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2696 NUM_BANKS(ADDR_SURF_4_BANK));
2697 break;
2698 case 6:
2699 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2700 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2701 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2702 NUM_BANKS(ADDR_SURF_2_BANK));
2703 break;
2704 case 8:
2705 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2706 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2707 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2708 NUM_BANKS(ADDR_SURF_16_BANK));
2709 break;
2710 case 9:
2711 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2712 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2713 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2714 NUM_BANKS(ADDR_SURF_16_BANK));
2715 break;
2716 case 10:
2717 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2718 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2719 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2720 NUM_BANKS(ADDR_SURF_16_BANK));
2721 break;
2722 case 11:
2723 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2724 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2725 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2726 NUM_BANKS(ADDR_SURF_16_BANK));
2727 break;
2728 case 12:
2729 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2730 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2731 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2732 NUM_BANKS(ADDR_SURF_8_BANK));
2733 break;
2734 case 13:
2735 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2736 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2737 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2738 NUM_BANKS(ADDR_SURF_4_BANK));
2739 break;
2740 case 14:
2741 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2742 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2743 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2744 NUM_BANKS(ADDR_SURF_2_BANK));
2745 break;
2746 default:
2747 gb_tile_moden = 0;
2748 break;
2749 }
32f79a8a 2750 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2751 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2752 }
2753 } else if (num_pipe_configs == 4) {
2754 if (num_rbs == 4) {
2755 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2756 switch (reg_offset) {
2757 case 0:
2758 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2759 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2760 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2761 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2762 break;
2763 case 1:
2764 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2765 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2766 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2767 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2768 break;
2769 case 2:
2770 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2771 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2772 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2773 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2774 break;
2775 case 3:
2776 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2777 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2778 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2779 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2780 break;
2781 case 4:
2782 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2783 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2784 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2785 TILE_SPLIT(split_equal_to_row_size));
2786 break;
2787 case 5:
2788 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2789 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2790 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2791 break;
2792 case 6:
2793 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2794 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2795 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2796 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2797 break;
2798 case 7:
2799 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2800 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2801 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2802 TILE_SPLIT(split_equal_to_row_size));
2803 break;
2804 case 8:
2805 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2806 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2807 break;
2808 case 9:
2809 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2810 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2811 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2812 break;
2813 case 10:
2814 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2815 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2816 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2817 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2818 break;
2819 case 11:
2820 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2821 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2822 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2823 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2824 break;
2825 case 12:
2826 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2827 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2828 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2829 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2830 break;
2831 case 13:
2832 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2833 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2834 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2835 break;
2836 case 14:
2837 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2838 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2839 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2840 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2841 break;
2842 case 16:
2843 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2844 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2845 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2846 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2847 break;
2848 case 17:
2849 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2850 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2851 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2852 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2853 break;
2854 case 27:
2855 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2856 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2857 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2858 break;
2859 case 28:
2860 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2861 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2862 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2863 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2864 break;
2865 case 29:
2866 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2867 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2868 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2869 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2870 break;
2871 case 30:
2872 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2873 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2874 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2875 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2876 break;
2877 default:
2878 gb_tile_moden = 0;
2879 break;
2880 }
39aee490 2881 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2882 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2883 }
2884 } else if (num_rbs < 4) {
2885 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2886 switch (reg_offset) {
2887 case 0:
2888 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2889 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2890 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2891 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2892 break;
2893 case 1:
2894 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2895 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2896 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2897 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2898 break;
2899 case 2:
2900 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2901 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2902 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2903 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2904 break;
2905 case 3:
2906 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2907 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2908 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2909 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2910 break;
2911 case 4:
2912 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2913 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2914 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2915 TILE_SPLIT(split_equal_to_row_size));
2916 break;
2917 case 5:
2918 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2919 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2920 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2921 break;
2922 case 6:
2923 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2924 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2925 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2926 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2927 break;
2928 case 7:
2929 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2930 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2931 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2932 TILE_SPLIT(split_equal_to_row_size));
2933 break;
2934 case 8:
2935 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2936 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2937 break;
2938 case 9:
2939 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2940 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2941 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2942 break;
2943 case 10:
2944 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2945 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2946 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2947 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2948 break;
2949 case 11:
2950 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2951 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2952 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2953 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2954 break;
2955 case 12:
2956 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2957 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2958 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2959 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2960 break;
2961 case 13:
2962 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2963 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2964 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2965 break;
2966 case 14:
2967 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2968 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2969 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2970 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2971 break;
2972 case 16:
2973 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2974 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2975 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2976 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2977 break;
2978 case 17:
2979 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2980 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2981 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2982 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2983 break;
2984 case 27:
2985 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2986 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2987 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2988 break;
2989 case 28:
2990 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2991 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2992 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2993 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2994 break;
2995 case 29:
2996 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2997 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2998 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2999 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3000 break;
3001 case 30:
3002 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3003 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3004 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
3005 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3006 break;
3007 default:
3008 gb_tile_moden = 0;
3009 break;
3010 }
39aee490 3011 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3012 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3013 }
3014 }
3015 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3016 switch (reg_offset) {
3017 case 0:
3018 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3019 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3020 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3021 NUM_BANKS(ADDR_SURF_16_BANK));
3022 break;
3023 case 1:
3024 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3025 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3026 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3027 NUM_BANKS(ADDR_SURF_16_BANK));
3028 break;
3029 case 2:
3030 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3031 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3032 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3033 NUM_BANKS(ADDR_SURF_16_BANK));
3034 break;
3035 case 3:
3036 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3037 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3038 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3039 NUM_BANKS(ADDR_SURF_16_BANK));
3040 break;
3041 case 4:
3042 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3043 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3044 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3045 NUM_BANKS(ADDR_SURF_16_BANK));
3046 break;
3047 case 5:
3048 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3049 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3050 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3051 NUM_BANKS(ADDR_SURF_8_BANK));
3052 break;
3053 case 6:
3054 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3055 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3056 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3057 NUM_BANKS(ADDR_SURF_4_BANK));
3058 break;
3059 case 8:
3060 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3061 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3062 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3063 NUM_BANKS(ADDR_SURF_16_BANK));
3064 break;
3065 case 9:
3066 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3067 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3068 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3069 NUM_BANKS(ADDR_SURF_16_BANK));
3070 break;
3071 case 10:
3072 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3073 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3074 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3075 NUM_BANKS(ADDR_SURF_16_BANK));
3076 break;
3077 case 11:
3078 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3079 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3080 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3081 NUM_BANKS(ADDR_SURF_16_BANK));
3082 break;
3083 case 12:
3084 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3085 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3086 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3087 NUM_BANKS(ADDR_SURF_16_BANK));
3088 break;
3089 case 13:
3090 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3091 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3092 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3093 NUM_BANKS(ADDR_SURF_8_BANK));
3094 break;
3095 case 14:
3096 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3097 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3098 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3099 NUM_BANKS(ADDR_SURF_4_BANK));
3100 break;
3101 default:
3102 gb_tile_moden = 0;
3103 break;
3104 }
32f79a8a 3105 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3106 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3107 }
3108 } else if (num_pipe_configs == 2) {
3109 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3110 switch (reg_offset) {
3111 case 0:
3112 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3113 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3114 PIPE_CONFIG(ADDR_SURF_P2) |
3115 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3116 break;
3117 case 1:
3118 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3119 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3120 PIPE_CONFIG(ADDR_SURF_P2) |
3121 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3122 break;
3123 case 2:
3124 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3125 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3126 PIPE_CONFIG(ADDR_SURF_P2) |
3127 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3128 break;
3129 case 3:
3130 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3131 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3132 PIPE_CONFIG(ADDR_SURF_P2) |
3133 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3134 break;
3135 case 4:
3136 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3137 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3138 PIPE_CONFIG(ADDR_SURF_P2) |
3139 TILE_SPLIT(split_equal_to_row_size));
3140 break;
3141 case 5:
3142 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3143 PIPE_CONFIG(ADDR_SURF_P2) |
8cc1a532
AD		 3144 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3145 break;
3146 case 6:
3147 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3148 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3149 PIPE_CONFIG(ADDR_SURF_P2) |
3150 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3151 break;
3152 case 7:
3153 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3154 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3155 PIPE_CONFIG(ADDR_SURF_P2) |
3156 TILE_SPLIT(split_equal_to_row_size));
3157 break;
3158 case 8:
020ff546
MO		 3159 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3160 PIPE_CONFIG(ADDR_SURF_P2);
8cc1a532
AD		 3161 break;
3162 case 9:
3163 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546
MO		 3164 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3165 PIPE_CONFIG(ADDR_SURF_P2));
8cc1a532
AD		 3166 break;
3167 case 10:
3168 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3169 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3170 PIPE_CONFIG(ADDR_SURF_P2) |
3171 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3172 break;
3173 case 11:
3174 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3175 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3176 PIPE_CONFIG(ADDR_SURF_P2) |
3177 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3178 break;
3179 case 12:
3180 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3181 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3182 PIPE_CONFIG(ADDR_SURF_P2) |
3183 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3184 break;
3185 case 13:
3186 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3187 PIPE_CONFIG(ADDR_SURF_P2) |
8cc1a532
AD		 3188 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3189 break;
3190 case 14:
3191 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3192 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3193 PIPE_CONFIG(ADDR_SURF_P2) |
3194 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3195 break;
3196 case 16:
3197 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3198 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3199 PIPE_CONFIG(ADDR_SURF_P2) |
3200 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3201 break;
3202 case 17:
3203 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3204 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3205 PIPE_CONFIG(ADDR_SURF_P2) |
3206 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3207 break;
3208 case 27:
3209 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546
MO		 3210 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3211 PIPE_CONFIG(ADDR_SURF_P2));
8cc1a532
AD		 3212 break;
3213 case 28:
3214 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3215 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3216 PIPE_CONFIG(ADDR_SURF_P2) |
3217 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3218 break;
3219 case 29:
3220 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3221 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3222 PIPE_CONFIG(ADDR_SURF_P2) |
3223 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3224 break;
3225 case 30:
3226 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3227 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3228 PIPE_CONFIG(ADDR_SURF_P2) |
3229 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3230 break;
3231 default:
3232 gb_tile_moden = 0;
3233 break;
3234 }
39aee490 3235 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3236 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3237 }
3238 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3239 switch (reg_offset) {
3240 case 0:
3241 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3242 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3243 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3244 NUM_BANKS(ADDR_SURF_16_BANK));
3245 break;
3246 case 1:
3247 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3248 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3249 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3250 NUM_BANKS(ADDR_SURF_16_BANK));
3251 break;
3252 case 2:
3253 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3254 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3255 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3256 NUM_BANKS(ADDR_SURF_16_BANK));
3257 break;
3258 case 3:
3259 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3260 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3261 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3262 NUM_BANKS(ADDR_SURF_16_BANK));
3263 break;
3264 case 4:
3265 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3266 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3267 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3268 NUM_BANKS(ADDR_SURF_16_BANK));
3269 break;
3270 case 5:
3271 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3272 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3273 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3274 NUM_BANKS(ADDR_SURF_16_BANK));
3275 break;
3276 case 6:
3277 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3278 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3279 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3280 NUM_BANKS(ADDR_SURF_8_BANK));
3281 break;
3282 case 8:
3283 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3284 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3285 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3286 NUM_BANKS(ADDR_SURF_16_BANK));
3287 break;
3288 case 9:
3289 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3290 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3291 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3292 NUM_BANKS(ADDR_SURF_16_BANK));
3293 break;
3294 case 10:
3295 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3296 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3297 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3298 NUM_BANKS(ADDR_SURF_16_BANK));
3299 break;
3300 case 11:
3301 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3302 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3303 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3304 NUM_BANKS(ADDR_SURF_16_BANK));
3305 break;
3306 case 12:
3307 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3308 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3309 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3310 NUM_BANKS(ADDR_SURF_16_BANK));
3311 break;
3312 case 13:
3313 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3314 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3315 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3316 NUM_BANKS(ADDR_SURF_16_BANK));
3317 break;
3318 case 14:
3319 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3320 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3321 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3322 NUM_BANKS(ADDR_SURF_8_BANK));
3323 break;
3324 default:
3325 gb_tile_moden = 0;
3326 break;
3327 }
32f79a8a 3328 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3329 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3330 }
3331 } else
3332 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3333}
3334
3335/**
3336 * cik_select_se_sh - select which SE, SH to address
3337 *
3338 * @rdev: radeon_device pointer
3339 * @se_num: shader engine to address
3340 * @sh_num: sh block to address
3341 *
3342 * Select which SE, SH combinations to address. Certain
3343 * registers are instanced per SE or SH. 0xffffffff means
3344 * broadcast to all SEs or SHs (CIK).
3345 */
3346static void cik_select_se_sh(struct radeon_device *rdev,
3347 u32 se_num, u32 sh_num)
3348{
3349 u32 data = INSTANCE_BROADCAST_WRITES;
3350
3351 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 3352 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 3353 else if (se_num == 0xffffffff)
3354 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3355 else if (sh_num == 0xffffffff)
3356 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3357 else
3358 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3359 WREG32(GRBM_GFX_INDEX, data);
3360}
3361
3362/**
3363 * cik_create_bitmask - create a bitmask
3364 *
3365 * @bit_width: length of the mask
3366 *
3367 * create a variable length bit mask (CIK).
3368 * Returns the bitmask.
3369 */
3370static u32 cik_create_bitmask(u32 bit_width)
3371{
3372 u32 i, mask = 0;
3373
3374 for (i = 0; i < bit_width; i++) {
3375 mask <<= 1;
3376 mask |= 1;
3377 }
3378 return mask;
3379}
3380
3381/**
972c5ddb 3382 * cik_get_rb_disabled - computes the mask of disabled RBs
8cc1a532
AD		 3383 *
3384 * @rdev: radeon_device pointer
3385 * @max_rb_num: max RBs (render backends) for the asic
3386 * @se_num: number of SEs (shader engines) for the asic
3387 * @sh_per_se: number of SH blocks per SE for the asic
3388 *
3389 * Calculates the bitmask of disabled RBs (CIK).
3390 * Returns the disabled RB bitmask.
3391 */
3392static u32 cik_get_rb_disabled(struct radeon_device *rdev,
9fadb352 3393 u32 max_rb_num_per_se,
8cc1a532
AD		 3394 u32 sh_per_se)
3395{
3396 u32 data, mask;
3397
3398 data = RREG32(CC_RB_BACKEND_DISABLE);
3399 if (data & 1)
3400 data &= BACKEND_DISABLE_MASK;
3401 else
3402 data = 0;
3403 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3404
3405 data >>= BACKEND_DISABLE_SHIFT;
3406
9fadb352 3407 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
8cc1a532
AD		 3408
3409 return data & mask;
3410}
3411
3412/**
3413 * cik_setup_rb - setup the RBs on the asic
3414 *
3415 * @rdev: radeon_device pointer
3416 * @se_num: number of SEs (shader engines) for the asic
3417 * @sh_per_se: number of SH blocks per SE for the asic
3418 * @max_rb_num: max RBs (render backends) for the asic
3419 *
3420 * Configures per-SE/SH RB registers (CIK).
3421 */
3422static void cik_setup_rb(struct radeon_device *rdev,
3423 u32 se_num, u32 sh_per_se,
9fadb352 3424 u32 max_rb_num_per_se)
8cc1a532
AD		 3425{
3426 int i, j;
3427 u32 data, mask;
3428 u32 disabled_rbs = 0;
3429 u32 enabled_rbs = 0;
3430
3431 for (i = 0; i < se_num; i++) {
3432 for (j = 0; j < sh_per_se; j++) {
3433 cik_select_se_sh(rdev, i, j);
9fadb352 3434 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
fc821b70
AD		 3435 if (rdev->family == CHIP_HAWAII)
3436 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3437 else
3438 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
8cc1a532
AD		 3439 }
3440 }
3441 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3442
3443 mask = 1;
9fadb352 3444 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
8cc1a532
AD		 3445 if (!(disabled_rbs & mask))
3446 enabled_rbs |= mask;
3447 mask <<= 1;
3448 }
3449
439a1cff
MO		 3450 rdev->config.cik.backend_enable_mask = enabled_rbs;
3451
8cc1a532
AD		 3452 for (i = 0; i < se_num; i++) {
3453 cik_select_se_sh(rdev, i, 0xffffffff);
3454 data = 0;
3455 for (j = 0; j < sh_per_se; j++) {
3456 switch (enabled_rbs & 3) {
fc821b70
AD		 3457 case 0:
3458 if (j == 0)
3459 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3460 else
3461 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3462 break;
8cc1a532
AD		 3463 case 1:
3464 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3465 break;
3466 case 2:
3467 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3468 break;
3469 case 3:
3470 default:
3471 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3472 break;
3473 }
3474 enabled_rbs >>= 2;
3475 }
3476 WREG32(PA_SC_RASTER_CONFIG, data);
3477 }
3478 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3479}
3480
3481/**
3482 * cik_gpu_init - setup the 3D engine
3483 *
3484 * @rdev: radeon_device pointer
3485 *
3486 * Configures the 3D engine and tiling configuration
3487 * registers so that the 3D engine is usable.
3488 */
3489static void cik_gpu_init(struct radeon_device *rdev)
3490{
3491 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3492 u32 mc_shared_chmap, mc_arb_ramcfg;
3493 u32 hdp_host_path_cntl;
3494 u32 tmp;
6101b3ae 3495 int i, j;
8cc1a532
AD		 3496
3497 switch (rdev->family) {
3498 case CHIP_BONAIRE:
3499 rdev->config.cik.max_shader_engines = 2;
3500 rdev->config.cik.max_tile_pipes = 4;
3501 rdev->config.cik.max_cu_per_sh = 7;
3502 rdev->config.cik.max_sh_per_se = 1;
3503 rdev->config.cik.max_backends_per_se = 2;
3504 rdev->config.cik.max_texture_channel_caches = 4;
3505 rdev->config.cik.max_gprs = 256;
3506 rdev->config.cik.max_gs_threads = 32;
3507 rdev->config.cik.max_hw_contexts = 8;
3508
3509 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3510 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3511 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3512 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3513 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3514 break;
b496038b
AD		 3515 case CHIP_HAWAII:
3516 rdev->config.cik.max_shader_engines = 4;
3517 rdev->config.cik.max_tile_pipes = 16;
3518 rdev->config.cik.max_cu_per_sh = 11;
3519 rdev->config.cik.max_sh_per_se = 1;
3520 rdev->config.cik.max_backends_per_se = 4;
3521 rdev->config.cik.max_texture_channel_caches = 16;
3522 rdev->config.cik.max_gprs = 256;
3523 rdev->config.cik.max_gs_threads = 32;
3524 rdev->config.cik.max_hw_contexts = 8;
3525
3526 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3527 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3528 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3529 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3530 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3531 break;
8cc1a532 3532 case CHIP_KAVERI:
b2e4c70a
AD		 3533 rdev->config.cik.max_shader_engines = 1;
3534 rdev->config.cik.max_tile_pipes = 4;
3535 if ((rdev->pdev->device == 0x1304) ||
3536 (rdev->pdev->device == 0x1305) ||
3537 (rdev->pdev->device == 0x130C) ||
3538 (rdev->pdev->device == 0x130F) ||
3539 (rdev->pdev->device == 0x1310) ||
3540 (rdev->pdev->device == 0x1311) ||
3541 (rdev->pdev->device == 0x131C)) {
3542 rdev->config.cik.max_cu_per_sh = 8;
3543 rdev->config.cik.max_backends_per_se = 2;
3544 } else if ((rdev->pdev->device == 0x1309) ||
3545 (rdev->pdev->device == 0x130A) ||
3546 (rdev->pdev->device == 0x130D) ||
7c4622d5
AD		 3547 (rdev->pdev->device == 0x1313) ||
3548 (rdev->pdev->device == 0x131D)) {
b2e4c70a
AD		 3549 rdev->config.cik.max_cu_per_sh = 6;
3550 rdev->config.cik.max_backends_per_se = 2;
3551 } else if ((rdev->pdev->device == 0x1306) ||
3552 (rdev->pdev->device == 0x1307) ||
3553 (rdev->pdev->device == 0x130B) ||
3554 (rdev->pdev->device == 0x130E) ||
3555 (rdev->pdev->device == 0x1315) ||
6dc14baf 3556 (rdev->pdev->device == 0x1318) ||
b2e4c70a
AD		 3557 (rdev->pdev->device == 0x131B)) {
3558 rdev->config.cik.max_cu_per_sh = 4;
3559 rdev->config.cik.max_backends_per_se = 1;
3560 } else {
3561 rdev->config.cik.max_cu_per_sh = 3;
3562 rdev->config.cik.max_backends_per_se = 1;
3563 }
3564 rdev->config.cik.max_sh_per_se = 1;
3565 rdev->config.cik.max_texture_channel_caches = 4;
3566 rdev->config.cik.max_gprs = 256;
3567 rdev->config.cik.max_gs_threads = 16;
3568 rdev->config.cik.max_hw_contexts = 8;
3569
3570 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3571 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3572 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3573 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3574 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 3575 break;
3576 case CHIP_KABINI:
f73a9e83 3577 case CHIP_MULLINS:
8cc1a532
AD		 3578 default:
3579 rdev->config.cik.max_shader_engines = 1;
3580 rdev->config.cik.max_tile_pipes = 2;
3581 rdev->config.cik.max_cu_per_sh = 2;
3582 rdev->config.cik.max_sh_per_se = 1;
3583 rdev->config.cik.max_backends_per_se = 1;
3584 rdev->config.cik.max_texture_channel_caches = 2;
3585 rdev->config.cik.max_gprs = 256;
3586 rdev->config.cik.max_gs_threads = 16;
3587 rdev->config.cik.max_hw_contexts = 8;
3588
3589 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3590 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3591 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3592 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3593 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3594 break;
3595 }
3596
3597 /* Initialize HDP */
3598 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3599 WREG32((0x2c14 + j), 0x00000000);
3600 WREG32((0x2c18 + j), 0x00000000);
3601 WREG32((0x2c1c + j), 0x00000000);
3602 WREG32((0x2c20 + j), 0x00000000);
3603 WREG32((0x2c24 + j), 0x00000000);
3604 }
3605
3606 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3607
3608 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3609
3610 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3611 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3612
3613 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3614 rdev->config.cik.mem_max_burst_length_bytes = 256;
3615 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3616 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3617 if (rdev->config.cik.mem_row_size_in_kb > 4)
3618 rdev->config.cik.mem_row_size_in_kb = 4;
3619 /* XXX use MC settings? */
3620 rdev->config.cik.shader_engine_tile_size = 32;
3621 rdev->config.cik.num_gpus = 1;
3622 rdev->config.cik.multi_gpu_tile_size = 64;
3623
3624 /* fix up row size */
3625 gb_addr_config &= ~ROW_SIZE_MASK;
3626 switch (rdev->config.cik.mem_row_size_in_kb) {
3627 case 1:
3628 default:
3629 gb_addr_config |= ROW_SIZE(0);
3630 break;
3631 case 2:
3632 gb_addr_config |= ROW_SIZE(1);
3633 break;
3634 case 4:
3635 gb_addr_config |= ROW_SIZE(2);
3636 break;
3637 }
3638
3639 /* setup tiling info dword. gb_addr_config is not adequate since it does
3640 * not have bank info, so create a custom tiling dword.
3641 * bits 3:0 num_pipes
3642 * bits 7:4 num_banks
3643 * bits 11:8 group_size
3644 * bits 15:12 row_size
3645 */
3646 rdev->config.cik.tile_config = 0;
3647 switch (rdev->config.cik.num_tile_pipes) {
3648 case 1:
3649 rdev->config.cik.tile_config |= (0 << 0);
3650 break;
3651 case 2:
3652 rdev->config.cik.tile_config |= (1 << 0);
3653 break;
3654 case 4:
3655 rdev->config.cik.tile_config |= (2 << 0);
3656 break;
3657 case 8:
3658 default:
3659 /* XXX what about 12? */
3660 rdev->config.cik.tile_config |= (3 << 0);
3661 break;
3662 }
a537314e
MD		 3663 rdev->config.cik.tile_config |=
3664 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
8cc1a532
AD		 3665 rdev->config.cik.tile_config |=
3666 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3667 rdev->config.cik.tile_config |=
3668 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3669
3670 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3671 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3672 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 3673 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3674 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 3675 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3676 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3677 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 3678
3679 cik_tiling_mode_table_init(rdev);
3680
3681 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3682 rdev->config.cik.max_sh_per_se,
3683 rdev->config.cik.max_backends_per_se);
3684
52da51f0 3685 rdev->config.cik.active_cus = 0;
65fcf668
AD		 3686 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3687 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6101b3ae
AD		 3688 rdev->config.cik.active_cus +=
3689 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
65fcf668
AD		 3690 }
3691 }
3692
8cc1a532
AD		 3693 /* set HW defaults for 3D engine */
3694 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3695
3696 WREG32(SX_DEBUG_1, 0x20);
3697
3698 WREG32(TA_CNTL_AUX, 0x00010000);
3699
3700 tmp = RREG32(SPI_CONFIG_CNTL);
3701 tmp |= 0x03000000;
3702 WREG32(SPI_CONFIG_CNTL, tmp);
3703
3704 WREG32(SQ_CONFIG, 1);
3705
3706 WREG32(DB_DEBUG, 0);
3707
3708 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3709 tmp |= 0x00000400;
3710 WREG32(DB_DEBUG2, tmp);
3711
3712 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3713 tmp |= 0x00020200;
3714 WREG32(DB_DEBUG3, tmp);
3715
3716 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3717 tmp |= 0x00018208;
3718 WREG32(CB_HW_CONTROL, tmp);
3719
3720 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3721
3722 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3723 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3724 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3725 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3726
3727 WREG32(VGT_NUM_INSTANCES, 1);
3728
3729 WREG32(CP_PERFMON_CNTL, 0);
3730
3731 WREG32(SQ_CONFIG, 0);
3732
3733 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3734 FORCE_EOV_MAX_REZ_CNT(255)));
3735
3736 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3737 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3738
3739 WREG32(VGT_GS_VERTEX_REUSE, 16);
3740 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3741
3742 tmp = RREG32(HDP_MISC_CNTL);
3743 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3744 WREG32(HDP_MISC_CNTL, tmp);
3745
3746 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3747 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3748
3749 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3750 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3751
3752 udelay(50);
3753}
3754
2cae3bc3
AD		 3755/*
3756 * GPU scratch registers helpers function.
3757 */
3758/**
3759 * cik_scratch_init - setup driver info for CP scratch regs
3760 *
3761 * @rdev: radeon_device pointer
3762 *
3763 * Set up the number and offset of the CP scratch registers.
3764 * NOTE: use of CP scratch registers is a legacy inferface and
3765 * is not used by default on newer asics (r6xx+). On newer asics,
3766 * memory buffers are used for fences rather than scratch regs.
3767 */
3768static void cik_scratch_init(struct radeon_device *rdev)
3769{
3770 int i;
3771
3772 rdev->scratch.num_reg = 7;
3773 rdev->scratch.reg_base = SCRATCH_REG0;
3774 for (i = 0; i < rdev->scratch.num_reg; i++) {
3775 rdev->scratch.free[i] = true;
3776 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3777 }
3778}
3779
fbc832c7
AD		 3780/**
3781 * cik_ring_test - basic gfx ring test
3782 *
3783 * @rdev: radeon_device pointer
3784 * @ring: radeon_ring structure holding ring information
3785 *
3786 * Allocate a scratch register and write to it using the gfx ring (CIK).
3787 * Provides a basic gfx ring test to verify that the ring is working.
3788 * Used by cik_cp_gfx_resume();
3789 * Returns 0 on success, error on failure.
3790 */
3791int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3792{
3793 uint32_t scratch;
3794 uint32_t tmp = 0;
3795 unsigned i;
3796 int r;
3797
3798 r = radeon_scratch_get(rdev, &scratch);
3799 if (r) {
3800 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3801 return r;
3802 }
3803 WREG32(scratch, 0xCAFEDEAD);
3804 r = radeon_ring_lock(rdev, ring, 3);
3805 if (r) {
3806 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3807 radeon_scratch_free(rdev, scratch);
3808 return r;
3809 }
3810 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3811 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3812 radeon_ring_write(ring, 0xDEADBEEF);
1538a9e0 3813 radeon_ring_unlock_commit(rdev, ring, false);
963e81f9 3814
fbc832c7
AD		 3815 for (i = 0; i < rdev->usec_timeout; i++) {
3816 tmp = RREG32(scratch);
3817 if (tmp == 0xDEADBEEF)
3818 break;
3819 DRM_UDELAY(1);
3820 }
3821 if (i < rdev->usec_timeout) {
3822 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3823 } else {
3824 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3825 ring->idx, scratch, tmp);
3826 r = -EINVAL;
3827 }
3828 radeon_scratch_free(rdev, scratch);
3829 return r;
3830}
3831
780f5ddd
AD		 3832/**
3833 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3834 *
3835 * @rdev: radeon_device pointer
3836 * @ridx: radeon ring index
3837 *
3838 * Emits an hdp flush on the cp.
3839 */
3840static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3841 int ridx)
3842{
3843 struct radeon_ring *ring = &rdev->ring[ridx];
5d259067 3844 u32 ref_and_mask;
780f5ddd 3845
5d259067
AD		 3846 switch (ring->idx) {
3847 case CAYMAN_RING_TYPE_CP1_INDEX:
3848 case CAYMAN_RING_TYPE_CP2_INDEX:
3849 default:
3850 switch (ring->me) {
3851 case 0:
3852 ref_and_mask = CP2 << ring->pipe;
3853 break;
3854 case 1:
3855 ref_and_mask = CP6 << ring->pipe;
3856 break;
3857 default:
3858 return;
3859 }
3860 break;
3861 case RADEON_RING_TYPE_GFX_INDEX:
3862 ref_and_mask = CP0;
3863 break;
3864 }
3865
3866 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3867 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3868 WAIT_REG_MEM_FUNCTION(3) | /* == */
3869 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3870 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3871 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3872 radeon_ring_write(ring, ref_and_mask);
3873 radeon_ring_write(ring, ref_and_mask);
3874 radeon_ring_write(ring, 0x20); /* poll interval */
780f5ddd
AD		 3875}
3876
2cae3bc3 3877/**
b07fdd38 3878 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3879 *
3880 * @rdev: radeon_device pointer
3881 * @fence: radeon fence object
3882 *
3883 * Emits a fence sequnce number on the gfx ring and flushes
3884 * GPU caches.
3885 */
b07fdd38
AD		 3886void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3887 struct radeon_fence *fence)
2cae3bc3
AD		 3888{
3889 struct radeon_ring *ring = &rdev->ring[fence->ring];
3890 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3891
3892 /* EVENT_WRITE_EOP - flush caches, send int */
3893 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3894 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3895 EOP_TC_ACTION_EN |
3896 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3897 EVENT_INDEX(5)));
3898 radeon_ring_write(ring, addr & 0xfffffffc);
3899 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3900 radeon_ring_write(ring, fence->seq);
3901 radeon_ring_write(ring, 0);
2cae3bc3
AD		 3902}
3903
b07fdd38
AD		 3904/**
3905 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3906 *
3907 * @rdev: radeon_device pointer
3908 * @fence: radeon fence object
3909 *
3910 * Emits a fence sequnce number on the compute ring and flushes
3911 * GPU caches.
3912 */
3913void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3914 struct radeon_fence *fence)
3915{
3916 struct radeon_ring *ring = &rdev->ring[fence->ring];
3917 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3918
3919 /* RELEASE_MEM - flush caches, send int */
3920 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3921 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3922 EOP_TC_ACTION_EN |
3923 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3924 EVENT_INDEX(5)));
3925 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3926 radeon_ring_write(ring, addr & 0xfffffffc);
3927 radeon_ring_write(ring, upper_32_bits(addr));
3928 radeon_ring_write(ring, fence->seq);
3929 radeon_ring_write(ring, 0);
b07fdd38
AD		 3930}
3931
86302eea
CK		 3932/**
3933 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3934 *
3935 * @rdev: radeon_device pointer
3936 * @ring: radeon ring buffer object
3937 * @semaphore: radeon semaphore object
3938 * @emit_wait: Is this a sempahore wait?
3939 *
3940 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3941 * from running ahead of semaphore waits.
3942 */
1654b817 3943bool cik_semaphore_ring_emit(struct radeon_device *rdev,
2cae3bc3
AD		 3944 struct radeon_ring *ring,
3945 struct radeon_semaphore *semaphore,
3946 bool emit_wait)
3947{
3948 uint64_t addr = semaphore->gpu_addr;
3949 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3950
3951 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
5e167cdb 3952 radeon_ring_write(ring, lower_32_bits(addr));
2cae3bc3 3953 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
1654b817 3954
86302eea
CK		 3955 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3956 /* Prevent the PFP from running ahead of the semaphore wait */
3957 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3958 radeon_ring_write(ring, 0x0);
3959 }
3960
1654b817 3961 return true;
2cae3bc3
AD		 3962}
3963
c9dbd705
AD		 3964/**
3965 * cik_copy_cpdma - copy pages using the CP DMA engine
3966 *
3967 * @rdev: radeon_device pointer
3968 * @src_offset: src GPU address
3969 * @dst_offset: dst GPU address
3970 * @num_gpu_pages: number of GPU pages to xfer
57d20a43 3971 * @resv: reservation object to sync to
c9dbd705
AD		 3972 *
3973 * Copy GPU paging using the CP DMA engine (CIK+).
3974 * Used by the radeon ttm implementation to move pages if
3975 * registered as the asic copy callback.
3976 */
57d20a43
CK		 3977struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
3978 uint64_t src_offset, uint64_t dst_offset,
3979 unsigned num_gpu_pages,
3980 struct reservation_object *resv)
c9dbd705
AD		 3981{
3982 struct radeon_semaphore *sem = NULL;
57d20a43 3983 struct radeon_fence *fence;
c9dbd705
AD		 3984 int ring_index = rdev->asic->copy.blit_ring_index;
3985 struct radeon_ring *ring = &rdev->ring[ring_index];
3986 u32 size_in_bytes, cur_size_in_bytes, control;
3987 int i, num_loops;
3988 int r = 0;
3989
3990 r = radeon_semaphore_create(rdev, &sem);
3991 if (r) {
3992 DRM_ERROR("radeon: moving bo (%d).\n", r);
57d20a43 3993 return ERR_PTR(r);
c9dbd705
AD		 3994 }
3995
3996 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3997 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3998 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3999 if (r) {
4000 DRM_ERROR("radeon: moving bo (%d).\n", r);
4001 radeon_semaphore_free(rdev, &sem, NULL);
57d20a43 4002 return ERR_PTR(r);
c9dbd705
AD		 4003 }
4004
392a250b 4005 radeon_semaphore_sync_resv(rdev, sem, resv, false);
1654b817 4006 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
c9dbd705
AD		 4007
4008 for (i = 0; i < num_loops; i++) {
4009 cur_size_in_bytes = size_in_bytes;
4010 if (cur_size_in_bytes > 0x1fffff)
4011 cur_size_in_bytes = 0x1fffff;
4012 size_in_bytes -= cur_size_in_bytes;
4013 control = 0;
4014 if (size_in_bytes == 0)
4015 control |= PACKET3_DMA_DATA_CP_SYNC;
4016 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4017 radeon_ring_write(ring, control);
4018 radeon_ring_write(ring, lower_32_bits(src_offset));
4019 radeon_ring_write(ring, upper_32_bits(src_offset));
4020 radeon_ring_write(ring, lower_32_bits(dst_offset));
4021 radeon_ring_write(ring, upper_32_bits(dst_offset));
4022 radeon_ring_write(ring, cur_size_in_bytes);
4023 src_offset += cur_size_in_bytes;
4024 dst_offset += cur_size_in_bytes;
4025 }
4026
57d20a43 4027 r = radeon_fence_emit(rdev, &fence, ring->idx);
c9dbd705
AD		 4028 if (r) {
4029 radeon_ring_unlock_undo(rdev, ring);
aa4c8b36 4030 radeon_semaphore_free(rdev, &sem, NULL);
57d20a43 4031 return ERR_PTR(r);
c9dbd705
AD		 4032 }
4033
1538a9e0 4034 radeon_ring_unlock_commit(rdev, ring, false);
57d20a43 4035 radeon_semaphore_free(rdev, &sem, fence);
c9dbd705 4036
57d20a43 4037 return fence;
c9dbd705
AD		 4038}
4039
2cae3bc3
AD		 4040/*
4041 * IB stuff
4042 */
4043/**
4044 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4045 *
4046 * @rdev: radeon_device pointer
4047 * @ib: radeon indirect buffer object
4048 *
4049 * Emits an DE (drawing engine) or CE (constant engine) IB
4050 * on the gfx ring. IBs are usually generated by userspace
4051 * acceleration drivers and submitted to the kernel for
4052 * sheduling on the ring. This function schedules the IB
4053 * on the gfx ring for execution by the GPU.
4054 */
4055void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4056{
4057 struct radeon_ring *ring = &rdev->ring[ib->ring];
4058 u32 header, control = INDIRECT_BUFFER_VALID;
4059
4060 if (ib->is_const_ib) {
4061 /* set switch buffer packet before const IB */
4062 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4063 radeon_ring_write(ring, 0);
4064
4065 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4066 } else {
4067 u32 next_rptr;
4068 if (ring->rptr_save_reg) {
4069 next_rptr = ring->wptr + 3 + 4;
4070 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4071 radeon_ring_write(ring, ((ring->rptr_save_reg -
4072 PACKET3_SET_UCONFIG_REG_START) >> 2));
4073 radeon_ring_write(ring, next_rptr);
4074 } else if (rdev->wb.enabled) {
4075 next_rptr = ring->wptr + 5 + 4;
4076 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4077 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4078 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5e167cdb 4079 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
2cae3bc3
AD		 4080 radeon_ring_write(ring, next_rptr);
4081 }
4082
4083 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4084 }
4085
4086 control |= ib->length_dw |
4087 (ib->vm ? (ib->vm->id << 24) : 0);
4088
4089 radeon_ring_write(ring, header);
4090 radeon_ring_write(ring,
4091#ifdef __BIG_ENDIAN
4092 (2 << 0) |
4093#endif
4094 (ib->gpu_addr & 0xFFFFFFFC));
4095 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4096 radeon_ring_write(ring, control);
4097}
4098
fbc832c7
AD		 4099/**
4100 * cik_ib_test - basic gfx ring IB test
4101 *
4102 * @rdev: radeon_device pointer
4103 * @ring: radeon_ring structure holding ring information
4104 *
4105 * Allocate an IB and execute it on the gfx ring (CIK).
4106 * Provides a basic gfx ring test to verify that IBs are working.
4107 * Returns 0 on success, error on failure.
4108 */
4109int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4110{
4111 struct radeon_ib ib;
4112 uint32_t scratch;
4113 uint32_t tmp = 0;
4114 unsigned i;
4115 int r;
4116
4117 r = radeon_scratch_get(rdev, &scratch);
4118 if (r) {
4119 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4120 return r;
4121 }
4122 WREG32(scratch, 0xCAFEDEAD);
4123 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4124 if (r) {
4125 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
5510f124 4126 radeon_scratch_free(rdev, scratch);
fbc832c7
AD		 4127 return r;
4128 }
4129 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4130 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4131 ib.ptr[2] = 0xDEADBEEF;
4132 ib.length_dw = 3;
1538a9e0 4133 r = radeon_ib_schedule(rdev, &ib, NULL, false);
fbc832c7
AD		 4134 if (r) {
4135 radeon_scratch_free(rdev, scratch);
4136 radeon_ib_free(rdev, &ib);
4137 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4138 return r;
4139 }
4140 r = radeon_fence_wait(ib.fence, false);
4141 if (r) {
4142 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
5510f124
CK		 4143 radeon_scratch_free(rdev, scratch);
4144 radeon_ib_free(rdev, &ib);
fbc832c7
AD		 4145 return r;
4146 }
4147 for (i = 0; i < rdev->usec_timeout; i++) {
4148 tmp = RREG32(scratch);
4149 if (tmp == 0xDEADBEEF)
4150 break;
4151 DRM_UDELAY(1);
4152 }
4153 if (i < rdev->usec_timeout) {
4154 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4155 } else {
4156 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4157 scratch, tmp);
4158 r = -EINVAL;
4159 }
4160 radeon_scratch_free(rdev, scratch);
4161 radeon_ib_free(rdev, &ib);
4162 return r;
4163}
4164
841cf442
AD		 4165/*
4166 * CP.
4167 * On CIK, gfx and compute now have independant command processors.
4168 *
4169 * GFX
4170 * Gfx consists of a single ring and can process both gfx jobs and
4171 * compute jobs. The gfx CP consists of three microengines (ME):
4172 * PFP - Pre-Fetch Parser
4173 * ME - Micro Engine
4174 * CE - Constant Engine
4175 * The PFP and ME make up what is considered the Drawing Engine (DE).
4176 * The CE is an asynchronous engine used for updating buffer desciptors
4177 * used by the DE so that they can be loaded into cache in parallel
4178 * while the DE is processing state update packets.
4179 *
4180 * Compute
4181 * The compute CP consists of two microengines (ME):
4182 * MEC1 - Compute MicroEngine 1
4183 * MEC2 - Compute MicroEngine 2
4184 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4185 * The queues are exposed to userspace and are programmed directly
4186 * by the compute runtime.
4187 */
4188/**
4189 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4190 *
4191 * @rdev: radeon_device pointer
4192 * @enable: enable or disable the MEs
4193 *
4194 * Halts or unhalts the gfx MEs.
4195 */
4196static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4197{
4198 if (enable)
4199 WREG32(CP_ME_CNTL, 0);
4200 else {
50efa51a
AD		 4201 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4202 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
841cf442
AD		 4203 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4204 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4205 }
4206 udelay(50);
4207}
4208
4209/**
4210 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4211 *
4212 * @rdev: radeon_device pointer
4213 *
4214 * Loads the gfx PFP, ME, and CE ucode.
4215 * Returns 0 for success, -EINVAL if the ucode is not available.
4216 */
4217static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4218{
841cf442
AD		 4219 int i;
4220
4221 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4222 return -EINVAL;
4223
4224 cik_cp_gfx_enable(rdev, false);
4225
f2c6b0f4
AD		 4226 if (rdev->new_fw) {
4227 const struct gfx_firmware_header_v1_0 *pfp_hdr =
4228 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4229 const struct gfx_firmware_header_v1_0 *ce_hdr =
4230 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4231 const struct gfx_firmware_header_v1_0 *me_hdr =
4232 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4233 const __le32 *fw_data;
4234 u32 fw_size;
4235
4236 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4237 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4238 radeon_ucode_print_gfx_hdr(&me_hdr->header);
4239
4240 /* PFP */
4241 fw_data = (const __le32 *)
4242 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4243 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4244 WREG32(CP_PFP_UCODE_ADDR, 0);
4245 for (i = 0; i < fw_size; i++)
4246 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4247 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
f2c6b0f4
AD		 4248
4249 /* CE */
4250 fw_data = (const __le32 *)
4251 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4252 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4253 WREG32(CP_CE_UCODE_ADDR, 0);
4254 for (i = 0; i < fw_size; i++)
4255 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4256 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
f2c6b0f4
AD		 4257
4258 /* ME */
4259 fw_data = (const __be32 *)
4260 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4261 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4262 WREG32(CP_ME_RAM_WADDR, 0);
4263 for (i = 0; i < fw_size; i++)
4264 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
38aea071
AD		 4265 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
4266 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
f2c6b0f4
AD		 4267 } else {
4268 const __be32 *fw_data;
4269
4270 /* PFP */
4271 fw_data = (const __be32 *)rdev->pfp_fw->data;
4272 WREG32(CP_PFP_UCODE_ADDR, 0);
4273 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4274 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4275 WREG32(CP_PFP_UCODE_ADDR, 0);
4276
4277 /* CE */
4278 fw_data = (const __be32 *)rdev->ce_fw->data;
4279 WREG32(CP_CE_UCODE_ADDR, 0);
4280 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4281 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4282 WREG32(CP_CE_UCODE_ADDR, 0);
4283
4284 /* ME */
4285 fw_data = (const __be32 *)rdev->me_fw->data;
4286 WREG32(CP_ME_RAM_WADDR, 0);
4287 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4288 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4289 WREG32(CP_ME_RAM_WADDR, 0);
4290 }
841cf442 4291
841cf442
AD		 4292 return 0;
4293}
4294
4295/**
4296 * cik_cp_gfx_start - start the gfx ring
4297 *
4298 * @rdev: radeon_device pointer
4299 *
4300 * Enables the ring and loads the clear state context and other
4301 * packets required to init the ring.
4302 * Returns 0 for success, error for failure.
4303 */
4304static int cik_cp_gfx_start(struct radeon_device *rdev)
4305{
4306 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4307 int r, i;
4308
4309 /* init the CP */
4310 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4311 WREG32(CP_ENDIAN_SWAP, 0);
4312 WREG32(CP_DEVICE_ID, 1);
4313
4314 cik_cp_gfx_enable(rdev, true);
4315
4316 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4317 if (r) {
4318 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4319 return r;
4320 }
4321
4322 /* init the CE partitions. CE only used for gfx on CIK */
4323 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4324 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4325 radeon_ring_write(ring, 0xc000);
4326 radeon_ring_write(ring, 0xc000);
4327
4328 /* setup clear context state */
4329 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4330 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4331
4332 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4333 radeon_ring_write(ring, 0x80000000);
4334 radeon_ring_write(ring, 0x80000000);
4335
4336 for (i = 0; i < cik_default_size; i++)
4337 radeon_ring_write(ring, cik_default_state[i]);
4338
4339 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4340 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4341
4342 /* set clear context state */
4343 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4344 radeon_ring_write(ring, 0);
4345
4346 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4347 radeon_ring_write(ring, 0x00000316);
4348 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4349 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4350
1538a9e0 4351 radeon_ring_unlock_commit(rdev, ring, false);
841cf442
AD		 4352
4353 return 0;
4354}
4355
4356/**
4357 * cik_cp_gfx_fini - stop the gfx ring
4358 *
4359 * @rdev: radeon_device pointer
4360 *
4361 * Stop the gfx ring and tear down the driver ring
4362 * info.
4363 */
4364static void cik_cp_gfx_fini(struct radeon_device *rdev)
4365{
4366 cik_cp_gfx_enable(rdev, false);
4367 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4368}
4369
4370/**
4371 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4372 *
4373 * @rdev: radeon_device pointer
4374 *
4375 * Program the location and size of the gfx ring buffer
4376 * and test it to make sure it's working.
4377 * Returns 0 for success, error for failure.
4378 */
4379static int cik_cp_gfx_resume(struct radeon_device *rdev)
4380{
4381 struct radeon_ring *ring;
4382 u32 tmp;
4383 u32 rb_bufsz;
4384 u64 rb_addr;
4385 int r;
4386
4387 WREG32(CP_SEM_WAIT_TIMER, 0x0);
939c0d3c
AD		 4388 if (rdev->family != CHIP_HAWAII)
4389 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
841cf442
AD		 4390
4391 /* Set the write pointer delay */
4392 WREG32(CP_RB_WPTR_DELAY, 0);
4393
4394 /* set the RB to use vmid 0 */
4395 WREG32(CP_RB_VMID, 0);
4396
4397 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4398
4399 /* ring 0 - compute and gfx */
4400 /* Set ring buffer size */
4401 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 4402 rb_bufsz = order_base_2(ring->ring_size / 8);
4403 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
841cf442
AD		 4404#ifdef __BIG_ENDIAN
4405 tmp |= BUF_SWAP_32BIT;
4406#endif
4407 WREG32(CP_RB0_CNTL, tmp);
4408
4409 /* Initialize the ring buffer's read and write pointers */
4410 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4411 ring->wptr = 0;
4412 WREG32(CP_RB0_WPTR, ring->wptr);
4413
4414 /* set the wb address wether it's enabled or not */
4415 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4416 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4417
4418 /* scratch register shadowing is no longer supported */
4419 WREG32(SCRATCH_UMSK, 0);
4420
4421 if (!rdev->wb.enabled)
4422 tmp |= RB_NO_UPDATE;
4423
4424 mdelay(1);
4425 WREG32(CP_RB0_CNTL, tmp);
4426
4427 rb_addr = ring->gpu_addr >> 8;
4428 WREG32(CP_RB0_BASE, rb_addr);
4429 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4430
841cf442
AD		 4431 /* start the ring */
4432 cik_cp_gfx_start(rdev);
4433 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4434 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4435 if (r) {
4436 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4437 return r;
4438 }
50efa51a
AD		 4439
4440 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4441 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4442
841cf442
AD		 4443 return 0;
4444}
4445
ea31bf69
AD		 4446u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4447 struct radeon_ring *ring)
963e81f9
AD		 4448{
4449 u32 rptr;
4450
ea31bf69
AD		 4451 if (rdev->wb.enabled)
4452 rptr = rdev->wb.wb[ring->rptr_offs/4];
4453 else
4454 rptr = RREG32(CP_RB0_RPTR);
4455
4456 return rptr;
4457}
4458
4459u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4460 struct radeon_ring *ring)
4461{
4462 u32 wptr;
4463
4464 wptr = RREG32(CP_RB0_WPTR);
963e81f9 4465
ea31bf69
AD		 4466 return wptr;
4467}
4468
4469void cik_gfx_set_wptr(struct radeon_device *rdev,
4470 struct radeon_ring *ring)
4471{
4472 WREG32(CP_RB0_WPTR, ring->wptr);
4473 (void)RREG32(CP_RB0_WPTR);
4474}
4475
4476u32 cik_compute_get_rptr(struct radeon_device *rdev,
4477 struct radeon_ring *ring)
4478{
4479 u32 rptr;
963e81f9
AD		 4480
4481 if (rdev->wb.enabled) {
ea31bf69 4482 rptr = rdev->wb.wb[ring->rptr_offs/4];
963e81f9 4483 } else {
f61d5b46 4484 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4485 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4486 rptr = RREG32(CP_HQD_PQ_RPTR);
4487 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4488 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4489 }
963e81f9
AD		 4490
4491 return rptr;
4492}
4493
ea31bf69
AD		 4494u32 cik_compute_get_wptr(struct radeon_device *rdev,
4495 struct radeon_ring *ring)
963e81f9
AD		 4496{
4497 u32 wptr;
4498
4499 if (rdev->wb.enabled) {
ea31bf69
AD		 4500 /* XXX check if swapping is necessary on BE */
4501 wptr = rdev->wb.wb[ring->wptr_offs/4];
963e81f9 4502 } else {
f61d5b46 4503 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4504 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4505 wptr = RREG32(CP_HQD_PQ_WPTR);
4506 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4507 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4508 }
963e81f9
AD		 4509
4510 return wptr;
4511}
4512
ea31bf69
AD		 4513void cik_compute_set_wptr(struct radeon_device *rdev,
4514 struct radeon_ring *ring)
963e81f9 4515{
ea31bf69
AD		 4516 /* XXX check if swapping is necessary on BE */
4517 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
d5754ab8 4518 WDOORBELL32(ring->doorbell_index, ring->wptr);
963e81f9
AD		 4519}
4520
841cf442
AD		 4521/**
4522 * cik_cp_compute_enable - enable/disable the compute CP MEs
4523 *
4524 * @rdev: radeon_device pointer
4525 * @enable: enable or disable the MEs
4526 *
4527 * Halts or unhalts the compute MEs.
4528 */
4529static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4530{
4531 if (enable)
4532 WREG32(CP_MEC_CNTL, 0);
b2b3d8d9 4533 else {
841cf442 4534 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
b2b3d8d9
AD		 4535 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4536 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4537 }
841cf442
AD		 4538 udelay(50);
4539}
4540
4541/**
4542 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4543 *
4544 * @rdev: radeon_device pointer
4545 *
4546 * Loads the compute MEC1&2 ucode.
4547 * Returns 0 for success, -EINVAL if the ucode is not available.
4548 */
4549static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4550{
841cf442
AD		 4551 int i;
4552
4553 if (!rdev->mec_fw)
4554 return -EINVAL;
4555
4556 cik_cp_compute_enable(rdev, false);
4557
f2c6b0f4
AD		 4558 if (rdev->new_fw) {
4559 const struct gfx_firmware_header_v1_0 *mec_hdr =
4560 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4561 const __le32 *fw_data;
4562 u32 fw_size;
4563
4564 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4565
4566 /* MEC1 */
4567 fw_data = (const __le32 *)
4568 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4569 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4570 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4571 for (i = 0; i < fw_size; i++)
4572 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4573 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
841cf442 4574
841cf442 4575 /* MEC2 */
f2c6b0f4
AD		 4576 if (rdev->family == CHIP_KAVERI) {
4577 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4578 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4579
4580 fw_data = (const __le32 *)
4581 (rdev->mec2_fw->data +
4582 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4583 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4584 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4585 for (i = 0; i < fw_size; i++)
4586 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4587 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
f2c6b0f4
AD		 4588 }
4589 } else {
4590 const __be32 *fw_data;
4591
4592 /* MEC1 */
841cf442 4593 fw_data = (const __be32 *)rdev->mec_fw->data;
f2c6b0f4 4594 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
841cf442 4595 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
f2c6b0f4
AD		 4596 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4597 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4598
4599 if (rdev->family == CHIP_KAVERI) {
4600 /* MEC2 */
4601 fw_data = (const __be32 *)rdev->mec_fw->data;
4602 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4603 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4604 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4605 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4606 }
841cf442
AD		 4607 }
4608
4609 return 0;
4610}
4611
4612/**
4613 * cik_cp_compute_start - start the compute queues
4614 *
4615 * @rdev: radeon_device pointer
4616 *
4617 * Enable the compute queues.
4618 * Returns 0 for success, error for failure.
4619 */
4620static int cik_cp_compute_start(struct radeon_device *rdev)
4621{
963e81f9
AD		 4622 cik_cp_compute_enable(rdev, true);
4623
841cf442
AD		 4624 return 0;
4625}
4626
4627/**
4628 * cik_cp_compute_fini - stop the compute queues
4629 *
4630 * @rdev: radeon_device pointer
4631 *
4632 * Stop the compute queues and tear down the driver queue
4633 * info.
4634 */
4635static void cik_cp_compute_fini(struct radeon_device *rdev)
4636{
963e81f9
AD		 4637 int i, idx, r;
4638
841cf442 4639 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 4640
4641 for (i = 0; i < 2; i++) {
4642 if (i == 0)
4643 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4644 else
4645 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4646
4647 if (rdev->ring[idx].mqd_obj) {
4648 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4649 if (unlikely(r != 0))
4650 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4651
4652 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4653 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4654
4655 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4656 rdev->ring[idx].mqd_obj = NULL;
4657 }
4658 }
841cf442
AD		 4659}
4660
963e81f9
AD		 4661static void cik_mec_fini(struct radeon_device *rdev)
4662{
4663 int r;
4664
4665 if (rdev->mec.hpd_eop_obj) {
4666 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4667 if (unlikely(r != 0))
4668 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4669 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4670 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4671
4672 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4673 rdev->mec.hpd_eop_obj = NULL;
4674 }
4675}
4676
4677#define MEC_HPD_SIZE 2048
4678
4679static int cik_mec_init(struct radeon_device *rdev)
4680{
4681 int r;
4682 u32 *hpd;
4683
4684 /*
4685 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4686 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
62a7b7fb
OG		 4687 * Nonetheless, we assign only 1 pipe because all other pipes will
4688 * be handled by KFD
963e81f9 4689 */
62a7b7fb
OG		 4690 rdev->mec.num_mec = 1;
4691 rdev->mec.num_pipe = 1;
963e81f9
AD		 4692 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4693
4694 if (rdev->mec.hpd_eop_obj == NULL) {
4695 r = radeon_bo_create(rdev,
4696 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4697 PAGE_SIZE, true,
831b6966 4698 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
963e81f9
AD		 4699 &rdev->mec.hpd_eop_obj);
4700 if (r) {
4701 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4702 return r;
4703 }
4704 }
4705
4706 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4707 if (unlikely(r != 0)) {
4708 cik_mec_fini(rdev);
4709 return r;
4710 }
4711 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4712 &rdev->mec.hpd_eop_gpu_addr);
4713 if (r) {
4714 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4715 cik_mec_fini(rdev);
4716 return r;
4717 }
4718 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4719 if (r) {
4720 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4721 cik_mec_fini(rdev);
4722 return r;
4723 }
4724
4725 /* clear memory. Not sure if this is required or not */
4726 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4727
4728 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4729 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4730
4731 return 0;
4732}
4733
4734struct hqd_registers
4735{
4736 u32 cp_mqd_base_addr;
4737 u32 cp_mqd_base_addr_hi;
4738 u32 cp_hqd_active;
4739 u32 cp_hqd_vmid;
4740 u32 cp_hqd_persistent_state;
4741 u32 cp_hqd_pipe_priority;
4742 u32 cp_hqd_queue_priority;
4743 u32 cp_hqd_quantum;
4744 u32 cp_hqd_pq_base;
4745 u32 cp_hqd_pq_base_hi;
4746 u32 cp_hqd_pq_rptr;
4747 u32 cp_hqd_pq_rptr_report_addr;
4748 u32 cp_hqd_pq_rptr_report_addr_hi;
4749 u32 cp_hqd_pq_wptr_poll_addr;
4750 u32 cp_hqd_pq_wptr_poll_addr_hi;
4751 u32 cp_hqd_pq_doorbell_control;
4752 u32 cp_hqd_pq_wptr;
4753 u32 cp_hqd_pq_control;
4754 u32 cp_hqd_ib_base_addr;
4755 u32 cp_hqd_ib_base_addr_hi;
4756 u32 cp_hqd_ib_rptr;
4757 u32 cp_hqd_ib_control;
4758 u32 cp_hqd_iq_timer;
4759 u32 cp_hqd_iq_rptr;
4760 u32 cp_hqd_dequeue_request;
4761 u32 cp_hqd_dma_offload;
4762 u32 cp_hqd_sema_cmd;
4763 u32 cp_hqd_msg_type;
4764 u32 cp_hqd_atomic0_preop_lo;
4765 u32 cp_hqd_atomic0_preop_hi;
4766 u32 cp_hqd_atomic1_preop_lo;
4767 u32 cp_hqd_atomic1_preop_hi;
4768 u32 cp_hqd_hq_scheduler0;
4769 u32 cp_hqd_hq_scheduler1;
4770 u32 cp_mqd_control;
4771};
4772
4773struct bonaire_mqd
4774{
4775 u32 header;
4776 u32 dispatch_initiator;
4777 u32 dimensions[3];
4778 u32 start_idx[3];
4779 u32 num_threads[3];
4780 u32 pipeline_stat_enable;
4781 u32 perf_counter_enable;
4782 u32 pgm[2];
4783 u32 tba[2];
4784 u32 tma[2];
4785 u32 pgm_rsrc[2];
4786 u32 vmid;
4787 u32 resource_limits;
4788 u32 static_thread_mgmt01[2];
4789 u32 tmp_ring_size;
4790 u32 static_thread_mgmt23[2];
4791 u32 restart[3];
4792 u32 thread_trace_enable;
4793 u32 reserved1;
4794 u32 user_data[16];
4795 u32 vgtcs_invoke_count[2];
4796 struct hqd_registers queue_state;
4797 u32 dequeue_cntr;
4798 u32 interrupt_queue[64];
4799};
4800
841cf442
AD		 4801/**
4802 * cik_cp_compute_resume - setup the compute queue registers
4803 *
4804 * @rdev: radeon_device pointer
4805 *
4806 * Program the compute queues and test them to make sure they
4807 * are working.
4808 * Returns 0 for success, error for failure.
4809 */
4810static int cik_cp_compute_resume(struct radeon_device *rdev)
4811{
370ce45b 4812 int r, i, j, idx;
963e81f9
AD		 4813 u32 tmp;
4814 bool use_doorbell = true;
4815 u64 hqd_gpu_addr;
4816 u64 mqd_gpu_addr;
4817 u64 eop_gpu_addr;
4818 u64 wb_gpu_addr;
4819 u32 *buf;
4820 struct bonaire_mqd *mqd;
841cf442 4821
841cf442
AD		 4822 r = cik_cp_compute_start(rdev);
4823 if (r)
4824 return r;
963e81f9
AD		 4825
4826 /* fix up chicken bits */
4827 tmp = RREG32(CP_CPF_DEBUG);
4828 tmp |= (1 << 23);
4829 WREG32(CP_CPF_DEBUG, tmp);
4830
4831 /* init the pipes */
f61d5b46 4832 mutex_lock(&rdev->srbm_mutex);
963e81f9 4833
62a7b7fb 4834 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr;
963e81f9 4835
62a7b7fb 4836 cik_srbm_select(rdev, 0, 0, 0, 0);
963e81f9 4837
62a7b7fb
OG		 4838 /* write the EOP addr */
4839 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4840 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
963e81f9 4841
62a7b7fb
OG		 4842 /* set the VMID assigned */
4843 WREG32(CP_HPD_EOP_VMID, 0);
4844
4845 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4846 tmp = RREG32(CP_HPD_EOP_CONTROL);
4847 tmp &= ~EOP_SIZE_MASK;
4848 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4849 WREG32(CP_HPD_EOP_CONTROL, tmp);
963e81f9 4850
f61d5b46 4851 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4852
4853 /* init the queues. Just two for now. */
4854 for (i = 0; i < 2; i++) {
4855 if (i == 0)
4856 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4857 else
4858 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4859
4860 if (rdev->ring[idx].mqd_obj == NULL) {
4861 r = radeon_bo_create(rdev,
4862 sizeof(struct bonaire_mqd),
4863 PAGE_SIZE, true,
02376d82 4864 RADEON_GEM_DOMAIN_GTT, 0, NULL,
831b6966 4865 NULL, &rdev->ring[idx].mqd_obj);
963e81f9
AD		 4866 if (r) {
4867 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4868 return r;
4869 }
4870 }
4871
4872 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4873 if (unlikely(r != 0)) {
4874 cik_cp_compute_fini(rdev);
4875 return r;
4876 }
4877 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4878 &mqd_gpu_addr);
4879 if (r) {
4880 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4881 cik_cp_compute_fini(rdev);
4882 return r;
4883 }
4884 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4885 if (r) {
4886 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4887 cik_cp_compute_fini(rdev);
4888 return r;
4889 }
4890
963e81f9
AD		 4891 /* init the mqd struct */
4892 memset(buf, 0, sizeof(struct bonaire_mqd));
4893
4894 mqd = (struct bonaire_mqd *)buf;
4895 mqd->header = 0xC0310800;
4896 mqd->static_thread_mgmt01[0] = 0xffffffff;
4897 mqd->static_thread_mgmt01[1] = 0xffffffff;
4898 mqd->static_thread_mgmt23[0] = 0xffffffff;
4899 mqd->static_thread_mgmt23[1] = 0xffffffff;
4900
f61d5b46 4901 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4902 cik_srbm_select(rdev, rdev->ring[idx].me,
4903 rdev->ring[idx].pipe,
4904 rdev->ring[idx].queue, 0);
4905
4906 /* disable wptr polling */
4907 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4908 tmp &= ~WPTR_POLL_EN;
4909 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4910
4911 /* enable doorbell? */
4912 mqd->queue_state.cp_hqd_pq_doorbell_control =
4913 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4914 if (use_doorbell)
4915 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4916 else
4917 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4918 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4919 mqd->queue_state.cp_hqd_pq_doorbell_control);
4920
4921 /* disable the queue if it's active */
4922 mqd->queue_state.cp_hqd_dequeue_request = 0;
4923 mqd->queue_state.cp_hqd_pq_rptr = 0;
4924 mqd->queue_state.cp_hqd_pq_wptr= 0;
4925 if (RREG32(CP_HQD_ACTIVE) & 1) {
4926 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
370ce45b 4927 for (j = 0; j < rdev->usec_timeout; j++) {
963e81f9
AD		 4928 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4929 break;
4930 udelay(1);
4931 }
4932 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4933 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4934 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4935 }
4936
4937 /* set the pointer to the MQD */
4938 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4939 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4940 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4941 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4942 /* set MQD vmid to 0 */
4943 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4944 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4945 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4946
4947 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4948 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4949 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4950 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4951 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4952 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4953
4954 /* set up the HQD, this is similar to CP_RB0_CNTL */
4955 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4956 mqd->queue_state.cp_hqd_pq_control &=
4957 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4958
4959 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4960 order_base_2(rdev->ring[idx].ring_size / 8);
963e81f9 4961 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4962 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
963e81f9
AD		 4963#ifdef __BIG_ENDIAN
4964 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4965#endif
4966 mqd->queue_state.cp_hqd_pq_control &=
4967 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4968 mqd->queue_state.cp_hqd_pq_control |=
4969 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4970 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4971
4972 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4973 if (i == 0)
4974 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4975 else
4976 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4977 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4978 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4979 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4980 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4981 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4982
4983 /* set the wb address wether it's enabled or not */
4984 if (i == 0)
4985 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4986 else
4987 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4988 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4989 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4990 upper_32_bits(wb_gpu_addr) & 0xffff;
4991 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4992 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4993 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4994 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4995
4996 /* enable the doorbell if requested */
4997 if (use_doorbell) {
4998 mqd->queue_state.cp_hqd_pq_doorbell_control =
4999 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
5000 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
5001 mqd->queue_state.cp_hqd_pq_doorbell_control |=
d5754ab8 5002 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
963e81f9
AD		 5003 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5004 mqd->queue_state.cp_hqd_pq_doorbell_control &=
5005 ~(DOORBELL_SOURCE | DOORBELL_HIT);
5006
5007 } else {
5008 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
5009 }
5010 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5011 mqd->queue_state.cp_hqd_pq_doorbell_control);
5012
5013 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
5014 rdev->ring[idx].wptr = 0;
5015 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
5016 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
ff212f25 5017 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
963e81f9
AD		 5018
5019 /* set the vmid for the queue */
5020 mqd->queue_state.cp_hqd_vmid = 0;
5021 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5022
5023 /* activate the queue */
5024 mqd->queue_state.cp_hqd_active = 1;
5025 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5026
5027 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5028 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 5029
5030 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5031 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5032
5033 rdev->ring[idx].ready = true;
5034 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5035 if (r)
5036 rdev->ring[idx].ready = false;
5037 }
5038
841cf442
AD		 5039 return 0;
5040}
5041
841cf442
AD		 5042static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5043{
5044 cik_cp_gfx_enable(rdev, enable);
5045 cik_cp_compute_enable(rdev, enable);
5046}
5047
841cf442
AD		 5048static int cik_cp_load_microcode(struct radeon_device *rdev)
5049{
5050 int r;
5051
5052 r = cik_cp_gfx_load_microcode(rdev);
5053 if (r)
5054 return r;
5055 r = cik_cp_compute_load_microcode(rdev);
5056 if (r)
5057 return r;
5058
5059 return 0;
5060}
5061
841cf442
AD		 5062static void cik_cp_fini(struct radeon_device *rdev)
5063{
5064 cik_cp_gfx_fini(rdev);
5065 cik_cp_compute_fini(rdev);
5066}
5067
841cf442
AD		 5068static int cik_cp_resume(struct radeon_device *rdev)
5069{
5070 int r;
5071
4214faf6
AD		 5072 cik_enable_gui_idle_interrupt(rdev, false);
5073
841cf442
AD		 5074 r = cik_cp_load_microcode(rdev);
5075 if (r)
5076 return r;
5077
5078 r = cik_cp_gfx_resume(rdev);
5079 if (r)
5080 return r;
5081 r = cik_cp_compute_resume(rdev);
5082 if (r)
5083 return r;
5084
4214faf6
AD		 5085 cik_enable_gui_idle_interrupt(rdev, true);
5086
841cf442
AD		 5087 return 0;
5088}
5089
cc066715 5090static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 5091{
6f2043ce
AD		 5092 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
5093 RREG32(GRBM_STATUS));
5094 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
5095 RREG32(GRBM_STATUS2));
5096 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
5097 RREG32(GRBM_STATUS_SE0));
5098 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
5099 RREG32(GRBM_STATUS_SE1));
5100 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
5101 RREG32(GRBM_STATUS_SE2));
5102 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
5103 RREG32(GRBM_STATUS_SE3));
5104 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
5105 RREG32(SRBM_STATUS));
5106 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
5107 RREG32(SRBM_STATUS2));
cc066715
AD		 5108 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
5109 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5110 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
5111 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 5112 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5113 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
5114 RREG32(CP_STALLED_STAT1));
5115 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
5116 RREG32(CP_STALLED_STAT2));
5117 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
5118 RREG32(CP_STALLED_STAT3));
5119 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
5120 RREG32(CP_CPF_BUSY_STAT));
5121 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
5122 RREG32(CP_CPF_STALLED_STAT1));
5123 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5124 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5125 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
5126 RREG32(CP_CPC_STALLED_STAT1));
5127 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 5128}
6f2043ce 5129
21a93e13 5130/**
cc066715 5131 * cik_gpu_check_soft_reset - check which blocks are busy
21a93e13
AD		 5132 *
5133 * @rdev: radeon_device pointer
21a93e13 5134 *
cc066715
AD		 5135 * Check which blocks are busy and return the relevant reset
5136 * mask to be used by cik_gpu_soft_reset().
5137 * Returns a mask of the blocks to be reset.
21a93e13 5138 */
2483b4ea 5139u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
21a93e13 5140{
cc066715
AD		 5141 u32 reset_mask = 0;
5142 u32 tmp;
21a93e13 5143
cc066715
AD		 5144 /* GRBM_STATUS */
5145 tmp = RREG32(GRBM_STATUS);
5146 if (tmp & (PA_BUSY | SC_BUSY |
5147 BCI_BUSY | SX_BUSY |
5148 TA_BUSY | VGT_BUSY |
5149 DB_BUSY | CB_BUSY |
5150 GDS_BUSY | SPI_BUSY |
5151 IA_BUSY | IA_BUSY_NO_DMA))
5152 reset_mask |= RADEON_RESET_GFX;
21a93e13 5153
cc066715
AD		 5154 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5155 reset_mask |= RADEON_RESET_CP;
21a93e13 5156
cc066715
AD		 5157 /* GRBM_STATUS2 */
5158 tmp = RREG32(GRBM_STATUS2);
5159 if (tmp & RLC_BUSY)
5160 reset_mask |= RADEON_RESET_RLC;
21a93e13 5161
cc066715
AD		 5162 /* SDMA0_STATUS_REG */
5163 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5164 if (!(tmp & SDMA_IDLE))
5165 reset_mask |= RADEON_RESET_DMA;
21a93e13 5166
cc066715
AD		 5167 /* SDMA1_STATUS_REG */
5168 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5169 if (!(tmp & SDMA_IDLE))
5170 reset_mask |= RADEON_RESET_DMA1;
21a93e13 5171
cc066715
AD		 5172 /* SRBM_STATUS2 */
5173 tmp = RREG32(SRBM_STATUS2);
5174 if (tmp & SDMA_BUSY)
5175 reset_mask |= RADEON_RESET_DMA;
21a93e13 5176
cc066715
AD		 5177 if (tmp & SDMA1_BUSY)
5178 reset_mask |= RADEON_RESET_DMA1;
21a93e13 5179
cc066715
AD		 5180 /* SRBM_STATUS */
5181 tmp = RREG32(SRBM_STATUS);
21a93e13 5182
cc066715
AD		 5183 if (tmp & IH_BUSY)
5184 reset_mask |= RADEON_RESET_IH;
21a93e13 5185
cc066715
AD		 5186 if (tmp & SEM_BUSY)
5187 reset_mask |= RADEON_RESET_SEM;
21a93e13 5188
cc066715
AD		 5189 if (tmp & GRBM_RQ_PENDING)
5190 reset_mask |= RADEON_RESET_GRBM;
21a93e13 5191
cc066715
AD		 5192 if (tmp & VMC_BUSY)
5193 reset_mask |= RADEON_RESET_VMC;
21a93e13 5194
cc066715
AD		 5195 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5196 MCC_BUSY | MCD_BUSY))
5197 reset_mask |= RADEON_RESET_MC;
21a93e13 5198
cc066715
AD		 5199 if (evergreen_is_display_hung(rdev))
5200 reset_mask |= RADEON_RESET_DISPLAY;
5201
5202 /* Skip MC reset as it's mostly likely not hung, just busy */
5203 if (reset_mask & RADEON_RESET_MC) {
5204 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5205 reset_mask &= ~RADEON_RESET_MC;
21a93e13 5206 }
cc066715
AD		 5207
5208 return reset_mask;
21a93e13
AD		 5209}
5210
5211/**
cc066715 5212 * cik_gpu_soft_reset - soft reset GPU
21a93e13
AD		 5213 *
5214 * @rdev: radeon_device pointer
cc066715 5215 * @reset_mask: mask of which blocks to reset
21a93e13 5216 *
cc066715 5217 * Soft reset the blocks specified in @reset_mask.
21a93e13 5218 */
cc066715 5219static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
21a93e13 5220{
6f2043ce 5221 struct evergreen_mc_save save;
cc066715
AD		 5222 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5223 u32 tmp;
21a93e13 5224
cc066715
AD		 5225 if (reset_mask == 0)
5226 return;
21a93e13 5227
cc066715 5228 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
21a93e13 5229
cc066715
AD		 5230 cik_print_gpu_status_regs(rdev);
5231 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5232 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5233 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5234 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
21a93e13 5235
fb2c7f4d
AD		 5236 /* disable CG/PG */
5237 cik_fini_pg(rdev);
5238 cik_fini_cg(rdev);
5239
cc066715
AD		 5240 /* stop the rlc */
5241 cik_rlc_stop(rdev);
21a93e13 5242
cc066715
AD		 5243 /* Disable GFX parsing/prefetching */
5244 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
21a93e13 5245
cc066715
AD		 5246 /* Disable MEC parsing/prefetching */
5247 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
21a93e13 5248
cc066715
AD		 5249 if (reset_mask & RADEON_RESET_DMA) {
5250 /* sdma0 */
5251 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5252 tmp |= SDMA_HALT;
5253 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5254 }
5255 if (reset_mask & RADEON_RESET_DMA1) {
5256 /* sdma1 */
5257 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5258 tmp |= SDMA_HALT;
5259 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5260 }
21a93e13 5261
6f2043ce 5262 evergreen_mc_stop(rdev, &save);
cc066715 5263 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 5264 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5265 }
21a93e13 5266
cc066715
AD		 5267 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5268 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
21a93e13 5269
cc066715
AD		 5270 if (reset_mask & RADEON_RESET_CP) {
5271 grbm_soft_reset |= SOFT_RESET_CP;
21a93e13 5272
cc066715
AD		 5273 srbm_soft_reset |= SOFT_RESET_GRBM;
5274 }
21a93e13 5275
cc066715
AD		 5276 if (reset_mask & RADEON_RESET_DMA)
5277 srbm_soft_reset |= SOFT_RESET_SDMA;
21a93e13 5278
cc066715
AD		 5279 if (reset_mask & RADEON_RESET_DMA1)
5280 srbm_soft_reset |= SOFT_RESET_SDMA1;
5281
5282 if (reset_mask & RADEON_RESET_DISPLAY)
5283 srbm_soft_reset |= SOFT_RESET_DC;
5284
5285 if (reset_mask & RADEON_RESET_RLC)
5286 grbm_soft_reset |= SOFT_RESET_RLC;
5287
5288 if (reset_mask & RADEON_RESET_SEM)
5289 srbm_soft_reset |= SOFT_RESET_SEM;
5290
5291 if (reset_mask & RADEON_RESET_IH)
5292 srbm_soft_reset |= SOFT_RESET_IH;
5293
5294 if (reset_mask & RADEON_RESET_GRBM)
5295 srbm_soft_reset |= SOFT_RESET_GRBM;
5296
5297 if (reset_mask & RADEON_RESET_VMC)
5298 srbm_soft_reset |= SOFT_RESET_VMC;
5299
5300 if (!(rdev->flags & RADEON_IS_IGP)) {
5301 if (reset_mask & RADEON_RESET_MC)
5302 srbm_soft_reset |= SOFT_RESET_MC;
21a93e13
AD		 5303 }
5304
cc066715
AD		 5305 if (grbm_soft_reset) {
5306 tmp = RREG32(GRBM_SOFT_RESET);
5307 tmp |= grbm_soft_reset;
5308 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5309 WREG32(GRBM_SOFT_RESET, tmp);
5310 tmp = RREG32(GRBM_SOFT_RESET);
21a93e13 5311
cc066715 5312 udelay(50);
21a93e13 5313
cc066715
AD		 5314 tmp &= ~grbm_soft_reset;
5315 WREG32(GRBM_SOFT_RESET, tmp);
5316 tmp = RREG32(GRBM_SOFT_RESET);
5317 }
21a93e13 5318
cc066715
AD		 5319 if (srbm_soft_reset) {
5320 tmp = RREG32(SRBM_SOFT_RESET);
5321 tmp |= srbm_soft_reset;
5322 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5323 WREG32(SRBM_SOFT_RESET, tmp);
5324 tmp = RREG32(SRBM_SOFT_RESET);
21a93e13 5325
cc066715 5326 udelay(50);
21a93e13 5327
cc066715
AD		 5328 tmp &= ~srbm_soft_reset;
5329 WREG32(SRBM_SOFT_RESET, tmp);
5330 tmp = RREG32(SRBM_SOFT_RESET);
5331 }
21a93e13 5332
6f2043ce
AD		 5333 /* Wait a little for things to settle down */
5334 udelay(50);
21a93e13 5335
6f2043ce 5336 evergreen_mc_resume(rdev, &save);
cc066715
AD		 5337 udelay(50);
5338
5339 cik_print_gpu_status_regs(rdev);
21a93e13
AD		 5340}
5341
0279ed19
AD		 5342struct kv_reset_save_regs {
5343 u32 gmcon_reng_execute;
5344 u32 gmcon_misc;
5345 u32 gmcon_misc3;
5346};
5347
5348static void kv_save_regs_for_reset(struct radeon_device *rdev,
5349 struct kv_reset_save_regs *save)
5350{
5351 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5352 save->gmcon_misc = RREG32(GMCON_MISC);
5353 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5354
5355 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5356 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5357 STCTRL_STUTTER_EN));
5358}
5359
5360static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5361 struct kv_reset_save_regs *save)
5362{
5363 int i;
5364
5365 WREG32(GMCON_PGFSM_WRITE, 0);
5366 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5367
5368 for (i = 0; i < 5; i++)
5369 WREG32(GMCON_PGFSM_WRITE, 0);
5370
5371 WREG32(GMCON_PGFSM_WRITE, 0);
5372 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5373
5374 for (i = 0; i < 5; i++)
5375 WREG32(GMCON_PGFSM_WRITE, 0);
5376
5377 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5378 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5379
5380 for (i = 0; i < 5; i++)
5381 WREG32(GMCON_PGFSM_WRITE, 0);
5382
5383 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5384 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5385
5386 for (i = 0; i < 5; i++)
5387 WREG32(GMCON_PGFSM_WRITE, 0);
5388
5389 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5390 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5391
5392 for (i = 0; i < 5; i++)
5393 WREG32(GMCON_PGFSM_WRITE, 0);
5394
5395 WREG32(GMCON_PGFSM_WRITE, 0);
5396 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5397
5398 for (i = 0; i < 5; i++)
5399 WREG32(GMCON_PGFSM_WRITE, 0);
5400
5401 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5402 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5403
5404 for (i = 0; i < 5; i++)
5405 WREG32(GMCON_PGFSM_WRITE, 0);
5406
5407 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5408 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5409
5410 for (i = 0; i < 5; i++)
5411 WREG32(GMCON_PGFSM_WRITE, 0);
5412
5413 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5414 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5415
5416 for (i = 0; i < 5; i++)
5417 WREG32(GMCON_PGFSM_WRITE, 0);
5418
5419 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5420 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5421
5422 for (i = 0; i < 5; i++)
5423 WREG32(GMCON_PGFSM_WRITE, 0);
5424
5425 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5426 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5427
5428 WREG32(GMCON_MISC3, save->gmcon_misc3);
5429 WREG32(GMCON_MISC, save->gmcon_misc);
5430 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5431}
5432
5433static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5434{
5435 struct evergreen_mc_save save;
5436 struct kv_reset_save_regs kv_save = { 0 };
5437 u32 tmp, i;
5438
5439 dev_info(rdev->dev, "GPU pci config reset\n");
5440
5441 /* disable dpm? */
5442
5443 /* disable cg/pg */
5444 cik_fini_pg(rdev);
5445 cik_fini_cg(rdev);
5446
5447 /* Disable GFX parsing/prefetching */
5448 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5449
5450 /* Disable MEC parsing/prefetching */
5451 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5452
5453 /* sdma0 */
5454 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5455 tmp |= SDMA_HALT;
5456 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5457 /* sdma1 */
5458 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5459 tmp |= SDMA_HALT;
5460 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5461 /* XXX other engines? */
5462
5463 /* halt the rlc, disable cp internal ints */
5464 cik_rlc_stop(rdev);
5465
5466 udelay(50);
5467
5468 /* disable mem access */
5469 evergreen_mc_stop(rdev, &save);
5470 if (evergreen_mc_wait_for_idle(rdev)) {
5471 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5472 }
5473
5474 if (rdev->flags & RADEON_IS_IGP)
5475 kv_save_regs_for_reset(rdev, &kv_save);
5476
5477 /* disable BM */
5478 pci_clear_master(rdev->pdev);
5479 /* reset */
5480 radeon_pci_config_reset(rdev);
5481
5482 udelay(100);
5483
5484 /* wait for asic to come out of reset */
5485 for (i = 0; i < rdev->usec_timeout; i++) {
5486 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5487 break;
5488 udelay(1);
5489 }
5490
5491 /* does asic init need to be run first??? */
5492 if (rdev->flags & RADEON_IS_IGP)
5493 kv_restore_regs_for_reset(rdev, &kv_save);
5494}
5495
21a93e13 5496/**
cc066715 5497 * cik_asic_reset - soft reset GPU
21a93e13
AD		 5498 *
5499 * @rdev: radeon_device pointer
5500 *
cc066715
AD		 5501 * Look up which blocks are hung and attempt
5502 * to reset them.
6f2043ce 5503 * Returns 0 for success.
21a93e13 5504 */
6f2043ce 5505int cik_asic_reset(struct radeon_device *rdev)
21a93e13 5506{
cc066715 5507 u32 reset_mask;
21a93e13 5508
cc066715 5509 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5510
cc066715
AD		 5511 if (reset_mask)
5512 r600_set_bios_scratch_engine_hung(rdev, true);
21a93e13 5513
0279ed19 5514 /* try soft reset */
cc066715 5515 cik_gpu_soft_reset(rdev, reset_mask);
21a93e13 5516
cc066715
AD		 5517 reset_mask = cik_gpu_check_soft_reset(rdev);
5518
0279ed19
AD		 5519 /* try pci config reset */
5520 if (reset_mask && radeon_hard_reset)
5521 cik_gpu_pci_config_reset(rdev);
5522
5523 reset_mask = cik_gpu_check_soft_reset(rdev);
5524
cc066715
AD		 5525 if (!reset_mask)
5526 r600_set_bios_scratch_engine_hung(rdev, false);
21a93e13
AD		 5527
5528 return 0;
5529}
5530
5531/**
cc066715 5532 * cik_gfx_is_lockup - check if the 3D engine is locked up
21a93e13
AD		 5533 *
5534 * @rdev: radeon_device pointer
cc066715 5535 * @ring: radeon_ring structure holding ring information
21a93e13 5536 *
cc066715
AD		 5537 * Check if the 3D engine is locked up (CIK).
5538 * Returns true if the engine is locked, false if not.
21a93e13 5539 */
cc066715 5540bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
21a93e13 5541{
cc066715 5542 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5543
cc066715
AD		 5544 if (!(reset_mask & (RADEON_RESET_GFX |
5545 RADEON_RESET_COMPUTE |
5546 RADEON_RESET_CP))) {
ff212f25 5547 radeon_ring_lockup_update(rdev, ring);
cc066715 5548 return false;
21a93e13 5549 }
cc066715 5550 return radeon_ring_test_lockup(rdev, ring);
21a93e13
AD		 5551}
5552
1c49165d 5553/* MC */
21a93e13 5554/**
1c49165d 5555 * cik_mc_program - program the GPU memory controller
21a93e13
AD		 5556 *
5557 * @rdev: radeon_device pointer
21a93e13 5558 *
1c49165d
AD		 5559 * Set the location of vram, gart, and AGP in the GPU's
5560 * physical address space (CIK).
21a93e13 5561 */
1c49165d 5562static void cik_mc_program(struct radeon_device *rdev)
21a93e13 5563{
1c49165d 5564 struct evergreen_mc_save save;
21a93e13 5565 u32 tmp;
1c49165d 5566 int i, j;
21a93e13 5567
1c49165d
AD		 5568 /* Initialize HDP */
5569 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5570 WREG32((0x2c14 + j), 0x00000000);
5571 WREG32((0x2c18 + j), 0x00000000);
5572 WREG32((0x2c1c + j), 0x00000000);
5573 WREG32((0x2c20 + j), 0x00000000);
5574 WREG32((0x2c24 + j), 0x00000000);
21a93e13 5575 }
1c49165d 5576 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
21a93e13 5577
1c49165d
AD		 5578 evergreen_mc_stop(rdev, &save);
5579 if (radeon_mc_wait_for_idle(rdev)) {
5580 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5581 }
1c49165d
AD		 5582 /* Lockout access through VGA aperture*/
5583 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5584 /* Update configuration */
5585 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5586 rdev->mc.vram_start >> 12);
5587 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5588 rdev->mc.vram_end >> 12);
5589 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5590 rdev->vram_scratch.gpu_addr >> 12);
5591 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5592 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5593 WREG32(MC_VM_FB_LOCATION, tmp);
5594 /* XXX double check these! */
5595 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5596 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5597 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5598 WREG32(MC_VM_AGP_BASE, 0);
5599 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5600 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5601 if (radeon_mc_wait_for_idle(rdev)) {
5602 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5603 }
1c49165d
AD		 5604 evergreen_mc_resume(rdev, &save);
5605 /* we need to own VRAM, so turn off the VGA renderer here
5606 * to stop it overwriting our objects */
5607 rv515_vga_render_disable(rdev);
21a93e13
AD		 5608}
5609
5610/**
1c49165d 5611 * cik_mc_init - initialize the memory controller driver params
21a93e13
AD		 5612 *
5613 * @rdev: radeon_device pointer
21a93e13 5614 *
1c49165d
AD		 5615 * Look up the amount of vram, vram width, and decide how to place
5616 * vram and gart within the GPU's physical address space (CIK).
5617 * Returns 0 for success.
21a93e13 5618 */
1c49165d 5619static int cik_mc_init(struct radeon_device *rdev)
21a93e13 5620{
1c49165d
AD		 5621 u32 tmp;
5622 int chansize, numchan;
21a93e13 5623
1c49165d
AD		 5624 /* Get VRAM informations */
5625 rdev->mc.vram_is_ddr = true;
5626 tmp = RREG32(MC_ARB_RAMCFG);
5627 if (tmp & CHANSIZE_MASK) {
5628 chansize = 64;
21a93e13 5629 } else {
1c49165d 5630 chansize = 32;
21a93e13 5631 }
1c49165d
AD		 5632 tmp = RREG32(MC_SHARED_CHMAP);
5633 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5634 case 0:
5635 default:
5636 numchan = 1;
5637 break;
5638 case 1:
5639 numchan = 2;
5640 break;
5641 case 2:
5642 numchan = 4;
5643 break;
5644 case 3:
5645 numchan = 8;
5646 break;
5647 case 4:
5648 numchan = 3;
5649 break;
5650 case 5:
5651 numchan = 6;
5652 break;
5653 case 6:
5654 numchan = 10;
5655 break;
5656 case 7:
5657 numchan = 12;
5658 break;
5659 case 8:
5660 numchan = 16;
5661 break;
5662 }
5663 rdev->mc.vram_width = numchan * chansize;
5664 /* Could aper size report 0 ? */
5665 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5666 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5667 /* size in MB on si */
13c5bfda
AD		 5668 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5669 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1c49165d
AD		 5670 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5671 si_vram_gtt_location(rdev, &rdev->mc);
5672 radeon_update_bandwidth_info(rdev);
5673
5674 return 0;
5675}
5676
5677/*
5678 * GART
5679 * VMID 0 is the physical GPU addresses as used by the kernel.
5680 * VMIDs 1-15 are used for userspace clients and are handled
5681 * by the radeon vm/hsa code.
5682 */
5683/**
5684 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5685 *
5686 * @rdev: radeon_device pointer
5687 *
5688 * Flush the TLB for the VMID 0 page table (CIK).
5689 */
5690void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5691{
5692 /* flush hdp cache */
5693 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5694
5695 /* bits 0-15 are the VM contexts0-15 */
5696 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5697}
5698
5699/**
5700 * cik_pcie_gart_enable - gart enable
5701 *
5702 * @rdev: radeon_device pointer
5703 *
5704 * This sets up the TLBs, programs the page tables for VMID0,
5705 * sets up the hw for VMIDs 1-15 which are allocated on
5706 * demand, and sets up the global locations for the LDS, GDS,
5707 * and GPUVM for FSA64 clients (CIK).
5708 * Returns 0 for success, errors for failure.
5709 */
5710static int cik_pcie_gart_enable(struct radeon_device *rdev)
5711{
5712 int r, i;
5713
5714 if (rdev->gart.robj == NULL) {
5715 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5716 return -EINVAL;
5717 }
5718 r = radeon_gart_table_vram_pin(rdev);
5719 if (r)
5720 return r;
1c49165d
AD		 5721 /* Setup TLB control */
5722 WREG32(MC_VM_MX_L1_TLB_CNTL,
5723 (0xA << 7) |
5724 ENABLE_L1_TLB |
ec3dbbcb 5725 ENABLE_L1_FRAGMENT_PROCESSING |
1c49165d
AD		 5726 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5727 ENABLE_ADVANCED_DRIVER_MODEL |
5728 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5729 /* Setup L2 cache */
5730 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5731 ENABLE_L2_FRAGMENT_PROCESSING |
5732 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5733 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5734 EFFECTIVE_L2_QUEUE_SIZE(7) |
5735 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5736 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5737 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
ec3dbbcb
CK		 5738 BANK_SELECT(4) |
5739 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
1c49165d
AD		 5740 /* setup context0 */
5741 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5742 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5743 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5744 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5745 (u32)(rdev->dummy_page.addr >> 12));
5746 WREG32(VM_CONTEXT0_CNTL2, 0);
5747 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5748 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5749
5750 WREG32(0x15D4, 0);
5751 WREG32(0x15D8, 0);
5752 WREG32(0x15DC, 0);
5753
054e01d6 5754 /* restore context1-15 */
1c49165d
AD		 5755 /* set vm size, must be a multiple of 4 */
5756 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5757 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5758 for (i = 1; i < 16; i++) {
5759 if (i < 8)
5760 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
054e01d6 5761 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5762 else
5763 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
054e01d6 5764 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5765 }
5766
5767 /* enable context1-15 */
5768 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5769 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 5770 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 5771 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4510fb98 5772 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
a00024b0
AD		 5773 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5774 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5775 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5776 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5777 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5778 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5779 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5780 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5781 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5782 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5783 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5784 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d 5785
1c49165d
AD		 5786 if (rdev->family == CHIP_KAVERI) {
5787 u32 tmp = RREG32(CHUB_CONTROL);
5788 tmp &= ~BYPASS_VM;
5789 WREG32(CHUB_CONTROL, tmp);
5790 }
5791
5792 /* XXX SH_MEM regs */
5793 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 5794 mutex_lock(&rdev->srbm_mutex);
1c49165d 5795 for (i = 0; i < 16; i++) {
b556b12e 5796 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 5797 /* CP and shaders */
1c49165d
AD		 5798 WREG32(SH_MEM_CONFIG, 0);
5799 WREG32(SH_MEM_APE1_BASE, 1);
5800 WREG32(SH_MEM_APE1_LIMIT, 0);
5801 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 5802 /* SDMA GFX */
5803 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5804 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5805 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5806 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5807 /* XXX SDMA RLC - todo */
1c49165d 5808 }
b556b12e 5809 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5810 mutex_unlock(&rdev->srbm_mutex);
1c49165d
AD		 5811
5812 cik_pcie_gart_tlb_flush(rdev);
5813 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5814 (unsigned)(rdev->mc.gtt_size >> 20),
5815 (unsigned long long)rdev->gart.table_addr);
5816 rdev->gart.ready = true;
5817 return 0;
5818}
5819
5820/**
5821 * cik_pcie_gart_disable - gart disable
5822 *
5823 * @rdev: radeon_device pointer
5824 *
5825 * This disables all VM page table (CIK).
5826 */
5827static void cik_pcie_gart_disable(struct radeon_device *rdev)
5828{
054e01d6
CK		 5829 unsigned i;
5830
5831 for (i = 1; i < 16; ++i) {
5832 uint32_t reg;
5833 if (i < 8)
5834 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5835 else
5836 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5837 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5838 }
5839
1c49165d
AD		 5840 /* Disable all tables */
5841 WREG32(VM_CONTEXT0_CNTL, 0);
5842 WREG32(VM_CONTEXT1_CNTL, 0);
5843 /* Setup TLB control */
5844 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5845 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5846 /* Setup L2 cache */
5847 WREG32(VM_L2_CNTL,
5848 ENABLE_L2_FRAGMENT_PROCESSING |
5849 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5850 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5851 EFFECTIVE_L2_QUEUE_SIZE(7) |
5852 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5853 WREG32(VM_L2_CNTL2, 0);
5854 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5855 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5856 radeon_gart_table_vram_unpin(rdev);
5857}
5858
5859/**
5860 * cik_pcie_gart_fini - vm fini callback
5861 *
5862 * @rdev: radeon_device pointer
5863 *
5864 * Tears down the driver GART/VM setup (CIK).
5865 */
5866static void cik_pcie_gart_fini(struct radeon_device *rdev)
5867{
5868 cik_pcie_gart_disable(rdev);
5869 radeon_gart_table_vram_free(rdev);
5870 radeon_gart_fini(rdev);
5871}
5872
5873/* vm parser */
5874/**
5875 * cik_ib_parse - vm ib_parse callback
5876 *
5877 * @rdev: radeon_device pointer
5878 * @ib: indirect buffer pointer
5879 *
5880 * CIK uses hw IB checking so this is a nop (CIK).
5881 */
5882int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5883{
5884 return 0;
5885}
5886
5887/*
5888 * vm
5889 * VMID 0 is the physical GPU addresses as used by the kernel.
5890 * VMIDs 1-15 are used for userspace clients and are handled
5891 * by the radeon vm/hsa code.
5892 */
5893/**
5894 * cik_vm_init - cik vm init callback
5895 *
5896 * @rdev: radeon_device pointer
5897 *
5898 * Inits cik specific vm parameters (number of VMs, base of vram for
5899 * VMIDs 1-15) (CIK).
5900 * Returns 0 for success.
5901 */
5902int cik_vm_init(struct radeon_device *rdev)
5903{
62a7b7fb
OG		 5904 /*
5905 * number of VMs
5906 * VMID 0 is reserved for System
5907 * radeon graphics/compute will use VMIDs 1-7
5908 * amdkfd will use VMIDs 8-15
5909 */
5910 rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS;
1c49165d
AD		 5911 /* base offset of vram pages */
5912 if (rdev->flags & RADEON_IS_IGP) {
5913 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5914 tmp <<= 22;
5915 rdev->vm_manager.vram_base_offset = tmp;
5916 } else
5917 rdev->vm_manager.vram_base_offset = 0;
5918
5919 return 0;
5920}
5921
5922/**
5923 * cik_vm_fini - cik vm fini callback
5924 *
5925 * @rdev: radeon_device pointer
5926 *
5927 * Tear down any asic specific VM setup (CIK).
5928 */
5929void cik_vm_fini(struct radeon_device *rdev)
5930{
5931}
5932
3ec7d11b
AD		 5933/**
5934 * cik_vm_decode_fault - print human readable fault info
5935 *
5936 * @rdev: radeon_device pointer
5937 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5938 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5939 *
5940 * Print human readable fault information (CIK).
5941 */
5942static void cik_vm_decode_fault(struct radeon_device *rdev,
5943 u32 status, u32 addr, u32 mc_client)
5944{
939c0d3c 5945 u32 mc_id;
3ec7d11b
AD		 5946 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5947 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
328a50c7
MD		 5948 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5949 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
3ec7d11b 5950
939c0d3c
AD		 5951 if (rdev->family == CHIP_HAWAII)
5952 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5953 else
5954 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5955
328a50c7 5956 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
3ec7d11b
AD		 5957 protections, vmid, addr,
5958 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
328a50c7 5959 block, mc_client, mc_id);
3ec7d11b
AD		 5960}
5961
f96ab484
AD		 5962/**
5963 * cik_vm_flush - cik vm flush using the CP
5964 *
5965 * @rdev: radeon_device pointer
5966 *
5967 * Update the page table base and flush the VM TLB
5968 * using the CP (CIK).
5969 */
5970void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5971{
5972 struct radeon_ring *ring = &rdev->ring[ridx];
f1d2a26b 5973 int usepfp = (ridx == RADEON_RING_TYPE_GFX_INDEX);
f96ab484
AD		 5974
5975 if (vm == NULL)
5976 return;
5977
5978 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
f1d2a26b 5979 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5980 WRITE_DATA_DST_SEL(0)));
5981 if (vm->id < 8) {
5982 radeon_ring_write(ring,
5983 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5984 } else {
5985 radeon_ring_write(ring,
5986 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5987 }
5988 radeon_ring_write(ring, 0);
5989 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5990
5991 /* update SH_MEM_* regs */
5992 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5993 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5994 WRITE_DATA_DST_SEL(0)));
5995 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5996 radeon_ring_write(ring, 0);
5997 radeon_ring_write(ring, VMID(vm->id));
5998
5999 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4fb0bbd5 6000 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6001 WRITE_DATA_DST_SEL(0)));
6002 radeon_ring_write(ring, SH_MEM_BASES >> 2);
6003 radeon_ring_write(ring, 0);
6004
6005 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
6006 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
6007 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
6008 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
21a93e13 6009
f96ab484 6010 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 6011 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6012 WRITE_DATA_DST_SEL(0)));
6013 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
6014 radeon_ring_write(ring, 0);
6015 radeon_ring_write(ring, VMID(0));
6f2043ce 6016
f96ab484 6017 /* HDP flush */
780f5ddd 6018 cik_hdp_flush_cp_ring_emit(rdev, ridx);
f96ab484
AD		 6019
6020 /* bits 0-15 are the VM contexts0-15 */
6021 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 6022 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6023 WRITE_DATA_DST_SEL(0)));
6024 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6025 radeon_ring_write(ring, 0);
6026 radeon_ring_write(ring, 1 << vm->id);
6027
b07fdd38 6028 /* compute doesn't have PFP */
f1d2a26b 6029 if (usepfp) {
b07fdd38
AD		 6030 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6031 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6032 radeon_ring_write(ring, 0x0);
6033 }
cc066715 6034}
6f2043ce 6035
f6796cae
AD		 6036/*
6037 * RLC
6038 * The RLC is a multi-purpose microengine that handles a
6039 * variety of functions, the most important of which is
6040 * the interrupt controller.
6041 */
866d83de
AD		 6042static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6043 bool enable)
f6796cae 6044{
866d83de 6045 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 6046
866d83de
AD		 6047 if (enable)
6048 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6049 else
6050 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 6051 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 6052}
f6796cae 6053
866d83de 6054static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
cc066715 6055{
cc066715 6056 u32 tmp;
6f2043ce 6057
866d83de
AD		 6058 tmp = RREG32(RLC_LB_CNTL);
6059 if (enable)
6060 tmp |= LOAD_BALANCE_ENABLE;
6061 else
6062 tmp &= ~LOAD_BALANCE_ENABLE;
6063 WREG32(RLC_LB_CNTL, tmp);
6064}
cc066715 6065
866d83de
AD		 6066static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6067{
6068 u32 i, j, k;
6069 u32 mask;
cc066715 6070
f6796cae
AD		 6071 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6072 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6073 cik_select_se_sh(rdev, i, j);
6074 for (k = 0; k < rdev->usec_timeout; k++) {
6075 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6076 break;
6077 udelay(1);
6078 }
6079 }
6080 }
6081 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
cc066715 6082
f6796cae
AD		 6083 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6084 for (k = 0; k < rdev->usec_timeout; k++) {
6085 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6086 break;
6087 udelay(1);
6088 }
6089}
cc066715 6090
22c775ce
AD		 6091static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6092{
6093 u32 tmp;
cc066715 6094
22c775ce
AD		 6095 tmp = RREG32(RLC_CNTL);
6096 if (tmp != rlc)
6097 WREG32(RLC_CNTL, rlc);
6098}
cc066715 6099
22c775ce
AD		 6100static u32 cik_halt_rlc(struct radeon_device *rdev)
6101{
6102 u32 data, orig;
cc066715 6103
22c775ce 6104 orig = data = RREG32(RLC_CNTL);
cc066715 6105
22c775ce
AD		 6106 if (data & RLC_ENABLE) {
6107 u32 i;
cc066715 6108
22c775ce
AD		 6109 data &= ~RLC_ENABLE;
6110 WREG32(RLC_CNTL, data);
cc066715 6111
22c775ce
AD		 6112 for (i = 0; i < rdev->usec_timeout; i++) {
6113 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6114 break;
6115 udelay(1);
6116 }
cc066715 6117
22c775ce
AD		 6118 cik_wait_for_rlc_serdes(rdev);
6119 }
cc066715 6120
22c775ce
AD		 6121 return orig;
6122}
cc066715 6123
a412fce0
AD		 6124void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6125{
6126 u32 tmp, i, mask;
6127
6128 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6129 WREG32(RLC_GPR_REG2, tmp);
6130
6131 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6132 for (i = 0; i < rdev->usec_timeout; i++) {
6133 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6134 break;
6135 udelay(1);
6136 }
6137
6138 for (i = 0; i < rdev->usec_timeout; i++) {
6139 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6140 break;
6141 udelay(1);
6142 }
6143}
6144
6145void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6146{
6147 u32 tmp;
6148
6149 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6150 WREG32(RLC_GPR_REG2, tmp);
6151}
6152
866d83de
AD		 6153/**
6154 * cik_rlc_stop - stop the RLC ME
6155 *
6156 * @rdev: radeon_device pointer
6157 *
6158 * Halt the RLC ME (MicroEngine) (CIK).
6159 */
6160static void cik_rlc_stop(struct radeon_device *rdev)
6161{
22c775ce 6162 WREG32(RLC_CNTL, 0);
866d83de
AD		 6163
6164 cik_enable_gui_idle_interrupt(rdev, false);
6165
866d83de
AD		 6166 cik_wait_for_rlc_serdes(rdev);
6167}
6168
f6796cae
AD		 6169/**
6170 * cik_rlc_start - start the RLC ME
6171 *
6172 * @rdev: radeon_device pointer
6173 *
6174 * Unhalt the RLC ME (MicroEngine) (CIK).
6175 */
6176static void cik_rlc_start(struct radeon_device *rdev)
6177{
f6796cae 6178 WREG32(RLC_CNTL, RLC_ENABLE);
cc066715 6179
866d83de 6180 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6181
f6796cae 6182 udelay(50);
6f2043ce
AD		 6183}
6184
6185/**
f6796cae 6186 * cik_rlc_resume - setup the RLC hw
6f2043ce
AD		 6187 *
6188 * @rdev: radeon_device pointer
6189 *
f6796cae
AD		 6190 * Initialize the RLC registers, load the ucode,
6191 * and start the RLC (CIK).
6192 * Returns 0 for success, -EINVAL if the ucode is not available.
6f2043ce 6193 */
f6796cae 6194static int cik_rlc_resume(struct radeon_device *rdev)
6f2043ce 6195{
22c775ce 6196 u32 i, size, tmp;
cc066715 6197
f6796cae
AD		 6198 if (!rdev->rlc_fw)
6199 return -EINVAL;
cc066715 6200
cc066715
AD		 6201 cik_rlc_stop(rdev);
6202
22c775ce
AD		 6203 /* disable CG */
6204 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6205 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
cc066715 6206
866d83de 6207 si_rlc_reset(rdev);
6f2043ce 6208
22c775ce 6209 cik_init_pg(rdev);
6f2043ce 6210
22c775ce 6211 cik_init_cg(rdev);
cc066715 6212
f6796cae
AD		 6213 WREG32(RLC_LB_CNTR_INIT, 0);
6214 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
cc066715 6215
f6796cae
AD		 6216 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6217 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6218 WREG32(RLC_LB_PARAMS, 0x00600408);
6219 WREG32(RLC_LB_CNTL, 0x80000004);
cc066715 6220
f6796cae
AD		 6221 WREG32(RLC_MC_CNTL, 0);
6222 WREG32(RLC_UCODE_CNTL, 0);
cc066715 6223
f2c6b0f4
AD		 6224 if (rdev->new_fw) {
6225 const struct rlc_firmware_header_v1_0 *hdr =
6226 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6227 const __le32 *fw_data = (const __le32 *)
6228 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6229
6230 radeon_ucode_print_rlc_hdr(&hdr->header);
6231
6232 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6233 WREG32(RLC_GPM_UCODE_ADDR, 0);
6234 for (i = 0; i < size; i++)
6235 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 6236 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
f2c6b0f4
AD		 6237 } else {
6238 const __be32 *fw_data;
6239
6240 switch (rdev->family) {
6241 case CHIP_BONAIRE:
6242 case CHIP_HAWAII:
6243 default:
6244 size = BONAIRE_RLC_UCODE_SIZE;
6245 break;
6246 case CHIP_KAVERI:
6247 size = KV_RLC_UCODE_SIZE;
6248 break;
6249 case CHIP_KABINI:
6250 size = KB_RLC_UCODE_SIZE;
6251 break;
6252 case CHIP_MULLINS:
6253 size = ML_RLC_UCODE_SIZE;
6254 break;
6255 }
6256
6257 fw_data = (const __be32 *)rdev->rlc_fw->data;
6258 WREG32(RLC_GPM_UCODE_ADDR, 0);
6259 for (i = 0; i < size; i++)
6260 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6261 WREG32(RLC_GPM_UCODE_ADDR, 0);
6262 }
cc066715 6263
866d83de
AD		 6264 /* XXX - find out what chips support lbpw */
6265 cik_enable_lbpw(rdev, false);
cc066715 6266
22c775ce
AD		 6267 if (rdev->family == CHIP_BONAIRE)
6268 WREG32(RLC_DRIVER_DMA_STATUS, 0);
cc066715 6269
f6796cae 6270 cik_rlc_start(rdev);
cc066715 6271
f6796cae
AD		 6272 return 0;
6273}
cc066715 6274
22c775ce
AD		 6275static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6276{
6277 u32 data, orig, tmp, tmp2;
cc066715 6278
22c775ce 6279 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
cc066715 6280
473359bc 6281 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6 6282 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6283
22c775ce 6284 tmp = cik_halt_rlc(rdev);
cc066715 6285
22c775ce
AD		 6286 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6287 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6288 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6289 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6290 WREG32(RLC_SERDES_WR_CTRL, tmp2);
cc066715 6291
22c775ce 6292 cik_update_rlc(rdev, tmp);
cc066715 6293
22c775ce
AD		 6294 data |= CGCG_EN | CGLS_EN;
6295 } else {
ddc76ff6 6296 cik_enable_gui_idle_interrupt(rdev, false);
cc066715 6297
22c775ce
AD		 6298 RREG32(CB_CGTT_SCLK_CTRL);
6299 RREG32(CB_CGTT_SCLK_CTRL);
6300 RREG32(CB_CGTT_SCLK_CTRL);
6301 RREG32(CB_CGTT_SCLK_CTRL);
cc066715 6302
22c775ce 6303 data &= ~(CGCG_EN | CGLS_EN);
cc066715 6304 }
6f2043ce 6305
22c775ce
AD		 6306 if (orig != data)
6307 WREG32(RLC_CGCG_CGLS_CTRL, data);
cc066715 6308
6f2043ce
AD		 6309}
6310
22c775ce 6311static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6f2043ce 6312{
22c775ce
AD		 6313 u32 data, orig, tmp = 0;
6314
473359bc
AD		 6315 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6316 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6317 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6318 orig = data = RREG32(CP_MEM_SLP_CNTL);
6319 data |= CP_MEM_LS_EN;
6320 if (orig != data)
6321 WREG32(CP_MEM_SLP_CNTL, data);
6322 }
6323 }
cc066715 6324
22c775ce
AD		 6325 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6326 data &= 0xfffffffd;
6327 if (orig != data)
6328 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6329
6330 tmp = cik_halt_rlc(rdev);
6331
6332 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6333 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6334 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6335 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6336 WREG32(RLC_SERDES_WR_CTRL, data);
6337
6338 cik_update_rlc(rdev, tmp);
6339
473359bc
AD		 6340 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6341 orig = data = RREG32(CGTS_SM_CTRL_REG);
6342 data &= ~SM_MODE_MASK;
6343 data |= SM_MODE(0x2);
6344 data |= SM_MODE_ENABLE;
6345 data &= ~CGTS_OVERRIDE;
6346 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6347 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6348 data &= ~CGTS_LS_OVERRIDE;
6349 data &= ~ON_MONITOR_ADD_MASK;
6350 data |= ON_MONITOR_ADD_EN;
6351 data |= ON_MONITOR_ADD(0x96);
6352 if (orig != data)
6353 WREG32(CGTS_SM_CTRL_REG, data);
6354 }
22c775ce
AD		 6355 } else {
6356 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6357 data |= 0x00000002;
6358 if (orig != data)
6359 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6360
6361 data = RREG32(RLC_MEM_SLP_CNTL);
6362 if (data & RLC_MEM_LS_EN) {
6363 data &= ~RLC_MEM_LS_EN;
6364 WREG32(RLC_MEM_SLP_CNTL, data);
6365 }
6f2043ce 6366
22c775ce
AD		 6367 data = RREG32(CP_MEM_SLP_CNTL);
6368 if (data & CP_MEM_LS_EN) {
6369 data &= ~CP_MEM_LS_EN;
6370 WREG32(CP_MEM_SLP_CNTL, data);
6371 }
cc066715 6372
22c775ce
AD		 6373 orig = data = RREG32(CGTS_SM_CTRL_REG);
6374 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6375 if (orig != data)
6376 WREG32(CGTS_SM_CTRL_REG, data);
cc066715 6377
22c775ce 6378 tmp = cik_halt_rlc(rdev);
cc066715 6379
22c775ce
AD		 6380 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6381 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6382 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6383 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6384 WREG32(RLC_SERDES_WR_CTRL, data);
cc066715 6385
22c775ce 6386 cik_update_rlc(rdev, tmp);
cc066715 6387 }
6f2043ce 6388}
1c49165d 6389
22c775ce 6390static const u32 mc_cg_registers[] =
21a93e13 6391{
22c775ce
AD		 6392 MC_HUB_MISC_HUB_CG,
6393 MC_HUB_MISC_SIP_CG,
6394 MC_HUB_MISC_VM_CG,
6395 MC_XPB_CLK_GAT,
6396 ATC_MISC_CG,
6397 MC_CITF_MISC_WR_CG,
6398 MC_CITF_MISC_RD_CG,
6399 MC_CITF_MISC_VM_CG,
6400 VM_L2_CG,
6401};
21a93e13 6402
22c775ce
AD		 6403static void cik_enable_mc_ls(struct radeon_device *rdev,
6404 bool enable)
1c49165d 6405{
22c775ce
AD		 6406 int i;
6407 u32 orig, data;
1c49165d 6408
22c775ce
AD		 6409 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6410 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6411 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 6412 data |= MC_LS_ENABLE;
6413 else
6414 data &= ~MC_LS_ENABLE;
6415 if (data != orig)
6416 WREG32(mc_cg_registers[i], data);
1c49165d 6417 }
22c775ce 6418}
1c49165d 6419
22c775ce
AD		 6420static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6421 bool enable)
6422{
6423 int i;
6424 u32 orig, data;
6425
6426 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6427 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6428 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 6429 data |= MC_CG_ENABLE;
6430 else
6431 data &= ~MC_CG_ENABLE;
6432 if (data != orig)
6433 WREG32(mc_cg_registers[i], data);
1c49165d 6434 }
1c49165d
AD		 6435}
6436
22c775ce
AD		 6437static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6438 bool enable)
1c49165d 6439{
22c775ce 6440 u32 orig, data;
1c49165d 6441
473359bc 6442 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 6443 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6444 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
1c49165d 6445 } else {
22c775ce
AD		 6446 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6447 data |= 0xff000000;
6448 if (data != orig)
6449 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6450
22c775ce
AD		 6451 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6452 data |= 0xff000000;
6453 if (data != orig)
6454 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6455 }
1c49165d
AD		 6456}
6457
22c775ce
AD		 6458static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6459 bool enable)
1c49165d 6460{
22c775ce
AD		 6461 u32 orig, data;
6462
473359bc 6463 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 6464 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6465 data |= 0x100;
6466 if (orig != data)
6467 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6468
6469 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6470 data |= 0x100;
6471 if (orig != data)
6472 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6473 } else {
6474 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6475 data &= ~0x100;
6476 if (orig != data)
6477 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6478
22c775ce
AD		 6479 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6480 data &= ~0x100;
6481 if (orig != data)
6482 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6483 }
1c49165d
AD		 6484}
6485
22c775ce
AD		 6486static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6487 bool enable)
1c49165d 6488{
22c775ce 6489 u32 orig, data;
1c49165d 6490
473359bc 6491 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 6492 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6493 data = 0xfff;
6494 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6495
22c775ce
AD		 6496 orig = data = RREG32(UVD_CGC_CTRL);
6497 data |= DCM;
6498 if (orig != data)
6499 WREG32(UVD_CGC_CTRL, data);
6500 } else {
6501 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6502 data &= ~0xfff;
6503 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6504
22c775ce
AD		 6505 orig = data = RREG32(UVD_CGC_CTRL);
6506 data &= ~DCM;
6507 if (orig != data)
6508 WREG32(UVD_CGC_CTRL, data);
1c49165d 6509 }
22c775ce 6510}
1c49165d 6511
473359bc
AD		 6512static void cik_enable_bif_mgls(struct radeon_device *rdev,
6513 bool enable)
6514{
6515 u32 orig, data;
1c49165d 6516
473359bc 6517 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
1c49165d 6518
473359bc
AD		 6519 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6520 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6521 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6522 else
6523 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6524 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
1c49165d 6525
473359bc
AD		 6526 if (orig != data)
6527 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6528}
1c49165d 6529
22c775ce
AD		 6530static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6531 bool enable)
6532{
6533 u32 orig, data;
1c49165d 6534
22c775ce 6535 orig = data = RREG32(HDP_HOST_PATH_CNTL);
1c49165d 6536
473359bc 6537 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 6538 data &= ~CLOCK_GATING_DIS;
6539 else
6540 data |= CLOCK_GATING_DIS;
6541
6542 if (orig != data)
6543 WREG32(HDP_HOST_PATH_CNTL, data);
1c49165d
AD		 6544}
6545
22c775ce
AD		 6546static void cik_enable_hdp_ls(struct radeon_device *rdev,
6547 bool enable)
1c49165d 6548{
22c775ce
AD		 6549 u32 orig, data;
6550
6551 orig = data = RREG32(HDP_MEM_POWER_LS);
6552
473359bc 6553 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 6554 data |= HDP_LS_ENABLE;
6555 else
6556 data &= ~HDP_LS_ENABLE;
6557
6558 if (orig != data)
6559 WREG32(HDP_MEM_POWER_LS, data);
6560}
6561
6562void cik_update_cg(struct radeon_device *rdev,
6563 u32 block, bool enable)
6564{
4214faf6 6565
22c775ce 6566 if (block & RADEON_CG_BLOCK_GFX) {
4214faf6 6567 cik_enable_gui_idle_interrupt(rdev, false);
22c775ce
AD		 6568 /* order matters! */
6569 if (enable) {
6570 cik_enable_mgcg(rdev, true);
6571 cik_enable_cgcg(rdev, true);
6572 } else {
6573 cik_enable_cgcg(rdev, false);
6574 cik_enable_mgcg(rdev, false);
6575 }
4214faf6 6576 cik_enable_gui_idle_interrupt(rdev, true);
22c775ce
AD		 6577 }
6578
6579 if (block & RADEON_CG_BLOCK_MC) {
6580 if (!(rdev->flags & RADEON_IS_IGP)) {
6581 cik_enable_mc_mgcg(rdev, enable);
6582 cik_enable_mc_ls(rdev, enable);
6583 }
6584 }
6585
6586 if (block & RADEON_CG_BLOCK_SDMA) {
6587 cik_enable_sdma_mgcg(rdev, enable);
6588 cik_enable_sdma_mgls(rdev, enable);
6589 }
6590
473359bc
AD		 6591 if (block & RADEON_CG_BLOCK_BIF) {
6592 cik_enable_bif_mgls(rdev, enable);
6593 }
6594
22c775ce
AD		 6595 if (block & RADEON_CG_BLOCK_UVD) {
6596 if (rdev->has_uvd)
6597 cik_enable_uvd_mgcg(rdev, enable);
6598 }
6599
6600 if (block & RADEON_CG_BLOCK_HDP) {
6601 cik_enable_hdp_mgcg(rdev, enable);
6602 cik_enable_hdp_ls(rdev, enable);
6603 }
a1d6f97c
AD		 6604
6605 if (block & RADEON_CG_BLOCK_VCE) {
6606 vce_v2_0_enable_mgcg(rdev, enable);
6607 }
1c49165d
AD		 6608}
6609
22c775ce 6610static void cik_init_cg(struct radeon_device *rdev)
1c49165d 6611{
22c775ce 6612
ddc76ff6 6613 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 6614
6615 if (rdev->has_uvd)
6616 si_init_uvd_internal_cg(rdev);
6617
6618 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6619 RADEON_CG_BLOCK_SDMA |
473359bc 6620 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 6621 RADEON_CG_BLOCK_UVD |
6622 RADEON_CG_BLOCK_HDP), true);
1c49165d
AD		 6623}
6624
473359bc 6625static void cik_fini_cg(struct radeon_device *rdev)
1c49165d 6626{
473359bc
AD		 6627 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6628 RADEON_CG_BLOCK_SDMA |
6629 RADEON_CG_BLOCK_BIF |
6630 RADEON_CG_BLOCK_UVD |
6631 RADEON_CG_BLOCK_HDP), false);
6632
6633 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
1c49165d
AD		 6634}
6635
22c775ce
AD		 6636static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6637 bool enable)
1c49165d 6638{
22c775ce 6639 u32 data, orig;
1c49165d 6640
22c775ce 6641 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6642 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6643 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6644 else
6645 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6646 if (orig != data)
6647 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6648}
6649
22c775ce
AD		 6650static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6651 bool enable)
1c49165d 6652{
22c775ce
AD		 6653 u32 data, orig;
6654
6655 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6656 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6657 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6658 else
6659 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6660 if (orig != data)
6661 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6662}
6663
22c775ce 6664static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
3ec7d11b 6665{
22c775ce 6666 u32 data, orig;
3ec7d11b 6667
22c775ce 6668 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6669 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 6670 data &= ~DISABLE_CP_PG;
6671 else
6672 data |= DISABLE_CP_PG;
6673 if (orig != data)
6674 WREG32(RLC_PG_CNTL, data);
3ec7d11b
AD		 6675}
6676
22c775ce 6677static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
f96ab484 6678{
22c775ce 6679 u32 data, orig;
f96ab484 6680
22c775ce 6681 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6682 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 6683 data &= ~DISABLE_GDS_PG;
6684 else
6685 data |= DISABLE_GDS_PG;
6686 if (orig != data)
6687 WREG32(RLC_PG_CNTL, data);
6688}
6689
6690#define CP_ME_TABLE_SIZE 96
6691#define CP_ME_TABLE_OFFSET 2048
6692#define CP_MEC_TABLE_OFFSET 4096
6693
6694void cik_init_cp_pg_table(struct radeon_device *rdev)
6695{
22c775ce
AD		 6696 volatile u32 *dst_ptr;
6697 int me, i, max_me = 4;
6698 u32 bo_offset = 0;
f2c6b0f4 6699 u32 table_offset, table_size;
22c775ce
AD		 6700
6701 if (rdev->family == CHIP_KAVERI)
6702 max_me = 5;
6703
6704 if (rdev->rlc.cp_table_ptr == NULL)
f96ab484
AD		 6705 return;
6706
22c775ce
AD		 6707 /* write the cp table buffer */
6708 dst_ptr = rdev->rlc.cp_table_ptr;
6709 for (me = 0; me < max_me; me++) {
f2c6b0f4
AD		 6710 if (rdev->new_fw) {
6711 const __le32 *fw_data;
6712 const struct gfx_firmware_header_v1_0 *hdr;
6713
6714 if (me == 0) {
6715 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6716 fw_data = (const __le32 *)
6717 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6718 table_offset = le32_to_cpu(hdr->jt_offset);
6719 table_size = le32_to_cpu(hdr->jt_size);
6720 } else if (me == 1) {
6721 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6722 fw_data = (const __le32 *)
6723 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6724 table_offset = le32_to_cpu(hdr->jt_offset);
6725 table_size = le32_to_cpu(hdr->jt_size);
6726 } else if (me == 2) {
6727 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6728 fw_data = (const __le32 *)
6729 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6730 table_offset = le32_to_cpu(hdr->jt_offset);
6731 table_size = le32_to_cpu(hdr->jt_size);
6732 } else if (me == 3) {
6733 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6734 fw_data = (const __le32 *)
6735 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6736 table_offset = le32_to_cpu(hdr->jt_offset);
6737 table_size = le32_to_cpu(hdr->jt_size);
6738 } else {
6739 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6740 fw_data = (const __le32 *)
6741 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6742 table_offset = le32_to_cpu(hdr->jt_offset);
6743 table_size = le32_to_cpu(hdr->jt_size);
6744 }
6745
6746 for (i = 0; i < table_size; i ++) {
6747 dst_ptr[bo_offset + i] =
6748 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6749 }
6750 bo_offset += table_size;
22c775ce 6751 } else {
f2c6b0f4
AD		 6752 const __be32 *fw_data;
6753 table_size = CP_ME_TABLE_SIZE;
6754
6755 if (me == 0) {
6756 fw_data = (const __be32 *)rdev->ce_fw->data;
6757 table_offset = CP_ME_TABLE_OFFSET;
6758 } else if (me == 1) {
6759 fw_data = (const __be32 *)rdev->pfp_fw->data;
6760 table_offset = CP_ME_TABLE_OFFSET;
6761 } else if (me == 2) {
6762 fw_data = (const __be32 *)rdev->me_fw->data;
6763 table_offset = CP_ME_TABLE_OFFSET;
6764 } else {
6765 fw_data = (const __be32 *)rdev->mec_fw->data;
6766 table_offset = CP_MEC_TABLE_OFFSET;
6767 }
22c775ce 6768
f2c6b0f4
AD		 6769 for (i = 0; i < table_size; i ++) {
6770 dst_ptr[bo_offset + i] =
6771 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6772 }
6773 bo_offset += table_size;
22c775ce 6774 }
f96ab484 6775 }
22c775ce 6776}
f96ab484 6777
22c775ce
AD		 6778static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6779 bool enable)
6780{
6781 u32 data, orig;
6782
2b19d17f 6783 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
22c775ce
AD		 6784 orig = data = RREG32(RLC_PG_CNTL);
6785 data |= GFX_PG_ENABLE;
6786 if (orig != data)
6787 WREG32(RLC_PG_CNTL, data);
6788
6789 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6790 data |= AUTO_PG_EN;
6791 if (orig != data)
6792 WREG32(RLC_AUTO_PG_CTRL, data);
6793 } else {
6794 orig = data = RREG32(RLC_PG_CNTL);
6795 data &= ~GFX_PG_ENABLE;
6796 if (orig != data)
6797 WREG32(RLC_PG_CNTL, data);
f96ab484 6798
22c775ce
AD		 6799 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6800 data &= ~AUTO_PG_EN;
6801 if (orig != data)
6802 WREG32(RLC_AUTO_PG_CTRL, data);
f96ab484 6803
22c775ce
AD		 6804 data = RREG32(DB_RENDER_CONTROL);
6805 }
6806}
f96ab484 6807
22c775ce
AD		 6808static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6809{
6810 u32 mask = 0, tmp, tmp1;
6811 int i;
f96ab484 6812
22c775ce
AD		 6813 cik_select_se_sh(rdev, se, sh);
6814 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6815 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6816 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
f96ab484 6817
22c775ce 6818 tmp &= 0xffff0000;
f96ab484 6819
22c775ce
AD		 6820 tmp |= tmp1;
6821 tmp >>= 16;
6822
6823 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6824 mask <<= 1;
6825 mask |= 1;
b07fdd38 6826 }
22c775ce
AD		 6827
6828 return (~tmp) & mask;
f96ab484
AD		 6829}
6830
22c775ce 6831static void cik_init_ao_cu_mask(struct radeon_device *rdev)
d0e092d9 6832{
22c775ce
AD		 6833 u32 i, j, k, active_cu_number = 0;
6834 u32 mask, counter, cu_bitmap;
6835 u32 tmp = 0;
d0e092d9 6836
22c775ce
AD		 6837 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6838 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6839 mask = 1;
6840 cu_bitmap = 0;
6841 counter = 0;
6842 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6843 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6844 if (counter < 2)
6845 cu_bitmap |= mask;
6846 counter ++;
d0e092d9 6847 }
22c775ce 6848 mask <<= 1;
d0e092d9 6849 }
d0e092d9 6850
22c775ce
AD		 6851 active_cu_number += counter;
6852 tmp |= (cu_bitmap << (i * 16 + j * 8));
d0e092d9 6853 }
d0e092d9 6854 }
22c775ce
AD		 6855
6856 WREG32(RLC_PG_AO_CU_MASK, tmp);
6857
6858 tmp = RREG32(RLC_MAX_PG_CU);
6859 tmp &= ~MAX_PU_CU_MASK;
6860 tmp |= MAX_PU_CU(active_cu_number);
6861 WREG32(RLC_MAX_PG_CU, tmp);
d0e092d9
AD		 6862}
6863
22c775ce
AD		 6864static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6865 bool enable)
605de6b9 6866{
22c775ce 6867 u32 data, orig;
605de6b9 6868
22c775ce 6869 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6870 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 6871 data |= STATIC_PER_CU_PG_ENABLE;
6872 else
6873 data &= ~STATIC_PER_CU_PG_ENABLE;
6874 if (orig != data)
6875 WREG32(RLC_PG_CNTL, data);
6876}
6877
6878static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6879 bool enable)
6880{
6881 u32 data, orig;
605de6b9 6882
22c775ce 6883 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6884 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce 6885 data |= DYN_PER_CU_PG_ENABLE;
605de6b9 6886 else
22c775ce
AD		 6887 data &= ~DYN_PER_CU_PG_ENABLE;
6888 if (orig != data)
6889 WREG32(RLC_PG_CNTL, data);
6890}
605de6b9 6891
22c775ce
AD		 6892#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6893#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6894
6895static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6896{
6897 u32 data, orig;
6898 u32 i;
6899
6900 if (rdev->rlc.cs_data) {
6901 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6902 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 6903 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce 6904 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
605de6b9 6905 } else {
22c775ce
AD		 6906 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6907 for (i = 0; i < 3; i++)
6908 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6909 }
6910 if (rdev->rlc.reg_list) {
6911 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6912 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6913 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
605de6b9 6914 }
605de6b9 6915
22c775ce
AD		 6916 orig = data = RREG32(RLC_PG_CNTL);
6917 data |= GFX_PG_SRC;
6918 if (orig != data)
6919 WREG32(RLC_PG_CNTL, data);
605de6b9 6920
22c775ce
AD		 6921 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6922 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
605de6b9 6923
22c775ce
AD		 6924 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6925 data &= ~IDLE_POLL_COUNT_MASK;
6926 data |= IDLE_POLL_COUNT(0x60);
6927 WREG32(CP_RB_WPTR_POLL_CNTL, data);
605de6b9 6928
22c775ce
AD		 6929 data = 0x10101010;
6930 WREG32(RLC_PG_DELAY, data);
605de6b9 6931
22c775ce
AD		 6932 data = RREG32(RLC_PG_DELAY_2);
6933 data &= ~0xff;
6934 data |= 0x3;
6935 WREG32(RLC_PG_DELAY_2, data);
605de6b9 6936
22c775ce
AD		 6937 data = RREG32(RLC_AUTO_PG_CTRL);
6938 data &= ~GRBM_REG_SGIT_MASK;
6939 data |= GRBM_REG_SGIT(0x700);
6940 WREG32(RLC_AUTO_PG_CTRL, data);
605de6b9 6941
605de6b9
AD		 6942}
6943
22c775ce 6944static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
f6796cae 6945{
473359bc
AD		 6946 cik_enable_gfx_cgpg(rdev, enable);
6947 cik_enable_gfx_static_mgpg(rdev, enable);
6948 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce 6949}
f6796cae 6950
a0f38609
AD		 6951u32 cik_get_csb_size(struct radeon_device *rdev)
6952{
6953 u32 count = 0;
6954 const struct cs_section_def *sect = NULL;
6955 const struct cs_extent_def *ext = NULL;
f6796cae 6956
a0f38609
AD		 6957 if (rdev->rlc.cs_data == NULL)
6958 return 0;
f6796cae 6959
a0f38609
AD		 6960 /* begin clear state */
6961 count += 2;
6962 /* context control state */
6963 count += 3;
6964
6965 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6966 for (ext = sect->section; ext->extent != NULL; ++ext) {
6967 if (sect->id == SECT_CONTEXT)
6968 count += 2 + ext->reg_count;
6969 else
6970 return 0;
f6796cae
AD		 6971 }
6972 }
a0f38609
AD		 6973 /* pa_sc_raster_config/pa_sc_raster_config1 */
6974 count += 4;
6975 /* end clear state */
6976 count += 2;
6977 /* clear state */
6978 count += 2;
f6796cae 6979
a0f38609 6980 return count;
f6796cae
AD		 6981}
6982
a0f38609 6983void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
f6796cae 6984{
a0f38609
AD		 6985 u32 count = 0, i;
6986 const struct cs_section_def *sect = NULL;
6987 const struct cs_extent_def *ext = NULL;
f6796cae 6988
a0f38609
AD		 6989 if (rdev->rlc.cs_data == NULL)
6990 return;
6991 if (buffer == NULL)
6992 return;
f6796cae 6993
6ba81e53
AD		 6994 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6995 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
a0f38609 6996
6ba81e53
AD		 6997 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6998 buffer[count++] = cpu_to_le32(0x80000000);
6999 buffer[count++] = cpu_to_le32(0x80000000);
a0f38609
AD		 7000
7001 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
7002 for (ext = sect->section; ext->extent != NULL; ++ext) {
7003 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 7004 buffer[count++] =
7005 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
7006 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
a0f38609 7007 for (i = 0; i < ext->reg_count; i++)
6ba81e53 7008 buffer[count++] = cpu_to_le32(ext->extent[i]);
a0f38609
AD		 7009 } else {
7010 return;
7011 }
7012 }
7013 }
f6796cae 7014
6ba81e53
AD		 7015 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
7016 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
f6796cae
AD		 7017 switch (rdev->family) {
7018 case CHIP_BONAIRE:
6ba81e53
AD		 7019 buffer[count++] = cpu_to_le32(0x16000012);
7020 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7021 break;
7022 case CHIP_KAVERI:
6ba81e53
AD		 7023 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7024 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7025 break;
7026 case CHIP_KABINI:
f73a9e83 7027 case CHIP_MULLINS:
6ba81e53
AD		 7028 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7029 buffer[count++] = cpu_to_le32(0x00000000);
a0f38609 7030 break;
bbfe90bd 7031 case CHIP_HAWAII:
a8947f57
AD		 7032 buffer[count++] = cpu_to_le32(0x3a00161a);
7033 buffer[count++] = cpu_to_le32(0x0000002e);
bbfe90bd 7034 break;
a0f38609 7035 default:
6ba81e53
AD		 7036 buffer[count++] = cpu_to_le32(0x00000000);
7037 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7038 break;
7039 }
7040
6ba81e53
AD		 7041 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7042 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
f6796cae 7043
6ba81e53
AD		 7044 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7045 buffer[count++] = cpu_to_le32(0);
a0f38609 7046}
f6796cae 7047
473359bc 7048static void cik_init_pg(struct radeon_device *rdev)
22c775ce 7049{
473359bc 7050 if (rdev->pg_flags) {
22c775ce
AD		 7051 cik_enable_sck_slowdown_on_pu(rdev, true);
7052 cik_enable_sck_slowdown_on_pd(rdev, true);
2b19d17f 7053 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 7054 cik_init_gfx_cgpg(rdev);
7055 cik_enable_cp_pg(rdev, true);
7056 cik_enable_gds_pg(rdev, true);
7057 }
22c775ce
AD		 7058 cik_init_ao_cu_mask(rdev);
7059 cik_update_gfx_pg(rdev, true);
7060 }
7061}
f6796cae 7062
473359bc
AD		 7063static void cik_fini_pg(struct radeon_device *rdev)
7064{
7065 if (rdev->pg_flags) {
7066 cik_update_gfx_pg(rdev, false);
2b19d17f 7067 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 7068 cik_enable_cp_pg(rdev, false);
7069 cik_enable_gds_pg(rdev, false);
7070 }
7071 }
f6796cae 7072}
a59781bb
AD		 7073
7074/*
7075 * Interrupts
7076 * Starting with r6xx, interrupts are handled via a ring buffer.
7077 * Ring buffers are areas of GPU accessible memory that the GPU
7078 * writes interrupt vectors into and the host reads vectors out of.
7079 * There is a rptr (read pointer) that determines where the
7080 * host is currently reading, and a wptr (write pointer)
7081 * which determines where the GPU has written. When the
7082 * pointers are equal, the ring is idle. When the GPU
7083 * writes vectors to the ring buffer, it increments the
7084 * wptr. When there is an interrupt, the host then starts
7085 * fetching commands and processing them until the pointers are
7086 * equal again at which point it updates the rptr.
7087 */
7088
7089/**
7090 * cik_enable_interrupts - Enable the interrupt ring buffer
7091 *
7092 * @rdev: radeon_device pointer
7093 *
7094 * Enable the interrupt ring buffer (CIK).
7095 */
7096static void cik_enable_interrupts(struct radeon_device *rdev)
7097{
7098 u32 ih_cntl = RREG32(IH_CNTL);
7099 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7100
7101 ih_cntl |= ENABLE_INTR;
7102 ih_rb_cntl |= IH_RB_ENABLE;
7103 WREG32(IH_CNTL, ih_cntl);
7104 WREG32(IH_RB_CNTL, ih_rb_cntl);
7105 rdev->ih.enabled = true;
7106}
7107
7108/**
7109 * cik_disable_interrupts - Disable the interrupt ring buffer
7110 *
7111 * @rdev: radeon_device pointer
7112 *
7113 * Disable the interrupt ring buffer (CIK).
7114 */
7115static void cik_disable_interrupts(struct radeon_device *rdev)
7116{
7117 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7118 u32 ih_cntl = RREG32(IH_CNTL);
7119
7120 ih_rb_cntl &= ~IH_RB_ENABLE;
7121 ih_cntl &= ~ENABLE_INTR;
7122 WREG32(IH_RB_CNTL, ih_rb_cntl);
7123 WREG32(IH_CNTL, ih_cntl);
7124 /* set rptr, wptr to 0 */
7125 WREG32(IH_RB_RPTR, 0);
7126 WREG32(IH_RB_WPTR, 0);
7127 rdev->ih.enabled = false;
7128 rdev->ih.rptr = 0;
7129}
7130
7131/**
7132 * cik_disable_interrupt_state - Disable all interrupt sources
7133 *
7134 * @rdev: radeon_device pointer
7135 *
7136 * Clear all interrupt enable bits used by the driver (CIK).
7137 */
7138static void cik_disable_interrupt_state(struct radeon_device *rdev)
7139{
7140 u32 tmp;
7141
7142 /* gfx ring */
4214faf6
AD		 7143 tmp = RREG32(CP_INT_CNTL_RING0) &
7144 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7145 WREG32(CP_INT_CNTL_RING0, tmp);
21a93e13
AD		 7146 /* sdma */
7147 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7148 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7149 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7150 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 7151 /* compute queues */
7152 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7153 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7154 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7155 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7156 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7157 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7158 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7159 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7160 /* grbm */
7161 WREG32(GRBM_INT_CNTL, 0);
7162 /* vline/vblank, etc. */
7163 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7164 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7165 if (rdev->num_crtc >= 4) {
7166 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7167 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7168 }
7169 if (rdev->num_crtc >= 6) {
7170 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7171 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7172 }
f5d636d2
CK		 7173 /* pflip */
7174 if (rdev->num_crtc >= 2) {
7175 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7176 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7177 }
7178 if (rdev->num_crtc >= 4) {
7179 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7180 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7181 }
7182 if (rdev->num_crtc >= 6) {
7183 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7184 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7185 }
a59781bb
AD		 7186
7187 /* dac hotplug */
7188 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7189
7190 /* digital hotplug */
7191 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7192 WREG32(DC_HPD1_INT_CONTROL, tmp);
7193 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7194 WREG32(DC_HPD2_INT_CONTROL, tmp);
7195 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7196 WREG32(DC_HPD3_INT_CONTROL, tmp);
7197 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7198 WREG32(DC_HPD4_INT_CONTROL, tmp);
7199 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7200 WREG32(DC_HPD5_INT_CONTROL, tmp);
7201 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7202 WREG32(DC_HPD6_INT_CONTROL, tmp);
7203
7204}
7205
7206/**
7207 * cik_irq_init - init and enable the interrupt ring
7208 *
7209 * @rdev: radeon_device pointer
7210 *
7211 * Allocate a ring buffer for the interrupt controller,
7212 * enable the RLC, disable interrupts, enable the IH
7213 * ring buffer and enable it (CIK).
7214 * Called at device load and reume.
7215 * Returns 0 for success, errors for failure.
7216 */
7217static int cik_irq_init(struct radeon_device *rdev)
7218{
7219 int ret = 0;
7220 int rb_bufsz;
7221 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7222
7223 /* allocate ring */
7224 ret = r600_ih_ring_alloc(rdev);
7225 if (ret)
7226 return ret;
7227
7228 /* disable irqs */
7229 cik_disable_interrupts(rdev);
7230
7231 /* init rlc */
7232 ret = cik_rlc_resume(rdev);
7233 if (ret) {
7234 r600_ih_ring_fini(rdev);
7235 return ret;
7236 }
7237
7238 /* setup interrupt control */
7239 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7240 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7241 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7242 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7243 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7244 */
7245 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7246 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7247 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7248 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7249
7250 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 7251 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
a59781bb
AD		 7252
7253 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7254 IH_WPTR_OVERFLOW_CLEAR |
7255 (rb_bufsz << 1));
7256
7257 if (rdev->wb.enabled)
7258 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7259
7260 /* set the writeback address whether it's enabled or not */
7261 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7262 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7263
7264 WREG32(IH_RB_CNTL, ih_rb_cntl);
7265
7266 /* set rptr, wptr to 0 */
7267 WREG32(IH_RB_RPTR, 0);
7268 WREG32(IH_RB_WPTR, 0);
7269
7270 /* Default settings for IH_CNTL (disabled at first) */
7271 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7272 /* RPTR_REARM only works if msi's are enabled */
7273 if (rdev->msi_enabled)
7274 ih_cntl |= RPTR_REARM;
7275 WREG32(IH_CNTL, ih_cntl);
7276
7277 /* force the active interrupt state to all disabled */
7278 cik_disable_interrupt_state(rdev);
7279
7280 pci_set_master(rdev->pdev);
7281
7282 /* enable irqs */
7283 cik_enable_interrupts(rdev);
7284
7285 return ret;
7286}
7287
7288/**
7289 * cik_irq_set - enable/disable interrupt sources
7290 *
7291 * @rdev: radeon_device pointer
7292 *
7293 * Enable interrupt sources on the GPU (vblanks, hpd,
7294 * etc.) (CIK).
7295 * Returns 0 for success, errors for failure.
7296 */
7297int cik_irq_set(struct radeon_device *rdev)
7298{
4214faf6 7299 u32 cp_int_cntl;
28b57b85 7300 u32 cp_m1p0;
a59781bb
AD		 7301 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7302 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7303 u32 grbm_int_cntl = 0;
21a93e13 7304 u32 dma_cntl, dma_cntl1;
41a524ab 7305 u32 thermal_int;
a59781bb
AD		 7306
7307 if (!rdev->irq.installed) {
7308 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7309 return -EINVAL;
7310 }
7311 /* don't enable anything if the ih is disabled */
7312 if (!rdev->ih.enabled) {
7313 cik_disable_interrupts(rdev);
7314 /* force the active interrupt state to all disabled */
7315 cik_disable_interrupt_state(rdev);
7316 return 0;
7317 }
7318
4214faf6
AD		 7319 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7320 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7321 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7322
a59781bb
AD		 7323 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
7324 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
7325 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
7326 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
7327 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
7328 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
7329
21a93e13
AD		 7330 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7331 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7332
2b0781a6 7333 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
2b0781a6 7334
cc8dbbb4
AD		 7335 if (rdev->flags & RADEON_IS_IGP)
7336 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
7337 ~(THERM_INTH_MASK | THERM_INTL_MASK);
7338 else
7339 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
7340 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
41a524ab 7341
a59781bb
AD		 7342 /* enable CP interrupts on all rings */
7343 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7344 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7345 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7346 }
2b0781a6
AD		 7347 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7348 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7349 DRM_DEBUG("si_irq_set: sw int cp1\n");
7350 if (ring->me == 1) {
7351 switch (ring->pipe) {
7352 case 0:
7353 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7354 break;
2b0781a6
AD		 7355 default:
7356 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7357 break;
7358 }
7359 } else {
7360 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7361 }
7362 }
7363 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7364 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7365 DRM_DEBUG("si_irq_set: sw int cp2\n");
7366 if (ring->me == 1) {
7367 switch (ring->pipe) {
7368 case 0:
7369 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7370 break;
2b0781a6
AD		 7371 default:
7372 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7373 break;
7374 }
7375 } else {
7376 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7377 }
7378 }
a59781bb 7379
21a93e13
AD		 7380 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7381 DRM_DEBUG("cik_irq_set: sw int dma\n");
7382 dma_cntl |= TRAP_ENABLE;
7383 }
7384
7385 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7386 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7387 dma_cntl1 |= TRAP_ENABLE;
7388 }
7389
a59781bb
AD		 7390 if (rdev->irq.crtc_vblank_int[0] ||
7391 atomic_read(&rdev->irq.pflip[0])) {
7392 DRM_DEBUG("cik_irq_set: vblank 0\n");
7393 crtc1 |= VBLANK_INTERRUPT_MASK;
7394 }
7395 if (rdev->irq.crtc_vblank_int[1] ||
7396 atomic_read(&rdev->irq.pflip[1])) {
7397 DRM_DEBUG("cik_irq_set: vblank 1\n");
7398 crtc2 |= VBLANK_INTERRUPT_MASK;
7399 }
7400 if (rdev->irq.crtc_vblank_int[2] ||
7401 atomic_read(&rdev->irq.pflip[2])) {
7402 DRM_DEBUG("cik_irq_set: vblank 2\n");
7403 crtc3 |= VBLANK_INTERRUPT_MASK;
7404 }
7405 if (rdev->irq.crtc_vblank_int[3] ||
7406 atomic_read(&rdev->irq.pflip[3])) {
7407 DRM_DEBUG("cik_irq_set: vblank 3\n");
7408 crtc4 |= VBLANK_INTERRUPT_MASK;
7409 }
7410 if (rdev->irq.crtc_vblank_int[4] ||
7411 atomic_read(&rdev->irq.pflip[4])) {
7412 DRM_DEBUG("cik_irq_set: vblank 4\n");
7413 crtc5 |= VBLANK_INTERRUPT_MASK;
7414 }
7415 if (rdev->irq.crtc_vblank_int[5] ||
7416 atomic_read(&rdev->irq.pflip[5])) {
7417 DRM_DEBUG("cik_irq_set: vblank 5\n");
7418 crtc6 |= VBLANK_INTERRUPT_MASK;
7419 }
7420 if (rdev->irq.hpd[0]) {
7421 DRM_DEBUG("cik_irq_set: hpd 1\n");
7422 hpd1 |= DC_HPDx_INT_EN;
7423 }
7424 if (rdev->irq.hpd[1]) {
7425 DRM_DEBUG("cik_irq_set: hpd 2\n");
7426 hpd2 |= DC_HPDx_INT_EN;
7427 }
7428 if (rdev->irq.hpd[2]) {
7429 DRM_DEBUG("cik_irq_set: hpd 3\n");
7430 hpd3 |= DC_HPDx_INT_EN;
7431 }
7432 if (rdev->irq.hpd[3]) {
7433 DRM_DEBUG("cik_irq_set: hpd 4\n");
7434 hpd4 |= DC_HPDx_INT_EN;
7435 }
7436 if (rdev->irq.hpd[4]) {
7437 DRM_DEBUG("cik_irq_set: hpd 5\n");
7438 hpd5 |= DC_HPDx_INT_EN;
7439 }
7440 if (rdev->irq.hpd[5]) {
7441 DRM_DEBUG("cik_irq_set: hpd 6\n");
7442 hpd6 |= DC_HPDx_INT_EN;
7443 }
7444
41a524ab
AD		 7445 if (rdev->irq.dpm_thermal) {
7446 DRM_DEBUG("dpm thermal\n");
cc8dbbb4
AD		 7447 if (rdev->flags & RADEON_IS_IGP)
7448 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7449 else
7450 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
41a524ab
AD		 7451 }
7452
a59781bb
AD		 7453 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7454
21a93e13
AD		 7455 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7456 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7457
2b0781a6 7458 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
2b0781a6 7459
a59781bb
AD		 7460 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7461
7462 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7463 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7464 if (rdev->num_crtc >= 4) {
7465 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7466 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7467 }
7468 if (rdev->num_crtc >= 6) {
7469 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7470 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7471 }
7472
f5d636d2
CK		 7473 if (rdev->num_crtc >= 2) {
7474 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7475 GRPH_PFLIP_INT_MASK);
7476 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7477 GRPH_PFLIP_INT_MASK);
7478 }
7479 if (rdev->num_crtc >= 4) {
7480 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7481 GRPH_PFLIP_INT_MASK);
7482 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7483 GRPH_PFLIP_INT_MASK);
7484 }
7485 if (rdev->num_crtc >= 6) {
7486 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7487 GRPH_PFLIP_INT_MASK);
7488 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7489 GRPH_PFLIP_INT_MASK);
7490 }
7491
a59781bb
AD		 7492 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7493 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7494 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7495 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7496 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7497 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7498
cc8dbbb4
AD		 7499 if (rdev->flags & RADEON_IS_IGP)
7500 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7501 else
7502 WREG32_SMC(CG_THERMAL_INT, thermal_int);
41a524ab 7503
a59781bb
AD		 7504 return 0;
7505}
7506
7507/**
7508 * cik_irq_ack - ack interrupt sources
7509 *
7510 * @rdev: radeon_device pointer
7511 *
7512 * Ack interrupt sources on the GPU (vblanks, hpd,
7513 * etc.) (CIK). Certain interrupts sources are sw
7514 * generated and do not require an explicit ack.
7515 */
7516static inline void cik_irq_ack(struct radeon_device *rdev)
7517{
7518 u32 tmp;
7519
7520 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7521 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7522 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7523 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7524 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7525 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7526 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7527
f5d636d2
CK		 7528 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7529 EVERGREEN_CRTC0_REGISTER_OFFSET);
7530 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7531 EVERGREEN_CRTC1_REGISTER_OFFSET);
7532 if (rdev->num_crtc >= 4) {
7533 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7534 EVERGREEN_CRTC2_REGISTER_OFFSET);
7535 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7536 EVERGREEN_CRTC3_REGISTER_OFFSET);
7537 }
7538 if (rdev->num_crtc >= 6) {
7539 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7540 EVERGREEN_CRTC4_REGISTER_OFFSET);
7541 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7542 EVERGREEN_CRTC5_REGISTER_OFFSET);
7543 }
7544
7545 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7546 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7547 GRPH_PFLIP_INT_CLEAR);
7548 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7549 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7550 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7551 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7552 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7553 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7554 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7555 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7556 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7557 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7558 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7559
7560 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 7561 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7562 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7563 GRPH_PFLIP_INT_CLEAR);
7564 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7565 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7566 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7567 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7568 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7569 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7570 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7571 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7572 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7573 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7574 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7575 }
7576
7577 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 7578 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7579 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7580 GRPH_PFLIP_INT_CLEAR);
7581 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7582 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7583 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7584 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7585 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7586 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7587 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7588 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7589 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7590 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7591 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7592 }
7593
7594 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7595 tmp = RREG32(DC_HPD1_INT_CONTROL);
7596 tmp |= DC_HPDx_INT_ACK;
7597 WREG32(DC_HPD1_INT_CONTROL, tmp);
7598 }
7599 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7600 tmp = RREG32(DC_HPD2_INT_CONTROL);
7601 tmp |= DC_HPDx_INT_ACK;
7602 WREG32(DC_HPD2_INT_CONTROL, tmp);
7603 }
7604 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7605 tmp = RREG32(DC_HPD3_INT_CONTROL);
7606 tmp |= DC_HPDx_INT_ACK;
7607 WREG32(DC_HPD3_INT_CONTROL, tmp);
7608 }
7609 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7610 tmp = RREG32(DC_HPD4_INT_CONTROL);
7611 tmp |= DC_HPDx_INT_ACK;
7612 WREG32(DC_HPD4_INT_CONTROL, tmp);
7613 }
7614 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7615 tmp = RREG32(DC_HPD5_INT_CONTROL);
7616 tmp |= DC_HPDx_INT_ACK;
7617 WREG32(DC_HPD5_INT_CONTROL, tmp);
7618 }
7619 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7620 tmp = RREG32(DC_HPD5_INT_CONTROL);
7621 tmp |= DC_HPDx_INT_ACK;
7622 WREG32(DC_HPD6_INT_CONTROL, tmp);
7623 }
7624}
7625
7626/**
7627 * cik_irq_disable - disable interrupts
7628 *
7629 * @rdev: radeon_device pointer
7630 *
7631 * Disable interrupts on the hw (CIK).
7632 */
7633static void cik_irq_disable(struct radeon_device *rdev)
7634{
7635 cik_disable_interrupts(rdev);
7636 /* Wait and acknowledge irq */
7637 mdelay(1);
7638 cik_irq_ack(rdev);
7639 cik_disable_interrupt_state(rdev);
7640}
7641
7642/**
7643 * cik_irq_disable - disable interrupts for suspend
7644 *
7645 * @rdev: radeon_device pointer
7646 *
7647 * Disable interrupts and stop the RLC (CIK).
7648 * Used for suspend.
7649 */
7650static void cik_irq_suspend(struct radeon_device *rdev)
7651{
7652 cik_irq_disable(rdev);
7653 cik_rlc_stop(rdev);
7654}
7655
7656/**
7657 * cik_irq_fini - tear down interrupt support
7658 *
7659 * @rdev: radeon_device pointer
7660 *
7661 * Disable interrupts on the hw and free the IH ring
7662 * buffer (CIK).
7663 * Used for driver unload.
7664 */
7665static void cik_irq_fini(struct radeon_device *rdev)
7666{
7667 cik_irq_suspend(rdev);
7668 r600_ih_ring_fini(rdev);
7669}
7670
7671/**
7672 * cik_get_ih_wptr - get the IH ring buffer wptr
7673 *
7674 * @rdev: radeon_device pointer
7675 *
7676 * Get the IH ring buffer wptr from either the register
7677 * or the writeback memory buffer (CIK). Also check for
7678 * ring buffer overflow and deal with it.
7679 * Used by cik_irq_process().
7680 * Returns the value of the wptr.
7681 */
7682static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7683{
7684 u32 wptr, tmp;
7685
7686 if (rdev->wb.enabled)
7687 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7688 else
7689 wptr = RREG32(IH_RB_WPTR);
7690
7691 if (wptr & RB_OVERFLOW) {
11bab0ae 7692 wptr &= ~RB_OVERFLOW;
a59781bb
AD		 7693 /* When a ring buffer overflow happen start parsing interrupt
7694 * from the last not overwritten vector (wptr + 16). Hopefully
7695 * this should allow us to catchup.
7696 */
6cc2fda2
MD		 7697 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7698 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
a59781bb
AD		 7699 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7700 tmp = RREG32(IH_RB_CNTL);
7701 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7702 WREG32(IH_RB_CNTL, tmp);
7703 }
7704 return (wptr & rdev->ih.ptr_mask);
7705}
7706
7707/* CIK IV Ring
7708 * Each IV ring entry is 128 bits:
7709 * [7:0] - interrupt source id
7710 * [31:8] - reserved
7711 * [59:32] - interrupt source data
7712 * [63:60] - reserved
21a93e13
AD		 7713 * [71:64] - RINGID
7714 * CP:
7715 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 7716 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7717 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7718 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7719 * PIPE_ID - ME0 0=3D
7720 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 7721 * SDMA:
7722 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7723 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7724 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 7725 * [79:72] - VMID
7726 * [95:80] - PASID
7727 * [127:96] - reserved
7728 */
7729/**
7730 * cik_irq_process - interrupt handler
7731 *
7732 * @rdev: radeon_device pointer
7733 *
7734 * Interrupt hander (CIK). Walk the IH ring,
7735 * ack interrupts and schedule work to handle
7736 * interrupt events.
7737 * Returns irq process return code.
7738 */
7739int cik_irq_process(struct radeon_device *rdev)
7740{
2b0781a6
AD		 7741 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7742 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 7743 u32 wptr;
7744 u32 rptr;
7745 u32 src_id, src_data, ring_id;
7746 u8 me_id, pipe_id, queue_id;
7747 u32 ring_index;
7748 bool queue_hotplug = false;
7749 bool queue_reset = false;
3ec7d11b 7750 u32 addr, status, mc_client;
41a524ab 7751 bool queue_thermal = false;
a59781bb
AD		 7752
7753 if (!rdev->ih.enabled || rdev->shutdown)
7754 return IRQ_NONE;
7755
7756 wptr = cik_get_ih_wptr(rdev);
7757
7758restart_ih:
7759 /* is somebody else already processing irqs? */
7760 if (atomic_xchg(&rdev->ih.lock, 1))
7761 return IRQ_NONE;
7762
7763 rptr = rdev->ih.rptr;
7764 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7765
7766 /* Order reading of wptr vs. reading of IH ring data */
7767 rmb();
7768
7769 /* display interrupts */
7770 cik_irq_ack(rdev);
7771
7772 while (rptr != wptr) {
7773 /* wptr/rptr are in bytes! */
7774 ring_index = rptr / 4;
7775 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7776 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7777 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 7778
7779 switch (src_id) {
7780 case 1: /* D1 vblank/vline */
7781 switch (src_data) {
7782 case 0: /* D1 vblank */
7783 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7784 if (rdev->irq.crtc_vblank_int[0]) {
7785 drm_handle_vblank(rdev->ddev, 0);
7786 rdev->pm.vblank_sync = true;
7787 wake_up(&rdev->irq.vblank_queue);
7788 }
7789 if (atomic_read(&rdev->irq.pflip[0]))
1a0e7918 7790 radeon_crtc_handle_vblank(rdev, 0);
a59781bb
AD		 7791 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7792 DRM_DEBUG("IH: D1 vblank\n");
7793 }
7794 break;
7795 case 1: /* D1 vline */
7796 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7797 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7798 DRM_DEBUG("IH: D1 vline\n");
7799 }
7800 break;
7801 default:
7802 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7803 break;
7804 }
7805 break;
7806 case 2: /* D2 vblank/vline */
7807 switch (src_data) {
7808 case 0: /* D2 vblank */
7809 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7810 if (rdev->irq.crtc_vblank_int[1]) {
7811 drm_handle_vblank(rdev->ddev, 1);
7812 rdev->pm.vblank_sync = true;
7813 wake_up(&rdev->irq.vblank_queue);
7814 }
7815 if (atomic_read(&rdev->irq.pflip[1]))
1a0e7918 7816 radeon_crtc_handle_vblank(rdev, 1);
a59781bb
AD		 7817 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7818 DRM_DEBUG("IH: D2 vblank\n");
7819 }
7820 break;
7821 case 1: /* D2 vline */
7822 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7823 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7824 DRM_DEBUG("IH: D2 vline\n");
7825 }
7826 break;
7827 default:
7828 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7829 break;
7830 }
7831 break;
7832 case 3: /* D3 vblank/vline */
7833 switch (src_data) {
7834 case 0: /* D3 vblank */
7835 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7836 if (rdev->irq.crtc_vblank_int[2]) {
7837 drm_handle_vblank(rdev->ddev, 2);
7838 rdev->pm.vblank_sync = true;
7839 wake_up(&rdev->irq.vblank_queue);
7840 }
7841 if (atomic_read(&rdev->irq.pflip[2]))
1a0e7918 7842 radeon_crtc_handle_vblank(rdev, 2);
a59781bb
AD		 7843 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7844 DRM_DEBUG("IH: D3 vblank\n");
7845 }
7846 break;
7847 case 1: /* D3 vline */
7848 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7849 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7850 DRM_DEBUG("IH: D3 vline\n");
7851 }
7852 break;
7853 default:
7854 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7855 break;
7856 }
7857 break;
7858 case 4: /* D4 vblank/vline */
7859 switch (src_data) {
7860 case 0: /* D4 vblank */
7861 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7862 if (rdev->irq.crtc_vblank_int[3]) {
7863 drm_handle_vblank(rdev->ddev, 3);
7864 rdev->pm.vblank_sync = true;
7865 wake_up(&rdev->irq.vblank_queue);
7866 }
7867 if (atomic_read(&rdev->irq.pflip[3]))
1a0e7918 7868 radeon_crtc_handle_vblank(rdev, 3);
a59781bb
AD		 7869 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7870 DRM_DEBUG("IH: D4 vblank\n");
7871 }
7872 break;
7873 case 1: /* D4 vline */
7874 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7875 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7876 DRM_DEBUG("IH: D4 vline\n");
7877 }
7878 break;
7879 default:
7880 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7881 break;
7882 }
7883 break;
7884 case 5: /* D5 vblank/vline */
7885 switch (src_data) {
7886 case 0: /* D5 vblank */
7887 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7888 if (rdev->irq.crtc_vblank_int[4]) {
7889 drm_handle_vblank(rdev->ddev, 4);
7890 rdev->pm.vblank_sync = true;
7891 wake_up(&rdev->irq.vblank_queue);
7892 }
7893 if (atomic_read(&rdev->irq.pflip[4]))
1a0e7918 7894 radeon_crtc_handle_vblank(rdev, 4);
a59781bb
AD		 7895 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7896 DRM_DEBUG("IH: D5 vblank\n");
7897 }
7898 break;
7899 case 1: /* D5 vline */
7900 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7901 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7902 DRM_DEBUG("IH: D5 vline\n");
7903 }
7904 break;
7905 default:
7906 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7907 break;
7908 }
7909 break;
7910 case 6: /* D6 vblank/vline */
7911 switch (src_data) {
7912 case 0: /* D6 vblank */
7913 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7914 if (rdev->irq.crtc_vblank_int[5]) {
7915 drm_handle_vblank(rdev->ddev, 5);
7916 rdev->pm.vblank_sync = true;
7917 wake_up(&rdev->irq.vblank_queue);
7918 }
7919 if (atomic_read(&rdev->irq.pflip[5]))
1a0e7918 7920 radeon_crtc_handle_vblank(rdev, 5);
a59781bb
AD		 7921 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7922 DRM_DEBUG("IH: D6 vblank\n");
7923 }
7924 break;
7925 case 1: /* D6 vline */
7926 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7927 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7928 DRM_DEBUG("IH: D6 vline\n");
7929 }
7930 break;
7931 default:
7932 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7933 break;
7934 }
7935 break;
f5d636d2
CK		 7936 case 8: /* D1 page flip */
7937 case 10: /* D2 page flip */
7938 case 12: /* D3 page flip */
7939 case 14: /* D4 page flip */
7940 case 16: /* D5 page flip */
7941 case 18: /* D6 page flip */
7942 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 7943 if (radeon_use_pflipirq > 0)
7944 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 7945 break;
a59781bb
AD		 7946 case 42: /* HPD hotplug */
7947 switch (src_data) {
7948 case 0:
7949 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7950 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7951 queue_hotplug = true;
7952 DRM_DEBUG("IH: HPD1\n");
7953 }
7954 break;
7955 case 1:
7956 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7957 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7958 queue_hotplug = true;
7959 DRM_DEBUG("IH: HPD2\n");
7960 }
7961 break;
7962 case 2:
7963 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7964 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7965 queue_hotplug = true;
7966 DRM_DEBUG("IH: HPD3\n");
7967 }
7968 break;
7969 case 3:
7970 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7971 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7972 queue_hotplug = true;
7973 DRM_DEBUG("IH: HPD4\n");
7974 }
7975 break;
7976 case 4:
7977 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7978 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7979 queue_hotplug = true;
7980 DRM_DEBUG("IH: HPD5\n");
7981 }
7982 break;
7983 case 5:
7984 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7985 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7986 queue_hotplug = true;
7987 DRM_DEBUG("IH: HPD6\n");
7988 }
7989 break;
7990 default:
7991 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7992 break;
7993 }
7994 break;
6a3808b8
CK		 7995 case 124: /* UVD */
7996 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7997 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7998 break;
9d97c99b
AD		 7999 case 146:
8000 case 147:
3ec7d11b
AD		 8001 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8002 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8003 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9b7d786b
CK		 8004 /* reset addr and status */
8005 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8006 if (addr == 0x0 && status == 0x0)
8007 break;
9d97c99b
AD		 8008 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8009 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 8010 addr);
9d97c99b 8011 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 8012 status);
8013 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b 8014 break;
d93f7937
CK		 8015 case 167: /* VCE */
8016 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8017 switch (src_data) {
8018 case 0:
8019 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8020 break;
8021 case 1:
8022 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8023 break;
8024 default:
8025 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8026 break;
8027 }
8028 break;
a59781bb
AD		 8029 case 176: /* GFX RB CP_INT */
8030 case 177: /* GFX IB CP_INT */
8031 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8032 break;
8033 case 181: /* CP EOP event */
8034 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 8035 /* XXX check the bitfield order! */
8036 me_id = (ring_id & 0x60) >> 5;
8037 pipe_id = (ring_id & 0x18) >> 3;
8038 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 8039 switch (me_id) {
8040 case 0:
8041 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8042 break;
8043 case 1:
a59781bb 8044 case 2:
2b0781a6
AD		 8045 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8046 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8047 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8048 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 8049 break;
8050 }
8051 break;
8052 case 184: /* CP Privileged reg access */
8053 DRM_ERROR("Illegal register access in command stream\n");
8054 /* XXX check the bitfield order! */
8055 me_id = (ring_id & 0x60) >> 5;
8056 pipe_id = (ring_id & 0x18) >> 3;
8057 queue_id = (ring_id & 0x7) >> 0;
8058 switch (me_id) {
8059 case 0:
8060 /* This results in a full GPU reset, but all we need to do is soft
8061 * reset the CP for gfx
8062 */
8063 queue_reset = true;
8064 break;
8065 case 1:
8066 /* XXX compute */
2b0781a6 8067 queue_reset = true;
a59781bb
AD		 8068 break;
8069 case 2:
8070 /* XXX compute */
2b0781a6 8071 queue_reset = true;
a59781bb
AD		 8072 break;
8073 }
8074 break;
8075 case 185: /* CP Privileged inst */
8076 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 8077 /* XXX check the bitfield order! */
8078 me_id = (ring_id & 0x60) >> 5;
8079 pipe_id = (ring_id & 0x18) >> 3;
8080 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 8081 switch (me_id) {
8082 case 0:
8083 /* This results in a full GPU reset, but all we need to do is soft
8084 * reset the CP for gfx
8085 */
8086 queue_reset = true;
8087 break;
8088 case 1:
8089 /* XXX compute */
2b0781a6 8090 queue_reset = true;
a59781bb
AD		 8091 break;
8092 case 2:
8093 /* XXX compute */
2b0781a6 8094 queue_reset = true;
a59781bb
AD		 8095 break;
8096 }
8097 break;
21a93e13
AD		 8098 case 224: /* SDMA trap event */
8099 /* XXX check the bitfield order! */
8100 me_id = (ring_id & 0x3) >> 0;
8101 queue_id = (ring_id & 0xc) >> 2;
8102 DRM_DEBUG("IH: SDMA trap\n");
8103 switch (me_id) {
8104 case 0:
8105 switch (queue_id) {
8106 case 0:
8107 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8108 break;
8109 case 1:
8110 /* XXX compute */
8111 break;
8112 case 2:
8113 /* XXX compute */
8114 break;
8115 }
8116 break;
8117 case 1:
8118 switch (queue_id) {
8119 case 0:
8120 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8121 break;
8122 case 1:
8123 /* XXX compute */
8124 break;
8125 case 2:
8126 /* XXX compute */
8127 break;
8128 }
8129 break;
8130 }
8131 break;
41a524ab
AD		 8132 case 230: /* thermal low to high */
8133 DRM_DEBUG("IH: thermal low to high\n");
8134 rdev->pm.dpm.thermal.high_to_low = false;
8135 queue_thermal = true;
8136 break;
8137 case 231: /* thermal high to low */
8138 DRM_DEBUG("IH: thermal high to low\n");
8139 rdev->pm.dpm.thermal.high_to_low = true;
8140 queue_thermal = true;
8141 break;
8142 case 233: /* GUI IDLE */
8143 DRM_DEBUG("IH: GUI idle\n");
8144 break;
21a93e13
AD		 8145 case 241: /* SDMA Privileged inst */
8146 case 247: /* SDMA Privileged inst */
8147 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8148 /* XXX check the bitfield order! */
8149 me_id = (ring_id & 0x3) >> 0;
8150 queue_id = (ring_id & 0xc) >> 2;
8151 switch (me_id) {
8152 case 0:
8153 switch (queue_id) {
8154 case 0:
8155 queue_reset = true;
8156 break;
8157 case 1:
8158 /* XXX compute */
8159 queue_reset = true;
8160 break;
8161 case 2:
8162 /* XXX compute */
8163 queue_reset = true;
8164 break;
8165 }
8166 break;
8167 case 1:
8168 switch (queue_id) {
8169 case 0:
8170 queue_reset = true;
8171 break;
8172 case 1:
8173 /* XXX compute */
8174 queue_reset = true;
8175 break;
8176 case 2:
8177 /* XXX compute */
8178 queue_reset = true;
8179 break;
8180 }
8181 break;
8182 }
8183 break;
a59781bb
AD		 8184 default:
8185 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8186 break;
8187 }
8188
8189 /* wptr/rptr are in bytes! */
8190 rptr += 16;
8191 rptr &= rdev->ih.ptr_mask;
f55e03b9 8192 WREG32(IH_RB_RPTR, rptr);
a59781bb
AD		 8193 }
8194 if (queue_hotplug)
8195 schedule_work(&rdev->hotplug_work);
3c036389
CK		 8196 if (queue_reset) {
8197 rdev->needs_reset = true;
8198 wake_up_all(&rdev->fence_queue);
8199 }
41a524ab
AD		 8200 if (queue_thermal)
8201 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb 8202 rdev->ih.rptr = rptr;
a59781bb
AD		 8203 atomic_set(&rdev->ih.lock, 0);
8204
8205 /* make sure wptr hasn't changed while processing */
8206 wptr = cik_get_ih_wptr(rdev);
8207 if (wptr != rptr)
8208 goto restart_ih;
8209
8210 return IRQ_HANDLED;
8211}
7bf94a2c
AD		 8212
8213/*
8214 * startup/shutdown callbacks
8215 */
8216/**
8217 * cik_startup - program the asic to a functional state
8218 *
8219 * @rdev: radeon_device pointer
8220 *
8221 * Programs the asic to a functional state (CIK).
8222 * Called by cik_init() and cik_resume().
8223 * Returns 0 for success, error for failure.
8224 */
8225static int cik_startup(struct radeon_device *rdev)
8226{
8227 struct radeon_ring *ring;
0e16e4cf 8228 u32 nop;
7bf94a2c
AD		 8229 int r;
8230
8a7cd276
AD		 8231 /* enable pcie gen2/3 link */
8232 cik_pcie_gen3_enable(rdev);
7235711a
AD		 8233 /* enable aspm */
8234 cik_program_aspm(rdev);
8a7cd276 8235
e5903d39
AD		 8236 /* scratch needs to be initialized before MC */
8237 r = r600_vram_scratch_init(rdev);
8238 if (r)
8239 return r;
8240
6fab3feb
AD		 8241 cik_mc_program(rdev);
8242
6c7bccea 8243 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7bf94a2c
AD		 8244 r = ci_mc_load_microcode(rdev);
8245 if (r) {
8246 DRM_ERROR("Failed to load MC firmware!\n");
8247 return r;
8248 }
8249 }
8250
7bf94a2c
AD		 8251 r = cik_pcie_gart_enable(rdev);
8252 if (r)
8253 return r;
8254 cik_gpu_init(rdev);
8255
8256 /* allocate rlc buffers */
22c775ce
AD		 8257 if (rdev->flags & RADEON_IS_IGP) {
8258 if (rdev->family == CHIP_KAVERI) {
8259 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8260 rdev->rlc.reg_list_size =
8261 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8262 } else {
8263 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8264 rdev->rlc.reg_list_size =
8265 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8266 }
8267 }
8268 rdev->rlc.cs_data = ci_cs_data;
8269 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
1fd11777 8270 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 8271 if (r) {
8272 DRM_ERROR("Failed to init rlc BOs!\n");
8273 return r;
8274 }
8275
8276 /* allocate wb buffer */
8277 r = radeon_wb_init(rdev);
8278 if (r)
8279 return r;
8280
963e81f9
AD		 8281 /* allocate mec buffers */
8282 r = cik_mec_init(rdev);
8283 if (r) {
8284 DRM_ERROR("Failed to init MEC BOs!\n");
8285 return r;
8286 }
8287
7bf94a2c
AD		 8288 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8289 if (r) {
8290 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8291 return r;
8292 }
8293
963e81f9
AD		 8294 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8295 if (r) {
8296 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8297 return r;
8298 }
8299
8300 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8301 if (r) {
8302 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8303 return r;
8304 }
8305
7bf94a2c
AD		 8306 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8307 if (r) {
8308 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8309 return r;
8310 }
8311
8312 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8313 if (r) {
8314 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8315 return r;
8316 }
8317
2ce529da 8318 r = radeon_uvd_resume(rdev);
87167bb1 8319 if (!r) {
2ce529da
AD		 8320 r = uvd_v4_2_resume(rdev);
8321 if (!r) {
8322 r = radeon_fence_driver_start_ring(rdev,
8323 R600_RING_TYPE_UVD_INDEX);
8324 if (r)
8325 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8326 }
87167bb1
CK		 8327 }
8328 if (r)
8329 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8330
d93f7937
CK		 8331 r = radeon_vce_resume(rdev);
8332 if (!r) {
8333 r = vce_v2_0_resume(rdev);
8334 if (!r)
8335 r = radeon_fence_driver_start_ring(rdev,
8336 TN_RING_TYPE_VCE1_INDEX);
8337 if (!r)
8338 r = radeon_fence_driver_start_ring(rdev,
8339 TN_RING_TYPE_VCE2_INDEX);
8340 }
8341 if (r) {
8342 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8343 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8344 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8345 }
8346
7bf94a2c
AD		 8347 /* Enable IRQ */
8348 if (!rdev->irq.installed) {
8349 r = radeon_irq_kms_init(rdev);
8350 if (r)
8351 return r;
8352 }
8353
8354 r = cik_irq_init(rdev);
8355 if (r) {
8356 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8357 radeon_irq_kms_fini(rdev);
8358 return r;
8359 }
8360 cik_irq_set(rdev);
8361
0e16e4cf 8362 if (rdev->family == CHIP_HAWAII) {
78cd3661
AD		 8363 if (rdev->new_fw)
8364 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8365 else
8366 nop = RADEON_CP_PACKET2;
0e16e4cf
AD		 8367 } else {
8368 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8369 }
8370
7bf94a2c
AD		 8371 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8372 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
0e16e4cf 8373 nop);
7bf94a2c
AD		 8374 if (r)
8375 return r;
8376
963e81f9 8377 /* set up the compute queues */
2615b53a 8378 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8379 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8380 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
0e16e4cf 8381 nop);
963e81f9
AD		 8382 if (r)
8383 return r;
8384 ring->me = 1; /* first MEC */
8385 ring->pipe = 0; /* first pipe */
8386 ring->queue = 0; /* first queue */
8387 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8388
2615b53a 8389 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8390 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8391 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
0e16e4cf 8392 nop);
963e81f9
AD		 8393 if (r)
8394 return r;
8395 /* dGPU only have 1 MEC */
8396 ring->me = 1; /* first MEC */
8397 ring->pipe = 0; /* first pipe */
8398 ring->queue = 1; /* second queue */
8399 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8400
7bf94a2c
AD		 8401 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8402 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 8403 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8404 if (r)
8405 return r;
8406
8407 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8408 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 8409 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8410 if (r)
8411 return r;
8412
8413 r = cik_cp_resume(rdev);
8414 if (r)
8415 return r;
8416
8417 r = cik_sdma_resume(rdev);
8418 if (r)
8419 return r;
8420
87167bb1
CK		 8421 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8422 if (ring->ring_size) {
02c9f7fa 8423 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 8424 RADEON_CP_PACKET2);
87167bb1 8425 if (!r)
e409b128 8426 r = uvd_v1_0_init(rdev);
87167bb1
CK		 8427 if (r)
8428 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8429 }
8430
d93f7937
CK		 8431 r = -ENOENT;
8432
8433 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8434 if (ring->ring_size)
8435 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8436 VCE_CMD_NO_OP);
8437
8438 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8439 if (ring->ring_size)
8440 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8441 VCE_CMD_NO_OP);
8442
8443 if (!r)
8444 r = vce_v1_0_init(rdev);
8445 else if (r != -ENOENT)
8446 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8447
7bf94a2c
AD		 8448 r = radeon_ib_pool_init(rdev);
8449 if (r) {
8450 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8451 return r;
8452 }
8453
8454 r = radeon_vm_manager_init(rdev);
8455 if (r) {
8456 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8457 return r;
8458 }
8459
b530602f
AD		 8460 r = dce6_audio_init(rdev);
8461 if (r)
8462 return r;
8463
7bf94a2c
AD		 8464 return 0;
8465}
8466
8467/**
8468 * cik_resume - resume the asic to a functional state
8469 *
8470 * @rdev: radeon_device pointer
8471 *
8472 * Programs the asic to a functional state (CIK).
8473 * Called at resume.
8474 * Returns 0 for success, error for failure.
8475 */
8476int cik_resume(struct radeon_device *rdev)
8477{
8478 int r;
8479
8480 /* post card */
8481 atom_asic_init(rdev->mode_info.atom_context);
8482
0aafd313
AD		 8483 /* init golden registers */
8484 cik_init_golden_registers(rdev);
8485
bc6a6295
AD		 8486 if (rdev->pm.pm_method == PM_METHOD_DPM)
8487 radeon_pm_resume(rdev);
6c7bccea 8488
7bf94a2c
AD		 8489 rdev->accel_working = true;
8490 r = cik_startup(rdev);
8491 if (r) {
8492 DRM_ERROR("cik startup failed on resume\n");
8493 rdev->accel_working = false;
8494 return r;
8495 }
8496
8497 return r;
8498
8499}
8500
8501/**
8502 * cik_suspend - suspend the asic
8503 *
8504 * @rdev: radeon_device pointer
8505 *
8506 * Bring the chip into a state suitable for suspend (CIK).
8507 * Called at suspend.
8508 * Returns 0 for success.
8509 */
8510int cik_suspend(struct radeon_device *rdev)
8511{
6c7bccea 8512 radeon_pm_suspend(rdev);
b530602f 8513 dce6_audio_fini(rdev);
7bf94a2c
AD		 8514 radeon_vm_manager_fini(rdev);
8515 cik_cp_enable(rdev, false);
8516 cik_sdma_enable(rdev, false);
e409b128 8517 uvd_v1_0_fini(rdev);
87167bb1 8518 radeon_uvd_suspend(rdev);
d93f7937 8519 radeon_vce_suspend(rdev);
473359bc
AD		 8520 cik_fini_pg(rdev);
8521 cik_fini_cg(rdev);
7bf94a2c
AD		 8522 cik_irq_suspend(rdev);
8523 radeon_wb_disable(rdev);
8524 cik_pcie_gart_disable(rdev);
8525 return 0;
8526}
8527
8528/* Plan is to move initialization in that function and use
8529 * helper function so that radeon_device_init pretty much
8530 * do nothing more than calling asic specific function. This
8531 * should also allow to remove a bunch of callback function
8532 * like vram_info.
8533 */
8534/**
8535 * cik_init - asic specific driver and hw init
8536 *
8537 * @rdev: radeon_device pointer
8538 *
8539 * Setup asic specific driver variables and program the hw
8540 * to a functional state (CIK).
8541 * Called at driver startup.
8542 * Returns 0 for success, errors for failure.
8543 */
8544int cik_init(struct radeon_device *rdev)
8545{
8546 struct radeon_ring *ring;
8547 int r;
8548
8549 /* Read BIOS */
8550 if (!radeon_get_bios(rdev)) {
8551 if (ASIC_IS_AVIVO(rdev))
8552 return -EINVAL;
8553 }
8554 /* Must be an ATOMBIOS */
8555 if (!rdev->is_atom_bios) {
8556 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8557 return -EINVAL;
8558 }
8559 r = radeon_atombios_init(rdev);
8560 if (r)
8561 return r;
8562
8563 /* Post card if necessary */
8564 if (!radeon_card_posted(rdev)) {
8565 if (!rdev->bios) {
8566 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8567 return -EINVAL;
8568 }
8569 DRM_INFO("GPU not posted. posting now...\n");
8570 atom_asic_init(rdev->mode_info.atom_context);
8571 }
0aafd313
AD		 8572 /* init golden registers */
8573 cik_init_golden_registers(rdev);
7bf94a2c
AD		 8574 /* Initialize scratch registers */
8575 cik_scratch_init(rdev);
8576 /* Initialize surface registers */
8577 radeon_surface_init(rdev);
8578 /* Initialize clocks */
8579 radeon_get_clock_info(rdev->ddev);
8580
8581 /* Fence driver */
8582 r = radeon_fence_driver_init(rdev);
8583 if (r)
8584 return r;
8585
8586 /* initialize memory controller */
8587 r = cik_mc_init(rdev);
8588 if (r)
8589 return r;
8590 /* Memory manager */
8591 r = radeon_bo_init(rdev);
8592 if (r)
8593 return r;
8594
01ac8794
AD		 8595 if (rdev->flags & RADEON_IS_IGP) {
8596 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8597 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8598 r = cik_init_microcode(rdev);
8599 if (r) {
8600 DRM_ERROR("Failed to load firmware!\n");
8601 return r;
8602 }
8603 }
8604 } else {
8605 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8606 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8607 !rdev->mc_fw) {
8608 r = cik_init_microcode(rdev);
8609 if (r) {
8610 DRM_ERROR("Failed to load firmware!\n");
8611 return r;
8612 }
8613 }
8614 }
8615
6c7bccea
AD		 8616 /* Initialize power management */
8617 radeon_pm_init(rdev);
8618
7bf94a2c
AD		 8619 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8620 ring->ring_obj = NULL;
8621 r600_ring_init(rdev, ring, 1024 * 1024);
8622
963e81f9
AD		 8623 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8624 ring->ring_obj = NULL;
8625 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8626 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8627 if (r)
8628 return r;
8629
8630 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8631 ring->ring_obj = NULL;
8632 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8633 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8634 if (r)
8635 return r;
8636
7bf94a2c
AD		 8637 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8638 ring->ring_obj = NULL;
8639 r600_ring_init(rdev, ring, 256 * 1024);
8640
8641 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8642 ring->ring_obj = NULL;
8643 r600_ring_init(rdev, ring, 256 * 1024);
8644
87167bb1
CK		 8645 r = radeon_uvd_init(rdev);
8646 if (!r) {
8647 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8648 ring->ring_obj = NULL;
8649 r600_ring_init(rdev, ring, 4096);
8650 }
8651
d93f7937
CK		 8652 r = radeon_vce_init(rdev);
8653 if (!r) {
8654 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8655 ring->ring_obj = NULL;
8656 r600_ring_init(rdev, ring, 4096);
8657
8658 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8659 ring->ring_obj = NULL;
8660 r600_ring_init(rdev, ring, 4096);
8661 }
8662
7bf94a2c
AD		 8663 rdev->ih.ring_obj = NULL;
8664 r600_ih_ring_init(rdev, 64 * 1024);
8665
8666 r = r600_pcie_gart_init(rdev);
8667 if (r)
8668 return r;
8669
8670 rdev->accel_working = true;
8671 r = cik_startup(rdev);
8672 if (r) {
8673 dev_err(rdev->dev, "disabling GPU acceleration\n");
8674 cik_cp_fini(rdev);
8675 cik_sdma_fini(rdev);
8676 cik_irq_fini(rdev);
1fd11777 8677 sumo_rlc_fini(rdev);
963e81f9 8678 cik_mec_fini(rdev);
7bf94a2c
AD		 8679 radeon_wb_fini(rdev);
8680 radeon_ib_pool_fini(rdev);
8681 radeon_vm_manager_fini(rdev);
8682 radeon_irq_kms_fini(rdev);
8683 cik_pcie_gart_fini(rdev);
8684 rdev->accel_working = false;
8685 }
8686
8687 /* Don't start up if the MC ucode is missing.
8688 * The default clocks and voltages before the MC ucode
8689 * is loaded are not suffient for advanced operations.
8690 */
8691 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8692 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8693 return -EINVAL;
8694 }
8695
8696 return 0;
8697}
8698
8699/**
8700 * cik_fini - asic specific driver and hw fini
8701 *
8702 * @rdev: radeon_device pointer
8703 *
8704 * Tear down the asic specific driver variables and program the hw
8705 * to an idle state (CIK).
8706 * Called at driver unload.
8707 */
8708void cik_fini(struct radeon_device *rdev)
8709{
6c7bccea 8710 radeon_pm_fini(rdev);
7bf94a2c
AD		 8711 cik_cp_fini(rdev);
8712 cik_sdma_fini(rdev);
473359bc
AD		 8713 cik_fini_pg(rdev);
8714 cik_fini_cg(rdev);
7bf94a2c 8715 cik_irq_fini(rdev);
1fd11777 8716 sumo_rlc_fini(rdev);
963e81f9 8717 cik_mec_fini(rdev);
7bf94a2c
AD		 8718 radeon_wb_fini(rdev);
8719 radeon_vm_manager_fini(rdev);
8720 radeon_ib_pool_fini(rdev);
8721 radeon_irq_kms_fini(rdev);
e409b128 8722 uvd_v1_0_fini(rdev);
87167bb1 8723 radeon_uvd_fini(rdev);
d93f7937 8724 radeon_vce_fini(rdev);
7bf94a2c
AD		 8725 cik_pcie_gart_fini(rdev);
8726 r600_vram_scratch_fini(rdev);
8727 radeon_gem_fini(rdev);
8728 radeon_fence_driver_fini(rdev);
8729 radeon_bo_fini(rdev);
8730 radeon_atombios_fini(rdev);
8731 kfree(rdev->bios);
8732 rdev->bios = NULL;
8733}
cd84a27d 8734
134b480f
AD		 8735void dce8_program_fmt(struct drm_encoder *encoder)
8736{
8737 struct drm_device *dev = encoder->dev;
8738 struct radeon_device *rdev = dev->dev_private;
8739 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8740 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8741 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8742 int bpc = 0;
8743 u32 tmp = 0;
6214bb74 8744 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 8745
6214bb74
AD		 8746 if (connector) {
8747 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 8748 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 8749 dither = radeon_connector->dither;
8750 }
134b480f
AD		 8751
8752 /* LVDS/eDP FMT is set up by atom */
8753 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8754 return;
8755
8756 /* not needed for analog */
8757 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8758 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8759 return;
8760
8761 if (bpc == 0)
8762 return;
8763
8764 switch (bpc) {
8765 case 6:
6214bb74 8766 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8767 /* XXX sort out optimal dither settings */
8768 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8769 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8770 else
8771 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8772 break;
8773 case 8:
6214bb74 8774 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8775 /* XXX sort out optimal dither settings */
8776 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8777 FMT_RGB_RANDOM_ENABLE |
8778 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8779 else
8780 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8781 break;
8782 case 10:
6214bb74 8783 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8784 /* XXX sort out optimal dither settings */
8785 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8786 FMT_RGB_RANDOM_ENABLE |
8787 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8788 else
8789 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8790 break;
8791 default:
8792 /* not needed */
8793 break;
8794 }
8795
8796 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8797}
8798
cd84a27d
AD		 8799/* display watermark setup */
8800/**
8801 * dce8_line_buffer_adjust - Set up the line buffer
8802 *
8803 * @rdev: radeon_device pointer
8804 * @radeon_crtc: the selected display controller
8805 * @mode: the current display mode on the selected display
8806 * controller
8807 *
8808 * Setup up the line buffer allocation for
8809 * the selected display controller (CIK).
8810 * Returns the line buffer size in pixels.
8811 */
8812static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8813 struct radeon_crtc *radeon_crtc,
8814 struct drm_display_mode *mode)
8815{
bc01a8c7
AD		 8816 u32 tmp, buffer_alloc, i;
8817 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 8818 /*
8819 * Line Buffer Setup
8820 * There are 6 line buffers, one for each display controllers.
8821 * There are 3 partitions per LB. Select the number of partitions
8822 * to enable based on the display width. For display widths larger
8823 * than 4096, you need use to use 2 display controllers and combine
8824 * them using the stereo blender.
8825 */
8826 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 8827 if (mode->crtc_hdisplay < 1920) {
cd84a27d 8828 tmp = 1;
bc01a8c7
AD		 8829 buffer_alloc = 2;
8830 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 8831 tmp = 2;
bc01a8c7
AD		 8832 buffer_alloc = 2;
8833 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 8834 tmp = 0;
bc01a8c7
AD		 8835 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8836 } else {
cd84a27d
AD		 8837 DRM_DEBUG_KMS("Mode too big for LB!\n");
8838 tmp = 0;
bc01a8c7 8839 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 8840 }
bc01a8c7 8841 } else {
cd84a27d 8842 tmp = 1;
bc01a8c7
AD		 8843 buffer_alloc = 0;
8844 }
cd84a27d
AD		 8845
8846 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8847 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8848
bc01a8c7
AD		 8849 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8850 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8851 for (i = 0; i < rdev->usec_timeout; i++) {
8852 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8853 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8854 break;
8855 udelay(1);
8856 }
8857
cd84a27d
AD		 8858 if (radeon_crtc->base.enabled && mode) {
8859 switch (tmp) {
8860 case 0:
8861 default:
8862 return 4096 * 2;
8863 case 1:
8864 return 1920 * 2;
8865 case 2:
8866 return 2560 * 2;
8867 }
8868 }
8869
8870 /* controller not enabled, so no lb used */
8871 return 0;
8872}
8873
8874/**
8875 * cik_get_number_of_dram_channels - get the number of dram channels
8876 *
8877 * @rdev: radeon_device pointer
8878 *
8879 * Look up the number of video ram channels (CIK).
8880 * Used for display watermark bandwidth calculations
8881 * Returns the number of dram channels
8882 */
8883static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8884{
8885 u32 tmp = RREG32(MC_SHARED_CHMAP);
8886
8887 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8888 case 0:
8889 default:
8890 return 1;
8891 case 1:
8892 return 2;
8893 case 2:
8894 return 4;
8895 case 3:
8896 return 8;
8897 case 4:
8898 return 3;
8899 case 5:
8900 return 6;
8901 case 6:
8902 return 10;
8903 case 7:
8904 return 12;
8905 case 8:
8906 return 16;
8907 }
8908}
8909
8910struct dce8_wm_params {
8911 u32 dram_channels; /* number of dram channels */
8912 u32 yclk; /* bandwidth per dram data pin in kHz */
8913 u32 sclk; /* engine clock in kHz */
8914 u32 disp_clk; /* display clock in kHz */
8915 u32 src_width; /* viewport width */
8916 u32 active_time; /* active display time in ns */
8917 u32 blank_time; /* blank time in ns */
8918 bool interlaced; /* mode is interlaced */
8919 fixed20_12 vsc; /* vertical scale ratio */
8920 u32 num_heads; /* number of active crtcs */
8921 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8922 u32 lb_size; /* line buffer allocated to pipe */
8923 u32 vtaps; /* vertical scaler taps */
8924};
8925
8926/**
8927 * dce8_dram_bandwidth - get the dram bandwidth
8928 *
8929 * @wm: watermark calculation data
8930 *
8931 * Calculate the raw dram bandwidth (CIK).
8932 * Used for display watermark bandwidth calculations
8933 * Returns the dram bandwidth in MBytes/s
8934 */
8935static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8936{
8937 /* Calculate raw DRAM Bandwidth */
8938 fixed20_12 dram_efficiency; /* 0.7 */
8939 fixed20_12 yclk, dram_channels, bandwidth;
8940 fixed20_12 a;
8941
8942 a.full = dfixed_const(1000);
8943 yclk.full = dfixed_const(wm->yclk);
8944 yclk.full = dfixed_div(yclk, a);
8945 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8946 a.full = dfixed_const(10);
8947 dram_efficiency.full = dfixed_const(7);
8948 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8949 bandwidth.full = dfixed_mul(dram_channels, yclk);
8950 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8951
8952 return dfixed_trunc(bandwidth);
8953}
8954
8955/**
8956 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8957 *
8958 * @wm: watermark calculation data
8959 *
8960 * Calculate the dram bandwidth used for display (CIK).
8961 * Used for display watermark bandwidth calculations
8962 * Returns the dram bandwidth for display in MBytes/s
8963 */
8964static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8965{
8966 /* Calculate DRAM Bandwidth and the part allocated to display. */
8967 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8968 fixed20_12 yclk, dram_channels, bandwidth;
8969 fixed20_12 a;
8970
8971 a.full = dfixed_const(1000);
8972 yclk.full = dfixed_const(wm->yclk);
8973 yclk.full = dfixed_div(yclk, a);
8974 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8975 a.full = dfixed_const(10);
8976 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8977 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8978 bandwidth.full = dfixed_mul(dram_channels, yclk);
8979 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8980
8981 return dfixed_trunc(bandwidth);
8982}
8983
8984/**
8985 * dce8_data_return_bandwidth - get the data return bandwidth
8986 *
8987 * @wm: watermark calculation data
8988 *
8989 * Calculate the data return bandwidth used for display (CIK).
8990 * Used for display watermark bandwidth calculations
8991 * Returns the data return bandwidth in MBytes/s
8992 */
8993static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8994{
8995 /* Calculate the display Data return Bandwidth */
8996 fixed20_12 return_efficiency; /* 0.8 */
8997 fixed20_12 sclk, bandwidth;
8998 fixed20_12 a;
8999
9000 a.full = dfixed_const(1000);
9001 sclk.full = dfixed_const(wm->sclk);
9002 sclk.full = dfixed_div(sclk, a);
9003 a.full = dfixed_const(10);
9004 return_efficiency.full = dfixed_const(8);
9005 return_efficiency.full = dfixed_div(return_efficiency, a);
9006 a.full = dfixed_const(32);
9007 bandwidth.full = dfixed_mul(a, sclk);
9008 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9009
9010 return dfixed_trunc(bandwidth);
9011}
9012
9013/**
9014 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9015 *
9016 * @wm: watermark calculation data
9017 *
9018 * Calculate the dmif bandwidth used for display (CIK).
9019 * Used for display watermark bandwidth calculations
9020 * Returns the dmif bandwidth in MBytes/s
9021 */
9022static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9023{
9024 /* Calculate the DMIF Request Bandwidth */
9025 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9026 fixed20_12 disp_clk, bandwidth;
9027 fixed20_12 a, b;
9028
9029 a.full = dfixed_const(1000);
9030 disp_clk.full = dfixed_const(wm->disp_clk);
9031 disp_clk.full = dfixed_div(disp_clk, a);
9032 a.full = dfixed_const(32);
9033 b.full = dfixed_mul(a, disp_clk);
9034
9035 a.full = dfixed_const(10);
9036 disp_clk_request_efficiency.full = dfixed_const(8);
9037 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9038
9039 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9040
9041 return dfixed_trunc(bandwidth);
9042}
9043
9044/**
9045 * dce8_available_bandwidth - get the min available bandwidth
9046 *
9047 * @wm: watermark calculation data
9048 *
9049 * Calculate the min available bandwidth used for display (CIK).
9050 * Used for display watermark bandwidth calculations
9051 * Returns the min available bandwidth in MBytes/s
9052 */
9053static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9054{
9055 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9056 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9057 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9058 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9059
9060 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9061}
9062
9063/**
9064 * dce8_average_bandwidth - get the average available bandwidth
9065 *
9066 * @wm: watermark calculation data
9067 *
9068 * Calculate the average available bandwidth used for display (CIK).
9069 * Used for display watermark bandwidth calculations
9070 * Returns the average available bandwidth in MBytes/s
9071 */
9072static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9073{
9074 /* Calculate the display mode Average Bandwidth
9075 * DisplayMode should contain the source and destination dimensions,
9076 * timing, etc.
9077 */
9078 fixed20_12 bpp;
9079 fixed20_12 line_time;
9080 fixed20_12 src_width;
9081 fixed20_12 bandwidth;
9082 fixed20_12 a;
9083
9084 a.full = dfixed_const(1000);
9085 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9086 line_time.full = dfixed_div(line_time, a);
9087 bpp.full = dfixed_const(wm->bytes_per_pixel);
9088 src_width.full = dfixed_const(wm->src_width);
9089 bandwidth.full = dfixed_mul(src_width, bpp);
9090 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9091 bandwidth.full = dfixed_div(bandwidth, line_time);
9092
9093 return dfixed_trunc(bandwidth);
9094}
9095
9096/**
9097 * dce8_latency_watermark - get the latency watermark
9098 *
9099 * @wm: watermark calculation data
9100 *
9101 * Calculate the latency watermark (CIK).
9102 * Used for display watermark bandwidth calculations
9103 * Returns the latency watermark in ns
9104 */
9105static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9106{
9107 /* First calculate the latency in ns */
9108 u32 mc_latency = 2000; /* 2000 ns. */
9109 u32 available_bandwidth = dce8_available_bandwidth(wm);
9110 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9111 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9112 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9113 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9114 (wm->num_heads * cursor_line_pair_return_time);
9115 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9116 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9117 u32 tmp, dmif_size = 12288;
9118 fixed20_12 a, b, c;
9119
9120 if (wm->num_heads == 0)
9121 return 0;
9122
9123 a.full = dfixed_const(2);
9124 b.full = dfixed_const(1);
9125 if ((wm->vsc.full > a.full) ||
9126 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9127 (wm->vtaps >= 5) ||
9128 ((wm->vsc.full >= a.full) && wm->interlaced))
9129 max_src_lines_per_dst_line = 4;
9130 else
9131 max_src_lines_per_dst_line = 2;
9132
9133 a.full = dfixed_const(available_bandwidth);
9134 b.full = dfixed_const(wm->num_heads);
9135 a.full = dfixed_div(a, b);
9136
9137 b.full = dfixed_const(mc_latency + 512);
9138 c.full = dfixed_const(wm->disp_clk);
9139 b.full = dfixed_div(b, c);
9140
9141 c.full = dfixed_const(dmif_size);
9142 b.full = dfixed_div(c, b);
9143
9144 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9145
9146 b.full = dfixed_const(1000);
9147 c.full = dfixed_const(wm->disp_clk);
9148 b.full = dfixed_div(c, b);
9149 c.full = dfixed_const(wm->bytes_per_pixel);
9150 b.full = dfixed_mul(b, c);
9151
9152 lb_fill_bw = min(tmp, dfixed_trunc(b));
9153
9154 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9155 b.full = dfixed_const(1000);
9156 c.full = dfixed_const(lb_fill_bw);
9157 b.full = dfixed_div(c, b);
9158 a.full = dfixed_div(a, b);
9159 line_fill_time = dfixed_trunc(a);
9160
9161 if (line_fill_time < wm->active_time)
9162 return latency;
9163 else
9164 return latency + (line_fill_time - wm->active_time);
9165
9166}
9167
9168/**
9169 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9170 * average and available dram bandwidth
9171 *
9172 * @wm: watermark calculation data
9173 *
9174 * Check if the display average bandwidth fits in the display
9175 * dram bandwidth (CIK).
9176 * Used for display watermark bandwidth calculations
9177 * Returns true if the display fits, false if not.
9178 */
9179static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9180{
9181 if (dce8_average_bandwidth(wm) <=
9182 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9183 return true;
9184 else
9185 return false;
9186}
9187
9188/**
9189 * dce8_average_bandwidth_vs_available_bandwidth - check
9190 * average and available bandwidth
9191 *
9192 * @wm: watermark calculation data
9193 *
9194 * Check if the display average bandwidth fits in the display
9195 * available bandwidth (CIK).
9196 * Used for display watermark bandwidth calculations
9197 * Returns true if the display fits, false if not.
9198 */
9199static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9200{
9201 if (dce8_average_bandwidth(wm) <=
9202 (dce8_available_bandwidth(wm) / wm->num_heads))
9203 return true;
9204 else
9205 return false;
9206}
9207
9208/**
9209 * dce8_check_latency_hiding - check latency hiding
9210 *
9211 * @wm: watermark calculation data
9212 *
9213 * Check latency hiding (CIK).
9214 * Used for display watermark bandwidth calculations
9215 * Returns true if the display fits, false if not.
9216 */
9217static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9218{
9219 u32 lb_partitions = wm->lb_size / wm->src_width;
9220 u32 line_time = wm->active_time + wm->blank_time;
9221 u32 latency_tolerant_lines;
9222 u32 latency_hiding;
9223 fixed20_12 a;
9224
9225 a.full = dfixed_const(1);
9226 if (wm->vsc.full > a.full)
9227 latency_tolerant_lines = 1;
9228 else {
9229 if (lb_partitions <= (wm->vtaps + 1))
9230 latency_tolerant_lines = 1;
9231 else
9232 latency_tolerant_lines = 2;
9233 }
9234
9235 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9236
9237 if (dce8_latency_watermark(wm) <= latency_hiding)
9238 return true;
9239 else
9240 return false;
9241}
9242
9243/**
9244 * dce8_program_watermarks - program display watermarks
9245 *
9246 * @rdev: radeon_device pointer
9247 * @radeon_crtc: the selected display controller
9248 * @lb_size: line buffer size
9249 * @num_heads: number of display controllers in use
9250 *
9251 * Calculate and program the display watermarks for the
9252 * selected display controller (CIK).
9253 */
9254static void dce8_program_watermarks(struct radeon_device *rdev,
9255 struct radeon_crtc *radeon_crtc,
9256 u32 lb_size, u32 num_heads)
9257{
9258 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 9259 struct dce8_wm_params wm_low, wm_high;
cd84a27d
AD		 9260 u32 pixel_period;
9261 u32 line_time = 0;
9262 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9263 u32 tmp, wm_mask;
9264
9265 if (radeon_crtc->base.enabled && num_heads && mode) {
9266 pixel_period = 1000000 / (u32)mode->clock;
9267 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9268
58ea2dea
AD		 9269 /* watermark for high clocks */
9270 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9271 rdev->pm.dpm_enabled) {
9272 wm_high.yclk =
9273 radeon_dpm_get_mclk(rdev, false) * 10;
9274 wm_high.sclk =
9275 radeon_dpm_get_sclk(rdev, false) * 10;
9276 } else {
9277 wm_high.yclk = rdev->pm.current_mclk * 10;
9278 wm_high.sclk = rdev->pm.current_sclk * 10;
9279 }
9280
9281 wm_high.disp_clk = mode->clock;
9282 wm_high.src_width = mode->crtc_hdisplay;
9283 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9284 wm_high.blank_time = line_time - wm_high.active_time;
9285 wm_high.interlaced = false;
cd84a27d 9286 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 9287 wm_high.interlaced = true;
9288 wm_high.vsc = radeon_crtc->vsc;
9289 wm_high.vtaps = 1;
cd84a27d 9290 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 9291 wm_high.vtaps = 2;
9292 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9293 wm_high.lb_size = lb_size;
9294 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9295 wm_high.num_heads = num_heads;
cd84a27d
AD		 9296
9297 /* set for high clocks */
58ea2dea
AD		 9298 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9299
9300 /* possibly force display priority to high */
9301 /* should really do this at mode validation time... */
9302 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9303 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9304 !dce8_check_latency_hiding(&wm_high) ||
9305 (rdev->disp_priority == 2)) {
9306 DRM_DEBUG_KMS("force priority to high\n");
9307 }
9308
9309 /* watermark for low clocks */
9310 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9311 rdev->pm.dpm_enabled) {
9312 wm_low.yclk =
9313 radeon_dpm_get_mclk(rdev, true) * 10;
9314 wm_low.sclk =
9315 radeon_dpm_get_sclk(rdev, true) * 10;
9316 } else {
9317 wm_low.yclk = rdev->pm.current_mclk * 10;
9318 wm_low.sclk = rdev->pm.current_sclk * 10;
9319 }
9320
9321 wm_low.disp_clk = mode->clock;
9322 wm_low.src_width = mode->crtc_hdisplay;
9323 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9324 wm_low.blank_time = line_time - wm_low.active_time;
9325 wm_low.interlaced = false;
9326 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9327 wm_low.interlaced = true;
9328 wm_low.vsc = radeon_crtc->vsc;
9329 wm_low.vtaps = 1;
9330 if (radeon_crtc->rmx_type != RMX_OFF)
9331 wm_low.vtaps = 2;
9332 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9333 wm_low.lb_size = lb_size;
9334 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9335 wm_low.num_heads = num_heads;
9336
cd84a27d 9337 /* set for low clocks */
58ea2dea 9338 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 9339
9340 /* possibly force display priority to high */
9341 /* should really do this at mode validation time... */
58ea2dea
AD		 9342 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9343 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9344 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 9345 (rdev->disp_priority == 2)) {
9346 DRM_DEBUG_KMS("force priority to high\n");
9347 }
9348 }
9349
9350 /* select wm A */
9351 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9352 tmp = wm_mask;
9353 tmp &= ~LATENCY_WATERMARK_MASK(3);
9354 tmp |= LATENCY_WATERMARK_MASK(1);
9355 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9356 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9357 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9358 LATENCY_HIGH_WATERMARK(line_time)));
9359 /* select wm B */
9360 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9361 tmp &= ~LATENCY_WATERMARK_MASK(3);
9362 tmp |= LATENCY_WATERMARK_MASK(2);
9363 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9364 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9365 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9366 LATENCY_HIGH_WATERMARK(line_time)));
9367 /* restore original selection */
9368 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 9369
9370 /* save values for DPM */
9371 radeon_crtc->line_time = line_time;
9372 radeon_crtc->wm_high = latency_watermark_a;
9373 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 9374}
9375
9376/**
9377 * dce8_bandwidth_update - program display watermarks
9378 *
9379 * @rdev: radeon_device pointer
9380 *
9381 * Calculate and program the display watermarks and line
9382 * buffer allocation (CIK).
9383 */
9384void dce8_bandwidth_update(struct radeon_device *rdev)
9385{
9386 struct drm_display_mode *mode = NULL;
9387 u32 num_heads = 0, lb_size;
9388 int i;
9389
9390 radeon_update_display_priority(rdev);
9391
9392 for (i = 0; i < rdev->num_crtc; i++) {
9393 if (rdev->mode_info.crtcs[i]->base.enabled)
9394 num_heads++;
9395 }
9396 for (i = 0; i < rdev->num_crtc; i++) {
9397 mode = &rdev->mode_info.crtcs[i]->base.mode;
9398 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9399 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9400 }
9401}
44fa346f
AD		 9402
9403/**
9404 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9405 *
9406 * @rdev: radeon_device pointer
9407 *
9408 * Fetches a GPU clock counter snapshot (SI).
9409 * Returns the 64 bit clock counter snapshot.
9410 */
9411uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9412{
9413 uint64_t clock;
9414
9415 mutex_lock(&rdev->gpu_clock_mutex);
9416 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9417 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9418 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9419 mutex_unlock(&rdev->gpu_clock_mutex);
9420 return clock;
9421}
9422
87167bb1
CK		 9423static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9424 u32 cntl_reg, u32 status_reg)
9425{
9426 int r, i;
9427 struct atom_clock_dividers dividers;
9428 uint32_t tmp;
9429
9430 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9431 clock, false, &dividers);
9432 if (r)
9433 return r;
9434
9435 tmp = RREG32_SMC(cntl_reg);
9436 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9437 tmp |= dividers.post_divider;
9438 WREG32_SMC(cntl_reg, tmp);
9439
9440 for (i = 0; i < 100; i++) {
9441 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9442 break;
9443 mdelay(10);
9444 }
9445 if (i == 100)
9446 return -ETIMEDOUT;
9447
9448 return 0;
9449}
9450
9451int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9452{
9453 int r = 0;
9454
9455 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9456 if (r)
9457 return r;
9458
9459 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9460 return r;
9461}
9462
5ad6bf91
AD		 9463int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9464{
9465 int r, i;
9466 struct atom_clock_dividers dividers;
9467 u32 tmp;
9468
9469 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9470 ecclk, false, &dividers);
9471 if (r)
9472 return r;
9473
9474 for (i = 0; i < 100; i++) {
9475 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9476 break;
9477 mdelay(10);
9478 }
9479 if (i == 100)
9480 return -ETIMEDOUT;
9481
9482 tmp = RREG32_SMC(CG_ECLK_CNTL);
9483 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9484 tmp |= dividers.post_divider;
9485 WREG32_SMC(CG_ECLK_CNTL, tmp);
9486
9487 for (i = 0; i < 100; i++) {
9488 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9489 break;
9490 mdelay(10);
9491 }
9492 if (i == 100)
9493 return -ETIMEDOUT;
9494
9495 return 0;
9496}
9497
8a7cd276 9498static void cik_pcie_gen3_enable(struct radeon_device *rdev)
87167bb1 9499{
8a7cd276
AD		 9500 struct pci_dev *root = rdev->pdev->bus->self;
9501 int bridge_pos, gpu_pos;
9502 u32 speed_cntl, mask, current_data_rate;
9503 int ret, i;
9504 u16 tmp16;
87167bb1 9505
0bd252de
AW		 9506 if (pci_is_root_bus(rdev->pdev->bus))
9507 return;
9508
8a7cd276
AD		 9509 if (radeon_pcie_gen2 == 0)
9510 return;
87167bb1 9511
8a7cd276
AD		 9512 if (rdev->flags & RADEON_IS_IGP)
9513 return;
87167bb1 9514
8a7cd276
AD		 9515 if (!(rdev->flags & RADEON_IS_PCIE))
9516 return;
87167bb1 9517
8a7cd276
AD		 9518 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9519 if (ret != 0)
9520 return;
87167bb1 9521
8a7cd276
AD		 9522 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9523 return;
87167bb1 9524
8a7cd276
AD		 9525 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9526 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9527 LC_CURRENT_DATA_RATE_SHIFT;
9528 if (mask & DRM_PCIE_SPEED_80) {
9529 if (current_data_rate == 2) {
9530 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9531 return;
9532 }
9533 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9534 } else if (mask & DRM_PCIE_SPEED_50) {
9535 if (current_data_rate == 1) {
9536 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9537 return;
9538 }
9539 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9540 }
87167bb1 9541
8a7cd276
AD		 9542 bridge_pos = pci_pcie_cap(root);
9543 if (!bridge_pos)
9544 return;
9545
9546 gpu_pos = pci_pcie_cap(rdev->pdev);
9547 if (!gpu_pos)
9548 return;
9549
9550 if (mask & DRM_PCIE_SPEED_80) {
9551 /* re-try equalization if gen3 is not already enabled */
9552 if (current_data_rate != 2) {
9553 u16 bridge_cfg, gpu_cfg;
9554 u16 bridge_cfg2, gpu_cfg2;
9555 u32 max_lw, current_lw, tmp;
9556
9557 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9558 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9559
9560 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9561 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9562
9563 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9564 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9565
9566 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9567 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9568 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9569
9570 if (current_lw < max_lw) {
9571 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9572 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9573 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9574 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9575 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9576 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9577 }
9578 }
9579
9580 for (i = 0; i < 10; i++) {
9581 /* check status */
9582 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9583 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9584 break;
9585
9586 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9587 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9588
9589 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9590 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9591
9592 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9593 tmp |= LC_SET_QUIESCE;
9594 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9595
9596 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9597 tmp |= LC_REDO_EQ;
9598 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9599
9600 mdelay(100);
9601
9602 /* linkctl */
9603 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9604 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9605 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9606 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9607
9608 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9609 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9610 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9611 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9612
9613 /* linkctl2 */
9614 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9615 tmp16 &= ~((1 << 4) | (7 << 9));
9616 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9617 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9618
9619 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9620 tmp16 &= ~((1 << 4) | (7 << 9));
9621 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9622 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9623
9624 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9625 tmp &= ~LC_SET_QUIESCE;
9626 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9627 }
9628 }
9629 }
9630
9631 /* set the link speed */
9632 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9633 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9634 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9635
9636 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9637 tmp16 &= ~0xf;
9638 if (mask & DRM_PCIE_SPEED_80)
9639 tmp16 |= 3; /* gen3 */
9640 else if (mask & DRM_PCIE_SPEED_50)
9641 tmp16 |= 2; /* gen2 */
9642 else
9643 tmp16 |= 1; /* gen1 */
9644 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9645
9646 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9647 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9648 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9649
9650 for (i = 0; i < rdev->usec_timeout; i++) {
9651 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9652 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9653 break;
9654 udelay(1);
9655 }
9656}
7235711a
AD		 9657
9658static void cik_program_aspm(struct radeon_device *rdev)
9659{
9660 u32 data, orig;
9661 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9662 bool disable_clkreq = false;
9663
9664 if (radeon_aspm == 0)
9665 return;
9666
9667 /* XXX double check IGPs */
9668 if (rdev->flags & RADEON_IS_IGP)
9669 return;
9670
9671 if (!(rdev->flags & RADEON_IS_PCIE))
9672 return;
9673
9674 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9675 data &= ~LC_XMIT_N_FTS_MASK;
9676 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9677 if (orig != data)
9678 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9679
9680 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9681 data |= LC_GO_TO_RECOVERY;
9682 if (orig != data)
9683 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9684
9685 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9686 data |= P_IGNORE_EDB_ERR;
9687 if (orig != data)
9688 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9689
9690 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9691 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9692 data |= LC_PMI_TO_L1_DIS;
9693 if (!disable_l0s)
9694 data |= LC_L0S_INACTIVITY(7);
9695
9696 if (!disable_l1) {
9697 data |= LC_L1_INACTIVITY(7);
9698 data &= ~LC_PMI_TO_L1_DIS;
9699 if (orig != data)
9700 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9701
9702 if (!disable_plloff_in_l1) {
9703 bool clk_req_support;
9704
9705 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9706 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9707 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9708 if (orig != data)
9709 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9710
9711 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9712 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9713 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9714 if (orig != data)
9715 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9716
9717 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9718 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9719 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9720 if (orig != data)
9721 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9722
9723 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9724 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9725 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9726 if (orig != data)
9727 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9728
9729 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9730 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9731 data |= LC_DYN_LANES_PWR_STATE(3);
9732 if (orig != data)
9733 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9734
0bd252de
AW		 9735 if (!disable_clkreq &&
9736 !pci_is_root_bus(rdev->pdev->bus)) {
7235711a
AD		 9737 struct pci_dev *root = rdev->pdev->bus->self;
9738 u32 lnkcap;
9739
9740 clk_req_support = false;
9741 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9742 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9743 clk_req_support = true;
9744 } else {
9745 clk_req_support = false;
9746 }
9747
9748 if (clk_req_support) {
9749 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9750 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9751 if (orig != data)
9752 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9753
9754 orig = data = RREG32_SMC(THM_CLK_CNTL);
9755 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9756 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9757 if (orig != data)
9758 WREG32_SMC(THM_CLK_CNTL, data);
9759
9760 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9761 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9762 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9763 if (orig != data)
9764 WREG32_SMC(MISC_CLK_CTRL, data);
9765
9766 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9767 data &= ~BCLK_AS_XCLK;
9768 if (orig != data)
9769 WREG32_SMC(CG_CLKPIN_CNTL, data);
9770
9771 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9772 data &= ~FORCE_BIF_REFCLK_EN;
9773 if (orig != data)
9774 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9775
9776 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9777 data &= ~MPLL_CLKOUT_SEL_MASK;
9778 data |= MPLL_CLKOUT_SEL(4);
9779 if (orig != data)
9780 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9781 }
9782 }
9783 } else {
9784 if (orig != data)
9785 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9786 }
9787
9788 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9789 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9790 if (orig != data)
9791 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9792
9793 if (!disable_l0s) {
9794 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9795 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9796 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9797 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9798 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9799 data &= ~LC_L0S_INACTIVITY_MASK;
9800 if (orig != data)
9801 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9802 }
9803 }
9804 }
87167bb1 9805}
Michael's Linux tree.