]> git.ipfire.org Git - people/ms/linux.git/blame - drivers/gpu/drm/radeon/cik.c
projects / people / ms / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon/vm: don't attempt to update ptes if ib allocation fails
[people/ms/linux.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
8cc1a532
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
6f2043ce 29#include "radeon_asic.h"
8cc1a532
AD		 30#include "cikd.h"
31#include "atom.h"
841cf442 32#include "cik_blit_shaders.h"
8c68e393 33#include "radeon_ucode.h"
22c775ce 34#include "clearstate_ci.h"
02c81327
AD		 35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 42MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 43MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
d4775655
AD		 44MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
45MODULE_FIRMWARE("radeon/HAWAII_me.bin");
46MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
47MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
48MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
49MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
50MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
51MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
02c81327
AD		 52MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
53MODULE_FIRMWARE("radeon/KAVERI_me.bin");
54MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
55MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
56MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 57MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
02c81327
AD		 58MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
59MODULE_FIRMWARE("radeon/KABINI_me.bin");
60MODULE_FIRMWARE("radeon/KABINI_ce.bin");
61MODULE_FIRMWARE("radeon/KABINI_mec.bin");
62MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 63MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
02c81327 64
a59781bb
AD		 65extern int r600_ih_ring_alloc(struct radeon_device *rdev);
66extern void r600_ih_ring_fini(struct radeon_device *rdev);
6f2043ce
AD		 67extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
68extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
cc066715 69extern bool evergreen_is_display_hung(struct radeon_device *rdev);
1fd11777
AD		 70extern void sumo_rlc_fini(struct radeon_device *rdev);
71extern int sumo_rlc_init(struct radeon_device *rdev);
1c49165d 72extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 73extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 74extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
2483b4ea
CK		 75extern int cik_sdma_resume(struct radeon_device *rdev);
76extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
77extern void cik_sdma_fini(struct radeon_device *rdev);
cc066715 78static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 79static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 80static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 81static void cik_init_pg(struct radeon_device *rdev);
82static void cik_init_cg(struct radeon_device *rdev);
fb2c7f4d
AD		 83static void cik_fini_pg(struct radeon_device *rdev);
84static void cik_fini_cg(struct radeon_device *rdev);
4214faf6
AD		 85static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
86 bool enable);
6f2043ce 87
286d9cc6
AD		 88/* get temperature in millidegrees */
89int ci_get_temp(struct radeon_device *rdev)
90{
91 u32 temp;
92 int actual_temp = 0;
93
94 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
95 CTF_TEMP_SHIFT;
96
97 if (temp & 0x200)
98 actual_temp = 255;
99 else
100 actual_temp = temp & 0x1ff;
101
102 actual_temp = actual_temp * 1000;
103
104 return actual_temp;
105}
106
107/* get temperature in millidegrees */
108int kv_get_temp(struct radeon_device *rdev)
109{
110 u32 temp;
111 int actual_temp = 0;
112
113 temp = RREG32_SMC(0xC0300E0C);
114
115 if (temp)
116 actual_temp = (temp / 8) - 49;
117 else
118 actual_temp = 0;
119
120 actual_temp = actual_temp * 1000;
121
122 return actual_temp;
123}
6f2043ce 124
6e2c3c0a
AD		 125/*
126 * Indirect registers accessor
127 */
128u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
129{
0a5b7b0b 130 unsigned long flags;
6e2c3c0a
AD		 131 u32 r;
132
0a5b7b0b 133 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 134 WREG32(PCIE_INDEX, reg);
135 (void)RREG32(PCIE_INDEX);
136 r = RREG32(PCIE_DATA);
0a5b7b0b 137 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 138 return r;
139}
140
141void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
142{
0a5b7b0b
AD		 143 unsigned long flags;
144
145 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 146 WREG32(PCIE_INDEX, reg);
147 (void)RREG32(PCIE_INDEX);
148 WREG32(PCIE_DATA, v);
149 (void)RREG32(PCIE_DATA);
0a5b7b0b 150 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 151}
152
22c775ce
AD		 153static const u32 spectre_rlc_save_restore_register_list[] =
154{
155 (0x0e00 << 16) | (0xc12c >> 2),
156 0x00000000,
157 (0x0e00 << 16) | (0xc140 >> 2),
158 0x00000000,
159 (0x0e00 << 16) | (0xc150 >> 2),
160 0x00000000,
161 (0x0e00 << 16) | (0xc15c >> 2),
162 0x00000000,
163 (0x0e00 << 16) | (0xc168 >> 2),
164 0x00000000,
165 (0x0e00 << 16) | (0xc170 >> 2),
166 0x00000000,
167 (0x0e00 << 16) | (0xc178 >> 2),
168 0x00000000,
169 (0x0e00 << 16) | (0xc204 >> 2),
170 0x00000000,
171 (0x0e00 << 16) | (0xc2b4 >> 2),
172 0x00000000,
173 (0x0e00 << 16) | (0xc2b8 >> 2),
174 0x00000000,
175 (0x0e00 << 16) | (0xc2bc >> 2),
176 0x00000000,
177 (0x0e00 << 16) | (0xc2c0 >> 2),
178 0x00000000,
179 (0x0e00 << 16) | (0x8228 >> 2),
180 0x00000000,
181 (0x0e00 << 16) | (0x829c >> 2),
182 0x00000000,
183 (0x0e00 << 16) | (0x869c >> 2),
184 0x00000000,
185 (0x0600 << 16) | (0x98f4 >> 2),
186 0x00000000,
187 (0x0e00 << 16) | (0x98f8 >> 2),
188 0x00000000,
189 (0x0e00 << 16) | (0x9900 >> 2),
190 0x00000000,
191 (0x0e00 << 16) | (0xc260 >> 2),
192 0x00000000,
193 (0x0e00 << 16) | (0x90e8 >> 2),
194 0x00000000,
195 (0x0e00 << 16) | (0x3c000 >> 2),
196 0x00000000,
197 (0x0e00 << 16) | (0x3c00c >> 2),
198 0x00000000,
199 (0x0e00 << 16) | (0x8c1c >> 2),
200 0x00000000,
201 (0x0e00 << 16) | (0x9700 >> 2),
202 0x00000000,
203 (0x0e00 << 16) | (0xcd20 >> 2),
204 0x00000000,
205 (0x4e00 << 16) | (0xcd20 >> 2),
206 0x00000000,
207 (0x5e00 << 16) | (0xcd20 >> 2),
208 0x00000000,
209 (0x6e00 << 16) | (0xcd20 >> 2),
210 0x00000000,
211 (0x7e00 << 16) | (0xcd20 >> 2),
212 0x00000000,
213 (0x8e00 << 16) | (0xcd20 >> 2),
214 0x00000000,
215 (0x9e00 << 16) | (0xcd20 >> 2),
216 0x00000000,
217 (0xae00 << 16) | (0xcd20 >> 2),
218 0x00000000,
219 (0xbe00 << 16) | (0xcd20 >> 2),
220 0x00000000,
221 (0x0e00 << 16) | (0x89bc >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0x8900 >> 2),
224 0x00000000,
225 0x3,
226 (0x0e00 << 16) | (0xc130 >> 2),
227 0x00000000,
228 (0x0e00 << 16) | (0xc134 >> 2),
229 0x00000000,
230 (0x0e00 << 16) | (0xc1fc >> 2),
231 0x00000000,
232 (0x0e00 << 16) | (0xc208 >> 2),
233 0x00000000,
234 (0x0e00 << 16) | (0xc264 >> 2),
235 0x00000000,
236 (0x0e00 << 16) | (0xc268 >> 2),
237 0x00000000,
238 (0x0e00 << 16) | (0xc26c >> 2),
239 0x00000000,
240 (0x0e00 << 16) | (0xc270 >> 2),
241 0x00000000,
242 (0x0e00 << 16) | (0xc274 >> 2),
243 0x00000000,
244 (0x0e00 << 16) | (0xc278 >> 2),
245 0x00000000,
246 (0x0e00 << 16) | (0xc27c >> 2),
247 0x00000000,
248 (0x0e00 << 16) | (0xc280 >> 2),
249 0x00000000,
250 (0x0e00 << 16) | (0xc284 >> 2),
251 0x00000000,
252 (0x0e00 << 16) | (0xc288 >> 2),
253 0x00000000,
254 (0x0e00 << 16) | (0xc28c >> 2),
255 0x00000000,
256 (0x0e00 << 16) | (0xc290 >> 2),
257 0x00000000,
258 (0x0e00 << 16) | (0xc294 >> 2),
259 0x00000000,
260 (0x0e00 << 16) | (0xc298 >> 2),
261 0x00000000,
262 (0x0e00 << 16) | (0xc29c >> 2),
263 0x00000000,
264 (0x0e00 << 16) | (0xc2a0 >> 2),
265 0x00000000,
266 (0x0e00 << 16) | (0xc2a4 >> 2),
267 0x00000000,
268 (0x0e00 << 16) | (0xc2a8 >> 2),
269 0x00000000,
270 (0x0e00 << 16) | (0xc2ac >> 2),
271 0x00000000,
272 (0x0e00 << 16) | (0xc2b0 >> 2),
273 0x00000000,
274 (0x0e00 << 16) | (0x301d0 >> 2),
275 0x00000000,
276 (0x0e00 << 16) | (0x30238 >> 2),
277 0x00000000,
278 (0x0e00 << 16) | (0x30250 >> 2),
279 0x00000000,
280 (0x0e00 << 16) | (0x30254 >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0x30258 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0x3025c >> 2),
285 0x00000000,
286 (0x4e00 << 16) | (0xc900 >> 2),
287 0x00000000,
288 (0x5e00 << 16) | (0xc900 >> 2),
289 0x00000000,
290 (0x6e00 << 16) | (0xc900 >> 2),
291 0x00000000,
292 (0x7e00 << 16) | (0xc900 >> 2),
293 0x00000000,
294 (0x8e00 << 16) | (0xc900 >> 2),
295 0x00000000,
296 (0x9e00 << 16) | (0xc900 >> 2),
297 0x00000000,
298 (0xae00 << 16) | (0xc900 >> 2),
299 0x00000000,
300 (0xbe00 << 16) | (0xc900 >> 2),
301 0x00000000,
302 (0x4e00 << 16) | (0xc904 >> 2),
303 0x00000000,
304 (0x5e00 << 16) | (0xc904 >> 2),
305 0x00000000,
306 (0x6e00 << 16) | (0xc904 >> 2),
307 0x00000000,
308 (0x7e00 << 16) | (0xc904 >> 2),
309 0x00000000,
310 (0x8e00 << 16) | (0xc904 >> 2),
311 0x00000000,
312 (0x9e00 << 16) | (0xc904 >> 2),
313 0x00000000,
314 (0xae00 << 16) | (0xc904 >> 2),
315 0x00000000,
316 (0xbe00 << 16) | (0xc904 >> 2),
317 0x00000000,
318 (0x4e00 << 16) | (0xc908 >> 2),
319 0x00000000,
320 (0x5e00 << 16) | (0xc908 >> 2),
321 0x00000000,
322 (0x6e00 << 16) | (0xc908 >> 2),
323 0x00000000,
324 (0x7e00 << 16) | (0xc908 >> 2),
325 0x00000000,
326 (0x8e00 << 16) | (0xc908 >> 2),
327 0x00000000,
328 (0x9e00 << 16) | (0xc908 >> 2),
329 0x00000000,
330 (0xae00 << 16) | (0xc908 >> 2),
331 0x00000000,
332 (0xbe00 << 16) | (0xc908 >> 2),
333 0x00000000,
334 (0x4e00 << 16) | (0xc90c >> 2),
335 0x00000000,
336 (0x5e00 << 16) | (0xc90c >> 2),
337 0x00000000,
338 (0x6e00 << 16) | (0xc90c >> 2),
339 0x00000000,
340 (0x7e00 << 16) | (0xc90c >> 2),
341 0x00000000,
342 (0x8e00 << 16) | (0xc90c >> 2),
343 0x00000000,
344 (0x9e00 << 16) | (0xc90c >> 2),
345 0x00000000,
346 (0xae00 << 16) | (0xc90c >> 2),
347 0x00000000,
348 (0xbe00 << 16) | (0xc90c >> 2),
349 0x00000000,
350 (0x4e00 << 16) | (0xc910 >> 2),
351 0x00000000,
352 (0x5e00 << 16) | (0xc910 >> 2),
353 0x00000000,
354 (0x6e00 << 16) | (0xc910 >> 2),
355 0x00000000,
356 (0x7e00 << 16) | (0xc910 >> 2),
357 0x00000000,
358 (0x8e00 << 16) | (0xc910 >> 2),
359 0x00000000,
360 (0x9e00 << 16) | (0xc910 >> 2),
361 0x00000000,
362 (0xae00 << 16) | (0xc910 >> 2),
363 0x00000000,
364 (0xbe00 << 16) | (0xc910 >> 2),
365 0x00000000,
366 (0x0e00 << 16) | (0xc99c >> 2),
367 0x00000000,
368 (0x0e00 << 16) | (0x9834 >> 2),
369 0x00000000,
370 (0x0000 << 16) | (0x30f00 >> 2),
371 0x00000000,
372 (0x0001 << 16) | (0x30f00 >> 2),
373 0x00000000,
374 (0x0000 << 16) | (0x30f04 >> 2),
375 0x00000000,
376 (0x0001 << 16) | (0x30f04 >> 2),
377 0x00000000,
378 (0x0000 << 16) | (0x30f08 >> 2),
379 0x00000000,
380 (0x0001 << 16) | (0x30f08 >> 2),
381 0x00000000,
382 (0x0000 << 16) | (0x30f0c >> 2),
383 0x00000000,
384 (0x0001 << 16) | (0x30f0c >> 2),
385 0x00000000,
386 (0x0600 << 16) | (0x9b7c >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0x8a14 >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0x8a18 >> 2),
391 0x00000000,
392 (0x0600 << 16) | (0x30a00 >> 2),
393 0x00000000,
394 (0x0e00 << 16) | (0x8bf0 >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0x8bcc >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x8b24 >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0x30a04 >> 2),
401 0x00000000,
402 (0x0600 << 16) | (0x30a10 >> 2),
403 0x00000000,
404 (0x0600 << 16) | (0x30a14 >> 2),
405 0x00000000,
406 (0x0600 << 16) | (0x30a18 >> 2),
407 0x00000000,
408 (0x0600 << 16) | (0x30a2c >> 2),
409 0x00000000,
410 (0x0e00 << 16) | (0xc700 >> 2),
411 0x00000000,
412 (0x0e00 << 16) | (0xc704 >> 2),
413 0x00000000,
414 (0x0e00 << 16) | (0xc708 >> 2),
415 0x00000000,
416 (0x0e00 << 16) | (0xc768 >> 2),
417 0x00000000,
418 (0x0400 << 16) | (0xc770 >> 2),
419 0x00000000,
420 (0x0400 << 16) | (0xc774 >> 2),
421 0x00000000,
422 (0x0400 << 16) | (0xc778 >> 2),
423 0x00000000,
424 (0x0400 << 16) | (0xc77c >> 2),
425 0x00000000,
426 (0x0400 << 16) | (0xc780 >> 2),
427 0x00000000,
428 (0x0400 << 16) | (0xc784 >> 2),
429 0x00000000,
430 (0x0400 << 16) | (0xc788 >> 2),
431 0x00000000,
432 (0x0400 << 16) | (0xc78c >> 2),
433 0x00000000,
434 (0x0400 << 16) | (0xc798 >> 2),
435 0x00000000,
436 (0x0400 << 16) | (0xc79c >> 2),
437 0x00000000,
438 (0x0400 << 16) | (0xc7a0 >> 2),
439 0x00000000,
440 (0x0400 << 16) | (0xc7a4 >> 2),
441 0x00000000,
442 (0x0400 << 16) | (0xc7a8 >> 2),
443 0x00000000,
444 (0x0400 << 16) | (0xc7ac >> 2),
445 0x00000000,
446 (0x0400 << 16) | (0xc7b0 >> 2),
447 0x00000000,
448 (0x0400 << 16) | (0xc7b4 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x9100 >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x3c010 >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x92a8 >> 2),
455 0x00000000,
456 (0x0e00 << 16) | (0x92ac >> 2),
457 0x00000000,
458 (0x0e00 << 16) | (0x92b4 >> 2),
459 0x00000000,
460 (0x0e00 << 16) | (0x92b8 >> 2),
461 0x00000000,
462 (0x0e00 << 16) | (0x92bc >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0x92c0 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0x92c4 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0x92c8 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0x92cc >> 2),
471 0x00000000,
472 (0x0e00 << 16) | (0x92d0 >> 2),
473 0x00000000,
474 (0x0e00 << 16) | (0x8c00 >> 2),
475 0x00000000,
476 (0x0e00 << 16) | (0x8c04 >> 2),
477 0x00000000,
478 (0x0e00 << 16) | (0x8c20 >> 2),
479 0x00000000,
480 (0x0e00 << 16) | (0x8c38 >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0x8c3c >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0xae00 >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0x9604 >> 2),
487 0x00000000,
488 (0x0e00 << 16) | (0xac08 >> 2),
489 0x00000000,
490 (0x0e00 << 16) | (0xac0c >> 2),
491 0x00000000,
492 (0x0e00 << 16) | (0xac10 >> 2),
493 0x00000000,
494 (0x0e00 << 16) | (0xac14 >> 2),
495 0x00000000,
496 (0x0e00 << 16) | (0xac58 >> 2),
497 0x00000000,
498 (0x0e00 << 16) | (0xac68 >> 2),
499 0x00000000,
500 (0x0e00 << 16) | (0xac6c >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0xac70 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0xac74 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0xac78 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0xac7c >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0xac80 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0xac84 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0xac88 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0xac8c >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x970c >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0x9714 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0x9718 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0x971c >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0x31068 >> 2),
527 0x00000000,
528 (0x4e00 << 16) | (0x31068 >> 2),
529 0x00000000,
530 (0x5e00 << 16) | (0x31068 >> 2),
531 0x00000000,
532 (0x6e00 << 16) | (0x31068 >> 2),
533 0x00000000,
534 (0x7e00 << 16) | (0x31068 >> 2),
535 0x00000000,
536 (0x8e00 << 16) | (0x31068 >> 2),
537 0x00000000,
538 (0x9e00 << 16) | (0x31068 >> 2),
539 0x00000000,
540 (0xae00 << 16) | (0x31068 >> 2),
541 0x00000000,
542 (0xbe00 << 16) | (0x31068 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0xcd10 >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0xcd14 >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0x88b0 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0x88b4 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0x88b8 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0x88bc >> 2),
555 0x00000000,
556 (0x0400 << 16) | (0x89c0 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0x88c4 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0x88c8 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0x88d0 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0x88d4 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x88d8 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x8980 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x30938 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x3093c >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x30940 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x89a0 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x30900 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x30904 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x89b4 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x3c210 >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x3c214 >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x3c218 >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x8904 >> 2),
591 0x00000000,
592 0x5,
593 (0x0e00 << 16) | (0x8c28 >> 2),
594 (0x0e00 << 16) | (0x8c2c >> 2),
595 (0x0e00 << 16) | (0x8c30 >> 2),
596 (0x0e00 << 16) | (0x8c34 >> 2),
597 (0x0e00 << 16) | (0x9600 >> 2),
598};
599
600static const u32 kalindi_rlc_save_restore_register_list[] =
601{
602 (0x0e00 << 16) | (0xc12c >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xc140 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xc150 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xc15c >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xc168 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xc170 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xc204 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xc2b4 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xc2b8 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xc2bc >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xc2c0 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x8228 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x829c >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x869c >> 2),
629 0x00000000,
630 (0x0600 << 16) | (0x98f4 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x98f8 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x9900 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0xc260 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x90e8 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x3c000 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x3c00c >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x8c1c >> 2),
645 0x00000000,
646 (0x0e00 << 16) | (0x9700 >> 2),
647 0x00000000,
648 (0x0e00 << 16) | (0xcd20 >> 2),
649 0x00000000,
650 (0x4e00 << 16) | (0xcd20 >> 2),
651 0x00000000,
652 (0x5e00 << 16) | (0xcd20 >> 2),
653 0x00000000,
654 (0x6e00 << 16) | (0xcd20 >> 2),
655 0x00000000,
656 (0x7e00 << 16) | (0xcd20 >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0x89bc >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0x8900 >> 2),
661 0x00000000,
662 0x3,
663 (0x0e00 << 16) | (0xc130 >> 2),
664 0x00000000,
665 (0x0e00 << 16) | (0xc134 >> 2),
666 0x00000000,
667 (0x0e00 << 16) | (0xc1fc >> 2),
668 0x00000000,
669 (0x0e00 << 16) | (0xc208 >> 2),
670 0x00000000,
671 (0x0e00 << 16) | (0xc264 >> 2),
672 0x00000000,
673 (0x0e00 << 16) | (0xc268 >> 2),
674 0x00000000,
675 (0x0e00 << 16) | (0xc26c >> 2),
676 0x00000000,
677 (0x0e00 << 16) | (0xc270 >> 2),
678 0x00000000,
679 (0x0e00 << 16) | (0xc274 >> 2),
680 0x00000000,
681 (0x0e00 << 16) | (0xc28c >> 2),
682 0x00000000,
683 (0x0e00 << 16) | (0xc290 >> 2),
684 0x00000000,
685 (0x0e00 << 16) | (0xc294 >> 2),
686 0x00000000,
687 (0x0e00 << 16) | (0xc298 >> 2),
688 0x00000000,
689 (0x0e00 << 16) | (0xc2a0 >> 2),
690 0x00000000,
691 (0x0e00 << 16) | (0xc2a4 >> 2),
692 0x00000000,
693 (0x0e00 << 16) | (0xc2a8 >> 2),
694 0x00000000,
695 (0x0e00 << 16) | (0xc2ac >> 2),
696 0x00000000,
697 (0x0e00 << 16) | (0x301d0 >> 2),
698 0x00000000,
699 (0x0e00 << 16) | (0x30238 >> 2),
700 0x00000000,
701 (0x0e00 << 16) | (0x30250 >> 2),
702 0x00000000,
703 (0x0e00 << 16) | (0x30254 >> 2),
704 0x00000000,
705 (0x0e00 << 16) | (0x30258 >> 2),
706 0x00000000,
707 (0x0e00 << 16) | (0x3025c >> 2),
708 0x00000000,
709 (0x4e00 << 16) | (0xc900 >> 2),
710 0x00000000,
711 (0x5e00 << 16) | (0xc900 >> 2),
712 0x00000000,
713 (0x6e00 << 16) | (0xc900 >> 2),
714 0x00000000,
715 (0x7e00 << 16) | (0xc900 >> 2),
716 0x00000000,
717 (0x4e00 << 16) | (0xc904 >> 2),
718 0x00000000,
719 (0x5e00 << 16) | (0xc904 >> 2),
720 0x00000000,
721 (0x6e00 << 16) | (0xc904 >> 2),
722 0x00000000,
723 (0x7e00 << 16) | (0xc904 >> 2),
724 0x00000000,
725 (0x4e00 << 16) | (0xc908 >> 2),
726 0x00000000,
727 (0x5e00 << 16) | (0xc908 >> 2),
728 0x00000000,
729 (0x6e00 << 16) | (0xc908 >> 2),
730 0x00000000,
731 (0x7e00 << 16) | (0xc908 >> 2),
732 0x00000000,
733 (0x4e00 << 16) | (0xc90c >> 2),
734 0x00000000,
735 (0x5e00 << 16) | (0xc90c >> 2),
736 0x00000000,
737 (0x6e00 << 16) | (0xc90c >> 2),
738 0x00000000,
739 (0x7e00 << 16) | (0xc90c >> 2),
740 0x00000000,
741 (0x4e00 << 16) | (0xc910 >> 2),
742 0x00000000,
743 (0x5e00 << 16) | (0xc910 >> 2),
744 0x00000000,
745 (0x6e00 << 16) | (0xc910 >> 2),
746 0x00000000,
747 (0x7e00 << 16) | (0xc910 >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0xc99c >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0x9834 >> 2),
752 0x00000000,
753 (0x0000 << 16) | (0x30f00 >> 2),
754 0x00000000,
755 (0x0000 << 16) | (0x30f04 >> 2),
756 0x00000000,
757 (0x0000 << 16) | (0x30f08 >> 2),
758 0x00000000,
759 (0x0000 << 16) | (0x30f0c >> 2),
760 0x00000000,
761 (0x0600 << 16) | (0x9b7c >> 2),
762 0x00000000,
763 (0x0e00 << 16) | (0x8a14 >> 2),
764 0x00000000,
765 (0x0e00 << 16) | (0x8a18 >> 2),
766 0x00000000,
767 (0x0600 << 16) | (0x30a00 >> 2),
768 0x00000000,
769 (0x0e00 << 16) | (0x8bf0 >> 2),
770 0x00000000,
771 (0x0e00 << 16) | (0x8bcc >> 2),
772 0x00000000,
773 (0x0e00 << 16) | (0x8b24 >> 2),
774 0x00000000,
775 (0x0e00 << 16) | (0x30a04 >> 2),
776 0x00000000,
777 (0x0600 << 16) | (0x30a10 >> 2),
778 0x00000000,
779 (0x0600 << 16) | (0x30a14 >> 2),
780 0x00000000,
781 (0x0600 << 16) | (0x30a18 >> 2),
782 0x00000000,
783 (0x0600 << 16) | (0x30a2c >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc700 >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0xc704 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0xc708 >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0xc768 >> 2),
792 0x00000000,
793 (0x0400 << 16) | (0xc770 >> 2),
794 0x00000000,
795 (0x0400 << 16) | (0xc774 >> 2),
796 0x00000000,
797 (0x0400 << 16) | (0xc798 >> 2),
798 0x00000000,
799 (0x0400 << 16) | (0xc79c >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0x9100 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0x3c010 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0x8c00 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0x8c04 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0x8c20 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0x8c38 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0x8c3c >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0xae00 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x9604 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0xac08 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0xac0c >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0xac10 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0xac14 >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0xac58 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0xac68 >> 2),
830 0x00000000,
831 (0x0e00 << 16) | (0xac6c >> 2),
832 0x00000000,
833 (0x0e00 << 16) | (0xac70 >> 2),
834 0x00000000,
835 (0x0e00 << 16) | (0xac74 >> 2),
836 0x00000000,
837 (0x0e00 << 16) | (0xac78 >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0xac7c >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0xac80 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0xac84 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0xac88 >> 2),
846 0x00000000,
847 (0x0e00 << 16) | (0xac8c >> 2),
848 0x00000000,
849 (0x0e00 << 16) | (0x970c >> 2),
850 0x00000000,
851 (0x0e00 << 16) | (0x9714 >> 2),
852 0x00000000,
853 (0x0e00 << 16) | (0x9718 >> 2),
854 0x00000000,
855 (0x0e00 << 16) | (0x971c >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0x31068 >> 2),
858 0x00000000,
859 (0x4e00 << 16) | (0x31068 >> 2),
860 0x00000000,
861 (0x5e00 << 16) | (0x31068 >> 2),
862 0x00000000,
863 (0x6e00 << 16) | (0x31068 >> 2),
864 0x00000000,
865 (0x7e00 << 16) | (0x31068 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0xcd10 >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0xcd14 >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0x88b0 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0x88b4 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0x88b8 >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0x88bc >> 2),
878 0x00000000,
879 (0x0400 << 16) | (0x89c0 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x88c4 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x88c8 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x88d0 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x88d4 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x88d8 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x8980 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x30938 >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0x3093c >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0x30940 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0x89a0 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0x30900 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x30904 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x89b4 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x3e1fc >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x3c210 >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x3c214 >> 2),
912 0x00000000,
913 (0x0e00 << 16) | (0x3c218 >> 2),
914 0x00000000,
915 (0x0e00 << 16) | (0x8904 >> 2),
916 0x00000000,
917 0x5,
918 (0x0e00 << 16) | (0x8c28 >> 2),
919 (0x0e00 << 16) | (0x8c2c >> 2),
920 (0x0e00 << 16) | (0x8c30 >> 2),
921 (0x0e00 << 16) | (0x8c34 >> 2),
922 (0x0e00 << 16) | (0x9600 >> 2),
923};
924
0aafd313
AD		 925static const u32 bonaire_golden_spm_registers[] =
926{
927 0x30800, 0xe0ffffff, 0xe0000000
928};
929
930static const u32 bonaire_golden_common_registers[] =
931{
932 0xc770, 0xffffffff, 0x00000800,
933 0xc774, 0xffffffff, 0x00000800,
934 0xc798, 0xffffffff, 0x00007fbf,
935 0xc79c, 0xffffffff, 0x00007faf
936};
937
938static const u32 bonaire_golden_registers[] =
939{
940 0x3354, 0x00000333, 0x00000333,
941 0x3350, 0x000c0fc0, 0x00040200,
942 0x9a10, 0x00010000, 0x00058208,
943 0x3c000, 0xffff1fff, 0x00140000,
944 0x3c200, 0xfdfc0fff, 0x00000100,
945 0x3c234, 0x40000000, 0x40000200,
946 0x9830, 0xffffffff, 0x00000000,
947 0x9834, 0xf00fffff, 0x00000400,
948 0x9838, 0x0002021c, 0x00020200,
949 0xc78, 0x00000080, 0x00000000,
950 0x5bb0, 0x000000f0, 0x00000070,
951 0x5bc0, 0xf0311fff, 0x80300000,
952 0x98f8, 0x73773777, 0x12010001,
953 0x350c, 0x00810000, 0x408af000,
954 0x7030, 0x31000111, 0x00000011,
955 0x2f48, 0x73773777, 0x12010001,
956 0x220c, 0x00007fb6, 0x0021a1b1,
957 0x2210, 0x00007fb6, 0x002021b1,
958 0x2180, 0x00007fb6, 0x00002191,
959 0x2218, 0x00007fb6, 0x002121b1,
960 0x221c, 0x00007fb6, 0x002021b1,
961 0x21dc, 0x00007fb6, 0x00002191,
962 0x21e0, 0x00007fb6, 0x00002191,
963 0x3628, 0x0000003f, 0x0000000a,
964 0x362c, 0x0000003f, 0x0000000a,
965 0x2ae4, 0x00073ffe, 0x000022a2,
966 0x240c, 0x000007ff, 0x00000000,
967 0x8a14, 0xf000003f, 0x00000007,
968 0x8bf0, 0x00002001, 0x00000001,
969 0x8b24, 0xffffffff, 0x00ffffff,
970 0x30a04, 0x0000ff0f, 0x00000000,
971 0x28a4c, 0x07ffffff, 0x06000000,
972 0x4d8, 0x00000fff, 0x00000100,
973 0x3e78, 0x00000001, 0x00000002,
974 0x9100, 0x03000000, 0x0362c688,
975 0x8c00, 0x000000ff, 0x00000001,
976 0xe40, 0x00001fff, 0x00001fff,
977 0x9060, 0x0000007f, 0x00000020,
978 0x9508, 0x00010000, 0x00010000,
979 0xac14, 0x000003ff, 0x000000f3,
980 0xac0c, 0xffffffff, 0x00001032
981};
982
983static const u32 bonaire_mgcg_cgcg_init[] =
984{
985 0xc420, 0xffffffff, 0xfffffffc,
986 0x30800, 0xffffffff, 0xe0000000,
987 0x3c2a0, 0xffffffff, 0x00000100,
988 0x3c208, 0xffffffff, 0x00000100,
989 0x3c2c0, 0xffffffff, 0xc0000100,
990 0x3c2c8, 0xffffffff, 0xc0000100,
991 0x3c2c4, 0xffffffff, 0xc0000100,
992 0x55e4, 0xffffffff, 0x00600100,
993 0x3c280, 0xffffffff, 0x00000100,
994 0x3c214, 0xffffffff, 0x06000100,
995 0x3c220, 0xffffffff, 0x00000100,
996 0x3c218, 0xffffffff, 0x06000100,
997 0x3c204, 0xffffffff, 0x00000100,
998 0x3c2e0, 0xffffffff, 0x00000100,
999 0x3c224, 0xffffffff, 0x00000100,
1000 0x3c200, 0xffffffff, 0x00000100,
1001 0x3c230, 0xffffffff, 0x00000100,
1002 0x3c234, 0xffffffff, 0x00000100,
1003 0x3c250, 0xffffffff, 0x00000100,
1004 0x3c254, 0xffffffff, 0x00000100,
1005 0x3c258, 0xffffffff, 0x00000100,
1006 0x3c25c, 0xffffffff, 0x00000100,
1007 0x3c260, 0xffffffff, 0x00000100,
1008 0x3c27c, 0xffffffff, 0x00000100,
1009 0x3c278, 0xffffffff, 0x00000100,
1010 0x3c210, 0xffffffff, 0x06000100,
1011 0x3c290, 0xffffffff, 0x00000100,
1012 0x3c274, 0xffffffff, 0x00000100,
1013 0x3c2b4, 0xffffffff, 0x00000100,
1014 0x3c2b0, 0xffffffff, 0x00000100,
1015 0x3c270, 0xffffffff, 0x00000100,
1016 0x30800, 0xffffffff, 0xe0000000,
1017 0x3c020, 0xffffffff, 0x00010000,
1018 0x3c024, 0xffffffff, 0x00030002,
1019 0x3c028, 0xffffffff, 0x00040007,
1020 0x3c02c, 0xffffffff, 0x00060005,
1021 0x3c030, 0xffffffff, 0x00090008,
1022 0x3c034, 0xffffffff, 0x00010000,
1023 0x3c038, 0xffffffff, 0x00030002,
1024 0x3c03c, 0xffffffff, 0x00040007,
1025 0x3c040, 0xffffffff, 0x00060005,
1026 0x3c044, 0xffffffff, 0x00090008,
1027 0x3c048, 0xffffffff, 0x00010000,
1028 0x3c04c, 0xffffffff, 0x00030002,
1029 0x3c050, 0xffffffff, 0x00040007,
1030 0x3c054, 0xffffffff, 0x00060005,
1031 0x3c058, 0xffffffff, 0x00090008,
1032 0x3c05c, 0xffffffff, 0x00010000,
1033 0x3c060, 0xffffffff, 0x00030002,
1034 0x3c064, 0xffffffff, 0x00040007,
1035 0x3c068, 0xffffffff, 0x00060005,
1036 0x3c06c, 0xffffffff, 0x00090008,
1037 0x3c070, 0xffffffff, 0x00010000,
1038 0x3c074, 0xffffffff, 0x00030002,
1039 0x3c078, 0xffffffff, 0x00040007,
1040 0x3c07c, 0xffffffff, 0x00060005,
1041 0x3c080, 0xffffffff, 0x00090008,
1042 0x3c084, 0xffffffff, 0x00010000,
1043 0x3c088, 0xffffffff, 0x00030002,
1044 0x3c08c, 0xffffffff, 0x00040007,
1045 0x3c090, 0xffffffff, 0x00060005,
1046 0x3c094, 0xffffffff, 0x00090008,
1047 0x3c098, 0xffffffff, 0x00010000,
1048 0x3c09c, 0xffffffff, 0x00030002,
1049 0x3c0a0, 0xffffffff, 0x00040007,
1050 0x3c0a4, 0xffffffff, 0x00060005,
1051 0x3c0a8, 0xffffffff, 0x00090008,
1052 0x3c000, 0xffffffff, 0x96e00200,
1053 0x8708, 0xffffffff, 0x00900100,
1054 0xc424, 0xffffffff, 0x0020003f,
1055 0x38, 0xffffffff, 0x0140001c,
1056 0x3c, 0x000f0000, 0x000f0000,
1057 0x220, 0xffffffff, 0xC060000C,
1058 0x224, 0xc0000fff, 0x00000100,
1059 0xf90, 0xffffffff, 0x00000100,
1060 0xf98, 0x00000101, 0x00000000,
1061 0x20a8, 0xffffffff, 0x00000104,
1062 0x55e4, 0xff000fff, 0x00000100,
1063 0x30cc, 0xc0000fff, 0x00000104,
1064 0xc1e4, 0x00000001, 0x00000001,
1065 0xd00c, 0xff000ff0, 0x00000100,
1066 0xd80c, 0xff000ff0, 0x00000100
1067};
1068
1069static const u32 spectre_golden_spm_registers[] =
1070{
1071 0x30800, 0xe0ffffff, 0xe0000000
1072};
1073
1074static const u32 spectre_golden_common_registers[] =
1075{
1076 0xc770, 0xffffffff, 0x00000800,
1077 0xc774, 0xffffffff, 0x00000800,
1078 0xc798, 0xffffffff, 0x00007fbf,
1079 0xc79c, 0xffffffff, 0x00007faf
1080};
1081
1082static const u32 spectre_golden_registers[] =
1083{
1084 0x3c000, 0xffff1fff, 0x96940200,
1085 0x3c00c, 0xffff0001, 0xff000000,
1086 0x3c200, 0xfffc0fff, 0x00000100,
1087 0x6ed8, 0x00010101, 0x00010000,
1088 0x9834, 0xf00fffff, 0x00000400,
1089 0x9838, 0xfffffffc, 0x00020200,
1090 0x5bb0, 0x000000f0, 0x00000070,
1091 0x5bc0, 0xf0311fff, 0x80300000,
1092 0x98f8, 0x73773777, 0x12010001,
1093 0x9b7c, 0x00ff0000, 0x00fc0000,
1094 0x2f48, 0x73773777, 0x12010001,
1095 0x8a14, 0xf000003f, 0x00000007,
1096 0x8b24, 0xffffffff, 0x00ffffff,
1097 0x28350, 0x3f3f3fff, 0x00000082,
1098 0x28355, 0x0000003f, 0x00000000,
1099 0x3e78, 0x00000001, 0x00000002,
1100 0x913c, 0xffff03df, 0x00000004,
1101 0xc768, 0x00000008, 0x00000008,
1102 0x8c00, 0x000008ff, 0x00000800,
1103 0x9508, 0x00010000, 0x00010000,
1104 0xac0c, 0xffffffff, 0x54763210,
1105 0x214f8, 0x01ff01ff, 0x00000002,
1106 0x21498, 0x007ff800, 0x00200000,
1107 0x2015c, 0xffffffff, 0x00000f40,
1108 0x30934, 0xffffffff, 0x00000001
1109};
1110
1111static const u32 spectre_mgcg_cgcg_init[] =
1112{
1113 0xc420, 0xffffffff, 0xfffffffc,
1114 0x30800, 0xffffffff, 0xe0000000,
1115 0x3c2a0, 0xffffffff, 0x00000100,
1116 0x3c208, 0xffffffff, 0x00000100,
1117 0x3c2c0, 0xffffffff, 0x00000100,
1118 0x3c2c8, 0xffffffff, 0x00000100,
1119 0x3c2c4, 0xffffffff, 0x00000100,
1120 0x55e4, 0xffffffff, 0x00600100,
1121 0x3c280, 0xffffffff, 0x00000100,
1122 0x3c214, 0xffffffff, 0x06000100,
1123 0x3c220, 0xffffffff, 0x00000100,
1124 0x3c218, 0xffffffff, 0x06000100,
1125 0x3c204, 0xffffffff, 0x00000100,
1126 0x3c2e0, 0xffffffff, 0x00000100,
1127 0x3c224, 0xffffffff, 0x00000100,
1128 0x3c200, 0xffffffff, 0x00000100,
1129 0x3c230, 0xffffffff, 0x00000100,
1130 0x3c234, 0xffffffff, 0x00000100,
1131 0x3c250, 0xffffffff, 0x00000100,
1132 0x3c254, 0xffffffff, 0x00000100,
1133 0x3c258, 0xffffffff, 0x00000100,
1134 0x3c25c, 0xffffffff, 0x00000100,
1135 0x3c260, 0xffffffff, 0x00000100,
1136 0x3c27c, 0xffffffff, 0x00000100,
1137 0x3c278, 0xffffffff, 0x00000100,
1138 0x3c210, 0xffffffff, 0x06000100,
1139 0x3c290, 0xffffffff, 0x00000100,
1140 0x3c274, 0xffffffff, 0x00000100,
1141 0x3c2b4, 0xffffffff, 0x00000100,
1142 0x3c2b0, 0xffffffff, 0x00000100,
1143 0x3c270, 0xffffffff, 0x00000100,
1144 0x30800, 0xffffffff, 0xe0000000,
1145 0x3c020, 0xffffffff, 0x00010000,
1146 0x3c024, 0xffffffff, 0x00030002,
1147 0x3c028, 0xffffffff, 0x00040007,
1148 0x3c02c, 0xffffffff, 0x00060005,
1149 0x3c030, 0xffffffff, 0x00090008,
1150 0x3c034, 0xffffffff, 0x00010000,
1151 0x3c038, 0xffffffff, 0x00030002,
1152 0x3c03c, 0xffffffff, 0x00040007,
1153 0x3c040, 0xffffffff, 0x00060005,
1154 0x3c044, 0xffffffff, 0x00090008,
1155 0x3c048, 0xffffffff, 0x00010000,
1156 0x3c04c, 0xffffffff, 0x00030002,
1157 0x3c050, 0xffffffff, 0x00040007,
1158 0x3c054, 0xffffffff, 0x00060005,
1159 0x3c058, 0xffffffff, 0x00090008,
1160 0x3c05c, 0xffffffff, 0x00010000,
1161 0x3c060, 0xffffffff, 0x00030002,
1162 0x3c064, 0xffffffff, 0x00040007,
1163 0x3c068, 0xffffffff, 0x00060005,
1164 0x3c06c, 0xffffffff, 0x00090008,
1165 0x3c070, 0xffffffff, 0x00010000,
1166 0x3c074, 0xffffffff, 0x00030002,
1167 0x3c078, 0xffffffff, 0x00040007,
1168 0x3c07c, 0xffffffff, 0x00060005,
1169 0x3c080, 0xffffffff, 0x00090008,
1170 0x3c084, 0xffffffff, 0x00010000,
1171 0x3c088, 0xffffffff, 0x00030002,
1172 0x3c08c, 0xffffffff, 0x00040007,
1173 0x3c090, 0xffffffff, 0x00060005,
1174 0x3c094, 0xffffffff, 0x00090008,
1175 0x3c098, 0xffffffff, 0x00010000,
1176 0x3c09c, 0xffffffff, 0x00030002,
1177 0x3c0a0, 0xffffffff, 0x00040007,
1178 0x3c0a4, 0xffffffff, 0x00060005,
1179 0x3c0a8, 0xffffffff, 0x00090008,
1180 0x3c0ac, 0xffffffff, 0x00010000,
1181 0x3c0b0, 0xffffffff, 0x00030002,
1182 0x3c0b4, 0xffffffff, 0x00040007,
1183 0x3c0b8, 0xffffffff, 0x00060005,
1184 0x3c0bc, 0xffffffff, 0x00090008,
1185 0x3c000, 0xffffffff, 0x96e00200,
1186 0x8708, 0xffffffff, 0x00900100,
1187 0xc424, 0xffffffff, 0x0020003f,
1188 0x38, 0xffffffff, 0x0140001c,
1189 0x3c, 0x000f0000, 0x000f0000,
1190 0x220, 0xffffffff, 0xC060000C,
1191 0x224, 0xc0000fff, 0x00000100,
1192 0xf90, 0xffffffff, 0x00000100,
1193 0xf98, 0x00000101, 0x00000000,
1194 0x20a8, 0xffffffff, 0x00000104,
1195 0x55e4, 0xff000fff, 0x00000100,
1196 0x30cc, 0xc0000fff, 0x00000104,
1197 0xc1e4, 0x00000001, 0x00000001,
1198 0xd00c, 0xff000ff0, 0x00000100,
1199 0xd80c, 0xff000ff0, 0x00000100
1200};
1201
1202static const u32 kalindi_golden_spm_registers[] =
1203{
1204 0x30800, 0xe0ffffff, 0xe0000000
1205};
1206
1207static const u32 kalindi_golden_common_registers[] =
1208{
1209 0xc770, 0xffffffff, 0x00000800,
1210 0xc774, 0xffffffff, 0x00000800,
1211 0xc798, 0xffffffff, 0x00007fbf,
1212 0xc79c, 0xffffffff, 0x00007faf
1213};
1214
1215static const u32 kalindi_golden_registers[] =
1216{
1217 0x3c000, 0xffffdfff, 0x6e944040,
1218 0x55e4, 0xff607fff, 0xfc000100,
1219 0x3c220, 0xff000fff, 0x00000100,
1220 0x3c224, 0xff000fff, 0x00000100,
1221 0x3c200, 0xfffc0fff, 0x00000100,
1222 0x6ed8, 0x00010101, 0x00010000,
1223 0x9830, 0xffffffff, 0x00000000,
1224 0x9834, 0xf00fffff, 0x00000400,
1225 0x5bb0, 0x000000f0, 0x00000070,
1226 0x5bc0, 0xf0311fff, 0x80300000,
1227 0x98f8, 0x73773777, 0x12010001,
1228 0x98fc, 0xffffffff, 0x00000010,
1229 0x9b7c, 0x00ff0000, 0x00fc0000,
1230 0x8030, 0x00001f0f, 0x0000100a,
1231 0x2f48, 0x73773777, 0x12010001,
1232 0x2408, 0x000fffff, 0x000c007f,
1233 0x8a14, 0xf000003f, 0x00000007,
1234 0x8b24, 0x3fff3fff, 0x00ffcfff,
1235 0x30a04, 0x0000ff0f, 0x00000000,
1236 0x28a4c, 0x07ffffff, 0x06000000,
1237 0x4d8, 0x00000fff, 0x00000100,
1238 0x3e78, 0x00000001, 0x00000002,
1239 0xc768, 0x00000008, 0x00000008,
1240 0x8c00, 0x000000ff, 0x00000003,
1241 0x214f8, 0x01ff01ff, 0x00000002,
1242 0x21498, 0x007ff800, 0x00200000,
1243 0x2015c, 0xffffffff, 0x00000f40,
1244 0x88c4, 0x001f3ae3, 0x00000082,
1245 0x88d4, 0x0000001f, 0x00000010,
1246 0x30934, 0xffffffff, 0x00000000
1247};
1248
1249static const u32 kalindi_mgcg_cgcg_init[] =
1250{
1251 0xc420, 0xffffffff, 0xfffffffc,
1252 0x30800, 0xffffffff, 0xe0000000,
1253 0x3c2a0, 0xffffffff, 0x00000100,
1254 0x3c208, 0xffffffff, 0x00000100,
1255 0x3c2c0, 0xffffffff, 0x00000100,
1256 0x3c2c8, 0xffffffff, 0x00000100,
1257 0x3c2c4, 0xffffffff, 0x00000100,
1258 0x55e4, 0xffffffff, 0x00600100,
1259 0x3c280, 0xffffffff, 0x00000100,
1260 0x3c214, 0xffffffff, 0x06000100,
1261 0x3c220, 0xffffffff, 0x00000100,
1262 0x3c218, 0xffffffff, 0x06000100,
1263 0x3c204, 0xffffffff, 0x00000100,
1264 0x3c2e0, 0xffffffff, 0x00000100,
1265 0x3c224, 0xffffffff, 0x00000100,
1266 0x3c200, 0xffffffff, 0x00000100,
1267 0x3c230, 0xffffffff, 0x00000100,
1268 0x3c234, 0xffffffff, 0x00000100,
1269 0x3c250, 0xffffffff, 0x00000100,
1270 0x3c254, 0xffffffff, 0x00000100,
1271 0x3c258, 0xffffffff, 0x00000100,
1272 0x3c25c, 0xffffffff, 0x00000100,
1273 0x3c260, 0xffffffff, 0x00000100,
1274 0x3c27c, 0xffffffff, 0x00000100,
1275 0x3c278, 0xffffffff, 0x00000100,
1276 0x3c210, 0xffffffff, 0x06000100,
1277 0x3c290, 0xffffffff, 0x00000100,
1278 0x3c274, 0xffffffff, 0x00000100,
1279 0x3c2b4, 0xffffffff, 0x00000100,
1280 0x3c2b0, 0xffffffff, 0x00000100,
1281 0x3c270, 0xffffffff, 0x00000100,
1282 0x30800, 0xffffffff, 0xe0000000,
1283 0x3c020, 0xffffffff, 0x00010000,
1284 0x3c024, 0xffffffff, 0x00030002,
1285 0x3c028, 0xffffffff, 0x00040007,
1286 0x3c02c, 0xffffffff, 0x00060005,
1287 0x3c030, 0xffffffff, 0x00090008,
1288 0x3c034, 0xffffffff, 0x00010000,
1289 0x3c038, 0xffffffff, 0x00030002,
1290 0x3c03c, 0xffffffff, 0x00040007,
1291 0x3c040, 0xffffffff, 0x00060005,
1292 0x3c044, 0xffffffff, 0x00090008,
1293 0x3c000, 0xffffffff, 0x96e00200,
1294 0x8708, 0xffffffff, 0x00900100,
1295 0xc424, 0xffffffff, 0x0020003f,
1296 0x38, 0xffffffff, 0x0140001c,
1297 0x3c, 0x000f0000, 0x000f0000,
1298 0x220, 0xffffffff, 0xC060000C,
1299 0x224, 0xc0000fff, 0x00000100,
1300 0x20a8, 0xffffffff, 0x00000104,
1301 0x55e4, 0xff000fff, 0x00000100,
1302 0x30cc, 0xc0000fff, 0x00000104,
1303 0xc1e4, 0x00000001, 0x00000001,
1304 0xd00c, 0xff000ff0, 0x00000100,
1305 0xd80c, 0xff000ff0, 0x00000100
1306};
1307
8efff337
AD		 1308static const u32 hawaii_golden_spm_registers[] =
1309{
1310 0x30800, 0xe0ffffff, 0xe0000000
1311};
1312
1313static const u32 hawaii_golden_common_registers[] =
1314{
1315 0x30800, 0xffffffff, 0xe0000000,
1316 0x28350, 0xffffffff, 0x3a00161a,
1317 0x28354, 0xffffffff, 0x0000002e,
1318 0x9a10, 0xffffffff, 0x00018208,
1319 0x98f8, 0xffffffff, 0x12011003
1320};
1321
1322static const u32 hawaii_golden_registers[] =
1323{
1324 0x3354, 0x00000333, 0x00000333,
1325 0x9a10, 0x00010000, 0x00058208,
1326 0x9830, 0xffffffff, 0x00000000,
1327 0x9834, 0xf00fffff, 0x00000400,
1328 0x9838, 0x0002021c, 0x00020200,
1329 0xc78, 0x00000080, 0x00000000,
1330 0x5bb0, 0x000000f0, 0x00000070,
1331 0x5bc0, 0xf0311fff, 0x80300000,
1332 0x350c, 0x00810000, 0x408af000,
1333 0x7030, 0x31000111, 0x00000011,
1334 0x2f48, 0x73773777, 0x12010001,
1335 0x2120, 0x0000007f, 0x0000001b,
1336 0x21dc, 0x00007fb6, 0x00002191,
1337 0x3628, 0x0000003f, 0x0000000a,
1338 0x362c, 0x0000003f, 0x0000000a,
1339 0x2ae4, 0x00073ffe, 0x000022a2,
1340 0x240c, 0x000007ff, 0x00000000,
1341 0x8bf0, 0x00002001, 0x00000001,
1342 0x8b24, 0xffffffff, 0x00ffffff,
1343 0x30a04, 0x0000ff0f, 0x00000000,
1344 0x28a4c, 0x07ffffff, 0x06000000,
1345 0x3e78, 0x00000001, 0x00000002,
1346 0xc768, 0x00000008, 0x00000008,
1347 0xc770, 0x00000f00, 0x00000800,
1348 0xc774, 0x00000f00, 0x00000800,
1349 0xc798, 0x00ffffff, 0x00ff7fbf,
1350 0xc79c, 0x00ffffff, 0x00ff7faf,
1351 0x8c00, 0x000000ff, 0x00000800,
1352 0xe40, 0x00001fff, 0x00001fff,
1353 0x9060, 0x0000007f, 0x00000020,
1354 0x9508, 0x00010000, 0x00010000,
1355 0xae00, 0x00100000, 0x000ff07c,
1356 0xac14, 0x000003ff, 0x0000000f,
1357 0xac10, 0xffffffff, 0x7564fdec,
1358 0xac0c, 0xffffffff, 0x3120b9a8,
1359 0xac08, 0x20000000, 0x0f9c0000
1360};
1361
1362static const u32 hawaii_mgcg_cgcg_init[] =
1363{
1364 0xc420, 0xffffffff, 0xfffffffd,
1365 0x30800, 0xffffffff, 0xe0000000,
1366 0x3c2a0, 0xffffffff, 0x00000100,
1367 0x3c208, 0xffffffff, 0x00000100,
1368 0x3c2c0, 0xffffffff, 0x00000100,
1369 0x3c2c8, 0xffffffff, 0x00000100,
1370 0x3c2c4, 0xffffffff, 0x00000100,
1371 0x55e4, 0xffffffff, 0x00200100,
1372 0x3c280, 0xffffffff, 0x00000100,
1373 0x3c214, 0xffffffff, 0x06000100,
1374 0x3c220, 0xffffffff, 0x00000100,
1375 0x3c218, 0xffffffff, 0x06000100,
1376 0x3c204, 0xffffffff, 0x00000100,
1377 0x3c2e0, 0xffffffff, 0x00000100,
1378 0x3c224, 0xffffffff, 0x00000100,
1379 0x3c200, 0xffffffff, 0x00000100,
1380 0x3c230, 0xffffffff, 0x00000100,
1381 0x3c234, 0xffffffff, 0x00000100,
1382 0x3c250, 0xffffffff, 0x00000100,
1383 0x3c254, 0xffffffff, 0x00000100,
1384 0x3c258, 0xffffffff, 0x00000100,
1385 0x3c25c, 0xffffffff, 0x00000100,
1386 0x3c260, 0xffffffff, 0x00000100,
1387 0x3c27c, 0xffffffff, 0x00000100,
1388 0x3c278, 0xffffffff, 0x00000100,
1389 0x3c210, 0xffffffff, 0x06000100,
1390 0x3c290, 0xffffffff, 0x00000100,
1391 0x3c274, 0xffffffff, 0x00000100,
1392 0x3c2b4, 0xffffffff, 0x00000100,
1393 0x3c2b0, 0xffffffff, 0x00000100,
1394 0x3c270, 0xffffffff, 0x00000100,
1395 0x30800, 0xffffffff, 0xe0000000,
1396 0x3c020, 0xffffffff, 0x00010000,
1397 0x3c024, 0xffffffff, 0x00030002,
1398 0x3c028, 0xffffffff, 0x00040007,
1399 0x3c02c, 0xffffffff, 0x00060005,
1400 0x3c030, 0xffffffff, 0x00090008,
1401 0x3c034, 0xffffffff, 0x00010000,
1402 0x3c038, 0xffffffff, 0x00030002,
1403 0x3c03c, 0xffffffff, 0x00040007,
1404 0x3c040, 0xffffffff, 0x00060005,
1405 0x3c044, 0xffffffff, 0x00090008,
1406 0x3c048, 0xffffffff, 0x00010000,
1407 0x3c04c, 0xffffffff, 0x00030002,
1408 0x3c050, 0xffffffff, 0x00040007,
1409 0x3c054, 0xffffffff, 0x00060005,
1410 0x3c058, 0xffffffff, 0x00090008,
1411 0x3c05c, 0xffffffff, 0x00010000,
1412 0x3c060, 0xffffffff, 0x00030002,
1413 0x3c064, 0xffffffff, 0x00040007,
1414 0x3c068, 0xffffffff, 0x00060005,
1415 0x3c06c, 0xffffffff, 0x00090008,
1416 0x3c070, 0xffffffff, 0x00010000,
1417 0x3c074, 0xffffffff, 0x00030002,
1418 0x3c078, 0xffffffff, 0x00040007,
1419 0x3c07c, 0xffffffff, 0x00060005,
1420 0x3c080, 0xffffffff, 0x00090008,
1421 0x3c084, 0xffffffff, 0x00010000,
1422 0x3c088, 0xffffffff, 0x00030002,
1423 0x3c08c, 0xffffffff, 0x00040007,
1424 0x3c090, 0xffffffff, 0x00060005,
1425 0x3c094, 0xffffffff, 0x00090008,
1426 0x3c098, 0xffffffff, 0x00010000,
1427 0x3c09c, 0xffffffff, 0x00030002,
1428 0x3c0a0, 0xffffffff, 0x00040007,
1429 0x3c0a4, 0xffffffff, 0x00060005,
1430 0x3c0a8, 0xffffffff, 0x00090008,
1431 0x3c0ac, 0xffffffff, 0x00010000,
1432 0x3c0b0, 0xffffffff, 0x00030002,
1433 0x3c0b4, 0xffffffff, 0x00040007,
1434 0x3c0b8, 0xffffffff, 0x00060005,
1435 0x3c0bc, 0xffffffff, 0x00090008,
1436 0x3c0c0, 0xffffffff, 0x00010000,
1437 0x3c0c4, 0xffffffff, 0x00030002,
1438 0x3c0c8, 0xffffffff, 0x00040007,
1439 0x3c0cc, 0xffffffff, 0x00060005,
1440 0x3c0d0, 0xffffffff, 0x00090008,
1441 0x3c0d4, 0xffffffff, 0x00010000,
1442 0x3c0d8, 0xffffffff, 0x00030002,
1443 0x3c0dc, 0xffffffff, 0x00040007,
1444 0x3c0e0, 0xffffffff, 0x00060005,
1445 0x3c0e4, 0xffffffff, 0x00090008,
1446 0x3c0e8, 0xffffffff, 0x00010000,
1447 0x3c0ec, 0xffffffff, 0x00030002,
1448 0x3c0f0, 0xffffffff, 0x00040007,
1449 0x3c0f4, 0xffffffff, 0x00060005,
1450 0x3c0f8, 0xffffffff, 0x00090008,
1451 0xc318, 0xffffffff, 0x00020200,
1452 0x3350, 0xffffffff, 0x00000200,
1453 0x15c0, 0xffffffff, 0x00000400,
1454 0x55e8, 0xffffffff, 0x00000000,
1455 0x2f50, 0xffffffff, 0x00000902,
1456 0x3c000, 0xffffffff, 0x96940200,
1457 0x8708, 0xffffffff, 0x00900100,
1458 0xc424, 0xffffffff, 0x0020003f,
1459 0x38, 0xffffffff, 0x0140001c,
1460 0x3c, 0x000f0000, 0x000f0000,
1461 0x220, 0xffffffff, 0xc060000c,
1462 0x224, 0xc0000fff, 0x00000100,
1463 0xf90, 0xffffffff, 0x00000100,
1464 0xf98, 0x00000101, 0x00000000,
1465 0x20a8, 0xffffffff, 0x00000104,
1466 0x55e4, 0xff000fff, 0x00000100,
1467 0x30cc, 0xc0000fff, 0x00000104,
1468 0xc1e4, 0x00000001, 0x00000001,
1469 0xd00c, 0xff000ff0, 0x00000100,
1470 0xd80c, 0xff000ff0, 0x00000100
1471};
1472
0aafd313
AD		 1473static void cik_init_golden_registers(struct radeon_device *rdev)
1474{
1475 switch (rdev->family) {
1476 case CHIP_BONAIRE:
1477 radeon_program_register_sequence(rdev,
1478 bonaire_mgcg_cgcg_init,
1479 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1480 radeon_program_register_sequence(rdev,
1481 bonaire_golden_registers,
1482 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1483 radeon_program_register_sequence(rdev,
1484 bonaire_golden_common_registers,
1485 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1486 radeon_program_register_sequence(rdev,
1487 bonaire_golden_spm_registers,
1488 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1489 break;
1490 case CHIP_KABINI:
1491 radeon_program_register_sequence(rdev,
1492 kalindi_mgcg_cgcg_init,
1493 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1494 radeon_program_register_sequence(rdev,
1495 kalindi_golden_registers,
1496 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1497 radeon_program_register_sequence(rdev,
1498 kalindi_golden_common_registers,
1499 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1500 radeon_program_register_sequence(rdev,
1501 kalindi_golden_spm_registers,
1502 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1503 break;
1504 case CHIP_KAVERI:
1505 radeon_program_register_sequence(rdev,
1506 spectre_mgcg_cgcg_init,
1507 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1508 radeon_program_register_sequence(rdev,
1509 spectre_golden_registers,
1510 (const u32)ARRAY_SIZE(spectre_golden_registers));
1511 radeon_program_register_sequence(rdev,
1512 spectre_golden_common_registers,
1513 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1514 radeon_program_register_sequence(rdev,
1515 spectre_golden_spm_registers,
1516 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1517 break;
8efff337
AD		 1518 case CHIP_HAWAII:
1519 radeon_program_register_sequence(rdev,
1520 hawaii_mgcg_cgcg_init,
1521 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1522 radeon_program_register_sequence(rdev,
1523 hawaii_golden_registers,
1524 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1525 radeon_program_register_sequence(rdev,
1526 hawaii_golden_common_registers,
1527 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1528 radeon_program_register_sequence(rdev,
1529 hawaii_golden_spm_registers,
1530 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1531 break;
0aafd313
AD		 1532 default:
1533 break;
1534 }
1535}
1536
2c67912c
AD		 1537/**
1538 * cik_get_xclk - get the xclk
1539 *
1540 * @rdev: radeon_device pointer
1541 *
1542 * Returns the reference clock used by the gfx engine
1543 * (CIK).
1544 */
1545u32 cik_get_xclk(struct radeon_device *rdev)
1546{
1547 u32 reference_clock = rdev->clock.spll.reference_freq;
1548
1549 if (rdev->flags & RADEON_IS_IGP) {
1550 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1551 return reference_clock / 2;
1552 } else {
1553 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1554 return reference_clock / 4;
1555 }
1556 return reference_clock;
1557}
1558
75efdee1
AD		 1559/**
1560 * cik_mm_rdoorbell - read a doorbell dword
1561 *
1562 * @rdev: radeon_device pointer
1563 * @offset: byte offset into the aperture
1564 *
1565 * Returns the value in the doorbell aperture at the
1566 * requested offset (CIK).
1567 */
1568u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset)
1569{
1570 if (offset < rdev->doorbell.size) {
1571 return readl(((void __iomem *)rdev->doorbell.ptr) + offset);
1572 } else {
1573 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset);
1574 return 0;
1575 }
1576}
1577
1578/**
1579 * cik_mm_wdoorbell - write a doorbell dword
1580 *
1581 * @rdev: radeon_device pointer
1582 * @offset: byte offset into the aperture
1583 * @v: value to write
1584 *
1585 * Writes @v to the doorbell aperture at the
1586 * requested offset (CIK).
1587 */
1588void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v)
1589{
1590 if (offset < rdev->doorbell.size) {
1591 writel(v, ((void __iomem *)rdev->doorbell.ptr) + offset);
1592 } else {
1593 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset);
1594 }
1595}
1596
bc8273fe
AD		 1597#define BONAIRE_IO_MC_REGS_SIZE 36
1598
1599static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1600{
1601 {0x00000070, 0x04400000},
1602 {0x00000071, 0x80c01803},
1603 {0x00000072, 0x00004004},
1604 {0x00000073, 0x00000100},
1605 {0x00000074, 0x00ff0000},
1606 {0x00000075, 0x34000000},
1607 {0x00000076, 0x08000014},
1608 {0x00000077, 0x00cc08ec},
1609 {0x00000078, 0x00000400},
1610 {0x00000079, 0x00000000},
1611 {0x0000007a, 0x04090000},
1612 {0x0000007c, 0x00000000},
1613 {0x0000007e, 0x4408a8e8},
1614 {0x0000007f, 0x00000304},
1615 {0x00000080, 0x00000000},
1616 {0x00000082, 0x00000001},
1617 {0x00000083, 0x00000002},
1618 {0x00000084, 0xf3e4f400},
1619 {0x00000085, 0x052024e3},
1620 {0x00000087, 0x00000000},
1621 {0x00000088, 0x01000000},
1622 {0x0000008a, 0x1c0a0000},
1623 {0x0000008b, 0xff010000},
1624 {0x0000008d, 0xffffefff},
1625 {0x0000008e, 0xfff3efff},
1626 {0x0000008f, 0xfff3efbf},
1627 {0x00000092, 0xf7ffffff},
1628 {0x00000093, 0xffffff7f},
1629 {0x00000095, 0x00101101},
1630 {0x00000096, 0x00000fff},
1631 {0x00000097, 0x00116fff},
1632 {0x00000098, 0x60010000},
1633 {0x00000099, 0x10010000},
1634 {0x0000009a, 0x00006000},
1635 {0x0000009b, 0x00001000},
1636 {0x0000009f, 0x00b48000}
1637};
1638
d4775655
AD		 1639#define HAWAII_IO_MC_REGS_SIZE 22
1640
1641static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1642{
1643 {0x0000007d, 0x40000000},
1644 {0x0000007e, 0x40180304},
1645 {0x0000007f, 0x0000ff00},
1646 {0x00000081, 0x00000000},
1647 {0x00000083, 0x00000800},
1648 {0x00000086, 0x00000000},
1649 {0x00000087, 0x00000100},
1650 {0x00000088, 0x00020100},
1651 {0x00000089, 0x00000000},
1652 {0x0000008b, 0x00040000},
1653 {0x0000008c, 0x00000100},
1654 {0x0000008e, 0xff010000},
1655 {0x00000090, 0xffffefff},
1656 {0x00000091, 0xfff3efff},
1657 {0x00000092, 0xfff3efbf},
1658 {0x00000093, 0xf7ffffff},
1659 {0x00000094, 0xffffff7f},
1660 {0x00000095, 0x00000fff},
1661 {0x00000096, 0x00116fff},
1662 {0x00000097, 0x60010000},
1663 {0x00000098, 0x10010000},
1664 {0x0000009f, 0x00c79000}
1665};
1666
1667
b556b12e
AD		 1668/**
1669 * cik_srbm_select - select specific register instances
1670 *
1671 * @rdev: radeon_device pointer
1672 * @me: selected ME (micro engine)
1673 * @pipe: pipe
1674 * @queue: queue
1675 * @vmid: VMID
1676 *
1677 * Switches the currently active registers instances. Some
1678 * registers are instanced per VMID, others are instanced per
1679 * me/pipe/queue combination.
1680 */
1681static void cik_srbm_select(struct radeon_device *rdev,
1682 u32 me, u32 pipe, u32 queue, u32 vmid)
1683{
1684 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1685 MEID(me & 0x3) |
1686 VMID(vmid & 0xf) |
1687 QUEUEID(queue & 0x7));
1688 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1689}
1690
bc8273fe
AD		 1691/* ucode loading */
1692/**
1693 * ci_mc_load_microcode - load MC ucode into the hw
1694 *
1695 * @rdev: radeon_device pointer
1696 *
1697 * Load the GDDR MC ucode into the hw (CIK).
1698 * Returns 0 on success, error on failure.
1699 */
1700static int ci_mc_load_microcode(struct radeon_device *rdev)
1701{
1702 const __be32 *fw_data;
1703 u32 running, blackout = 0;
1704 u32 *io_mc_regs;
1705 int i, ucode_size, regs_size;
1706
1707 if (!rdev->mc_fw)
1708 return -EINVAL;
1709
1710 switch (rdev->family) {
1711 case CHIP_BONAIRE:
bc8273fe
AD		 1712 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1713 ucode_size = CIK_MC_UCODE_SIZE;
1714 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1715 break;
d4775655
AD		 1716 case CHIP_HAWAII:
1717 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1718 ucode_size = HAWAII_MC_UCODE_SIZE;
1719 regs_size = HAWAII_IO_MC_REGS_SIZE;
1720 break;
1721 default:
1722 return -EINVAL;
bc8273fe
AD		 1723 }
1724
1725 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1726
1727 if (running == 0) {
1728 if (running) {
1729 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1730 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1731 }
1732
1733 /* reset the engine and set to writable */
1734 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1735 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1736
1737 /* load mc io regs */
1738 for (i = 0; i < regs_size; i++) {
1739 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1740 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1741 }
1742 /* load the MC ucode */
1743 fw_data = (const __be32 *)rdev->mc_fw->data;
1744 for (i = 0; i < ucode_size; i++)
1745 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1746
1747 /* put the engine back into the active state */
1748 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1749 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1750 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1751
1752 /* wait for training to complete */
1753 for (i = 0; i < rdev->usec_timeout; i++) {
1754 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1755 break;
1756 udelay(1);
1757 }
1758 for (i = 0; i < rdev->usec_timeout; i++) {
1759 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1760 break;
1761 udelay(1);
1762 }
1763
1764 if (running)
1765 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1766 }
1767
1768 return 0;
1769}
1770
02c81327
AD		 1771/**
1772 * cik_init_microcode - load ucode images from disk
1773 *
1774 * @rdev: radeon_device pointer
1775 *
1776 * Use the firmware interface to load the ucode images into
1777 * the driver (not loaded into hw).
1778 * Returns 0 on success, error on failure.
1779 */
1780static int cik_init_microcode(struct radeon_device *rdev)
1781{
02c81327
AD		 1782 const char *chip_name;
1783 size_t pfp_req_size, me_req_size, ce_req_size,
d4775655
AD		 1784 mec_req_size, rlc_req_size, mc_req_size = 0,
1785 sdma_req_size, smc_req_size = 0;
02c81327
AD		 1786 char fw_name[30];
1787 int err;
1788
1789 DRM_DEBUG("\n");
1790
02c81327
AD		 1791 switch (rdev->family) {
1792 case CHIP_BONAIRE:
1793 chip_name = "BONAIRE";
1794 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1795 me_req_size = CIK_ME_UCODE_SIZE * 4;
1796 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1797 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1798 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1799 mc_req_size = CIK_MC_UCODE_SIZE * 4;
21a93e13 1800 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 1801 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
02c81327 1802 break;
d4775655
AD		 1803 case CHIP_HAWAII:
1804 chip_name = "HAWAII";
1805 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1806 me_req_size = CIK_ME_UCODE_SIZE * 4;
1807 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1808 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1809 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1810 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1811 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1812 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1813 break;
02c81327
AD		 1814 case CHIP_KAVERI:
1815 chip_name = "KAVERI";
1816 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1817 me_req_size = CIK_ME_UCODE_SIZE * 4;
1818 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1819 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1820 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 1821 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
02c81327
AD		 1822 break;
1823 case CHIP_KABINI:
1824 chip_name = "KABINI";
1825 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1826 me_req_size = CIK_ME_UCODE_SIZE * 4;
1827 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1828 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1829 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 1830 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
02c81327
AD		 1831 break;
1832 default: BUG();
1833 }
1834
1835 DRM_INFO("Loading %s Microcode\n", chip_name);
1836
1837 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
0a168933 1838 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
02c81327
AD		 1839 if (err)
1840 goto out;
1841 if (rdev->pfp_fw->size != pfp_req_size) {
1842 printk(KERN_ERR
1843 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1844 rdev->pfp_fw->size, fw_name);
1845 err = -EINVAL;
1846 goto out;
1847 }
1848
1849 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
0a168933 1850 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
02c81327
AD		 1851 if (err)
1852 goto out;
1853 if (rdev->me_fw->size != me_req_size) {
1854 printk(KERN_ERR
1855 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1856 rdev->me_fw->size, fw_name);
1857 err = -EINVAL;
1858 }
1859
1860 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
0a168933 1861 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
02c81327
AD		 1862 if (err)
1863 goto out;
1864 if (rdev->ce_fw->size != ce_req_size) {
1865 printk(KERN_ERR
1866 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1867 rdev->ce_fw->size, fw_name);
1868 err = -EINVAL;
1869 }
1870
1871 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
0a168933 1872 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
02c81327
AD		 1873 if (err)
1874 goto out;
1875 if (rdev->mec_fw->size != mec_req_size) {
1876 printk(KERN_ERR
1877 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1878 rdev->mec_fw->size, fw_name);
1879 err = -EINVAL;
1880 }
1881
1882 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
0a168933 1883 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
02c81327
AD		 1884 if (err)
1885 goto out;
1886 if (rdev->rlc_fw->size != rlc_req_size) {
1887 printk(KERN_ERR
1888 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1889 rdev->rlc_fw->size, fw_name);
1890 err = -EINVAL;
1891 }
1892
21a93e13 1893 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
0a168933 1894 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
21a93e13
AD		 1895 if (err)
1896 goto out;
1897 if (rdev->sdma_fw->size != sdma_req_size) {
1898 printk(KERN_ERR
1899 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1900 rdev->sdma_fw->size, fw_name);
1901 err = -EINVAL;
1902 }
1903
cc8dbbb4 1904 /* No SMC, MC ucode on APUs */
02c81327
AD		 1905 if (!(rdev->flags & RADEON_IS_IGP)) {
1906 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
0a168933 1907 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
02c81327
AD		 1908 if (err)
1909 goto out;
1910 if (rdev->mc_fw->size != mc_req_size) {
1911 printk(KERN_ERR
1912 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1913 rdev->mc_fw->size, fw_name);
1914 err = -EINVAL;
1915 }
cc8dbbb4
AD		 1916
1917 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1918 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1919 if (err) {
1920 printk(KERN_ERR
1921 "smc: error loading firmware \"%s\"\n",
1922 fw_name);
1923 release_firmware(rdev->smc_fw);
1924 rdev->smc_fw = NULL;
d30d71e8 1925 err = 0;
cc8dbbb4
AD		 1926 } else if (rdev->smc_fw->size != smc_req_size) {
1927 printk(KERN_ERR
1928 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1929 rdev->smc_fw->size, fw_name);
1930 err = -EINVAL;
1931 }
02c81327
AD		 1932 }
1933
1934out:
02c81327
AD		 1935 if (err) {
1936 if (err != -EINVAL)
1937 printk(KERN_ERR
1938 "cik_cp: Failed to load firmware \"%s\"\n",
1939 fw_name);
1940 release_firmware(rdev->pfp_fw);
1941 rdev->pfp_fw = NULL;
1942 release_firmware(rdev->me_fw);
1943 rdev->me_fw = NULL;
1944 release_firmware(rdev->ce_fw);
1945 rdev->ce_fw = NULL;
1946 release_firmware(rdev->rlc_fw);
1947 rdev->rlc_fw = NULL;
1948 release_firmware(rdev->mc_fw);
1949 rdev->mc_fw = NULL;
cc8dbbb4
AD		 1950 release_firmware(rdev->smc_fw);
1951 rdev->smc_fw = NULL;
02c81327
AD		 1952 }
1953 return err;
1954}
1955
8cc1a532
AD		 1956/*
1957 * Core functions
1958 */
1959/**
1960 * cik_tiling_mode_table_init - init the hw tiling table
1961 *
1962 * @rdev: radeon_device pointer
1963 *
1964 * Starting with SI, the tiling setup is done globally in a
1965 * set of 32 tiling modes. Rather than selecting each set of
1966 * parameters per surface as on older asics, we just select
1967 * which index in the tiling table we want to use, and the
1968 * surface uses those parameters (CIK).
1969 */
1970static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1971{
1972 const u32 num_tile_mode_states = 32;
1973 const u32 num_secondary_tile_mode_states = 16;
1974 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1975 u32 num_pipe_configs;
1976 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1977 rdev->config.cik.max_shader_engines;
1978
1979 switch (rdev->config.cik.mem_row_size_in_kb) {
1980 case 1:
1981 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1982 break;
1983 case 2:
1984 default:
1985 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1986 break;
1987 case 4:
1988 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1989 break;
1990 }
1991
1992 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1993 if (num_pipe_configs > 8)
21e438af 1994 num_pipe_configs = 16;
8cc1a532 1995
21e438af
AD		 1996 if (num_pipe_configs == 16) {
1997 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1998 switch (reg_offset) {
1999 case 0:
2000 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2001 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2002 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2003 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2004 break;
2005 case 1:
2006 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2007 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2008 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2009 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2010 break;
2011 case 2:
2012 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2013 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2014 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2015 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2016 break;
2017 case 3:
2018 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2019 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2020 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2021 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2022 break;
2023 case 4:
2024 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2025 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2026 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2027 TILE_SPLIT(split_equal_to_row_size));
2028 break;
2029 case 5:
2030 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2031 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2032 break;
2033 case 6:
2034 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2035 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2036 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2037 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2038 break;
2039 case 7:
2040 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2041 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2042 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2043 TILE_SPLIT(split_equal_to_row_size));
2044 break;
2045 case 8:
2046 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2047 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2048 break;
2049 case 9:
2050 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2051 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2052 break;
2053 case 10:
2054 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2055 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2056 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2057 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2058 break;
2059 case 11:
2060 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2061 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2062 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2063 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2064 break;
2065 case 12:
2066 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2067 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2068 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2069 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2070 break;
2071 case 13:
2072 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2073 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2074 break;
2075 case 14:
2076 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2077 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2078 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2079 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2080 break;
2081 case 16:
2082 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2083 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2084 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2085 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2086 break;
2087 case 17:
2088 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2089 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2090 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2091 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2092 break;
2093 case 27:
2094 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2095 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2096 break;
2097 case 28:
2098 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2099 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2100 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2101 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2102 break;
2103 case 29:
2104 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2105 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2106 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2107 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2108 break;
2109 case 30:
2110 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2111 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2112 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2113 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2114 break;
2115 default:
2116 gb_tile_moden = 0;
2117 break;
2118 }
2119 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2120 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2121 }
2122 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2123 switch (reg_offset) {
2124 case 0:
2125 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2126 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2127 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2128 NUM_BANKS(ADDR_SURF_16_BANK));
2129 break;
2130 case 1:
2131 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2132 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2133 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2134 NUM_BANKS(ADDR_SURF_16_BANK));
2135 break;
2136 case 2:
2137 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2138 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2139 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2140 NUM_BANKS(ADDR_SURF_16_BANK));
2141 break;
2142 case 3:
2143 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2144 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2145 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2146 NUM_BANKS(ADDR_SURF_16_BANK));
2147 break;
2148 case 4:
2149 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2150 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2151 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2152 NUM_BANKS(ADDR_SURF_8_BANK));
2153 break;
2154 case 5:
2155 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2156 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2157 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2158 NUM_BANKS(ADDR_SURF_4_BANK));
2159 break;
2160 case 6:
2161 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2162 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2163 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2164 NUM_BANKS(ADDR_SURF_2_BANK));
2165 break;
2166 case 8:
2167 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2168 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2169 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2170 NUM_BANKS(ADDR_SURF_16_BANK));
2171 break;
2172 case 9:
2173 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2174 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2175 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2176 NUM_BANKS(ADDR_SURF_16_BANK));
2177 break;
2178 case 10:
2179 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2180 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2181 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2182 NUM_BANKS(ADDR_SURF_16_BANK));
2183 break;
2184 case 11:
2185 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2186 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2187 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2188 NUM_BANKS(ADDR_SURF_8_BANK));
2189 break;
2190 case 12:
2191 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2192 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2193 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2194 NUM_BANKS(ADDR_SURF_4_BANK));
2195 break;
2196 case 13:
2197 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2198 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2199 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2200 NUM_BANKS(ADDR_SURF_2_BANK));
2201 break;
2202 case 14:
2203 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2204 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2205 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2206 NUM_BANKS(ADDR_SURF_2_BANK));
2207 break;
2208 default:
2209 gb_tile_moden = 0;
2210 break;
2211 }
2212 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2213 }
2214 } else if (num_pipe_configs == 8) {
8cc1a532
AD		 2215 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2216 switch (reg_offset) {
2217 case 0:
2218 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2219 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2220 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2221 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2222 break;
2223 case 1:
2224 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2225 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2226 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2227 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2228 break;
2229 case 2:
2230 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2231 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2232 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2233 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2234 break;
2235 case 3:
2236 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2237 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2238 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2239 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2240 break;
2241 case 4:
2242 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2243 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2244 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2245 TILE_SPLIT(split_equal_to_row_size));
2246 break;
2247 case 5:
2248 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2249 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2250 break;
2251 case 6:
2252 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2253 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2254 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2255 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2256 break;
2257 case 7:
2258 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2259 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2260 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2261 TILE_SPLIT(split_equal_to_row_size));
2262 break;
2263 case 8:
2264 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2265 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2266 break;
2267 case 9:
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2269 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2270 break;
2271 case 10:
2272 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2273 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2274 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2275 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2276 break;
2277 case 11:
2278 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2279 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2280 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2281 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2282 break;
2283 case 12:
2284 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2285 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2286 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2287 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2288 break;
2289 case 13:
2290 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2291 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2292 break;
2293 case 14:
2294 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2295 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2296 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2297 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2298 break;
2299 case 16:
2300 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2301 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2302 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2303 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2304 break;
2305 case 17:
2306 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2307 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2308 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2310 break;
2311 case 27:
2312 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2313 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2314 break;
2315 case 28:
2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2317 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2318 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2319 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2320 break;
2321 case 29:
2322 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2323 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2324 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2325 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2326 break;
2327 case 30:
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2329 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2331 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2332 break;
2333 default:
2334 gb_tile_moden = 0;
2335 break;
2336 }
39aee490 2337 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2338 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2339 }
2340 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2341 switch (reg_offset) {
2342 case 0:
2343 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2346 NUM_BANKS(ADDR_SURF_16_BANK));
2347 break;
2348 case 1:
2349 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2350 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2351 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2352 NUM_BANKS(ADDR_SURF_16_BANK));
2353 break;
2354 case 2:
2355 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2357 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2358 NUM_BANKS(ADDR_SURF_16_BANK));
2359 break;
2360 case 3:
2361 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2362 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2363 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2364 NUM_BANKS(ADDR_SURF_16_BANK));
2365 break;
2366 case 4:
2367 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2368 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2369 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2370 NUM_BANKS(ADDR_SURF_8_BANK));
2371 break;
2372 case 5:
2373 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2374 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2375 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2376 NUM_BANKS(ADDR_SURF_4_BANK));
2377 break;
2378 case 6:
2379 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2380 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2381 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2382 NUM_BANKS(ADDR_SURF_2_BANK));
2383 break;
2384 case 8:
2385 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2386 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2387 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2388 NUM_BANKS(ADDR_SURF_16_BANK));
2389 break;
2390 case 9:
2391 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2392 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2393 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2394 NUM_BANKS(ADDR_SURF_16_BANK));
2395 break;
2396 case 10:
2397 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2398 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2399 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2400 NUM_BANKS(ADDR_SURF_16_BANK));
2401 break;
2402 case 11:
2403 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2404 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2405 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2406 NUM_BANKS(ADDR_SURF_16_BANK));
2407 break;
2408 case 12:
2409 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2410 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2411 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2412 NUM_BANKS(ADDR_SURF_8_BANK));
2413 break;
2414 case 13:
2415 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2416 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2417 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2418 NUM_BANKS(ADDR_SURF_4_BANK));
2419 break;
2420 case 14:
2421 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2422 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2423 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2424 NUM_BANKS(ADDR_SURF_2_BANK));
2425 break;
2426 default:
2427 gb_tile_moden = 0;
2428 break;
2429 }
2430 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2431 }
2432 } else if (num_pipe_configs == 4) {
2433 if (num_rbs == 4) {
2434 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2435 switch (reg_offset) {
2436 case 0:
2437 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2438 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2439 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2440 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2441 break;
2442 case 1:
2443 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2444 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2445 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2446 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2447 break;
2448 case 2:
2449 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2450 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2451 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2452 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2453 break;
2454 case 3:
2455 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2456 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2457 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2458 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2459 break;
2460 case 4:
2461 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2462 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2463 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2464 TILE_SPLIT(split_equal_to_row_size));
2465 break;
2466 case 5:
2467 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2468 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2469 break;
2470 case 6:
2471 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2472 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2473 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2474 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2475 break;
2476 case 7:
2477 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2478 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2479 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2480 TILE_SPLIT(split_equal_to_row_size));
2481 break;
2482 case 8:
2483 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2484 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2485 break;
2486 case 9:
2487 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2488 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2489 break;
2490 case 10:
2491 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2492 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2493 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2494 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2495 break;
2496 case 11:
2497 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2498 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2499 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2500 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2501 break;
2502 case 12:
2503 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2504 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2505 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2506 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2507 break;
2508 case 13:
2509 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2510 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2511 break;
2512 case 14:
2513 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2515 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2516 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2517 break;
2518 case 16:
2519 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2520 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2521 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2522 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2523 break;
2524 case 17:
2525 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2526 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2528 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2529 break;
2530 case 27:
2531 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2532 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2533 break;
2534 case 28:
2535 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2536 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2537 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2538 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2539 break;
2540 case 29:
2541 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2542 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2543 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2544 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2545 break;
2546 case 30:
2547 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2548 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2549 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2550 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2551 break;
2552 default:
2553 gb_tile_moden = 0;
2554 break;
2555 }
39aee490 2556 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2557 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2558 }
2559 } else if (num_rbs < 4) {
2560 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2561 switch (reg_offset) {
2562 case 0:
2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2566 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2567 break;
2568 case 1:
2569 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2570 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2571 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2572 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2573 break;
2574 case 2:
2575 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2576 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2577 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2578 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2579 break;
2580 case 3:
2581 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2583 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2584 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2585 break;
2586 case 4:
2587 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2588 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2589 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2590 TILE_SPLIT(split_equal_to_row_size));
2591 break;
2592 case 5:
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2594 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2595 break;
2596 case 6:
2597 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2598 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2599 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2600 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2601 break;
2602 case 7:
2603 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2604 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2605 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2606 TILE_SPLIT(split_equal_to_row_size));
2607 break;
2608 case 8:
2609 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2610 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2611 break;
2612 case 9:
2613 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2614 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2615 break;
2616 case 10:
2617 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2618 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2619 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2620 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2621 break;
2622 case 11:
2623 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2624 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2625 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2626 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2627 break;
2628 case 12:
2629 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2630 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2631 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2632 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2633 break;
2634 case 13:
2635 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2636 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2637 break;
2638 case 14:
2639 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2640 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2641 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2642 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2643 break;
2644 case 16:
2645 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2646 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2647 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2648 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2649 break;
2650 case 17:
2651 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2652 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2653 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2654 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2655 break;
2656 case 27:
2657 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2658 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2659 break;
2660 case 28:
2661 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2662 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2663 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2664 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2665 break;
2666 case 29:
2667 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2668 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2669 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2670 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2671 break;
2672 case 30:
2673 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2674 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2675 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2676 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2677 break;
2678 default:
2679 gb_tile_moden = 0;
2680 break;
2681 }
39aee490 2682 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2683 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2684 }
2685 }
2686 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2687 switch (reg_offset) {
2688 case 0:
2689 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2690 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2691 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2692 NUM_BANKS(ADDR_SURF_16_BANK));
2693 break;
2694 case 1:
2695 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2696 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2697 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2698 NUM_BANKS(ADDR_SURF_16_BANK));
2699 break;
2700 case 2:
2701 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2702 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2703 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2704 NUM_BANKS(ADDR_SURF_16_BANK));
2705 break;
2706 case 3:
2707 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2708 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2709 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2710 NUM_BANKS(ADDR_SURF_16_BANK));
2711 break;
2712 case 4:
2713 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2714 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2715 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2716 NUM_BANKS(ADDR_SURF_16_BANK));
2717 break;
2718 case 5:
2719 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2720 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2721 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2722 NUM_BANKS(ADDR_SURF_8_BANK));
2723 break;
2724 case 6:
2725 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2726 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2727 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2728 NUM_BANKS(ADDR_SURF_4_BANK));
2729 break;
2730 case 8:
2731 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2732 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2733 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2734 NUM_BANKS(ADDR_SURF_16_BANK));
2735 break;
2736 case 9:
2737 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2738 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2739 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2740 NUM_BANKS(ADDR_SURF_16_BANK));
2741 break;
2742 case 10:
2743 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2744 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2745 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2746 NUM_BANKS(ADDR_SURF_16_BANK));
2747 break;
2748 case 11:
2749 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2750 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2751 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2752 NUM_BANKS(ADDR_SURF_16_BANK));
2753 break;
2754 case 12:
2755 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2756 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2757 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2758 NUM_BANKS(ADDR_SURF_16_BANK));
2759 break;
2760 case 13:
2761 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2762 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2763 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2764 NUM_BANKS(ADDR_SURF_8_BANK));
2765 break;
2766 case 14:
2767 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2768 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2769 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2770 NUM_BANKS(ADDR_SURF_4_BANK));
2771 break;
2772 default:
2773 gb_tile_moden = 0;
2774 break;
2775 }
2776 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2777 }
2778 } else if (num_pipe_configs == 2) {
2779 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2780 switch (reg_offset) {
2781 case 0:
2782 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2783 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2784 PIPE_CONFIG(ADDR_SURF_P2) |
2785 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2786 break;
2787 case 1:
2788 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2789 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2790 PIPE_CONFIG(ADDR_SURF_P2) |
2791 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2792 break;
2793 case 2:
2794 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2795 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2796 PIPE_CONFIG(ADDR_SURF_P2) |
2797 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2798 break;
2799 case 3:
2800 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2801 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2802 PIPE_CONFIG(ADDR_SURF_P2) |
2803 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2804 break;
2805 case 4:
2806 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2807 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2808 PIPE_CONFIG(ADDR_SURF_P2) |
2809 TILE_SPLIT(split_equal_to_row_size));
2810 break;
2811 case 5:
2812 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2813 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2814 break;
2815 case 6:
2816 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2817 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2818 PIPE_CONFIG(ADDR_SURF_P2) |
2819 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2820 break;
2821 case 7:
2822 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2823 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2824 PIPE_CONFIG(ADDR_SURF_P2) |
2825 TILE_SPLIT(split_equal_to_row_size));
2826 break;
2827 case 8:
2828 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2829 break;
2830 case 9:
2831 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2832 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2833 break;
2834 case 10:
2835 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2836 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2837 PIPE_CONFIG(ADDR_SURF_P2) |
2838 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2839 break;
2840 case 11:
2841 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2842 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2843 PIPE_CONFIG(ADDR_SURF_P2) |
2844 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2845 break;
2846 case 12:
2847 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2848 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2849 PIPE_CONFIG(ADDR_SURF_P2) |
2850 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2851 break;
2852 case 13:
2853 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2854 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2855 break;
2856 case 14:
2857 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2858 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2859 PIPE_CONFIG(ADDR_SURF_P2) |
2860 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2861 break;
2862 case 16:
2863 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2864 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2865 PIPE_CONFIG(ADDR_SURF_P2) |
2866 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2867 break;
2868 case 17:
2869 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2870 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2871 PIPE_CONFIG(ADDR_SURF_P2) |
2872 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2873 break;
2874 case 27:
2875 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2876 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2877 break;
2878 case 28:
2879 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2880 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2881 PIPE_CONFIG(ADDR_SURF_P2) |
2882 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2883 break;
2884 case 29:
2885 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2886 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2887 PIPE_CONFIG(ADDR_SURF_P2) |
2888 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2889 break;
2890 case 30:
2891 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2892 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2893 PIPE_CONFIG(ADDR_SURF_P2) |
2894 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2895 break;
2896 default:
2897 gb_tile_moden = 0;
2898 break;
2899 }
39aee490 2900 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2901 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2902 }
2903 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2904 switch (reg_offset) {
2905 case 0:
2906 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2907 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2908 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2909 NUM_BANKS(ADDR_SURF_16_BANK));
2910 break;
2911 case 1:
2912 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2913 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2914 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2915 NUM_BANKS(ADDR_SURF_16_BANK));
2916 break;
2917 case 2:
2918 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2919 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2920 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2921 NUM_BANKS(ADDR_SURF_16_BANK));
2922 break;
2923 case 3:
2924 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2925 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2926 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2927 NUM_BANKS(ADDR_SURF_16_BANK));
2928 break;
2929 case 4:
2930 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2931 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2932 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2933 NUM_BANKS(ADDR_SURF_16_BANK));
2934 break;
2935 case 5:
2936 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2937 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2938 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2939 NUM_BANKS(ADDR_SURF_16_BANK));
2940 break;
2941 case 6:
2942 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2943 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2944 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2945 NUM_BANKS(ADDR_SURF_8_BANK));
2946 break;
2947 case 8:
2948 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2949 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2950 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2951 NUM_BANKS(ADDR_SURF_16_BANK));
2952 break;
2953 case 9:
2954 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2955 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2956 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2957 NUM_BANKS(ADDR_SURF_16_BANK));
2958 break;
2959 case 10:
2960 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2961 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2962 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2963 NUM_BANKS(ADDR_SURF_16_BANK));
2964 break;
2965 case 11:
2966 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2967 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2968 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2969 NUM_BANKS(ADDR_SURF_16_BANK));
2970 break;
2971 case 12:
2972 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2973 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2974 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2975 NUM_BANKS(ADDR_SURF_16_BANK));
2976 break;
2977 case 13:
2978 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2979 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2980 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2981 NUM_BANKS(ADDR_SURF_16_BANK));
2982 break;
2983 case 14:
2984 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2985 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2986 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2987 NUM_BANKS(ADDR_SURF_8_BANK));
2988 break;
2989 default:
2990 gb_tile_moden = 0;
2991 break;
2992 }
2993 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2994 }
2995 } else
2996 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
2997}
2998
2999/**
3000 * cik_select_se_sh - select which SE, SH to address
3001 *
3002 * @rdev: radeon_device pointer
3003 * @se_num: shader engine to address
3004 * @sh_num: sh block to address
3005 *
3006 * Select which SE, SH combinations to address. Certain
3007 * registers are instanced per SE or SH. 0xffffffff means
3008 * broadcast to all SEs or SHs (CIK).
3009 */
3010static void cik_select_se_sh(struct radeon_device *rdev,
3011 u32 se_num, u32 sh_num)
3012{
3013 u32 data = INSTANCE_BROADCAST_WRITES;
3014
3015 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 3016 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 3017 else if (se_num == 0xffffffff)
3018 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3019 else if (sh_num == 0xffffffff)
3020 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3021 else
3022 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3023 WREG32(GRBM_GFX_INDEX, data);
3024}
3025
3026/**
3027 * cik_create_bitmask - create a bitmask
3028 *
3029 * @bit_width: length of the mask
3030 *
3031 * create a variable length bit mask (CIK).
3032 * Returns the bitmask.
3033 */
3034static u32 cik_create_bitmask(u32 bit_width)
3035{
3036 u32 i, mask = 0;
3037
3038 for (i = 0; i < bit_width; i++) {
3039 mask <<= 1;
3040 mask |= 1;
3041 }
3042 return mask;
3043}
3044
3045/**
3046 * cik_select_se_sh - select which SE, SH to address
3047 *
3048 * @rdev: radeon_device pointer
3049 * @max_rb_num: max RBs (render backends) for the asic
3050 * @se_num: number of SEs (shader engines) for the asic
3051 * @sh_per_se: number of SH blocks per SE for the asic
3052 *
3053 * Calculates the bitmask of disabled RBs (CIK).
3054 * Returns the disabled RB bitmask.
3055 */
3056static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3057 u32 max_rb_num, u32 se_num,
3058 u32 sh_per_se)
3059{
3060 u32 data, mask;
3061
3062 data = RREG32(CC_RB_BACKEND_DISABLE);
3063 if (data & 1)
3064 data &= BACKEND_DISABLE_MASK;
3065 else
3066 data = 0;
3067 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3068
3069 data >>= BACKEND_DISABLE_SHIFT;
3070
3071 mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
3072
3073 return data & mask;
3074}
3075
3076/**
3077 * cik_setup_rb - setup the RBs on the asic
3078 *
3079 * @rdev: radeon_device pointer
3080 * @se_num: number of SEs (shader engines) for the asic
3081 * @sh_per_se: number of SH blocks per SE for the asic
3082 * @max_rb_num: max RBs (render backends) for the asic
3083 *
3084 * Configures per-SE/SH RB registers (CIK).
3085 */
3086static void cik_setup_rb(struct radeon_device *rdev,
3087 u32 se_num, u32 sh_per_se,
3088 u32 max_rb_num)
3089{
3090 int i, j;
3091 u32 data, mask;
3092 u32 disabled_rbs = 0;
3093 u32 enabled_rbs = 0;
3094
3095 for (i = 0; i < se_num; i++) {
3096 for (j = 0; j < sh_per_se; j++) {
3097 cik_select_se_sh(rdev, i, j);
3098 data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
fc821b70
AD		 3099 if (rdev->family == CHIP_HAWAII)
3100 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3101 else
3102 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
8cc1a532
AD		 3103 }
3104 }
3105 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3106
3107 mask = 1;
3108 for (i = 0; i < max_rb_num; i++) {
3109 if (!(disabled_rbs & mask))
3110 enabled_rbs |= mask;
3111 mask <<= 1;
3112 }
3113
3114 for (i = 0; i < se_num; i++) {
3115 cik_select_se_sh(rdev, i, 0xffffffff);
3116 data = 0;
3117 for (j = 0; j < sh_per_se; j++) {
3118 switch (enabled_rbs & 3) {
fc821b70
AD		 3119 case 0:
3120 if (j == 0)
3121 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3122 else
3123 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3124 break;
8cc1a532
AD		 3125 case 1:
3126 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3127 break;
3128 case 2:
3129 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3130 break;
3131 case 3:
3132 default:
3133 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3134 break;
3135 }
3136 enabled_rbs >>= 2;
3137 }
3138 WREG32(PA_SC_RASTER_CONFIG, data);
3139 }
3140 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3141}
3142
3143/**
3144 * cik_gpu_init - setup the 3D engine
3145 *
3146 * @rdev: radeon_device pointer
3147 *
3148 * Configures the 3D engine and tiling configuration
3149 * registers so that the 3D engine is usable.
3150 */
3151static void cik_gpu_init(struct radeon_device *rdev)
3152{
3153 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3154 u32 mc_shared_chmap, mc_arb_ramcfg;
3155 u32 hdp_host_path_cntl;
3156 u32 tmp;
3157 int i, j;
3158
3159 switch (rdev->family) {
3160 case CHIP_BONAIRE:
3161 rdev->config.cik.max_shader_engines = 2;
3162 rdev->config.cik.max_tile_pipes = 4;
3163 rdev->config.cik.max_cu_per_sh = 7;
3164 rdev->config.cik.max_sh_per_se = 1;
3165 rdev->config.cik.max_backends_per_se = 2;
3166 rdev->config.cik.max_texture_channel_caches = 4;
3167 rdev->config.cik.max_gprs = 256;
3168 rdev->config.cik.max_gs_threads = 32;
3169 rdev->config.cik.max_hw_contexts = 8;
3170
3171 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3172 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3173 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3174 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3175 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3176 break;
b496038b
AD		 3177 case CHIP_HAWAII:
3178 rdev->config.cik.max_shader_engines = 4;
3179 rdev->config.cik.max_tile_pipes = 16;
3180 rdev->config.cik.max_cu_per_sh = 11;
3181 rdev->config.cik.max_sh_per_se = 1;
3182 rdev->config.cik.max_backends_per_se = 4;
3183 rdev->config.cik.max_texture_channel_caches = 16;
3184 rdev->config.cik.max_gprs = 256;
3185 rdev->config.cik.max_gs_threads = 32;
3186 rdev->config.cik.max_hw_contexts = 8;
3187
3188 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3189 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3190 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3191 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3192 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3193 break;
8cc1a532 3194 case CHIP_KAVERI:
b2e4c70a
AD		 3195 rdev->config.cik.max_shader_engines = 1;
3196 rdev->config.cik.max_tile_pipes = 4;
3197 if ((rdev->pdev->device == 0x1304) ||
3198 (rdev->pdev->device == 0x1305) ||
3199 (rdev->pdev->device == 0x130C) ||
3200 (rdev->pdev->device == 0x130F) ||
3201 (rdev->pdev->device == 0x1310) ||
3202 (rdev->pdev->device == 0x1311) ||
3203 (rdev->pdev->device == 0x131C)) {
3204 rdev->config.cik.max_cu_per_sh = 8;
3205 rdev->config.cik.max_backends_per_se = 2;
3206 } else if ((rdev->pdev->device == 0x1309) ||
3207 (rdev->pdev->device == 0x130A) ||
3208 (rdev->pdev->device == 0x130D) ||
7c4622d5
AD		 3209 (rdev->pdev->device == 0x1313) ||
3210 (rdev->pdev->device == 0x131D)) {
b2e4c70a
AD		 3211 rdev->config.cik.max_cu_per_sh = 6;
3212 rdev->config.cik.max_backends_per_se = 2;
3213 } else if ((rdev->pdev->device == 0x1306) ||
3214 (rdev->pdev->device == 0x1307) ||
3215 (rdev->pdev->device == 0x130B) ||
3216 (rdev->pdev->device == 0x130E) ||
3217 (rdev->pdev->device == 0x1315) ||
3218 (rdev->pdev->device == 0x131B)) {
3219 rdev->config.cik.max_cu_per_sh = 4;
3220 rdev->config.cik.max_backends_per_se = 1;
3221 } else {
3222 rdev->config.cik.max_cu_per_sh = 3;
3223 rdev->config.cik.max_backends_per_se = 1;
3224 }
3225 rdev->config.cik.max_sh_per_se = 1;
3226 rdev->config.cik.max_texture_channel_caches = 4;
3227 rdev->config.cik.max_gprs = 256;
3228 rdev->config.cik.max_gs_threads = 16;
3229 rdev->config.cik.max_hw_contexts = 8;
3230
3231 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3232 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3233 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3234 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3235 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 3236 break;
3237 case CHIP_KABINI:
3238 default:
3239 rdev->config.cik.max_shader_engines = 1;
3240 rdev->config.cik.max_tile_pipes = 2;
3241 rdev->config.cik.max_cu_per_sh = 2;
3242 rdev->config.cik.max_sh_per_se = 1;
3243 rdev->config.cik.max_backends_per_se = 1;
3244 rdev->config.cik.max_texture_channel_caches = 2;
3245 rdev->config.cik.max_gprs = 256;
3246 rdev->config.cik.max_gs_threads = 16;
3247 rdev->config.cik.max_hw_contexts = 8;
3248
3249 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3250 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3251 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3252 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3253 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3254 break;
3255 }
3256
3257 /* Initialize HDP */
3258 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3259 WREG32((0x2c14 + j), 0x00000000);
3260 WREG32((0x2c18 + j), 0x00000000);
3261 WREG32((0x2c1c + j), 0x00000000);
3262 WREG32((0x2c20 + j), 0x00000000);
3263 WREG32((0x2c24 + j), 0x00000000);
3264 }
3265
3266 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3267
3268 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3269
3270 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3271 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3272
3273 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3274 rdev->config.cik.mem_max_burst_length_bytes = 256;
3275 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3276 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3277 if (rdev->config.cik.mem_row_size_in_kb > 4)
3278 rdev->config.cik.mem_row_size_in_kb = 4;
3279 /* XXX use MC settings? */
3280 rdev->config.cik.shader_engine_tile_size = 32;
3281 rdev->config.cik.num_gpus = 1;
3282 rdev->config.cik.multi_gpu_tile_size = 64;
3283
3284 /* fix up row size */
3285 gb_addr_config &= ~ROW_SIZE_MASK;
3286 switch (rdev->config.cik.mem_row_size_in_kb) {
3287 case 1:
3288 default:
3289 gb_addr_config |= ROW_SIZE(0);
3290 break;
3291 case 2:
3292 gb_addr_config |= ROW_SIZE(1);
3293 break;
3294 case 4:
3295 gb_addr_config |= ROW_SIZE(2);
3296 break;
3297 }
3298
3299 /* setup tiling info dword. gb_addr_config is not adequate since it does
3300 * not have bank info, so create a custom tiling dword.
3301 * bits 3:0 num_pipes
3302 * bits 7:4 num_banks
3303 * bits 11:8 group_size
3304 * bits 15:12 row_size
3305 */
3306 rdev->config.cik.tile_config = 0;
3307 switch (rdev->config.cik.num_tile_pipes) {
3308 case 1:
3309 rdev->config.cik.tile_config |= (0 << 0);
3310 break;
3311 case 2:
3312 rdev->config.cik.tile_config |= (1 << 0);
3313 break;
3314 case 4:
3315 rdev->config.cik.tile_config |= (2 << 0);
3316 break;
3317 case 8:
3318 default:
3319 /* XXX what about 12? */
3320 rdev->config.cik.tile_config |= (3 << 0);
3321 break;
3322 }
a537314e
MD		 3323 rdev->config.cik.tile_config |=
3324 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
8cc1a532
AD		 3325 rdev->config.cik.tile_config |=
3326 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3327 rdev->config.cik.tile_config |=
3328 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3329
3330 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3331 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3332 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 3333 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3334 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 3335 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3336 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3337 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 3338
3339 cik_tiling_mode_table_init(rdev);
3340
3341 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3342 rdev->config.cik.max_sh_per_se,
3343 rdev->config.cik.max_backends_per_se);
3344
3345 /* set HW defaults for 3D engine */
3346 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3347
3348 WREG32(SX_DEBUG_1, 0x20);
3349
3350 WREG32(TA_CNTL_AUX, 0x00010000);
3351
3352 tmp = RREG32(SPI_CONFIG_CNTL);
3353 tmp |= 0x03000000;
3354 WREG32(SPI_CONFIG_CNTL, tmp);
3355
3356 WREG32(SQ_CONFIG, 1);
3357
3358 WREG32(DB_DEBUG, 0);
3359
3360 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3361 tmp |= 0x00000400;
3362 WREG32(DB_DEBUG2, tmp);
3363
3364 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3365 tmp |= 0x00020200;
3366 WREG32(DB_DEBUG3, tmp);
3367
3368 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3369 tmp |= 0x00018208;
3370 WREG32(CB_HW_CONTROL, tmp);
3371
3372 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3373
3374 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3375 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3376 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3377 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3378
3379 WREG32(VGT_NUM_INSTANCES, 1);
3380
3381 WREG32(CP_PERFMON_CNTL, 0);
3382
3383 WREG32(SQ_CONFIG, 0);
3384
3385 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3386 FORCE_EOV_MAX_REZ_CNT(255)));
3387
3388 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3389 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3390
3391 WREG32(VGT_GS_VERTEX_REUSE, 16);
3392 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3393
3394 tmp = RREG32(HDP_MISC_CNTL);
3395 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3396 WREG32(HDP_MISC_CNTL, tmp);
3397
3398 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3399 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3400
3401 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3402 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3403
3404 udelay(50);
3405}
3406
2cae3bc3
AD		 3407/*
3408 * GPU scratch registers helpers function.
3409 */
3410/**
3411 * cik_scratch_init - setup driver info for CP scratch regs
3412 *
3413 * @rdev: radeon_device pointer
3414 *
3415 * Set up the number and offset of the CP scratch registers.
3416 * NOTE: use of CP scratch registers is a legacy inferface and
3417 * is not used by default on newer asics (r6xx+). On newer asics,
3418 * memory buffers are used for fences rather than scratch regs.
3419 */
3420static void cik_scratch_init(struct radeon_device *rdev)
3421{
3422 int i;
3423
3424 rdev->scratch.num_reg = 7;
3425 rdev->scratch.reg_base = SCRATCH_REG0;
3426 for (i = 0; i < rdev->scratch.num_reg; i++) {
3427 rdev->scratch.free[i] = true;
3428 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3429 }
3430}
3431
fbc832c7
AD		 3432/**
3433 * cik_ring_test - basic gfx ring test
3434 *
3435 * @rdev: radeon_device pointer
3436 * @ring: radeon_ring structure holding ring information
3437 *
3438 * Allocate a scratch register and write to it using the gfx ring (CIK).
3439 * Provides a basic gfx ring test to verify that the ring is working.
3440 * Used by cik_cp_gfx_resume();
3441 * Returns 0 on success, error on failure.
3442 */
3443int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3444{
3445 uint32_t scratch;
3446 uint32_t tmp = 0;
3447 unsigned i;
3448 int r;
3449
3450 r = radeon_scratch_get(rdev, &scratch);
3451 if (r) {
3452 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3453 return r;
3454 }
3455 WREG32(scratch, 0xCAFEDEAD);
3456 r = radeon_ring_lock(rdev, ring, 3);
3457 if (r) {
3458 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3459 radeon_scratch_free(rdev, scratch);
3460 return r;
3461 }
3462 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3463 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3464 radeon_ring_write(ring, 0xDEADBEEF);
3465 radeon_ring_unlock_commit(rdev, ring);
963e81f9 3466
fbc832c7
AD		 3467 for (i = 0; i < rdev->usec_timeout; i++) {
3468 tmp = RREG32(scratch);
3469 if (tmp == 0xDEADBEEF)
3470 break;
3471 DRM_UDELAY(1);
3472 }
3473 if (i < rdev->usec_timeout) {
3474 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3475 } else {
3476 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3477 ring->idx, scratch, tmp);
3478 r = -EINVAL;
3479 }
3480 radeon_scratch_free(rdev, scratch);
3481 return r;
3482}
3483
2cae3bc3 3484/**
b07fdd38 3485 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3486 *
3487 * @rdev: radeon_device pointer
3488 * @fence: radeon fence object
3489 *
3490 * Emits a fence sequnce number on the gfx ring and flushes
3491 * GPU caches.
3492 */
b07fdd38
AD		 3493void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3494 struct radeon_fence *fence)
2cae3bc3
AD		 3495{
3496 struct radeon_ring *ring = &rdev->ring[fence->ring];
3497 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3498
3499 /* EVENT_WRITE_EOP - flush caches, send int */
3500 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3501 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3502 EOP_TC_ACTION_EN |
3503 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3504 EVENT_INDEX(5)));
3505 radeon_ring_write(ring, addr & 0xfffffffc);
3506 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3507 radeon_ring_write(ring, fence->seq);
3508 radeon_ring_write(ring, 0);
3509 /* HDP flush */
3510 /* We should be using the new WAIT_REG_MEM special op packet here
3511 * but it causes the CP to hang
3512 */
3513 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3514 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3515 WRITE_DATA_DST_SEL(0)));
3516 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3517 radeon_ring_write(ring, 0);
3518 radeon_ring_write(ring, 0);
3519}
3520
b07fdd38
AD		 3521/**
3522 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3523 *
3524 * @rdev: radeon_device pointer
3525 * @fence: radeon fence object
3526 *
3527 * Emits a fence sequnce number on the compute ring and flushes
3528 * GPU caches.
3529 */
3530void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3531 struct radeon_fence *fence)
3532{
3533 struct radeon_ring *ring = &rdev->ring[fence->ring];
3534 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3535
3536 /* RELEASE_MEM - flush caches, send int */
3537 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3538 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3539 EOP_TC_ACTION_EN |
3540 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3541 EVENT_INDEX(5)));
3542 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3543 radeon_ring_write(ring, addr & 0xfffffffc);
3544 radeon_ring_write(ring, upper_32_bits(addr));
3545 radeon_ring_write(ring, fence->seq);
3546 radeon_ring_write(ring, 0);
3547 /* HDP flush */
3548 /* We should be using the new WAIT_REG_MEM special op packet here
3549 * but it causes the CP to hang
3550 */
3551 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3552 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3553 WRITE_DATA_DST_SEL(0)));
3554 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3555 radeon_ring_write(ring, 0);
3556 radeon_ring_write(ring, 0);
3557}
3558
1654b817 3559bool cik_semaphore_ring_emit(struct radeon_device *rdev,
2cae3bc3
AD		 3560 struct radeon_ring *ring,
3561 struct radeon_semaphore *semaphore,
3562 bool emit_wait)
3563{
99b4f251
CK		 3564/* TODO: figure out why semaphore cause lockups */
3565#if 0
2cae3bc3
AD		 3566 uint64_t addr = semaphore->gpu_addr;
3567 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3568
3569 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3570 radeon_ring_write(ring, addr & 0xffffffff);
3571 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
1654b817
CK		 3572
3573 return true;
99b4f251
CK		 3574#else
3575 return false;
3576#endif
2cae3bc3
AD		 3577}
3578
c9dbd705
AD		 3579/**
3580 * cik_copy_cpdma - copy pages using the CP DMA engine
3581 *
3582 * @rdev: radeon_device pointer
3583 * @src_offset: src GPU address
3584 * @dst_offset: dst GPU address
3585 * @num_gpu_pages: number of GPU pages to xfer
3586 * @fence: radeon fence object
3587 *
3588 * Copy GPU paging using the CP DMA engine (CIK+).
3589 * Used by the radeon ttm implementation to move pages if
3590 * registered as the asic copy callback.
3591 */
3592int cik_copy_cpdma(struct radeon_device *rdev,
3593 uint64_t src_offset, uint64_t dst_offset,
3594 unsigned num_gpu_pages,
3595 struct radeon_fence **fence)
3596{
3597 struct radeon_semaphore *sem = NULL;
3598 int ring_index = rdev->asic->copy.blit_ring_index;
3599 struct radeon_ring *ring = &rdev->ring[ring_index];
3600 u32 size_in_bytes, cur_size_in_bytes, control;
3601 int i, num_loops;
3602 int r = 0;
3603
3604 r = radeon_semaphore_create(rdev, &sem);
3605 if (r) {
3606 DRM_ERROR("radeon: moving bo (%d).\n", r);
3607 return r;
3608 }
3609
3610 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3611 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3612 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3613 if (r) {
3614 DRM_ERROR("radeon: moving bo (%d).\n", r);
3615 radeon_semaphore_free(rdev, &sem, NULL);
3616 return r;
3617 }
3618
1654b817
CK		 3619 radeon_semaphore_sync_to(sem, *fence);
3620 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
c9dbd705
AD		 3621
3622 for (i = 0; i < num_loops; i++) {
3623 cur_size_in_bytes = size_in_bytes;
3624 if (cur_size_in_bytes > 0x1fffff)
3625 cur_size_in_bytes = 0x1fffff;
3626 size_in_bytes -= cur_size_in_bytes;
3627 control = 0;
3628 if (size_in_bytes == 0)
3629 control |= PACKET3_DMA_DATA_CP_SYNC;
3630 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3631 radeon_ring_write(ring, control);
3632 radeon_ring_write(ring, lower_32_bits(src_offset));
3633 radeon_ring_write(ring, upper_32_bits(src_offset));
3634 radeon_ring_write(ring, lower_32_bits(dst_offset));
3635 radeon_ring_write(ring, upper_32_bits(dst_offset));
3636 radeon_ring_write(ring, cur_size_in_bytes);
3637 src_offset += cur_size_in_bytes;
3638 dst_offset += cur_size_in_bytes;
3639 }
3640
3641 r = radeon_fence_emit(rdev, fence, ring->idx);
3642 if (r) {
3643 radeon_ring_unlock_undo(rdev, ring);
3644 return r;
3645 }
3646
3647 radeon_ring_unlock_commit(rdev, ring);
3648 radeon_semaphore_free(rdev, &sem, *fence);
3649
3650 return r;
3651}
3652
2cae3bc3
AD		 3653/*
3654 * IB stuff
3655 */
3656/**
3657 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3658 *
3659 * @rdev: radeon_device pointer
3660 * @ib: radeon indirect buffer object
3661 *
3662 * Emits an DE (drawing engine) or CE (constant engine) IB
3663 * on the gfx ring. IBs are usually generated by userspace
3664 * acceleration drivers and submitted to the kernel for
3665 * sheduling on the ring. This function schedules the IB
3666 * on the gfx ring for execution by the GPU.
3667 */
3668void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3669{
3670 struct radeon_ring *ring = &rdev->ring[ib->ring];
3671 u32 header, control = INDIRECT_BUFFER_VALID;
3672
3673 if (ib->is_const_ib) {
3674 /* set switch buffer packet before const IB */
3675 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3676 radeon_ring_write(ring, 0);
3677
3678 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3679 } else {
3680 u32 next_rptr;
3681 if (ring->rptr_save_reg) {
3682 next_rptr = ring->wptr + 3 + 4;
3683 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3684 radeon_ring_write(ring, ((ring->rptr_save_reg -
3685 PACKET3_SET_UCONFIG_REG_START) >> 2));
3686 radeon_ring_write(ring, next_rptr);
3687 } else if (rdev->wb.enabled) {
3688 next_rptr = ring->wptr + 5 + 4;
3689 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3690 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3691 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3692 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3693 radeon_ring_write(ring, next_rptr);
3694 }
3695
3696 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3697 }
3698
3699 control |= ib->length_dw |
3700 (ib->vm ? (ib->vm->id << 24) : 0);
3701
3702 radeon_ring_write(ring, header);
3703 radeon_ring_write(ring,
3704#ifdef __BIG_ENDIAN
3705 (2 << 0) |
3706#endif
3707 (ib->gpu_addr & 0xFFFFFFFC));
3708 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3709 radeon_ring_write(ring, control);
3710}
3711
fbc832c7
AD		 3712/**
3713 * cik_ib_test - basic gfx ring IB test
3714 *
3715 * @rdev: radeon_device pointer
3716 * @ring: radeon_ring structure holding ring information
3717 *
3718 * Allocate an IB and execute it on the gfx ring (CIK).
3719 * Provides a basic gfx ring test to verify that IBs are working.
3720 * Returns 0 on success, error on failure.
3721 */
3722int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3723{
3724 struct radeon_ib ib;
3725 uint32_t scratch;
3726 uint32_t tmp = 0;
3727 unsigned i;
3728 int r;
3729
3730 r = radeon_scratch_get(rdev, &scratch);
3731 if (r) {
3732 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3733 return r;
3734 }
3735 WREG32(scratch, 0xCAFEDEAD);
3736 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3737 if (r) {
3738 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
5510f124 3739 radeon_scratch_free(rdev, scratch);
fbc832c7
AD		 3740 return r;
3741 }
3742 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3743 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3744 ib.ptr[2] = 0xDEADBEEF;
3745 ib.length_dw = 3;
3746 r = radeon_ib_schedule(rdev, &ib, NULL);
3747 if (r) {
3748 radeon_scratch_free(rdev, scratch);
3749 radeon_ib_free(rdev, &ib);
3750 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3751 return r;
3752 }
3753 r = radeon_fence_wait(ib.fence, false);
3754 if (r) {
3755 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
5510f124
CK		 3756 radeon_scratch_free(rdev, scratch);
3757 radeon_ib_free(rdev, &ib);
fbc832c7
AD		 3758 return r;
3759 }
3760 for (i = 0; i < rdev->usec_timeout; i++) {
3761 tmp = RREG32(scratch);
3762 if (tmp == 0xDEADBEEF)
3763 break;
3764 DRM_UDELAY(1);
3765 }
3766 if (i < rdev->usec_timeout) {
3767 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3768 } else {
3769 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3770 scratch, tmp);
3771 r = -EINVAL;
3772 }
3773 radeon_scratch_free(rdev, scratch);
3774 radeon_ib_free(rdev, &ib);
3775 return r;
3776}
3777
841cf442
AD		 3778/*
3779 * CP.
3780 * On CIK, gfx and compute now have independant command processors.
3781 *
3782 * GFX
3783 * Gfx consists of a single ring and can process both gfx jobs and
3784 * compute jobs. The gfx CP consists of three microengines (ME):
3785 * PFP - Pre-Fetch Parser
3786 * ME - Micro Engine
3787 * CE - Constant Engine
3788 * The PFP and ME make up what is considered the Drawing Engine (DE).
3789 * The CE is an asynchronous engine used for updating buffer desciptors
3790 * used by the DE so that they can be loaded into cache in parallel
3791 * while the DE is processing state update packets.
3792 *
3793 * Compute
3794 * The compute CP consists of two microengines (ME):
3795 * MEC1 - Compute MicroEngine 1
3796 * MEC2 - Compute MicroEngine 2
3797 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3798 * The queues are exposed to userspace and are programmed directly
3799 * by the compute runtime.
3800 */
3801/**
3802 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3803 *
3804 * @rdev: radeon_device pointer
3805 * @enable: enable or disable the MEs
3806 *
3807 * Halts or unhalts the gfx MEs.
3808 */
3809static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3810{
3811 if (enable)
3812 WREG32(CP_ME_CNTL, 0);
3813 else {
3814 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3815 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3816 }
3817 udelay(50);
3818}
3819
3820/**
3821 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3822 *
3823 * @rdev: radeon_device pointer
3824 *
3825 * Loads the gfx PFP, ME, and CE ucode.
3826 * Returns 0 for success, -EINVAL if the ucode is not available.
3827 */
3828static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3829{
3830 const __be32 *fw_data;
3831 int i;
3832
3833 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3834 return -EINVAL;
3835
3836 cik_cp_gfx_enable(rdev, false);
3837
3838 /* PFP */
3839 fw_data = (const __be32 *)rdev->pfp_fw->data;
3840 WREG32(CP_PFP_UCODE_ADDR, 0);
3841 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3842 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3843 WREG32(CP_PFP_UCODE_ADDR, 0);
3844
3845 /* CE */
3846 fw_data = (const __be32 *)rdev->ce_fw->data;
3847 WREG32(CP_CE_UCODE_ADDR, 0);
3848 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3849 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3850 WREG32(CP_CE_UCODE_ADDR, 0);
3851
3852 /* ME */
3853 fw_data = (const __be32 *)rdev->me_fw->data;
3854 WREG32(CP_ME_RAM_WADDR, 0);
3855 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3856 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3857 WREG32(CP_ME_RAM_WADDR, 0);
3858
3859 WREG32(CP_PFP_UCODE_ADDR, 0);
3860 WREG32(CP_CE_UCODE_ADDR, 0);
3861 WREG32(CP_ME_RAM_WADDR, 0);
3862 WREG32(CP_ME_RAM_RADDR, 0);
3863 return 0;
3864}
3865
3866/**
3867 * cik_cp_gfx_start - start the gfx ring
3868 *
3869 * @rdev: radeon_device pointer
3870 *
3871 * Enables the ring and loads the clear state context and other
3872 * packets required to init the ring.
3873 * Returns 0 for success, error for failure.
3874 */
3875static int cik_cp_gfx_start(struct radeon_device *rdev)
3876{
3877 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3878 int r, i;
3879
3880 /* init the CP */
3881 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3882 WREG32(CP_ENDIAN_SWAP, 0);
3883 WREG32(CP_DEVICE_ID, 1);
3884
3885 cik_cp_gfx_enable(rdev, true);
3886
3887 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3888 if (r) {
3889 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3890 return r;
3891 }
3892
3893 /* init the CE partitions. CE only used for gfx on CIK */
3894 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3895 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3896 radeon_ring_write(ring, 0xc000);
3897 radeon_ring_write(ring, 0xc000);
3898
3899 /* setup clear context state */
3900 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3901 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3902
3903 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3904 radeon_ring_write(ring, 0x80000000);
3905 radeon_ring_write(ring, 0x80000000);
3906
3907 for (i = 0; i < cik_default_size; i++)
3908 radeon_ring_write(ring, cik_default_state[i]);
3909
3910 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3911 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3912
3913 /* set clear context state */
3914 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3915 radeon_ring_write(ring, 0);
3916
3917 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3918 radeon_ring_write(ring, 0x00000316);
3919 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3920 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3921
3922 radeon_ring_unlock_commit(rdev, ring);
3923
3924 return 0;
3925}
3926
3927/**
3928 * cik_cp_gfx_fini - stop the gfx ring
3929 *
3930 * @rdev: radeon_device pointer
3931 *
3932 * Stop the gfx ring and tear down the driver ring
3933 * info.
3934 */
3935static void cik_cp_gfx_fini(struct radeon_device *rdev)
3936{
3937 cik_cp_gfx_enable(rdev, false);
3938 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3939}
3940
3941/**
3942 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3943 *
3944 * @rdev: radeon_device pointer
3945 *
3946 * Program the location and size of the gfx ring buffer
3947 * and test it to make sure it's working.
3948 * Returns 0 for success, error for failure.
3949 */
3950static int cik_cp_gfx_resume(struct radeon_device *rdev)
3951{
3952 struct radeon_ring *ring;
3953 u32 tmp;
3954 u32 rb_bufsz;
3955 u64 rb_addr;
3956 int r;
3957
3958 WREG32(CP_SEM_WAIT_TIMER, 0x0);
939c0d3c
AD		 3959 if (rdev->family != CHIP_HAWAII)
3960 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
841cf442
AD		 3961
3962 /* Set the write pointer delay */
3963 WREG32(CP_RB_WPTR_DELAY, 0);
3964
3965 /* set the RB to use vmid 0 */
3966 WREG32(CP_RB_VMID, 0);
3967
3968 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3969
3970 /* ring 0 - compute and gfx */
3971 /* Set ring buffer size */
3972 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 3973 rb_bufsz = order_base_2(ring->ring_size / 8);
3974 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
841cf442
AD		 3975#ifdef __BIG_ENDIAN
3976 tmp |= BUF_SWAP_32BIT;
3977#endif
3978 WREG32(CP_RB0_CNTL, tmp);
3979
3980 /* Initialize the ring buffer's read and write pointers */
3981 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3982 ring->wptr = 0;
3983 WREG32(CP_RB0_WPTR, ring->wptr);
3984
3985 /* set the wb address wether it's enabled or not */
3986 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3987 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3988
3989 /* scratch register shadowing is no longer supported */
3990 WREG32(SCRATCH_UMSK, 0);
3991
3992 if (!rdev->wb.enabled)
3993 tmp |= RB_NO_UPDATE;
3994
3995 mdelay(1);
3996 WREG32(CP_RB0_CNTL, tmp);
3997
3998 rb_addr = ring->gpu_addr >> 8;
3999 WREG32(CP_RB0_BASE, rb_addr);
4000 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4001
4002 ring->rptr = RREG32(CP_RB0_RPTR);
4003
4004 /* start the ring */
4005 cik_cp_gfx_start(rdev);
4006 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4007 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4008 if (r) {
4009 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4010 return r;
4011 }
4012 return 0;
4013}
4014
963e81f9
AD		 4015u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
4016 struct radeon_ring *ring)
4017{
4018 u32 rptr;
4019
4020
4021
4022 if (rdev->wb.enabled) {
4023 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
4024 } else {
f61d5b46 4025 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4026 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4027 rptr = RREG32(CP_HQD_PQ_RPTR);
4028 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4029 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4030 }
963e81f9
AD		 4031
4032 return rptr;
4033}
4034
4035u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
4036 struct radeon_ring *ring)
4037{
4038 u32 wptr;
4039
4040 if (rdev->wb.enabled) {
4041 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
4042 } else {
f61d5b46 4043 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4044 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4045 wptr = RREG32(CP_HQD_PQ_WPTR);
4046 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4047 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4048 }
963e81f9
AD		 4049
4050 return wptr;
4051}
4052
4053void cik_compute_ring_set_wptr(struct radeon_device *rdev,
4054 struct radeon_ring *ring)
4055{
2e1e6dad
CK		 4056 rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(ring->wptr);
4057 WDOORBELL32(ring->doorbell_offset, ring->wptr);
963e81f9
AD		 4058}
4059
841cf442
AD		 4060/**
4061 * cik_cp_compute_enable - enable/disable the compute CP MEs
4062 *
4063 * @rdev: radeon_device pointer
4064 * @enable: enable or disable the MEs
4065 *
4066 * Halts or unhalts the compute MEs.
4067 */
4068static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4069{
4070 if (enable)
4071 WREG32(CP_MEC_CNTL, 0);
4072 else
4073 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4074 udelay(50);
4075}
4076
4077/**
4078 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4079 *
4080 * @rdev: radeon_device pointer
4081 *
4082 * Loads the compute MEC1&2 ucode.
4083 * Returns 0 for success, -EINVAL if the ucode is not available.
4084 */
4085static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4086{
4087 const __be32 *fw_data;
4088 int i;
4089
4090 if (!rdev->mec_fw)
4091 return -EINVAL;
4092
4093 cik_cp_compute_enable(rdev, false);
4094
4095 /* MEC1 */
4096 fw_data = (const __be32 *)rdev->mec_fw->data;
4097 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4098 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4099 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4100 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4101
4102 if (rdev->family == CHIP_KAVERI) {
4103 /* MEC2 */
4104 fw_data = (const __be32 *)rdev->mec_fw->data;
4105 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4106 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4107 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4108 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4109 }
4110
4111 return 0;
4112}
4113
4114/**
4115 * cik_cp_compute_start - start the compute queues
4116 *
4117 * @rdev: radeon_device pointer
4118 *
4119 * Enable the compute queues.
4120 * Returns 0 for success, error for failure.
4121 */
4122static int cik_cp_compute_start(struct radeon_device *rdev)
4123{
963e81f9
AD		 4124 cik_cp_compute_enable(rdev, true);
4125
841cf442
AD		 4126 return 0;
4127}
4128
4129/**
4130 * cik_cp_compute_fini - stop the compute queues
4131 *
4132 * @rdev: radeon_device pointer
4133 *
4134 * Stop the compute queues and tear down the driver queue
4135 * info.
4136 */
4137static void cik_cp_compute_fini(struct radeon_device *rdev)
4138{
963e81f9
AD		 4139 int i, idx, r;
4140
841cf442 4141 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 4142
4143 for (i = 0; i < 2; i++) {
4144 if (i == 0)
4145 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4146 else
4147 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4148
4149 if (rdev->ring[idx].mqd_obj) {
4150 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4151 if (unlikely(r != 0))
4152 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4153
4154 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4155 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4156
4157 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4158 rdev->ring[idx].mqd_obj = NULL;
4159 }
4160 }
841cf442
AD		 4161}
4162
963e81f9
AD		 4163static void cik_mec_fini(struct radeon_device *rdev)
4164{
4165 int r;
4166
4167 if (rdev->mec.hpd_eop_obj) {
4168 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4169 if (unlikely(r != 0))
4170 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4171 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4172 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4173
4174 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4175 rdev->mec.hpd_eop_obj = NULL;
4176 }
4177}
4178
4179#define MEC_HPD_SIZE 2048
4180
4181static int cik_mec_init(struct radeon_device *rdev)
4182{
4183 int r;
4184 u32 *hpd;
4185
4186 /*
4187 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4188 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4189 */
4190 if (rdev->family == CHIP_KAVERI)
4191 rdev->mec.num_mec = 2;
4192 else
4193 rdev->mec.num_mec = 1;
4194 rdev->mec.num_pipe = 4;
4195 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4196
4197 if (rdev->mec.hpd_eop_obj == NULL) {
4198 r = radeon_bo_create(rdev,
4199 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4200 PAGE_SIZE, true,
4201 RADEON_GEM_DOMAIN_GTT, NULL,
4202 &rdev->mec.hpd_eop_obj);
4203 if (r) {
4204 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4205 return r;
4206 }
4207 }
4208
4209 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4210 if (unlikely(r != 0)) {
4211 cik_mec_fini(rdev);
4212 return r;
4213 }
4214 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4215 &rdev->mec.hpd_eop_gpu_addr);
4216 if (r) {
4217 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4218 cik_mec_fini(rdev);
4219 return r;
4220 }
4221 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4222 if (r) {
4223 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4224 cik_mec_fini(rdev);
4225 return r;
4226 }
4227
4228 /* clear memory. Not sure if this is required or not */
4229 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4230
4231 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4232 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4233
4234 return 0;
4235}
4236
4237struct hqd_registers
4238{
4239 u32 cp_mqd_base_addr;
4240 u32 cp_mqd_base_addr_hi;
4241 u32 cp_hqd_active;
4242 u32 cp_hqd_vmid;
4243 u32 cp_hqd_persistent_state;
4244 u32 cp_hqd_pipe_priority;
4245 u32 cp_hqd_queue_priority;
4246 u32 cp_hqd_quantum;
4247 u32 cp_hqd_pq_base;
4248 u32 cp_hqd_pq_base_hi;
4249 u32 cp_hqd_pq_rptr;
4250 u32 cp_hqd_pq_rptr_report_addr;
4251 u32 cp_hqd_pq_rptr_report_addr_hi;
4252 u32 cp_hqd_pq_wptr_poll_addr;
4253 u32 cp_hqd_pq_wptr_poll_addr_hi;
4254 u32 cp_hqd_pq_doorbell_control;
4255 u32 cp_hqd_pq_wptr;
4256 u32 cp_hqd_pq_control;
4257 u32 cp_hqd_ib_base_addr;
4258 u32 cp_hqd_ib_base_addr_hi;
4259 u32 cp_hqd_ib_rptr;
4260 u32 cp_hqd_ib_control;
4261 u32 cp_hqd_iq_timer;
4262 u32 cp_hqd_iq_rptr;
4263 u32 cp_hqd_dequeue_request;
4264 u32 cp_hqd_dma_offload;
4265 u32 cp_hqd_sema_cmd;
4266 u32 cp_hqd_msg_type;
4267 u32 cp_hqd_atomic0_preop_lo;
4268 u32 cp_hqd_atomic0_preop_hi;
4269 u32 cp_hqd_atomic1_preop_lo;
4270 u32 cp_hqd_atomic1_preop_hi;
4271 u32 cp_hqd_hq_scheduler0;
4272 u32 cp_hqd_hq_scheduler1;
4273 u32 cp_mqd_control;
4274};
4275
4276struct bonaire_mqd
4277{
4278 u32 header;
4279 u32 dispatch_initiator;
4280 u32 dimensions[3];
4281 u32 start_idx[3];
4282 u32 num_threads[3];
4283 u32 pipeline_stat_enable;
4284 u32 perf_counter_enable;
4285 u32 pgm[2];
4286 u32 tba[2];
4287 u32 tma[2];
4288 u32 pgm_rsrc[2];
4289 u32 vmid;
4290 u32 resource_limits;
4291 u32 static_thread_mgmt01[2];
4292 u32 tmp_ring_size;
4293 u32 static_thread_mgmt23[2];
4294 u32 restart[3];
4295 u32 thread_trace_enable;
4296 u32 reserved1;
4297 u32 user_data[16];
4298 u32 vgtcs_invoke_count[2];
4299 struct hqd_registers queue_state;
4300 u32 dequeue_cntr;
4301 u32 interrupt_queue[64];
4302};
4303
841cf442
AD		 4304/**
4305 * cik_cp_compute_resume - setup the compute queue registers
4306 *
4307 * @rdev: radeon_device pointer
4308 *
4309 * Program the compute queues and test them to make sure they
4310 * are working.
4311 * Returns 0 for success, error for failure.
4312 */
4313static int cik_cp_compute_resume(struct radeon_device *rdev)
4314{
963e81f9
AD		 4315 int r, i, idx;
4316 u32 tmp;
4317 bool use_doorbell = true;
4318 u64 hqd_gpu_addr;
4319 u64 mqd_gpu_addr;
4320 u64 eop_gpu_addr;
4321 u64 wb_gpu_addr;
4322 u32 *buf;
4323 struct bonaire_mqd *mqd;
841cf442 4324
841cf442
AD		 4325 r = cik_cp_compute_start(rdev);
4326 if (r)
4327 return r;
963e81f9
AD		 4328
4329 /* fix up chicken bits */
4330 tmp = RREG32(CP_CPF_DEBUG);
4331 tmp |= (1 << 23);
4332 WREG32(CP_CPF_DEBUG, tmp);
4333
4334 /* init the pipes */
f61d5b46 4335 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4336 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4337 int me = (i < 4) ? 1 : 2;
4338 int pipe = (i < 4) ? i : (i - 4);
4339
4340 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4341
4342 cik_srbm_select(rdev, me, pipe, 0, 0);
4343
4344 /* write the EOP addr */
4345 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4346 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4347
4348 /* set the VMID assigned */
4349 WREG32(CP_HPD_EOP_VMID, 0);
4350
4351 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4352 tmp = RREG32(CP_HPD_EOP_CONTROL);
4353 tmp &= ~EOP_SIZE_MASK;
b72a8925 4354 tmp |= order_base_2(MEC_HPD_SIZE / 8);
963e81f9
AD		 4355 WREG32(CP_HPD_EOP_CONTROL, tmp);
4356 }
4357 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4358 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4359
4360 /* init the queues. Just two for now. */
4361 for (i = 0; i < 2; i++) {
4362 if (i == 0)
4363 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4364 else
4365 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4366
4367 if (rdev->ring[idx].mqd_obj == NULL) {
4368 r = radeon_bo_create(rdev,
4369 sizeof(struct bonaire_mqd),
4370 PAGE_SIZE, true,
4371 RADEON_GEM_DOMAIN_GTT, NULL,
4372 &rdev->ring[idx].mqd_obj);
4373 if (r) {
4374 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4375 return r;
4376 }
4377 }
4378
4379 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4380 if (unlikely(r != 0)) {
4381 cik_cp_compute_fini(rdev);
4382 return r;
4383 }
4384 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4385 &mqd_gpu_addr);
4386 if (r) {
4387 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4388 cik_cp_compute_fini(rdev);
4389 return r;
4390 }
4391 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4392 if (r) {
4393 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4394 cik_cp_compute_fini(rdev);
4395 return r;
4396 }
4397
4398 /* doorbell offset */
4399 rdev->ring[idx].doorbell_offset =
4400 (rdev->ring[idx].doorbell_page_num * PAGE_SIZE) + 0;
4401
4402 /* init the mqd struct */
4403 memset(buf, 0, sizeof(struct bonaire_mqd));
4404
4405 mqd = (struct bonaire_mqd *)buf;
4406 mqd->header = 0xC0310800;
4407 mqd->static_thread_mgmt01[0] = 0xffffffff;
4408 mqd->static_thread_mgmt01[1] = 0xffffffff;
4409 mqd->static_thread_mgmt23[0] = 0xffffffff;
4410 mqd->static_thread_mgmt23[1] = 0xffffffff;
4411
f61d5b46 4412 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4413 cik_srbm_select(rdev, rdev->ring[idx].me,
4414 rdev->ring[idx].pipe,
4415 rdev->ring[idx].queue, 0);
4416
4417 /* disable wptr polling */
4418 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4419 tmp &= ~WPTR_POLL_EN;
4420 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4421
4422 /* enable doorbell? */
4423 mqd->queue_state.cp_hqd_pq_doorbell_control =
4424 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4425 if (use_doorbell)
4426 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4427 else
4428 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4429 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4430 mqd->queue_state.cp_hqd_pq_doorbell_control);
4431
4432 /* disable the queue if it's active */
4433 mqd->queue_state.cp_hqd_dequeue_request = 0;
4434 mqd->queue_state.cp_hqd_pq_rptr = 0;
4435 mqd->queue_state.cp_hqd_pq_wptr= 0;
4436 if (RREG32(CP_HQD_ACTIVE) & 1) {
4437 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4438 for (i = 0; i < rdev->usec_timeout; i++) {
4439 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4440 break;
4441 udelay(1);
4442 }
4443 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4444 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4445 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4446 }
4447
4448 /* set the pointer to the MQD */
4449 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4450 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4451 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4452 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4453 /* set MQD vmid to 0 */
4454 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4455 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4456 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4457
4458 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4459 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4460 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4461 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4462 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4463 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4464
4465 /* set up the HQD, this is similar to CP_RB0_CNTL */
4466 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4467 mqd->queue_state.cp_hqd_pq_control &=
4468 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4469
4470 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4471 order_base_2(rdev->ring[idx].ring_size / 8);
963e81f9 4472 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4473 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
963e81f9
AD		 4474#ifdef __BIG_ENDIAN
4475 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4476#endif
4477 mqd->queue_state.cp_hqd_pq_control &=
4478 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4479 mqd->queue_state.cp_hqd_pq_control |=
4480 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4481 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4482
4483 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4484 if (i == 0)
4485 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4486 else
4487 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4488 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4489 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4490 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4491 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4492 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4493
4494 /* set the wb address wether it's enabled or not */
4495 if (i == 0)
4496 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4497 else
4498 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4499 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4500 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4501 upper_32_bits(wb_gpu_addr) & 0xffff;
4502 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4503 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4504 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4505 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4506
4507 /* enable the doorbell if requested */
4508 if (use_doorbell) {
4509 mqd->queue_state.cp_hqd_pq_doorbell_control =
4510 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4511 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4512 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4513 DOORBELL_OFFSET(rdev->ring[idx].doorbell_offset / 4);
4514 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4515 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4516 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4517
4518 } else {
4519 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4520 }
4521 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4522 mqd->queue_state.cp_hqd_pq_doorbell_control);
4523
4524 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4525 rdev->ring[idx].wptr = 0;
4526 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4527 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4528 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
4529 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
4530
4531 /* set the vmid for the queue */
4532 mqd->queue_state.cp_hqd_vmid = 0;
4533 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4534
4535 /* activate the queue */
4536 mqd->queue_state.cp_hqd_active = 1;
4537 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4538
4539 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4540 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4541
4542 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4543 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4544
4545 rdev->ring[idx].ready = true;
4546 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4547 if (r)
4548 rdev->ring[idx].ready = false;
4549 }
4550
841cf442
AD		 4551 return 0;
4552}
4553
841cf442
AD		 4554static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4555{
4556 cik_cp_gfx_enable(rdev, enable);
4557 cik_cp_compute_enable(rdev, enable);
4558}
4559
841cf442
AD		 4560static int cik_cp_load_microcode(struct radeon_device *rdev)
4561{
4562 int r;
4563
4564 r = cik_cp_gfx_load_microcode(rdev);
4565 if (r)
4566 return r;
4567 r = cik_cp_compute_load_microcode(rdev);
4568 if (r)
4569 return r;
4570
4571 return 0;
4572}
4573
841cf442
AD		 4574static void cik_cp_fini(struct radeon_device *rdev)
4575{
4576 cik_cp_gfx_fini(rdev);
4577 cik_cp_compute_fini(rdev);
4578}
4579
841cf442
AD		 4580static int cik_cp_resume(struct radeon_device *rdev)
4581{
4582 int r;
4583
4214faf6
AD		 4584 cik_enable_gui_idle_interrupt(rdev, false);
4585
841cf442
AD		 4586 r = cik_cp_load_microcode(rdev);
4587 if (r)
4588 return r;
4589
4590 r = cik_cp_gfx_resume(rdev);
4591 if (r)
4592 return r;
4593 r = cik_cp_compute_resume(rdev);
4594 if (r)
4595 return r;
4596
4214faf6
AD		 4597 cik_enable_gui_idle_interrupt(rdev, true);
4598
841cf442
AD		 4599 return 0;
4600}
4601
cc066715 4602static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 4603{
6f2043ce
AD		 4604 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4605 RREG32(GRBM_STATUS));
4606 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4607 RREG32(GRBM_STATUS2));
4608 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4609 RREG32(GRBM_STATUS_SE0));
4610 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4611 RREG32(GRBM_STATUS_SE1));
4612 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4613 RREG32(GRBM_STATUS_SE2));
4614 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4615 RREG32(GRBM_STATUS_SE3));
4616 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4617 RREG32(SRBM_STATUS));
4618 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4619 RREG32(SRBM_STATUS2));
cc066715
AD		 4620 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4621 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4622 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4623 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 4624 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4625 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4626 RREG32(CP_STALLED_STAT1));
4627 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4628 RREG32(CP_STALLED_STAT2));
4629 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4630 RREG32(CP_STALLED_STAT3));
4631 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4632 RREG32(CP_CPF_BUSY_STAT));
4633 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4634 RREG32(CP_CPF_STALLED_STAT1));
4635 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4636 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4637 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4638 RREG32(CP_CPC_STALLED_STAT1));
4639 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 4640}
6f2043ce 4641
21a93e13 4642/**
cc066715 4643 * cik_gpu_check_soft_reset - check which blocks are busy
21a93e13
AD		 4644 *
4645 * @rdev: radeon_device pointer
21a93e13 4646 *
cc066715
AD		 4647 * Check which blocks are busy and return the relevant reset
4648 * mask to be used by cik_gpu_soft_reset().
4649 * Returns a mask of the blocks to be reset.
21a93e13 4650 */
2483b4ea 4651u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
21a93e13 4652{
cc066715
AD		 4653 u32 reset_mask = 0;
4654 u32 tmp;
21a93e13 4655
cc066715
AD		 4656 /* GRBM_STATUS */
4657 tmp = RREG32(GRBM_STATUS);
4658 if (tmp & (PA_BUSY | SC_BUSY |
4659 BCI_BUSY | SX_BUSY |
4660 TA_BUSY | VGT_BUSY |
4661 DB_BUSY | CB_BUSY |
4662 GDS_BUSY | SPI_BUSY |
4663 IA_BUSY | IA_BUSY_NO_DMA))
4664 reset_mask |= RADEON_RESET_GFX;
21a93e13 4665
cc066715
AD		 4666 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4667 reset_mask |= RADEON_RESET_CP;
21a93e13 4668
cc066715
AD		 4669 /* GRBM_STATUS2 */
4670 tmp = RREG32(GRBM_STATUS2);
4671 if (tmp & RLC_BUSY)
4672 reset_mask |= RADEON_RESET_RLC;
21a93e13 4673
cc066715
AD		 4674 /* SDMA0_STATUS_REG */
4675 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4676 if (!(tmp & SDMA_IDLE))
4677 reset_mask |= RADEON_RESET_DMA;
21a93e13 4678
cc066715
AD		 4679 /* SDMA1_STATUS_REG */
4680 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4681 if (!(tmp & SDMA_IDLE))
4682 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4683
cc066715
AD		 4684 /* SRBM_STATUS2 */
4685 tmp = RREG32(SRBM_STATUS2);
4686 if (tmp & SDMA_BUSY)
4687 reset_mask |= RADEON_RESET_DMA;
21a93e13 4688
cc066715
AD		 4689 if (tmp & SDMA1_BUSY)
4690 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4691
cc066715
AD		 4692 /* SRBM_STATUS */
4693 tmp = RREG32(SRBM_STATUS);
21a93e13 4694
cc066715
AD		 4695 if (tmp & IH_BUSY)
4696 reset_mask |= RADEON_RESET_IH;
21a93e13 4697
cc066715
AD		 4698 if (tmp & SEM_BUSY)
4699 reset_mask |= RADEON_RESET_SEM;
21a93e13 4700
cc066715
AD		 4701 if (tmp & GRBM_RQ_PENDING)
4702 reset_mask |= RADEON_RESET_GRBM;
21a93e13 4703
cc066715
AD		 4704 if (tmp & VMC_BUSY)
4705 reset_mask |= RADEON_RESET_VMC;
21a93e13 4706
cc066715
AD		 4707 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4708 MCC_BUSY | MCD_BUSY))
4709 reset_mask |= RADEON_RESET_MC;
21a93e13 4710
cc066715
AD		 4711 if (evergreen_is_display_hung(rdev))
4712 reset_mask |= RADEON_RESET_DISPLAY;
4713
4714 /* Skip MC reset as it's mostly likely not hung, just busy */
4715 if (reset_mask & RADEON_RESET_MC) {
4716 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4717 reset_mask &= ~RADEON_RESET_MC;
21a93e13 4718 }
cc066715
AD		 4719
4720 return reset_mask;
21a93e13
AD		 4721}
4722
4723/**
cc066715 4724 * cik_gpu_soft_reset - soft reset GPU
21a93e13
AD		 4725 *
4726 * @rdev: radeon_device pointer
cc066715 4727 * @reset_mask: mask of which blocks to reset
21a93e13 4728 *
cc066715 4729 * Soft reset the blocks specified in @reset_mask.
21a93e13 4730 */
cc066715 4731static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
21a93e13 4732{
6f2043ce 4733 struct evergreen_mc_save save;
cc066715
AD		 4734 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4735 u32 tmp;
21a93e13 4736
cc066715
AD		 4737 if (reset_mask == 0)
4738 return;
21a93e13 4739
cc066715 4740 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
21a93e13 4741
cc066715
AD		 4742 cik_print_gpu_status_regs(rdev);
4743 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4744 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4745 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4746 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
21a93e13 4747
fb2c7f4d
AD		 4748 /* disable CG/PG */
4749 cik_fini_pg(rdev);
4750 cik_fini_cg(rdev);
4751
cc066715
AD		 4752 /* stop the rlc */
4753 cik_rlc_stop(rdev);
21a93e13 4754
cc066715
AD		 4755 /* Disable GFX parsing/prefetching */
4756 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
21a93e13 4757
cc066715
AD		 4758 /* Disable MEC parsing/prefetching */
4759 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
21a93e13 4760
cc066715
AD		 4761 if (reset_mask & RADEON_RESET_DMA) {
4762 /* sdma0 */
4763 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4764 tmp |= SDMA_HALT;
4765 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4766 }
4767 if (reset_mask & RADEON_RESET_DMA1) {
4768 /* sdma1 */
4769 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4770 tmp |= SDMA_HALT;
4771 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4772 }
21a93e13 4773
6f2043ce 4774 evergreen_mc_stop(rdev, &save);
cc066715 4775 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 4776 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4777 }
21a93e13 4778
cc066715
AD		 4779 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4780 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
21a93e13 4781
cc066715
AD		 4782 if (reset_mask & RADEON_RESET_CP) {
4783 grbm_soft_reset |= SOFT_RESET_CP;
21a93e13 4784
cc066715
AD		 4785 srbm_soft_reset |= SOFT_RESET_GRBM;
4786 }
21a93e13 4787
cc066715
AD		 4788 if (reset_mask & RADEON_RESET_DMA)
4789 srbm_soft_reset |= SOFT_RESET_SDMA;
21a93e13 4790
cc066715
AD		 4791 if (reset_mask & RADEON_RESET_DMA1)
4792 srbm_soft_reset |= SOFT_RESET_SDMA1;
4793
4794 if (reset_mask & RADEON_RESET_DISPLAY)
4795 srbm_soft_reset |= SOFT_RESET_DC;
4796
4797 if (reset_mask & RADEON_RESET_RLC)
4798 grbm_soft_reset |= SOFT_RESET_RLC;
4799
4800 if (reset_mask & RADEON_RESET_SEM)
4801 srbm_soft_reset |= SOFT_RESET_SEM;
4802
4803 if (reset_mask & RADEON_RESET_IH)
4804 srbm_soft_reset |= SOFT_RESET_IH;
4805
4806 if (reset_mask & RADEON_RESET_GRBM)
4807 srbm_soft_reset |= SOFT_RESET_GRBM;
4808
4809 if (reset_mask & RADEON_RESET_VMC)
4810 srbm_soft_reset |= SOFT_RESET_VMC;
4811
4812 if (!(rdev->flags & RADEON_IS_IGP)) {
4813 if (reset_mask & RADEON_RESET_MC)
4814 srbm_soft_reset |= SOFT_RESET_MC;
21a93e13
AD		 4815 }
4816
cc066715
AD		 4817 if (grbm_soft_reset) {
4818 tmp = RREG32(GRBM_SOFT_RESET);
4819 tmp |= grbm_soft_reset;
4820 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4821 WREG32(GRBM_SOFT_RESET, tmp);
4822 tmp = RREG32(GRBM_SOFT_RESET);
21a93e13 4823
cc066715 4824 udelay(50);
21a93e13 4825
cc066715
AD		 4826 tmp &= ~grbm_soft_reset;
4827 WREG32(GRBM_SOFT_RESET, tmp);
4828 tmp = RREG32(GRBM_SOFT_RESET);
4829 }
21a93e13 4830
cc066715
AD		 4831 if (srbm_soft_reset) {
4832 tmp = RREG32(SRBM_SOFT_RESET);
4833 tmp |= srbm_soft_reset;
4834 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4835 WREG32(SRBM_SOFT_RESET, tmp);
4836 tmp = RREG32(SRBM_SOFT_RESET);
21a93e13 4837
cc066715 4838 udelay(50);
21a93e13 4839
cc066715
AD		 4840 tmp &= ~srbm_soft_reset;
4841 WREG32(SRBM_SOFT_RESET, tmp);
4842 tmp = RREG32(SRBM_SOFT_RESET);
4843 }
21a93e13 4844
6f2043ce
AD		 4845 /* Wait a little for things to settle down */
4846 udelay(50);
21a93e13 4847
6f2043ce 4848 evergreen_mc_resume(rdev, &save);
cc066715
AD		 4849 udelay(50);
4850
4851 cik_print_gpu_status_regs(rdev);
21a93e13
AD		 4852}
4853
4854/**
cc066715 4855 * cik_asic_reset - soft reset GPU
21a93e13
AD		 4856 *
4857 * @rdev: radeon_device pointer
4858 *
cc066715
AD		 4859 * Look up which blocks are hung and attempt
4860 * to reset them.
6f2043ce 4861 * Returns 0 for success.
21a93e13 4862 */
6f2043ce 4863int cik_asic_reset(struct radeon_device *rdev)
21a93e13 4864{
cc066715 4865 u32 reset_mask;
21a93e13 4866
cc066715 4867 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 4868
cc066715
AD		 4869 if (reset_mask)
4870 r600_set_bios_scratch_engine_hung(rdev, true);
21a93e13 4871
cc066715 4872 cik_gpu_soft_reset(rdev, reset_mask);
21a93e13 4873
cc066715
AD		 4874 reset_mask = cik_gpu_check_soft_reset(rdev);
4875
4876 if (!reset_mask)
4877 r600_set_bios_scratch_engine_hung(rdev, false);
21a93e13
AD		 4878
4879 return 0;
4880}
4881
4882/**
cc066715 4883 * cik_gfx_is_lockup - check if the 3D engine is locked up
21a93e13
AD		 4884 *
4885 * @rdev: radeon_device pointer
cc066715 4886 * @ring: radeon_ring structure holding ring information
21a93e13 4887 *
cc066715
AD		 4888 * Check if the 3D engine is locked up (CIK).
4889 * Returns true if the engine is locked, false if not.
21a93e13 4890 */
cc066715 4891bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
21a93e13 4892{
cc066715 4893 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 4894
cc066715
AD		 4895 if (!(reset_mask & (RADEON_RESET_GFX |
4896 RADEON_RESET_COMPUTE |
4897 RADEON_RESET_CP))) {
4898 radeon_ring_lockup_update(ring);
4899 return false;
21a93e13 4900 }
cc066715
AD		 4901 /* force CP activities */
4902 radeon_ring_force_activity(rdev, ring);
4903 return radeon_ring_test_lockup(rdev, ring);
21a93e13
AD		 4904}
4905
1c49165d 4906/* MC */
21a93e13 4907/**
1c49165d 4908 * cik_mc_program - program the GPU memory controller
21a93e13
AD		 4909 *
4910 * @rdev: radeon_device pointer
21a93e13 4911 *
1c49165d
AD		 4912 * Set the location of vram, gart, and AGP in the GPU's
4913 * physical address space (CIK).
21a93e13 4914 */
1c49165d 4915static void cik_mc_program(struct radeon_device *rdev)
21a93e13 4916{
1c49165d 4917 struct evergreen_mc_save save;
21a93e13 4918 u32 tmp;
1c49165d 4919 int i, j;
21a93e13 4920
1c49165d
AD		 4921 /* Initialize HDP */
4922 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4923 WREG32((0x2c14 + j), 0x00000000);
4924 WREG32((0x2c18 + j), 0x00000000);
4925 WREG32((0x2c1c + j), 0x00000000);
4926 WREG32((0x2c20 + j), 0x00000000);
4927 WREG32((0x2c24 + j), 0x00000000);
21a93e13 4928 }
1c49165d 4929 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
21a93e13 4930
1c49165d
AD		 4931 evergreen_mc_stop(rdev, &save);
4932 if (radeon_mc_wait_for_idle(rdev)) {
4933 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 4934 }
1c49165d
AD		 4935 /* Lockout access through VGA aperture*/
4936 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4937 /* Update configuration */
4938 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4939 rdev->mc.vram_start >> 12);
4940 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4941 rdev->mc.vram_end >> 12);
4942 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4943 rdev->vram_scratch.gpu_addr >> 12);
4944 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4945 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4946 WREG32(MC_VM_FB_LOCATION, tmp);
4947 /* XXX double check these! */
4948 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4949 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4950 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4951 WREG32(MC_VM_AGP_BASE, 0);
4952 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4953 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4954 if (radeon_mc_wait_for_idle(rdev)) {
4955 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 4956 }
1c49165d
AD		 4957 evergreen_mc_resume(rdev, &save);
4958 /* we need to own VRAM, so turn off the VGA renderer here
4959 * to stop it overwriting our objects */
4960 rv515_vga_render_disable(rdev);
21a93e13
AD		 4961}
4962
4963/**
1c49165d 4964 * cik_mc_init - initialize the memory controller driver params
21a93e13
AD		 4965 *
4966 * @rdev: radeon_device pointer
21a93e13 4967 *
1c49165d
AD		 4968 * Look up the amount of vram, vram width, and decide how to place
4969 * vram and gart within the GPU's physical address space (CIK).
4970 * Returns 0 for success.
21a93e13 4971 */
1c49165d 4972static int cik_mc_init(struct radeon_device *rdev)
21a93e13 4973{
1c49165d
AD		 4974 u32 tmp;
4975 int chansize, numchan;
21a93e13 4976
1c49165d
AD		 4977 /* Get VRAM informations */
4978 rdev->mc.vram_is_ddr = true;
4979 tmp = RREG32(MC_ARB_RAMCFG);
4980 if (tmp & CHANSIZE_MASK) {
4981 chansize = 64;
21a93e13 4982 } else {
1c49165d 4983 chansize = 32;
21a93e13 4984 }
1c49165d
AD		 4985 tmp = RREG32(MC_SHARED_CHMAP);
4986 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4987 case 0:
4988 default:
4989 numchan = 1;
4990 break;
4991 case 1:
4992 numchan = 2;
4993 break;
4994 case 2:
4995 numchan = 4;
4996 break;
4997 case 3:
4998 numchan = 8;
4999 break;
5000 case 4:
5001 numchan = 3;
5002 break;
5003 case 5:
5004 numchan = 6;
5005 break;
5006 case 6:
5007 numchan = 10;
5008 break;
5009 case 7:
5010 numchan = 12;
5011 break;
5012 case 8:
5013 numchan = 16;
5014 break;
5015 }
5016 rdev->mc.vram_width = numchan * chansize;
5017 /* Could aper size report 0 ? */
5018 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5019 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5020 /* size in MB on si */
13c5bfda
AD		 5021 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5022 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1c49165d
AD		 5023 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5024 si_vram_gtt_location(rdev, &rdev->mc);
5025 radeon_update_bandwidth_info(rdev);
5026
5027 return 0;
5028}
5029
5030/*
5031 * GART
5032 * VMID 0 is the physical GPU addresses as used by the kernel.
5033 * VMIDs 1-15 are used for userspace clients and are handled
5034 * by the radeon vm/hsa code.
5035 */
5036/**
5037 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5038 *
5039 * @rdev: radeon_device pointer
5040 *
5041 * Flush the TLB for the VMID 0 page table (CIK).
5042 */
5043void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5044{
5045 /* flush hdp cache */
5046 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5047
5048 /* bits 0-15 are the VM contexts0-15 */
5049 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5050}
5051
5052/**
5053 * cik_pcie_gart_enable - gart enable
5054 *
5055 * @rdev: radeon_device pointer
5056 *
5057 * This sets up the TLBs, programs the page tables for VMID0,
5058 * sets up the hw for VMIDs 1-15 which are allocated on
5059 * demand, and sets up the global locations for the LDS, GDS,
5060 * and GPUVM for FSA64 clients (CIK).
5061 * Returns 0 for success, errors for failure.
5062 */
5063static int cik_pcie_gart_enable(struct radeon_device *rdev)
5064{
5065 int r, i;
5066
5067 if (rdev->gart.robj == NULL) {
5068 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5069 return -EINVAL;
5070 }
5071 r = radeon_gart_table_vram_pin(rdev);
5072 if (r)
5073 return r;
5074 radeon_gart_restore(rdev);
5075 /* Setup TLB control */
5076 WREG32(MC_VM_MX_L1_TLB_CNTL,
5077 (0xA << 7) |
5078 ENABLE_L1_TLB |
5079 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5080 ENABLE_ADVANCED_DRIVER_MODEL |
5081 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5082 /* Setup L2 cache */
5083 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5084 ENABLE_L2_FRAGMENT_PROCESSING |
5085 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5086 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5087 EFFECTIVE_L2_QUEUE_SIZE(7) |
5088 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5089 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5090 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5091 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5092 /* setup context0 */
5093 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5094 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5095 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5096 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5097 (u32)(rdev->dummy_page.addr >> 12));
5098 WREG32(VM_CONTEXT0_CNTL2, 0);
5099 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5100 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5101
5102 WREG32(0x15D4, 0);
5103 WREG32(0x15D8, 0);
5104 WREG32(0x15DC, 0);
5105
5106 /* empty context1-15 */
5107 /* FIXME start with 4G, once using 2 level pt switch to full
5108 * vm size space
5109 */
5110 /* set vm size, must be a multiple of 4 */
5111 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5112 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5113 for (i = 1; i < 16; i++) {
5114 if (i < 8)
5115 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5116 rdev->gart.table_addr >> 12);
5117 else
5118 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5119 rdev->gart.table_addr >> 12);
5120 }
5121
5122 /* enable context1-15 */
5123 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5124 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 5125 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 5126 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
a00024b0
AD		 5127 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5128 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5129 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5130 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5131 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5132 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5133 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5134 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5135 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5136 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5137 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5138 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d
AD		 5139
5140 /* TC cache setup ??? */
5141 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
5142 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
5143 WREG32(TC_CFG_L1_STORE_POLICY, 0);
5144
5145 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
5146 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
5147 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
5148 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
5149 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
5150
5151 WREG32(TC_CFG_L1_VOLATILE, 0);
5152 WREG32(TC_CFG_L2_VOLATILE, 0);
5153
5154 if (rdev->family == CHIP_KAVERI) {
5155 u32 tmp = RREG32(CHUB_CONTROL);
5156 tmp &= ~BYPASS_VM;
5157 WREG32(CHUB_CONTROL, tmp);
5158 }
5159
5160 /* XXX SH_MEM regs */
5161 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 5162 mutex_lock(&rdev->srbm_mutex);
1c49165d 5163 for (i = 0; i < 16; i++) {
b556b12e 5164 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 5165 /* CP and shaders */
1c49165d
AD		 5166 WREG32(SH_MEM_CONFIG, 0);
5167 WREG32(SH_MEM_APE1_BASE, 1);
5168 WREG32(SH_MEM_APE1_LIMIT, 0);
5169 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 5170 /* SDMA GFX */
5171 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5172 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5173 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5174 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5175 /* XXX SDMA RLC - todo */
1c49165d 5176 }
b556b12e 5177 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5178 mutex_unlock(&rdev->srbm_mutex);
1c49165d
AD		 5179
5180 cik_pcie_gart_tlb_flush(rdev);
5181 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5182 (unsigned)(rdev->mc.gtt_size >> 20),
5183 (unsigned long long)rdev->gart.table_addr);
5184 rdev->gart.ready = true;
5185 return 0;
5186}
5187
5188/**
5189 * cik_pcie_gart_disable - gart disable
5190 *
5191 * @rdev: radeon_device pointer
5192 *
5193 * This disables all VM page table (CIK).
5194 */
5195static void cik_pcie_gart_disable(struct radeon_device *rdev)
5196{
5197 /* Disable all tables */
5198 WREG32(VM_CONTEXT0_CNTL, 0);
5199 WREG32(VM_CONTEXT1_CNTL, 0);
5200 /* Setup TLB control */
5201 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5202 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5203 /* Setup L2 cache */
5204 WREG32(VM_L2_CNTL,
5205 ENABLE_L2_FRAGMENT_PROCESSING |
5206 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5207 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5208 EFFECTIVE_L2_QUEUE_SIZE(7) |
5209 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5210 WREG32(VM_L2_CNTL2, 0);
5211 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5212 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5213 radeon_gart_table_vram_unpin(rdev);
5214}
5215
5216/**
5217 * cik_pcie_gart_fini - vm fini callback
5218 *
5219 * @rdev: radeon_device pointer
5220 *
5221 * Tears down the driver GART/VM setup (CIK).
5222 */
5223static void cik_pcie_gart_fini(struct radeon_device *rdev)
5224{
5225 cik_pcie_gart_disable(rdev);
5226 radeon_gart_table_vram_free(rdev);
5227 radeon_gart_fini(rdev);
5228}
5229
5230/* vm parser */
5231/**
5232 * cik_ib_parse - vm ib_parse callback
5233 *
5234 * @rdev: radeon_device pointer
5235 * @ib: indirect buffer pointer
5236 *
5237 * CIK uses hw IB checking so this is a nop (CIK).
5238 */
5239int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5240{
5241 return 0;
5242}
5243
5244/*
5245 * vm
5246 * VMID 0 is the physical GPU addresses as used by the kernel.
5247 * VMIDs 1-15 are used for userspace clients and are handled
5248 * by the radeon vm/hsa code.
5249 */
5250/**
5251 * cik_vm_init - cik vm init callback
5252 *
5253 * @rdev: radeon_device pointer
5254 *
5255 * Inits cik specific vm parameters (number of VMs, base of vram for
5256 * VMIDs 1-15) (CIK).
5257 * Returns 0 for success.
5258 */
5259int cik_vm_init(struct radeon_device *rdev)
5260{
5261 /* number of VMs */
5262 rdev->vm_manager.nvm = 16;
5263 /* base offset of vram pages */
5264 if (rdev->flags & RADEON_IS_IGP) {
5265 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5266 tmp <<= 22;
5267 rdev->vm_manager.vram_base_offset = tmp;
5268 } else
5269 rdev->vm_manager.vram_base_offset = 0;
5270
5271 return 0;
5272}
5273
5274/**
5275 * cik_vm_fini - cik vm fini callback
5276 *
5277 * @rdev: radeon_device pointer
5278 *
5279 * Tear down any asic specific VM setup (CIK).
5280 */
5281void cik_vm_fini(struct radeon_device *rdev)
5282{
5283}
5284
3ec7d11b
AD		 5285/**
5286 * cik_vm_decode_fault - print human readable fault info
5287 *
5288 * @rdev: radeon_device pointer
5289 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5290 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5291 *
5292 * Print human readable fault information (CIK).
5293 */
5294static void cik_vm_decode_fault(struct radeon_device *rdev,
5295 u32 status, u32 addr, u32 mc_client)
5296{
939c0d3c 5297 u32 mc_id;
3ec7d11b
AD		 5298 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5299 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
328a50c7
MD		 5300 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5301 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
3ec7d11b 5302
939c0d3c
AD		 5303 if (rdev->family == CHIP_HAWAII)
5304 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5305 else
5306 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5307
328a50c7 5308 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
3ec7d11b
AD		 5309 protections, vmid, addr,
5310 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
328a50c7 5311 block, mc_client, mc_id);
3ec7d11b
AD		 5312}
5313
f96ab484
AD		 5314/**
5315 * cik_vm_flush - cik vm flush using the CP
5316 *
5317 * @rdev: radeon_device pointer
5318 *
5319 * Update the page table base and flush the VM TLB
5320 * using the CP (CIK).
5321 */
5322void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5323{
5324 struct radeon_ring *ring = &rdev->ring[ridx];
5325
5326 if (vm == NULL)
5327 return;
5328
5329 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5330 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5331 WRITE_DATA_DST_SEL(0)));
5332 if (vm->id < 8) {
5333 radeon_ring_write(ring,
5334 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5335 } else {
5336 radeon_ring_write(ring,
5337 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5338 }
5339 radeon_ring_write(ring, 0);
5340 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5341
5342 /* update SH_MEM_* regs */
5343 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5344 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5345 WRITE_DATA_DST_SEL(0)));
5346 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5347 radeon_ring_write(ring, 0);
5348 radeon_ring_write(ring, VMID(vm->id));
5349
5350 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5351 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5352 WRITE_DATA_DST_SEL(0)));
5353 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5354 radeon_ring_write(ring, 0);
5355
5356 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5357 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5358 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5359 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
21a93e13 5360
f96ab484
AD		 5361 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5362 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5363 WRITE_DATA_DST_SEL(0)));
5364 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5365 radeon_ring_write(ring, 0);
5366 radeon_ring_write(ring, VMID(0));
6f2043ce 5367
f96ab484
AD		 5368 /* HDP flush */
5369 /* We should be using the WAIT_REG_MEM packet here like in
5370 * cik_fence_ring_emit(), but it causes the CP to hang in this
5371 * context...
5372 */
5373 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5374 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5375 WRITE_DATA_DST_SEL(0)));
5376 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5377 radeon_ring_write(ring, 0);
5378 radeon_ring_write(ring, 0);
5379
5380 /* bits 0-15 are the VM contexts0-15 */
5381 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5382 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5383 WRITE_DATA_DST_SEL(0)));
5384 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5385 radeon_ring_write(ring, 0);
5386 radeon_ring_write(ring, 1 << vm->id);
5387
b07fdd38
AD		 5388 /* compute doesn't have PFP */
5389 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5390 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5391 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5392 radeon_ring_write(ring, 0x0);
5393 }
cc066715 5394}
6f2043ce 5395
f6796cae
AD		 5396/*
5397 * RLC
5398 * The RLC is a multi-purpose microengine that handles a
5399 * variety of functions, the most important of which is
5400 * the interrupt controller.
5401 */
866d83de
AD		 5402static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5403 bool enable)
f6796cae 5404{
866d83de 5405 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 5406
866d83de
AD		 5407 if (enable)
5408 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5409 else
5410 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 5411 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 5412}
f6796cae 5413
866d83de 5414static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
cc066715 5415{
cc066715 5416 u32 tmp;
6f2043ce 5417
866d83de
AD		 5418 tmp = RREG32(RLC_LB_CNTL);
5419 if (enable)
5420 tmp |= LOAD_BALANCE_ENABLE;
5421 else
5422 tmp &= ~LOAD_BALANCE_ENABLE;
5423 WREG32(RLC_LB_CNTL, tmp);
5424}
cc066715 5425
866d83de
AD		 5426static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5427{
5428 u32 i, j, k;
5429 u32 mask;
cc066715 5430
f6796cae
AD		 5431 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5432 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5433 cik_select_se_sh(rdev, i, j);
5434 for (k = 0; k < rdev->usec_timeout; k++) {
5435 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5436 break;
5437 udelay(1);
5438 }
5439 }
5440 }
5441 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
cc066715 5442
f6796cae
AD		 5443 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5444 for (k = 0; k < rdev->usec_timeout; k++) {
5445 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5446 break;
5447 udelay(1);
5448 }
5449}
cc066715 5450
22c775ce
AD		 5451static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5452{
5453 u32 tmp;
cc066715 5454
22c775ce
AD		 5455 tmp = RREG32(RLC_CNTL);
5456 if (tmp != rlc)
5457 WREG32(RLC_CNTL, rlc);
5458}
cc066715 5459
22c775ce
AD		 5460static u32 cik_halt_rlc(struct radeon_device *rdev)
5461{
5462 u32 data, orig;
cc066715 5463
22c775ce 5464 orig = data = RREG32(RLC_CNTL);
cc066715 5465
22c775ce
AD		 5466 if (data & RLC_ENABLE) {
5467 u32 i;
cc066715 5468
22c775ce
AD		 5469 data &= ~RLC_ENABLE;
5470 WREG32(RLC_CNTL, data);
cc066715 5471
22c775ce
AD		 5472 for (i = 0; i < rdev->usec_timeout; i++) {
5473 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5474 break;
5475 udelay(1);
5476 }
cc066715 5477
22c775ce
AD		 5478 cik_wait_for_rlc_serdes(rdev);
5479 }
cc066715 5480
22c775ce
AD		 5481 return orig;
5482}
cc066715 5483
a412fce0
AD		 5484void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5485{
5486 u32 tmp, i, mask;
5487
5488 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5489 WREG32(RLC_GPR_REG2, tmp);
5490
5491 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5492 for (i = 0; i < rdev->usec_timeout; i++) {
5493 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5494 break;
5495 udelay(1);
5496 }
5497
5498 for (i = 0; i < rdev->usec_timeout; i++) {
5499 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5500 break;
5501 udelay(1);
5502 }
5503}
5504
5505void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5506{
5507 u32 tmp;
5508
5509 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5510 WREG32(RLC_GPR_REG2, tmp);
5511}
5512
866d83de
AD		 5513/**
5514 * cik_rlc_stop - stop the RLC ME
5515 *
5516 * @rdev: radeon_device pointer
5517 *
5518 * Halt the RLC ME (MicroEngine) (CIK).
5519 */
5520static void cik_rlc_stop(struct radeon_device *rdev)
5521{
22c775ce 5522 WREG32(RLC_CNTL, 0);
866d83de
AD		 5523
5524 cik_enable_gui_idle_interrupt(rdev, false);
5525
866d83de
AD		 5526 cik_wait_for_rlc_serdes(rdev);
5527}
5528
f6796cae
AD		 5529/**
5530 * cik_rlc_start - start the RLC ME
5531 *
5532 * @rdev: radeon_device pointer
5533 *
5534 * Unhalt the RLC ME (MicroEngine) (CIK).
5535 */
5536static void cik_rlc_start(struct radeon_device *rdev)
5537{
f6796cae 5538 WREG32(RLC_CNTL, RLC_ENABLE);
cc066715 5539
866d83de 5540 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 5541
f6796cae 5542 udelay(50);
6f2043ce
AD		 5543}
5544
5545/**
f6796cae 5546 * cik_rlc_resume - setup the RLC hw
6f2043ce
AD		 5547 *
5548 * @rdev: radeon_device pointer
5549 *
f6796cae
AD		 5550 * Initialize the RLC registers, load the ucode,
5551 * and start the RLC (CIK).
5552 * Returns 0 for success, -EINVAL if the ucode is not available.
6f2043ce 5553 */
f6796cae 5554static int cik_rlc_resume(struct radeon_device *rdev)
6f2043ce 5555{
22c775ce 5556 u32 i, size, tmp;
f6796cae 5557 const __be32 *fw_data;
cc066715 5558
f6796cae
AD		 5559 if (!rdev->rlc_fw)
5560 return -EINVAL;
cc066715 5561
f6796cae
AD		 5562 switch (rdev->family) {
5563 case CHIP_BONAIRE:
d4775655 5564 case CHIP_HAWAII:
f6796cae
AD		 5565 default:
5566 size = BONAIRE_RLC_UCODE_SIZE;
5567 break;
5568 case CHIP_KAVERI:
5569 size = KV_RLC_UCODE_SIZE;
5570 break;
5571 case CHIP_KABINI:
5572 size = KB_RLC_UCODE_SIZE;
5573 break;
5574 }
cc066715 5575
cc066715
AD		 5576 cik_rlc_stop(rdev);
5577
22c775ce
AD		 5578 /* disable CG */
5579 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5580 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
cc066715 5581
866d83de 5582 si_rlc_reset(rdev);
6f2043ce 5583
22c775ce 5584 cik_init_pg(rdev);
6f2043ce 5585
22c775ce 5586 cik_init_cg(rdev);
cc066715 5587
f6796cae
AD		 5588 WREG32(RLC_LB_CNTR_INIT, 0);
5589 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
cc066715 5590
f6796cae
AD		 5591 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5592 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5593 WREG32(RLC_LB_PARAMS, 0x00600408);
5594 WREG32(RLC_LB_CNTL, 0x80000004);
cc066715 5595
f6796cae
AD		 5596 WREG32(RLC_MC_CNTL, 0);
5597 WREG32(RLC_UCODE_CNTL, 0);
cc066715 5598
f6796cae
AD		 5599 fw_data = (const __be32 *)rdev->rlc_fw->data;
5600 WREG32(RLC_GPM_UCODE_ADDR, 0);
5601 for (i = 0; i < size; i++)
5602 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5603 WREG32(RLC_GPM_UCODE_ADDR, 0);
cc066715 5604
866d83de
AD		 5605 /* XXX - find out what chips support lbpw */
5606 cik_enable_lbpw(rdev, false);
cc066715 5607
22c775ce
AD		 5608 if (rdev->family == CHIP_BONAIRE)
5609 WREG32(RLC_DRIVER_DMA_STATUS, 0);
cc066715 5610
f6796cae 5611 cik_rlc_start(rdev);
cc066715 5612
f6796cae
AD		 5613 return 0;
5614}
cc066715 5615
22c775ce
AD		 5616static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5617{
5618 u32 data, orig, tmp, tmp2;
cc066715 5619
22c775ce 5620 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
cc066715 5621
473359bc 5622 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6 5623 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 5624
22c775ce 5625 tmp = cik_halt_rlc(rdev);
cc066715 5626
22c775ce
AD		 5627 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5628 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5629 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5630 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5631 WREG32(RLC_SERDES_WR_CTRL, tmp2);
cc066715 5632
22c775ce 5633 cik_update_rlc(rdev, tmp);
cc066715 5634
22c775ce
AD		 5635 data |= CGCG_EN | CGLS_EN;
5636 } else {
ddc76ff6 5637 cik_enable_gui_idle_interrupt(rdev, false);
cc066715 5638
22c775ce
AD		 5639 RREG32(CB_CGTT_SCLK_CTRL);
5640 RREG32(CB_CGTT_SCLK_CTRL);
5641 RREG32(CB_CGTT_SCLK_CTRL);
5642 RREG32(CB_CGTT_SCLK_CTRL);
cc066715 5643
22c775ce 5644 data &= ~(CGCG_EN | CGLS_EN);
cc066715 5645 }
6f2043ce 5646
22c775ce
AD		 5647 if (orig != data)
5648 WREG32(RLC_CGCG_CGLS_CTRL, data);
cc066715 5649
6f2043ce
AD		 5650}
5651
22c775ce 5652static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6f2043ce 5653{
22c775ce
AD		 5654 u32 data, orig, tmp = 0;
5655
473359bc
AD		 5656 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5657 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5658 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5659 orig = data = RREG32(CP_MEM_SLP_CNTL);
5660 data |= CP_MEM_LS_EN;
5661 if (orig != data)
5662 WREG32(CP_MEM_SLP_CNTL, data);
5663 }
5664 }
cc066715 5665
22c775ce
AD		 5666 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5667 data &= 0xfffffffd;
5668 if (orig != data)
5669 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5670
5671 tmp = cik_halt_rlc(rdev);
5672
5673 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5674 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5675 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5676 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5677 WREG32(RLC_SERDES_WR_CTRL, data);
5678
5679 cik_update_rlc(rdev, tmp);
5680
473359bc
AD		 5681 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5682 orig = data = RREG32(CGTS_SM_CTRL_REG);
5683 data &= ~SM_MODE_MASK;
5684 data |= SM_MODE(0x2);
5685 data |= SM_MODE_ENABLE;
5686 data &= ~CGTS_OVERRIDE;
5687 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5688 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5689 data &= ~CGTS_LS_OVERRIDE;
5690 data &= ~ON_MONITOR_ADD_MASK;
5691 data |= ON_MONITOR_ADD_EN;
5692 data |= ON_MONITOR_ADD(0x96);
5693 if (orig != data)
5694 WREG32(CGTS_SM_CTRL_REG, data);
5695 }
22c775ce
AD		 5696 } else {
5697 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5698 data |= 0x00000002;
5699 if (orig != data)
5700 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5701
5702 data = RREG32(RLC_MEM_SLP_CNTL);
5703 if (data & RLC_MEM_LS_EN) {
5704 data &= ~RLC_MEM_LS_EN;
5705 WREG32(RLC_MEM_SLP_CNTL, data);
5706 }
6f2043ce 5707
22c775ce
AD		 5708 data = RREG32(CP_MEM_SLP_CNTL);
5709 if (data & CP_MEM_LS_EN) {
5710 data &= ~CP_MEM_LS_EN;
5711 WREG32(CP_MEM_SLP_CNTL, data);
5712 }
cc066715 5713
22c775ce
AD		 5714 orig = data = RREG32(CGTS_SM_CTRL_REG);
5715 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5716 if (orig != data)
5717 WREG32(CGTS_SM_CTRL_REG, data);
cc066715 5718
22c775ce 5719 tmp = cik_halt_rlc(rdev);
cc066715 5720
22c775ce
AD		 5721 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5722 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5723 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5724 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5725 WREG32(RLC_SERDES_WR_CTRL, data);
cc066715 5726
22c775ce 5727 cik_update_rlc(rdev, tmp);
cc066715 5728 }
6f2043ce 5729}
1c49165d 5730
22c775ce 5731static const u32 mc_cg_registers[] =
21a93e13 5732{
22c775ce
AD		 5733 MC_HUB_MISC_HUB_CG,
5734 MC_HUB_MISC_SIP_CG,
5735 MC_HUB_MISC_VM_CG,
5736 MC_XPB_CLK_GAT,
5737 ATC_MISC_CG,
5738 MC_CITF_MISC_WR_CG,
5739 MC_CITF_MISC_RD_CG,
5740 MC_CITF_MISC_VM_CG,
5741 VM_L2_CG,
5742};
21a93e13 5743
22c775ce
AD		 5744static void cik_enable_mc_ls(struct radeon_device *rdev,
5745 bool enable)
1c49165d 5746{
22c775ce
AD		 5747 int i;
5748 u32 orig, data;
1c49165d 5749
22c775ce
AD		 5750 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5751 orig = data = RREG32(mc_cg_registers[i]);
473359bc 5752 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 5753 data |= MC_LS_ENABLE;
5754 else
5755 data &= ~MC_LS_ENABLE;
5756 if (data != orig)
5757 WREG32(mc_cg_registers[i], data);
1c49165d 5758 }
22c775ce 5759}
1c49165d 5760
22c775ce
AD		 5761static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5762 bool enable)
5763{
5764 int i;
5765 u32 orig, data;
5766
5767 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5768 orig = data = RREG32(mc_cg_registers[i]);
473359bc 5769 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 5770 data |= MC_CG_ENABLE;
5771 else
5772 data &= ~MC_CG_ENABLE;
5773 if (data != orig)
5774 WREG32(mc_cg_registers[i], data);
1c49165d 5775 }
1c49165d
AD		 5776}
5777
22c775ce
AD		 5778static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5779 bool enable)
1c49165d 5780{
22c775ce 5781 u32 orig, data;
1c49165d 5782
473359bc 5783 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 5784 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5785 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
1c49165d 5786 } else {
22c775ce
AD		 5787 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5788 data |= 0xff000000;
5789 if (data != orig)
5790 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
1c49165d 5791
22c775ce
AD		 5792 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5793 data |= 0xff000000;
5794 if (data != orig)
5795 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5796 }
1c49165d
AD		 5797}
5798
22c775ce
AD		 5799static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5800 bool enable)
1c49165d 5801{
22c775ce
AD		 5802 u32 orig, data;
5803
473359bc 5804 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 5805 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5806 data |= 0x100;
5807 if (orig != data)
5808 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5809
5810 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5811 data |= 0x100;
5812 if (orig != data)
5813 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5814 } else {
5815 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5816 data &= ~0x100;
5817 if (orig != data)
5818 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
1c49165d 5819
22c775ce
AD		 5820 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5821 data &= ~0x100;
5822 if (orig != data)
5823 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5824 }
1c49165d
AD		 5825}
5826
22c775ce
AD		 5827static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
5828 bool enable)
1c49165d 5829{
22c775ce 5830 u32 orig, data;
1c49165d 5831
473359bc 5832 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 5833 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5834 data = 0xfff;
5835 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 5836
22c775ce
AD		 5837 orig = data = RREG32(UVD_CGC_CTRL);
5838 data |= DCM;
5839 if (orig != data)
5840 WREG32(UVD_CGC_CTRL, data);
5841 } else {
5842 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5843 data &= ~0xfff;
5844 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 5845
22c775ce
AD		 5846 orig = data = RREG32(UVD_CGC_CTRL);
5847 data &= ~DCM;
5848 if (orig != data)
5849 WREG32(UVD_CGC_CTRL, data);
1c49165d 5850 }
22c775ce 5851}
1c49165d 5852
473359bc
AD		 5853static void cik_enable_bif_mgls(struct radeon_device *rdev,
5854 bool enable)
5855{
5856 u32 orig, data;
1c49165d 5857
473359bc 5858 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
1c49165d 5859
473359bc
AD		 5860 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5861 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5862 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5863 else
5864 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5865 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
1c49165d 5866
473359bc
AD		 5867 if (orig != data)
5868 WREG32_PCIE_PORT(PCIE_CNTL2, data);
5869}
1c49165d 5870
22c775ce
AD		 5871static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
5872 bool enable)
5873{
5874 u32 orig, data;
1c49165d 5875
22c775ce 5876 orig = data = RREG32(HDP_HOST_PATH_CNTL);
1c49165d 5877
473359bc 5878 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 5879 data &= ~CLOCK_GATING_DIS;
5880 else
5881 data |= CLOCK_GATING_DIS;
5882
5883 if (orig != data)
5884 WREG32(HDP_HOST_PATH_CNTL, data);
1c49165d
AD		 5885}
5886
22c775ce
AD		 5887static void cik_enable_hdp_ls(struct radeon_device *rdev,
5888 bool enable)
1c49165d 5889{
22c775ce
AD		 5890 u32 orig, data;
5891
5892 orig = data = RREG32(HDP_MEM_POWER_LS);
5893
473359bc 5894 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 5895 data |= HDP_LS_ENABLE;
5896 else
5897 data &= ~HDP_LS_ENABLE;
5898
5899 if (orig != data)
5900 WREG32(HDP_MEM_POWER_LS, data);
5901}
5902
5903void cik_update_cg(struct radeon_device *rdev,
5904 u32 block, bool enable)
5905{
4214faf6 5906
22c775ce 5907 if (block & RADEON_CG_BLOCK_GFX) {
4214faf6 5908 cik_enable_gui_idle_interrupt(rdev, false);
22c775ce
AD		 5909 /* order matters! */
5910 if (enable) {
5911 cik_enable_mgcg(rdev, true);
5912 cik_enable_cgcg(rdev, true);
5913 } else {
5914 cik_enable_cgcg(rdev, false);
5915 cik_enable_mgcg(rdev, false);
5916 }
4214faf6 5917 cik_enable_gui_idle_interrupt(rdev, true);
22c775ce
AD		 5918 }
5919
5920 if (block & RADEON_CG_BLOCK_MC) {
5921 if (!(rdev->flags & RADEON_IS_IGP)) {
5922 cik_enable_mc_mgcg(rdev, enable);
5923 cik_enable_mc_ls(rdev, enable);
5924 }
5925 }
5926
5927 if (block & RADEON_CG_BLOCK_SDMA) {
5928 cik_enable_sdma_mgcg(rdev, enable);
5929 cik_enable_sdma_mgls(rdev, enable);
5930 }
5931
473359bc
AD		 5932 if (block & RADEON_CG_BLOCK_BIF) {
5933 cik_enable_bif_mgls(rdev, enable);
5934 }
5935
22c775ce
AD		 5936 if (block & RADEON_CG_BLOCK_UVD) {
5937 if (rdev->has_uvd)
5938 cik_enable_uvd_mgcg(rdev, enable);
5939 }
5940
5941 if (block & RADEON_CG_BLOCK_HDP) {
5942 cik_enable_hdp_mgcg(rdev, enable);
5943 cik_enable_hdp_ls(rdev, enable);
5944 }
1c49165d
AD		 5945}
5946
22c775ce 5947static void cik_init_cg(struct radeon_device *rdev)
1c49165d 5948{
22c775ce 5949
ddc76ff6 5950 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 5951
5952 if (rdev->has_uvd)
5953 si_init_uvd_internal_cg(rdev);
5954
5955 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5956 RADEON_CG_BLOCK_SDMA |
473359bc 5957 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 5958 RADEON_CG_BLOCK_UVD |
5959 RADEON_CG_BLOCK_HDP), true);
1c49165d
AD		 5960}
5961
473359bc 5962static void cik_fini_cg(struct radeon_device *rdev)
1c49165d 5963{
473359bc
AD		 5964 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5965 RADEON_CG_BLOCK_SDMA |
5966 RADEON_CG_BLOCK_BIF |
5967 RADEON_CG_BLOCK_UVD |
5968 RADEON_CG_BLOCK_HDP), false);
5969
5970 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
1c49165d
AD		 5971}
5972
22c775ce
AD		 5973static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
5974 bool enable)
1c49165d 5975{
22c775ce 5976 u32 data, orig;
1c49165d 5977
22c775ce 5978 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5979 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 5980 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5981 else
5982 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5983 if (orig != data)
5984 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 5985}
5986
22c775ce
AD		 5987static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
5988 bool enable)
1c49165d 5989{
22c775ce
AD		 5990 u32 data, orig;
5991
5992 orig = data = RREG32(RLC_PG_CNTL);
473359bc 5993 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 5994 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5995 else
5996 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5997 if (orig != data)
5998 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 5999}
6000
22c775ce 6001static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
3ec7d11b 6002{
22c775ce 6003 u32 data, orig;
3ec7d11b 6004
22c775ce 6005 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6006 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 6007 data &= ~DISABLE_CP_PG;
6008 else
6009 data |= DISABLE_CP_PG;
6010 if (orig != data)
6011 WREG32(RLC_PG_CNTL, data);
3ec7d11b
AD		 6012}
6013
22c775ce 6014static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
f96ab484 6015{
22c775ce 6016 u32 data, orig;
f96ab484 6017
22c775ce 6018 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6019 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 6020 data &= ~DISABLE_GDS_PG;
6021 else
6022 data |= DISABLE_GDS_PG;
6023 if (orig != data)
6024 WREG32(RLC_PG_CNTL, data);
6025}
6026
6027#define CP_ME_TABLE_SIZE 96
6028#define CP_ME_TABLE_OFFSET 2048
6029#define CP_MEC_TABLE_OFFSET 4096
6030
6031void cik_init_cp_pg_table(struct radeon_device *rdev)
6032{
6033 const __be32 *fw_data;
6034 volatile u32 *dst_ptr;
6035 int me, i, max_me = 4;
6036 u32 bo_offset = 0;
6037 u32 table_offset;
6038
6039 if (rdev->family == CHIP_KAVERI)
6040 max_me = 5;
6041
6042 if (rdev->rlc.cp_table_ptr == NULL)
f96ab484
AD		 6043 return;
6044
22c775ce
AD		 6045 /* write the cp table buffer */
6046 dst_ptr = rdev->rlc.cp_table_ptr;
6047 for (me = 0; me < max_me; me++) {
6048 if (me == 0) {
6049 fw_data = (const __be32 *)rdev->ce_fw->data;
6050 table_offset = CP_ME_TABLE_OFFSET;
6051 } else if (me == 1) {
6052 fw_data = (const __be32 *)rdev->pfp_fw->data;
6053 table_offset = CP_ME_TABLE_OFFSET;
6054 } else if (me == 2) {
6055 fw_data = (const __be32 *)rdev->me_fw->data;
6056 table_offset = CP_ME_TABLE_OFFSET;
6057 } else {
6058 fw_data = (const __be32 *)rdev->mec_fw->data;
6059 table_offset = CP_MEC_TABLE_OFFSET;
6060 }
6061
6062 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6ba81e53 6063 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
22c775ce
AD		 6064 }
6065 bo_offset += CP_ME_TABLE_SIZE;
f96ab484 6066 }
22c775ce 6067}
f96ab484 6068
22c775ce
AD		 6069static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6070 bool enable)
6071{
6072 u32 data, orig;
6073
2b19d17f 6074 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
22c775ce
AD		 6075 orig = data = RREG32(RLC_PG_CNTL);
6076 data |= GFX_PG_ENABLE;
6077 if (orig != data)
6078 WREG32(RLC_PG_CNTL, data);
6079
6080 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6081 data |= AUTO_PG_EN;
6082 if (orig != data)
6083 WREG32(RLC_AUTO_PG_CTRL, data);
6084 } else {
6085 orig = data = RREG32(RLC_PG_CNTL);
6086 data &= ~GFX_PG_ENABLE;
6087 if (orig != data)
6088 WREG32(RLC_PG_CNTL, data);
f96ab484 6089
22c775ce
AD		 6090 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6091 data &= ~AUTO_PG_EN;
6092 if (orig != data)
6093 WREG32(RLC_AUTO_PG_CTRL, data);
f96ab484 6094
22c775ce
AD		 6095 data = RREG32(DB_RENDER_CONTROL);
6096 }
6097}
f96ab484 6098
22c775ce
AD		 6099static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6100{
6101 u32 mask = 0, tmp, tmp1;
6102 int i;
f96ab484 6103
22c775ce
AD		 6104 cik_select_se_sh(rdev, se, sh);
6105 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6106 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6107 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
f96ab484 6108
22c775ce 6109 tmp &= 0xffff0000;
f96ab484 6110
22c775ce
AD		 6111 tmp |= tmp1;
6112 tmp >>= 16;
6113
6114 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6115 mask <<= 1;
6116 mask |= 1;
b07fdd38 6117 }
22c775ce
AD		 6118
6119 return (~tmp) & mask;
f96ab484
AD		 6120}
6121
22c775ce 6122static void cik_init_ao_cu_mask(struct radeon_device *rdev)
d0e092d9 6123{
22c775ce
AD		 6124 u32 i, j, k, active_cu_number = 0;
6125 u32 mask, counter, cu_bitmap;
6126 u32 tmp = 0;
d0e092d9 6127
22c775ce
AD		 6128 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6129 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6130 mask = 1;
6131 cu_bitmap = 0;
6132 counter = 0;
6133 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6134 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6135 if (counter < 2)
6136 cu_bitmap |= mask;
6137 counter ++;
d0e092d9 6138 }
22c775ce 6139 mask <<= 1;
d0e092d9 6140 }
d0e092d9 6141
22c775ce
AD		 6142 active_cu_number += counter;
6143 tmp |= (cu_bitmap << (i * 16 + j * 8));
d0e092d9 6144 }
d0e092d9 6145 }
22c775ce
AD		 6146
6147 WREG32(RLC_PG_AO_CU_MASK, tmp);
6148
6149 tmp = RREG32(RLC_MAX_PG_CU);
6150 tmp &= ~MAX_PU_CU_MASK;
6151 tmp |= MAX_PU_CU(active_cu_number);
6152 WREG32(RLC_MAX_PG_CU, tmp);
d0e092d9
AD		 6153}
6154
22c775ce
AD		 6155static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6156 bool enable)
605de6b9 6157{
22c775ce 6158 u32 data, orig;
605de6b9 6159
22c775ce 6160 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6161 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 6162 data |= STATIC_PER_CU_PG_ENABLE;
6163 else
6164 data &= ~STATIC_PER_CU_PG_ENABLE;
6165 if (orig != data)
6166 WREG32(RLC_PG_CNTL, data);
6167}
6168
6169static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6170 bool enable)
6171{
6172 u32 data, orig;
605de6b9 6173
22c775ce 6174 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6175 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce 6176 data |= DYN_PER_CU_PG_ENABLE;
605de6b9 6177 else
22c775ce
AD		 6178 data &= ~DYN_PER_CU_PG_ENABLE;
6179 if (orig != data)
6180 WREG32(RLC_PG_CNTL, data);
6181}
605de6b9 6182
22c775ce
AD		 6183#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6184#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6185
6186static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6187{
6188 u32 data, orig;
6189 u32 i;
6190
6191 if (rdev->rlc.cs_data) {
6192 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6193 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 6194 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce 6195 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
605de6b9 6196 } else {
22c775ce
AD		 6197 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6198 for (i = 0; i < 3; i++)
6199 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6200 }
6201 if (rdev->rlc.reg_list) {
6202 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6203 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6204 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
605de6b9 6205 }
605de6b9 6206
22c775ce
AD		 6207 orig = data = RREG32(RLC_PG_CNTL);
6208 data |= GFX_PG_SRC;
6209 if (orig != data)
6210 WREG32(RLC_PG_CNTL, data);
605de6b9 6211
22c775ce
AD		 6212 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6213 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
605de6b9 6214
22c775ce
AD		 6215 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6216 data &= ~IDLE_POLL_COUNT_MASK;
6217 data |= IDLE_POLL_COUNT(0x60);
6218 WREG32(CP_RB_WPTR_POLL_CNTL, data);
605de6b9 6219
22c775ce
AD		 6220 data = 0x10101010;
6221 WREG32(RLC_PG_DELAY, data);
605de6b9 6222
22c775ce
AD		 6223 data = RREG32(RLC_PG_DELAY_2);
6224 data &= ~0xff;
6225 data |= 0x3;
6226 WREG32(RLC_PG_DELAY_2, data);
605de6b9 6227
22c775ce
AD		 6228 data = RREG32(RLC_AUTO_PG_CTRL);
6229 data &= ~GRBM_REG_SGIT_MASK;
6230 data |= GRBM_REG_SGIT(0x700);
6231 WREG32(RLC_AUTO_PG_CTRL, data);
605de6b9 6232
605de6b9
AD		 6233}
6234
22c775ce 6235static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
f6796cae 6236{
473359bc
AD		 6237 cik_enable_gfx_cgpg(rdev, enable);
6238 cik_enable_gfx_static_mgpg(rdev, enable);
6239 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce 6240}
f6796cae 6241
a0f38609
AD		 6242u32 cik_get_csb_size(struct radeon_device *rdev)
6243{
6244 u32 count = 0;
6245 const struct cs_section_def *sect = NULL;
6246 const struct cs_extent_def *ext = NULL;
f6796cae 6247
a0f38609
AD		 6248 if (rdev->rlc.cs_data == NULL)
6249 return 0;
f6796cae 6250
a0f38609
AD		 6251 /* begin clear state */
6252 count += 2;
6253 /* context control state */
6254 count += 3;
6255
6256 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6257 for (ext = sect->section; ext->extent != NULL; ++ext) {
6258 if (sect->id == SECT_CONTEXT)
6259 count += 2 + ext->reg_count;
6260 else
6261 return 0;
f6796cae
AD		 6262 }
6263 }
a0f38609
AD		 6264 /* pa_sc_raster_config/pa_sc_raster_config1 */
6265 count += 4;
6266 /* end clear state */
6267 count += 2;
6268 /* clear state */
6269 count += 2;
f6796cae 6270
a0f38609 6271 return count;
f6796cae
AD		 6272}
6273
a0f38609 6274void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
f6796cae 6275{
a0f38609
AD		 6276 u32 count = 0, i;
6277 const struct cs_section_def *sect = NULL;
6278 const struct cs_extent_def *ext = NULL;
f6796cae 6279
a0f38609
AD		 6280 if (rdev->rlc.cs_data == NULL)
6281 return;
6282 if (buffer == NULL)
6283 return;
f6796cae 6284
6ba81e53
AD		 6285 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6286 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
a0f38609 6287
6ba81e53
AD		 6288 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6289 buffer[count++] = cpu_to_le32(0x80000000);
6290 buffer[count++] = cpu_to_le32(0x80000000);
a0f38609
AD		 6291
6292 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6293 for (ext = sect->section; ext->extent != NULL; ++ext) {
6294 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 6295 buffer[count++] =
6296 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6297 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
a0f38609 6298 for (i = 0; i < ext->reg_count; i++)
6ba81e53 6299 buffer[count++] = cpu_to_le32(ext->extent[i]);
a0f38609
AD		 6300 } else {
6301 return;
6302 }
6303 }
6304 }
f6796cae 6305
6ba81e53
AD		 6306 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6307 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
f6796cae
AD		 6308 switch (rdev->family) {
6309 case CHIP_BONAIRE:
6ba81e53
AD		 6310 buffer[count++] = cpu_to_le32(0x16000012);
6311 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6312 break;
6313 case CHIP_KAVERI:
6ba81e53
AD		 6314 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6315 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6316 break;
6317 case CHIP_KABINI:
6ba81e53
AD		 6318 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6319 buffer[count++] = cpu_to_le32(0x00000000);
a0f38609 6320 break;
bbfe90bd
AD		 6321 case CHIP_HAWAII:
6322 buffer[count++] = 0x3a00161a;
6323 buffer[count++] = 0x0000002e;
6324 break;
a0f38609 6325 default:
6ba81e53
AD		 6326 buffer[count++] = cpu_to_le32(0x00000000);
6327 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6328 break;
6329 }
6330
6ba81e53
AD		 6331 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6332 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
f6796cae 6333
6ba81e53
AD		 6334 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6335 buffer[count++] = cpu_to_le32(0);
a0f38609 6336}
f6796cae 6337
473359bc 6338static void cik_init_pg(struct radeon_device *rdev)
22c775ce 6339{
473359bc 6340 if (rdev->pg_flags) {
22c775ce
AD		 6341 cik_enable_sck_slowdown_on_pu(rdev, true);
6342 cik_enable_sck_slowdown_on_pd(rdev, true);
2b19d17f 6343 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 6344 cik_init_gfx_cgpg(rdev);
6345 cik_enable_cp_pg(rdev, true);
6346 cik_enable_gds_pg(rdev, true);
6347 }
22c775ce
AD		 6348 cik_init_ao_cu_mask(rdev);
6349 cik_update_gfx_pg(rdev, true);
6350 }
6351}
f6796cae 6352
473359bc
AD		 6353static void cik_fini_pg(struct radeon_device *rdev)
6354{
6355 if (rdev->pg_flags) {
6356 cik_update_gfx_pg(rdev, false);
2b19d17f 6357 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 6358 cik_enable_cp_pg(rdev, false);
6359 cik_enable_gds_pg(rdev, false);
6360 }
6361 }
f6796cae 6362}
a59781bb
AD		 6363
6364/*
6365 * Interrupts
6366 * Starting with r6xx, interrupts are handled via a ring buffer.
6367 * Ring buffers are areas of GPU accessible memory that the GPU
6368 * writes interrupt vectors into and the host reads vectors out of.
6369 * There is a rptr (read pointer) that determines where the
6370 * host is currently reading, and a wptr (write pointer)
6371 * which determines where the GPU has written. When the
6372 * pointers are equal, the ring is idle. When the GPU
6373 * writes vectors to the ring buffer, it increments the
6374 * wptr. When there is an interrupt, the host then starts
6375 * fetching commands and processing them until the pointers are
6376 * equal again at which point it updates the rptr.
6377 */
6378
6379/**
6380 * cik_enable_interrupts - Enable the interrupt ring buffer
6381 *
6382 * @rdev: radeon_device pointer
6383 *
6384 * Enable the interrupt ring buffer (CIK).
6385 */
6386static void cik_enable_interrupts(struct radeon_device *rdev)
6387{
6388 u32 ih_cntl = RREG32(IH_CNTL);
6389 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6390
6391 ih_cntl |= ENABLE_INTR;
6392 ih_rb_cntl |= IH_RB_ENABLE;
6393 WREG32(IH_CNTL, ih_cntl);
6394 WREG32(IH_RB_CNTL, ih_rb_cntl);
6395 rdev->ih.enabled = true;
6396}
6397
6398/**
6399 * cik_disable_interrupts - Disable the interrupt ring buffer
6400 *
6401 * @rdev: radeon_device pointer
6402 *
6403 * Disable the interrupt ring buffer (CIK).
6404 */
6405static void cik_disable_interrupts(struct radeon_device *rdev)
6406{
6407 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6408 u32 ih_cntl = RREG32(IH_CNTL);
6409
6410 ih_rb_cntl &= ~IH_RB_ENABLE;
6411 ih_cntl &= ~ENABLE_INTR;
6412 WREG32(IH_RB_CNTL, ih_rb_cntl);
6413 WREG32(IH_CNTL, ih_cntl);
6414 /* set rptr, wptr to 0 */
6415 WREG32(IH_RB_RPTR, 0);
6416 WREG32(IH_RB_WPTR, 0);
6417 rdev->ih.enabled = false;
6418 rdev->ih.rptr = 0;
6419}
6420
6421/**
6422 * cik_disable_interrupt_state - Disable all interrupt sources
6423 *
6424 * @rdev: radeon_device pointer
6425 *
6426 * Clear all interrupt enable bits used by the driver (CIK).
6427 */
6428static void cik_disable_interrupt_state(struct radeon_device *rdev)
6429{
6430 u32 tmp;
6431
6432 /* gfx ring */
4214faf6
AD		 6433 tmp = RREG32(CP_INT_CNTL_RING0) &
6434 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6435 WREG32(CP_INT_CNTL_RING0, tmp);
21a93e13
AD		 6436 /* sdma */
6437 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6438 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6439 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6440 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 6441 /* compute queues */
6442 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6443 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6444 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6445 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6446 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6447 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6448 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6449 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6450 /* grbm */
6451 WREG32(GRBM_INT_CNTL, 0);
6452 /* vline/vblank, etc. */
6453 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6454 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6455 if (rdev->num_crtc >= 4) {
6456 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6457 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6458 }
6459 if (rdev->num_crtc >= 6) {
6460 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6461 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6462 }
6463
6464 /* dac hotplug */
6465 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6466
6467 /* digital hotplug */
6468 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6469 WREG32(DC_HPD1_INT_CONTROL, tmp);
6470 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6471 WREG32(DC_HPD2_INT_CONTROL, tmp);
6472 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6473 WREG32(DC_HPD3_INT_CONTROL, tmp);
6474 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6475 WREG32(DC_HPD4_INT_CONTROL, tmp);
6476 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6477 WREG32(DC_HPD5_INT_CONTROL, tmp);
6478 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6479 WREG32(DC_HPD6_INT_CONTROL, tmp);
6480
6481}
6482
6483/**
6484 * cik_irq_init - init and enable the interrupt ring
6485 *
6486 * @rdev: radeon_device pointer
6487 *
6488 * Allocate a ring buffer for the interrupt controller,
6489 * enable the RLC, disable interrupts, enable the IH
6490 * ring buffer and enable it (CIK).
6491 * Called at device load and reume.
6492 * Returns 0 for success, errors for failure.
6493 */
6494static int cik_irq_init(struct radeon_device *rdev)
6495{
6496 int ret = 0;
6497 int rb_bufsz;
6498 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6499
6500 /* allocate ring */
6501 ret = r600_ih_ring_alloc(rdev);
6502 if (ret)
6503 return ret;
6504
6505 /* disable irqs */
6506 cik_disable_interrupts(rdev);
6507
6508 /* init rlc */
6509 ret = cik_rlc_resume(rdev);
6510 if (ret) {
6511 r600_ih_ring_fini(rdev);
6512 return ret;
6513 }
6514
6515 /* setup interrupt control */
6516 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6517 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6518 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6519 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6520 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6521 */
6522 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6523 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6524 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6525 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6526
6527 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 6528 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
a59781bb
AD		 6529
6530 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6531 IH_WPTR_OVERFLOW_CLEAR |
6532 (rb_bufsz << 1));
6533
6534 if (rdev->wb.enabled)
6535 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6536
6537 /* set the writeback address whether it's enabled or not */
6538 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6539 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6540
6541 WREG32(IH_RB_CNTL, ih_rb_cntl);
6542
6543 /* set rptr, wptr to 0 */
6544 WREG32(IH_RB_RPTR, 0);
6545 WREG32(IH_RB_WPTR, 0);
6546
6547 /* Default settings for IH_CNTL (disabled at first) */
6548 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6549 /* RPTR_REARM only works if msi's are enabled */
6550 if (rdev->msi_enabled)
6551 ih_cntl |= RPTR_REARM;
6552 WREG32(IH_CNTL, ih_cntl);
6553
6554 /* force the active interrupt state to all disabled */
6555 cik_disable_interrupt_state(rdev);
6556
6557 pci_set_master(rdev->pdev);
6558
6559 /* enable irqs */
6560 cik_enable_interrupts(rdev);
6561
6562 return ret;
6563}
6564
6565/**
6566 * cik_irq_set - enable/disable interrupt sources
6567 *
6568 * @rdev: radeon_device pointer
6569 *
6570 * Enable interrupt sources on the GPU (vblanks, hpd,
6571 * etc.) (CIK).
6572 * Returns 0 for success, errors for failure.
6573 */
6574int cik_irq_set(struct radeon_device *rdev)
6575{
4214faf6 6576 u32 cp_int_cntl;
2b0781a6
AD		 6577 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6578 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
a59781bb
AD		 6579 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6580 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6581 u32 grbm_int_cntl = 0;
21a93e13 6582 u32 dma_cntl, dma_cntl1;
41a524ab 6583 u32 thermal_int;
a59781bb
AD		 6584
6585 if (!rdev->irq.installed) {
6586 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6587 return -EINVAL;
6588 }
6589 /* don't enable anything if the ih is disabled */
6590 if (!rdev->ih.enabled) {
6591 cik_disable_interrupts(rdev);
6592 /* force the active interrupt state to all disabled */
6593 cik_disable_interrupt_state(rdev);
6594 return 0;
6595 }
6596
4214faf6
AD		 6597 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6598 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6599 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6600
a59781bb
AD		 6601 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6602 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6603 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6604 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6605 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6606 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6607
21a93e13
AD		 6608 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6609 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6610
2b0781a6
AD		 6611 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6612 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6613 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6614 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6615 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6616 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6617 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6618 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6619
cc8dbbb4
AD		 6620 if (rdev->flags & RADEON_IS_IGP)
6621 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6622 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6623 else
6624 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6625 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
41a524ab 6626
a59781bb
AD		 6627 /* enable CP interrupts on all rings */
6628 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6629 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6630 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6631 }
2b0781a6
AD		 6632 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6633 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6634 DRM_DEBUG("si_irq_set: sw int cp1\n");
6635 if (ring->me == 1) {
6636 switch (ring->pipe) {
6637 case 0:
6638 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6639 break;
6640 case 1:
6641 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6642 break;
6643 case 2:
6644 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6645 break;
6646 case 3:
6647 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6648 break;
6649 default:
6650 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6651 break;
6652 }
6653 } else if (ring->me == 2) {
6654 switch (ring->pipe) {
6655 case 0:
6656 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6657 break;
6658 case 1:
6659 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6660 break;
6661 case 2:
6662 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6663 break;
6664 case 3:
6665 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6666 break;
6667 default:
6668 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6669 break;
6670 }
6671 } else {
6672 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6673 }
6674 }
6675 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6676 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6677 DRM_DEBUG("si_irq_set: sw int cp2\n");
6678 if (ring->me == 1) {
6679 switch (ring->pipe) {
6680 case 0:
6681 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6682 break;
6683 case 1:
6684 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6685 break;
6686 case 2:
6687 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6688 break;
6689 case 3:
6690 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6691 break;
6692 default:
6693 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6694 break;
6695 }
6696 } else if (ring->me == 2) {
6697 switch (ring->pipe) {
6698 case 0:
6699 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6700 break;
6701 case 1:
6702 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6703 break;
6704 case 2:
6705 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6706 break;
6707 case 3:
6708 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6709 break;
6710 default:
6711 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6712 break;
6713 }
6714 } else {
6715 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6716 }
6717 }
a59781bb 6718
21a93e13
AD		 6719 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6720 DRM_DEBUG("cik_irq_set: sw int dma\n");
6721 dma_cntl |= TRAP_ENABLE;
6722 }
6723
6724 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6725 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6726 dma_cntl1 |= TRAP_ENABLE;
6727 }
6728
a59781bb
AD		 6729 if (rdev->irq.crtc_vblank_int[0] ||
6730 atomic_read(&rdev->irq.pflip[0])) {
6731 DRM_DEBUG("cik_irq_set: vblank 0\n");
6732 crtc1 |= VBLANK_INTERRUPT_MASK;
6733 }
6734 if (rdev->irq.crtc_vblank_int[1] ||
6735 atomic_read(&rdev->irq.pflip[1])) {
6736 DRM_DEBUG("cik_irq_set: vblank 1\n");
6737 crtc2 |= VBLANK_INTERRUPT_MASK;
6738 }
6739 if (rdev->irq.crtc_vblank_int[2] ||
6740 atomic_read(&rdev->irq.pflip[2])) {
6741 DRM_DEBUG("cik_irq_set: vblank 2\n");
6742 crtc3 |= VBLANK_INTERRUPT_MASK;
6743 }
6744 if (rdev->irq.crtc_vblank_int[3] ||
6745 atomic_read(&rdev->irq.pflip[3])) {
6746 DRM_DEBUG("cik_irq_set: vblank 3\n");
6747 crtc4 |= VBLANK_INTERRUPT_MASK;
6748 }
6749 if (rdev->irq.crtc_vblank_int[4] ||
6750 atomic_read(&rdev->irq.pflip[4])) {
6751 DRM_DEBUG("cik_irq_set: vblank 4\n");
6752 crtc5 |= VBLANK_INTERRUPT_MASK;
6753 }
6754 if (rdev->irq.crtc_vblank_int[5] ||
6755 atomic_read(&rdev->irq.pflip[5])) {
6756 DRM_DEBUG("cik_irq_set: vblank 5\n");
6757 crtc6 |= VBLANK_INTERRUPT_MASK;
6758 }
6759 if (rdev->irq.hpd[0]) {
6760 DRM_DEBUG("cik_irq_set: hpd 1\n");
6761 hpd1 |= DC_HPDx_INT_EN;
6762 }
6763 if (rdev->irq.hpd[1]) {
6764 DRM_DEBUG("cik_irq_set: hpd 2\n");
6765 hpd2 |= DC_HPDx_INT_EN;
6766 }
6767 if (rdev->irq.hpd[2]) {
6768 DRM_DEBUG("cik_irq_set: hpd 3\n");
6769 hpd3 |= DC_HPDx_INT_EN;
6770 }
6771 if (rdev->irq.hpd[3]) {
6772 DRM_DEBUG("cik_irq_set: hpd 4\n");
6773 hpd4 |= DC_HPDx_INT_EN;
6774 }
6775 if (rdev->irq.hpd[4]) {
6776 DRM_DEBUG("cik_irq_set: hpd 5\n");
6777 hpd5 |= DC_HPDx_INT_EN;
6778 }
6779 if (rdev->irq.hpd[5]) {
6780 DRM_DEBUG("cik_irq_set: hpd 6\n");
6781 hpd6 |= DC_HPDx_INT_EN;
6782 }
6783
41a524ab
AD		 6784 if (rdev->irq.dpm_thermal) {
6785 DRM_DEBUG("dpm thermal\n");
cc8dbbb4
AD		 6786 if (rdev->flags & RADEON_IS_IGP)
6787 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6788 else
6789 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
41a524ab
AD		 6790 }
6791
a59781bb
AD		 6792 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6793
21a93e13
AD		 6794 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6795 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6796
2b0781a6
AD		 6797 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6798 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6799 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6800 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6801 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6802 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6803 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6804 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6805
a59781bb
AD		 6806 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6807
6808 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6809 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6810 if (rdev->num_crtc >= 4) {
6811 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6812 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6813 }
6814 if (rdev->num_crtc >= 6) {
6815 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6816 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6817 }
6818
6819 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6820 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6821 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6822 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6823 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6824 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6825
cc8dbbb4
AD		 6826 if (rdev->flags & RADEON_IS_IGP)
6827 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
6828 else
6829 WREG32_SMC(CG_THERMAL_INT, thermal_int);
41a524ab 6830
a59781bb
AD		 6831 return 0;
6832}
6833
6834/**
6835 * cik_irq_ack - ack interrupt sources
6836 *
6837 * @rdev: radeon_device pointer
6838 *
6839 * Ack interrupt sources on the GPU (vblanks, hpd,
6840 * etc.) (CIK). Certain interrupts sources are sw
6841 * generated and do not require an explicit ack.
6842 */
6843static inline void cik_irq_ack(struct radeon_device *rdev)
6844{
6845 u32 tmp;
6846
6847 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6848 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6849 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6850 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6851 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6852 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6853 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
6854
6855 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
6856 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6857 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
6858 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6859 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6860 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6861 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6862 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6863
6864 if (rdev->num_crtc >= 4) {
6865 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6866 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6867 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6868 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6869 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6870 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6871 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6872 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6873 }
6874
6875 if (rdev->num_crtc >= 6) {
6876 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6877 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6878 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6879 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6880 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6881 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6882 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6883 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6884 }
6885
6886 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6887 tmp = RREG32(DC_HPD1_INT_CONTROL);
6888 tmp |= DC_HPDx_INT_ACK;
6889 WREG32(DC_HPD1_INT_CONTROL, tmp);
6890 }
6891 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6892 tmp = RREG32(DC_HPD2_INT_CONTROL);
6893 tmp |= DC_HPDx_INT_ACK;
6894 WREG32(DC_HPD2_INT_CONTROL, tmp);
6895 }
6896 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6897 tmp = RREG32(DC_HPD3_INT_CONTROL);
6898 tmp |= DC_HPDx_INT_ACK;
6899 WREG32(DC_HPD3_INT_CONTROL, tmp);
6900 }
6901 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6902 tmp = RREG32(DC_HPD4_INT_CONTROL);
6903 tmp |= DC_HPDx_INT_ACK;
6904 WREG32(DC_HPD4_INT_CONTROL, tmp);
6905 }
6906 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6907 tmp = RREG32(DC_HPD5_INT_CONTROL);
6908 tmp |= DC_HPDx_INT_ACK;
6909 WREG32(DC_HPD5_INT_CONTROL, tmp);
6910 }
6911 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6912 tmp = RREG32(DC_HPD5_INT_CONTROL);
6913 tmp |= DC_HPDx_INT_ACK;
6914 WREG32(DC_HPD6_INT_CONTROL, tmp);
6915 }
6916}
6917
6918/**
6919 * cik_irq_disable - disable interrupts
6920 *
6921 * @rdev: radeon_device pointer
6922 *
6923 * Disable interrupts on the hw (CIK).
6924 */
6925static void cik_irq_disable(struct radeon_device *rdev)
6926{
6927 cik_disable_interrupts(rdev);
6928 /* Wait and acknowledge irq */
6929 mdelay(1);
6930 cik_irq_ack(rdev);
6931 cik_disable_interrupt_state(rdev);
6932}
6933
6934/**
6935 * cik_irq_disable - disable interrupts for suspend
6936 *
6937 * @rdev: radeon_device pointer
6938 *
6939 * Disable interrupts and stop the RLC (CIK).
6940 * Used for suspend.
6941 */
6942static void cik_irq_suspend(struct radeon_device *rdev)
6943{
6944 cik_irq_disable(rdev);
6945 cik_rlc_stop(rdev);
6946}
6947
6948/**
6949 * cik_irq_fini - tear down interrupt support
6950 *
6951 * @rdev: radeon_device pointer
6952 *
6953 * Disable interrupts on the hw and free the IH ring
6954 * buffer (CIK).
6955 * Used for driver unload.
6956 */
6957static void cik_irq_fini(struct radeon_device *rdev)
6958{
6959 cik_irq_suspend(rdev);
6960 r600_ih_ring_fini(rdev);
6961}
6962
6963/**
6964 * cik_get_ih_wptr - get the IH ring buffer wptr
6965 *
6966 * @rdev: radeon_device pointer
6967 *
6968 * Get the IH ring buffer wptr from either the register
6969 * or the writeback memory buffer (CIK). Also check for
6970 * ring buffer overflow and deal with it.
6971 * Used by cik_irq_process().
6972 * Returns the value of the wptr.
6973 */
6974static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
6975{
6976 u32 wptr, tmp;
6977
6978 if (rdev->wb.enabled)
6979 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6980 else
6981 wptr = RREG32(IH_RB_WPTR);
6982
6983 if (wptr & RB_OVERFLOW) {
6984 /* When a ring buffer overflow happen start parsing interrupt
6985 * from the last not overwritten vector (wptr + 16). Hopefully
6986 * this should allow us to catchup.
6987 */
6988 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6989 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6990 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6991 tmp = RREG32(IH_RB_CNTL);
6992 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6993 WREG32(IH_RB_CNTL, tmp);
6994 }
6995 return (wptr & rdev->ih.ptr_mask);
6996}
6997
6998/* CIK IV Ring
6999 * Each IV ring entry is 128 bits:
7000 * [7:0] - interrupt source id
7001 * [31:8] - reserved
7002 * [59:32] - interrupt source data
7003 * [63:60] - reserved
21a93e13
AD		 7004 * [71:64] - RINGID
7005 * CP:
7006 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 7007 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7008 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7009 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7010 * PIPE_ID - ME0 0=3D
7011 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 7012 * SDMA:
7013 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7014 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7015 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 7016 * [79:72] - VMID
7017 * [95:80] - PASID
7018 * [127:96] - reserved
7019 */
7020/**
7021 * cik_irq_process - interrupt handler
7022 *
7023 * @rdev: radeon_device pointer
7024 *
7025 * Interrupt hander (CIK). Walk the IH ring,
7026 * ack interrupts and schedule work to handle
7027 * interrupt events.
7028 * Returns irq process return code.
7029 */
7030int cik_irq_process(struct radeon_device *rdev)
7031{
2b0781a6
AD		 7032 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7033 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 7034 u32 wptr;
7035 u32 rptr;
7036 u32 src_id, src_data, ring_id;
7037 u8 me_id, pipe_id, queue_id;
7038 u32 ring_index;
7039 bool queue_hotplug = false;
7040 bool queue_reset = false;
3ec7d11b 7041 u32 addr, status, mc_client;
41a524ab 7042 bool queue_thermal = false;
a59781bb
AD		 7043
7044 if (!rdev->ih.enabled || rdev->shutdown)
7045 return IRQ_NONE;
7046
7047 wptr = cik_get_ih_wptr(rdev);
7048
7049restart_ih:
7050 /* is somebody else already processing irqs? */
7051 if (atomic_xchg(&rdev->ih.lock, 1))
7052 return IRQ_NONE;
7053
7054 rptr = rdev->ih.rptr;
7055 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7056
7057 /* Order reading of wptr vs. reading of IH ring data */
7058 rmb();
7059
7060 /* display interrupts */
7061 cik_irq_ack(rdev);
7062
7063 while (rptr != wptr) {
7064 /* wptr/rptr are in bytes! */
7065 ring_index = rptr / 4;
7066 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7067 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7068 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 7069
7070 switch (src_id) {
7071 case 1: /* D1 vblank/vline */
7072 switch (src_data) {
7073 case 0: /* D1 vblank */
7074 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7075 if (rdev->irq.crtc_vblank_int[0]) {
7076 drm_handle_vblank(rdev->ddev, 0);
7077 rdev->pm.vblank_sync = true;
7078 wake_up(&rdev->irq.vblank_queue);
7079 }
7080 if (atomic_read(&rdev->irq.pflip[0]))
7081 radeon_crtc_handle_flip(rdev, 0);
7082 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7083 DRM_DEBUG("IH: D1 vblank\n");
7084 }
7085 break;
7086 case 1: /* D1 vline */
7087 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7088 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7089 DRM_DEBUG("IH: D1 vline\n");
7090 }
7091 break;
7092 default:
7093 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7094 break;
7095 }
7096 break;
7097 case 2: /* D2 vblank/vline */
7098 switch (src_data) {
7099 case 0: /* D2 vblank */
7100 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7101 if (rdev->irq.crtc_vblank_int[1]) {
7102 drm_handle_vblank(rdev->ddev, 1);
7103 rdev->pm.vblank_sync = true;
7104 wake_up(&rdev->irq.vblank_queue);
7105 }
7106 if (atomic_read(&rdev->irq.pflip[1]))
7107 radeon_crtc_handle_flip(rdev, 1);
7108 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7109 DRM_DEBUG("IH: D2 vblank\n");
7110 }
7111 break;
7112 case 1: /* D2 vline */
7113 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7114 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7115 DRM_DEBUG("IH: D2 vline\n");
7116 }
7117 break;
7118 default:
7119 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7120 break;
7121 }
7122 break;
7123 case 3: /* D3 vblank/vline */
7124 switch (src_data) {
7125 case 0: /* D3 vblank */
7126 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7127 if (rdev->irq.crtc_vblank_int[2]) {
7128 drm_handle_vblank(rdev->ddev, 2);
7129 rdev->pm.vblank_sync = true;
7130 wake_up(&rdev->irq.vblank_queue);
7131 }
7132 if (atomic_read(&rdev->irq.pflip[2]))
7133 radeon_crtc_handle_flip(rdev, 2);
7134 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7135 DRM_DEBUG("IH: D3 vblank\n");
7136 }
7137 break;
7138 case 1: /* D3 vline */
7139 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7140 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7141 DRM_DEBUG("IH: D3 vline\n");
7142 }
7143 break;
7144 default:
7145 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7146 break;
7147 }
7148 break;
7149 case 4: /* D4 vblank/vline */
7150 switch (src_data) {
7151 case 0: /* D4 vblank */
7152 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7153 if (rdev->irq.crtc_vblank_int[3]) {
7154 drm_handle_vblank(rdev->ddev, 3);
7155 rdev->pm.vblank_sync = true;
7156 wake_up(&rdev->irq.vblank_queue);
7157 }
7158 if (atomic_read(&rdev->irq.pflip[3]))
7159 radeon_crtc_handle_flip(rdev, 3);
7160 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7161 DRM_DEBUG("IH: D4 vblank\n");
7162 }
7163 break;
7164 case 1: /* D4 vline */
7165 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7166 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7167 DRM_DEBUG("IH: D4 vline\n");
7168 }
7169 break;
7170 default:
7171 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7172 break;
7173 }
7174 break;
7175 case 5: /* D5 vblank/vline */
7176 switch (src_data) {
7177 case 0: /* D5 vblank */
7178 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7179 if (rdev->irq.crtc_vblank_int[4]) {
7180 drm_handle_vblank(rdev->ddev, 4);
7181 rdev->pm.vblank_sync = true;
7182 wake_up(&rdev->irq.vblank_queue);
7183 }
7184 if (atomic_read(&rdev->irq.pflip[4]))
7185 radeon_crtc_handle_flip(rdev, 4);
7186 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7187 DRM_DEBUG("IH: D5 vblank\n");
7188 }
7189 break;
7190 case 1: /* D5 vline */
7191 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7192 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7193 DRM_DEBUG("IH: D5 vline\n");
7194 }
7195 break;
7196 default:
7197 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7198 break;
7199 }
7200 break;
7201 case 6: /* D6 vblank/vline */
7202 switch (src_data) {
7203 case 0: /* D6 vblank */
7204 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7205 if (rdev->irq.crtc_vblank_int[5]) {
7206 drm_handle_vblank(rdev->ddev, 5);
7207 rdev->pm.vblank_sync = true;
7208 wake_up(&rdev->irq.vblank_queue);
7209 }
7210 if (atomic_read(&rdev->irq.pflip[5]))
7211 radeon_crtc_handle_flip(rdev, 5);
7212 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7213 DRM_DEBUG("IH: D6 vblank\n");
7214 }
7215 break;
7216 case 1: /* D6 vline */
7217 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7218 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7219 DRM_DEBUG("IH: D6 vline\n");
7220 }
7221 break;
7222 default:
7223 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7224 break;
7225 }
7226 break;
7227 case 42: /* HPD hotplug */
7228 switch (src_data) {
7229 case 0:
7230 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7231 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7232 queue_hotplug = true;
7233 DRM_DEBUG("IH: HPD1\n");
7234 }
7235 break;
7236 case 1:
7237 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7238 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7239 queue_hotplug = true;
7240 DRM_DEBUG("IH: HPD2\n");
7241 }
7242 break;
7243 case 2:
7244 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7245 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7246 queue_hotplug = true;
7247 DRM_DEBUG("IH: HPD3\n");
7248 }
7249 break;
7250 case 3:
7251 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7252 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7253 queue_hotplug = true;
7254 DRM_DEBUG("IH: HPD4\n");
7255 }
7256 break;
7257 case 4:
7258 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7259 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7260 queue_hotplug = true;
7261 DRM_DEBUG("IH: HPD5\n");
7262 }
7263 break;
7264 case 5:
7265 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7266 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7267 queue_hotplug = true;
7268 DRM_DEBUG("IH: HPD6\n");
7269 }
7270 break;
7271 default:
7272 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7273 break;
7274 }
7275 break;
6a3808b8
CK		 7276 case 124: /* UVD */
7277 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7278 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7279 break;
9d97c99b
AD		 7280 case 146:
7281 case 147:
3ec7d11b
AD		 7282 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7283 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7284 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9d97c99b
AD		 7285 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7286 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 7287 addr);
9d97c99b 7288 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 7289 status);
7290 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b
AD		 7291 /* reset addr and status */
7292 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7293 break;
a59781bb
AD		 7294 case 176: /* GFX RB CP_INT */
7295 case 177: /* GFX IB CP_INT */
7296 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7297 break;
7298 case 181: /* CP EOP event */
7299 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 7300 /* XXX check the bitfield order! */
7301 me_id = (ring_id & 0x60) >> 5;
7302 pipe_id = (ring_id & 0x18) >> 3;
7303 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 7304 switch (me_id) {
7305 case 0:
7306 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7307 break;
7308 case 1:
a59781bb 7309 case 2:
2b0781a6
AD		 7310 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7311 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7312 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7313 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 7314 break;
7315 }
7316 break;
7317 case 184: /* CP Privileged reg access */
7318 DRM_ERROR("Illegal register access in command stream\n");
7319 /* XXX check the bitfield order! */
7320 me_id = (ring_id & 0x60) >> 5;
7321 pipe_id = (ring_id & 0x18) >> 3;
7322 queue_id = (ring_id & 0x7) >> 0;
7323 switch (me_id) {
7324 case 0:
7325 /* This results in a full GPU reset, but all we need to do is soft
7326 * reset the CP for gfx
7327 */
7328 queue_reset = true;
7329 break;
7330 case 1:
7331 /* XXX compute */
2b0781a6 7332 queue_reset = true;
a59781bb
AD		 7333 break;
7334 case 2:
7335 /* XXX compute */
2b0781a6 7336 queue_reset = true;
a59781bb
AD		 7337 break;
7338 }
7339 break;
7340 case 185: /* CP Privileged inst */
7341 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 7342 /* XXX check the bitfield order! */
7343 me_id = (ring_id & 0x60) >> 5;
7344 pipe_id = (ring_id & 0x18) >> 3;
7345 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 7346 switch (me_id) {
7347 case 0:
7348 /* This results in a full GPU reset, but all we need to do is soft
7349 * reset the CP for gfx
7350 */
7351 queue_reset = true;
7352 break;
7353 case 1:
7354 /* XXX compute */
2b0781a6 7355 queue_reset = true;
a59781bb
AD		 7356 break;
7357 case 2:
7358 /* XXX compute */
2b0781a6 7359 queue_reset = true;
a59781bb
AD		 7360 break;
7361 }
7362 break;
21a93e13
AD		 7363 case 224: /* SDMA trap event */
7364 /* XXX check the bitfield order! */
7365 me_id = (ring_id & 0x3) >> 0;
7366 queue_id = (ring_id & 0xc) >> 2;
7367 DRM_DEBUG("IH: SDMA trap\n");
7368 switch (me_id) {
7369 case 0:
7370 switch (queue_id) {
7371 case 0:
7372 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7373 break;
7374 case 1:
7375 /* XXX compute */
7376 break;
7377 case 2:
7378 /* XXX compute */
7379 break;
7380 }
7381 break;
7382 case 1:
7383 switch (queue_id) {
7384 case 0:
7385 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7386 break;
7387 case 1:
7388 /* XXX compute */
7389 break;
7390 case 2:
7391 /* XXX compute */
7392 break;
7393 }
7394 break;
7395 }
7396 break;
41a524ab
AD		 7397 case 230: /* thermal low to high */
7398 DRM_DEBUG("IH: thermal low to high\n");
7399 rdev->pm.dpm.thermal.high_to_low = false;
7400 queue_thermal = true;
7401 break;
7402 case 231: /* thermal high to low */
7403 DRM_DEBUG("IH: thermal high to low\n");
7404 rdev->pm.dpm.thermal.high_to_low = true;
7405 queue_thermal = true;
7406 break;
7407 case 233: /* GUI IDLE */
7408 DRM_DEBUG("IH: GUI idle\n");
7409 break;
21a93e13
AD		 7410 case 241: /* SDMA Privileged inst */
7411 case 247: /* SDMA Privileged inst */
7412 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7413 /* XXX check the bitfield order! */
7414 me_id = (ring_id & 0x3) >> 0;
7415 queue_id = (ring_id & 0xc) >> 2;
7416 switch (me_id) {
7417 case 0:
7418 switch (queue_id) {
7419 case 0:
7420 queue_reset = true;
7421 break;
7422 case 1:
7423 /* XXX compute */
7424 queue_reset = true;
7425 break;
7426 case 2:
7427 /* XXX compute */
7428 queue_reset = true;
7429 break;
7430 }
7431 break;
7432 case 1:
7433 switch (queue_id) {
7434 case 0:
7435 queue_reset = true;
7436 break;
7437 case 1:
7438 /* XXX compute */
7439 queue_reset = true;
7440 break;
7441 case 2:
7442 /* XXX compute */
7443 queue_reset = true;
7444 break;
7445 }
7446 break;
7447 }
7448 break;
a59781bb
AD		 7449 default:
7450 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7451 break;
7452 }
7453
7454 /* wptr/rptr are in bytes! */
7455 rptr += 16;
7456 rptr &= rdev->ih.ptr_mask;
7457 }
7458 if (queue_hotplug)
7459 schedule_work(&rdev->hotplug_work);
7460 if (queue_reset)
7461 schedule_work(&rdev->reset_work);
41a524ab
AD		 7462 if (queue_thermal)
7463 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb
AD		 7464 rdev->ih.rptr = rptr;
7465 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7466 atomic_set(&rdev->ih.lock, 0);
7467
7468 /* make sure wptr hasn't changed while processing */
7469 wptr = cik_get_ih_wptr(rdev);
7470 if (wptr != rptr)
7471 goto restart_ih;
7472
7473 return IRQ_HANDLED;
7474}
7bf94a2c
AD		 7475
7476/*
7477 * startup/shutdown callbacks
7478 */
7479/**
7480 * cik_startup - program the asic to a functional state
7481 *
7482 * @rdev: radeon_device pointer
7483 *
7484 * Programs the asic to a functional state (CIK).
7485 * Called by cik_init() and cik_resume().
7486 * Returns 0 for success, error for failure.
7487 */
7488static int cik_startup(struct radeon_device *rdev)
7489{
7490 struct radeon_ring *ring;
7491 int r;
7492
8a7cd276
AD		 7493 /* enable pcie gen2/3 link */
7494 cik_pcie_gen3_enable(rdev);
7235711a
AD		 7495 /* enable aspm */
7496 cik_program_aspm(rdev);
8a7cd276 7497
e5903d39
AD		 7498 /* scratch needs to be initialized before MC */
7499 r = r600_vram_scratch_init(rdev);
7500 if (r)
7501 return r;
7502
6fab3feb
AD		 7503 cik_mc_program(rdev);
7504
7bf94a2c
AD		 7505 if (rdev->flags & RADEON_IS_IGP) {
7506 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7507 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
7508 r = cik_init_microcode(rdev);
7509 if (r) {
7510 DRM_ERROR("Failed to load firmware!\n");
7511 return r;
7512 }
7513 }
7514 } else {
7515 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7516 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
7517 !rdev->mc_fw) {
7518 r = cik_init_microcode(rdev);
7519 if (r) {
7520 DRM_ERROR("Failed to load firmware!\n");
7521 return r;
7522 }
7523 }
7524
7525 r = ci_mc_load_microcode(rdev);
7526 if (r) {
7527 DRM_ERROR("Failed to load MC firmware!\n");
7528 return r;
7529 }
7530 }
7531
7bf94a2c
AD		 7532 r = cik_pcie_gart_enable(rdev);
7533 if (r)
7534 return r;
7535 cik_gpu_init(rdev);
7536
7537 /* allocate rlc buffers */
22c775ce
AD		 7538 if (rdev->flags & RADEON_IS_IGP) {
7539 if (rdev->family == CHIP_KAVERI) {
7540 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7541 rdev->rlc.reg_list_size =
7542 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7543 } else {
7544 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7545 rdev->rlc.reg_list_size =
7546 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7547 }
7548 }
7549 rdev->rlc.cs_data = ci_cs_data;
7550 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
1fd11777 7551 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 7552 if (r) {
7553 DRM_ERROR("Failed to init rlc BOs!\n");
7554 return r;
7555 }
7556
7557 /* allocate wb buffer */
7558 r = radeon_wb_init(rdev);
7559 if (r)
7560 return r;
7561
963e81f9
AD		 7562 /* allocate mec buffers */
7563 r = cik_mec_init(rdev);
7564 if (r) {
7565 DRM_ERROR("Failed to init MEC BOs!\n");
7566 return r;
7567 }
7568
7bf94a2c
AD		 7569 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7570 if (r) {
7571 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7572 return r;
7573 }
7574
963e81f9
AD		 7575 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7576 if (r) {
7577 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7578 return r;
7579 }
7580
7581 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7582 if (r) {
7583 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7584 return r;
7585 }
7586
7bf94a2c
AD		 7587 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7588 if (r) {
7589 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7590 return r;
7591 }
7592
7593 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7594 if (r) {
7595 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7596 return r;
7597 }
7598
2ce529da 7599 r = radeon_uvd_resume(rdev);
87167bb1 7600 if (!r) {
2ce529da
AD		 7601 r = uvd_v4_2_resume(rdev);
7602 if (!r) {
7603 r = radeon_fence_driver_start_ring(rdev,
7604 R600_RING_TYPE_UVD_INDEX);
7605 if (r)
7606 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7607 }
87167bb1
CK		 7608 }
7609 if (r)
7610 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7611
7bf94a2c
AD		 7612 /* Enable IRQ */
7613 if (!rdev->irq.installed) {
7614 r = radeon_irq_kms_init(rdev);
7615 if (r)
7616 return r;
7617 }
7618
7619 r = cik_irq_init(rdev);
7620 if (r) {
7621 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7622 radeon_irq_kms_fini(rdev);
7623 return r;
7624 }
7625 cik_irq_set(rdev);
7626
7627 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7628 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7629 CP_RB0_RPTR, CP_RB0_WPTR,
1dac28eb 7630 PACKET3(PACKET3_NOP, 0x3FFF));
7bf94a2c
AD		 7631 if (r)
7632 return r;
7633
963e81f9 7634 /* set up the compute queues */
2615b53a 7635 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 7636 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7637 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7638 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
2e1e6dad 7639 PACKET3(PACKET3_NOP, 0x3FFF));
963e81f9
AD		 7640 if (r)
7641 return r;
7642 ring->me = 1; /* first MEC */
7643 ring->pipe = 0; /* first pipe */
7644 ring->queue = 0; /* first queue */
7645 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7646
2615b53a 7647 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 7648 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7649 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7650 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
2e1e6dad 7651 PACKET3(PACKET3_NOP, 0x3FFF));
963e81f9
AD		 7652 if (r)
7653 return r;
7654 /* dGPU only have 1 MEC */
7655 ring->me = 1; /* first MEC */
7656 ring->pipe = 0; /* first pipe */
7657 ring->queue = 1; /* second queue */
7658 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7659
7bf94a2c
AD		 7660 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7661 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7662 SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
7663 SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
2e1e6dad 7664 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 7665 if (r)
7666 return r;
7667
7668 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7669 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7670 SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
7671 SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
2e1e6dad 7672 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 7673 if (r)
7674 return r;
7675
7676 r = cik_cp_resume(rdev);
7677 if (r)
7678 return r;
7679
7680 r = cik_sdma_resume(rdev);
7681 if (r)
7682 return r;
7683
87167bb1
CK		 7684 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7685 if (ring->ring_size) {
02c9f7fa 7686 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
87167bb1 7687 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2e1e6dad 7688 RADEON_CP_PACKET2);
87167bb1 7689 if (!r)
e409b128 7690 r = uvd_v1_0_init(rdev);
87167bb1
CK		 7691 if (r)
7692 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7693 }
7694
7bf94a2c
AD		 7695 r = radeon_ib_pool_init(rdev);
7696 if (r) {
7697 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7698 return r;
7699 }
7700
7701 r = radeon_vm_manager_init(rdev);
7702 if (r) {
7703 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7704 return r;
7705 }
7706
b530602f
AD		 7707 r = dce6_audio_init(rdev);
7708 if (r)
7709 return r;
7710
7bf94a2c
AD		 7711 return 0;
7712}
7713
7714/**
7715 * cik_resume - resume the asic to a functional state
7716 *
7717 * @rdev: radeon_device pointer
7718 *
7719 * Programs the asic to a functional state (CIK).
7720 * Called at resume.
7721 * Returns 0 for success, error for failure.
7722 */
7723int cik_resume(struct radeon_device *rdev)
7724{
7725 int r;
7726
7727 /* post card */
7728 atom_asic_init(rdev->mode_info.atom_context);
7729
0aafd313
AD		 7730 /* init golden registers */
7731 cik_init_golden_registers(rdev);
7732
7bf94a2c
AD		 7733 rdev->accel_working = true;
7734 r = cik_startup(rdev);
7735 if (r) {
7736 DRM_ERROR("cik startup failed on resume\n");
7737 rdev->accel_working = false;
7738 return r;
7739 }
7740
7741 return r;
7742
7743}
7744
7745/**
7746 * cik_suspend - suspend the asic
7747 *
7748 * @rdev: radeon_device pointer
7749 *
7750 * Bring the chip into a state suitable for suspend (CIK).
7751 * Called at suspend.
7752 * Returns 0 for success.
7753 */
7754int cik_suspend(struct radeon_device *rdev)
7755{
b530602f 7756 dce6_audio_fini(rdev);
7bf94a2c
AD		 7757 radeon_vm_manager_fini(rdev);
7758 cik_cp_enable(rdev, false);
7759 cik_sdma_enable(rdev, false);
e409b128 7760 uvd_v1_0_fini(rdev);
87167bb1 7761 radeon_uvd_suspend(rdev);
473359bc
AD		 7762 cik_fini_pg(rdev);
7763 cik_fini_cg(rdev);
7bf94a2c
AD		 7764 cik_irq_suspend(rdev);
7765 radeon_wb_disable(rdev);
7766 cik_pcie_gart_disable(rdev);
7767 return 0;
7768}
7769
7770/* Plan is to move initialization in that function and use
7771 * helper function so that radeon_device_init pretty much
7772 * do nothing more than calling asic specific function. This
7773 * should also allow to remove a bunch of callback function
7774 * like vram_info.
7775 */
7776/**
7777 * cik_init - asic specific driver and hw init
7778 *
7779 * @rdev: radeon_device pointer
7780 *
7781 * Setup asic specific driver variables and program the hw
7782 * to a functional state (CIK).
7783 * Called at driver startup.
7784 * Returns 0 for success, errors for failure.
7785 */
7786int cik_init(struct radeon_device *rdev)
7787{
7788 struct radeon_ring *ring;
7789 int r;
7790
7791 /* Read BIOS */
7792 if (!radeon_get_bios(rdev)) {
7793 if (ASIC_IS_AVIVO(rdev))
7794 return -EINVAL;
7795 }
7796 /* Must be an ATOMBIOS */
7797 if (!rdev->is_atom_bios) {
7798 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7799 return -EINVAL;
7800 }
7801 r = radeon_atombios_init(rdev);
7802 if (r)
7803 return r;
7804
7805 /* Post card if necessary */
7806 if (!radeon_card_posted(rdev)) {
7807 if (!rdev->bios) {
7808 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7809 return -EINVAL;
7810 }
7811 DRM_INFO("GPU not posted. posting now...\n");
7812 atom_asic_init(rdev->mode_info.atom_context);
7813 }
0aafd313
AD		 7814 /* init golden registers */
7815 cik_init_golden_registers(rdev);
7bf94a2c
AD		 7816 /* Initialize scratch registers */
7817 cik_scratch_init(rdev);
7818 /* Initialize surface registers */
7819 radeon_surface_init(rdev);
7820 /* Initialize clocks */
7821 radeon_get_clock_info(rdev->ddev);
7822
7823 /* Fence driver */
7824 r = radeon_fence_driver_init(rdev);
7825 if (r)
7826 return r;
7827
7828 /* initialize memory controller */
7829 r = cik_mc_init(rdev);
7830 if (r)
7831 return r;
7832 /* Memory manager */
7833 r = radeon_bo_init(rdev);
7834 if (r)
7835 return r;
7836
7837 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7838 ring->ring_obj = NULL;
7839 r600_ring_init(rdev, ring, 1024 * 1024);
7840
963e81f9
AD		 7841 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7842 ring->ring_obj = NULL;
7843 r600_ring_init(rdev, ring, 1024 * 1024);
7844 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7845 if (r)
7846 return r;
7847
7848 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7849 ring->ring_obj = NULL;
7850 r600_ring_init(rdev, ring, 1024 * 1024);
7851 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7852 if (r)
7853 return r;
7854
7bf94a2c
AD		 7855 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7856 ring->ring_obj = NULL;
7857 r600_ring_init(rdev, ring, 256 * 1024);
7858
7859 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7860 ring->ring_obj = NULL;
7861 r600_ring_init(rdev, ring, 256 * 1024);
7862
87167bb1
CK		 7863 r = radeon_uvd_init(rdev);
7864 if (!r) {
7865 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7866 ring->ring_obj = NULL;
7867 r600_ring_init(rdev, ring, 4096);
7868 }
7869
7bf94a2c
AD		 7870 rdev->ih.ring_obj = NULL;
7871 r600_ih_ring_init(rdev, 64 * 1024);
7872
7873 r = r600_pcie_gart_init(rdev);
7874 if (r)
7875 return r;
7876
7877 rdev->accel_working = true;
7878 r = cik_startup(rdev);
7879 if (r) {
7880 dev_err(rdev->dev, "disabling GPU acceleration\n");
7881 cik_cp_fini(rdev);
7882 cik_sdma_fini(rdev);
7883 cik_irq_fini(rdev);
1fd11777 7884 sumo_rlc_fini(rdev);
963e81f9 7885 cik_mec_fini(rdev);
7bf94a2c
AD		 7886 radeon_wb_fini(rdev);
7887 radeon_ib_pool_fini(rdev);
7888 radeon_vm_manager_fini(rdev);
7889 radeon_irq_kms_fini(rdev);
7890 cik_pcie_gart_fini(rdev);
7891 rdev->accel_working = false;
7892 }
7893
7894 /* Don't start up if the MC ucode is missing.
7895 * The default clocks and voltages before the MC ucode
7896 * is loaded are not suffient for advanced operations.
7897 */
7898 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
7899 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7900 return -EINVAL;
7901 }
7902
7903 return 0;
7904}
7905
7906/**
7907 * cik_fini - asic specific driver and hw fini
7908 *
7909 * @rdev: radeon_device pointer
7910 *
7911 * Tear down the asic specific driver variables and program the hw
7912 * to an idle state (CIK).
7913 * Called at driver unload.
7914 */
7915void cik_fini(struct radeon_device *rdev)
7916{
7917 cik_cp_fini(rdev);
7918 cik_sdma_fini(rdev);
473359bc
AD		 7919 cik_fini_pg(rdev);
7920 cik_fini_cg(rdev);
7bf94a2c 7921 cik_irq_fini(rdev);
1fd11777 7922 sumo_rlc_fini(rdev);
963e81f9 7923 cik_mec_fini(rdev);
7bf94a2c
AD		 7924 radeon_wb_fini(rdev);
7925 radeon_vm_manager_fini(rdev);
7926 radeon_ib_pool_fini(rdev);
7927 radeon_irq_kms_fini(rdev);
e409b128 7928 uvd_v1_0_fini(rdev);
87167bb1 7929 radeon_uvd_fini(rdev);
7bf94a2c
AD		 7930 cik_pcie_gart_fini(rdev);
7931 r600_vram_scratch_fini(rdev);
7932 radeon_gem_fini(rdev);
7933 radeon_fence_driver_fini(rdev);
7934 radeon_bo_fini(rdev);
7935 radeon_atombios_fini(rdev);
7936 kfree(rdev->bios);
7937 rdev->bios = NULL;
7938}
cd84a27d 7939
134b480f
AD		 7940void dce8_program_fmt(struct drm_encoder *encoder)
7941{
7942 struct drm_device *dev = encoder->dev;
7943 struct radeon_device *rdev = dev->dev_private;
7944 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
7945 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
7946 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
7947 int bpc = 0;
7948 u32 tmp = 0;
6214bb74 7949 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 7950
6214bb74
AD		 7951 if (connector) {
7952 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 7953 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 7954 dither = radeon_connector->dither;
7955 }
134b480f
AD		 7956
7957 /* LVDS/eDP FMT is set up by atom */
7958 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
7959 return;
7960
7961 /* not needed for analog */
7962 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
7963 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
7964 return;
7965
7966 if (bpc == 0)
7967 return;
7968
7969 switch (bpc) {
7970 case 6:
6214bb74 7971 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 7972 /* XXX sort out optimal dither settings */
7973 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7974 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
7975 else
7976 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
7977 break;
7978 case 8:
6214bb74 7979 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 7980 /* XXX sort out optimal dither settings */
7981 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7982 FMT_RGB_RANDOM_ENABLE |
7983 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
7984 else
7985 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
7986 break;
7987 case 10:
6214bb74 7988 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 7989 /* XXX sort out optimal dither settings */
7990 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7991 FMT_RGB_RANDOM_ENABLE |
7992 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
7993 else
7994 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
7995 break;
7996 default:
7997 /* not needed */
7998 break;
7999 }
8000
8001 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8002}
8003
cd84a27d
AD		 8004/* display watermark setup */
8005/**
8006 * dce8_line_buffer_adjust - Set up the line buffer
8007 *
8008 * @rdev: radeon_device pointer
8009 * @radeon_crtc: the selected display controller
8010 * @mode: the current display mode on the selected display
8011 * controller
8012 *
8013 * Setup up the line buffer allocation for
8014 * the selected display controller (CIK).
8015 * Returns the line buffer size in pixels.
8016 */
8017static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8018 struct radeon_crtc *radeon_crtc,
8019 struct drm_display_mode *mode)
8020{
bc01a8c7
AD		 8021 u32 tmp, buffer_alloc, i;
8022 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 8023 /*
8024 * Line Buffer Setup
8025 * There are 6 line buffers, one for each display controllers.
8026 * There are 3 partitions per LB. Select the number of partitions
8027 * to enable based on the display width. For display widths larger
8028 * than 4096, you need use to use 2 display controllers and combine
8029 * them using the stereo blender.
8030 */
8031 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 8032 if (mode->crtc_hdisplay < 1920) {
cd84a27d 8033 tmp = 1;
bc01a8c7
AD		 8034 buffer_alloc = 2;
8035 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 8036 tmp = 2;
bc01a8c7
AD		 8037 buffer_alloc = 2;
8038 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 8039 tmp = 0;
bc01a8c7
AD		 8040 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8041 } else {
cd84a27d
AD		 8042 DRM_DEBUG_KMS("Mode too big for LB!\n");
8043 tmp = 0;
bc01a8c7 8044 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 8045 }
bc01a8c7 8046 } else {
cd84a27d 8047 tmp = 1;
bc01a8c7
AD		 8048 buffer_alloc = 0;
8049 }
cd84a27d
AD		 8050
8051 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8052 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8053
bc01a8c7
AD		 8054 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8055 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8056 for (i = 0; i < rdev->usec_timeout; i++) {
8057 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8058 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8059 break;
8060 udelay(1);
8061 }
8062
cd84a27d
AD		 8063 if (radeon_crtc->base.enabled && mode) {
8064 switch (tmp) {
8065 case 0:
8066 default:
8067 return 4096 * 2;
8068 case 1:
8069 return 1920 * 2;
8070 case 2:
8071 return 2560 * 2;
8072 }
8073 }
8074
8075 /* controller not enabled, so no lb used */
8076 return 0;
8077}
8078
8079/**
8080 * cik_get_number_of_dram_channels - get the number of dram channels
8081 *
8082 * @rdev: radeon_device pointer
8083 *
8084 * Look up the number of video ram channels (CIK).
8085 * Used for display watermark bandwidth calculations
8086 * Returns the number of dram channels
8087 */
8088static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8089{
8090 u32 tmp = RREG32(MC_SHARED_CHMAP);
8091
8092 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8093 case 0:
8094 default:
8095 return 1;
8096 case 1:
8097 return 2;
8098 case 2:
8099 return 4;
8100 case 3:
8101 return 8;
8102 case 4:
8103 return 3;
8104 case 5:
8105 return 6;
8106 case 6:
8107 return 10;
8108 case 7:
8109 return 12;
8110 case 8:
8111 return 16;
8112 }
8113}
8114
8115struct dce8_wm_params {
8116 u32 dram_channels; /* number of dram channels */
8117 u32 yclk; /* bandwidth per dram data pin in kHz */
8118 u32 sclk; /* engine clock in kHz */
8119 u32 disp_clk; /* display clock in kHz */
8120 u32 src_width; /* viewport width */
8121 u32 active_time; /* active display time in ns */
8122 u32 blank_time; /* blank time in ns */
8123 bool interlaced; /* mode is interlaced */
8124 fixed20_12 vsc; /* vertical scale ratio */
8125 u32 num_heads; /* number of active crtcs */
8126 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8127 u32 lb_size; /* line buffer allocated to pipe */
8128 u32 vtaps; /* vertical scaler taps */
8129};
8130
8131/**
8132 * dce8_dram_bandwidth - get the dram bandwidth
8133 *
8134 * @wm: watermark calculation data
8135 *
8136 * Calculate the raw dram bandwidth (CIK).
8137 * Used for display watermark bandwidth calculations
8138 * Returns the dram bandwidth in MBytes/s
8139 */
8140static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8141{
8142 /* Calculate raw DRAM Bandwidth */
8143 fixed20_12 dram_efficiency; /* 0.7 */
8144 fixed20_12 yclk, dram_channels, bandwidth;
8145 fixed20_12 a;
8146
8147 a.full = dfixed_const(1000);
8148 yclk.full = dfixed_const(wm->yclk);
8149 yclk.full = dfixed_div(yclk, a);
8150 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8151 a.full = dfixed_const(10);
8152 dram_efficiency.full = dfixed_const(7);
8153 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8154 bandwidth.full = dfixed_mul(dram_channels, yclk);
8155 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8156
8157 return dfixed_trunc(bandwidth);
8158}
8159
8160/**
8161 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8162 *
8163 * @wm: watermark calculation data
8164 *
8165 * Calculate the dram bandwidth used for display (CIK).
8166 * Used for display watermark bandwidth calculations
8167 * Returns the dram bandwidth for display in MBytes/s
8168 */
8169static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8170{
8171 /* Calculate DRAM Bandwidth and the part allocated to display. */
8172 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8173 fixed20_12 yclk, dram_channels, bandwidth;
8174 fixed20_12 a;
8175
8176 a.full = dfixed_const(1000);
8177 yclk.full = dfixed_const(wm->yclk);
8178 yclk.full = dfixed_div(yclk, a);
8179 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8180 a.full = dfixed_const(10);
8181 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8182 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8183 bandwidth.full = dfixed_mul(dram_channels, yclk);
8184 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8185
8186 return dfixed_trunc(bandwidth);
8187}
8188
8189/**
8190 * dce8_data_return_bandwidth - get the data return bandwidth
8191 *
8192 * @wm: watermark calculation data
8193 *
8194 * Calculate the data return bandwidth used for display (CIK).
8195 * Used for display watermark bandwidth calculations
8196 * Returns the data return bandwidth in MBytes/s
8197 */
8198static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8199{
8200 /* Calculate the display Data return Bandwidth */
8201 fixed20_12 return_efficiency; /* 0.8 */
8202 fixed20_12 sclk, bandwidth;
8203 fixed20_12 a;
8204
8205 a.full = dfixed_const(1000);
8206 sclk.full = dfixed_const(wm->sclk);
8207 sclk.full = dfixed_div(sclk, a);
8208 a.full = dfixed_const(10);
8209 return_efficiency.full = dfixed_const(8);
8210 return_efficiency.full = dfixed_div(return_efficiency, a);
8211 a.full = dfixed_const(32);
8212 bandwidth.full = dfixed_mul(a, sclk);
8213 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8214
8215 return dfixed_trunc(bandwidth);
8216}
8217
8218/**
8219 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8220 *
8221 * @wm: watermark calculation data
8222 *
8223 * Calculate the dmif bandwidth used for display (CIK).
8224 * Used for display watermark bandwidth calculations
8225 * Returns the dmif bandwidth in MBytes/s
8226 */
8227static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8228{
8229 /* Calculate the DMIF Request Bandwidth */
8230 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8231 fixed20_12 disp_clk, bandwidth;
8232 fixed20_12 a, b;
8233
8234 a.full = dfixed_const(1000);
8235 disp_clk.full = dfixed_const(wm->disp_clk);
8236 disp_clk.full = dfixed_div(disp_clk, a);
8237 a.full = dfixed_const(32);
8238 b.full = dfixed_mul(a, disp_clk);
8239
8240 a.full = dfixed_const(10);
8241 disp_clk_request_efficiency.full = dfixed_const(8);
8242 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8243
8244 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8245
8246 return dfixed_trunc(bandwidth);
8247}
8248
8249/**
8250 * dce8_available_bandwidth - get the min available bandwidth
8251 *
8252 * @wm: watermark calculation data
8253 *
8254 * Calculate the min available bandwidth used for display (CIK).
8255 * Used for display watermark bandwidth calculations
8256 * Returns the min available bandwidth in MBytes/s
8257 */
8258static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8259{
8260 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8261 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8262 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8263 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8264
8265 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8266}
8267
8268/**
8269 * dce8_average_bandwidth - get the average available bandwidth
8270 *
8271 * @wm: watermark calculation data
8272 *
8273 * Calculate the average available bandwidth used for display (CIK).
8274 * Used for display watermark bandwidth calculations
8275 * Returns the average available bandwidth in MBytes/s
8276 */
8277static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8278{
8279 /* Calculate the display mode Average Bandwidth
8280 * DisplayMode should contain the source and destination dimensions,
8281 * timing, etc.
8282 */
8283 fixed20_12 bpp;
8284 fixed20_12 line_time;
8285 fixed20_12 src_width;
8286 fixed20_12 bandwidth;
8287 fixed20_12 a;
8288
8289 a.full = dfixed_const(1000);
8290 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8291 line_time.full = dfixed_div(line_time, a);
8292 bpp.full = dfixed_const(wm->bytes_per_pixel);
8293 src_width.full = dfixed_const(wm->src_width);
8294 bandwidth.full = dfixed_mul(src_width, bpp);
8295 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8296 bandwidth.full = dfixed_div(bandwidth, line_time);
8297
8298 return dfixed_trunc(bandwidth);
8299}
8300
8301/**
8302 * dce8_latency_watermark - get the latency watermark
8303 *
8304 * @wm: watermark calculation data
8305 *
8306 * Calculate the latency watermark (CIK).
8307 * Used for display watermark bandwidth calculations
8308 * Returns the latency watermark in ns
8309 */
8310static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8311{
8312 /* First calculate the latency in ns */
8313 u32 mc_latency = 2000; /* 2000 ns. */
8314 u32 available_bandwidth = dce8_available_bandwidth(wm);
8315 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8316 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8317 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8318 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8319 (wm->num_heads * cursor_line_pair_return_time);
8320 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8321 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8322 u32 tmp, dmif_size = 12288;
8323 fixed20_12 a, b, c;
8324
8325 if (wm->num_heads == 0)
8326 return 0;
8327
8328 a.full = dfixed_const(2);
8329 b.full = dfixed_const(1);
8330 if ((wm->vsc.full > a.full) ||
8331 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8332 (wm->vtaps >= 5) ||
8333 ((wm->vsc.full >= a.full) && wm->interlaced))
8334 max_src_lines_per_dst_line = 4;
8335 else
8336 max_src_lines_per_dst_line = 2;
8337
8338 a.full = dfixed_const(available_bandwidth);
8339 b.full = dfixed_const(wm->num_heads);
8340 a.full = dfixed_div(a, b);
8341
8342 b.full = dfixed_const(mc_latency + 512);
8343 c.full = dfixed_const(wm->disp_clk);
8344 b.full = dfixed_div(b, c);
8345
8346 c.full = dfixed_const(dmif_size);
8347 b.full = dfixed_div(c, b);
8348
8349 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8350
8351 b.full = dfixed_const(1000);
8352 c.full = dfixed_const(wm->disp_clk);
8353 b.full = dfixed_div(c, b);
8354 c.full = dfixed_const(wm->bytes_per_pixel);
8355 b.full = dfixed_mul(b, c);
8356
8357 lb_fill_bw = min(tmp, dfixed_trunc(b));
8358
8359 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8360 b.full = dfixed_const(1000);
8361 c.full = dfixed_const(lb_fill_bw);
8362 b.full = dfixed_div(c, b);
8363 a.full = dfixed_div(a, b);
8364 line_fill_time = dfixed_trunc(a);
8365
8366 if (line_fill_time < wm->active_time)
8367 return latency;
8368 else
8369 return latency + (line_fill_time - wm->active_time);
8370
8371}
8372
8373/**
8374 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8375 * average and available dram bandwidth
8376 *
8377 * @wm: watermark calculation data
8378 *
8379 * Check if the display average bandwidth fits in the display
8380 * dram bandwidth (CIK).
8381 * Used for display watermark bandwidth calculations
8382 * Returns true if the display fits, false if not.
8383 */
8384static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8385{
8386 if (dce8_average_bandwidth(wm) <=
8387 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8388 return true;
8389 else
8390 return false;
8391}
8392
8393/**
8394 * dce8_average_bandwidth_vs_available_bandwidth - check
8395 * average and available bandwidth
8396 *
8397 * @wm: watermark calculation data
8398 *
8399 * Check if the display average bandwidth fits in the display
8400 * available bandwidth (CIK).
8401 * Used for display watermark bandwidth calculations
8402 * Returns true if the display fits, false if not.
8403 */
8404static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8405{
8406 if (dce8_average_bandwidth(wm) <=
8407 (dce8_available_bandwidth(wm) / wm->num_heads))
8408 return true;
8409 else
8410 return false;
8411}
8412
8413/**
8414 * dce8_check_latency_hiding - check latency hiding
8415 *
8416 * @wm: watermark calculation data
8417 *
8418 * Check latency hiding (CIK).
8419 * Used for display watermark bandwidth calculations
8420 * Returns true if the display fits, false if not.
8421 */
8422static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8423{
8424 u32 lb_partitions = wm->lb_size / wm->src_width;
8425 u32 line_time = wm->active_time + wm->blank_time;
8426 u32 latency_tolerant_lines;
8427 u32 latency_hiding;
8428 fixed20_12 a;
8429
8430 a.full = dfixed_const(1);
8431 if (wm->vsc.full > a.full)
8432 latency_tolerant_lines = 1;
8433 else {
8434 if (lb_partitions <= (wm->vtaps + 1))
8435 latency_tolerant_lines = 1;
8436 else
8437 latency_tolerant_lines = 2;
8438 }
8439
8440 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8441
8442 if (dce8_latency_watermark(wm) <= latency_hiding)
8443 return true;
8444 else
8445 return false;
8446}
8447
8448/**
8449 * dce8_program_watermarks - program display watermarks
8450 *
8451 * @rdev: radeon_device pointer
8452 * @radeon_crtc: the selected display controller
8453 * @lb_size: line buffer size
8454 * @num_heads: number of display controllers in use
8455 *
8456 * Calculate and program the display watermarks for the
8457 * selected display controller (CIK).
8458 */
8459static void dce8_program_watermarks(struct radeon_device *rdev,
8460 struct radeon_crtc *radeon_crtc,
8461 u32 lb_size, u32 num_heads)
8462{
8463 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 8464 struct dce8_wm_params wm_low, wm_high;
cd84a27d
AD		 8465 u32 pixel_period;
8466 u32 line_time = 0;
8467 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8468 u32 tmp, wm_mask;
8469
8470 if (radeon_crtc->base.enabled && num_heads && mode) {
8471 pixel_period = 1000000 / (u32)mode->clock;
8472 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8473
58ea2dea
AD		 8474 /* watermark for high clocks */
8475 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8476 rdev->pm.dpm_enabled) {
8477 wm_high.yclk =
8478 radeon_dpm_get_mclk(rdev, false) * 10;
8479 wm_high.sclk =
8480 radeon_dpm_get_sclk(rdev, false) * 10;
8481 } else {
8482 wm_high.yclk = rdev->pm.current_mclk * 10;
8483 wm_high.sclk = rdev->pm.current_sclk * 10;
8484 }
8485
8486 wm_high.disp_clk = mode->clock;
8487 wm_high.src_width = mode->crtc_hdisplay;
8488 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8489 wm_high.blank_time = line_time - wm_high.active_time;
8490 wm_high.interlaced = false;
cd84a27d 8491 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 8492 wm_high.interlaced = true;
8493 wm_high.vsc = radeon_crtc->vsc;
8494 wm_high.vtaps = 1;
cd84a27d 8495 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 8496 wm_high.vtaps = 2;
8497 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8498 wm_high.lb_size = lb_size;
8499 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8500 wm_high.num_heads = num_heads;
cd84a27d
AD		 8501
8502 /* set for high clocks */
58ea2dea
AD		 8503 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8504
8505 /* possibly force display priority to high */
8506 /* should really do this at mode validation time... */
8507 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8508 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8509 !dce8_check_latency_hiding(&wm_high) ||
8510 (rdev->disp_priority == 2)) {
8511 DRM_DEBUG_KMS("force priority to high\n");
8512 }
8513
8514 /* watermark for low clocks */
8515 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8516 rdev->pm.dpm_enabled) {
8517 wm_low.yclk =
8518 radeon_dpm_get_mclk(rdev, true) * 10;
8519 wm_low.sclk =
8520 radeon_dpm_get_sclk(rdev, true) * 10;
8521 } else {
8522 wm_low.yclk = rdev->pm.current_mclk * 10;
8523 wm_low.sclk = rdev->pm.current_sclk * 10;
8524 }
8525
8526 wm_low.disp_clk = mode->clock;
8527 wm_low.src_width = mode->crtc_hdisplay;
8528 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8529 wm_low.blank_time = line_time - wm_low.active_time;
8530 wm_low.interlaced = false;
8531 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8532 wm_low.interlaced = true;
8533 wm_low.vsc = radeon_crtc->vsc;
8534 wm_low.vtaps = 1;
8535 if (radeon_crtc->rmx_type != RMX_OFF)
8536 wm_low.vtaps = 2;
8537 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8538 wm_low.lb_size = lb_size;
8539 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8540 wm_low.num_heads = num_heads;
8541
cd84a27d 8542 /* set for low clocks */
58ea2dea 8543 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 8544
8545 /* possibly force display priority to high */
8546 /* should really do this at mode validation time... */
58ea2dea
AD		 8547 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8548 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8549 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 8550 (rdev->disp_priority == 2)) {
8551 DRM_DEBUG_KMS("force priority to high\n");
8552 }
8553 }
8554
8555 /* select wm A */
8556 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8557 tmp = wm_mask;
8558 tmp &= ~LATENCY_WATERMARK_MASK(3);
8559 tmp |= LATENCY_WATERMARK_MASK(1);
8560 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8561 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8562 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8563 LATENCY_HIGH_WATERMARK(line_time)));
8564 /* select wm B */
8565 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8566 tmp &= ~LATENCY_WATERMARK_MASK(3);
8567 tmp |= LATENCY_WATERMARK_MASK(2);
8568 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8569 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8570 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8571 LATENCY_HIGH_WATERMARK(line_time)));
8572 /* restore original selection */
8573 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 8574
8575 /* save values for DPM */
8576 radeon_crtc->line_time = line_time;
8577 radeon_crtc->wm_high = latency_watermark_a;
8578 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 8579}
8580
8581/**
8582 * dce8_bandwidth_update - program display watermarks
8583 *
8584 * @rdev: radeon_device pointer
8585 *
8586 * Calculate and program the display watermarks and line
8587 * buffer allocation (CIK).
8588 */
8589void dce8_bandwidth_update(struct radeon_device *rdev)
8590{
8591 struct drm_display_mode *mode = NULL;
8592 u32 num_heads = 0, lb_size;
8593 int i;
8594
8595 radeon_update_display_priority(rdev);
8596
8597 for (i = 0; i < rdev->num_crtc; i++) {
8598 if (rdev->mode_info.crtcs[i]->base.enabled)
8599 num_heads++;
8600 }
8601 for (i = 0; i < rdev->num_crtc; i++) {
8602 mode = &rdev->mode_info.crtcs[i]->base.mode;
8603 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8604 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8605 }
8606}
44fa346f
AD		 8607
8608/**
8609 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8610 *
8611 * @rdev: radeon_device pointer
8612 *
8613 * Fetches a GPU clock counter snapshot (SI).
8614 * Returns the 64 bit clock counter snapshot.
8615 */
8616uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8617{
8618 uint64_t clock;
8619
8620 mutex_lock(&rdev->gpu_clock_mutex);
8621 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8622 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8623 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8624 mutex_unlock(&rdev->gpu_clock_mutex);
8625 return clock;
8626}
8627
87167bb1
CK		 8628static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8629 u32 cntl_reg, u32 status_reg)
8630{
8631 int r, i;
8632 struct atom_clock_dividers dividers;
8633 uint32_t tmp;
8634
8635 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8636 clock, false, &dividers);
8637 if (r)
8638 return r;
8639
8640 tmp = RREG32_SMC(cntl_reg);
8641 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8642 tmp |= dividers.post_divider;
8643 WREG32_SMC(cntl_reg, tmp);
8644
8645 for (i = 0; i < 100; i++) {
8646 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8647 break;
8648 mdelay(10);
8649 }
8650 if (i == 100)
8651 return -ETIMEDOUT;
8652
8653 return 0;
8654}
8655
8656int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8657{
8658 int r = 0;
8659
8660 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8661 if (r)
8662 return r;
8663
8664 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8665 return r;
8666}
8667
8a7cd276 8668static void cik_pcie_gen3_enable(struct radeon_device *rdev)
87167bb1 8669{
8a7cd276
AD		 8670 struct pci_dev *root = rdev->pdev->bus->self;
8671 int bridge_pos, gpu_pos;
8672 u32 speed_cntl, mask, current_data_rate;
8673 int ret, i;
8674 u16 tmp16;
87167bb1 8675
8a7cd276
AD		 8676 if (radeon_pcie_gen2 == 0)
8677 return;
87167bb1 8678
8a7cd276
AD		 8679 if (rdev->flags & RADEON_IS_IGP)
8680 return;
87167bb1 8681
8a7cd276
AD		 8682 if (!(rdev->flags & RADEON_IS_PCIE))
8683 return;
87167bb1 8684
8a7cd276
AD		 8685 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8686 if (ret != 0)
8687 return;
87167bb1 8688
8a7cd276
AD		 8689 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8690 return;
87167bb1 8691
8a7cd276
AD		 8692 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8693 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8694 LC_CURRENT_DATA_RATE_SHIFT;
8695 if (mask & DRM_PCIE_SPEED_80) {
8696 if (current_data_rate == 2) {
8697 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8698 return;
8699 }
8700 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8701 } else if (mask & DRM_PCIE_SPEED_50) {
8702 if (current_data_rate == 1) {
8703 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8704 return;
8705 }
8706 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8707 }
87167bb1 8708
8a7cd276
AD		 8709 bridge_pos = pci_pcie_cap(root);
8710 if (!bridge_pos)
8711 return;
8712
8713 gpu_pos = pci_pcie_cap(rdev->pdev);
8714 if (!gpu_pos)
8715 return;
8716
8717 if (mask & DRM_PCIE_SPEED_80) {
8718 /* re-try equalization if gen3 is not already enabled */
8719 if (current_data_rate != 2) {
8720 u16 bridge_cfg, gpu_cfg;
8721 u16 bridge_cfg2, gpu_cfg2;
8722 u32 max_lw, current_lw, tmp;
8723
8724 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8725 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8726
8727 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8728 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8729
8730 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8731 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8732
8733 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8734 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8735 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8736
8737 if (current_lw < max_lw) {
8738 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8739 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8740 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8741 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8742 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8743 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8744 }
8745 }
8746
8747 for (i = 0; i < 10; i++) {
8748 /* check status */
8749 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8750 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8751 break;
8752
8753 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8754 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8755
8756 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8757 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8758
8759 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8760 tmp |= LC_SET_QUIESCE;
8761 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8762
8763 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8764 tmp |= LC_REDO_EQ;
8765 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8766
8767 mdelay(100);
8768
8769 /* linkctl */
8770 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8771 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8772 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8773 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8774
8775 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8776 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8777 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8778 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8779
8780 /* linkctl2 */
8781 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8782 tmp16 &= ~((1 << 4) | (7 << 9));
8783 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8784 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8785
8786 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8787 tmp16 &= ~((1 << 4) | (7 << 9));
8788 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8789 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8790
8791 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8792 tmp &= ~LC_SET_QUIESCE;
8793 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8794 }
8795 }
8796 }
8797
8798 /* set the link speed */
8799 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8800 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8801 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8802
8803 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8804 tmp16 &= ~0xf;
8805 if (mask & DRM_PCIE_SPEED_80)
8806 tmp16 |= 3; /* gen3 */
8807 else if (mask & DRM_PCIE_SPEED_50)
8808 tmp16 |= 2; /* gen2 */
8809 else
8810 tmp16 |= 1; /* gen1 */
8811 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8812
8813 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8814 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
8815 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8816
8817 for (i = 0; i < rdev->usec_timeout; i++) {
8818 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8819 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
8820 break;
8821 udelay(1);
8822 }
8823}
7235711a
AD		 8824
8825static void cik_program_aspm(struct radeon_device *rdev)
8826{
8827 u32 data, orig;
8828 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
8829 bool disable_clkreq = false;
8830
8831 if (radeon_aspm == 0)
8832 return;
8833
8834 /* XXX double check IGPs */
8835 if (rdev->flags & RADEON_IS_IGP)
8836 return;
8837
8838 if (!(rdev->flags & RADEON_IS_PCIE))
8839 return;
8840
8841 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8842 data &= ~LC_XMIT_N_FTS_MASK;
8843 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
8844 if (orig != data)
8845 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
8846
8847 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
8848 data |= LC_GO_TO_RECOVERY;
8849 if (orig != data)
8850 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
8851
8852 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
8853 data |= P_IGNORE_EDB_ERR;
8854 if (orig != data)
8855 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
8856
8857 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8858 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
8859 data |= LC_PMI_TO_L1_DIS;
8860 if (!disable_l0s)
8861 data |= LC_L0S_INACTIVITY(7);
8862
8863 if (!disable_l1) {
8864 data |= LC_L1_INACTIVITY(7);
8865 data &= ~LC_PMI_TO_L1_DIS;
8866 if (orig != data)
8867 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8868
8869 if (!disable_plloff_in_l1) {
8870 bool clk_req_support;
8871
8872 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
8873 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8874 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8875 if (orig != data)
8876 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
8877
8878 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
8879 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8880 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8881 if (orig != data)
8882 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
8883
8884 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
8885 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8886 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8887 if (orig != data)
8888 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
8889
8890 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
8891 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8892 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8893 if (orig != data)
8894 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
8895
8896 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8897 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
8898 data |= LC_DYN_LANES_PWR_STATE(3);
8899 if (orig != data)
8900 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
8901
8902 if (!disable_clkreq) {
8903 struct pci_dev *root = rdev->pdev->bus->self;
8904 u32 lnkcap;
8905
8906 clk_req_support = false;
8907 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
8908 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
8909 clk_req_support = true;
8910 } else {
8911 clk_req_support = false;
8912 }
8913
8914 if (clk_req_support) {
8915 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
8916 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
8917 if (orig != data)
8918 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
8919
8920 orig = data = RREG32_SMC(THM_CLK_CNTL);
8921 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
8922 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8923 if (orig != data)
8924 WREG32_SMC(THM_CLK_CNTL, data);
8925
8926 orig = data = RREG32_SMC(MISC_CLK_CTRL);
8927 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
8928 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8929 if (orig != data)
8930 WREG32_SMC(MISC_CLK_CTRL, data);
8931
8932 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
8933 data &= ~BCLK_AS_XCLK;
8934 if (orig != data)
8935 WREG32_SMC(CG_CLKPIN_CNTL, data);
8936
8937 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
8938 data &= ~FORCE_BIF_REFCLK_EN;
8939 if (orig != data)
8940 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
8941
8942 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
8943 data &= ~MPLL_CLKOUT_SEL_MASK;
8944 data |= MPLL_CLKOUT_SEL(4);
8945 if (orig != data)
8946 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
8947 }
8948 }
8949 } else {
8950 if (orig != data)
8951 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8952 }
8953
8954 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
8955 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
8956 if (orig != data)
8957 WREG32_PCIE_PORT(PCIE_CNTL2, data);
8958
8959 if (!disable_l0s) {
8960 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8961 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
8962 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8963 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
8964 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8965 data &= ~LC_L0S_INACTIVITY_MASK;
8966 if (orig != data)
8967 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8968 }
8969 }
8970 }
87167bb1 8971}
Michael's Linux tree.