]>
Commit | Line | Data |
---|---|---|
f9df67e9 HZ |
1 | /* |
2 | * Copyright 2019 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | */ | |
23 | #include <linux/firmware.h> | |
d7929c1e | 24 | #include <linux/pci.h> |
3616d49d AL |
25 | |
26 | #include <drm/drm_cache.h> | |
27 | ||
f9df67e9 HZ |
28 | #include "amdgpu.h" |
29 | #include "amdgpu_atomfirmware.h" | |
30 | #include "gmc_v10_0.h" | |
01eee24f | 31 | #include "umc_v8_7.h" |
f9df67e9 | 32 | |
ea930000 AS |
33 | #include "athub/athub_2_0_0_sh_mask.h" |
34 | #include "athub/athub_2_0_0_offset.h" | |
f9df67e9 HZ |
35 | #include "dcn/dcn_2_0_0_offset.h" |
36 | #include "dcn/dcn_2_0_0_sh_mask.h" | |
37 | #include "oss/osssys_5_0_0_offset.h" | |
38 | #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" | |
39 | #include "navi10_enum.h" | |
40 | ||
41 | #include "soc15.h" | |
ea930000 | 42 | #include "soc15d.h" |
f9df67e9 HZ |
43 | #include "soc15_common.h" |
44 | ||
45 | #include "nbio_v2_3.h" | |
46 | ||
47 | #include "gfxhub_v2_0.h" | |
0b3df16b | 48 | #include "gfxhub_v2_1.h" |
f9df67e9 | 49 | #include "mmhub_v2_0.h" |
4d8d75a4 | 50 | #include "mmhub_v2_3.h" |
f9df67e9 | 51 | #include "athub_v2_0.h" |
920a4cd3 | 52 | #include "athub_v2_1.h" |
f9df67e9 | 53 | |
01eee24f JC |
54 | static int gmc_v10_0_ecc_interrupt_state(struct amdgpu_device *adev, |
55 | struct amdgpu_irq_src *src, | |
8612a435 | 56 | unsigned int type, |
01eee24f JC |
57 | enum amdgpu_interrupt_state state) |
58 | { | |
59 | return 0; | |
60 | } | |
61 | ||
f9df67e9 HZ |
62 | static int |
63 | gmc_v10_0_vm_fault_interrupt_state(struct amdgpu_device *adev, | |
8612a435 | 64 | struct amdgpu_irq_src *src, unsigned int type, |
f9df67e9 HZ |
65 | enum amdgpu_interrupt_state state) |
66 | { | |
f9df67e9 HZ |
67 | switch (state) { |
68 | case AMDGPU_IRQ_STATE_DISABLE: | |
69 | /* MM HUB */ | |
f4caf584 | 70 | amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB0(0), false); |
f9df67e9 | 71 | /* GFX HUB */ |
d5d29009 AD |
72 | /* This works because this interrupt is only |
73 | * enabled at init/resume and disabled in | |
74 | * fini/suspend, so the overall state doesn't | |
75 | * change over the course of suspend/resume. | |
76 | */ | |
77 | if (!adev->in_s0ix) | |
f4caf584 | 78 | amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB(0), false); |
f9df67e9 HZ |
79 | break; |
80 | case AMDGPU_IRQ_STATE_ENABLE: | |
81 | /* MM HUB */ | |
f4caf584 | 82 | amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB0(0), true); |
f9df67e9 | 83 | /* GFX HUB */ |
d5d29009 AD |
84 | /* This works because this interrupt is only |
85 | * enabled at init/resume and disabled in | |
86 | * fini/suspend, so the overall state doesn't | |
87 | * change over the course of suspend/resume. | |
88 | */ | |
89 | if (!adev->in_s0ix) | |
f4caf584 | 90 | amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB(0), true); |
f9df67e9 HZ |
91 | break; |
92 | default: | |
93 | break; | |
94 | } | |
95 | ||
96 | return 0; | |
97 | } | |
98 | ||
99 | static int gmc_v10_0_process_interrupt(struct amdgpu_device *adev, | |
100 | struct amdgpu_irq_src *source, | |
101 | struct amdgpu_iv_entry *entry) | |
102 | { | |
6f38bdb8 LY |
103 | uint32_t vmhub_index = entry->client_id == SOC15_IH_CLIENTID_VMC ? |
104 | AMDGPU_MMHUB0(0) : AMDGPU_GFXHUB(0); | |
105 | struct amdgpu_vmhub *hub = &adev->vmhub[vmhub_index]; | |
a2a8857c | 106 | bool retry_fault = !!(entry->src_data[1] & 0x80); |
ff891a2e | 107 | bool write_fault = !!(entry->src_data[1] & 0x20); |
45d87b85 | 108 | struct amdgpu_task_info task_info; |
f9df67e9 HZ |
109 | uint32_t status = 0; |
110 | u64 addr; | |
111 | ||
112 | addr = (u64)entry->src_data[0] << 12; | |
113 | addr |= ((u64)entry->src_data[1] & 0xf) << 44; | |
114 | ||
a2a8857c CK |
115 | if (retry_fault) { |
116 | /* Returning 1 here also prevents sending the IV to the KFD */ | |
117 | ||
118 | /* Process it onyl if it's the first fault for this address */ | |
119 | if (entry->ih != &adev->irq.ih_soft && | |
3c2d6ea2 | 120 | amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid, |
a2a8857c CK |
121 | entry->timestamp)) |
122 | return 1; | |
123 | ||
124 | /* Delegate it to a different ring if the hardware hasn't | |
125 | * already done it. | |
126 | */ | |
58df0d71 | 127 | if (entry->ih == &adev->irq.ih) { |
a2a8857c CK |
128 | amdgpu_irq_delegate(adev, entry, 8); |
129 | return 1; | |
130 | } | |
131 | ||
132 | /* Try to handle the recoverable page faults by filling page | |
133 | * tables | |
134 | */ | |
5fb34bd9 | 135 | if (amdgpu_vm_handle_fault(adev, entry->pasid, 0, 0, addr, write_fault)) |
a2a8857c CK |
136 | return 1; |
137 | } | |
138 | ||
f9df67e9 | 139 | if (!amdgpu_sriov_vf(adev)) { |
53499173 XY |
140 | /* |
141 | * Issue a dummy read to wait for the status register to | |
142 | * be updated to avoid reading an incorrect value due to | |
143 | * the new fast GRBM interface. | |
144 | */ | |
f4caf584 | 145 | if ((entry->vmid_src == AMDGPU_GFXHUB(0)) && |
4e8303cf LL |
146 | (amdgpu_ip_version(adev, GC_HWIP, 0) < |
147 | IP_VERSION(10, 3, 0))) | |
53499173 XY |
148 | RREG32(hub->vm_l2_pro_fault_status); |
149 | ||
f9df67e9 HZ |
150 | status = RREG32(hub->vm_l2_pro_fault_status); |
151 | WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); | |
8dbf1ba8 AD |
152 | |
153 | amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, | |
154 | entry->vmid_src ? AMDGPU_MMHUB0(0) : AMDGPU_GFXHUB(0)); | |
f9df67e9 HZ |
155 | } |
156 | ||
45d87b85 CK |
157 | if (!printk_ratelimit()) |
158 | return 0; | |
159 | ||
160 | memset(&task_info, 0, sizeof(struct amdgpu_task_info)); | |
161 | amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); | |
162 | ||
163 | dev_err(adev->dev, | |
8612a435 | 164 | "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, for process %s pid %d thread %s pid %d)\n", |
45d87b85 CK |
165 | entry->vmid_src ? "mmhub" : "gfxhub", |
166 | entry->src_id, entry->ring_id, entry->vmid, | |
167 | entry->pasid, task_info.process_name, task_info.tgid, | |
168 | task_info.task_name, task_info.pid); | |
be14729a YZ |
169 | dev_err(adev->dev, " in page starting at address 0x%016llx from client 0x%x (%s)\n", |
170 | addr, entry->client_id, | |
171 | soc15_ih_clientid_name[entry->client_id]); | |
45d87b85 CK |
172 | |
173 | if (!amdgpu_sriov_vf(adev)) | |
174 | hub->vmhub_funcs->print_l2_protection_fault_status(adev, | |
175 | status); | |
f9df67e9 HZ |
176 | |
177 | return 0; | |
178 | } | |
179 | ||
180 | static const struct amdgpu_irq_src_funcs gmc_v10_0_irq_funcs = { | |
181 | .set = gmc_v10_0_vm_fault_interrupt_state, | |
182 | .process = gmc_v10_0_process_interrupt, | |
183 | }; | |
184 | ||
01eee24f JC |
185 | static const struct amdgpu_irq_src_funcs gmc_v10_0_ecc_funcs = { |
186 | .set = gmc_v10_0_ecc_interrupt_state, | |
187 | .process = amdgpu_umc_process_ecc_irq, | |
188 | }; | |
189 | ||
94ba290d | 190 | static void gmc_v10_0_set_irq_funcs(struct amdgpu_device *adev) |
f9df67e9 HZ |
191 | { |
192 | adev->gmc.vm_fault.num_types = 1; | |
193 | adev->gmc.vm_fault.funcs = &gmc_v10_0_irq_funcs; | |
01eee24f JC |
194 | |
195 | if (!amdgpu_sriov_vf(adev)) { | |
196 | adev->gmc.ecc_irq.num_types = 1; | |
197 | adev->gmc.ecc_irq.funcs = &gmc_v10_0_ecc_funcs; | |
198 | } | |
f9df67e9 HZ |
199 | } |
200 | ||
f271fe18 | 201 | /** |
202 | * gmc_v10_0_use_invalidate_semaphore - judge whether to use semaphore | |
203 | * | |
204 | * @adev: amdgpu_device pointer | |
205 | * @vmhub: vmhub type | |
206 | * | |
207 | */ | |
208 | static bool gmc_v10_0_use_invalidate_semaphore(struct amdgpu_device *adev, | |
209 | uint32_t vmhub) | |
210 | { | |
f4caf584 | 211 | return ((vmhub == AMDGPU_MMHUB0(0)) && |
f271fe18 | 212 | (!amdgpu_sriov_vf(adev))); |
213 | } | |
214 | ||
ea930000 AS |
215 | static bool gmc_v10_0_get_atc_vmid_pasid_mapping_info( |
216 | struct amdgpu_device *adev, | |
217 | uint8_t vmid, uint16_t *p_pasid) | |
218 | { | |
219 | uint32_t value; | |
220 | ||
221 | value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) | |
222 | + vmid); | |
223 | *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK; | |
224 | ||
225 | return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK); | |
226 | } | |
227 | ||
f9df67e9 HZ |
228 | /* |
229 | * GART | |
230 | * VMID 0 is the physical GPU addresses as used by the kernel. | |
231 | * VMIDs 1-15 are used for userspace clients and are handled | |
232 | * by the amdgpu vm/hsa code. | |
233 | */ | |
234 | ||
a70cb217 CK |
235 | /** |
236 | * gmc_v10_0_flush_gpu_tlb - gart tlb flush callback | |
237 | * | |
238 | * @adev: amdgpu_device pointer | |
239 | * @vmid: vm instance to flush | |
240 | * @vmhub: vmhub type | |
241 | * @flush_type: the flush type | |
242 | * | |
243 | * Flush the TLB for the requested page table. | |
244 | */ | |
245 | static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, | |
246 | uint32_t vmhub, uint32_t flush_type) | |
f9df67e9 | 247 | { |
f271fe18 | 248 | bool use_semaphore = gmc_v10_0_use_invalidate_semaphore(adev, vmhub); |
f9df67e9 | 249 | struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; |
caa9f483 | 250 | u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type); |
f9df67e9 | 251 | /* Use register 17 for GART */ |
8612a435 | 252 | const unsigned int eng = 17; |
6ba3f59e | 253 | unsigned char hub_ip = 0; |
a70cb217 CK |
254 | u32 sem, req, ack; |
255 | unsigned int i; | |
256 | u32 tmp; | |
6ba3f59e | 257 | |
a70cb217 CK |
258 | sem = hub->vm_inv_eng0_sem + hub->eng_distance * eng; |
259 | req = hub->vm_inv_eng0_req + hub->eng_distance * eng; | |
260 | ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; | |
261 | ||
262 | /* flush hdp cache */ | |
263 | adev->hdp.funcs->flush_hdp(adev, NULL); | |
264 | ||
265 | /* For SRIOV run time, driver shouldn't access the register through MMIO | |
266 | * Directly use kiq to do the vm invalidation instead | |
267 | */ | |
268 | if (adev->gfx.kiq[0].ring.sched.ready && !adev->enable_mes && | |
e2e37888 | 269 | (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) { |
a70cb217 | 270 | amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req, |
bc3c5660 | 271 | 1 << vmid, GET_INST(GC, 0)); |
a70cb217 CK |
272 | return; |
273 | } | |
274 | ||
275 | hub_ip = (vmhub == AMDGPU_GFXHUB(0)) ? GC_HWIP : MMHUB_HWIP; | |
f9df67e9 | 276 | |
f920d1bb | 277 | spin_lock(&adev->gmc.invalidate_lock); |
278 | /* | |
279 | * It may lose gpuvm invalidate acknowldege state across power-gating | |
280 | * off cycle, add semaphore acquire before invalidation and semaphore | |
281 | * release after invalidation to avoid entering power gated state | |
282 | * to WA the Issue | |
283 | */ | |
284 | ||
285 | /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ | |
f271fe18 | 286 | if (use_semaphore) { |
f920d1bb | 287 | for (i = 0; i < adev->usec_timeout; i++) { |
288 | /* a read return value of 1 means semaphore acuqire */ | |
a70cb217 | 289 | tmp = RREG32_RLC_NO_KIQ(sem, hub_ip); |
f920d1bb | 290 | if (tmp & 0x1) |
291 | break; | |
292 | udelay(1); | |
293 | } | |
294 | ||
295 | if (i >= adev->usec_timeout) | |
296 | DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); | |
297 | } | |
298 | ||
a70cb217 | 299 | WREG32_RLC_NO_KIQ(req, inv_req, hub_ip); |
f9df67e9 | 300 | |
53499173 XY |
301 | /* |
302 | * Issue a dummy read to wait for the ACK register to be cleared | |
303 | * to avoid a false ACK due to the new fast GRBM interface. | |
304 | */ | |
f4caf584 | 305 | if ((vmhub == AMDGPU_GFXHUB(0)) && |
4e8303cf | 306 | (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(10, 3, 0))) |
a70cb217 | 307 | RREG32_RLC_NO_KIQ(req, hub_ip); |
53499173 | 308 | |
f9df67e9 HZ |
309 | /* Wait for ACK with a delay.*/ |
310 | for (i = 0; i < adev->usec_timeout; i++) { | |
a70cb217 | 311 | tmp = RREG32_RLC_NO_KIQ(ack, hub_ip); |
f9df67e9 HZ |
312 | tmp &= 1 << vmid; |
313 | if (tmp) | |
314 | break; | |
315 | ||
316 | udelay(1); | |
317 | } | |
318 | ||
f920d1bb | 319 | /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ |
f271fe18 | 320 | if (use_semaphore) |
a70cb217 | 321 | WREG32_RLC_NO_KIQ(sem, 0, hub_ip); |
f920d1bb | 322 | |
323 | spin_unlock(&adev->gmc.invalidate_lock); | |
324 | ||
a70cb217 CK |
325 | if (i >= adev->usec_timeout) |
326 | dev_err(adev->dev, "Timeout waiting for VM flush hub: %d!\n", | |
327 | vmhub); | |
f9df67e9 HZ |
328 | } |
329 | ||
ea930000 AS |
330 | /** |
331 | * gmc_v10_0_flush_gpu_tlb_pasid - tlb flush via pasid | |
332 | * | |
333 | * @adev: amdgpu_device pointer | |
334 | * @pasid: pasid to be flush | |
185ef9ef LJ |
335 | * @flush_type: the flush type |
336 | * @all_hub: Used with PACKET3_INVALIDATE_TLBS_ALL_HUB() | |
1bae03aa | 337 | * @inst: is used to select which instance of KIQ to use for the invalidation |
ea930000 AS |
338 | * |
339 | * Flush the TLB for the requested pasid. | |
340 | */ | |
3983c9fd CK |
341 | static void gmc_v10_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, |
342 | uint16_t pasid, uint32_t flush_type, | |
343 | bool all_hub, uint32_t inst) | |
ea930000 | 344 | { |
72cc9920 | 345 | uint16_t queried; |
ea930000 | 346 | int vmid, i; |
ea930000 | 347 | |
68fce5f0 | 348 | for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { |
72cc9920 CK |
349 | bool valid; |
350 | ||
351 | valid = gmc_v10_0_get_atc_vmid_pasid_mapping_info(adev, vmid, | |
352 | &queried); | |
353 | if (!valid || queried != pasid) | |
354 | continue; | |
355 | ||
356 | if (all_hub) { | |
357 | for_each_set_bit(i, adev->vmhubs_mask, | |
358 | AMDGPU_MAX_VMHUBS) | |
359 | gmc_v10_0_flush_gpu_tlb(adev, vmid, i, | |
360 | flush_type); | |
361 | } else { | |
362 | gmc_v10_0_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(0), | |
363 | flush_type); | |
ea930000 AS |
364 | } |
365 | } | |
ea930000 AS |
366 | } |
367 | ||
f9df67e9 | 368 | static uint64_t gmc_v10_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, |
8612a435 | 369 | unsigned int vmid, uint64_t pd_addr) |
f9df67e9 | 370 | { |
0530553b LM |
371 | bool use_semaphore = gmc_v10_0_use_invalidate_semaphore(ring->adev, ring->vm_hub); |
372 | struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; | |
caa9f483 | 373 | uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0); |
8612a435 | 374 | unsigned int eng = ring->vm_inv_eng; |
f9df67e9 | 375 | |
f920d1bb | 376 | /* |
377 | * It may lose gpuvm invalidate acknowldege state across power-gating | |
378 | * off cycle, add semaphore acquire before invalidation and semaphore | |
379 | * release after invalidation to avoid entering power gated state | |
380 | * to WA the Issue | |
381 | */ | |
382 | ||
383 | /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ | |
f271fe18 | 384 | if (use_semaphore) |
f920d1bb | 385 | /* a read return value of 1 means semaphore acuqire */ |
386 | amdgpu_ring_emit_reg_wait(ring, | |
af6c5c4f HR |
387 | hub->vm_inv_eng0_sem + |
388 | hub->eng_distance * eng, 0x1, 0x1); | |
f920d1bb | 389 | |
af6c5c4f HR |
390 | amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + |
391 | (hub->ctx_addr_distance * vmid), | |
f9df67e9 HZ |
392 | lower_32_bits(pd_addr)); |
393 | ||
af6c5c4f HR |
394 | amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + |
395 | (hub->ctx_addr_distance * vmid), | |
f9df67e9 HZ |
396 | upper_32_bits(pd_addr)); |
397 | ||
af6c5c4f HR |
398 | amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + |
399 | hub->eng_distance * eng, | |
400 | hub->vm_inv_eng0_ack + | |
401 | hub->eng_distance * eng, | |
589b64a7 | 402 | req, 1 << vmid); |
f9df67e9 | 403 | |
f920d1bb | 404 | /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ |
f271fe18 | 405 | if (use_semaphore) |
f920d1bb | 406 | /* |
407 | * add semaphore release after invalidation, | |
408 | * write with 0 means semaphore release | |
409 | */ | |
af6c5c4f HR |
410 | amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem + |
411 | hub->eng_distance * eng, 0); | |
f920d1bb | 412 | |
f9df67e9 HZ |
413 | return pd_addr; |
414 | } | |
415 | ||
8612a435 SS |
416 | static void gmc_v10_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int vmid, |
417 | unsigned int pasid) | |
f9df67e9 HZ |
418 | { |
419 | struct amdgpu_device *adev = ring->adev; | |
420 | uint32_t reg; | |
421 | ||
56443402 JX |
422 | /* MES fw manages IH_VMID_x_LUT updating */ |
423 | if (ring->is_mes_queue) | |
424 | return; | |
425 | ||
f4caf584 | 426 | if (ring->vm_hub == AMDGPU_GFXHUB(0)) |
f9df67e9 HZ |
427 | reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; |
428 | else | |
429 | reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; | |
430 | ||
431 | amdgpu_ring_emit_wreg(ring, reg, pasid); | |
432 | } | |
433 | ||
434 | /* | |
435 | * PTE format on NAVI 10: | |
436 | * 63:59 reserved | |
4005809b LG |
437 | * 58 reserved and for sienna_cichlid is used for MALL noalloc |
438 | * 57 reserved | |
f9df67e9 HZ |
439 | * 56 F |
440 | * 55 L | |
441 | * 54 reserved | |
442 | * 53:52 SW | |
443 | * 51 T | |
444 | * 50:48 mtype | |
445 | * 47:12 4k physical page base address | |
446 | * 11:7 fragment | |
447 | * 6 write | |
448 | * 5 read | |
449 | * 4 exe | |
450 | * 3 Z | |
451 | * 2 snooped | |
452 | * 1 system | |
453 | * 0 valid | |
454 | * | |
455 | * PDE format on NAVI 10: | |
456 | * 63:59 block fragment size | |
457 | * 58:55 reserved | |
458 | * 54 P | |
459 | * 53:48 reserved | |
460 | * 47:6 physical base address of PD or PTE | |
461 | * 5:3 reserved | |
462 | * 2 C | |
463 | * 1 system | |
464 | * 0 valid | |
465 | */ | |
f9df67e9 | 466 | |
71776b6d CK |
467 | static uint64_t gmc_v10_0_map_mtype(struct amdgpu_device *adev, uint32_t flags) |
468 | { | |
469 | switch (flags) { | |
f9df67e9 | 470 | case AMDGPU_VM_MTYPE_DEFAULT: |
71776b6d | 471 | return AMDGPU_PTE_MTYPE_NV10(MTYPE_NC); |
f9df67e9 | 472 | case AMDGPU_VM_MTYPE_NC: |
71776b6d | 473 | return AMDGPU_PTE_MTYPE_NV10(MTYPE_NC); |
f9df67e9 | 474 | case AMDGPU_VM_MTYPE_WC: |
71776b6d | 475 | return AMDGPU_PTE_MTYPE_NV10(MTYPE_WC); |
f9df67e9 | 476 | case AMDGPU_VM_MTYPE_CC: |
71776b6d | 477 | return AMDGPU_PTE_MTYPE_NV10(MTYPE_CC); |
f9df67e9 | 478 | case AMDGPU_VM_MTYPE_UC: |
71776b6d | 479 | return AMDGPU_PTE_MTYPE_NV10(MTYPE_UC); |
f9df67e9 | 480 | default: |
71776b6d | 481 | return AMDGPU_PTE_MTYPE_NV10(MTYPE_NC); |
f9df67e9 | 482 | } |
f9df67e9 HZ |
483 | } |
484 | ||
485 | static void gmc_v10_0_get_vm_pde(struct amdgpu_device *adev, int level, | |
486 | uint64_t *addr, uint64_t *flags) | |
487 | { | |
488 | if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) | |
0ca565ab | 489 | *addr = amdgpu_gmc_vram_mc2pa(adev, *addr); |
f9df67e9 HZ |
490 | BUG_ON(*addr & 0xFFFF00000000003FULL); |
491 | ||
492 | if (!adev->gmc.translate_further) | |
493 | return; | |
494 | ||
495 | if (level == AMDGPU_VM_PDB1) { | |
496 | /* Set the block fragment size */ | |
497 | if (!(*flags & AMDGPU_PDE_PTE)) | |
498 | *flags |= AMDGPU_PDE_BFS(0x9); | |
499 | ||
500 | } else if (level == AMDGPU_VM_PDB0) { | |
501 | if (*flags & AMDGPU_PDE_PTE) | |
502 | *flags &= ~AMDGPU_PDE_PTE; | |
503 | else | |
504 | *flags |= AMDGPU_PTE_TF; | |
505 | } | |
506 | } | |
507 | ||
cbfae36c CK |
508 | static void gmc_v10_0_get_vm_pte(struct amdgpu_device *adev, |
509 | struct amdgpu_bo_va_mapping *mapping, | |
510 | uint64_t *flags) | |
511 | { | |
d1a372af FK |
512 | struct amdgpu_bo *bo = mapping->bo_va->base.bo; |
513 | ||
cbfae36c CK |
514 | *flags &= ~AMDGPU_PTE_EXECUTABLE; |
515 | *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; | |
516 | ||
517 | *flags &= ~AMDGPU_PTE_MTYPE_NV10_MASK; | |
518 | *flags |= (mapping->flags & AMDGPU_PTE_MTYPE_NV10_MASK); | |
519 | ||
b6c65a2c CK |
520 | *flags &= ~AMDGPU_PTE_NOALLOC; |
521 | *flags |= (mapping->flags & AMDGPU_PTE_NOALLOC); | |
522 | ||
cbfae36c CK |
523 | if (mapping->flags & AMDGPU_PTE_PRT) { |
524 | *flags |= AMDGPU_PTE_PRT; | |
525 | *flags |= AMDGPU_PTE_SNOOPED; | |
526 | *flags |= AMDGPU_PTE_LOG; | |
527 | *flags |= AMDGPU_PTE_SYSTEM; | |
528 | *flags &= ~AMDGPU_PTE_VALID; | |
529 | } | |
d1a372af FK |
530 | |
531 | if (bo && bo->flags & (AMDGPU_GEM_CREATE_COHERENT | | |
5f248462 | 532 | AMDGPU_GEM_CREATE_EXT_COHERENT | |
d1a372af FK |
533 | AMDGPU_GEM_CREATE_UNCACHED)) |
534 | *flags = (*flags & ~AMDGPU_PTE_MTYPE_NV10_MASK) | | |
535 | AMDGPU_PTE_MTYPE_NV10(MTYPE_UC); | |
cbfae36c CK |
536 | } |
537 | ||
8612a435 | 538 | static unsigned int gmc_v10_0_get_vbios_fb_size(struct amdgpu_device *adev) |
7348c20a AD |
539 | { |
540 | u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); | |
8612a435 | 541 | unsigned int size; |
7348c20a AD |
542 | |
543 | if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { | |
544 | size = AMDGPU_VBIOS_VGA_ALLOCATION; | |
545 | } else { | |
546 | u32 viewport; | |
547 | u32 pitch; | |
548 | ||
549 | viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); | |
550 | pitch = RREG32_SOC15(DCE, 0, mmHUBPREQ0_DCSURF_SURFACE_PITCH); | |
551 | size = (REG_GET_FIELD(viewport, | |
552 | HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * | |
553 | REG_GET_FIELD(pitch, HUBPREQ0_DCSURF_SURFACE_PITCH, PITCH) * | |
554 | 4); | |
555 | } | |
556 | ||
557 | return size; | |
558 | } | |
559 | ||
f9df67e9 HZ |
560 | static const struct amdgpu_gmc_funcs gmc_v10_0_gmc_funcs = { |
561 | .flush_gpu_tlb = gmc_v10_0_flush_gpu_tlb, | |
ea930000 | 562 | .flush_gpu_tlb_pasid = gmc_v10_0_flush_gpu_tlb_pasid, |
f9df67e9 HZ |
563 | .emit_flush_gpu_tlb = gmc_v10_0_emit_flush_gpu_tlb, |
564 | .emit_pasid_mapping = gmc_v10_0_emit_pasid_mapping, | |
71776b6d | 565 | .map_mtype = gmc_v10_0_map_mtype, |
cbfae36c | 566 | .get_vm_pde = gmc_v10_0_get_vm_pde, |
7348c20a AD |
567 | .get_vm_pte = gmc_v10_0_get_vm_pte, |
568 | .get_vbios_fb_size = gmc_v10_0_get_vbios_fb_size, | |
f9df67e9 HZ |
569 | }; |
570 | ||
571 | static void gmc_v10_0_set_gmc_funcs(struct amdgpu_device *adev) | |
572 | { | |
573 | if (adev->gmc.gmc_funcs == NULL) | |
574 | adev->gmc.gmc_funcs = &gmc_v10_0_gmc_funcs; | |
575 | } | |
576 | ||
01eee24f JC |
577 | static void gmc_v10_0_set_umc_funcs(struct amdgpu_device *adev) |
578 | { | |
4e8303cf | 579 | switch (amdgpu_ip_version(adev, UMC_HWIP, 0)) { |
5db9d065 | 580 | case IP_VERSION(8, 7, 0): |
01eee24f JC |
581 | adev->umc.max_ras_err_cnt_per_query = UMC_V8_7_TOTAL_CHANNEL_NUM; |
582 | adev->umc.channel_inst_num = UMC_V8_7_CHANNEL_INSTANCE_NUM; | |
583 | adev->umc.umc_inst_num = UMC_V8_7_UMC_INSTANCE_NUM; | |
584 | adev->umc.channel_offs = UMC_V8_7_PER_CHANNEL_OFFSET_SIENNA; | |
e69c7857 | 585 | adev->umc.retire_unit = 1; |
01eee24f | 586 | adev->umc.channel_idx_tbl = &umc_v8_7_channel_idx_tbl[0][0]; |
efe17d5a | 587 | adev->umc.ras = &umc_v8_7_ras; |
01eee24f JC |
588 | break; |
589 | default: | |
590 | break; | |
591 | } | |
592 | } | |
593 | ||
9fb1506e OZ |
594 | static void gmc_v10_0_set_mmhub_funcs(struct amdgpu_device *adev) |
595 | { | |
4e8303cf | 596 | switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { |
5db9d065 AD |
597 | case IP_VERSION(2, 3, 0): |
598 | case IP_VERSION(2, 4, 0): | |
a142606d | 599 | case IP_VERSION(2, 4, 1): |
4d8d75a4 HR |
600 | adev->mmhub.funcs = &mmhub_v2_3_funcs; |
601 | break; | |
602 | default: | |
603 | adev->mmhub.funcs = &mmhub_v2_0_funcs; | |
604 | break; | |
605 | } | |
9fb1506e OZ |
606 | } |
607 | ||
8ffff9b4 OZ |
608 | static void gmc_v10_0_set_gfxhub_funcs(struct amdgpu_device *adev) |
609 | { | |
4e8303cf | 610 | switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { |
5db9d065 AD |
611 | case IP_VERSION(10, 3, 0): |
612 | case IP_VERSION(10, 3, 2): | |
613 | case IP_VERSION(10, 3, 1): | |
614 | case IP_VERSION(10, 3, 4): | |
615 | case IP_VERSION(10, 3, 5): | |
a142606d | 616 | case IP_VERSION(10, 3, 6): |
5db9d065 | 617 | case IP_VERSION(10, 3, 3): |
97437f47 | 618 | case IP_VERSION(10, 3, 7): |
8ffff9b4 OZ |
619 | adev->gfxhub.funcs = &gfxhub_v2_1_funcs; |
620 | break; | |
621 | default: | |
622 | adev->gfxhub.funcs = &gfxhub_v2_0_funcs; | |
623 | break; | |
624 | } | |
625 | } | |
626 | ||
627 | ||
f9df67e9 HZ |
628 | static int gmc_v10_0_early_init(void *handle) |
629 | { | |
630 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
631 | ||
9fb1506e | 632 | gmc_v10_0_set_mmhub_funcs(adev); |
8ffff9b4 | 633 | gmc_v10_0_set_gfxhub_funcs(adev); |
f9df67e9 HZ |
634 | gmc_v10_0_set_gmc_funcs(adev); |
635 | gmc_v10_0_set_irq_funcs(adev); | |
01eee24f | 636 | gmc_v10_0_set_umc_funcs(adev); |
f9df67e9 HZ |
637 | |
638 | adev->gmc.shared_aperture_start = 0x2000000000000000ULL; | |
639 | adev->gmc.shared_aperture_end = | |
640 | adev->gmc.shared_aperture_start + (4ULL << 30) - 1; | |
641 | adev->gmc.private_aperture_start = 0x1000000000000000ULL; | |
642 | adev->gmc.private_aperture_end = | |
643 | adev->gmc.private_aperture_start + (4ULL << 30) - 1; | |
e77673d1 | 644 | adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF; |
f9df67e9 HZ |
645 | |
646 | return 0; | |
647 | } | |
648 | ||
649 | static int gmc_v10_0_late_init(void *handle) | |
650 | { | |
651 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
5677c520 | 652 | int r; |
f9df67e9 | 653 | |
5677c520 AD |
654 | r = amdgpu_gmc_allocate_vm_inv_eng(adev); |
655 | if (r) | |
656 | return r; | |
f9df67e9 | 657 | |
0ad7a64d JC |
658 | r = amdgpu_gmc_ras_late_init(adev); |
659 | if (r) | |
660 | return r; | |
661 | ||
f9df67e9 HZ |
662 | return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); |
663 | } | |
664 | ||
665 | static void gmc_v10_0_vram_gtt_location(struct amdgpu_device *adev, | |
666 | struct amdgpu_gmc *mc) | |
667 | { | |
668 | u64 base = 0; | |
669 | ||
8ffff9b4 | 670 | base = adev->gfxhub.funcs->get_fb_location(adev); |
f9df67e9 | 671 | |
fdb8483b JC |
672 | /* add the xgmi offset of the physical node */ |
673 | base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; | |
674 | ||
bff3315b | 675 | amdgpu_gmc_set_agp_default(adev, mc); |
f9df67e9 | 676 | amdgpu_gmc_vram_location(adev, &adev->gmc, base); |
917f91d8 | 677 | amdgpu_gmc_gart_location(adev, mc, AMDGPU_GART_PLACEMENT_BEST_FIT); |
61fc9369 | 678 | if (!amdgpu_sriov_vf(adev) && (amdgpu_agp == 1)) |
de59b699 | 679 | amdgpu_gmc_agp_location(adev, mc); |
f9df67e9 HZ |
680 | |
681 | /* base offset of vram pages */ | |
8ffff9b4 | 682 | adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev); |
fdb8483b JC |
683 | |
684 | /* add the xgmi offset of the physical node */ | |
685 | adev->vm_manager.vram_base_offset += | |
686 | adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; | |
f9df67e9 HZ |
687 | } |
688 | ||
689 | /** | |
690 | * gmc_v10_0_mc_init - initialize the memory controller driver params | |
691 | * | |
692 | * @adev: amdgpu_device pointer | |
693 | * | |
694 | * Look up the amount of vram, vram width, and decide how to place | |
695 | * vram and gart within the GPU's physical address space. | |
696 | * Returns 0 for success. | |
697 | */ | |
698 | static int gmc_v10_0_mc_init(struct amdgpu_device *adev) | |
699 | { | |
78b7dfd9 | 700 | int r; |
f9df67e9 HZ |
701 | |
702 | /* size in MB on si */ | |
703 | adev->gmc.mc_vram_size = | |
bebc0762 | 704 | adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL; |
f9df67e9 | 705 | adev->gmc.real_vram_size = adev->gmc.mc_vram_size; |
78b7dfd9 AS |
706 | |
707 | if (!(adev->flags & AMD_IS_APU)) { | |
708 | r = amdgpu_device_resize_fb_bar(adev); | |
709 | if (r) | |
710 | return r; | |
711 | } | |
712 | adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); | |
713 | adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); | |
f9df67e9 | 714 | |
6405e627 | 715 | #ifdef CONFIG_X86_64 |
b818a5d3 | 716 | if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) { |
6405e627 HR |
717 | adev->gmc.aper_base = adev->gfxhub.funcs->get_mc_fb_offset(adev); |
718 | adev->gmc.aper_size = adev->gmc.real_vram_size; | |
719 | } | |
720 | #endif | |
721 | ||
78b7dfd9 | 722 | adev->gmc.visible_vram_size = adev->gmc.aper_size; |
f9df67e9 HZ |
723 | |
724 | /* set the gart size */ | |
88c775bb | 725 | if (amdgpu_gart_size == -1) { |
4e8303cf | 726 | switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { |
88c775bb AD |
727 | default: |
728 | adev->gmc.gart_size = 512ULL << 20; | |
729 | break; | |
730 | case IP_VERSION(10, 3, 1): /* DCE SG support */ | |
731 | case IP_VERSION(10, 3, 3): /* DCE SG support */ | |
732 | case IP_VERSION(10, 3, 6): /* DCE SG support */ | |
733 | case IP_VERSION(10, 3, 7): /* DCE SG support */ | |
734 | adev->gmc.gart_size = 1024ULL << 20; | |
735 | break; | |
736 | } | |
737 | } else { | |
f9df67e9 | 738 | adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; |
88c775bb | 739 | } |
f9df67e9 HZ |
740 | |
741 | gmc_v10_0_vram_gtt_location(adev, &adev->gmc); | |
742 | ||
743 | return 0; | |
744 | } | |
745 | ||
746 | static int gmc_v10_0_gart_init(struct amdgpu_device *adev) | |
747 | { | |
748 | int r; | |
749 | ||
750 | if (adev->gart.bo) { | |
751 | WARN(1, "NAVI10 PCIE GART already initialized\n"); | |
752 | return 0; | |
753 | } | |
754 | ||
755 | /* Initialize common gart structure */ | |
756 | r = amdgpu_gart_init(adev); | |
757 | if (r) | |
758 | return r; | |
759 | ||
760 | adev->gart.table_size = adev->gart.num_gpu_pages * 8; | |
761 | adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_NV10(MTYPE_UC) | | |
762 | AMDGPU_PTE_EXECUTABLE; | |
763 | ||
764 | return amdgpu_gart_table_vram_alloc(adev); | |
765 | } | |
766 | ||
f9df67e9 HZ |
767 | static int gmc_v10_0_sw_init(void *handle) |
768 | { | |
ad02e08e | 769 | int r, vram_width = 0, vram_type = 0, vram_vendor = 0; |
f9df67e9 HZ |
770 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
771 | ||
8ffff9b4 | 772 | adev->gfxhub.funcs->init(adev); |
0b3df16b | 773 | |
9fb1506e | 774 | adev->mmhub.funcs->init(adev); |
f9df67e9 HZ |
775 | |
776 | spin_lock_init(&adev->gmc.invalidate_lock); | |
777 | ||
6405e627 HR |
778 | if ((adev->flags & AMD_IS_APU) && amdgpu_emu_mode == 1) { |
779 | adev->gmc.vram_type = AMDGPU_VRAM_TYPE_DDR4; | |
780 | adev->gmc.vram_width = 64; | |
781 | } else if (amdgpu_emu_mode == 1) { | |
0b3df16b | 782 | adev->gmc.vram_type = AMDGPU_VRAM_TYPE_GDDR6; |
631cdbd2 | 783 | adev->gmc.vram_width = 1 * 128; /* numchan * chansize */ |
0b3df16b LG |
784 | } else { |
785 | r = amdgpu_atomfirmware_get_vram_info(adev, | |
786 | &vram_width, &vram_type, &vram_vendor); | |
787 | adev->gmc.vram_width = vram_width; | |
788 | ||
789 | adev->gmc.vram_type = vram_type; | |
790 | adev->gmc.vram_vendor = vram_vendor; | |
791 | } | |
631cdbd2 | 792 | |
4e8303cf | 793 | switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { |
053d35de AD |
794 | case IP_VERSION(10, 3, 0): |
795 | adev->gmc.mall_size = 128 * 1024 * 1024; | |
796 | break; | |
797 | case IP_VERSION(10, 3, 2): | |
798 | adev->gmc.mall_size = 96 * 1024 * 1024; | |
799 | break; | |
800 | case IP_VERSION(10, 3, 4): | |
801 | adev->gmc.mall_size = 32 * 1024 * 1024; | |
802 | break; | |
803 | case IP_VERSION(10, 3, 5): | |
804 | adev->gmc.mall_size = 16 * 1024 * 1024; | |
805 | break; | |
806 | default: | |
807 | adev->gmc.mall_size = 0; | |
808 | break; | |
809 | } | |
810 | ||
4e8303cf | 811 | switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { |
5db9d065 AD |
812 | case IP_VERSION(10, 1, 10): |
813 | case IP_VERSION(10, 1, 1): | |
814 | case IP_VERSION(10, 1, 2): | |
815 | case IP_VERSION(10, 1, 3): | |
f9ed188d | 816 | case IP_VERSION(10, 1, 4): |
5db9d065 AD |
817 | case IP_VERSION(10, 3, 0): |
818 | case IP_VERSION(10, 3, 2): | |
819 | case IP_VERSION(10, 3, 1): | |
820 | case IP_VERSION(10, 3, 4): | |
821 | case IP_VERSION(10, 3, 5): | |
a142606d | 822 | case IP_VERSION(10, 3, 6): |
5db9d065 | 823 | case IP_VERSION(10, 3, 3): |
97437f47 | 824 | case IP_VERSION(10, 3, 7): |
d9426c3d LM |
825 | set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask); |
826 | set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask); | |
f9df67e9 HZ |
827 | /* |
828 | * To fulfill 4-level page support, | |
4a0e815f | 829 | * vm size is 256TB (48bit), maximum size of Navi10/Navi14/Navi12, |
f9df67e9 HZ |
830 | * block size 512 (9bit) |
831 | */ | |
832 | amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); | |
833 | break; | |
834 | default: | |
835 | break; | |
836 | } | |
837 | ||
838 | /* This interrupt is VMC page fault.*/ | |
839 | r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, | |
840 | VMC_1_0__SRCID__VM_FAULT, | |
841 | &adev->gmc.vm_fault); | |
5021e9a8 ND |
842 | |
843 | if (r) | |
844 | return r; | |
845 | ||
f9df67e9 HZ |
846 | r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, |
847 | UTCL2_1_0__SRCID__FAULT, | |
848 | &adev->gmc.vm_fault); | |
849 | if (r) | |
850 | return r; | |
851 | ||
01eee24f JC |
852 | if (!amdgpu_sriov_vf(adev)) { |
853 | /* interrupt sent to DF. */ | |
854 | r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0, | |
855 | &adev->gmc.ecc_irq); | |
856 | if (r) | |
857 | return r; | |
858 | } | |
859 | ||
f9df67e9 HZ |
860 | /* |
861 | * Set the internal MC address mask This is the max address of the GPU's | |
862 | * internal address space. | |
863 | */ | |
864 | adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ | |
865 | ||
244511f3 | 866 | r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44)); |
f9df67e9 | 867 | if (r) { |
e2710187 | 868 | dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n"); |
244511f3 | 869 | return r; |
f9df67e9 HZ |
870 | } |
871 | ||
3616d49d AL |
872 | adev->need_swiotlb = drm_need_swiotlb(44); |
873 | ||
f9df67e9 HZ |
874 | r = gmc_v10_0_mc_init(adev); |
875 | if (r) | |
876 | return r; | |
877 | ||
7348c20a | 878 | amdgpu_gmc_get_vbios_allocations(adev); |
f9df67e9 HZ |
879 | |
880 | /* Memory manager */ | |
881 | r = amdgpu_bo_init(adev); | |
882 | if (r) | |
883 | return r; | |
884 | ||
885 | r = gmc_v10_0_gart_init(adev); | |
886 | if (r) | |
887 | return r; | |
888 | ||
889 | /* | |
890 | * number of VMs | |
891 | * VMID 0 is reserved for System | |
892 | * amdgpu graphics/compute will use VMIDs 1-7 | |
893 | * amdkfd will use VMIDs 8-15 | |
894 | */ | |
40111ec2 | 895 | adev->vm_manager.first_kfd_vmid = 8; |
f9df67e9 HZ |
896 | |
897 | amdgpu_vm_manager_init(adev); | |
898 | ||
a6dcf9a7 HZ |
899 | r = amdgpu_gmc_ras_sw_init(adev); |
900 | if (r) | |
901 | return r; | |
902 | ||
f9df67e9 HZ |
903 | return 0; |
904 | } | |
905 | ||
906 | /** | |
2cce318c | 907 | * gmc_v10_0_gart_fini - vm fini callback |
f9df67e9 HZ |
908 | * |
909 | * @adev: amdgpu_device pointer | |
910 | * | |
911 | * Tears down the driver GART/VM setup (CIK). | |
912 | */ | |
913 | static void gmc_v10_0_gart_fini(struct amdgpu_device *adev) | |
914 | { | |
915 | amdgpu_gart_table_vram_free(adev); | |
f9df67e9 HZ |
916 | } |
917 | ||
918 | static int gmc_v10_0_sw_fini(void *handle) | |
919 | { | |
920 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
921 | ||
922 | amdgpu_vm_manager_fini(adev); | |
923 | gmc_v10_0_gart_fini(adev); | |
924 | amdgpu_gem_force_release(adev); | |
925 | amdgpu_bo_fini(adev); | |
926 | ||
927 | return 0; | |
928 | } | |
929 | ||
930 | static void gmc_v10_0_init_golden_registers(struct amdgpu_device *adev) | |
931 | { | |
f9df67e9 HZ |
932 | } |
933 | ||
934 | /** | |
935 | * gmc_v10_0_gart_enable - gart enable | |
936 | * | |
937 | * @adev: amdgpu_device pointer | |
938 | */ | |
939 | static int gmc_v10_0_gart_enable(struct amdgpu_device *adev) | |
940 | { | |
941 | int r; | |
942 | bool value; | |
f9df67e9 HZ |
943 | |
944 | if (adev->gart.bo == NULL) { | |
945 | dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); | |
946 | return -EINVAL; | |
947 | } | |
948 | ||
1b08dfb8 | 949 | amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr); |
d5d29009 AD |
950 | |
951 | if (!adev->in_s0ix) { | |
952 | r = adev->gfxhub.funcs->gart_enable(adev); | |
953 | if (r) | |
954 | return r; | |
955 | } | |
f9df67e9 | 956 | |
9fb1506e | 957 | r = adev->mmhub.funcs->gart_enable(adev); |
f9df67e9 HZ |
958 | if (r) |
959 | return r; | |
960 | ||
bf087285 | 961 | adev->hdp.funcs->init_registers(adev); |
f9df67e9 HZ |
962 | |
963 | /* Flush HDP after it is initialized */ | |
bf087285 | 964 | adev->hdp.funcs->flush_hdp(adev, NULL); |
f9df67e9 HZ |
965 | |
966 | value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ? | |
967 | false : true; | |
968 | ||
d5d29009 AD |
969 | if (!adev->in_s0ix) |
970 | adev->gfxhub.funcs->set_fault_enable_default(adev, value); | |
9fb1506e | 971 | adev->mmhub.funcs->set_fault_enable_default(adev, value); |
f4caf584 | 972 | gmc_v10_0_flush_gpu_tlb(adev, 0, AMDGPU_MMHUB0(0), 0); |
d5d29009 | 973 | if (!adev->in_s0ix) |
f4caf584 | 974 | gmc_v10_0_flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0); |
f9df67e9 HZ |
975 | |
976 | DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", | |
8612a435 | 977 | (unsigned int)(adev->gmc.gart_size >> 20), |
f9df67e9 HZ |
978 | (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); |
979 | ||
f9df67e9 HZ |
980 | return 0; |
981 | } | |
982 | ||
983 | static int gmc_v10_0_hw_init(void *handle) | |
984 | { | |
f9df67e9 | 985 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
72cc9920 CK |
986 | int r; |
987 | ||
988 | adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode; | |
f9df67e9 HZ |
989 | |
990 | /* The sequence of these two function calls matters.*/ | |
991 | gmc_v10_0_init_golden_registers(adev); | |
992 | ||
b3accd6f XH |
993 | /* |
994 | * harvestable groups in gc_utcl2 need to be programmed before any GFX block | |
995 | * register setup within GMC, or else system hang when harvesting SA. | |
996 | */ | |
d5d29009 | 997 | if (!adev->in_s0ix && adev->gfxhub.funcs && adev->gfxhub.funcs->utcl2_harvest) |
b3accd6f XH |
998 | adev->gfxhub.funcs->utcl2_harvest(adev); |
999 | ||
f9df67e9 HZ |
1000 | r = gmc_v10_0_gart_enable(adev); |
1001 | if (r) | |
1002 | return r; | |
1003 | ||
479e3b02 XD |
1004 | if (amdgpu_emu_mode == 1) { |
1005 | r = amdgpu_gmc_vram_checking(adev); | |
1006 | if (r) | |
1007 | return r; | |
1008 | } | |
1009 | ||
01eee24f JC |
1010 | if (adev->umc.funcs && adev->umc.funcs->init_registers) |
1011 | adev->umc.funcs->init_registers(adev); | |
1012 | ||
f9df67e9 HZ |
1013 | return 0; |
1014 | } | |
1015 | ||
1016 | /** | |
1017 | * gmc_v10_0_gart_disable - gart disable | |
1018 | * | |
1019 | * @adev: amdgpu_device pointer | |
1020 | * | |
1021 | * This disables all VM page table. | |
1022 | */ | |
1023 | static void gmc_v10_0_gart_disable(struct amdgpu_device *adev) | |
1024 | { | |
d5d29009 AD |
1025 | if (!adev->in_s0ix) |
1026 | adev->gfxhub.funcs->gart_disable(adev); | |
9fb1506e | 1027 | adev->mmhub.funcs->gart_disable(adev); |
f9df67e9 HZ |
1028 | } |
1029 | ||
1030 | static int gmc_v10_0_hw_fini(void *handle) | |
1031 | { | |
1032 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1033 | ||
71cf9e72 LS |
1034 | gmc_v10_0_gart_disable(adev); |
1035 | ||
f9df67e9 HZ |
1036 | if (amdgpu_sriov_vf(adev)) { |
1037 | /* full access mode, so don't touch any GMC register */ | |
1038 | DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); | |
1039 | return 0; | |
1040 | } | |
1041 | ||
1042 | amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); | |
f9df67e9 HZ |
1043 | |
1044 | return 0; | |
1045 | } | |
1046 | ||
1047 | static int gmc_v10_0_suspend(void *handle) | |
1048 | { | |
1049 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1050 | ||
1051 | gmc_v10_0_hw_fini(adev); | |
1052 | ||
1053 | return 0; | |
1054 | } | |
1055 | ||
1056 | static int gmc_v10_0_resume(void *handle) | |
1057 | { | |
1058 | int r; | |
1059 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1060 | ||
1061 | r = gmc_v10_0_hw_init(adev); | |
1062 | if (r) | |
1063 | return r; | |
1064 | ||
1065 | amdgpu_vmid_reset_all(adev); | |
1066 | ||
1067 | return 0; | |
1068 | } | |
1069 | ||
1070 | static bool gmc_v10_0_is_idle(void *handle) | |
1071 | { | |
1072 | /* MC is always ready in GMC v10.*/ | |
1073 | return true; | |
1074 | } | |
1075 | ||
1076 | static int gmc_v10_0_wait_for_idle(void *handle) | |
1077 | { | |
1078 | /* There is no need to wait for MC idle in GMC v10.*/ | |
1079 | return 0; | |
1080 | } | |
1081 | ||
1082 | static int gmc_v10_0_soft_reset(void *handle) | |
1083 | { | |
1084 | return 0; | |
1085 | } | |
1086 | ||
1087 | static int gmc_v10_0_set_clockgating_state(void *handle, | |
1088 | enum amd_clockgating_state state) | |
1089 | { | |
1090 | int r; | |
1091 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1092 | ||
3bbeaa30 PL |
1093 | /* |
1094 | * The issue mmhub can't disconnect from DF with MMHUB clock gating being disabled | |
1095 | * is a new problem observed at DF 3.0.3, however with the same suspend sequence not | |
1096 | * seen any issue on the DF 3.0.2 series platform. | |
1097 | */ | |
4e8303cf LL |
1098 | if (adev->in_s0ix && |
1099 | amdgpu_ip_version(adev, DF_HWIP, 0) > IP_VERSION(3, 0, 2)) { | |
3bbeaa30 PL |
1100 | dev_dbg(adev->dev, "keep mmhub clock gating being enabled for s0ix\n"); |
1101 | return 0; | |
1102 | } | |
1103 | ||
9fb1506e | 1104 | r = adev->mmhub.funcs->set_clockgating(adev, state); |
f9df67e9 HZ |
1105 | if (r) |
1106 | return r; | |
1107 | ||
4e8303cf | 1108 | if (amdgpu_ip_version(adev, ATHUB_HWIP, 0) >= IP_VERSION(2, 1, 0)) |
920a4cd3 LG |
1109 | return athub_v2_1_set_clockgating(adev, state); |
1110 | else | |
1111 | return athub_v2_0_set_clockgating(adev, state); | |
f9df67e9 HZ |
1112 | } |
1113 | ||
25faeddc | 1114 | static void gmc_v10_0_get_clockgating_state(void *handle, u64 *flags) |
f9df67e9 HZ |
1115 | { |
1116 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
1117 | ||
4e8303cf LL |
1118 | if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(10, 1, 3) || |
1119 | amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(10, 1, 4)) | |
d2895ec4 LY |
1120 | return; |
1121 | ||
9fb1506e | 1122 | adev->mmhub.funcs->get_clockgating(adev, flags); |
f9df67e9 | 1123 | |
4e8303cf | 1124 | if (amdgpu_ip_version(adev, ATHUB_HWIP, 0) >= IP_VERSION(2, 1, 0)) |
920a4cd3 LG |
1125 | athub_v2_1_get_clockgating(adev, flags); |
1126 | else | |
1127 | athub_v2_0_get_clockgating(adev, flags); | |
f9df67e9 HZ |
1128 | } |
1129 | ||
1130 | static int gmc_v10_0_set_powergating_state(void *handle, | |
1131 | enum amd_powergating_state state) | |
1132 | { | |
1133 | return 0; | |
1134 | } | |
1135 | ||
1136 | const struct amd_ip_funcs gmc_v10_0_ip_funcs = { | |
1137 | .name = "gmc_v10_0", | |
1138 | .early_init = gmc_v10_0_early_init, | |
1139 | .late_init = gmc_v10_0_late_init, | |
1140 | .sw_init = gmc_v10_0_sw_init, | |
1141 | .sw_fini = gmc_v10_0_sw_fini, | |
1142 | .hw_init = gmc_v10_0_hw_init, | |
1143 | .hw_fini = gmc_v10_0_hw_fini, | |
1144 | .suspend = gmc_v10_0_suspend, | |
1145 | .resume = gmc_v10_0_resume, | |
1146 | .is_idle = gmc_v10_0_is_idle, | |
1147 | .wait_for_idle = gmc_v10_0_wait_for_idle, | |
1148 | .soft_reset = gmc_v10_0_soft_reset, | |
1149 | .set_clockgating_state = gmc_v10_0_set_clockgating_state, | |
1150 | .set_powergating_state = gmc_v10_0_set_powergating_state, | |
1151 | .get_clockgating_state = gmc_v10_0_get_clockgating_state, | |
1152 | }; | |
1153 | ||
8612a435 | 1154 | const struct amdgpu_ip_block_version gmc_v10_0_ip_block = { |
f9df67e9 HZ |
1155 | .type = AMD_IP_BLOCK_TYPE_GMC, |
1156 | .major = 10, | |
1157 | .minor = 0, | |
1158 | .rev = 0, | |
1159 | .funcs = &gmc_v10_0_ip_funcs, | |
1160 | }; |