2 * Copyright 2009 Jerome Glisse.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
33 #include <linux/dma-mapping.h>
34 #include <linux/iommu.h>
35 #include <linux/pagemap.h>
36 #include <linux/sched/task.h>
37 #include <linux/sched/mm.h>
38 #include <linux/seq_file.h>
39 #include <linux/slab.h>
40 #include <linux/swap.h>
41 #include <linux/dma-buf.h>
42 #include <linux/sizes.h>
43 #include <linux/module.h>
45 #include <drm/drm_drv.h>
46 #include <drm/ttm/ttm_bo.h>
47 #include <drm/ttm/ttm_placement.h>
48 #include <drm/ttm/ttm_range_manager.h>
49 #include <drm/ttm/ttm_tt.h>
51 #include <drm/amdgpu_drm.h>
54 #include "amdgpu_object.h"
55 #include "amdgpu_trace.h"
56 #include "amdgpu_amdkfd.h"
57 #include "amdgpu_sdma.h"
58 #include "amdgpu_ras.h"
59 #include "amdgpu_hmm.h"
60 #include "amdgpu_atomfirmware.h"
61 #include "amdgpu_res_cursor.h"
62 #include "bif/bif_4_1_d.h"
64 MODULE_IMPORT_NS(DMA_BUF
);
66 #define AMDGPU_TTM_VRAM_MAX_DW_READ ((size_t)128)
68 static int amdgpu_ttm_backend_bind(struct ttm_device
*bdev
,
70 struct ttm_resource
*bo_mem
);
71 static void amdgpu_ttm_backend_unbind(struct ttm_device
*bdev
,
74 static int amdgpu_ttm_init_on_chip(struct amdgpu_device
*adev
,
76 uint64_t size_in_page
)
78 return ttm_range_man_init(&adev
->mman
.bdev
, type
,
83 * amdgpu_evict_flags - Compute placement flags
85 * @bo: The buffer object to evict
86 * @placement: Possible destination(s) for evicted BO
88 * Fill in placement data when ttm_bo_evict() is called
90 static void amdgpu_evict_flags(struct ttm_buffer_object
*bo
,
91 struct ttm_placement
*placement
)
93 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->bdev
);
94 struct amdgpu_bo
*abo
;
95 static const struct ttm_place placements
= {
98 .mem_type
= TTM_PL_SYSTEM
,
102 /* Don't handle scatter gather BOs */
103 if (bo
->type
== ttm_bo_type_sg
) {
104 placement
->num_placement
= 0;
108 /* Object isn't an AMDGPU object so ignore */
109 if (!amdgpu_bo_is_amdgpu_bo(bo
)) {
110 placement
->placement
= &placements
;
111 placement
->num_placement
= 1;
115 abo
= ttm_to_amdgpu_bo(bo
);
116 if (abo
->flags
& AMDGPU_GEM_CREATE_DISCARDABLE
) {
117 placement
->num_placement
= 0;
121 switch (bo
->resource
->mem_type
) {
125 case AMDGPU_PL_DOORBELL
:
126 placement
->num_placement
= 0;
130 if (!adev
->mman
.buffer_funcs_enabled
) {
131 /* Move to system memory */
132 amdgpu_bo_placement_from_domain(abo
, AMDGPU_GEM_DOMAIN_CPU
);
134 } else if (!amdgpu_gmc_vram_full_visible(&adev
->gmc
) &&
135 !(abo
->flags
& AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
) &&
136 amdgpu_bo_in_cpu_visible_vram(abo
)) {
138 /* Try evicting to the CPU inaccessible part of VRAM
139 * first, but only set GTT as busy placement, so this
140 * BO will be evicted to GTT rather than causing other
141 * BOs to be evicted from VRAM
143 amdgpu_bo_placement_from_domain(abo
, AMDGPU_GEM_DOMAIN_VRAM
|
144 AMDGPU_GEM_DOMAIN_GTT
|
145 AMDGPU_GEM_DOMAIN_CPU
);
146 abo
->placements
[0].fpfn
= adev
->gmc
.visible_vram_size
>> PAGE_SHIFT
;
147 abo
->placements
[0].lpfn
= 0;
148 abo
->placements
[0].flags
|= TTM_PL_FLAG_DESIRED
;
150 /* Move to GTT memory */
151 amdgpu_bo_placement_from_domain(abo
, AMDGPU_GEM_DOMAIN_GTT
|
152 AMDGPU_GEM_DOMAIN_CPU
);
156 case AMDGPU_PL_PREEMPT
:
158 amdgpu_bo_placement_from_domain(abo
, AMDGPU_GEM_DOMAIN_CPU
);
161 *placement
= abo
->placement
;
165 * amdgpu_ttm_map_buffer - Map memory into the GART windows
166 * @bo: buffer object to map
167 * @mem: memory object to map
168 * @mm_cur: range to map
169 * @window: which GART window to use
170 * @ring: DMA ring to use for the copy
171 * @tmz: if we should setup a TMZ enabled mapping
172 * @size: in number of bytes to map, out number of bytes mapped
173 * @addr: resulting address inside the MC address space
175 * Setup one of the GART windows to access a specific piece of memory or return
176 * the physical address for local memory.
178 static int amdgpu_ttm_map_buffer(struct ttm_buffer_object
*bo
,
179 struct ttm_resource
*mem
,
180 struct amdgpu_res_cursor
*mm_cur
,
181 unsigned int window
, struct amdgpu_ring
*ring
,
182 bool tmz
, uint64_t *size
, uint64_t *addr
)
184 struct amdgpu_device
*adev
= ring
->adev
;
185 unsigned int offset
, num_pages
, num_dw
, num_bytes
;
186 uint64_t src_addr
, dst_addr
;
187 struct amdgpu_job
*job
;
193 BUG_ON(adev
->mman
.buffer_funcs
->copy_max_bytes
<
194 AMDGPU_GTT_MAX_TRANSFER_SIZE
* 8);
196 if (WARN_ON(mem
->mem_type
== AMDGPU_PL_PREEMPT
))
199 /* Map only what can't be accessed directly */
200 if (!tmz
&& mem
->start
!= AMDGPU_BO_INVALID_OFFSET
) {
201 *addr
= amdgpu_ttm_domain_start(adev
, mem
->mem_type
) +
208 * If start begins at an offset inside the page, then adjust the size
209 * and addr accordingly
211 offset
= mm_cur
->start
& ~PAGE_MASK
;
213 num_pages
= PFN_UP(*size
+ offset
);
214 num_pages
= min_t(uint32_t, num_pages
, AMDGPU_GTT_MAX_TRANSFER_SIZE
);
216 *size
= min(*size
, (uint64_t)num_pages
* PAGE_SIZE
- offset
);
218 *addr
= adev
->gmc
.gart_start
;
219 *addr
+= (u64
)window
* AMDGPU_GTT_MAX_TRANSFER_SIZE
*
220 AMDGPU_GPU_PAGE_SIZE
;
223 num_dw
= ALIGN(adev
->mman
.buffer_funcs
->copy_num_dw
, 8);
224 num_bytes
= num_pages
* 8 * AMDGPU_GPU_PAGES_IN_CPU_PAGE
;
226 r
= amdgpu_job_alloc_with_ib(adev
, &adev
->mman
.high_pr
,
227 AMDGPU_FENCE_OWNER_UNDEFINED
,
228 num_dw
* 4 + num_bytes
,
229 AMDGPU_IB_POOL_DELAYED
, &job
);
233 src_addr
= num_dw
* 4;
234 src_addr
+= job
->ibs
[0].gpu_addr
;
236 dst_addr
= amdgpu_bo_gpu_offset(adev
->gart
.bo
);
237 dst_addr
+= window
* AMDGPU_GTT_MAX_TRANSFER_SIZE
* 8;
238 amdgpu_emit_copy_buffer(adev
, &job
->ibs
[0], src_addr
,
239 dst_addr
, num_bytes
, false);
241 amdgpu_ring_pad_ib(ring
, &job
->ibs
[0]);
242 WARN_ON(job
->ibs
[0].length_dw
> num_dw
);
244 flags
= amdgpu_ttm_tt_pte_flags(adev
, bo
->ttm
, mem
);
246 flags
|= AMDGPU_PTE_TMZ
;
248 cpu_addr
= &job
->ibs
[0].ptr
[num_dw
];
250 if (mem
->mem_type
== TTM_PL_TT
) {
251 dma_addr_t
*dma_addr
;
253 dma_addr
= &bo
->ttm
->dma_address
[mm_cur
->start
>> PAGE_SHIFT
];
254 amdgpu_gart_map(adev
, 0, num_pages
, dma_addr
, flags
, cpu_addr
);
256 dma_addr_t dma_address
;
258 dma_address
= mm_cur
->start
;
259 dma_address
+= adev
->vm_manager
.vram_base_offset
;
261 for (i
= 0; i
< num_pages
; ++i
) {
262 amdgpu_gart_map(adev
, i
<< PAGE_SHIFT
, 1, &dma_address
,
264 dma_address
+= PAGE_SIZE
;
268 dma_fence_put(amdgpu_job_submit(job
));
273 * amdgpu_ttm_copy_mem_to_mem - Helper function for copy
274 * @adev: amdgpu device
275 * @src: buffer/address where to read from
276 * @dst: buffer/address where to write to
277 * @size: number of bytes to copy
278 * @tmz: if a secure copy should be used
279 * @resv: resv object to sync to
280 * @f: Returns the last fence if multiple jobs are submitted.
282 * The function copies @size bytes from {src->mem + src->offset} to
283 * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
284 * move and different for a BO to BO copy.
287 int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device
*adev
,
288 const struct amdgpu_copy_mem
*src
,
289 const struct amdgpu_copy_mem
*dst
,
290 uint64_t size
, bool tmz
,
291 struct dma_resv
*resv
,
292 struct dma_fence
**f
)
294 struct amdgpu_ring
*ring
= adev
->mman
.buffer_funcs_ring
;
295 struct amdgpu_res_cursor src_mm
, dst_mm
;
296 struct dma_fence
*fence
= NULL
;
299 if (!adev
->mman
.buffer_funcs_enabled
) {
300 DRM_ERROR("Trying to move memory with ring turned off.\n");
304 amdgpu_res_first(src
->mem
, src
->offset
, size
, &src_mm
);
305 amdgpu_res_first(dst
->mem
, dst
->offset
, size
, &dst_mm
);
307 mutex_lock(&adev
->mman
.gtt_window_lock
);
308 while (src_mm
.remaining
) {
309 uint64_t from
, to
, cur_size
;
310 struct dma_fence
*next
;
312 /* Never copy more than 256MiB at once to avoid a timeout */
313 cur_size
= min3(src_mm
.size
, dst_mm
.size
, 256ULL << 20);
315 /* Map src to window 0 and dst to window 1. */
316 r
= amdgpu_ttm_map_buffer(src
->bo
, src
->mem
, &src_mm
,
317 0, ring
, tmz
, &cur_size
, &from
);
321 r
= amdgpu_ttm_map_buffer(dst
->bo
, dst
->mem
, &dst_mm
,
322 1, ring
, tmz
, &cur_size
, &to
);
326 r
= amdgpu_copy_buffer(ring
, from
, to
, cur_size
,
327 resv
, &next
, false, true, tmz
);
331 dma_fence_put(fence
);
334 amdgpu_res_next(&src_mm
, cur_size
);
335 amdgpu_res_next(&dst_mm
, cur_size
);
338 mutex_unlock(&adev
->mman
.gtt_window_lock
);
340 *f
= dma_fence_get(fence
);
341 dma_fence_put(fence
);
346 * amdgpu_move_blit - Copy an entire buffer to another buffer
348 * This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to
349 * help move buffers to and from VRAM.
351 static int amdgpu_move_blit(struct ttm_buffer_object
*bo
,
353 struct ttm_resource
*new_mem
,
354 struct ttm_resource
*old_mem
)
356 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->bdev
);
357 struct amdgpu_bo
*abo
= ttm_to_amdgpu_bo(bo
);
358 struct amdgpu_copy_mem src
, dst
;
359 struct dma_fence
*fence
= NULL
;
369 r
= amdgpu_ttm_copy_mem_to_mem(adev
, &src
, &dst
,
371 amdgpu_bo_encrypted(abo
),
372 bo
->base
.resv
, &fence
);
376 /* clear the space being freed */
377 if (old_mem
->mem_type
== TTM_PL_VRAM
&&
378 (abo
->flags
& AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE
)) {
379 struct dma_fence
*wipe_fence
= NULL
;
381 r
= amdgpu_fill_buffer(abo
, AMDGPU_POISON
, NULL
, &wipe_fence
,
385 } else if (wipe_fence
) {
386 dma_fence_put(fence
);
391 /* Always block for VM page tables before committing the new location */
392 if (bo
->type
== ttm_bo_type_kernel
)
393 r
= ttm_bo_move_accel_cleanup(bo
, fence
, true, false, new_mem
);
395 r
= ttm_bo_move_accel_cleanup(bo
, fence
, evict
, true, new_mem
);
396 dma_fence_put(fence
);
401 dma_fence_wait(fence
, false);
402 dma_fence_put(fence
);
407 * amdgpu_mem_visible - Check that memory can be accessed by ttm_bo_move_memcpy
409 * Called by amdgpu_bo_move()
411 static bool amdgpu_mem_visible(struct amdgpu_device
*adev
,
412 struct ttm_resource
*mem
)
414 u64 mem_size
= (u64
)mem
->size
;
415 struct amdgpu_res_cursor cursor
;
418 if (mem
->mem_type
== TTM_PL_SYSTEM
||
419 mem
->mem_type
== TTM_PL_TT
)
421 if (mem
->mem_type
!= TTM_PL_VRAM
)
424 amdgpu_res_first(mem
, 0, mem_size
, &cursor
);
425 end
= cursor
.start
+ cursor
.size
;
426 while (cursor
.remaining
) {
427 amdgpu_res_next(&cursor
, cursor
.size
);
429 if (!cursor
.remaining
)
432 /* ttm_resource_ioremap only supports contiguous memory */
433 if (end
!= cursor
.start
)
436 end
= cursor
.start
+ cursor
.size
;
439 return end
<= adev
->gmc
.visible_vram_size
;
443 * amdgpu_bo_move - Move a buffer object to a new memory location
445 * Called by ttm_bo_handle_move_mem()
447 static int amdgpu_bo_move(struct ttm_buffer_object
*bo
, bool evict
,
448 struct ttm_operation_ctx
*ctx
,
449 struct ttm_resource
*new_mem
,
450 struct ttm_place
*hop
)
452 struct amdgpu_device
*adev
;
453 struct amdgpu_bo
*abo
;
454 struct ttm_resource
*old_mem
= bo
->resource
;
457 if (new_mem
->mem_type
== TTM_PL_TT
||
458 new_mem
->mem_type
== AMDGPU_PL_PREEMPT
) {
459 r
= amdgpu_ttm_backend_bind(bo
->bdev
, bo
->ttm
, new_mem
);
464 abo
= ttm_to_amdgpu_bo(bo
);
465 adev
= amdgpu_ttm_adev(bo
->bdev
);
467 if (!old_mem
|| (old_mem
->mem_type
== TTM_PL_SYSTEM
&&
469 ttm_bo_move_null(bo
, new_mem
);
472 if (old_mem
->mem_type
== TTM_PL_SYSTEM
&&
473 (new_mem
->mem_type
== TTM_PL_TT
||
474 new_mem
->mem_type
== AMDGPU_PL_PREEMPT
)) {
475 ttm_bo_move_null(bo
, new_mem
);
478 if ((old_mem
->mem_type
== TTM_PL_TT
||
479 old_mem
->mem_type
== AMDGPU_PL_PREEMPT
) &&
480 new_mem
->mem_type
== TTM_PL_SYSTEM
) {
481 r
= ttm_bo_wait_ctx(bo
, ctx
);
485 amdgpu_ttm_backend_unbind(bo
->bdev
, bo
->ttm
);
486 ttm_resource_free(bo
, &bo
->resource
);
487 ttm_bo_assign_mem(bo
, new_mem
);
491 if (old_mem
->mem_type
== AMDGPU_PL_GDS
||
492 old_mem
->mem_type
== AMDGPU_PL_GWS
||
493 old_mem
->mem_type
== AMDGPU_PL_OA
||
494 old_mem
->mem_type
== AMDGPU_PL_DOORBELL
||
495 new_mem
->mem_type
== AMDGPU_PL_GDS
||
496 new_mem
->mem_type
== AMDGPU_PL_GWS
||
497 new_mem
->mem_type
== AMDGPU_PL_OA
||
498 new_mem
->mem_type
== AMDGPU_PL_DOORBELL
) {
499 /* Nothing to save here */
500 ttm_bo_move_null(bo
, new_mem
);
504 if (bo
->type
== ttm_bo_type_device
&&
505 new_mem
->mem_type
== TTM_PL_VRAM
&&
506 old_mem
->mem_type
!= TTM_PL_VRAM
) {
507 /* amdgpu_bo_fault_reserve_notify will re-set this if the CPU
508 * accesses the BO after it's moved.
510 abo
->flags
&= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
;
513 if (adev
->mman
.buffer_funcs_enabled
) {
514 if (((old_mem
->mem_type
== TTM_PL_SYSTEM
&&
515 new_mem
->mem_type
== TTM_PL_VRAM
) ||
516 (old_mem
->mem_type
== TTM_PL_VRAM
&&
517 new_mem
->mem_type
== TTM_PL_SYSTEM
))) {
520 hop
->mem_type
= TTM_PL_TT
;
521 hop
->flags
= TTM_PL_FLAG_TEMPORARY
;
525 r
= amdgpu_move_blit(bo
, evict
, new_mem
, old_mem
);
531 /* Check that all memory is CPU accessible */
532 if (!amdgpu_mem_visible(adev
, old_mem
) ||
533 !amdgpu_mem_visible(adev
, new_mem
)) {
534 pr_err("Move buffer fallback to memcpy unavailable\n");
538 r
= ttm_bo_move_memcpy(bo
, ctx
, new_mem
);
543 trace_amdgpu_bo_move(abo
, new_mem
->mem_type
, old_mem
->mem_type
);
545 /* update statistics */
546 atomic64_add(bo
->base
.size
, &adev
->num_bytes_moved
);
547 amdgpu_bo_move_notify(bo
, evict
);
552 * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
554 * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault()
556 static int amdgpu_ttm_io_mem_reserve(struct ttm_device
*bdev
,
557 struct ttm_resource
*mem
)
559 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bdev
);
560 size_t bus_size
= (size_t)mem
->size
;
562 switch (mem
->mem_type
) {
567 case AMDGPU_PL_PREEMPT
:
570 mem
->bus
.offset
= mem
->start
<< PAGE_SHIFT
;
571 /* check if it's visible */
572 if ((mem
->bus
.offset
+ bus_size
) > adev
->gmc
.visible_vram_size
)
575 if (adev
->mman
.aper_base_kaddr
&&
576 mem
->placement
& TTM_PL_FLAG_CONTIGUOUS
)
577 mem
->bus
.addr
= (u8
*)adev
->mman
.aper_base_kaddr
+
580 mem
->bus
.offset
+= adev
->gmc
.aper_base
;
581 mem
->bus
.is_iomem
= true;
583 case AMDGPU_PL_DOORBELL
:
584 mem
->bus
.offset
= mem
->start
<< PAGE_SHIFT
;
585 mem
->bus
.offset
+= adev
->doorbell
.base
;
586 mem
->bus
.is_iomem
= true;
587 mem
->bus
.caching
= ttm_uncached
;
595 static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object
*bo
,
596 unsigned long page_offset
)
598 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->bdev
);
599 struct amdgpu_res_cursor cursor
;
601 amdgpu_res_first(bo
->resource
, (u64
)page_offset
<< PAGE_SHIFT
, 0,
604 if (bo
->resource
->mem_type
== AMDGPU_PL_DOORBELL
)
605 return ((uint64_t)(adev
->doorbell
.base
+ cursor
.start
)) >> PAGE_SHIFT
;
607 return (adev
->gmc
.aper_base
+ cursor
.start
) >> PAGE_SHIFT
;
611 * amdgpu_ttm_domain_start - Returns GPU start address
612 * @adev: amdgpu device object
613 * @type: type of the memory
616 * GPU start address of a memory domain
619 uint64_t amdgpu_ttm_domain_start(struct amdgpu_device
*adev
, uint32_t type
)
623 return adev
->gmc
.gart_start
;
625 return adev
->gmc
.vram_start
;
632 * TTM backend functions.
634 struct amdgpu_ttm_tt
{
636 struct drm_gem_object
*gobj
;
639 struct task_struct
*usertask
;
645 #define ttm_to_amdgpu_ttm_tt(ptr) container_of(ptr, struct amdgpu_ttm_tt, ttm)
647 #ifdef CONFIG_DRM_AMDGPU_USERPTR
649 * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
650 * memory and start HMM tracking CPU page table update
652 * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only
653 * once afterwards to stop HMM tracking
655 int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo
*bo
, struct page
**pages
,
656 struct hmm_range
**range
)
658 struct ttm_tt
*ttm
= bo
->tbo
.ttm
;
659 struct amdgpu_ttm_tt
*gtt
= ttm_to_amdgpu_ttm_tt(ttm
);
660 unsigned long start
= gtt
->userptr
;
661 struct vm_area_struct
*vma
;
662 struct mm_struct
*mm
;
666 /* Make sure get_user_pages_done() can cleanup gracefully */
669 mm
= bo
->notifier
.mm
;
671 DRM_DEBUG_DRIVER("BO is not registered?\n");
675 if (!mmget_not_zero(mm
)) /* Happens during process shutdown */
679 vma
= vma_lookup(mm
, start
);
680 if (unlikely(!vma
)) {
684 if (unlikely((gtt
->userflags
& AMDGPU_GEM_USERPTR_ANONONLY
) &&
690 readonly
= amdgpu_ttm_tt_is_readonly(ttm
);
691 r
= amdgpu_hmm_range_get_pages(&bo
->notifier
, start
, ttm
->num_pages
,
692 readonly
, NULL
, pages
, range
);
694 mmap_read_unlock(mm
);
696 pr_debug("failed %d to get user pages 0x%lx\n", r
, start
);
703 /* amdgpu_ttm_tt_discard_user_pages - Discard range and pfn array allocations
705 void amdgpu_ttm_tt_discard_user_pages(struct ttm_tt
*ttm
,
706 struct hmm_range
*range
)
708 struct amdgpu_ttm_tt
*gtt
= (void *)ttm
;
710 if (gtt
&& gtt
->userptr
&& range
)
711 amdgpu_hmm_range_get_pages_done(range
);
715 * amdgpu_ttm_tt_get_user_pages_done - stop HMM track the CPU page table change
716 * Check if the pages backing this ttm range have been invalidated
718 * Returns: true if pages are still valid
720 bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt
*ttm
,
721 struct hmm_range
*range
)
723 struct amdgpu_ttm_tt
*gtt
= ttm_to_amdgpu_ttm_tt(ttm
);
725 if (!gtt
|| !gtt
->userptr
|| !range
)
728 DRM_DEBUG_DRIVER("user_pages_done 0x%llx pages 0x%x\n",
729 gtt
->userptr
, ttm
->num_pages
);
731 WARN_ONCE(!range
->hmm_pfns
, "No user pages to check\n");
733 return !amdgpu_hmm_range_get_pages_done(range
);
738 * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
740 * Called by amdgpu_cs_list_validate(). This creates the page list
741 * that backs user memory and will ultimately be mapped into the device
744 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt
*ttm
, struct page
**pages
)
748 for (i
= 0; i
< ttm
->num_pages
; ++i
)
749 ttm
->pages
[i
] = pages
? pages
[i
] : NULL
;
753 * amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages
755 * Called by amdgpu_ttm_backend_bind()
757 static int amdgpu_ttm_tt_pin_userptr(struct ttm_device
*bdev
,
760 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bdev
);
761 struct amdgpu_ttm_tt
*gtt
= ttm_to_amdgpu_ttm_tt(ttm
);
762 int write
= !(gtt
->userflags
& AMDGPU_GEM_USERPTR_READONLY
);
763 enum dma_data_direction direction
= write
?
764 DMA_BIDIRECTIONAL
: DMA_TO_DEVICE
;
767 /* Allocate an SG array and squash pages into it */
768 r
= sg_alloc_table_from_pages(ttm
->sg
, ttm
->pages
, ttm
->num_pages
, 0,
769 (u64
)ttm
->num_pages
<< PAGE_SHIFT
,
774 /* Map SG to device */
775 r
= dma_map_sgtable(adev
->dev
, ttm
->sg
, direction
, 0);
779 /* convert SG to linear array of pages and dma addresses */
780 drm_prime_sg_to_dma_addr_array(ttm
->sg
, gtt
->ttm
.dma_address
,
792 * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
794 static void amdgpu_ttm_tt_unpin_userptr(struct ttm_device
*bdev
,
797 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bdev
);
798 struct amdgpu_ttm_tt
*gtt
= ttm_to_amdgpu_ttm_tt(ttm
);
799 int write
= !(gtt
->userflags
& AMDGPU_GEM_USERPTR_READONLY
);
800 enum dma_data_direction direction
= write
?
801 DMA_BIDIRECTIONAL
: DMA_TO_DEVICE
;
803 /* double check that we don't free the table twice */
804 if (!ttm
->sg
|| !ttm
->sg
->sgl
)
807 /* unmap the pages mapped to the device */
808 dma_unmap_sgtable(adev
->dev
, ttm
->sg
, direction
, 0);
809 sg_free_table(ttm
->sg
);
813 * total_pages is constructed as MQD0+CtrlStack0 + MQD1+CtrlStack1 + ...
814 * MQDn+CtrlStackn where n is the number of XCCs per partition.
815 * pages_per_xcc is the size of one MQD+CtrlStack. The first page is MQD
816 * and uses memory type default, UC. The rest of pages_per_xcc are
817 * Ctrl stack and modify their memory type to NC.
819 static void amdgpu_ttm_gart_bind_gfx9_mqd(struct amdgpu_device
*adev
,
820 struct ttm_tt
*ttm
, uint64_t flags
)
822 struct amdgpu_ttm_tt
*gtt
= (void *)ttm
;
823 uint64_t total_pages
= ttm
->num_pages
;
824 int num_xcc
= max(1U, adev
->gfx
.num_xcc_per_xcp
);
825 uint64_t page_idx
, pages_per_xcc
;
827 uint64_t ctrl_flags
= (flags
& ~AMDGPU_PTE_MTYPE_VG10_MASK
) |
828 AMDGPU_PTE_MTYPE_VG10(AMDGPU_MTYPE_NC
);
830 pages_per_xcc
= total_pages
;
831 do_div(pages_per_xcc
, num_xcc
);
833 for (i
= 0, page_idx
= 0; i
< num_xcc
; i
++, page_idx
+= pages_per_xcc
) {
834 /* MQD page: use default flags */
835 amdgpu_gart_bind(adev
,
836 gtt
->offset
+ (page_idx
<< PAGE_SHIFT
),
837 1, >t
->ttm
.dma_address
[page_idx
], flags
);
839 * Ctrl pages - modify the memory type to NC (ctrl_flags) from
840 * the second page of the BO onward.
842 amdgpu_gart_bind(adev
,
843 gtt
->offset
+ ((page_idx
+ 1) << PAGE_SHIFT
),
845 >t
->ttm
.dma_address
[page_idx
+ 1],
850 static void amdgpu_ttm_gart_bind(struct amdgpu_device
*adev
,
851 struct ttm_buffer_object
*tbo
,
854 struct amdgpu_bo
*abo
= ttm_to_amdgpu_bo(tbo
);
855 struct ttm_tt
*ttm
= tbo
->ttm
;
856 struct amdgpu_ttm_tt
*gtt
= ttm_to_amdgpu_ttm_tt(ttm
);
858 if (amdgpu_bo_encrypted(abo
))
859 flags
|= AMDGPU_PTE_TMZ
;
861 if (abo
->flags
& AMDGPU_GEM_CREATE_CP_MQD_GFX9
) {
862 amdgpu_ttm_gart_bind_gfx9_mqd(adev
, ttm
, flags
);
864 amdgpu_gart_bind(adev
, gtt
->offset
, ttm
->num_pages
,
865 gtt
->ttm
.dma_address
, flags
);
870 * amdgpu_ttm_backend_bind - Bind GTT memory
872 * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem().
873 * This handles binding GTT memory to the device address space.
875 static int amdgpu_ttm_backend_bind(struct ttm_device
*bdev
,
877 struct ttm_resource
*bo_mem
)
879 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bdev
);
880 struct amdgpu_ttm_tt
*gtt
= ttm_to_amdgpu_ttm_tt(ttm
);
891 r
= amdgpu_ttm_tt_pin_userptr(bdev
, ttm
);
893 DRM_ERROR("failed to pin userptr\n");
896 } else if (ttm
->page_flags
& TTM_TT_FLAG_EXTERNAL
) {
898 struct dma_buf_attachment
*attach
;
899 struct sg_table
*sgt
;
901 attach
= gtt
->gobj
->import_attach
;
902 sgt
= dma_buf_map_attachment(attach
, DMA_BIDIRECTIONAL
);
909 drm_prime_sg_to_dma_addr_array(ttm
->sg
, gtt
->ttm
.dma_address
,
913 if (!ttm
->num_pages
) {
914 WARN(1, "nothing to bind %u pages for mreg %p back %p!\n",
915 ttm
->num_pages
, bo_mem
, ttm
);
918 if (bo_mem
->mem_type
!= TTM_PL_TT
||
919 !amdgpu_gtt_mgr_has_gart_addr(bo_mem
)) {
920 gtt
->offset
= AMDGPU_BO_INVALID_OFFSET
;
924 /* compute PTE flags relevant to this BO memory */
925 flags
= amdgpu_ttm_tt_pte_flags(adev
, ttm
, bo_mem
);
927 /* bind pages into GART page tables */
928 gtt
->offset
= (u64
)bo_mem
->start
<< PAGE_SHIFT
;
929 amdgpu_gart_bind(adev
, gtt
->offset
, ttm
->num_pages
,
930 gtt
->ttm
.dma_address
, flags
);
936 * amdgpu_ttm_alloc_gart - Make sure buffer object is accessible either
937 * through AGP or GART aperture.
939 * If bo is accessible through AGP aperture, then use AGP aperture
940 * to access bo; otherwise allocate logical space in GART aperture
941 * and map bo to GART aperture.
943 int amdgpu_ttm_alloc_gart(struct ttm_buffer_object
*bo
)
945 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->bdev
);
946 struct ttm_operation_ctx ctx
= { false, false };
947 struct amdgpu_ttm_tt
*gtt
= ttm_to_amdgpu_ttm_tt(bo
->ttm
);
948 struct ttm_placement placement
;
949 struct ttm_place placements
;
950 struct ttm_resource
*tmp
;
951 uint64_t addr
, flags
;
954 if (bo
->resource
->start
!= AMDGPU_BO_INVALID_OFFSET
)
957 addr
= amdgpu_gmc_agp_addr(bo
);
958 if (addr
!= AMDGPU_BO_INVALID_OFFSET
)
961 /* allocate GART space */
962 placement
.num_placement
= 1;
963 placement
.placement
= &placements
;
965 placements
.lpfn
= adev
->gmc
.gart_size
>> PAGE_SHIFT
;
966 placements
.mem_type
= TTM_PL_TT
;
967 placements
.flags
= bo
->resource
->placement
;
969 r
= ttm_bo_mem_space(bo
, &placement
, &tmp
, &ctx
);
973 /* compute PTE flags for this buffer object */
974 flags
= amdgpu_ttm_tt_pte_flags(adev
, bo
->ttm
, tmp
);
977 gtt
->offset
= (u64
)tmp
->start
<< PAGE_SHIFT
;
978 amdgpu_ttm_gart_bind(adev
, bo
, flags
);
979 amdgpu_gart_invalidate_tlb(adev
);
980 ttm_resource_free(bo
, &bo
->resource
);
981 ttm_bo_assign_mem(bo
, tmp
);
987 * amdgpu_ttm_recover_gart - Rebind GTT pages
989 * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to
990 * rebind GTT pages during a GPU reset.
992 void amdgpu_ttm_recover_gart(struct ttm_buffer_object
*tbo
)
994 struct amdgpu_device
*adev
= amdgpu_ttm_adev(tbo
->bdev
);
1000 flags
= amdgpu_ttm_tt_pte_flags(adev
, tbo
->ttm
, tbo
->resource
);
1001 amdgpu_ttm_gart_bind(adev
, tbo
, flags
);
1005 * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
1007 * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and
1010 static void amdgpu_ttm_backend_unbind(struct ttm_device
*bdev
,
1013 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bdev
);
1014 struct amdgpu_ttm_tt
*gtt
= ttm_to_amdgpu_ttm_tt(ttm
);
1016 /* if the pages have userptr pinning then clear that first */
1018 amdgpu_ttm_tt_unpin_userptr(bdev
, ttm
);
1019 } else if (ttm
->sg
&& gtt
->gobj
->import_attach
) {
1020 struct dma_buf_attachment
*attach
;
1022 attach
= gtt
->gobj
->import_attach
;
1023 dma_buf_unmap_attachment(attach
, ttm
->sg
, DMA_BIDIRECTIONAL
);
1030 if (gtt
->offset
== AMDGPU_BO_INVALID_OFFSET
)
1033 /* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
1034 amdgpu_gart_unbind(adev
, gtt
->offset
, ttm
->num_pages
);
1038 static void amdgpu_ttm_backend_destroy(struct ttm_device
*bdev
,
1041 struct amdgpu_ttm_tt
*gtt
= ttm_to_amdgpu_ttm_tt(ttm
);
1044 put_task_struct(gtt
->usertask
);
1046 ttm_tt_fini(>t
->ttm
);
1051 * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
1053 * @bo: The buffer object to create a GTT ttm_tt object around
1054 * @page_flags: Page flags to be added to the ttm_tt object
1056 * Called by ttm_tt_create().
1058 static struct ttm_tt
*amdgpu_ttm_tt_create(struct ttm_buffer_object
*bo
,
1059 uint32_t page_flags
)
1061 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->bdev
);
1062 struct amdgpu_bo
*abo
= ttm_to_amdgpu_bo(bo
);
1063 struct amdgpu_ttm_tt
*gtt
;
1064 enum ttm_caching caching
;
1066 gtt
= kzalloc(sizeof(struct amdgpu_ttm_tt
), GFP_KERNEL
);
1070 gtt
->gobj
= &bo
->base
;
1071 if (adev
->gmc
.mem_partitions
&& abo
->xcp_id
>= 0)
1072 gtt
->pool_id
= KFD_XCP_MEM_ID(adev
, abo
->xcp_id
);
1074 gtt
->pool_id
= abo
->xcp_id
;
1076 if (abo
->flags
& AMDGPU_GEM_CREATE_CPU_GTT_USWC
)
1077 caching
= ttm_write_combined
;
1079 caching
= ttm_cached
;
1081 /* allocate space for the uninitialized page entries */
1082 if (ttm_sg_tt_init(>t
->ttm
, bo
, page_flags
, caching
)) {
1090 * amdgpu_ttm_tt_populate - Map GTT pages visible to the device
1092 * Map the pages of a ttm_tt object to an address space visible
1093 * to the underlying device.
1095 static int amdgpu_ttm_tt_populate(struct ttm_device
*bdev
,
1097 struct ttm_operation_ctx
*ctx
)
1099 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bdev
);
1100 struct amdgpu_ttm_tt
*gtt
= ttm_to_amdgpu_ttm_tt(ttm
);
1101 struct ttm_pool
*pool
;
1105 /* user pages are bound by amdgpu_ttm_tt_pin_userptr() */
1107 ttm
->sg
= kzalloc(sizeof(struct sg_table
), GFP_KERNEL
);
1113 if (ttm
->page_flags
& TTM_TT_FLAG_EXTERNAL
)
1116 if (adev
->mman
.ttm_pools
&& gtt
->pool_id
>= 0)
1117 pool
= &adev
->mman
.ttm_pools
[gtt
->pool_id
];
1119 pool
= &adev
->mman
.bdev
.pool
;
1120 ret
= ttm_pool_alloc(pool
, ttm
, ctx
);
1124 for (i
= 0; i
< ttm
->num_pages
; ++i
)
1125 ttm
->pages
[i
]->mapping
= bdev
->dev_mapping
;
1131 * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
1133 * Unmaps pages of a ttm_tt object from the device address space and
1134 * unpopulates the page array backing it.
1136 static void amdgpu_ttm_tt_unpopulate(struct ttm_device
*bdev
,
1139 struct amdgpu_ttm_tt
*gtt
= ttm_to_amdgpu_ttm_tt(ttm
);
1140 struct amdgpu_device
*adev
;
1141 struct ttm_pool
*pool
;
1144 amdgpu_ttm_backend_unbind(bdev
, ttm
);
1147 amdgpu_ttm_tt_set_user_pages(ttm
, NULL
);
1153 if (ttm
->page_flags
& TTM_TT_FLAG_EXTERNAL
)
1156 for (i
= 0; i
< ttm
->num_pages
; ++i
)
1157 ttm
->pages
[i
]->mapping
= NULL
;
1159 adev
= amdgpu_ttm_adev(bdev
);
1161 if (adev
->mman
.ttm_pools
&& gtt
->pool_id
>= 0)
1162 pool
= &adev
->mman
.ttm_pools
[gtt
->pool_id
];
1164 pool
= &adev
->mman
.bdev
.pool
;
1166 return ttm_pool_free(pool
, ttm
);
1170 * amdgpu_ttm_tt_get_userptr - Return the userptr GTT ttm_tt for the current
1173 * @tbo: The ttm_buffer_object that contains the userptr
1174 * @user_addr: The returned value
1176 int amdgpu_ttm_tt_get_userptr(const struct ttm_buffer_object
*tbo
,
1177 uint64_t *user_addr
)
1179 struct amdgpu_ttm_tt
*gtt
;
1184 gtt
= (void *)tbo
->ttm
;
1185 *user_addr
= gtt
->userptr
;
1190 * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
1193 * @bo: The ttm_buffer_object to bind this userptr to
1194 * @addr: The address in the current tasks VM space to use
1195 * @flags: Requirements of userptr object.
1197 * Called by amdgpu_gem_userptr_ioctl() and kfd_ioctl_alloc_memory_of_gpu() to
1198 * bind userptr pages to current task and by kfd_ioctl_acquire_vm() to
1199 * initialize GPU VM for a KFD process.
1201 int amdgpu_ttm_tt_set_userptr(struct ttm_buffer_object
*bo
,
1202 uint64_t addr
, uint32_t flags
)
1204 struct amdgpu_ttm_tt
*gtt
;
1207 /* TODO: We want a separate TTM object type for userptrs */
1208 bo
->ttm
= amdgpu_ttm_tt_create(bo
, 0);
1209 if (bo
->ttm
== NULL
)
1213 /* Set TTM_TT_FLAG_EXTERNAL before populate but after create. */
1214 bo
->ttm
->page_flags
|= TTM_TT_FLAG_EXTERNAL
;
1216 gtt
= ttm_to_amdgpu_ttm_tt(bo
->ttm
);
1217 gtt
->userptr
= addr
;
1218 gtt
->userflags
= flags
;
1221 put_task_struct(gtt
->usertask
);
1222 gtt
->usertask
= current
->group_leader
;
1223 get_task_struct(gtt
->usertask
);
1229 * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
1231 struct mm_struct
*amdgpu_ttm_tt_get_usermm(struct ttm_tt
*ttm
)
1233 struct amdgpu_ttm_tt
*gtt
= ttm_to_amdgpu_ttm_tt(ttm
);
1238 if (gtt
->usertask
== NULL
)
1241 return gtt
->usertask
->mm
;
1245 * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an
1246 * address range for the current task.
1249 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt
*ttm
, unsigned long start
,
1250 unsigned long end
, unsigned long *userptr
)
1252 struct amdgpu_ttm_tt
*gtt
= ttm_to_amdgpu_ttm_tt(ttm
);
1255 if (gtt
== NULL
|| !gtt
->userptr
)
1258 /* Return false if no part of the ttm_tt object lies within
1261 size
= (unsigned long)gtt
->ttm
.num_pages
* PAGE_SIZE
;
1262 if (gtt
->userptr
> end
|| gtt
->userptr
+ size
<= start
)
1266 *userptr
= gtt
->userptr
;
1271 * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
1273 bool amdgpu_ttm_tt_is_userptr(struct ttm_tt
*ttm
)
1275 struct amdgpu_ttm_tt
*gtt
= ttm_to_amdgpu_ttm_tt(ttm
);
1277 if (gtt
== NULL
|| !gtt
->userptr
)
1284 * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
1286 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt
*ttm
)
1288 struct amdgpu_ttm_tt
*gtt
= ttm_to_amdgpu_ttm_tt(ttm
);
1293 return !!(gtt
->userflags
& AMDGPU_GEM_USERPTR_READONLY
);
1297 * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object
1299 * @ttm: The ttm_tt object to compute the flags for
1300 * @mem: The memory registry backing this ttm_tt object
1302 * Figure out the flags to use for a VM PDE (Page Directory Entry).
1304 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt
*ttm
, struct ttm_resource
*mem
)
1308 if (mem
&& mem
->mem_type
!= TTM_PL_SYSTEM
)
1309 flags
|= AMDGPU_PTE_VALID
;
1311 if (mem
&& (mem
->mem_type
== TTM_PL_TT
||
1312 mem
->mem_type
== AMDGPU_PL_DOORBELL
||
1313 mem
->mem_type
== AMDGPU_PL_PREEMPT
)) {
1314 flags
|= AMDGPU_PTE_SYSTEM
;
1316 if (ttm
->caching
== ttm_cached
)
1317 flags
|= AMDGPU_PTE_SNOOPED
;
1320 if (mem
&& mem
->mem_type
== TTM_PL_VRAM
&&
1321 mem
->bus
.caching
== ttm_cached
)
1322 flags
|= AMDGPU_PTE_SNOOPED
;
1328 * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
1330 * @adev: amdgpu_device pointer
1331 * @ttm: The ttm_tt object to compute the flags for
1332 * @mem: The memory registry backing this ttm_tt object
1334 * Figure out the flags to use for a VM PTE (Page Table Entry).
1336 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device
*adev
, struct ttm_tt
*ttm
,
1337 struct ttm_resource
*mem
)
1339 uint64_t flags
= amdgpu_ttm_tt_pde_flags(ttm
, mem
);
1341 flags
|= adev
->gart
.gart_pte_flags
;
1342 flags
|= AMDGPU_PTE_READABLE
;
1344 if (!amdgpu_ttm_tt_is_readonly(ttm
))
1345 flags
|= AMDGPU_PTE_WRITEABLE
;
1351 * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer
1354 * Return true if eviction is sensible. Called by ttm_mem_evict_first() on
1355 * behalf of ttm_bo_mem_force_space() which tries to evict buffer objects until
1356 * it can find space for a new object and by ttm_bo_force_list_clean() which is
1357 * used to clean out a memory space.
1359 static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object
*bo
,
1360 const struct ttm_place
*place
)
1362 struct dma_resv_iter resv_cursor
;
1363 struct dma_fence
*f
;
1365 if (!amdgpu_bo_is_amdgpu_bo(bo
))
1366 return ttm_bo_eviction_valuable(bo
, place
);
1369 if (bo
->resource
->mem_type
== TTM_PL_SYSTEM
)
1372 if (bo
->type
== ttm_bo_type_kernel
&&
1373 !amdgpu_vm_evictable(ttm_to_amdgpu_bo(bo
)))
1376 /* If bo is a KFD BO, check if the bo belongs to the current process.
1377 * If true, then return false as any KFD process needs all its BOs to
1378 * be resident to run successfully
1380 dma_resv_for_each_fence(&resv_cursor
, bo
->base
.resv
,
1381 DMA_RESV_USAGE_BOOKKEEP
, f
) {
1382 if (amdkfd_fence_check_mm(f
, current
->mm
))
1386 /* Preemptible BOs don't own system resources managed by the
1387 * driver (pages, VRAM, GART space). They point to resources
1388 * owned by someone else (e.g. pageable memory in user mode
1389 * or a DMABuf). They are used in a preemptible context so we
1390 * can guarantee no deadlocks and good QoS in case of MMU
1391 * notifiers or DMABuf move notifiers from the resource owner.
1393 if (bo
->resource
->mem_type
== AMDGPU_PL_PREEMPT
)
1396 if (bo
->resource
->mem_type
== TTM_PL_TT
&&
1397 amdgpu_bo_encrypted(ttm_to_amdgpu_bo(bo
)))
1400 return ttm_bo_eviction_valuable(bo
, place
);
1403 static void amdgpu_ttm_vram_mm_access(struct amdgpu_device
*adev
, loff_t pos
,
1404 void *buf
, size_t size
, bool write
)
1407 uint64_t aligned_pos
= ALIGN_DOWN(pos
, 4);
1408 uint64_t bytes
= 4 - (pos
& 0x3);
1409 uint32_t shift
= (pos
& 0x3) * 8;
1410 uint32_t mask
= 0xffffffff << shift
;
1414 mask
&= 0xffffffff >> (bytes
- size
) * 8;
1418 if (mask
!= 0xffffffff) {
1419 amdgpu_device_mm_access(adev
, aligned_pos
, &value
, 4, false);
1422 value
|= (*(uint32_t *)buf
<< shift
) & mask
;
1423 amdgpu_device_mm_access(adev
, aligned_pos
, &value
, 4, true);
1425 value
= (value
& mask
) >> shift
;
1426 memcpy(buf
, &value
, bytes
);
1429 amdgpu_device_mm_access(adev
, aligned_pos
, buf
, 4, write
);
1438 static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object
*bo
,
1439 unsigned long offset
, void *buf
,
1442 struct amdgpu_bo
*abo
= ttm_to_amdgpu_bo(bo
);
1443 struct amdgpu_device
*adev
= amdgpu_ttm_adev(abo
->tbo
.bdev
);
1444 struct amdgpu_res_cursor src_mm
;
1445 struct amdgpu_job
*job
;
1446 struct dma_fence
*fence
;
1447 uint64_t src_addr
, dst_addr
;
1448 unsigned int num_dw
;
1451 if (len
!= PAGE_SIZE
)
1454 if (!adev
->mman
.sdma_access_ptr
)
1457 if (!drm_dev_enter(adev_to_drm(adev
), &idx
))
1461 memcpy(adev
->mman
.sdma_access_ptr
, buf
, len
);
1463 num_dw
= ALIGN(adev
->mman
.buffer_funcs
->copy_num_dw
, 8);
1464 r
= amdgpu_job_alloc_with_ib(adev
, &adev
->mman
.high_pr
,
1465 AMDGPU_FENCE_OWNER_UNDEFINED
,
1466 num_dw
* 4, AMDGPU_IB_POOL_DELAYED
,
1471 amdgpu_res_first(abo
->tbo
.resource
, offset
, len
, &src_mm
);
1472 src_addr
= amdgpu_ttm_domain_start(adev
, bo
->resource
->mem_type
) +
1474 dst_addr
= amdgpu_bo_gpu_offset(adev
->mman
.sdma_access_bo
);
1476 swap(src_addr
, dst_addr
);
1478 amdgpu_emit_copy_buffer(adev
, &job
->ibs
[0], src_addr
, dst_addr
,
1481 amdgpu_ring_pad_ib(adev
->mman
.buffer_funcs_ring
, &job
->ibs
[0]);
1482 WARN_ON(job
->ibs
[0].length_dw
> num_dw
);
1484 fence
= amdgpu_job_submit(job
);
1486 if (!dma_fence_wait_timeout(fence
, false, adev
->sdma_timeout
))
1488 dma_fence_put(fence
);
1491 memcpy(buf
, adev
->mman
.sdma_access_ptr
, len
);
1498 * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object.
1500 * @bo: The buffer object to read/write
1501 * @offset: Offset into buffer object
1502 * @buf: Secondary buffer to write/read from
1503 * @len: Length in bytes of access
1504 * @write: true if writing
1506 * This is used to access VRAM that backs a buffer object via MMIO
1507 * access for debugging purposes.
1509 static int amdgpu_ttm_access_memory(struct ttm_buffer_object
*bo
,
1510 unsigned long offset
, void *buf
, int len
,
1513 struct amdgpu_bo
*abo
= ttm_to_amdgpu_bo(bo
);
1514 struct amdgpu_device
*adev
= amdgpu_ttm_adev(abo
->tbo
.bdev
);
1515 struct amdgpu_res_cursor cursor
;
1518 if (bo
->resource
->mem_type
!= TTM_PL_VRAM
)
1521 if (amdgpu_device_has_timeouts_enabled(adev
) &&
1522 !amdgpu_ttm_access_memory_sdma(bo
, offset
, buf
, len
, write
))
1525 amdgpu_res_first(bo
->resource
, offset
, len
, &cursor
);
1526 while (cursor
.remaining
) {
1527 size_t count
, size
= cursor
.size
;
1528 loff_t pos
= cursor
.start
;
1530 count
= amdgpu_device_aper_access(adev
, pos
, buf
, size
, write
);
1533 /* using MM to access rest vram and handle un-aligned address */
1536 amdgpu_ttm_vram_mm_access(adev
, pos
, buf
, size
, write
);
1541 amdgpu_res_next(&cursor
, cursor
.size
);
1548 amdgpu_bo_delete_mem_notify(struct ttm_buffer_object
*bo
)
1550 amdgpu_bo_move_notify(bo
, false);
1553 static struct ttm_device_funcs amdgpu_bo_driver
= {
1554 .ttm_tt_create
= &amdgpu_ttm_tt_create
,
1555 .ttm_tt_populate
= &amdgpu_ttm_tt_populate
,
1556 .ttm_tt_unpopulate
= &amdgpu_ttm_tt_unpopulate
,
1557 .ttm_tt_destroy
= &amdgpu_ttm_backend_destroy
,
1558 .eviction_valuable
= amdgpu_ttm_bo_eviction_valuable
,
1559 .evict_flags
= &amdgpu_evict_flags
,
1560 .move
= &amdgpu_bo_move
,
1561 .delete_mem_notify
= &amdgpu_bo_delete_mem_notify
,
1562 .release_notify
= &amdgpu_bo_release_notify
,
1563 .io_mem_reserve
= &amdgpu_ttm_io_mem_reserve
,
1564 .io_mem_pfn
= amdgpu_ttm_io_mem_pfn
,
1565 .access_memory
= &amdgpu_ttm_access_memory
,
1569 * Firmware Reservation functions
1572 * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram
1574 * @adev: amdgpu_device pointer
1576 * free fw reserved vram if it has been reserved.
1578 static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device
*adev
)
1580 amdgpu_bo_free_kernel(&adev
->mman
.fw_vram_usage_reserved_bo
,
1581 NULL
, &adev
->mman
.fw_vram_usage_va
);
1585 * Driver Reservation functions
1588 * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
1590 * @adev: amdgpu_device pointer
1592 * free drv reserved vram if it has been reserved.
1594 static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device
*adev
)
1596 amdgpu_bo_free_kernel(&adev
->mman
.drv_vram_usage_reserved_bo
,
1598 &adev
->mman
.drv_vram_usage_va
);
1602 * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
1604 * @adev: amdgpu_device pointer
1606 * create bo vram reservation from fw.
1608 static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device
*adev
)
1610 uint64_t vram_size
= adev
->gmc
.visible_vram_size
;
1612 adev
->mman
.fw_vram_usage_va
= NULL
;
1613 adev
->mman
.fw_vram_usage_reserved_bo
= NULL
;
1615 if (adev
->mman
.fw_vram_usage_size
== 0 ||
1616 adev
->mman
.fw_vram_usage_size
> vram_size
)
1619 return amdgpu_bo_create_kernel_at(adev
,
1620 adev
->mman
.fw_vram_usage_start_offset
,
1621 adev
->mman
.fw_vram_usage_size
,
1622 &adev
->mman
.fw_vram_usage_reserved_bo
,
1623 &adev
->mman
.fw_vram_usage_va
);
1627 * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
1629 * @adev: amdgpu_device pointer
1631 * create bo vram reservation from drv.
1633 static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device
*adev
)
1635 u64 vram_size
= adev
->gmc
.visible_vram_size
;
1637 adev
->mman
.drv_vram_usage_va
= NULL
;
1638 adev
->mman
.drv_vram_usage_reserved_bo
= NULL
;
1640 if (adev
->mman
.drv_vram_usage_size
== 0 ||
1641 adev
->mman
.drv_vram_usage_size
> vram_size
)
1644 return amdgpu_bo_create_kernel_at(adev
,
1645 adev
->mman
.drv_vram_usage_start_offset
,
1646 adev
->mman
.drv_vram_usage_size
,
1647 &adev
->mman
.drv_vram_usage_reserved_bo
,
1648 &adev
->mman
.drv_vram_usage_va
);
1652 * Memoy training reservation functions
1656 * amdgpu_ttm_training_reserve_vram_fini - free memory training reserved vram
1658 * @adev: amdgpu_device pointer
1660 * free memory training reserved vram if it has been reserved.
1662 static int amdgpu_ttm_training_reserve_vram_fini(struct amdgpu_device
*adev
)
1664 struct psp_memory_training_context
*ctx
= &adev
->psp
.mem_train_ctx
;
1666 ctx
->init
= PSP_MEM_TRAIN_NOT_SUPPORT
;
1667 amdgpu_bo_free_kernel(&ctx
->c2p_bo
, NULL
, NULL
);
1673 static void amdgpu_ttm_training_data_block_init(struct amdgpu_device
*adev
,
1674 uint32_t reserve_size
)
1676 struct psp_memory_training_context
*ctx
= &adev
->psp
.mem_train_ctx
;
1678 memset(ctx
, 0, sizeof(*ctx
));
1680 ctx
->c2p_train_data_offset
=
1681 ALIGN((adev
->gmc
.mc_vram_size
- reserve_size
- SZ_1M
), SZ_1M
);
1682 ctx
->p2c_train_data_offset
=
1683 (adev
->gmc
.mc_vram_size
- GDDR6_MEM_TRAINING_OFFSET
);
1684 ctx
->train_data_size
=
1685 GDDR6_MEM_TRAINING_DATA_SIZE_IN_BYTES
;
1687 DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
1688 ctx
->train_data_size
,
1689 ctx
->p2c_train_data_offset
,
1690 ctx
->c2p_train_data_offset
);
1694 * reserve TMR memory at the top of VRAM which holds
1695 * IP Discovery data and is protected by PSP.
1697 static int amdgpu_ttm_reserve_tmr(struct amdgpu_device
*adev
)
1699 struct psp_memory_training_context
*ctx
= &adev
->psp
.mem_train_ctx
;
1700 bool mem_train_support
= false;
1701 uint32_t reserve_size
= 0;
1704 if (adev
->bios
&& !amdgpu_sriov_vf(adev
)) {
1705 if (amdgpu_atomfirmware_mem_training_supported(adev
))
1706 mem_train_support
= true;
1708 DRM_DEBUG("memory training does not support!\n");
1712 * Query reserved tmr size through atom firmwareinfo for Sienna_Cichlid and onwards for all
1713 * the use cases (IP discovery/G6 memory training/profiling/diagnostic data.etc)
1715 * Otherwise, fallback to legacy approach to check and reserve tmr block for ip
1716 * discovery data and G6 memory training data respectively
1720 amdgpu_atomfirmware_get_fw_reserved_fb_size(adev
);
1723 amdgpu_ip_version(adev
, GC_HWIP
, 0) == IP_VERSION(9, 4, 3))
1724 reserve_size
= max(reserve_size
, (uint32_t)280 << 20);
1725 else if (!reserve_size
)
1726 reserve_size
= DISCOVERY_TMR_OFFSET
;
1728 if (mem_train_support
) {
1729 /* reserve vram for mem train according to TMR location */
1730 amdgpu_ttm_training_data_block_init(adev
, reserve_size
);
1731 ret
= amdgpu_bo_create_kernel_at(adev
,
1732 ctx
->c2p_train_data_offset
,
1733 ctx
->train_data_size
,
1737 DRM_ERROR("alloc c2p_bo failed(%d)!\n", ret
);
1738 amdgpu_ttm_training_reserve_vram_fini(adev
);
1741 ctx
->init
= PSP_MEM_TRAIN_RESERVE_SUCCESS
;
1744 if (!adev
->gmc
.is_app_apu
) {
1745 ret
= amdgpu_bo_create_kernel_at(
1746 adev
, adev
->gmc
.real_vram_size
- reserve_size
,
1747 reserve_size
, &adev
->mman
.fw_reserved_memory
, NULL
);
1749 DRM_ERROR("alloc tmr failed(%d)!\n", ret
);
1750 amdgpu_bo_free_kernel(&adev
->mman
.fw_reserved_memory
,
1755 DRM_DEBUG_DRIVER("backdoor fw loading path for PSP TMR, no reservation needed\n");
1761 static int amdgpu_ttm_pools_init(struct amdgpu_device
*adev
)
1765 if (!adev
->gmc
.is_app_apu
|| !adev
->gmc
.num_mem_partitions
)
1768 adev
->mman
.ttm_pools
= kcalloc(adev
->gmc
.num_mem_partitions
,
1769 sizeof(*adev
->mman
.ttm_pools
),
1771 if (!adev
->mman
.ttm_pools
)
1774 for (i
= 0; i
< adev
->gmc
.num_mem_partitions
; i
++) {
1775 ttm_pool_init(&adev
->mman
.ttm_pools
[i
], adev
->dev
,
1776 adev
->gmc
.mem_partitions
[i
].numa
.node
,
1782 static void amdgpu_ttm_pools_fini(struct amdgpu_device
*adev
)
1786 if (!adev
->gmc
.is_app_apu
|| !adev
->mman
.ttm_pools
)
1789 for (i
= 0; i
< adev
->gmc
.num_mem_partitions
; i
++)
1790 ttm_pool_fini(&adev
->mman
.ttm_pools
[i
]);
1792 kfree(adev
->mman
.ttm_pools
);
1793 adev
->mman
.ttm_pools
= NULL
;
1797 * amdgpu_ttm_init - Init the memory management (ttm) as well as various
1798 * gtt/vram related fields.
1800 * This initializes all of the memory space pools that the TTM layer
1801 * will need such as the GTT space (system memory mapped to the device),
1802 * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
1803 * can be mapped per VMID.
1805 int amdgpu_ttm_init(struct amdgpu_device
*adev
)
1810 mutex_init(&adev
->mman
.gtt_window_lock
);
1812 /* No others user of address space so set it to 0 */
1813 r
= ttm_device_init(&adev
->mman
.bdev
, &amdgpu_bo_driver
, adev
->dev
,
1814 adev_to_drm(adev
)->anon_inode
->i_mapping
,
1815 adev_to_drm(adev
)->vma_offset_manager
,
1817 dma_addressing_limited(adev
->dev
));
1819 DRM_ERROR("failed initializing buffer object driver(%d).\n", r
);
1823 r
= amdgpu_ttm_pools_init(adev
);
1825 DRM_ERROR("failed to init ttm pools(%d).\n", r
);
1828 adev
->mman
.initialized
= true;
1830 /* Initialize VRAM pool with all of VRAM divided into pages */
1831 r
= amdgpu_vram_mgr_init(adev
);
1833 DRM_ERROR("Failed initializing VRAM heap.\n");
1837 /* Change the size here instead of the init above so only lpfn is affected */
1838 amdgpu_ttm_set_buffer_funcs_status(adev
, false);
1841 if (adev
->gmc
.xgmi
.connected_to_cpu
)
1842 adev
->mman
.aper_base_kaddr
= ioremap_cache(adev
->gmc
.aper_base
,
1843 adev
->gmc
.visible_vram_size
);
1845 else if (adev
->gmc
.is_app_apu
)
1847 "No need to ioremap when real vram size is 0\n");
1850 adev
->mman
.aper_base_kaddr
= ioremap_wc(adev
->gmc
.aper_base
,
1851 adev
->gmc
.visible_vram_size
);
1855 *The reserved vram for firmware must be pinned to the specified
1856 *place on the VRAM, so reserve it early.
1858 r
= amdgpu_ttm_fw_reserve_vram_init(adev
);
1863 *The reserved vram for driver must be pinned to the specified
1864 *place on the VRAM, so reserve it early.
1866 r
= amdgpu_ttm_drv_reserve_vram_init(adev
);
1871 * only NAVI10 and onwards ASIC support for IP discovery.
1872 * If IP discovery enabled, a block of memory should be
1873 * reserved for IP discovey.
1875 if (adev
->mman
.discovery_bin
) {
1876 r
= amdgpu_ttm_reserve_tmr(adev
);
1881 /* allocate memory as required for VGA
1882 * This is used for VGA emulation and pre-OS scanout buffers to
1883 * avoid display artifacts while transitioning between pre-OS
1886 if (!adev
->gmc
.is_app_apu
) {
1887 r
= amdgpu_bo_create_kernel_at(adev
, 0,
1888 adev
->mman
.stolen_vga_size
,
1889 &adev
->mman
.stolen_vga_memory
,
1894 r
= amdgpu_bo_create_kernel_at(adev
, adev
->mman
.stolen_vga_size
,
1895 adev
->mman
.stolen_extended_size
,
1896 &adev
->mman
.stolen_extended_memory
,
1902 r
= amdgpu_bo_create_kernel_at(adev
,
1903 adev
->mman
.stolen_reserved_offset
,
1904 adev
->mman
.stolen_reserved_size
,
1905 &adev
->mman
.stolen_reserved_memory
,
1910 DRM_DEBUG_DRIVER("Skipped stolen memory reservation\n");
1913 DRM_INFO("amdgpu: %uM of VRAM memory ready\n",
1914 (unsigned int)(adev
->gmc
.real_vram_size
/ (1024 * 1024)));
1916 /* Compute GTT size, either based on TTM limit
1917 * or whatever the user passed on module init.
1919 if (amdgpu_gtt_size
== -1)
1920 gtt_size
= ttm_tt_pages_limit() << PAGE_SHIFT
;
1922 gtt_size
= (uint64_t)amdgpu_gtt_size
<< 20;
1924 /* Initialize GTT memory pool */
1925 r
= amdgpu_gtt_mgr_init(adev
, gtt_size
);
1927 DRM_ERROR("Failed initializing GTT heap.\n");
1930 DRM_INFO("amdgpu: %uM of GTT memory ready.\n",
1931 (unsigned int)(gtt_size
/ (1024 * 1024)));
1933 /* Initiailize doorbell pool on PCI BAR */
1934 r
= amdgpu_ttm_init_on_chip(adev
, AMDGPU_PL_DOORBELL
, adev
->doorbell
.size
/ PAGE_SIZE
);
1936 DRM_ERROR("Failed initializing doorbell heap.\n");
1940 /* Create a boorbell page for kernel usages */
1941 r
= amdgpu_doorbell_create_kernel_doorbells(adev
);
1943 DRM_ERROR("Failed to initialize kernel doorbells.\n");
1947 /* Initialize preemptible memory pool */
1948 r
= amdgpu_preempt_mgr_init(adev
);
1950 DRM_ERROR("Failed initializing PREEMPT heap.\n");
1954 /* Initialize various on-chip memory pools */
1955 r
= amdgpu_ttm_init_on_chip(adev
, AMDGPU_PL_GDS
, adev
->gds
.gds_size
);
1957 DRM_ERROR("Failed initializing GDS heap.\n");
1961 r
= amdgpu_ttm_init_on_chip(adev
, AMDGPU_PL_GWS
, adev
->gds
.gws_size
);
1963 DRM_ERROR("Failed initializing gws heap.\n");
1967 r
= amdgpu_ttm_init_on_chip(adev
, AMDGPU_PL_OA
, adev
->gds
.oa_size
);
1969 DRM_ERROR("Failed initializing oa heap.\n");
1972 if (amdgpu_bo_create_kernel(adev
, PAGE_SIZE
, PAGE_SIZE
,
1973 AMDGPU_GEM_DOMAIN_GTT
,
1974 &adev
->mman
.sdma_access_bo
, NULL
,
1975 &adev
->mman
.sdma_access_ptr
))
1976 DRM_WARN("Debug VRAM access will use slowpath MM access\n");
1982 * amdgpu_ttm_fini - De-initialize the TTM memory pools
1984 void amdgpu_ttm_fini(struct amdgpu_device
*adev
)
1988 if (!adev
->mman
.initialized
)
1991 amdgpu_ttm_pools_fini(adev
);
1993 amdgpu_ttm_training_reserve_vram_fini(adev
);
1994 /* return the stolen vga memory back to VRAM */
1995 if (!adev
->gmc
.is_app_apu
) {
1996 amdgpu_bo_free_kernel(&adev
->mman
.stolen_vga_memory
, NULL
, NULL
);
1997 amdgpu_bo_free_kernel(&adev
->mman
.stolen_extended_memory
, NULL
, NULL
);
1998 /* return the FW reserved memory back to VRAM */
1999 amdgpu_bo_free_kernel(&adev
->mman
.fw_reserved_memory
, NULL
,
2001 if (adev
->mman
.stolen_reserved_size
)
2002 amdgpu_bo_free_kernel(&adev
->mman
.stolen_reserved_memory
,
2005 amdgpu_bo_free_kernel(&adev
->mman
.sdma_access_bo
, NULL
,
2006 &adev
->mman
.sdma_access_ptr
);
2007 amdgpu_ttm_fw_reserve_vram_fini(adev
);
2008 amdgpu_ttm_drv_reserve_vram_fini(adev
);
2010 if (drm_dev_enter(adev_to_drm(adev
), &idx
)) {
2012 if (adev
->mman
.aper_base_kaddr
)
2013 iounmap(adev
->mman
.aper_base_kaddr
);
2014 adev
->mman
.aper_base_kaddr
= NULL
;
2019 amdgpu_vram_mgr_fini(adev
);
2020 amdgpu_gtt_mgr_fini(adev
);
2021 amdgpu_preempt_mgr_fini(adev
);
2022 ttm_range_man_fini(&adev
->mman
.bdev
, AMDGPU_PL_GDS
);
2023 ttm_range_man_fini(&adev
->mman
.bdev
, AMDGPU_PL_GWS
);
2024 ttm_range_man_fini(&adev
->mman
.bdev
, AMDGPU_PL_OA
);
2025 ttm_device_fini(&adev
->mman
.bdev
);
2026 adev
->mman
.initialized
= false;
2027 DRM_INFO("amdgpu: ttm finalized\n");
2031 * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
2033 * @adev: amdgpu_device pointer
2034 * @enable: true when we can use buffer functions.
2036 * Enable/disable use of buffer functions during suspend/resume. This should
2037 * only be called at bootup or when userspace isn't running.
2039 void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device
*adev
, bool enable
)
2041 struct ttm_resource_manager
*man
= ttm_manager_type(&adev
->mman
.bdev
, TTM_PL_VRAM
);
2045 if (!adev
->mman
.initialized
|| amdgpu_in_reset(adev
) ||
2046 adev
->mman
.buffer_funcs_enabled
== enable
|| adev
->gmc
.is_app_apu
)
2050 struct amdgpu_ring
*ring
;
2051 struct drm_gpu_scheduler
*sched
;
2053 ring
= adev
->mman
.buffer_funcs_ring
;
2054 sched
= &ring
->sched
;
2055 r
= drm_sched_entity_init(&adev
->mman
.high_pr
,
2056 DRM_SCHED_PRIORITY_KERNEL
, &sched
,
2059 DRM_ERROR("Failed setting up TTM BO move entity (%d)\n",
2064 r
= drm_sched_entity_init(&adev
->mman
.low_pr
,
2065 DRM_SCHED_PRIORITY_NORMAL
, &sched
,
2068 DRM_ERROR("Failed setting up TTM BO move entity (%d)\n",
2070 goto error_free_entity
;
2073 drm_sched_entity_destroy(&adev
->mman
.high_pr
);
2074 drm_sched_entity_destroy(&adev
->mman
.low_pr
);
2075 dma_fence_put(man
->move
);
2079 /* this just adjusts TTM size idea, which sets lpfn to the correct value */
2081 size
= adev
->gmc
.real_vram_size
;
2083 size
= adev
->gmc
.visible_vram_size
;
2085 adev
->mman
.buffer_funcs_enabled
= enable
;
2090 drm_sched_entity_destroy(&adev
->mman
.high_pr
);
2093 static int amdgpu_ttm_prepare_job(struct amdgpu_device
*adev
,
2095 unsigned int num_dw
,
2096 struct dma_resv
*resv
,
2097 bool vm_needs_flush
,
2098 struct amdgpu_job
**job
,
2101 enum amdgpu_ib_pool_type pool
= direct_submit
?
2102 AMDGPU_IB_POOL_DIRECT
:
2103 AMDGPU_IB_POOL_DELAYED
;
2105 struct drm_sched_entity
*entity
= delayed
? &adev
->mman
.low_pr
:
2106 &adev
->mman
.high_pr
;
2107 r
= amdgpu_job_alloc_with_ib(adev
, entity
,
2108 AMDGPU_FENCE_OWNER_UNDEFINED
,
2109 num_dw
* 4, pool
, job
);
2113 if (vm_needs_flush
) {
2114 (*job
)->vm_pd_addr
= amdgpu_gmc_pd_addr(adev
->gmc
.pdb0_bo
?
2117 (*job
)->vm_needs_flush
= true;
2122 return drm_sched_job_add_resv_dependencies(&(*job
)->base
, resv
,
2123 DMA_RESV_USAGE_BOOKKEEP
);
2126 int amdgpu_copy_buffer(struct amdgpu_ring
*ring
, uint64_t src_offset
,
2127 uint64_t dst_offset
, uint32_t byte_count
,
2128 struct dma_resv
*resv
,
2129 struct dma_fence
**fence
, bool direct_submit
,
2130 bool vm_needs_flush
, bool tmz
)
2132 struct amdgpu_device
*adev
= ring
->adev
;
2133 unsigned int num_loops
, num_dw
;
2134 struct amdgpu_job
*job
;
2139 if (!direct_submit
&& !ring
->sched
.ready
) {
2140 DRM_ERROR("Trying to move memory with ring turned off.\n");
2144 max_bytes
= adev
->mman
.buffer_funcs
->copy_max_bytes
;
2145 num_loops
= DIV_ROUND_UP(byte_count
, max_bytes
);
2146 num_dw
= ALIGN(num_loops
* adev
->mman
.buffer_funcs
->copy_num_dw
, 8);
2147 r
= amdgpu_ttm_prepare_job(adev
, direct_submit
, num_dw
,
2148 resv
, vm_needs_flush
, &job
, false);
2152 for (i
= 0; i
< num_loops
; i
++) {
2153 uint32_t cur_size_in_bytes
= min(byte_count
, max_bytes
);
2155 amdgpu_emit_copy_buffer(adev
, &job
->ibs
[0], src_offset
,
2156 dst_offset
, cur_size_in_bytes
, tmz
);
2158 src_offset
+= cur_size_in_bytes
;
2159 dst_offset
+= cur_size_in_bytes
;
2160 byte_count
-= cur_size_in_bytes
;
2163 amdgpu_ring_pad_ib(ring
, &job
->ibs
[0]);
2164 WARN_ON(job
->ibs
[0].length_dw
> num_dw
);
2166 r
= amdgpu_job_submit_direct(job
, ring
, fence
);
2168 *fence
= amdgpu_job_submit(job
);
2175 amdgpu_job_free(job
);
2176 DRM_ERROR("Error scheduling IBs (%d)\n", r
);
2180 static int amdgpu_ttm_fill_mem(struct amdgpu_ring
*ring
, uint32_t src_data
,
2181 uint64_t dst_addr
, uint32_t byte_count
,
2182 struct dma_resv
*resv
,
2183 struct dma_fence
**fence
,
2184 bool vm_needs_flush
, bool delayed
)
2186 struct amdgpu_device
*adev
= ring
->adev
;
2187 unsigned int num_loops
, num_dw
;
2188 struct amdgpu_job
*job
;
2193 max_bytes
= adev
->mman
.buffer_funcs
->fill_max_bytes
;
2194 num_loops
= DIV_ROUND_UP_ULL(byte_count
, max_bytes
);
2195 num_dw
= ALIGN(num_loops
* adev
->mman
.buffer_funcs
->fill_num_dw
, 8);
2196 r
= amdgpu_ttm_prepare_job(adev
, false, num_dw
, resv
, vm_needs_flush
,
2201 for (i
= 0; i
< num_loops
; i
++) {
2202 uint32_t cur_size
= min(byte_count
, max_bytes
);
2204 amdgpu_emit_fill_buffer(adev
, &job
->ibs
[0], src_data
, dst_addr
,
2207 dst_addr
+= cur_size
;
2208 byte_count
-= cur_size
;
2211 amdgpu_ring_pad_ib(ring
, &job
->ibs
[0]);
2212 WARN_ON(job
->ibs
[0].length_dw
> num_dw
);
2213 *fence
= amdgpu_job_submit(job
);
2217 int amdgpu_fill_buffer(struct amdgpu_bo
*bo
,
2219 struct dma_resv
*resv
,
2220 struct dma_fence
**f
,
2223 struct amdgpu_device
*adev
= amdgpu_ttm_adev(bo
->tbo
.bdev
);
2224 struct amdgpu_ring
*ring
= adev
->mman
.buffer_funcs_ring
;
2225 struct dma_fence
*fence
= NULL
;
2226 struct amdgpu_res_cursor dst
;
2229 if (!adev
->mman
.buffer_funcs_enabled
) {
2230 DRM_ERROR("Trying to clear memory with ring turned off.\n");
2234 amdgpu_res_first(bo
->tbo
.resource
, 0, amdgpu_bo_size(bo
), &dst
);
2236 mutex_lock(&adev
->mman
.gtt_window_lock
);
2237 while (dst
.remaining
) {
2238 struct dma_fence
*next
;
2239 uint64_t cur_size
, to
;
2241 /* Never fill more than 256MiB at once to avoid timeouts */
2242 cur_size
= min(dst
.size
, 256ULL << 20);
2244 r
= amdgpu_ttm_map_buffer(&bo
->tbo
, bo
->tbo
.resource
, &dst
,
2245 1, ring
, false, &cur_size
, &to
);
2249 r
= amdgpu_ttm_fill_mem(ring
, src_data
, to
, cur_size
, resv
,
2250 &next
, true, delayed
);
2254 dma_fence_put(fence
);
2257 amdgpu_res_next(&dst
, cur_size
);
2260 mutex_unlock(&adev
->mman
.gtt_window_lock
);
2262 *f
= dma_fence_get(fence
);
2263 dma_fence_put(fence
);
2268 * amdgpu_ttm_evict_resources - evict memory buffers
2269 * @adev: amdgpu device object
2270 * @mem_type: evicted BO's memory type
2272 * Evicts all @mem_type buffers on the lru list of the memory type.
2275 * 0 for success or a negative error code on failure.
2277 int amdgpu_ttm_evict_resources(struct amdgpu_device
*adev
, int mem_type
)
2279 struct ttm_resource_manager
*man
;
2287 man
= ttm_manager_type(&adev
->mman
.bdev
, mem_type
);
2290 DRM_ERROR("Trying to evict invalid memory type\n");
2294 return ttm_resource_manager_evict_all(&adev
->mman
.bdev
, man
);
2297 #if defined(CONFIG_DEBUG_FS)
2299 static int amdgpu_ttm_page_pool_show(struct seq_file
*m
, void *unused
)
2301 struct amdgpu_device
*adev
= m
->private;
2303 return ttm_pool_debugfs(&adev
->mman
.bdev
.pool
, m
);
2306 DEFINE_SHOW_ATTRIBUTE(amdgpu_ttm_page_pool
);
2309 * amdgpu_ttm_vram_read - Linear read access to VRAM
2311 * Accesses VRAM via MMIO for debugging purposes.
2313 static ssize_t
amdgpu_ttm_vram_read(struct file
*f
, char __user
*buf
,
2314 size_t size
, loff_t
*pos
)
2316 struct amdgpu_device
*adev
= file_inode(f
)->i_private
;
2319 if (size
& 0x3 || *pos
& 0x3)
2322 if (*pos
>= adev
->gmc
.mc_vram_size
)
2325 size
= min(size
, (size_t)(adev
->gmc
.mc_vram_size
- *pos
));
2327 size_t bytes
= min(size
, AMDGPU_TTM_VRAM_MAX_DW_READ
* 4);
2328 uint32_t value
[AMDGPU_TTM_VRAM_MAX_DW_READ
];
2330 amdgpu_device_vram_access(adev
, *pos
, value
, bytes
, false);
2331 if (copy_to_user(buf
, value
, bytes
))
2344 * amdgpu_ttm_vram_write - Linear write access to VRAM
2346 * Accesses VRAM via MMIO for debugging purposes.
2348 static ssize_t
amdgpu_ttm_vram_write(struct file
*f
, const char __user
*buf
,
2349 size_t size
, loff_t
*pos
)
2351 struct amdgpu_device
*adev
= file_inode(f
)->i_private
;
2355 if (size
& 0x3 || *pos
& 0x3)
2358 if (*pos
>= adev
->gmc
.mc_vram_size
)
2364 if (*pos
>= adev
->gmc
.mc_vram_size
)
2367 r
= get_user(value
, (uint32_t *)buf
);
2371 amdgpu_device_mm_access(adev
, *pos
, &value
, 4, true);
2382 static const struct file_operations amdgpu_ttm_vram_fops
= {
2383 .owner
= THIS_MODULE
,
2384 .read
= amdgpu_ttm_vram_read
,
2385 .write
= amdgpu_ttm_vram_write
,
2386 .llseek
= default_llseek
,
2390 * amdgpu_iomem_read - Virtual read access to GPU mapped memory
2392 * This function is used to read memory that has been mapped to the
2393 * GPU and the known addresses are not physical addresses but instead
2394 * bus addresses (e.g., what you'd put in an IB or ring buffer).
2396 static ssize_t
amdgpu_iomem_read(struct file
*f
, char __user
*buf
,
2397 size_t size
, loff_t
*pos
)
2399 struct amdgpu_device
*adev
= file_inode(f
)->i_private
;
2400 struct iommu_domain
*dom
;
2404 /* retrieve the IOMMU domain if any for this device */
2405 dom
= iommu_get_domain_for_dev(adev
->dev
);
2408 phys_addr_t addr
= *pos
& PAGE_MASK
;
2409 loff_t off
= *pos
& ~PAGE_MASK
;
2410 size_t bytes
= PAGE_SIZE
- off
;
2415 bytes
= min(bytes
, size
);
2417 /* Translate the bus address to a physical address. If
2418 * the domain is NULL it means there is no IOMMU active
2419 * and the address translation is the identity
2421 addr
= dom
? iommu_iova_to_phys(dom
, addr
) : addr
;
2423 pfn
= addr
>> PAGE_SHIFT
;
2424 if (!pfn_valid(pfn
))
2427 p
= pfn_to_page(pfn
);
2428 if (p
->mapping
!= adev
->mman
.bdev
.dev_mapping
)
2431 ptr
= kmap_local_page(p
);
2432 r
= copy_to_user(buf
, ptr
+ off
, bytes
);
2446 * amdgpu_iomem_write - Virtual write access to GPU mapped memory
2448 * This function is used to write memory that has been mapped to the
2449 * GPU and the known addresses are not physical addresses but instead
2450 * bus addresses (e.g., what you'd put in an IB or ring buffer).
2452 static ssize_t
amdgpu_iomem_write(struct file
*f
, const char __user
*buf
,
2453 size_t size
, loff_t
*pos
)
2455 struct amdgpu_device
*adev
= file_inode(f
)->i_private
;
2456 struct iommu_domain
*dom
;
2460 dom
= iommu_get_domain_for_dev(adev
->dev
);
2463 phys_addr_t addr
= *pos
& PAGE_MASK
;
2464 loff_t off
= *pos
& ~PAGE_MASK
;
2465 size_t bytes
= PAGE_SIZE
- off
;
2470 bytes
= min(bytes
, size
);
2472 addr
= dom
? iommu_iova_to_phys(dom
, addr
) : addr
;
2474 pfn
= addr
>> PAGE_SHIFT
;
2475 if (!pfn_valid(pfn
))
2478 p
= pfn_to_page(pfn
);
2479 if (p
->mapping
!= adev
->mman
.bdev
.dev_mapping
)
2482 ptr
= kmap_local_page(p
);
2483 r
= copy_from_user(ptr
+ off
, buf
, bytes
);
2496 static const struct file_operations amdgpu_ttm_iomem_fops
= {
2497 .owner
= THIS_MODULE
,
2498 .read
= amdgpu_iomem_read
,
2499 .write
= amdgpu_iomem_write
,
2500 .llseek
= default_llseek
2505 void amdgpu_ttm_debugfs_init(struct amdgpu_device
*adev
)
2507 #if defined(CONFIG_DEBUG_FS)
2508 struct drm_minor
*minor
= adev_to_drm(adev
)->primary
;
2509 struct dentry
*root
= minor
->debugfs_root
;
2511 debugfs_create_file_size("amdgpu_vram", 0444, root
, adev
,
2512 &amdgpu_ttm_vram_fops
, adev
->gmc
.mc_vram_size
);
2513 debugfs_create_file("amdgpu_iomem", 0444, root
, adev
,
2514 &amdgpu_ttm_iomem_fops
);
2515 debugfs_create_file("ttm_page_pool", 0444, root
, adev
,
2516 &amdgpu_ttm_page_pool_fops
);
2517 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev
->mman
.bdev
,
2519 root
, "amdgpu_vram_mm");
2520 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev
->mman
.bdev
,
2522 root
, "amdgpu_gtt_mm");
2523 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev
->mman
.bdev
,
2525 root
, "amdgpu_gds_mm");
2526 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev
->mman
.bdev
,
2528 root
, "amdgpu_gws_mm");
2529 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev
->mman
.bdev
,
2531 root
, "amdgpu_oa_mm");