2 * Copyright 2019 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include "amdgpu_vm.h"
24 #include "amdgpu_job.h"
25 #include "amdgpu_object.h"
26 #include "amdgpu_trace.h"
28 #define AMDGPU_VM_SDMA_MIN_NUM_DW 256u
29 #define AMDGPU_VM_SDMA_MAX_NUM_DW (16u * 1024u)
32 * amdgpu_vm_sdma_map_table - make sure new PDs/PTs are GTT mapped
34 * @table: newly allocated or validated PD/PT
36 static int amdgpu_vm_sdma_map_table(struct amdgpu_bo
*table
)
40 r
= amdgpu_ttm_alloc_gart(&table
->tbo
);
45 r
= amdgpu_ttm_alloc_gart(&table
->shadow
->tbo
);
51 * amdgpu_vm_sdma_prepare - prepare SDMA command submission
53 * @p: see amdgpu_vm_update_params definition
54 * @owner: owner we need to sync to
55 * @exclusive: exclusive move fence we need to sync to
58 * Negativ errno, 0 for success.
60 static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params
*p
,
61 struct dma_resv
*resv
,
62 enum amdgpu_sync_mode sync_mode
)
64 enum amdgpu_ib_pool_type pool
= p
->immediate
? AMDGPU_IB_POOL_IMMEDIATE
65 : AMDGPU_IB_POOL_DELAYED
;
66 unsigned int ndw
= AMDGPU_VM_SDMA_MIN_NUM_DW
;
69 r
= amdgpu_job_alloc_with_ib(p
->adev
, ndw
* 4, pool
, &p
->job
);
78 return amdgpu_sync_resv(p
->adev
, &p
->job
->sync
, resv
, sync_mode
, p
->vm
);
82 * amdgpu_vm_sdma_commit - commit SDMA command submission
84 * @p: see amdgpu_vm_update_params definition
85 * @fence: resulting fence
88 * Negativ errno, 0 for success.
90 static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params
*p
,
91 struct dma_fence
**fence
)
93 struct amdgpu_ib
*ib
= p
->job
->ibs
;
94 struct drm_sched_entity
*entity
;
95 struct amdgpu_ring
*ring
;
99 entity
= p
->immediate
? &p
->vm
->immediate
: &p
->vm
->delayed
;
100 ring
= container_of(entity
->rq
->sched
, struct amdgpu_ring
, sched
);
102 WARN_ON(ib
->length_dw
== 0);
103 amdgpu_ring_pad_ib(ring
, ib
);
104 WARN_ON(ib
->length_dw
> p
->num_dw_left
);
105 r
= amdgpu_job_submit(p
->job
, entity
, AMDGPU_FENCE_OWNER_VM
, &f
);
110 struct dma_fence
*tmp
= dma_fence_get(f
);
112 swap(p
->vm
->last_unlocked
, f
);
115 amdgpu_bo_fence(p
->vm
->root
.base
.bo
, f
, true);
118 if (fence
&& !p
->immediate
)
124 amdgpu_job_free(p
->job
);
129 * amdgpu_vm_sdma_copy_ptes - copy the PTEs from mapping
131 * @p: see amdgpu_vm_update_params definition
132 * @bo: PD/PT to update
133 * @pe: addr of the page entry
134 * @count: number of page entries to copy
136 * Traces the parameters and calls the DMA function to copy the PTEs.
138 static void amdgpu_vm_sdma_copy_ptes(struct amdgpu_vm_update_params
*p
,
139 struct amdgpu_bo
*bo
, uint64_t pe
,
142 struct amdgpu_ib
*ib
= p
->job
->ibs
;
143 uint64_t src
= ib
->gpu_addr
;
145 src
+= p
->num_dw_left
* 4;
147 pe
+= amdgpu_gmc_sign_extend(bo
->tbo
.offset
);
148 trace_amdgpu_vm_copy_ptes(pe
, src
, count
, p
->immediate
);
150 amdgpu_vm_copy_pte(p
->adev
, ib
, pe
, src
, count
);
154 * amdgpu_vm_sdma_set_ptes - helper to call the right asic function
156 * @p: see amdgpu_vm_update_params definition
157 * @bo: PD/PT to update
158 * @pe: addr of the page entry
159 * @addr: dst addr to write into pe
160 * @count: number of page entries to update
161 * @incr: increase next addr by incr bytes
162 * @flags: hw access flags
164 * Traces the parameters and calls the right asic functions
165 * to setup the page table using the DMA.
167 static void amdgpu_vm_sdma_set_ptes(struct amdgpu_vm_update_params
*p
,
168 struct amdgpu_bo
*bo
, uint64_t pe
,
169 uint64_t addr
, unsigned count
,
170 uint32_t incr
, uint64_t flags
)
172 struct amdgpu_ib
*ib
= p
->job
->ibs
;
174 pe
+= amdgpu_gmc_sign_extend(bo
->tbo
.offset
);
175 trace_amdgpu_vm_set_ptes(pe
, addr
, count
, incr
, flags
, p
->immediate
);
177 amdgpu_vm_write_pte(p
->adev
, ib
, pe
, addr
| flags
,
180 amdgpu_vm_set_pte_pde(p
->adev
, ib
, pe
, addr
,
186 * amdgpu_vm_sdma_update - execute VM update
188 * @p: see amdgpu_vm_update_params definition
189 * @bo: PD/PT to update
190 * @pe: addr of the page entry
191 * @addr: dst addr to write into pe
192 * @count: number of page entries to update
193 * @incr: increase next addr by incr bytes
194 * @flags: hw access flags
196 * Reserve space in the IB, setup mapping buffer on demand and write commands to
199 static int amdgpu_vm_sdma_update(struct amdgpu_vm_update_params
*p
,
200 struct amdgpu_bo
*bo
, uint64_t pe
,
201 uint64_t addr
, unsigned count
, uint32_t incr
,
204 enum amdgpu_ib_pool_type pool
= p
->immediate
? AMDGPU_IB_POOL_IMMEDIATE
205 : AMDGPU_IB_POOL_DELAYED
;
206 unsigned int i
, ndw
, nptes
;
210 /* Wait for PD/PT moves to be completed */
211 r
= amdgpu_sync_fence(&p
->job
->sync
, bo
->tbo
.moving
, false);
216 ndw
= p
->num_dw_left
;
217 ndw
-= p
->job
->ibs
->length_dw
;
220 r
= amdgpu_vm_sdma_commit(p
, NULL
);
224 /* estimate how many dw we need */
228 ndw
= max(ndw
, AMDGPU_VM_SDMA_MIN_NUM_DW
);
229 ndw
= min(ndw
, AMDGPU_VM_SDMA_MAX_NUM_DW
);
231 r
= amdgpu_job_alloc_with_ib(p
->adev
, ndw
* 4, pool
,
236 p
->num_dw_left
= ndw
;
239 if (!p
->pages_addr
) {
240 /* set page commands needed */
242 amdgpu_vm_sdma_set_ptes(p
, bo
->shadow
, pe
, addr
,
244 amdgpu_vm_sdma_set_ptes(p
, bo
, pe
, addr
, count
,
249 /* copy commands needed */
250 ndw
-= p
->adev
->vm_manager
.vm_pte_funcs
->copy_pte_num_dw
*
251 (bo
->shadow
? 2 : 1);
256 nptes
= min(count
, ndw
/ 2);
258 /* Put the PTEs at the end of the IB. */
259 p
->num_dw_left
-= nptes
* 2;
260 pte
= (uint64_t *)&(p
->job
->ibs
->ptr
[p
->num_dw_left
]);
261 for (i
= 0; i
< nptes
; ++i
, addr
+= incr
) {
262 pte
[i
] = amdgpu_vm_map_gart(p
->pages_addr
, addr
);
267 amdgpu_vm_sdma_copy_ptes(p
, bo
->shadow
, pe
, nptes
);
268 amdgpu_vm_sdma_copy_ptes(p
, bo
, pe
, nptes
);
277 const struct amdgpu_vm_update_funcs amdgpu_vm_sdma_funcs
= {
278 .map_table
= amdgpu_vm_sdma_map_table
,
279 .prepare
= amdgpu_vm_sdma_prepare
,
280 .update
= amdgpu_vm_sdma_update
,
281 .commit
= amdgpu_vm_sdma_commit