2 * Copyright © 2016 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include <linux/sched/mm.h>
26 #include <linux/dma-fence-array.h>
27 #include <drm/drm_gem.h>
29 #include "display/intel_frontbuffer.h"
30 #include "gem/i915_gem_lmem.h"
31 #include "gem/i915_gem_tiling.h"
32 #include "gt/intel_engine.h"
33 #include "gt/intel_engine_heartbeat.h"
34 #include "gt/intel_gt.h"
35 #include "gt/intel_gt_requests.h"
38 #include "i915_gem_evict.h"
39 #include "i915_sw_fence_work.h"
40 #include "i915_trace.h"
42 #include "i915_vma_resource.h"
44 static inline void assert_vma_held_evict(const struct i915_vma
*vma
)
47 * We may be forced to unbind when the vm is dead, to clean it up.
48 * This is the only exception to the requirement of the object lock
51 if (kref_read(&vma
->vm
->ref
))
52 assert_object_held_shared(vma
->obj
);
55 static struct kmem_cache
*slab_vmas
;
57 static struct i915_vma
*i915_vma_alloc(void)
59 return kmem_cache_zalloc(slab_vmas
, GFP_KERNEL
);
62 static void i915_vma_free(struct i915_vma
*vma
)
64 return kmem_cache_free(slab_vmas
, vma
);
67 #if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM) && IS_ENABLED(CONFIG_DRM_DEBUG_MM)
69 #include <linux/stackdepot.h>
71 static void vma_print_allocator(struct i915_vma
*vma
, const char *reason
)
75 if (!vma
->node
.stack
) {
76 DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: unknown owner\n",
77 vma
->node
.start
, vma
->node
.size
, reason
);
81 stack_depot_snprint(vma
->node
.stack
, buf
, sizeof(buf
), 0);
82 DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: inserted at %s\n",
83 vma
->node
.start
, vma
->node
.size
, reason
, buf
);
88 static void vma_print_allocator(struct i915_vma
*vma
, const char *reason
)
94 static inline struct i915_vma
*active_to_vma(struct i915_active
*ref
)
96 return container_of(ref
, typeof(struct i915_vma
), active
);
99 static int __i915_vma_active(struct i915_active
*ref
)
101 return i915_vma_tryget(active_to_vma(ref
)) ? 0 : -ENOENT
;
104 static void __i915_vma_retire(struct i915_active
*ref
)
106 i915_vma_put(active_to_vma(ref
));
109 static struct i915_vma
*
110 vma_create(struct drm_i915_gem_object
*obj
,
111 struct i915_address_space
*vm
,
112 const struct i915_ggtt_view
*view
)
114 struct i915_vma
*pos
= ERR_PTR(-E2BIG
);
115 struct i915_vma
*vma
;
116 struct rb_node
*rb
, **p
;
119 /* The aliasing_ppgtt should never be used directly! */
120 GEM_BUG_ON(vm
== &vm
->gt
->ggtt
->alias
->vm
);
122 vma
= i915_vma_alloc();
124 return ERR_PTR(-ENOMEM
);
126 vma
->ops
= &vm
->vma_ops
;
128 vma
->size
= obj
->base
.size
;
129 vma
->display_alignment
= I915_GTT_MIN_ALIGNMENT
;
131 i915_active_init(&vma
->active
, __i915_vma_active
, __i915_vma_retire
, 0);
133 /* Declare ourselves safe for use inside shrinkers */
134 if (IS_ENABLED(CONFIG_LOCKDEP
)) {
135 fs_reclaim_acquire(GFP_KERNEL
);
136 might_lock(&vma
->active
.mutex
);
137 fs_reclaim_release(GFP_KERNEL
);
140 INIT_LIST_HEAD(&vma
->closed_link
);
141 INIT_LIST_HEAD(&vma
->obj_link
);
142 RB_CLEAR_NODE(&vma
->obj_node
);
144 if (view
&& view
->type
!= I915_GGTT_VIEW_NORMAL
) {
145 vma
->ggtt_view
= *view
;
146 if (view
->type
== I915_GGTT_VIEW_PARTIAL
) {
147 GEM_BUG_ON(range_overflows_t(u64
,
148 view
->partial
.offset
,
150 obj
->base
.size
>> PAGE_SHIFT
));
151 vma
->size
= view
->partial
.size
;
152 vma
->size
<<= PAGE_SHIFT
;
153 GEM_BUG_ON(vma
->size
> obj
->base
.size
);
154 } else if (view
->type
== I915_GGTT_VIEW_ROTATED
) {
155 vma
->size
= intel_rotation_info_size(&view
->rotated
);
156 vma
->size
<<= PAGE_SHIFT
;
157 } else if (view
->type
== I915_GGTT_VIEW_REMAPPED
) {
158 vma
->size
= intel_remapped_info_size(&view
->remapped
);
159 vma
->size
<<= PAGE_SHIFT
;
163 if (unlikely(vma
->size
> vm
->total
))
166 GEM_BUG_ON(!IS_ALIGNED(vma
->size
, I915_GTT_PAGE_SIZE
));
168 err
= mutex_lock_interruptible(&vm
->mutex
);
175 list_add_tail(&vma
->vm_link
, &vm
->unbound_list
);
177 spin_lock(&obj
->vma
.lock
);
178 if (i915_is_ggtt(vm
)) {
179 if (unlikely(overflows_type(vma
->size
, u32
)))
182 vma
->fence_size
= i915_gem_fence_size(vm
->i915
, vma
->size
,
183 i915_gem_object_get_tiling(obj
),
184 i915_gem_object_get_stride(obj
));
185 if (unlikely(vma
->fence_size
< vma
->size
|| /* overflow */
186 vma
->fence_size
> vm
->total
))
189 GEM_BUG_ON(!IS_ALIGNED(vma
->fence_size
, I915_GTT_MIN_ALIGNMENT
));
191 vma
->fence_alignment
= i915_gem_fence_alignment(vm
->i915
, vma
->size
,
192 i915_gem_object_get_tiling(obj
),
193 i915_gem_object_get_stride(obj
));
194 GEM_BUG_ON(!is_power_of_2(vma
->fence_alignment
));
196 __set_bit(I915_VMA_GGTT_BIT
, __i915_vma_flags(vma
));
200 p
= &obj
->vma
.tree
.rb_node
;
205 pos
= rb_entry(rb
, struct i915_vma
, obj_node
);
208 * If the view already exists in the tree, another thread
209 * already created a matching vma, so return the older instance
210 * and dispose of ours.
212 cmp
= i915_vma_compare(pos
, vm
, view
);
220 rb_link_node(&vma
->obj_node
, rb
, p
);
221 rb_insert_color(&vma
->obj_node
, &obj
->vma
.tree
);
223 if (i915_vma_is_ggtt(vma
))
225 * We put the GGTT vma at the start of the vma-list, followed
226 * by the ppGGTT vma. This allows us to break early when
227 * iterating over only the GGTT vma for an object, see
228 * for_each_ggtt_vma()
230 list_add(&vma
->obj_link
, &obj
->vma
.list
);
232 list_add_tail(&vma
->obj_link
, &obj
->vma
.list
);
234 spin_unlock(&obj
->vma
.lock
);
235 mutex_unlock(&vm
->mutex
);
240 spin_unlock(&obj
->vma
.lock
);
241 list_del_init(&vma
->vm_link
);
242 mutex_unlock(&vm
->mutex
);
248 static struct i915_vma
*
249 i915_vma_lookup(struct drm_i915_gem_object
*obj
,
250 struct i915_address_space
*vm
,
251 const struct i915_ggtt_view
*view
)
255 rb
= obj
->vma
.tree
.rb_node
;
257 struct i915_vma
*vma
= rb_entry(rb
, struct i915_vma
, obj_node
);
260 cmp
= i915_vma_compare(vma
, vm
, view
);
274 * i915_vma_instance - return the singleton instance of the VMA
275 * @obj: parent &struct drm_i915_gem_object to be mapped
276 * @vm: address space in which the mapping is located
277 * @view: additional mapping requirements
279 * i915_vma_instance() looks up an existing VMA of the @obj in the @vm with
280 * the same @view characteristics. If a match is not found, one is created.
281 * Once created, the VMA is kept until either the object is freed, or the
282 * address space is closed.
284 * Returns the vma, or an error pointer.
287 i915_vma_instance(struct drm_i915_gem_object
*obj
,
288 struct i915_address_space
*vm
,
289 const struct i915_ggtt_view
*view
)
291 struct i915_vma
*vma
;
293 GEM_BUG_ON(view
&& !i915_is_ggtt_or_dpt(vm
));
294 GEM_BUG_ON(!kref_read(&vm
->ref
));
296 spin_lock(&obj
->vma
.lock
);
297 vma
= i915_vma_lookup(obj
, vm
, view
);
298 spin_unlock(&obj
->vma
.lock
);
300 /* vma_create() will resolve the race if another creates the vma */
302 vma
= vma_create(obj
, vm
, view
);
304 GEM_BUG_ON(!IS_ERR(vma
) && i915_vma_compare(vma
, vm
, view
));
308 struct i915_vma_work
{
309 struct dma_fence_work base
;
310 struct i915_address_space
*vm
;
311 struct i915_vm_pt_stash stash
;
312 struct i915_vma_resource
*vma_res
;
313 struct drm_i915_gem_object
*pinned
;
314 struct i915_sw_dma_fence_cb cb
;
315 enum i915_cache_level cache_level
;
319 static void __vma_bind(struct dma_fence_work
*work
)
321 struct i915_vma_work
*vw
= container_of(work
, typeof(*vw
), base
);
322 struct i915_vma_resource
*vma_res
= vw
->vma_res
;
324 vma_res
->ops
->bind_vma(vma_res
->vm
, &vw
->stash
,
325 vma_res
, vw
->cache_level
, vw
->flags
);
329 static void __vma_release(struct dma_fence_work
*work
)
331 struct i915_vma_work
*vw
= container_of(work
, typeof(*vw
), base
);
334 i915_gem_object_put(vw
->pinned
);
336 i915_vm_free_pt_stash(vw
->vm
, &vw
->stash
);
338 i915_vma_resource_put(vw
->vma_res
);
341 static const struct dma_fence_work_ops bind_ops
= {
344 .release
= __vma_release
,
347 struct i915_vma_work
*i915_vma_work(void)
349 struct i915_vma_work
*vw
;
351 vw
= kzalloc(sizeof(*vw
), GFP_KERNEL
);
355 dma_fence_work_init(&vw
->base
, &bind_ops
);
356 vw
->base
.dma
.error
= -EAGAIN
; /* disable the worker by default */
361 int i915_vma_wait_for_bind(struct i915_vma
*vma
)
365 if (rcu_access_pointer(vma
->active
.excl
.fence
)) {
366 struct dma_fence
*fence
;
369 fence
= dma_fence_get_rcu_safe(&vma
->active
.excl
.fence
);
372 err
= dma_fence_wait(fence
, true);
373 dma_fence_put(fence
);
380 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
381 static int i915_vma_verify_bind_complete(struct i915_vma
*vma
)
383 struct dma_fence
*fence
= i915_active_fence_get(&vma
->active
.excl
);
389 if (dma_fence_is_signaled(fence
))
394 dma_fence_put(fence
);
399 #define i915_vma_verify_bind_complete(_vma) 0
402 I915_SELFTEST_EXPORT
void
403 i915_vma_resource_init_from_vma(struct i915_vma_resource
*vma_res
,
404 struct i915_vma
*vma
)
406 struct drm_i915_gem_object
*obj
= vma
->obj
;
408 i915_vma_resource_init(vma_res
, vma
->vm
, vma
->pages
, &vma
->page_sizes
,
409 obj
->mm
.rsgt
, i915_gem_object_is_readonly(obj
),
410 i915_gem_object_is_lmem(obj
), obj
->mm
.region
,
411 vma
->ops
, vma
->private, vma
->node
.start
,
412 vma
->node
.size
, vma
->size
);
416 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
418 * @cache_level: mapping cache level
419 * @flags: flags like global or local mapping
420 * @work: preallocated worker for allocating and binding the PTE
421 * @vma_res: pointer to a preallocated vma resource. The resource is either
424 * DMA addresses are taken from the scatter-gather table of this object (or of
425 * this VMA in case of non-default GGTT views) and PTE entries set up.
426 * Note that DMA addresses are also the only part of the SG table we care about.
428 int i915_vma_bind(struct i915_vma
*vma
,
429 enum i915_cache_level cache_level
,
431 struct i915_vma_work
*work
,
432 struct i915_vma_resource
*vma_res
)
438 lockdep_assert_held(&vma
->vm
->mutex
);
439 GEM_BUG_ON(!drm_mm_node_allocated(&vma
->node
));
440 GEM_BUG_ON(vma
->size
> vma
->node
.size
);
442 if (GEM_DEBUG_WARN_ON(range_overflows(vma
->node
.start
,
445 i915_vma_resource_free(vma_res
);
449 if (GEM_DEBUG_WARN_ON(!flags
)) {
450 i915_vma_resource_free(vma_res
);
455 bind_flags
&= I915_VMA_GLOBAL_BIND
| I915_VMA_LOCAL_BIND
;
457 vma_flags
= atomic_read(&vma
->flags
);
458 vma_flags
&= I915_VMA_GLOBAL_BIND
| I915_VMA_LOCAL_BIND
;
460 bind_flags
&= ~vma_flags
;
461 if (bind_flags
== 0) {
462 i915_vma_resource_free(vma_res
);
466 GEM_BUG_ON(!atomic_read(&vma
->pages_count
));
468 /* Wait for or await async unbinds touching our range */
469 if (work
&& bind_flags
& vma
->vm
->bind_async_flags
)
470 ret
= i915_vma_resource_bind_dep_await(vma
->vm
,
476 __GFP_RETRY_MAYFAIL
|
479 ret
= i915_vma_resource_bind_dep_sync(vma
->vm
, vma
->node
.start
,
480 vma
->node
.size
, true);
482 i915_vma_resource_free(vma_res
);
486 if (vma
->resource
|| !vma_res
) {
487 /* Rebinding with an additional I915_VMA_*_BIND */
488 GEM_WARN_ON(!vma_flags
);
489 i915_vma_resource_free(vma_res
);
491 i915_vma_resource_init_from_vma(vma_res
, vma
);
492 vma
->resource
= vma_res
;
494 trace_i915_vma_bind(vma
, bind_flags
);
495 if (work
&& bind_flags
& vma
->vm
->bind_async_flags
) {
496 struct dma_fence
*prev
;
498 work
->vma_res
= i915_vma_resource_get(vma
->resource
);
499 work
->cache_level
= cache_level
;
500 work
->flags
= bind_flags
;
503 * Note we only want to chain up to the migration fence on
504 * the pages (not the object itself). As we don't track that,
505 * yet, we have to use the exclusive fence instead.
507 * Also note that we do not want to track the async vma as
508 * part of the obj->resv->excl_fence as it only affects
509 * execution and not content or object's backing store lifetime.
511 prev
= i915_active_set_exclusive(&vma
->active
, &work
->base
.dma
);
513 __i915_sw_fence_await_dma_fence(&work
->base
.chain
,
519 work
->base
.dma
.error
= 0; /* enable the queue_work() */
522 * If we don't have the refcounted pages list, keep a reference
523 * on the object to avoid waiting for the async bind to
524 * complete in the object destruction path.
526 if (!work
->vma_res
->bi
.pages_rsgt
)
527 work
->pinned
= i915_gem_object_get(vma
->obj
);
529 ret
= i915_gem_object_wait_moving_fence(vma
->obj
, true);
531 i915_vma_resource_free(vma
->resource
);
532 vma
->resource
= NULL
;
536 vma
->ops
->bind_vma(vma
->vm
, NULL
, vma
->resource
, cache_level
,
540 set_bit(I915_BO_WAS_BOUND_BIT
, &vma
->obj
->flags
);
542 atomic_or(bind_flags
, &vma
->flags
);
546 void __iomem
*i915_vma_pin_iomap(struct i915_vma
*vma
)
551 if (WARN_ON_ONCE(vma
->obj
->flags
& I915_BO_ALLOC_GPU_ONLY
))
552 return IOMEM_ERR_PTR(-EINVAL
);
554 if (!i915_gem_object_is_lmem(vma
->obj
)) {
555 if (GEM_WARN_ON(!i915_vma_is_map_and_fenceable(vma
))) {
561 GEM_BUG_ON(!i915_vma_is_ggtt(vma
));
562 GEM_BUG_ON(!i915_vma_is_bound(vma
, I915_VMA_GLOBAL_BIND
));
563 GEM_BUG_ON(i915_vma_verify_bind_complete(vma
));
565 ptr
= READ_ONCE(vma
->iomap
);
568 * TODO: consider just using i915_gem_object_pin_map() for lmem
569 * instead, which already supports mapping non-contiguous chunks
570 * of pages, that way we can also drop the
571 * I915_BO_ALLOC_CONTIGUOUS when allocating the object.
573 if (i915_gem_object_is_lmem(vma
->obj
))
574 ptr
= i915_gem_object_lmem_io_map(vma
->obj
, 0,
575 vma
->obj
->base
.size
);
577 ptr
= io_mapping_map_wc(&i915_vm_to_ggtt(vma
->vm
)->iomap
,
585 if (unlikely(cmpxchg(&vma
->iomap
, NULL
, ptr
))) {
586 io_mapping_unmap(ptr
);
593 err
= i915_vma_pin_fence(vma
);
597 i915_vma_set_ggtt_write(vma
);
599 /* NB Access through the GTT requires the device to be awake. */
603 __i915_vma_unpin(vma
);
605 return IOMEM_ERR_PTR(err
);
608 void i915_vma_flush_writes(struct i915_vma
*vma
)
610 if (i915_vma_unset_ggtt_write(vma
))
611 intel_gt_flush_ggtt_writes(vma
->vm
->gt
);
614 void i915_vma_unpin_iomap(struct i915_vma
*vma
)
616 GEM_BUG_ON(vma
->iomap
== NULL
);
618 i915_vma_flush_writes(vma
);
620 i915_vma_unpin_fence(vma
);
624 void i915_vma_unpin_and_release(struct i915_vma
**p_vma
, unsigned int flags
)
626 struct i915_vma
*vma
;
627 struct drm_i915_gem_object
*obj
;
629 vma
= fetch_and_zero(p_vma
);
638 if (flags
& I915_VMA_RELEASE_MAP
)
639 i915_gem_object_unpin_map(obj
);
641 i915_gem_object_put(obj
);
644 bool i915_vma_misplaced(const struct i915_vma
*vma
,
645 u64 size
, u64 alignment
, u64 flags
)
647 if (!drm_mm_node_allocated(&vma
->node
))
650 if (test_bit(I915_VMA_ERROR_BIT
, __i915_vma_flags(vma
)))
653 if (vma
->node
.size
< size
)
656 GEM_BUG_ON(alignment
&& !is_power_of_2(alignment
));
657 if (alignment
&& !IS_ALIGNED(vma
->node
.start
, alignment
))
660 if (flags
& PIN_MAPPABLE
&& !i915_vma_is_map_and_fenceable(vma
))
663 if (flags
& PIN_OFFSET_BIAS
&&
664 vma
->node
.start
< (flags
& PIN_OFFSET_MASK
))
667 if (flags
& PIN_OFFSET_FIXED
&&
668 vma
->node
.start
!= (flags
& PIN_OFFSET_MASK
))
674 void __i915_vma_set_map_and_fenceable(struct i915_vma
*vma
)
676 bool mappable
, fenceable
;
678 GEM_BUG_ON(!i915_vma_is_ggtt(vma
));
679 GEM_BUG_ON(!vma
->fence_size
);
681 fenceable
= (vma
->node
.size
>= vma
->fence_size
&&
682 IS_ALIGNED(vma
->node
.start
, vma
->fence_alignment
));
684 mappable
= vma
->node
.start
+ vma
->fence_size
<= i915_vm_to_ggtt(vma
->vm
)->mappable_end
;
686 if (mappable
&& fenceable
)
687 set_bit(I915_VMA_CAN_FENCE_BIT
, __i915_vma_flags(vma
));
689 clear_bit(I915_VMA_CAN_FENCE_BIT
, __i915_vma_flags(vma
));
692 bool i915_gem_valid_gtt_space(struct i915_vma
*vma
, unsigned long color
)
694 struct drm_mm_node
*node
= &vma
->node
;
695 struct drm_mm_node
*other
;
698 * On some machines we have to be careful when putting differing types
699 * of snoopable memory together to avoid the prefetcher crossing memory
700 * domains and dying. During vm initialisation, we decide whether or not
701 * these constraints apply and set the drm_mm.color_adjust
704 if (!i915_vm_has_cache_coloring(vma
->vm
))
707 /* Only valid to be called on an already inserted vma */
708 GEM_BUG_ON(!drm_mm_node_allocated(node
));
709 GEM_BUG_ON(list_empty(&node
->node_list
));
711 other
= list_prev_entry(node
, node_list
);
712 if (i915_node_color_differs(other
, color
) &&
713 !drm_mm_hole_follows(other
))
716 other
= list_next_entry(node
, node_list
);
717 if (i915_node_color_differs(other
, color
) &&
718 !drm_mm_hole_follows(node
))
725 * i915_vma_insert - finds a slot for the vma in its address space
727 * @size: requested size in bytes (can be larger than the VMA)
728 * @alignment: required alignment
729 * @flags: mask of PIN_* flags to use
731 * First we try to allocate some free space that meets the requirements for
732 * the VMA. Failiing that, if the flags permit, it will evict an old VMA,
733 * preferrably the oldest idle entry to make room for the new VMA.
736 * 0 on success, negative error code otherwise.
739 i915_vma_insert(struct i915_vma
*vma
, struct i915_gem_ww_ctx
*ww
,
740 u64 size
, u64 alignment
, u64 flags
)
746 GEM_BUG_ON(i915_vma_is_bound(vma
, I915_VMA_GLOBAL_BIND
| I915_VMA_LOCAL_BIND
));
747 GEM_BUG_ON(drm_mm_node_allocated(&vma
->node
));
749 size
= max(size
, vma
->size
);
750 alignment
= max(alignment
, vma
->display_alignment
);
751 if (flags
& PIN_MAPPABLE
) {
752 size
= max_t(typeof(size
), size
, vma
->fence_size
);
753 alignment
= max_t(typeof(alignment
),
754 alignment
, vma
->fence_alignment
);
757 GEM_BUG_ON(!IS_ALIGNED(size
, I915_GTT_PAGE_SIZE
));
758 GEM_BUG_ON(!IS_ALIGNED(alignment
, I915_GTT_MIN_ALIGNMENT
));
759 GEM_BUG_ON(!is_power_of_2(alignment
));
761 start
= flags
& PIN_OFFSET_BIAS
? flags
& PIN_OFFSET_MASK
: 0;
762 GEM_BUG_ON(!IS_ALIGNED(start
, I915_GTT_PAGE_SIZE
));
764 end
= vma
->vm
->total
;
765 if (flags
& PIN_MAPPABLE
)
766 end
= min_t(u64
, end
, i915_vm_to_ggtt(vma
->vm
)->mappable_end
);
767 if (flags
& PIN_ZONE_4G
)
768 end
= min_t(u64
, end
, (1ULL << 32) - I915_GTT_PAGE_SIZE
);
769 GEM_BUG_ON(!IS_ALIGNED(end
, I915_GTT_PAGE_SIZE
));
771 alignment
= max(alignment
, i915_vm_obj_min_alignment(vma
->vm
, vma
->obj
));
773 * for compact-pt we round up the reservation to prevent
774 * any smaller pages being used within the same PDE
776 if (NEEDS_COMPACT_PT(vma
->vm
->i915
))
777 size
= round_up(size
, alignment
);
779 /* If binding the object/GGTT view requires more space than the entire
780 * aperture has, reject it early before evicting everything in a vain
781 * attempt to find space.
784 DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n",
785 size
, flags
& PIN_MAPPABLE
? "mappable" : "total",
792 if (i915_vm_has_cache_coloring(vma
->vm
))
793 color
= vma
->obj
->cache_level
;
795 if (flags
& PIN_OFFSET_FIXED
) {
796 u64 offset
= flags
& PIN_OFFSET_MASK
;
797 if (!IS_ALIGNED(offset
, alignment
) ||
798 range_overflows(offset
, size
, end
))
801 ret
= i915_gem_gtt_reserve(vma
->vm
, ww
, &vma
->node
,
808 * We only support huge gtt pages through the 48b PPGTT,
809 * however we also don't want to force any alignment for
810 * objects which need to be tightly packed into the low 32bits.
812 * Note that we assume that GGTT are limited to 4GiB for the
813 * forseeable future. See also i915_ggtt_offset().
815 if (upper_32_bits(end
- 1) &&
816 vma
->page_sizes
.sg
> I915_GTT_PAGE_SIZE
) {
818 * We can't mix 64K and 4K PTEs in the same page-table
819 * (2M block), and so to avoid the ugliness and
820 * complexity of coloring we opt for just aligning 64K
824 rounddown_pow_of_two(vma
->page_sizes
.sg
|
825 I915_GTT_PAGE_SIZE_2M
);
828 * Check we don't expand for the limited Global GTT
829 * (mappable aperture is even more precious!). This
830 * also checks that we exclude the aliasing-ppgtt.
832 GEM_BUG_ON(i915_vma_is_ggtt(vma
));
834 alignment
= max(alignment
, page_alignment
);
836 if (vma
->page_sizes
.sg
& I915_GTT_PAGE_SIZE_64K
)
837 size
= round_up(size
, I915_GTT_PAGE_SIZE_2M
);
840 ret
= i915_gem_gtt_insert(vma
->vm
, ww
, &vma
->node
,
841 size
, alignment
, color
,
846 GEM_BUG_ON(vma
->node
.start
< start
);
847 GEM_BUG_ON(vma
->node
.start
+ vma
->node
.size
> end
);
849 GEM_BUG_ON(!drm_mm_node_allocated(&vma
->node
));
850 GEM_BUG_ON(!i915_gem_valid_gtt_space(vma
, color
));
852 list_move_tail(&vma
->vm_link
, &vma
->vm
->bound_list
);
858 i915_vma_detach(struct i915_vma
*vma
)
860 GEM_BUG_ON(!drm_mm_node_allocated(&vma
->node
));
861 GEM_BUG_ON(i915_vma_is_bound(vma
, I915_VMA_GLOBAL_BIND
| I915_VMA_LOCAL_BIND
));
864 * And finally now the object is completely decoupled from this
865 * vma, we can drop its hold on the backing storage and allow
866 * it to be reaped by the shrinker.
868 list_move_tail(&vma
->vm_link
, &vma
->vm
->unbound_list
);
871 static bool try_qad_pin(struct i915_vma
*vma
, unsigned int flags
)
875 bound
= atomic_read(&vma
->flags
);
877 if (flags
& PIN_VALIDATE
) {
878 flags
&= I915_VMA_BIND_MASK
;
880 return (flags
& bound
) == flags
;
883 /* with the lock mandatory for unbind, we don't race here */
884 flags
&= I915_VMA_BIND_MASK
;
886 if (unlikely(flags
& ~bound
))
889 if (unlikely(bound
& (I915_VMA_OVERFLOW
| I915_VMA_ERROR
)))
892 GEM_BUG_ON(((bound
+ 1) & I915_VMA_PIN_MASK
) == 0);
893 } while (!atomic_try_cmpxchg(&vma
->flags
, &bound
, bound
+ 1));
898 static struct scatterlist
*
899 rotate_pages(struct drm_i915_gem_object
*obj
, unsigned int offset
,
900 unsigned int width
, unsigned int height
,
901 unsigned int src_stride
, unsigned int dst_stride
,
902 struct sg_table
*st
, struct scatterlist
*sg
)
904 unsigned int column
, row
;
905 unsigned int src_idx
;
907 for (column
= 0; column
< width
; column
++) {
910 src_idx
= src_stride
* (height
- 1) + column
+ offset
;
911 for (row
= 0; row
< height
; row
++) {
914 * We don't need the pages, but need to initialize
915 * the entries so the sg list can be happily traversed.
916 * The only thing we need are DMA addresses.
918 sg_set_page(sg
, NULL
, I915_GTT_PAGE_SIZE
, 0);
920 i915_gem_object_get_dma_address(obj
, src_idx
);
921 sg_dma_len(sg
) = I915_GTT_PAGE_SIZE
;
923 src_idx
-= src_stride
;
926 left
= (dst_stride
- height
) * I915_GTT_PAGE_SIZE
;
934 * The DE ignores the PTEs for the padding tiles, the sg entry
935 * here is just a conenience to indicate how many padding PTEs
936 * to insert at this spot.
938 sg_set_page(sg
, NULL
, left
, 0);
939 sg_dma_address(sg
) = 0;
940 sg_dma_len(sg
) = left
;
947 static noinline
struct sg_table
*
948 intel_rotate_pages(struct intel_rotation_info
*rot_info
,
949 struct drm_i915_gem_object
*obj
)
951 unsigned int size
= intel_rotation_info_size(rot_info
);
952 struct drm_i915_private
*i915
= to_i915(obj
->base
.dev
);
954 struct scatterlist
*sg
;
958 /* Allocate target SG list. */
959 st
= kmalloc(sizeof(*st
), GFP_KERNEL
);
963 ret
= sg_alloc_table(st
, size
, GFP_KERNEL
);
970 for (i
= 0 ; i
< ARRAY_SIZE(rot_info
->plane
); i
++)
971 sg
= rotate_pages(obj
, rot_info
->plane
[i
].offset
,
972 rot_info
->plane
[i
].width
, rot_info
->plane
[i
].height
,
973 rot_info
->plane
[i
].src_stride
,
974 rot_info
->plane
[i
].dst_stride
,
983 drm_dbg(&i915
->drm
, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
984 obj
->base
.size
, rot_info
->plane
[0].width
,
985 rot_info
->plane
[0].height
, size
);
990 static struct scatterlist
*
991 add_padding_pages(unsigned int count
,
992 struct sg_table
*st
, struct scatterlist
*sg
)
997 * The DE ignores the PTEs for the padding tiles, the sg entry
998 * here is just a convenience to indicate how many padding PTEs
999 * to insert at this spot.
1001 sg_set_page(sg
, NULL
, count
* I915_GTT_PAGE_SIZE
, 0);
1002 sg_dma_address(sg
) = 0;
1003 sg_dma_len(sg
) = count
* I915_GTT_PAGE_SIZE
;
1009 static struct scatterlist
*
1010 remap_tiled_color_plane_pages(struct drm_i915_gem_object
*obj
,
1011 unsigned int offset
, unsigned int alignment_pad
,
1012 unsigned int width
, unsigned int height
,
1013 unsigned int src_stride
, unsigned int dst_stride
,
1014 struct sg_table
*st
, struct scatterlist
*sg
,
1015 unsigned int *gtt_offset
)
1019 if (!width
|| !height
)
1023 sg
= add_padding_pages(alignment_pad
, st
, sg
);
1025 for (row
= 0; row
< height
; row
++) {
1026 unsigned int left
= width
* I915_GTT_PAGE_SIZE
;
1030 unsigned int length
;
1033 * We don't need the pages, but need to initialize
1034 * the entries so the sg list can be happily traversed.
1035 * The only thing we need are DMA addresses.
1038 addr
= i915_gem_object_get_dma_address_len(obj
, offset
, &length
);
1040 length
= min(left
, length
);
1044 sg_set_page(sg
, NULL
, length
, 0);
1045 sg_dma_address(sg
) = addr
;
1046 sg_dma_len(sg
) = length
;
1049 offset
+= length
/ I915_GTT_PAGE_SIZE
;
1053 offset
+= src_stride
- width
;
1055 left
= (dst_stride
- width
) * I915_GTT_PAGE_SIZE
;
1060 sg
= add_padding_pages(left
>> PAGE_SHIFT
, st
, sg
);
1063 *gtt_offset
+= alignment_pad
+ dst_stride
* height
;
1068 static struct scatterlist
*
1069 remap_contiguous_pages(struct drm_i915_gem_object
*obj
,
1070 unsigned int obj_offset
,
1072 struct sg_table
*st
, struct scatterlist
*sg
)
1074 struct scatterlist
*iter
;
1075 unsigned int offset
;
1077 iter
= i915_gem_object_get_sg_dma(obj
, obj_offset
, &offset
);
1083 len
= min(sg_dma_len(iter
) - (offset
<< PAGE_SHIFT
),
1084 count
<< PAGE_SHIFT
);
1085 sg_set_page(sg
, NULL
, len
, 0);
1086 sg_dma_address(sg
) =
1087 sg_dma_address(iter
) + (offset
<< PAGE_SHIFT
);
1088 sg_dma_len(sg
) = len
;
1091 count
-= len
>> PAGE_SHIFT
;
1096 iter
= __sg_next(iter
);
1101 static struct scatterlist
*
1102 remap_linear_color_plane_pages(struct drm_i915_gem_object
*obj
,
1103 unsigned int obj_offset
, unsigned int alignment_pad
,
1105 struct sg_table
*st
, struct scatterlist
*sg
,
1106 unsigned int *gtt_offset
)
1112 sg
= add_padding_pages(alignment_pad
, st
, sg
);
1114 sg
= remap_contiguous_pages(obj
, obj_offset
, size
, st
, sg
);
1117 *gtt_offset
+= alignment_pad
+ size
;
1122 static struct scatterlist
*
1123 remap_color_plane_pages(const struct intel_remapped_info
*rem_info
,
1124 struct drm_i915_gem_object
*obj
,
1126 struct sg_table
*st
, struct scatterlist
*sg
,
1127 unsigned int *gtt_offset
)
1129 unsigned int alignment_pad
= 0;
1131 if (rem_info
->plane_alignment
)
1132 alignment_pad
= ALIGN(*gtt_offset
, rem_info
->plane_alignment
) - *gtt_offset
;
1134 if (rem_info
->plane
[color_plane
].linear
)
1135 sg
= remap_linear_color_plane_pages(obj
,
1136 rem_info
->plane
[color_plane
].offset
,
1138 rem_info
->plane
[color_plane
].size
,
1143 sg
= remap_tiled_color_plane_pages(obj
,
1144 rem_info
->plane
[color_plane
].offset
,
1146 rem_info
->plane
[color_plane
].width
,
1147 rem_info
->plane
[color_plane
].height
,
1148 rem_info
->plane
[color_plane
].src_stride
,
1149 rem_info
->plane
[color_plane
].dst_stride
,
1156 static noinline
struct sg_table
*
1157 intel_remap_pages(struct intel_remapped_info
*rem_info
,
1158 struct drm_i915_gem_object
*obj
)
1160 unsigned int size
= intel_remapped_info_size(rem_info
);
1161 struct drm_i915_private
*i915
= to_i915(obj
->base
.dev
);
1162 struct sg_table
*st
;
1163 struct scatterlist
*sg
;
1164 unsigned int gtt_offset
= 0;
1168 /* Allocate target SG list. */
1169 st
= kmalloc(sizeof(*st
), GFP_KERNEL
);
1173 ret
= sg_alloc_table(st
, size
, GFP_KERNEL
);
1180 for (i
= 0 ; i
< ARRAY_SIZE(rem_info
->plane
); i
++)
1181 sg
= remap_color_plane_pages(rem_info
, obj
, i
, st
, sg
, >t_offset
);
1191 drm_dbg(&i915
->drm
, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n",
1192 obj
->base
.size
, rem_info
->plane
[0].width
,
1193 rem_info
->plane
[0].height
, size
);
1195 return ERR_PTR(ret
);
1198 static noinline
struct sg_table
*
1199 intel_partial_pages(const struct i915_ggtt_view
*view
,
1200 struct drm_i915_gem_object
*obj
)
1202 struct sg_table
*st
;
1203 struct scatterlist
*sg
;
1204 unsigned int count
= view
->partial
.size
;
1207 st
= kmalloc(sizeof(*st
), GFP_KERNEL
);
1211 ret
= sg_alloc_table(st
, count
, GFP_KERNEL
);
1217 sg
= remap_contiguous_pages(obj
, view
->partial
.offset
, count
, st
, st
->sgl
);
1220 i915_sg_trim(st
); /* Drop any unused tail entries. */
1227 return ERR_PTR(ret
);
1231 __i915_vma_get_pages(struct i915_vma
*vma
)
1233 struct sg_table
*pages
;
1236 * The vma->pages are only valid within the lifespan of the borrowed
1237 * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
1238 * must be the vma->pages. A simple rule is that vma->pages must only
1239 * be accessed when the obj->mm.pages are pinned.
1241 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma
->obj
));
1243 switch (vma
->ggtt_view
.type
) {
1245 GEM_BUG_ON(vma
->ggtt_view
.type
);
1247 case I915_GGTT_VIEW_NORMAL
:
1248 pages
= vma
->obj
->mm
.pages
;
1251 case I915_GGTT_VIEW_ROTATED
:
1253 intel_rotate_pages(&vma
->ggtt_view
.rotated
, vma
->obj
);
1256 case I915_GGTT_VIEW_REMAPPED
:
1258 intel_remap_pages(&vma
->ggtt_view
.remapped
, vma
->obj
);
1261 case I915_GGTT_VIEW_PARTIAL
:
1262 pages
= intel_partial_pages(&vma
->ggtt_view
, vma
->obj
);
1266 if (IS_ERR(pages
)) {
1267 drm_err(&vma
->vm
->i915
->drm
,
1268 "Failed to get pages for VMA view type %u (%ld)!\n",
1269 vma
->ggtt_view
.type
, PTR_ERR(pages
));
1270 return PTR_ERR(pages
);
1278 I915_SELFTEST_EXPORT
int i915_vma_get_pages(struct i915_vma
*vma
)
1282 if (atomic_add_unless(&vma
->pages_count
, 1, 0))
1285 err
= i915_gem_object_pin_pages(vma
->obj
);
1289 err
= __i915_vma_get_pages(vma
);
1293 vma
->page_sizes
= vma
->obj
->mm
.page_sizes
;
1294 atomic_inc(&vma
->pages_count
);
1299 __i915_gem_object_unpin_pages(vma
->obj
);
1304 static void __vma_put_pages(struct i915_vma
*vma
, unsigned int count
)
1306 /* We allocate under vma_get_pages, so beware the shrinker */
1307 GEM_BUG_ON(atomic_read(&vma
->pages_count
) < count
);
1309 if (atomic_sub_return(count
, &vma
->pages_count
) == 0) {
1310 if (vma
->pages
!= vma
->obj
->mm
.pages
) {
1311 sg_free_table(vma
->pages
);
1316 i915_gem_object_unpin_pages(vma
->obj
);
1320 I915_SELFTEST_EXPORT
void i915_vma_put_pages(struct i915_vma
*vma
)
1322 if (atomic_add_unless(&vma
->pages_count
, -1, 1))
1325 __vma_put_pages(vma
, 1);
1328 static void vma_unbind_pages(struct i915_vma
*vma
)
1332 lockdep_assert_held(&vma
->vm
->mutex
);
1334 /* The upper portion of pages_count is the number of bindings */
1335 count
= atomic_read(&vma
->pages_count
);
1336 count
>>= I915_VMA_PAGES_BIAS
;
1339 __vma_put_pages(vma
, count
| count
<< I915_VMA_PAGES_BIAS
);
1342 int i915_vma_pin_ww(struct i915_vma
*vma
, struct i915_gem_ww_ctx
*ww
,
1343 u64 size
, u64 alignment
, u64 flags
)
1345 struct i915_vma_work
*work
= NULL
;
1346 struct dma_fence
*moving
= NULL
;
1347 struct i915_vma_resource
*vma_res
= NULL
;
1348 intel_wakeref_t wakeref
= 0;
1352 assert_vma_held(vma
);
1355 BUILD_BUG_ON(PIN_GLOBAL
!= I915_VMA_GLOBAL_BIND
);
1356 BUILD_BUG_ON(PIN_USER
!= I915_VMA_LOCAL_BIND
);
1358 GEM_BUG_ON(!(flags
& (PIN_USER
| PIN_GLOBAL
)));
1360 /* First try and grab the pin without rebinding the vma */
1361 if (try_qad_pin(vma
, flags
))
1364 err
= i915_vma_get_pages(vma
);
1368 if (flags
& PIN_GLOBAL
)
1369 wakeref
= intel_runtime_pm_get(&vma
->vm
->i915
->runtime_pm
);
1371 if (flags
& vma
->vm
->bind_async_flags
) {
1373 err
= i915_vm_lock_objects(vma
->vm
, ww
);
1377 work
= i915_vma_work();
1385 err
= i915_gem_object_get_moving_fence(vma
->obj
, &moving
);
1389 dma_fence_work_chain(&work
->base
, moving
);
1391 /* Allocate enough page directories to used PTE */
1392 if (vma
->vm
->allocate_va_range
) {
1393 err
= i915_vm_alloc_pt_stash(vma
->vm
,
1399 err
= i915_vm_map_pt_stash(vma
->vm
, &work
->stash
);
1405 vma_res
= i915_vma_resource_alloc();
1406 if (IS_ERR(vma_res
)) {
1407 err
= PTR_ERR(vma_res
);
1412 * Differentiate between user/kernel vma inside the aliasing-ppgtt.
1414 * We conflate the Global GTT with the user's vma when using the
1415 * aliasing-ppgtt, but it is still vitally important to try and
1416 * keep the use cases distinct. For example, userptr objects are
1417 * not allowed inside the Global GTT as that will cause lock
1418 * inversions when we have to evict them the mmu_notifier callbacks -
1419 * but they are allowed to be part of the user ppGTT which can never
1420 * be mapped. As such we try to give the distinct users of the same
1421 * mutex, distinct lockclasses [equivalent to how we keep i915_ggtt
1422 * and i915_ppgtt separate].
1424 * NB this may cause us to mask real lock inversions -- while the
1425 * code is safe today, lockdep may not be able to spot future
1428 err
= mutex_lock_interruptible_nested(&vma
->vm
->mutex
,
1429 !(flags
& PIN_GLOBAL
));
1433 /* No more allocations allowed now we hold vm->mutex */
1435 if (unlikely(i915_vma_is_closed(vma
))) {
1440 bound
= atomic_read(&vma
->flags
);
1441 if (unlikely(bound
& I915_VMA_ERROR
)) {
1446 if (unlikely(!((bound
+ 1) & I915_VMA_PIN_MASK
))) {
1447 err
= -EAGAIN
; /* pins are meant to be fairly temporary */
1451 if (unlikely(!(flags
& ~bound
& I915_VMA_BIND_MASK
))) {
1452 if (!(flags
& PIN_VALIDATE
))
1453 __i915_vma_pin(vma
);
1457 err
= i915_active_acquire(&vma
->active
);
1461 if (!(bound
& I915_VMA_BIND_MASK
)) {
1462 err
= i915_vma_insert(vma
, ww
, size
, alignment
, flags
);
1466 if (i915_is_ggtt(vma
->vm
))
1467 __i915_vma_set_map_and_fenceable(vma
);
1470 GEM_BUG_ON(!vma
->pages
);
1471 err
= i915_vma_bind(vma
,
1472 vma
->obj
->cache_level
,
1473 flags
, work
, vma_res
);
1478 /* There should only be at most 2 active bindings (user, global) */
1479 GEM_BUG_ON(bound
+ I915_VMA_PAGES_ACTIVE
< bound
);
1480 atomic_add(I915_VMA_PAGES_ACTIVE
, &vma
->pages_count
);
1481 list_move_tail(&vma
->vm_link
, &vma
->vm
->bound_list
);
1483 if (!(flags
& PIN_VALIDATE
)) {
1484 __i915_vma_pin(vma
);
1485 GEM_BUG_ON(!i915_vma_is_pinned(vma
));
1487 GEM_BUG_ON(!i915_vma_is_bound(vma
, flags
));
1488 GEM_BUG_ON(i915_vma_misplaced(vma
, size
, alignment
, flags
));
1491 if (!i915_vma_is_bound(vma
, I915_VMA_BIND_MASK
)) {
1492 i915_vma_detach(vma
);
1493 drm_mm_remove_node(&vma
->node
);
1496 i915_active_release(&vma
->active
);
1498 mutex_unlock(&vma
->vm
->mutex
);
1500 i915_vma_resource_free(vma_res
);
1503 dma_fence_work_commit_imm(&work
->base
);
1506 intel_runtime_pm_put(&vma
->vm
->i915
->runtime_pm
, wakeref
);
1509 dma_fence_put(moving
);
1511 i915_vma_put_pages(vma
);
1515 static void flush_idle_contexts(struct intel_gt
*gt
)
1517 struct intel_engine_cs
*engine
;
1518 enum intel_engine_id id
;
1520 for_each_engine(engine
, gt
, id
)
1521 intel_engine_flush_barriers(engine
);
1523 intel_gt_wait_for_idle(gt
, MAX_SCHEDULE_TIMEOUT
);
1526 static int __i915_ggtt_pin(struct i915_vma
*vma
, struct i915_gem_ww_ctx
*ww
,
1527 u32 align
, unsigned int flags
)
1529 struct i915_address_space
*vm
= vma
->vm
;
1533 err
= i915_vma_pin_ww(vma
, ww
, 0, align
, flags
| PIN_GLOBAL
);
1535 if (err
!= -ENOSPC
) {
1537 err
= i915_vma_wait_for_bind(vma
);
1539 i915_vma_unpin(vma
);
1544 /* Unlike i915_vma_pin, we don't take no for an answer! */
1545 flush_idle_contexts(vm
->gt
);
1546 if (mutex_lock_interruptible(&vm
->mutex
) == 0) {
1548 * We pass NULL ww here, as we don't want to unbind
1549 * locked objects when called from execbuf when pinning
1550 * is removed. This would probably regress badly.
1552 i915_gem_evict_vm(vm
, NULL
);
1553 mutex_unlock(&vm
->mutex
);
1558 int i915_ggtt_pin(struct i915_vma
*vma
, struct i915_gem_ww_ctx
*ww
,
1559 u32 align
, unsigned int flags
)
1561 struct i915_gem_ww_ctx _ww
;
1564 GEM_BUG_ON(!i915_vma_is_ggtt(vma
));
1567 return __i915_ggtt_pin(vma
, ww
, align
, flags
);
1569 lockdep_assert_not_held(&vma
->obj
->base
.resv
->lock
.base
);
1571 for_i915_gem_ww(&_ww
, err
, true) {
1572 err
= i915_gem_object_lock(vma
->obj
, &_ww
);
1574 err
= __i915_ggtt_pin(vma
, &_ww
, align
, flags
);
1580 static void __vma_close(struct i915_vma
*vma
, struct intel_gt
*gt
)
1583 * We defer actually closing, unbinding and destroying the VMA until
1584 * the next idle point, or if the object is freed in the meantime. By
1585 * postponing the unbind, we allow for it to be resurrected by the
1586 * client, avoiding the work required to rebind the VMA. This is
1587 * advantageous for DRI, where the client/server pass objects
1588 * between themselves, temporarily opening a local VMA to the
1589 * object, and then closing it again. The same object is then reused
1590 * on the next frame (or two, depending on the depth of the swap queue)
1591 * causing us to rebind the VMA once more. This ends up being a lot
1592 * of wasted work for the steady state.
1594 GEM_BUG_ON(i915_vma_is_closed(vma
));
1595 list_add(&vma
->closed_link
, >
->closed_vma
);
1598 void i915_vma_close(struct i915_vma
*vma
)
1600 struct intel_gt
*gt
= vma
->vm
->gt
;
1601 unsigned long flags
;
1603 if (i915_vma_is_ggtt(vma
))
1606 GEM_BUG_ON(!atomic_read(&vma
->open_count
));
1607 if (atomic_dec_and_lock_irqsave(&vma
->open_count
,
1610 __vma_close(vma
, gt
);
1611 spin_unlock_irqrestore(>
->closed_lock
, flags
);
1615 static void __i915_vma_remove_closed(struct i915_vma
*vma
)
1617 list_del_init(&vma
->closed_link
);
1620 void i915_vma_reopen(struct i915_vma
*vma
)
1622 struct intel_gt
*gt
= vma
->vm
->gt
;
1624 spin_lock_irq(>
->closed_lock
);
1625 if (i915_vma_is_closed(vma
))
1626 __i915_vma_remove_closed(vma
);
1627 spin_unlock_irq(>
->closed_lock
);
1630 static void force_unbind(struct i915_vma
*vma
)
1632 if (!drm_mm_node_allocated(&vma
->node
))
1635 atomic_and(~I915_VMA_PIN_MASK
, &vma
->flags
);
1636 WARN_ON(__i915_vma_unbind(vma
));
1637 GEM_BUG_ON(drm_mm_node_allocated(&vma
->node
));
1640 static void release_references(struct i915_vma
*vma
, bool vm_ddestroy
)
1642 struct drm_i915_gem_object
*obj
= vma
->obj
;
1643 struct intel_gt
*gt
= vma
->vm
->gt
;
1645 GEM_BUG_ON(i915_vma_is_active(vma
));
1647 spin_lock(&obj
->vma
.lock
);
1648 list_del(&vma
->obj_link
);
1649 if (!RB_EMPTY_NODE(&vma
->obj_node
))
1650 rb_erase(&vma
->obj_node
, &obj
->vma
.tree
);
1652 spin_unlock(&obj
->vma
.lock
);
1654 spin_lock_irq(>
->closed_lock
);
1655 __i915_vma_remove_closed(vma
);
1656 spin_unlock_irq(>
->closed_lock
);
1659 i915_vm_resv_put(vma
->vm
);
1661 i915_active_fini(&vma
->active
);
1662 GEM_WARN_ON(vma
->resource
);
1667 * i915_vma_destroy_locked - Remove all weak reference to the vma and put
1668 * the initial reference.
1670 * This function should be called when it's decided the vma isn't needed
1671 * anymore. The caller must assure that it doesn't race with another lookup
1672 * plus destroy, typically by taking an appropriate reference.
1674 * Current callsites are
1675 * - __i915_gem_object_pages_fini()
1676 * - __i915_vm_close() - Blocks the above function by taking a reference on
1678 * - __i915_vma_parked() - Blocks the above functions by taking a reference
1679 * on the vm and a reference on the object. Also takes the object lock so
1680 * destruction from __i915_vma_parked() can be blocked by holding the
1681 * object lock. Since the object lock is only allowed from within i915 with
1682 * an object refcount, holding the object lock also implicitly blocks the
1683 * vma freeing from __i915_gem_object_pages_fini().
1685 * Because of locks taken during destruction, a vma is also guaranteed to
1686 * stay alive while the following locks are held if it was looked up while
1687 * holding one of the locks:
1692 void i915_vma_destroy_locked(struct i915_vma
*vma
)
1694 lockdep_assert_held(&vma
->vm
->mutex
);
1697 list_del_init(&vma
->vm_link
);
1698 release_references(vma
, false);
1701 void i915_vma_destroy(struct i915_vma
*vma
)
1705 mutex_lock(&vma
->vm
->mutex
);
1707 list_del_init(&vma
->vm_link
);
1708 vm_ddestroy
= vma
->vm_ddestroy
;
1709 vma
->vm_ddestroy
= false;
1710 mutex_unlock(&vma
->vm
->mutex
);
1711 release_references(vma
, vm_ddestroy
);
1714 void i915_vma_parked(struct intel_gt
*gt
)
1716 struct i915_vma
*vma
, *next
;
1719 spin_lock_irq(>
->closed_lock
);
1720 list_for_each_entry_safe(vma
, next
, >
->closed_vma
, closed_link
) {
1721 struct drm_i915_gem_object
*obj
= vma
->obj
;
1722 struct i915_address_space
*vm
= vma
->vm
;
1724 /* XXX All to avoid keeping a reference on i915_vma itself */
1726 if (!kref_get_unless_zero(&obj
->base
.refcount
))
1729 if (!i915_vm_tryget(vm
)) {
1730 i915_gem_object_put(obj
);
1734 list_move(&vma
->closed_link
, &closed
);
1736 spin_unlock_irq(>
->closed_lock
);
1738 /* As the GT is held idle, no vma can be reopened as we destroy them */
1739 list_for_each_entry_safe(vma
, next
, &closed
, closed_link
) {
1740 struct drm_i915_gem_object
*obj
= vma
->obj
;
1741 struct i915_address_space
*vm
= vma
->vm
;
1743 if (i915_gem_object_trylock(obj
, NULL
)) {
1744 INIT_LIST_HEAD(&vma
->closed_link
);
1745 i915_vma_destroy(vma
);
1746 i915_gem_object_unlock(obj
);
1749 spin_lock_irq(>
->closed_lock
);
1750 list_add(&vma
->closed_link
, >
->closed_vma
);
1751 spin_unlock_irq(>
->closed_lock
);
1754 i915_gem_object_put(obj
);
1759 static void __i915_vma_iounmap(struct i915_vma
*vma
)
1761 GEM_BUG_ON(i915_vma_is_pinned(vma
));
1763 if (vma
->iomap
== NULL
)
1766 io_mapping_unmap(vma
->iomap
);
1770 void i915_vma_revoke_mmap(struct i915_vma
*vma
)
1772 struct drm_vma_offset_node
*node
;
1775 if (!i915_vma_has_userfault(vma
))
1778 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma
));
1779 GEM_BUG_ON(!vma
->obj
->userfault_count
);
1781 node
= &vma
->mmo
->vma_node
;
1782 vma_offset
= vma
->ggtt_view
.partial
.offset
<< PAGE_SHIFT
;
1783 unmap_mapping_range(vma
->vm
->i915
->drm
.anon_inode
->i_mapping
,
1784 drm_vma_node_offset_addr(node
) + vma_offset
,
1788 i915_vma_unset_userfault(vma
);
1789 if (!--vma
->obj
->userfault_count
)
1790 list_del(&vma
->obj
->userfault_link
);
1794 __i915_request_await_bind(struct i915_request
*rq
, struct i915_vma
*vma
)
1796 return __i915_request_await_exclusive(rq
, &vma
->active
);
1799 static int __i915_vma_move_to_active(struct i915_vma
*vma
, struct i915_request
*rq
)
1803 /* Wait for the vma to be bound before we start! */
1804 err
= __i915_request_await_bind(rq
, vma
);
1808 return i915_active_add_request(&vma
->active
, rq
);
1811 int _i915_vma_move_to_active(struct i915_vma
*vma
,
1812 struct i915_request
*rq
,
1813 struct dma_fence
*fence
,
1816 struct drm_i915_gem_object
*obj
= vma
->obj
;
1819 assert_object_held(obj
);
1821 GEM_BUG_ON(!vma
->pages
);
1823 err
= __i915_vma_move_to_active(vma
, rq
);
1828 * Reserve fences slot early to prevent an allocation after preparing
1829 * the workload and associating fences with dma_resv.
1831 if (fence
&& !(flags
& __EXEC_OBJECT_NO_RESERVE
)) {
1832 struct dma_fence
*curr
;
1835 dma_fence_array_for_each(curr
, idx
, fence
)
1837 err
= dma_resv_reserve_fences(vma
->obj
->base
.resv
, idx
);
1842 if (flags
& EXEC_OBJECT_WRITE
) {
1843 struct intel_frontbuffer
*front
;
1845 front
= __intel_frontbuffer_get(obj
);
1846 if (unlikely(front
)) {
1847 if (intel_frontbuffer_invalidate(front
, ORIGIN_CS
))
1848 i915_active_add_request(&front
->write
, rq
);
1849 intel_frontbuffer_put(front
);
1854 struct dma_fence
*curr
;
1855 enum dma_resv_usage usage
;
1858 obj
->read_domains
= 0;
1859 if (flags
& EXEC_OBJECT_WRITE
) {
1860 usage
= DMA_RESV_USAGE_WRITE
;
1861 obj
->write_domain
= I915_GEM_DOMAIN_RENDER
;
1863 usage
= DMA_RESV_USAGE_READ
;
1866 dma_fence_array_for_each(curr
, idx
, fence
)
1867 dma_resv_add_fence(vma
->obj
->base
.resv
, curr
, usage
);
1870 if (flags
& EXEC_OBJECT_NEEDS_FENCE
&& vma
->fence
)
1871 i915_active_add_request(&vma
->fence
->active
, rq
);
1873 obj
->read_domains
|= I915_GEM_GPU_DOMAINS
;
1874 obj
->mm
.dirty
= true;
1876 GEM_BUG_ON(!i915_vma_is_active(vma
));
1880 struct dma_fence
*__i915_vma_evict(struct i915_vma
*vma
, bool async
)
1882 struct i915_vma_resource
*vma_res
= vma
->resource
;
1883 struct dma_fence
*unbind_fence
;
1885 GEM_BUG_ON(i915_vma_is_pinned(vma
));
1886 assert_vma_held_evict(vma
);
1888 if (i915_vma_is_map_and_fenceable(vma
)) {
1889 /* Force a pagefault for domain tracking on next user access */
1890 i915_vma_revoke_mmap(vma
);
1893 * Check that we have flushed all writes through the GGTT
1894 * before the unbind, other due to non-strict nature of those
1895 * indirect writes they may end up referencing the GGTT PTE
1898 * Note that we may be concurrently poking at the GGTT_WRITE
1899 * bit from set-domain, as we mark all GGTT vma associated
1900 * with an object. We know this is for another vma, as we
1901 * are currently unbinding this one -- so if this vma will be
1902 * reused, it will be refaulted and have its dirty bit set
1903 * before the next write.
1905 i915_vma_flush_writes(vma
);
1907 /* release the fence reg _after_ flushing */
1908 i915_vma_revoke_fence(vma
);
1910 __i915_vma_iounmap(vma
);
1911 clear_bit(I915_VMA_CAN_FENCE_BIT
, __i915_vma_flags(vma
));
1913 GEM_BUG_ON(vma
->fence
);
1914 GEM_BUG_ON(i915_vma_has_userfault(vma
));
1916 /* Object backend must be async capable. */
1917 GEM_WARN_ON(async
&& !vma
->resource
->bi
.pages_rsgt
);
1919 /* If vm is not open, unbind is a nop. */
1920 vma_res
->needs_wakeref
= i915_vma_is_bound(vma
, I915_VMA_GLOBAL_BIND
) &&
1921 kref_read(&vma
->vm
->ref
);
1922 vma_res
->skip_pte_rewrite
= !kref_read(&vma
->vm
->ref
) ||
1923 vma
->vm
->skip_pte_rewrite
;
1924 trace_i915_vma_unbind(vma
);
1926 unbind_fence
= i915_vma_resource_unbind(vma_res
);
1927 vma
->resource
= NULL
;
1929 atomic_and(~(I915_VMA_BIND_MASK
| I915_VMA_ERROR
| I915_VMA_GGTT_WRITE
),
1932 i915_vma_detach(vma
);
1934 if (!async
&& unbind_fence
) {
1935 dma_fence_wait(unbind_fence
, false);
1936 dma_fence_put(unbind_fence
);
1937 unbind_fence
= NULL
;
1941 * Binding itself may not have completed until the unbind fence signals,
1942 * so don't drop the pages until that happens, unless the resource is
1946 vma_unbind_pages(vma
);
1947 return unbind_fence
;
1950 int __i915_vma_unbind(struct i915_vma
*vma
)
1954 lockdep_assert_held(&vma
->vm
->mutex
);
1955 assert_vma_held_evict(vma
);
1957 if (!drm_mm_node_allocated(&vma
->node
))
1960 if (i915_vma_is_pinned(vma
)) {
1961 vma_print_allocator(vma
, "is pinned");
1966 * After confirming that no one else is pinning this vma, wait for
1967 * any laggards who may have crept in during the wait (through
1968 * a residual pin skipping the vm->mutex) to complete.
1970 ret
= i915_vma_sync(vma
);
1974 GEM_BUG_ON(i915_vma_is_active(vma
));
1975 __i915_vma_evict(vma
, false);
1977 drm_mm_remove_node(&vma
->node
); /* pairs with i915_vma_release() */
1981 static struct dma_fence
*__i915_vma_unbind_async(struct i915_vma
*vma
)
1983 struct dma_fence
*fence
;
1985 lockdep_assert_held(&vma
->vm
->mutex
);
1987 if (!drm_mm_node_allocated(&vma
->node
))
1990 if (i915_vma_is_pinned(vma
) ||
1991 &vma
->obj
->mm
.rsgt
->table
!= vma
->resource
->bi
.pages
)
1992 return ERR_PTR(-EAGAIN
);
1995 * We probably need to replace this with awaiting the fences of the
1996 * object's dma_resv when the vma active goes away. When doing that
1997 * we need to be careful to not add the vma_resource unbind fence
1998 * immediately to the object's dma_resv, because then unbinding
1999 * the next vma from the object, in case there are many, will
2000 * actually await the unbinding of the previous vmas, which is
2003 if (i915_sw_fence_await_active(&vma
->resource
->chain
, &vma
->active
,
2004 I915_ACTIVE_AWAIT_EXCL
|
2005 I915_ACTIVE_AWAIT_ACTIVE
) < 0) {
2006 return ERR_PTR(-EBUSY
);
2009 fence
= __i915_vma_evict(vma
, true);
2011 drm_mm_remove_node(&vma
->node
); /* pairs with i915_vma_release() */
2016 int i915_vma_unbind(struct i915_vma
*vma
)
2018 struct i915_address_space
*vm
= vma
->vm
;
2019 intel_wakeref_t wakeref
= 0;
2022 assert_object_held_shared(vma
->obj
);
2024 /* Optimistic wait before taking the mutex */
2025 err
= i915_vma_sync(vma
);
2029 if (!drm_mm_node_allocated(&vma
->node
))
2032 if (i915_vma_is_pinned(vma
)) {
2033 vma_print_allocator(vma
, "is pinned");
2037 if (i915_vma_is_bound(vma
, I915_VMA_GLOBAL_BIND
))
2038 /* XXX not always required: nop_clear_range */
2039 wakeref
= intel_runtime_pm_get(&vm
->i915
->runtime_pm
);
2041 err
= mutex_lock_interruptible_nested(&vma
->vm
->mutex
, !wakeref
);
2045 err
= __i915_vma_unbind(vma
);
2046 mutex_unlock(&vm
->mutex
);
2050 intel_runtime_pm_put(&vm
->i915
->runtime_pm
, wakeref
);
2054 int i915_vma_unbind_async(struct i915_vma
*vma
, bool trylock_vm
)
2056 struct drm_i915_gem_object
*obj
= vma
->obj
;
2057 struct i915_address_space
*vm
= vma
->vm
;
2058 intel_wakeref_t wakeref
= 0;
2059 struct dma_fence
*fence
;
2063 * We need the dma-resv lock since we add the
2064 * unbind fence to the dma-resv object.
2066 assert_object_held(obj
);
2068 if (!drm_mm_node_allocated(&vma
->node
))
2071 if (i915_vma_is_pinned(vma
)) {
2072 vma_print_allocator(vma
, "is pinned");
2079 err
= dma_resv_reserve_fences(obj
->base
.resv
, 1);
2084 * It would be great if we could grab this wakeref from the
2085 * async unbind work if needed, but we can't because it uses
2086 * kmalloc and it's in the dma-fence signalling critical path.
2088 if (i915_vma_is_bound(vma
, I915_VMA_GLOBAL_BIND
))
2089 wakeref
= intel_runtime_pm_get(&vm
->i915
->runtime_pm
);
2091 if (trylock_vm
&& !mutex_trylock(&vm
->mutex
)) {
2094 } else if (!trylock_vm
) {
2095 err
= mutex_lock_interruptible_nested(&vm
->mutex
, !wakeref
);
2100 fence
= __i915_vma_unbind_async(vma
);
2101 mutex_unlock(&vm
->mutex
);
2102 if (IS_ERR_OR_NULL(fence
)) {
2103 err
= PTR_ERR_OR_ZERO(fence
);
2107 dma_resv_add_fence(obj
->base
.resv
, fence
, DMA_RESV_USAGE_READ
);
2108 dma_fence_put(fence
);
2112 intel_runtime_pm_put(&vm
->i915
->runtime_pm
, wakeref
);
2116 int i915_vma_unbind_unlocked(struct i915_vma
*vma
)
2120 i915_gem_object_lock(vma
->obj
, NULL
);
2121 err
= i915_vma_unbind(vma
);
2122 i915_gem_object_unlock(vma
->obj
);
2127 struct i915_vma
*i915_vma_make_unshrinkable(struct i915_vma
*vma
)
2129 i915_gem_object_make_unshrinkable(vma
->obj
);
2133 void i915_vma_make_shrinkable(struct i915_vma
*vma
)
2135 i915_gem_object_make_shrinkable(vma
->obj
);
2138 void i915_vma_make_purgeable(struct i915_vma
*vma
)
2140 i915_gem_object_make_purgeable(vma
->obj
);
2143 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2144 #include "selftests/i915_vma.c"
2147 void i915_vma_module_exit(void)
2149 kmem_cache_destroy(slab_vmas
);
2152 int __init
i915_vma_module_init(void)
2154 slab_vmas
= KMEM_CACHE(i915_vma
, SLAB_HWCACHE_ALIGN
);