]>
Commit | Line | Data |
---|---|---|
f033428d CW |
1 | /* |
2 | * SPDX-License-Identifier: MIT | |
3 | * | |
4 | * Copyright © 2014-2016 Intel Corporation | |
5 | */ | |
6 | ||
7 | #include "i915_drv.h" | |
8 | #include "i915_gem_object.h" | |
37d63f8f | 9 | #include "i915_scatterlist.h" |
f033428d CW |
10 | |
11 | void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, | |
12 | struct sg_table *pages, | |
13 | unsigned int sg_page_sizes) | |
14 | { | |
15 | struct drm_i915_private *i915 = to_i915(obj->base.dev); | |
16 | unsigned long supported = INTEL_INFO(i915)->page_sizes; | |
17 | int i; | |
18 | ||
19 | lockdep_assert_held(&obj->mm.lock); | |
20 | ||
21 | /* Make the pages coherent with the GPU (flushing any swapin). */ | |
22 | if (obj->cache_dirty) { | |
23 | obj->write_domain = 0; | |
24 | if (i915_gem_object_has_struct_page(obj)) | |
25 | drm_clflush_sg(pages); | |
26 | obj->cache_dirty = false; | |
27 | } | |
28 | ||
29 | obj->mm.get_page.sg_pos = pages->sgl; | |
30 | obj->mm.get_page.sg_idx = 0; | |
31 | ||
32 | obj->mm.pages = pages; | |
33 | ||
34 | if (i915_gem_object_is_tiled(obj) && | |
35 | i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { | |
36 | GEM_BUG_ON(obj->mm.quirked); | |
37 | __i915_gem_object_pin_pages(obj); | |
38 | obj->mm.quirked = true; | |
39 | } | |
40 | ||
41 | GEM_BUG_ON(!sg_page_sizes); | |
42 | obj->mm.page_sizes.phys = sg_page_sizes; | |
43 | ||
44 | /* | |
45 | * Calculate the supported page-sizes which fit into the given | |
46 | * sg_page_sizes. This will give us the page-sizes which we may be able | |
47 | * to use opportunistically when later inserting into the GTT. For | |
48 | * example if phys=2G, then in theory we should be able to use 1G, 2M, | |
49 | * 64K or 4K pages, although in practice this will depend on a number of | |
50 | * other factors. | |
51 | */ | |
52 | obj->mm.page_sizes.sg = 0; | |
53 | for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { | |
54 | if (obj->mm.page_sizes.phys & ~0u << i) | |
55 | obj->mm.page_sizes.sg |= BIT(i); | |
56 | } | |
57 | GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg)); | |
58 | ||
d82b4b26 | 59 | if (i915_gem_object_is_shrinkable(obj)) { |
ecab9be1 | 60 | struct list_head *list; |
a8cff4c8 CW |
61 | unsigned long flags; |
62 | ||
63 | spin_lock_irqsave(&i915->mm.obj_lock, flags); | |
64 | ||
d82b4b26 CW |
65 | i915->mm.shrink_count++; |
66 | i915->mm.shrink_memory += obj->base.size; | |
ecab9be1 CW |
67 | |
68 | if (obj->mm.madv != I915_MADV_WILLNEED) | |
69 | list = &i915->mm.purge_list; | |
70 | else | |
71 | list = &i915->mm.shrink_list; | |
72 | list_add_tail(&obj->mm.link, list); | |
a8cff4c8 CW |
73 | |
74 | spin_unlock_irqrestore(&i915->mm.obj_lock, flags); | |
d82b4b26 | 75 | } |
f033428d CW |
76 | } |
77 | ||
78 | int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj) | |
79 | { | |
80 | int err; | |
81 | ||
82 | if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) { | |
83 | DRM_DEBUG("Attempting to obtain a purgeable object\n"); | |
84 | return -EFAULT; | |
85 | } | |
86 | ||
87 | err = obj->ops->get_pages(obj); | |
88 | GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj)); | |
89 | ||
90 | return err; | |
91 | } | |
92 | ||
93 | /* Ensure that the associated pages are gathered from the backing storage | |
94 | * and pinned into our object. i915_gem_object_pin_pages() may be called | |
95 | * multiple times before they are released by a single call to | |
96 | * i915_gem_object_unpin_pages() - once the pages are no longer referenced | |
97 | * either as a result of memory pressure (reaping pages under the shrinker) | |
98 | * or as the object is itself released. | |
99 | */ | |
100 | int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj) | |
101 | { | |
102 | int err; | |
103 | ||
104 | err = mutex_lock_interruptible(&obj->mm.lock); | |
105 | if (err) | |
106 | return err; | |
107 | ||
108 | if (unlikely(!i915_gem_object_has_pages(obj))) { | |
109 | GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); | |
110 | ||
111 | err = ____i915_gem_object_get_pages(obj); | |
112 | if (err) | |
113 | goto unlock; | |
114 | ||
115 | smp_mb__before_atomic(); | |
116 | } | |
117 | atomic_inc(&obj->mm.pages_pin_count); | |
118 | ||
119 | unlock: | |
120 | mutex_unlock(&obj->mm.lock); | |
121 | return err; | |
122 | } | |
123 | ||
124 | /* Immediately discard the backing storage */ | |
125 | void i915_gem_object_truncate(struct drm_i915_gem_object *obj) | |
126 | { | |
127 | drm_gem_free_mmap_offset(&obj->base); | |
128 | if (obj->ops->truncate) | |
129 | obj->ops->truncate(obj); | |
130 | } | |
131 | ||
132 | /* Try to discard unwanted pages */ | |
133 | void i915_gem_object_writeback(struct drm_i915_gem_object *obj) | |
134 | { | |
135 | lockdep_assert_held(&obj->mm.lock); | |
136 | GEM_BUG_ON(i915_gem_object_has_pages(obj)); | |
137 | ||
138 | if (obj->ops->writeback) | |
139 | obj->ops->writeback(obj); | |
140 | } | |
141 | ||
142 | static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj) | |
143 | { | |
144 | struct radix_tree_iter iter; | |
145 | void __rcu **slot; | |
146 | ||
147 | rcu_read_lock(); | |
148 | radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0) | |
149 | radix_tree_delete(&obj->mm.get_page.radix, iter.index); | |
150 | rcu_read_unlock(); | |
151 | } | |
152 | ||
153 | struct sg_table * | |
154 | __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) | |
155 | { | |
156 | struct drm_i915_private *i915 = to_i915(obj->base.dev); | |
157 | struct sg_table *pages; | |
158 | ||
159 | pages = fetch_and_zero(&obj->mm.pages); | |
160 | if (IS_ERR_OR_NULL(pages)) | |
161 | return pages; | |
162 | ||
d82b4b26 | 163 | if (i915_gem_object_is_shrinkable(obj)) { |
a8cff4c8 CW |
164 | unsigned long flags; |
165 | ||
166 | spin_lock_irqsave(&i915->mm.obj_lock, flags); | |
167 | ||
d82b4b26 CW |
168 | list_del(&obj->mm.link); |
169 | i915->mm.shrink_count--; | |
170 | i915->mm.shrink_memory -= obj->base.size; | |
a8cff4c8 CW |
171 | |
172 | spin_unlock_irqrestore(&i915->mm.obj_lock, flags); | |
d82b4b26 | 173 | } |
f033428d CW |
174 | |
175 | if (obj->mm.mapping) { | |
176 | void *ptr; | |
177 | ||
178 | ptr = page_mask_bits(obj->mm.mapping); | |
179 | if (is_vmalloc_addr(ptr)) | |
180 | vunmap(ptr); | |
181 | else | |
182 | kunmap(kmap_to_page(ptr)); | |
183 | ||
184 | obj->mm.mapping = NULL; | |
185 | } | |
186 | ||
187 | __i915_gem_object_reset_page_iter(obj); | |
188 | obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0; | |
189 | ||
190 | return pages; | |
191 | } | |
192 | ||
193 | int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, | |
194 | enum i915_mm_subclass subclass) | |
195 | { | |
196 | struct sg_table *pages; | |
197 | int err; | |
198 | ||
199 | if (i915_gem_object_has_pinned_pages(obj)) | |
200 | return -EBUSY; | |
201 | ||
ecab9be1 | 202 | GEM_BUG_ON(atomic_read(&obj->bind_count)); |
f033428d CW |
203 | |
204 | /* May be called by shrinker from within get_pages() (on another bo) */ | |
205 | mutex_lock_nested(&obj->mm.lock, subclass); | |
206 | if (unlikely(atomic_read(&obj->mm.pages_pin_count))) { | |
207 | err = -EBUSY; | |
208 | goto unlock; | |
209 | } | |
210 | ||
211 | /* | |
212 | * ->put_pages might need to allocate memory for the bit17 swizzle | |
213 | * array, hence protect them from being reaped by removing them from gtt | |
214 | * lists early. | |
215 | */ | |
216 | pages = __i915_gem_object_unset_pages(obj); | |
217 | ||
218 | /* | |
219 | * XXX Temporary hijinx to avoid updating all backends to handle | |
220 | * NULL pages. In the future, when we have more asynchronous | |
221 | * get_pages backends we should be better able to handle the | |
222 | * cancellation of the async task in a more uniform manner. | |
223 | */ | |
224 | if (!pages && !i915_gem_object_needs_async_cancel(obj)) | |
225 | pages = ERR_PTR(-EINVAL); | |
226 | ||
227 | if (!IS_ERR(pages)) | |
228 | obj->ops->put_pages(obj, pages); | |
229 | ||
230 | err = 0; | |
231 | unlock: | |
232 | mutex_unlock(&obj->mm.lock); | |
233 | ||
234 | return err; | |
235 | } | |
236 | ||
237 | /* The 'mapping' part of i915_gem_object_pin_map() below */ | |
238 | static void *i915_gem_object_map(const struct drm_i915_gem_object *obj, | |
239 | enum i915_map_type type) | |
240 | { | |
241 | unsigned long n_pages = obj->base.size >> PAGE_SHIFT; | |
242 | struct sg_table *sgt = obj->mm.pages; | |
243 | struct sgt_iter sgt_iter; | |
244 | struct page *page; | |
245 | struct page *stack_pages[32]; | |
246 | struct page **pages = stack_pages; | |
247 | unsigned long i = 0; | |
248 | pgprot_t pgprot; | |
249 | void *addr; | |
250 | ||
251 | /* A single page can always be kmapped */ | |
252 | if (n_pages == 1 && type == I915_MAP_WB) | |
253 | return kmap(sg_page(sgt->sgl)); | |
254 | ||
255 | if (n_pages > ARRAY_SIZE(stack_pages)) { | |
256 | /* Too big for stack -- allocate temporary array instead */ | |
257 | pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL); | |
258 | if (!pages) | |
259 | return NULL; | |
260 | } | |
261 | ||
262 | for_each_sgt_page(page, sgt_iter, sgt) | |
263 | pages[i++] = page; | |
264 | ||
265 | /* Check that we have the expected number of pages */ | |
266 | GEM_BUG_ON(i != n_pages); | |
267 | ||
268 | switch (type) { | |
269 | default: | |
270 | MISSING_CASE(type); | |
2defb94e | 271 | /* fallthrough - to use PAGE_KERNEL anyway */ |
f033428d CW |
272 | case I915_MAP_WB: |
273 | pgprot = PAGE_KERNEL; | |
274 | break; | |
275 | case I915_MAP_WC: | |
276 | pgprot = pgprot_writecombine(PAGE_KERNEL_IO); | |
277 | break; | |
278 | } | |
279 | addr = vmap(pages, n_pages, 0, pgprot); | |
280 | ||
281 | if (pages != stack_pages) | |
282 | kvfree(pages); | |
283 | ||
284 | return addr; | |
285 | } | |
286 | ||
287 | /* get, pin, and map the pages of the object into kernel space */ | |
288 | void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, | |
289 | enum i915_map_type type) | |
290 | { | |
291 | enum i915_map_type has_type; | |
292 | bool pinned; | |
293 | void *ptr; | |
294 | int err; | |
295 | ||
296 | if (unlikely(!i915_gem_object_has_struct_page(obj))) | |
297 | return ERR_PTR(-ENXIO); | |
298 | ||
299 | err = mutex_lock_interruptible(&obj->mm.lock); | |
300 | if (err) | |
301 | return ERR_PTR(err); | |
302 | ||
303 | pinned = !(type & I915_MAP_OVERRIDE); | |
304 | type &= ~I915_MAP_OVERRIDE; | |
305 | ||
306 | if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) { | |
307 | if (unlikely(!i915_gem_object_has_pages(obj))) { | |
308 | GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); | |
309 | ||
310 | err = ____i915_gem_object_get_pages(obj); | |
311 | if (err) | |
312 | goto err_unlock; | |
313 | ||
314 | smp_mb__before_atomic(); | |
315 | } | |
316 | atomic_inc(&obj->mm.pages_pin_count); | |
317 | pinned = false; | |
318 | } | |
319 | GEM_BUG_ON(!i915_gem_object_has_pages(obj)); | |
320 | ||
321 | ptr = page_unpack_bits(obj->mm.mapping, &has_type); | |
322 | if (ptr && has_type != type) { | |
323 | if (pinned) { | |
324 | err = -EBUSY; | |
325 | goto err_unpin; | |
326 | } | |
327 | ||
328 | if (is_vmalloc_addr(ptr)) | |
329 | vunmap(ptr); | |
330 | else | |
331 | kunmap(kmap_to_page(ptr)); | |
332 | ||
333 | ptr = obj->mm.mapping = NULL; | |
334 | } | |
335 | ||
336 | if (!ptr) { | |
337 | ptr = i915_gem_object_map(obj, type); | |
338 | if (!ptr) { | |
339 | err = -ENOMEM; | |
340 | goto err_unpin; | |
341 | } | |
342 | ||
343 | obj->mm.mapping = page_pack_bits(ptr, type); | |
344 | } | |
345 | ||
346 | out_unlock: | |
347 | mutex_unlock(&obj->mm.lock); | |
348 | return ptr; | |
349 | ||
350 | err_unpin: | |
351 | atomic_dec(&obj->mm.pages_pin_count); | |
352 | err_unlock: | |
353 | ptr = ERR_PTR(err); | |
354 | goto out_unlock; | |
355 | } | |
356 | ||
357 | void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, | |
358 | unsigned long offset, | |
359 | unsigned long size) | |
360 | { | |
361 | enum i915_map_type has_type; | |
362 | void *ptr; | |
363 | ||
364 | GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); | |
365 | GEM_BUG_ON(range_overflows_t(typeof(obj->base.size), | |
366 | offset, size, obj->base.size)); | |
367 | ||
368 | obj->mm.dirty = true; | |
369 | ||
370 | if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) | |
371 | return; | |
372 | ||
373 | ptr = page_unpack_bits(obj->mm.mapping, &has_type); | |
374 | if (has_type == I915_MAP_WC) | |
375 | return; | |
376 | ||
377 | drm_clflush_virt_range(ptr + offset, size); | |
378 | if (size == obj->base.size) { | |
379 | obj->write_domain &= ~I915_GEM_DOMAIN_CPU; | |
380 | obj->cache_dirty = false; | |
381 | } | |
382 | } | |
383 | ||
384 | struct scatterlist * | |
385 | i915_gem_object_get_sg(struct drm_i915_gem_object *obj, | |
386 | unsigned int n, | |
387 | unsigned int *offset) | |
388 | { | |
389 | struct i915_gem_object_page_iter *iter = &obj->mm.get_page; | |
390 | struct scatterlist *sg; | |
391 | unsigned int idx, count; | |
392 | ||
393 | might_sleep(); | |
394 | GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT); | |
395 | GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); | |
396 | ||
397 | /* As we iterate forward through the sg, we record each entry in a | |
398 | * radixtree for quick repeated (backwards) lookups. If we have seen | |
399 | * this index previously, we will have an entry for it. | |
400 | * | |
401 | * Initial lookup is O(N), but this is amortized to O(1) for | |
402 | * sequential page access (where each new request is consecutive | |
403 | * to the previous one). Repeated lookups are O(lg(obj->base.size)), | |
404 | * i.e. O(1) with a large constant! | |
405 | */ | |
406 | if (n < READ_ONCE(iter->sg_idx)) | |
407 | goto lookup; | |
408 | ||
409 | mutex_lock(&iter->lock); | |
410 | ||
411 | /* We prefer to reuse the last sg so that repeated lookup of this | |
412 | * (or the subsequent) sg are fast - comparing against the last | |
413 | * sg is faster than going through the radixtree. | |
414 | */ | |
415 | ||
416 | sg = iter->sg_pos; | |
417 | idx = iter->sg_idx; | |
418 | count = __sg_page_count(sg); | |
419 | ||
420 | while (idx + count <= n) { | |
421 | void *entry; | |
422 | unsigned long i; | |
423 | int ret; | |
424 | ||
425 | /* If we cannot allocate and insert this entry, or the | |
426 | * individual pages from this range, cancel updating the | |
427 | * sg_idx so that on this lookup we are forced to linearly | |
428 | * scan onwards, but on future lookups we will try the | |
429 | * insertion again (in which case we need to be careful of | |
430 | * the error return reporting that we have already inserted | |
431 | * this index). | |
432 | */ | |
433 | ret = radix_tree_insert(&iter->radix, idx, sg); | |
434 | if (ret && ret != -EEXIST) | |
435 | goto scan; | |
436 | ||
437 | entry = xa_mk_value(idx); | |
438 | for (i = 1; i < count; i++) { | |
439 | ret = radix_tree_insert(&iter->radix, idx + i, entry); | |
440 | if (ret && ret != -EEXIST) | |
441 | goto scan; | |
442 | } | |
443 | ||
444 | idx += count; | |
445 | sg = ____sg_next(sg); | |
446 | count = __sg_page_count(sg); | |
447 | } | |
448 | ||
449 | scan: | |
450 | iter->sg_pos = sg; | |
451 | iter->sg_idx = idx; | |
452 | ||
453 | mutex_unlock(&iter->lock); | |
454 | ||
455 | if (unlikely(n < idx)) /* insertion completed by another thread */ | |
456 | goto lookup; | |
457 | ||
458 | /* In case we failed to insert the entry into the radixtree, we need | |
459 | * to look beyond the current sg. | |
460 | */ | |
461 | while (idx + count <= n) { | |
462 | idx += count; | |
463 | sg = ____sg_next(sg); | |
464 | count = __sg_page_count(sg); | |
465 | } | |
466 | ||
467 | *offset = n - idx; | |
468 | return sg; | |
469 | ||
470 | lookup: | |
471 | rcu_read_lock(); | |
472 | ||
473 | sg = radix_tree_lookup(&iter->radix, n); | |
474 | GEM_BUG_ON(!sg); | |
475 | ||
476 | /* If this index is in the middle of multi-page sg entry, | |
477 | * the radix tree will contain a value entry that points | |
478 | * to the start of that range. We will return the pointer to | |
479 | * the base page and the offset of this page within the | |
480 | * sg entry's range. | |
481 | */ | |
482 | *offset = 0; | |
483 | if (unlikely(xa_is_value(sg))) { | |
484 | unsigned long base = xa_to_value(sg); | |
485 | ||
486 | sg = radix_tree_lookup(&iter->radix, base); | |
487 | GEM_BUG_ON(!sg); | |
488 | ||
489 | *offset = n - base; | |
490 | } | |
491 | ||
492 | rcu_read_unlock(); | |
493 | ||
494 | return sg; | |
495 | } | |
496 | ||
497 | struct page * | |
498 | i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n) | |
499 | { | |
500 | struct scatterlist *sg; | |
501 | unsigned int offset; | |
502 | ||
503 | GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); | |
504 | ||
505 | sg = i915_gem_object_get_sg(obj, n, &offset); | |
506 | return nth_page(sg_page(sg), offset); | |
507 | } | |
508 | ||
509 | /* Like i915_gem_object_get_page(), but mark the returned page dirty */ | |
510 | struct page * | |
511 | i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, | |
512 | unsigned int n) | |
513 | { | |
514 | struct page *page; | |
515 | ||
516 | page = i915_gem_object_get_page(obj, n); | |
517 | if (!obj->mm.dirty) | |
518 | set_page_dirty(page); | |
519 | ||
520 | return page; | |
521 | } | |
522 | ||
523 | dma_addr_t | |
524 | i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, | |
525 | unsigned long n, | |
526 | unsigned int *len) | |
527 | { | |
528 | struct scatterlist *sg; | |
529 | unsigned int offset; | |
530 | ||
531 | sg = i915_gem_object_get_sg(obj, n, &offset); | |
532 | ||
533 | if (len) | |
534 | *len = sg_dma_len(sg) - (offset << PAGE_SHIFT); | |
535 | ||
536 | return sg_dma_address(sg) + (offset << PAGE_SHIFT); | |
537 | } | |
538 | ||
539 | dma_addr_t | |
540 | i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, | |
541 | unsigned long n) | |
542 | { | |
543 | return i915_gem_object_get_dma_address_len(obj, n, NULL); | |
544 | } |