1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #include <uapi/misc/habanalabs.h>
9 #include "habanalabs.h"
10 #include "include/hw_ip/mmu/mmu_general.h"
12 #include <linux/uaccess.h>
13 #include <linux/slab.h>
14 #include <linux/genalloc.h>
16 #define PGS_IN_2MB_PAGE (PAGE_SIZE_2MB >> PAGE_SHIFT)
17 #define HL_MMU_DEBUG 0
20 * The va ranges in context object contain a list with the available chunks of
21 * device virtual memory.
22 * There is one range for host allocations and one for DRAM allocations.
24 * On initialization each range contains one chunk of all of its available
25 * virtual range which is a half of the total device virtual range.
27 * On each mapping of physical pages, a suitable virtual range chunk (with a
28 * minimum size) is selected from the list. If the chunk size equals the
29 * requested size, the chunk is returned. Otherwise, the chunk is split into
30 * two chunks - one to return as result and a remainder to stay in the list.
32 * On each Unmapping of a virtual address, the relevant virtual chunk is
33 * returned to the list. The chunk is added to the list and if its edges match
34 * the edges of the adjacent chunks (means a contiguous chunk can be created),
35 * the chunks are merged.
37 * On finish, the list is checked to have only one chunk of all the relevant
38 * virtual range (which is a half of the device total virtual range).
39 * If not (means not all mappings were unmapped), a warning is printed.
43 * alloc_device_memory - allocate device memory
45 * @ctx : current context
46 * @args : host parameters containing the requested size
47 * @ret_handle : result handle
49 * This function does the following:
50 * - Allocate the requested size rounded up to 2MB pages
51 * - Return unique handle
53 static int alloc_device_memory(struct hl_ctx
*ctx
, struct hl_mem_in
*args
,
56 struct hl_device
*hdev
= ctx
->hdev
;
57 struct hl_vm
*vm
= &hdev
->vm
;
58 struct hl_vm_phys_pg_pack
*phys_pg_pack
;
60 u32 total_size
, num_pgs
, num_curr_pgs
, page_size
, page_shift
;
65 page_size
= hdev
->asic_prop
.dram_page_size
;
66 page_shift
= __ffs(page_size
);
67 num_pgs
= (args
->alloc
.mem_size
+ (page_size
- 1)) >> page_shift
;
68 total_size
= num_pgs
<< page_shift
;
70 contiguous
= args
->flags
& HL_MEM_CONTIGUOUS
;
73 paddr
= (u64
) gen_pool_alloc(vm
->dram_pg_pool
, total_size
);
76 "failed to allocate %u huge contiguous pages\n",
82 phys_pg_pack
= kzalloc(sizeof(*phys_pg_pack
), GFP_KERNEL
);
88 phys_pg_pack
->vm_type
= VM_TYPE_PHYS_PACK
;
89 phys_pg_pack
->asid
= ctx
->asid
;
90 phys_pg_pack
->npages
= num_pgs
;
91 phys_pg_pack
->page_size
= page_size
;
92 phys_pg_pack
->total_size
= total_size
;
93 phys_pg_pack
->flags
= args
->flags
;
94 phys_pg_pack
->contiguous
= contiguous
;
96 phys_pg_pack
->pages
= kcalloc(num_pgs
, sizeof(u64
), GFP_KERNEL
);
97 if (!phys_pg_pack
->pages
) {
102 if (phys_pg_pack
->contiguous
) {
103 for (i
= 0 ; i
< num_pgs
; i
++)
104 phys_pg_pack
->pages
[i
] = paddr
+ i
* page_size
;
106 for (i
= 0 ; i
< num_pgs
; i
++) {
107 phys_pg_pack
->pages
[i
] = (u64
) gen_pool_alloc(
110 if (!phys_pg_pack
->pages
[i
]) {
112 "ioctl failed to allocate page\n");
121 spin_lock(&vm
->idr_lock
);
122 handle
= idr_alloc(&vm
->phys_pg_pack_handles
, phys_pg_pack
, 1, 0,
124 spin_unlock(&vm
->idr_lock
);
127 dev_err(hdev
->dev
, "Failed to get handle for page\n");
132 for (i
= 0 ; i
< num_pgs
; i
++)
133 kref_get(&vm
->dram_pg_pool_refcount
);
135 phys_pg_pack
->handle
= handle
;
137 atomic64_add(phys_pg_pack
->total_size
, &ctx
->dram_phys_mem
);
138 atomic64_add(phys_pg_pack
->total_size
, &hdev
->dram_used_mem
);
140 *ret_handle
= handle
;
146 if (!phys_pg_pack
->contiguous
)
147 for (i
= 0 ; i
< num_curr_pgs
; i
++)
148 gen_pool_free(vm
->dram_pg_pool
, phys_pg_pack
->pages
[i
],
151 kfree(phys_pg_pack
->pages
);
156 gen_pool_free(vm
->dram_pg_pool
, paddr
, total_size
);
162 * get_userptr_from_host_va - initialize userptr structure from given host
165 * @hdev : habanalabs device structure
166 * @args : parameters containing the virtual address and size
167 * @p_userptr : pointer to result userptr structure
169 * This function does the following:
170 * - Allocate userptr structure
171 * - Pin the given host memory using the userptr structure
172 * - Perform DMA mapping to have the DMA addresses of the pages
174 static int get_userptr_from_host_va(struct hl_device
*hdev
,
175 struct hl_mem_in
*args
, struct hl_userptr
**p_userptr
)
177 struct hl_userptr
*userptr
;
180 userptr
= kzalloc(sizeof(*userptr
), GFP_KERNEL
);
186 rc
= hl_pin_host_memory(hdev
, args
->map_host
.host_virt_addr
,
187 args
->map_host
.mem_size
, userptr
);
189 dev_err(hdev
->dev
, "Failed to pin host memory\n");
193 rc
= hdev
->asic_funcs
->asic_dma_map_sg(hdev
, userptr
->sgt
->sgl
,
194 userptr
->sgt
->nents
, DMA_BIDIRECTIONAL
);
196 dev_err(hdev
->dev
, "failed to map sgt with DMA region\n");
200 userptr
->dma_mapped
= true;
201 userptr
->dir
= DMA_BIDIRECTIONAL
;
202 userptr
->vm_type
= VM_TYPE_USERPTR
;
204 *p_userptr
= userptr
;
209 hl_unpin_host_memory(hdev
, userptr
);
218 * free_userptr - free userptr structure
220 * @hdev : habanalabs device structure
221 * @userptr : userptr to free
223 * This function does the following:
224 * - Unpins the physical pages
225 * - Frees the userptr structure
227 static void free_userptr(struct hl_device
*hdev
, struct hl_userptr
*userptr
)
229 hl_unpin_host_memory(hdev
, userptr
);
234 * dram_pg_pool_do_release - free DRAM pages pool
236 * @ref : pointer to reference object
238 * This function does the following:
239 * - Frees the idr structure of physical pages handles
240 * - Frees the generic pool of DRAM physical pages
242 static void dram_pg_pool_do_release(struct kref
*ref
)
244 struct hl_vm
*vm
= container_of(ref
, struct hl_vm
,
245 dram_pg_pool_refcount
);
248 * free the idr here as only here we know for sure that there are no
249 * allocated physical pages and hence there are no handles in use
251 idr_destroy(&vm
->phys_pg_pack_handles
);
252 gen_pool_destroy(vm
->dram_pg_pool
);
256 * free_phys_pg_pack - free physical page pack
258 * @hdev : habanalabs device structure
259 * @phys_pg_pack : physical page pack to free
261 * This function does the following:
262 * - For DRAM memory only, iterate over the pack and free each physical block
263 * structure by returning it to the general pool
264 * - Free the hl_vm_phys_pg_pack structure
266 static void free_phys_pg_pack(struct hl_device
*hdev
,
267 struct hl_vm_phys_pg_pack
*phys_pg_pack
)
269 struct hl_vm
*vm
= &hdev
->vm
;
272 if (!phys_pg_pack
->created_from_userptr
) {
273 if (phys_pg_pack
->contiguous
) {
274 gen_pool_free(vm
->dram_pg_pool
, phys_pg_pack
->pages
[0],
275 phys_pg_pack
->total_size
);
277 for (i
= 0; i
< phys_pg_pack
->npages
; i
++)
278 kref_put(&vm
->dram_pg_pool_refcount
,
279 dram_pg_pool_do_release
);
281 for (i
= 0 ; i
< phys_pg_pack
->npages
; i
++) {
282 gen_pool_free(vm
->dram_pg_pool
,
283 phys_pg_pack
->pages
[i
],
284 phys_pg_pack
->page_size
);
285 kref_put(&vm
->dram_pg_pool_refcount
,
286 dram_pg_pool_do_release
);
291 kfree(phys_pg_pack
->pages
);
296 * free_device_memory - free device memory
298 * @ctx : current context
299 * @handle : handle of the memory chunk to free
301 * This function does the following:
302 * - Free the device memory related to the given handle
304 static int free_device_memory(struct hl_ctx
*ctx
, u32 handle
)
306 struct hl_device
*hdev
= ctx
->hdev
;
307 struct hl_vm
*vm
= &hdev
->vm
;
308 struct hl_vm_phys_pg_pack
*phys_pg_pack
;
310 spin_lock(&vm
->idr_lock
);
311 phys_pg_pack
= idr_find(&vm
->phys_pg_pack_handles
, handle
);
313 if (atomic_read(&phys_pg_pack
->mapping_cnt
) > 0) {
314 dev_err(hdev
->dev
, "handle %u is mapped, cannot free\n",
316 spin_unlock(&vm
->idr_lock
);
321 * must remove from idr before the freeing of the physical
322 * pages as the refcount of the pool is also the trigger of the
325 idr_remove(&vm
->phys_pg_pack_handles
, handle
);
326 spin_unlock(&vm
->idr_lock
);
328 atomic64_sub(phys_pg_pack
->total_size
, &ctx
->dram_phys_mem
);
329 atomic64_sub(phys_pg_pack
->total_size
, &hdev
->dram_used_mem
);
331 free_phys_pg_pack(hdev
, phys_pg_pack
);
333 spin_unlock(&vm
->idr_lock
);
335 "free device memory failed, no match for handle %u\n",
344 * clear_va_list_locked - free virtual addresses list
346 * @hdev : habanalabs device structure
347 * @va_list : list of virtual addresses to free
349 * This function does the following:
350 * - Iterate over the list and free each virtual addresses block
352 * This function should be called only when va_list lock is taken
354 static void clear_va_list_locked(struct hl_device
*hdev
,
355 struct list_head
*va_list)
357 struct hl_vm_va_block
*va_block
, *tmp
;
359 list_for_each_entry_safe(va_block
, tmp
, va_list, node
) {
360 list_del(&va_block
->node
);
366 * print_va_list_locked - print virtual addresses list
368 * @hdev : habanalabs device structure
369 * @va_list : list of virtual addresses to print
371 * This function does the following:
372 * - Iterate over the list and print each virtual addresses block
374 * This function should be called only when va_list lock is taken
376 static void print_va_list_locked(struct hl_device
*hdev
,
377 struct list_head
*va_list)
380 struct hl_vm_va_block
*va_block
;
382 dev_dbg(hdev
->dev
, "print va list:\n");
384 list_for_each_entry(va_block
, va_list, node
)
386 "va block, start: 0x%llx, end: 0x%llx, size: %llu\n",
387 va_block
->start
, va_block
->end
, va_block
->size
);
392 * merge_va_blocks_locked - merge a virtual block if possible
394 * @hdev : pointer to the habanalabs device structure
395 * @va_list : pointer to the virtual addresses block list
396 * @va_block : virtual block to merge with adjacent blocks
398 * This function does the following:
399 * - Merge the given blocks with the adjacent blocks if their virtual ranges
400 * create a contiguous virtual range
402 * This Function should be called only when va_list lock is taken
404 static void merge_va_blocks_locked(struct hl_device
*hdev
,
405 struct list_head
*va_list, struct hl_vm_va_block
*va_block
)
407 struct hl_vm_va_block
*prev
, *next
;
409 prev
= list_prev_entry(va_block
, node
);
410 if (&prev
->node
!= va_list && prev
->end
+ 1 == va_block
->start
) {
411 prev
->end
= va_block
->end
;
412 prev
->size
= prev
->end
- prev
->start
;
413 list_del(&va_block
->node
);
418 next
= list_next_entry(va_block
, node
);
419 if (&next
->node
!= va_list && va_block
->end
+ 1 == next
->start
) {
420 next
->start
= va_block
->start
;
421 next
->size
= next
->end
- next
->start
;
422 list_del(&va_block
->node
);
428 * add_va_block_locked - add a virtual block to the virtual addresses list
430 * @hdev : pointer to the habanalabs device structure
431 * @va_list : pointer to the virtual addresses block list
432 * @start : start virtual address
433 * @end : end virtual address
435 * This function does the following:
436 * - Add the given block to the virtual blocks list and merge with other
437 * blocks if a contiguous virtual block can be created
439 * This Function should be called only when va_list lock is taken
441 static int add_va_block_locked(struct hl_device
*hdev
,
442 struct list_head
*va_list, u64 start
, u64 end
)
444 struct hl_vm_va_block
*va_block
, *res
= NULL
;
445 u64 size
= end
- start
;
447 print_va_list_locked(hdev
, va_list);
449 list_for_each_entry(va_block
, va_list, node
) {
450 /* TODO: remove upon matureness */
451 if (hl_mem_area_crosses_range(start
, size
, va_block
->start
,
454 "block crossing ranges at start 0x%llx, end 0x%llx\n",
455 va_block
->start
, va_block
->end
);
459 if (va_block
->end
< start
)
463 va_block
= kmalloc(sizeof(*va_block
), GFP_KERNEL
);
467 va_block
->start
= start
;
469 va_block
->size
= size
;
472 list_add(&va_block
->node
, va_list);
474 list_add(&va_block
->node
, &res
->node
);
476 merge_va_blocks_locked(hdev
, va_list, va_block
);
478 print_va_list_locked(hdev
, va_list);
484 * add_va_block - wrapper for add_va_block_locked
486 * @hdev : pointer to the habanalabs device structure
487 * @va_list : pointer to the virtual addresses block list
488 * @start : start virtual address
489 * @end : end virtual address
491 * This function does the following:
492 * - Takes the list lock and calls add_va_block_locked
494 static inline int add_va_block(struct hl_device
*hdev
,
495 struct hl_va_range
*va_range
, u64 start
, u64 end
)
499 mutex_lock(&va_range
->lock
);
500 rc
= add_va_block_locked(hdev
, &va_range
->list
, start
, end
);
501 mutex_unlock(&va_range
->lock
);
507 * get_va_block - get a virtual block with the requested size
509 * @hdev : pointer to the habanalabs device structure
510 * @va_range : pointer to the virtual addresses range
511 * @size : requested block size
512 * @hint_addr : hint for request address by the user
513 * @is_userptr : is host or DRAM memory
515 * This function does the following:
516 * - Iterate on the virtual block list to find a suitable virtual block for the
518 * - Reserve the requested block and update the list
519 * - Return the start address of the virtual block
521 static u64
get_va_block(struct hl_device
*hdev
,
522 struct hl_va_range
*va_range
, u32 size
, u64 hint_addr
,
525 struct hl_vm_va_block
*va_block
, *new_va_block
= NULL
;
526 u64 valid_start
, valid_size
, prev_start
, prev_end
, page_mask
,
527 res_valid_start
= 0, res_valid_size
= 0;
529 bool add_prev
= false;
533 * We cannot know if the user allocated memory with huge pages
534 * or not, hence we continue with the biggest possible
537 page_size
= PAGE_SIZE_2MB
;
538 page_mask
= PAGE_MASK_2MB
;
540 page_size
= hdev
->asic_prop
.dram_page_size
;
541 page_mask
= ~((u64
)page_size
- 1);
544 mutex_lock(&va_range
->lock
);
546 print_va_list_locked(hdev
, &va_range
->list
);
548 list_for_each_entry(va_block
, &va_range
->list
, node
) {
549 /* calc the first possible aligned addr */
550 valid_start
= va_block
->start
;
553 if (valid_start
& (page_size
- 1)) {
554 valid_start
&= page_mask
;
555 valid_start
+= page_size
;
556 if (valid_start
> va_block
->end
)
560 valid_size
= va_block
->end
- valid_start
;
562 if (valid_size
>= size
&&
563 (!new_va_block
|| valid_size
< res_valid_size
)) {
565 new_va_block
= va_block
;
566 res_valid_start
= valid_start
;
567 res_valid_size
= valid_size
;
570 if (hint_addr
&& hint_addr
>= valid_start
&&
571 ((hint_addr
+ size
) <= va_block
->end
)) {
572 new_va_block
= va_block
;
573 res_valid_start
= hint_addr
;
574 res_valid_size
= valid_size
;
580 dev_err(hdev
->dev
, "no available va block for size %u\n", size
);
584 if (res_valid_start
> new_va_block
->start
) {
585 prev_start
= new_va_block
->start
;
586 prev_end
= res_valid_start
- 1;
588 new_va_block
->start
= res_valid_start
;
589 new_va_block
->size
= res_valid_size
;
594 if (new_va_block
->size
> size
) {
595 new_va_block
->start
+= size
;
596 new_va_block
->size
= new_va_block
->end
- new_va_block
->start
;
598 list_del(&new_va_block
->node
);
603 add_va_block_locked(hdev
, &va_range
->list
, prev_start
,
606 print_va_list_locked(hdev
, &va_range
->list
);
608 mutex_unlock(&va_range
->lock
);
610 return res_valid_start
;
614 * get_sg_info - get number of pages and the DMA address from SG list
617 * @dma_addr : pointer to DMA address to return
619 * Calculate the number of consecutive pages described by the SG list. Take the
620 * offset of the address in the first page, add to it the length and round it up
621 * to the number of needed pages.
623 static u32
get_sg_info(struct scatterlist
*sg
, dma_addr_t
*dma_addr
)
625 *dma_addr
= sg_dma_address(sg
);
627 return ((((*dma_addr
) & (PAGE_SIZE
- 1)) + sg_dma_len(sg
)) +
628 (PAGE_SIZE
- 1)) >> PAGE_SHIFT
;
632 * init_phys_pg_pack_from_userptr - initialize physical page pack from host
635 * @ctx : current context
636 * @userptr : userptr to initialize from
637 * @pphys_pg_pack : res pointer
639 * This function does the following:
640 * - Pin the physical pages related to the given virtual block
641 * - Create a physical page pack from the physical pages related to the given
644 static int init_phys_pg_pack_from_userptr(struct hl_ctx
*ctx
,
645 struct hl_userptr
*userptr
,
646 struct hl_vm_phys_pg_pack
**pphys_pg_pack
)
648 struct hl_vm_phys_pg_pack
*phys_pg_pack
;
649 struct scatterlist
*sg
;
652 u32 npages
, total_npages
, page_size
= PAGE_SIZE
;
653 bool first
= true, is_huge_page_opt
= true;
656 phys_pg_pack
= kzalloc(sizeof(*phys_pg_pack
), GFP_KERNEL
);
660 phys_pg_pack
->vm_type
= userptr
->vm_type
;
661 phys_pg_pack
->created_from_userptr
= true;
662 phys_pg_pack
->asid
= ctx
->asid
;
663 atomic_set(&phys_pg_pack
->mapping_cnt
, 1);
665 /* Only if all dma_addrs are aligned to 2MB and their
666 * sizes is at least 2MB, we can use huge page mapping.
667 * We limit the 2MB optimization to this condition,
668 * since later on we acquire the related VA range as one
672 for_each_sg(userptr
->sgt
->sgl
, sg
, userptr
->sgt
->nents
, i
) {
673 npages
= get_sg_info(sg
, &dma_addr
);
675 total_npages
+= npages
;
679 dma_addr
&= PAGE_MASK_2MB
;
682 if ((npages
% PGS_IN_2MB_PAGE
) ||
683 (dma_addr
& (PAGE_SIZE_2MB
- 1)))
684 is_huge_page_opt
= false;
687 if (is_huge_page_opt
) {
688 page_size
= PAGE_SIZE_2MB
;
689 total_npages
/= PGS_IN_2MB_PAGE
;
692 page_mask
= ~(((u64
) page_size
) - 1);
694 phys_pg_pack
->pages
= kcalloc(total_npages
, sizeof(u64
), GFP_KERNEL
);
695 if (!phys_pg_pack
->pages
) {
697 goto page_pack_arr_mem_err
;
700 phys_pg_pack
->npages
= total_npages
;
701 phys_pg_pack
->page_size
= page_size
;
702 phys_pg_pack
->total_size
= total_npages
* page_size
;
706 for_each_sg(userptr
->sgt
->sgl
, sg
, userptr
->sgt
->nents
, i
) {
707 npages
= get_sg_info(sg
, &dma_addr
);
709 /* align down to physical page size and save the offset */
712 phys_pg_pack
->offset
= dma_addr
& (page_size
- 1);
713 dma_addr
&= page_mask
;
717 phys_pg_pack
->pages
[j
++] = dma_addr
;
718 dma_addr
+= page_size
;
720 if (is_huge_page_opt
)
721 npages
-= PGS_IN_2MB_PAGE
;
727 *pphys_pg_pack
= phys_pg_pack
;
731 page_pack_arr_mem_err
:
738 * map_phys_page_pack - maps the physical page pack
740 * @ctx : current context
741 * @vaddr : start address of the virtual area to map from
742 * @phys_pg_pack : the pack of physical pages to map to
744 * This function does the following:
745 * - Maps each chunk of virtual memory to matching physical chunk
746 * - Stores number of successful mappings in the given argument
747 * - Returns 0 on success, error code otherwise.
749 static int map_phys_page_pack(struct hl_ctx
*ctx
, u64 vaddr
,
750 struct hl_vm_phys_pg_pack
*phys_pg_pack
)
752 struct hl_device
*hdev
= ctx
->hdev
;
753 u64 next_vaddr
= vaddr
, paddr
;
754 u32 page_size
= phys_pg_pack
->page_size
;
755 int i
, rc
= 0, mapped_pg_cnt
= 0;
757 for (i
= 0 ; i
< phys_pg_pack
->npages
; i
++) {
758 paddr
= phys_pg_pack
->pages
[i
];
760 /* For accessing the host we need to turn on bit 39 */
761 if (phys_pg_pack
->created_from_userptr
)
762 paddr
+= hdev
->asic_prop
.host_phys_base_address
;
764 rc
= hl_mmu_map(ctx
, next_vaddr
, paddr
, page_size
);
767 "map failed for handle %u, npages: %d, mapped: %d",
768 phys_pg_pack
->handle
, phys_pg_pack
->npages
,
774 next_vaddr
+= page_size
;
781 for (i
= 0 ; i
< mapped_pg_cnt
; i
++) {
782 if (hl_mmu_unmap(ctx
, next_vaddr
, page_size
))
783 dev_warn_ratelimited(hdev
->dev
,
784 "failed to unmap handle %u, va: 0x%llx, pa: 0x%llx, page size: %u\n",
785 phys_pg_pack
->handle
, next_vaddr
,
786 phys_pg_pack
->pages
[i
], page_size
);
788 next_vaddr
+= page_size
;
794 static int get_paddr_from_handle(struct hl_ctx
*ctx
, struct hl_mem_in
*args
,
797 struct hl_device
*hdev
= ctx
->hdev
;
798 struct hl_vm
*vm
= &hdev
->vm
;
799 struct hl_vm_phys_pg_pack
*phys_pg_pack
;
802 handle
= lower_32_bits(args
->map_device
.handle
);
803 spin_lock(&vm
->idr_lock
);
804 phys_pg_pack
= idr_find(&vm
->phys_pg_pack_handles
, handle
);
806 spin_unlock(&vm
->idr_lock
);
807 dev_err(hdev
->dev
, "no match for handle %u\n", handle
);
811 *paddr
= phys_pg_pack
->pages
[0];
813 spin_unlock(&vm
->idr_lock
);
819 * map_device_va - map the given memory
821 * @ctx : current context
822 * @args : host parameters with handle/host virtual address
823 * @device_addr : pointer to result device virtual address
825 * This function does the following:
826 * - If given a physical device memory handle, map to a device virtual block
827 * and return the start address of this block
828 * - If given a host virtual address and size, find the related physical pages,
829 * map a device virtual block to this pages and return the start address of
832 static int map_device_va(struct hl_ctx
*ctx
, struct hl_mem_in
*args
,
835 struct hl_device
*hdev
= ctx
->hdev
;
836 struct hl_vm
*vm
= &hdev
->vm
;
837 struct hl_vm_phys_pg_pack
*phys_pg_pack
;
838 struct hl_userptr
*userptr
= NULL
;
839 struct hl_vm_hash_node
*hnode
;
840 enum vm_type_t
*vm_type
;
841 u64 ret_vaddr
, hint_addr
;
844 bool is_userptr
= args
->flags
& HL_MEM_USERPTR
;
850 rc
= get_userptr_from_host_va(hdev
, args
, &userptr
);
852 dev_err(hdev
->dev
, "failed to get userptr from va\n");
856 rc
= init_phys_pg_pack_from_userptr(ctx
, userptr
,
860 "unable to init page pack for vaddr 0x%llx\n",
861 args
->map_host
.host_virt_addr
);
862 goto init_page_pack_err
;
865 vm_type
= (enum vm_type_t
*) userptr
;
866 hint_addr
= args
->map_host
.hint_addr
;
868 handle
= lower_32_bits(args
->map_device
.handle
);
870 spin_lock(&vm
->idr_lock
);
871 phys_pg_pack
= idr_find(&vm
->phys_pg_pack_handles
, handle
);
873 spin_unlock(&vm
->idr_lock
);
875 "no match for handle %u\n", handle
);
879 /* increment now to avoid freeing device memory while mapping */
880 atomic_inc(&phys_pg_pack
->mapping_cnt
);
882 spin_unlock(&vm
->idr_lock
);
884 vm_type
= (enum vm_type_t
*) phys_pg_pack
;
886 hint_addr
= args
->map_device
.hint_addr
;
890 * relevant for mapping device physical memory only, as host memory is
893 if (!is_userptr
&& !(phys_pg_pack
->flags
& HL_MEM_SHARED
) &&
894 phys_pg_pack
->asid
!= ctx
->asid
) {
896 "Failed to map memory, handle %u is not shared\n",
902 hnode
= kzalloc(sizeof(*hnode
), GFP_KERNEL
);
908 ret_vaddr
= get_va_block(hdev
,
909 is_userptr
? &ctx
->host_va_range
: &ctx
->dram_va_range
,
910 phys_pg_pack
->total_size
, hint_addr
, is_userptr
);
912 dev_err(hdev
->dev
, "no available va block for handle %u\n",
918 mutex_lock(&ctx
->mmu_lock
);
920 rc
= map_phys_page_pack(ctx
, ret_vaddr
, phys_pg_pack
);
922 mutex_unlock(&ctx
->mmu_lock
);
923 dev_err(hdev
->dev
, "mapping page pack failed for handle %u\n",
928 hdev
->asic_funcs
->mmu_invalidate_cache(hdev
, false);
930 mutex_unlock(&ctx
->mmu_lock
);
932 ret_vaddr
+= phys_pg_pack
->offset
;
934 hnode
->ptr
= vm_type
;
935 hnode
->vaddr
= ret_vaddr
;
937 mutex_lock(&ctx
->mem_hash_lock
);
938 hash_add(ctx
->mem_hash
, &hnode
->node
, ret_vaddr
);
939 mutex_unlock(&ctx
->mem_hash_lock
);
941 *device_addr
= ret_vaddr
;
944 free_phys_pg_pack(hdev
, phys_pg_pack
);
949 if (add_va_block(hdev
,
950 is_userptr
? &ctx
->host_va_range
: &ctx
->dram_va_range
,
952 ret_vaddr
+ phys_pg_pack
->total_size
- 1))
954 "release va block failed for handle 0x%x, vaddr: 0x%llx\n",
961 atomic_dec(&phys_pg_pack
->mapping_cnt
);
963 free_phys_pg_pack(hdev
, phys_pg_pack
);
966 free_userptr(hdev
, userptr
);
972 * unmap_device_va - unmap the given device virtual address
974 * @ctx : current context
975 * @vaddr : device virtual address to unmap
977 * This function does the following:
978 * - Unmap the physical pages related to the given virtual address
979 * - return the device virtual block to the virtual block list
981 static int unmap_device_va(struct hl_ctx
*ctx
, u64 vaddr
)
983 struct hl_device
*hdev
= ctx
->hdev
;
984 struct hl_vm_phys_pg_pack
*phys_pg_pack
= NULL
;
985 struct hl_vm_hash_node
*hnode
= NULL
;
986 struct hl_userptr
*userptr
= NULL
;
987 enum vm_type_t
*vm_type
;
993 /* protect from double entrance */
994 mutex_lock(&ctx
->mem_hash_lock
);
995 hash_for_each_possible(ctx
->mem_hash
, hnode
, node
, (unsigned long)vaddr
)
996 if (vaddr
== hnode
->vaddr
)
1000 mutex_unlock(&ctx
->mem_hash_lock
);
1002 "unmap failed, no mem hnode for vaddr 0x%llx\n",
1007 hash_del(&hnode
->node
);
1008 mutex_unlock(&ctx
->mem_hash_lock
);
1010 vm_type
= hnode
->ptr
;
1012 if (*vm_type
== VM_TYPE_USERPTR
) {
1014 userptr
= hnode
->ptr
;
1015 rc
= init_phys_pg_pack_from_userptr(ctx
, userptr
,
1019 "unable to init page pack for vaddr 0x%llx\n",
1023 } else if (*vm_type
== VM_TYPE_PHYS_PACK
) {
1025 phys_pg_pack
= hnode
->ptr
;
1028 "unmap failed, unknown vm desc for vaddr 0x%llx\n",
1034 if (atomic_read(&phys_pg_pack
->mapping_cnt
) == 0) {
1035 dev_err(hdev
->dev
, "vaddr 0x%llx is not mapped\n", vaddr
);
1037 goto mapping_cnt_err
;
1040 page_size
= phys_pg_pack
->page_size
;
1041 vaddr
&= ~(((u64
) page_size
) - 1);
1045 mutex_lock(&ctx
->mmu_lock
);
1047 for (i
= 0 ; i
< phys_pg_pack
->npages
; i
++, next_vaddr
+= page_size
)
1048 if (hl_mmu_unmap(ctx
, next_vaddr
, page_size
))
1049 dev_warn_ratelimited(hdev
->dev
,
1050 "unmap failed for vaddr: 0x%llx\n", next_vaddr
);
1052 hdev
->asic_funcs
->mmu_invalidate_cache(hdev
, true);
1054 mutex_unlock(&ctx
->mmu_lock
);
1056 if (add_va_block(hdev
,
1057 is_userptr
? &ctx
->host_va_range
: &ctx
->dram_va_range
,
1059 vaddr
+ phys_pg_pack
->total_size
- 1))
1060 dev_warn(hdev
->dev
, "add va block failed for vaddr: 0x%llx\n",
1063 atomic_dec(&phys_pg_pack
->mapping_cnt
);
1067 free_phys_pg_pack(hdev
, phys_pg_pack
);
1068 free_userptr(hdev
, userptr
);
1075 free_phys_pg_pack(hdev
, phys_pg_pack
);
1077 mutex_lock(&ctx
->mem_hash_lock
);
1078 hash_add(ctx
->mem_hash
, &hnode
->node
, vaddr
);
1079 mutex_unlock(&ctx
->mem_hash_lock
);
1084 int hl_mem_ioctl(struct hl_fpriv
*hpriv
, void *data
)
1086 union hl_mem_args
*args
= data
;
1087 struct hl_device
*hdev
= hpriv
->hdev
;
1088 struct hl_ctx
*ctx
= hpriv
->ctx
;
1089 u64 device_addr
= 0;
1093 if (hl_device_disabled_or_in_reset(hdev
)) {
1094 dev_warn_ratelimited(hdev
->dev
,
1095 "Device is disabled or in reset. Can't execute memory IOCTL\n");
1099 if (hdev
->mmu_enable
) {
1100 switch (args
->in
.op
) {
1101 case HL_MEM_OP_ALLOC
:
1102 if (!hdev
->dram_supports_virtual_memory
) {
1104 "DRAM alloc is not supported\n");
1108 if (args
->in
.alloc
.mem_size
== 0) {
1110 "alloc size must be larger than 0\n");
1114 rc
= alloc_device_memory(ctx
, &args
->in
, &handle
);
1116 memset(args
, 0, sizeof(*args
));
1117 args
->out
.handle
= (__u64
) handle
;
1120 case HL_MEM_OP_FREE
:
1121 if (!hdev
->dram_supports_virtual_memory
) {
1123 "DRAM free is not supported\n");
1127 rc
= free_device_memory(ctx
, args
->in
.free
.handle
);
1131 rc
= map_device_va(ctx
, &args
->in
, &device_addr
);
1133 memset(args
, 0, sizeof(*args
));
1134 args
->out
.device_virt_addr
= device_addr
;
1137 case HL_MEM_OP_UNMAP
:
1138 rc
= unmap_device_va(ctx
,
1139 args
->in
.unmap
.device_virt_addr
);
1143 dev_err(hdev
->dev
, "Unknown opcode for memory IOCTL\n");
1148 switch (args
->in
.op
) {
1149 case HL_MEM_OP_ALLOC
:
1150 if (args
->in
.alloc
.mem_size
== 0) {
1152 "alloc size must be larger than 0\n");
1157 /* Force contiguous as there are no real MMU
1158 * translations to overcome physical memory gaps
1160 args
->in
.flags
|= HL_MEM_CONTIGUOUS
;
1161 rc
= alloc_device_memory(ctx
, &args
->in
, &handle
);
1163 memset(args
, 0, sizeof(*args
));
1164 args
->out
.handle
= (__u64
) handle
;
1167 case HL_MEM_OP_FREE
:
1168 rc
= free_device_memory(ctx
, args
->in
.free
.handle
);
1172 if (args
->in
.flags
& HL_MEM_USERPTR
) {
1173 device_addr
= args
->in
.map_host
.host_virt_addr
;
1176 rc
= get_paddr_from_handle(ctx
, &args
->in
,
1180 memset(args
, 0, sizeof(*args
));
1181 args
->out
.device_virt_addr
= device_addr
;
1184 case HL_MEM_OP_UNMAP
:
1189 dev_err(hdev
->dev
, "Unknown opcode for memory IOCTL\n");
1200 * hl_pin_host_memory - pins a chunk of host memory
1202 * @hdev : pointer to the habanalabs device structure
1203 * @addr : the user-space virtual address of the memory area
1204 * @size : the size of the memory area
1205 * @userptr : pointer to hl_userptr structure
1207 * This function does the following:
1208 * - Pins the physical pages
1209 * - Create a SG list from those pages
1211 int hl_pin_host_memory(struct hl_device
*hdev
, u64 addr
, u64 size
,
1212 struct hl_userptr
*userptr
)
1219 dev_err(hdev
->dev
, "size to pin is invalid - %llu\n", size
);
1223 if (!access_ok((void __user
*) (uintptr_t) addr
, size
)) {
1224 dev_err(hdev
->dev
, "user pointer is invalid - 0x%llx\n", addr
);
1229 * If the combination of the address and size requested for this memory
1230 * region causes an integer overflow, return error.
1232 if (((addr
+ size
) < addr
) ||
1233 PAGE_ALIGN(addr
+ size
) < (addr
+ size
)) {
1235 "user pointer 0x%llx + %llu causes integer overflow\n",
1240 start
= addr
& PAGE_MASK
;
1241 offset
= addr
& ~PAGE_MASK
;
1242 end
= PAGE_ALIGN(addr
+ size
);
1243 npages
= (end
- start
) >> PAGE_SHIFT
;
1245 userptr
->size
= size
;
1246 userptr
->addr
= addr
;
1247 userptr
->dma_mapped
= false;
1248 INIT_LIST_HEAD(&userptr
->job_node
);
1250 userptr
->vec
= frame_vector_create(npages
);
1251 if (!userptr
->vec
) {
1252 dev_err(hdev
->dev
, "Failed to create frame vector\n");
1256 rc
= get_vaddr_frames(start
, npages
, FOLL_FORCE
| FOLL_WRITE
,
1261 "Failed to map host memory, user ptr probably wrong\n");
1263 goto destroy_framevec
;
1268 if (frame_vector_to_pages(userptr
->vec
) < 0) {
1270 "Failed to translate frame vector to pages\n");
1275 userptr
->sgt
= kzalloc(sizeof(*userptr
->sgt
), GFP_ATOMIC
);
1276 if (!userptr
->sgt
) {
1281 rc
= sg_alloc_table_from_pages(userptr
->sgt
,
1282 frame_vector_pages(userptr
->vec
),
1283 npages
, offset
, size
, GFP_ATOMIC
);
1285 dev_err(hdev
->dev
, "failed to create SG table from pages\n");
1289 hl_debugfs_add_userptr(hdev
, userptr
);
1294 kfree(userptr
->sgt
);
1296 put_vaddr_frames(userptr
->vec
);
1298 frame_vector_destroy(userptr
->vec
);
1303 * hl_unpin_host_memory - unpins a chunk of host memory
1305 * @hdev : pointer to the habanalabs device structure
1306 * @userptr : pointer to hl_userptr structure
1308 * This function does the following:
1309 * - Unpins the physical pages related to the host memory
1310 * - Free the SG list
1312 int hl_unpin_host_memory(struct hl_device
*hdev
, struct hl_userptr
*userptr
)
1314 struct page
**pages
;
1316 hl_debugfs_remove_userptr(hdev
, userptr
);
1318 if (userptr
->dma_mapped
)
1319 hdev
->asic_funcs
->hl_dma_unmap_sg(hdev
,
1321 userptr
->sgt
->nents
,
1324 pages
= frame_vector_pages(userptr
->vec
);
1325 if (!IS_ERR(pages
)) {
1328 for (i
= 0; i
< frame_vector_count(userptr
->vec
); i
++)
1329 set_page_dirty_lock(pages
[i
]);
1331 put_vaddr_frames(userptr
->vec
);
1332 frame_vector_destroy(userptr
->vec
);
1334 list_del(&userptr
->job_node
);
1336 sg_free_table(userptr
->sgt
);
1337 kfree(userptr
->sgt
);
1343 * hl_userptr_delete_list - clear userptr list
1345 * @hdev : pointer to the habanalabs device structure
1346 * @userptr_list : pointer to the list to clear
1348 * This function does the following:
1349 * - Iterates over the list and unpins the host memory and frees the userptr
1352 void hl_userptr_delete_list(struct hl_device
*hdev
,
1353 struct list_head
*userptr_list
)
1355 struct hl_userptr
*userptr
, *tmp
;
1357 list_for_each_entry_safe(userptr
, tmp
, userptr_list
, job_node
) {
1358 hl_unpin_host_memory(hdev
, userptr
);
1362 INIT_LIST_HEAD(userptr_list
);
1366 * hl_userptr_is_pinned - returns whether the given userptr is pinned
1368 * @hdev : pointer to the habanalabs device structure
1369 * @userptr_list : pointer to the list to clear
1370 * @userptr : pointer to userptr to check
1372 * This function does the following:
1373 * - Iterates over the list and checks if the given userptr is in it, means is
1374 * pinned. If so, returns true, otherwise returns false.
1376 bool hl_userptr_is_pinned(struct hl_device
*hdev
, u64 addr
,
1377 u32 size
, struct list_head
*userptr_list
,
1378 struct hl_userptr
**userptr
)
1380 list_for_each_entry((*userptr
), userptr_list
, job_node
) {
1381 if ((addr
== (*userptr
)->addr
) && (size
== (*userptr
)->size
))
1389 * hl_va_range_init - initialize virtual addresses range
1391 * @hdev : pointer to the habanalabs device structure
1392 * @va_range : pointer to the range to initialize
1393 * @start : range start address
1394 * @end : range end address
1396 * This function does the following:
1397 * - Initializes the virtual addresses list of the given range with the given
1400 static int hl_va_range_init(struct hl_device
*hdev
,
1401 struct hl_va_range
*va_range
, u64 start
, u64 end
)
1405 INIT_LIST_HEAD(&va_range
->list
);
1407 /* PAGE_SIZE alignment */
1409 if (start
& (PAGE_SIZE
- 1)) {
1414 if (end
& (PAGE_SIZE
- 1))
1418 dev_err(hdev
->dev
, "too small vm range for va list\n");
1422 rc
= add_va_block(hdev
, va_range
, start
, end
);
1425 dev_err(hdev
->dev
, "Failed to init host va list\n");
1429 va_range
->start_addr
= start
;
1430 va_range
->end_addr
= end
;
1436 * hl_vm_ctx_init_with_ranges - initialize virtual memory for context
1438 * @ctx : pointer to the habanalabs context structure
1439 * @host_range_start : host virtual addresses range start
1440 * @host_range_end : host virtual addresses range end
1441 * @dram_range_start : dram virtual addresses range start
1442 * @dram_range_end : dram virtual addresses range end
1444 * This function initializes the following:
1446 * - Virtual address to area descriptor hashtable
1447 * - Virtual block list of available virtual memory
1449 static int hl_vm_ctx_init_with_ranges(struct hl_ctx
*ctx
, u64 host_range_start
,
1450 u64 host_range_end
, u64 dram_range_start
,
1453 struct hl_device
*hdev
= ctx
->hdev
;
1456 rc
= hl_mmu_ctx_init(ctx
);
1458 dev_err(hdev
->dev
, "failed to init context %d\n", ctx
->asid
);
1462 mutex_init(&ctx
->mem_hash_lock
);
1463 hash_init(ctx
->mem_hash
);
1465 mutex_init(&ctx
->host_va_range
.lock
);
1467 rc
= hl_va_range_init(hdev
, &ctx
->host_va_range
, host_range_start
,
1470 dev_err(hdev
->dev
, "failed to init host vm range\n");
1474 mutex_init(&ctx
->dram_va_range
.lock
);
1476 rc
= hl_va_range_init(hdev
, &ctx
->dram_va_range
, dram_range_start
,
1479 dev_err(hdev
->dev
, "failed to init dram vm range\n");
1483 hl_debugfs_add_ctx_mem_hash(hdev
, ctx
);
1488 mutex_destroy(&ctx
->dram_va_range
.lock
);
1490 mutex_lock(&ctx
->host_va_range
.lock
);
1491 clear_va_list_locked(hdev
, &ctx
->host_va_range
.list
);
1492 mutex_unlock(&ctx
->host_va_range
.lock
);
1494 mutex_destroy(&ctx
->host_va_range
.lock
);
1495 mutex_destroy(&ctx
->mem_hash_lock
);
1496 hl_mmu_ctx_fini(ctx
);
1501 int hl_vm_ctx_init(struct hl_ctx
*ctx
)
1503 struct asic_fixed_properties
*prop
= &ctx
->hdev
->asic_prop
;
1504 u64 host_range_start
, host_range_end
, dram_range_start
,
1507 atomic64_set(&ctx
->dram_phys_mem
, 0);
1510 * - If MMU is enabled, init the ranges as usual.
1511 * - If MMU is disabled, in case of host mapping, the returned address
1513 * In case of DRAM mapping, the returned address is the physical
1514 * address of the memory related to the given handle.
1516 if (ctx
->hdev
->mmu_enable
) {
1517 dram_range_start
= prop
->va_space_dram_start_address
;
1518 dram_range_end
= prop
->va_space_dram_end_address
;
1519 host_range_start
= prop
->va_space_host_start_address
;
1520 host_range_end
= prop
->va_space_host_end_address
;
1522 dram_range_start
= prop
->dram_user_base_address
;
1523 dram_range_end
= prop
->dram_end_address
;
1524 host_range_start
= prop
->dram_user_base_address
;
1525 host_range_end
= prop
->dram_end_address
;
1528 return hl_vm_ctx_init_with_ranges(ctx
, host_range_start
, host_range_end
,
1529 dram_range_start
, dram_range_end
);
1533 * hl_va_range_fini - clear a virtual addresses range
1535 * @hdev : pointer to the habanalabs structure
1536 * va_range : pointer to virtual addresses range
1538 * This function initializes the following:
1539 * - Checks that the given range contains the whole initial range
1540 * - Frees the virtual addresses block list and its lock
1542 static void hl_va_range_fini(struct hl_device
*hdev
,
1543 struct hl_va_range
*va_range
)
1545 struct hl_vm_va_block
*va_block
;
1547 if (list_empty(&va_range
->list
)) {
1549 "va list should not be empty on cleanup!\n");
1553 if (!list_is_singular(&va_range
->list
)) {
1555 "va list should not contain multiple blocks on cleanup!\n");
1559 va_block
= list_first_entry(&va_range
->list
, typeof(*va_block
), node
);
1561 if (va_block
->start
!= va_range
->start_addr
||
1562 va_block
->end
!= va_range
->end_addr
) {
1564 "wrong va block on cleanup, from 0x%llx to 0x%llx\n",
1565 va_block
->start
, va_block
->end
);
1570 mutex_lock(&va_range
->lock
);
1571 clear_va_list_locked(hdev
, &va_range
->list
);
1572 mutex_unlock(&va_range
->lock
);
1575 mutex_destroy(&va_range
->lock
);
1579 * hl_vm_ctx_fini - virtual memory teardown of context
1581 * @ctx : pointer to the habanalabs context structure
1583 * This function perform teardown the following:
1584 * - Virtual block list of available virtual memory
1585 * - Virtual address to area descriptor hashtable
1588 * In addition this function does the following:
1589 * - Unmaps the existing hashtable nodes if the hashtable is not empty. The
1590 * hashtable should be empty as no valid mappings should exist at this
1592 * - Frees any existing physical page list from the idr which relates to the
1593 * current context asid.
1594 * - This function checks the virtual block list for correctness. At this point
1595 * the list should contain one element which describes the whole virtual
1596 * memory range of the context. Otherwise, a warning is printed.
1598 void hl_vm_ctx_fini(struct hl_ctx
*ctx
)
1600 struct hl_device
*hdev
= ctx
->hdev
;
1601 struct hl_vm
*vm
= &hdev
->vm
;
1602 struct hl_vm_phys_pg_pack
*phys_pg_list
;
1603 struct hl_vm_hash_node
*hnode
;
1604 struct hlist_node
*tmp_node
;
1607 hl_debugfs_remove_ctx_mem_hash(hdev
, ctx
);
1609 if (!hash_empty(ctx
->mem_hash
))
1610 dev_notice(hdev
->dev
, "ctx is freed while it has va in use\n");
1612 hash_for_each_safe(ctx
->mem_hash
, i
, tmp_node
, hnode
, node
) {
1614 "hl_mem_hash_node of vaddr 0x%llx of asid %d is still alive\n",
1615 hnode
->vaddr
, ctx
->asid
);
1616 unmap_device_va(ctx
, hnode
->vaddr
);
1619 spin_lock(&vm
->idr_lock
);
1620 idr_for_each_entry(&vm
->phys_pg_pack_handles
, phys_pg_list
, i
)
1621 if (phys_pg_list
->asid
== ctx
->asid
) {
1623 "page list 0x%p of asid %d is still alive\n",
1624 phys_pg_list
, ctx
->asid
);
1625 free_phys_pg_pack(hdev
, phys_pg_list
);
1626 idr_remove(&vm
->phys_pg_pack_handles
, i
);
1628 spin_unlock(&vm
->idr_lock
);
1630 hl_va_range_fini(hdev
, &ctx
->dram_va_range
);
1631 hl_va_range_fini(hdev
, &ctx
->host_va_range
);
1633 mutex_destroy(&ctx
->mem_hash_lock
);
1634 hl_mmu_ctx_fini(ctx
);
1638 * hl_vm_init - initialize virtual memory module
1640 * @hdev : pointer to the habanalabs device structure
1642 * This function initializes the following:
1644 * - DRAM physical pages pool of 2MB
1645 * - Idr for device memory allocation handles
1647 int hl_vm_init(struct hl_device
*hdev
)
1649 struct asic_fixed_properties
*prop
= &hdev
->asic_prop
;
1650 struct hl_vm
*vm
= &hdev
->vm
;
1653 rc
= hl_mmu_init(hdev
);
1655 dev_err(hdev
->dev
, "Failed to init MMU\n");
1659 vm
->dram_pg_pool
= gen_pool_create(__ffs(prop
->dram_page_size
), -1);
1660 if (!vm
->dram_pg_pool
) {
1661 dev_err(hdev
->dev
, "Failed to create dram page pool\n");
1663 goto pool_create_err
;
1666 kref_init(&vm
->dram_pg_pool_refcount
);
1668 rc
= gen_pool_add(vm
->dram_pg_pool
, prop
->dram_user_base_address
,
1669 prop
->dram_end_address
- prop
->dram_user_base_address
,
1674 "Failed to add memory to dram page pool %d\n", rc
);
1678 spin_lock_init(&vm
->idr_lock
);
1679 idr_init(&vm
->phys_pg_pack_handles
);
1681 atomic64_set(&hdev
->dram_used_mem
, 0);
1683 vm
->init_done
= true;
1688 gen_pool_destroy(vm
->dram_pg_pool
);
1696 * hl_vm_fini - virtual memory module teardown
1698 * @hdev : pointer to the habanalabs device structure
1700 * This function perform teardown to the following:
1701 * - Idr for device memory allocation handles
1702 * - DRAM physical pages pool of 2MB
1705 void hl_vm_fini(struct hl_device
*hdev
)
1707 struct hl_vm
*vm
= &hdev
->vm
;
1713 * At this point all the contexts should be freed and hence no DRAM
1714 * memory should be in use. Hence the DRAM pool should be freed here.
1716 if (kref_put(&vm
->dram_pg_pool_refcount
, dram_pg_pool_do_release
) != 1)
1717 dev_warn(hdev
->dev
, "dram_pg_pool was not destroyed on %s\n",
1722 vm
->init_done
= false;