2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <rdma/ib_umem.h>
34 #include <rdma/ib_umem_odp.h>
35 #include <linux/kernel.h>
40 #include <linux/mlx5/eq.h>
42 /* Contains the details of a pagefault. */
43 struct mlx5_pagefault
{
49 /* Initiator or send message responder pagefault details. */
51 /* Received packet size, only valid for responders. */
54 * Number of resource holding WQE, depends on type.
58 * WQE index. Refers to either the send queue or
59 * receive queue, according to event_subtype.
63 /* RDMA responder pagefault details */
67 * Received packet size, minimal size page fault
68 * resolution required for forward progress.
76 struct mlx5_ib_pf_eq
*eq
;
77 struct work_struct work
;
80 #define MAX_PREFETCH_LEN (4*1024*1024U)
82 /* Timeout in ms to wait for an active mmu notifier to complete when handling
84 #define MMU_NOTIFIER_TIMEOUT 1000
86 #define MLX5_IMR_MTT_BITS (30 - PAGE_SHIFT)
87 #define MLX5_IMR_MTT_SHIFT (MLX5_IMR_MTT_BITS + PAGE_SHIFT)
88 #define MLX5_IMR_MTT_ENTRIES BIT_ULL(MLX5_IMR_MTT_BITS)
89 #define MLX5_IMR_MTT_SIZE BIT_ULL(MLX5_IMR_MTT_SHIFT)
90 #define MLX5_IMR_MTT_MASK (~(MLX5_IMR_MTT_SIZE - 1))
92 #define MLX5_KSM_PAGE_SHIFT MLX5_IMR_MTT_SHIFT
94 static u64 mlx5_imr_ksm_entries
;
96 static int check_parent(struct ib_umem_odp
*odp
,
97 struct mlx5_ib_mr
*parent
)
99 struct mlx5_ib_mr
*mr
= odp
->private;
101 return mr
&& mr
->parent
== parent
&& !odp
->dying
;
104 static struct ib_ucontext_per_mm
*mr_to_per_mm(struct mlx5_ib_mr
*mr
)
106 if (WARN_ON(!mr
|| !is_odp_mr(mr
)))
109 return to_ib_umem_odp(mr
->umem
)->per_mm
;
112 static struct ib_umem_odp
*odp_next(struct ib_umem_odp
*odp
)
114 struct mlx5_ib_mr
*mr
= odp
->private, *parent
= mr
->parent
;
115 struct ib_ucontext_per_mm
*per_mm
= odp
->per_mm
;
118 down_read(&per_mm
->umem_rwsem
);
120 rb
= rb_next(&odp
->interval_tree
.rb
);
123 odp
= rb_entry(rb
, struct ib_umem_odp
, interval_tree
.rb
);
124 if (check_parent(odp
, parent
))
130 up_read(&per_mm
->umem_rwsem
);
134 static struct ib_umem_odp
*odp_lookup(u64 start
, u64 length
,
135 struct mlx5_ib_mr
*parent
)
137 struct ib_ucontext_per_mm
*per_mm
= mr_to_per_mm(parent
);
138 struct ib_umem_odp
*odp
;
141 down_read(&per_mm
->umem_rwsem
);
142 odp
= rbt_ib_umem_lookup(&per_mm
->umem_tree
, start
, length
);
147 if (check_parent(odp
, parent
))
149 rb
= rb_next(&odp
->interval_tree
.rb
);
152 odp
= rb_entry(rb
, struct ib_umem_odp
, interval_tree
.rb
);
153 if (ib_umem_start(&odp
->umem
) > start
+ length
)
159 up_read(&per_mm
->umem_rwsem
);
163 void mlx5_odp_populate_klm(struct mlx5_klm
*pklm
, size_t offset
,
164 size_t nentries
, struct mlx5_ib_mr
*mr
, int flags
)
166 struct ib_pd
*pd
= mr
->ibmr
.pd
;
167 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
168 struct ib_umem_odp
*odp
;
172 if (flags
& MLX5_IB_UPD_XLT_ZAP
) {
173 for (i
= 0; i
< nentries
; i
++, pklm
++) {
174 pklm
->bcount
= cpu_to_be32(MLX5_IMR_MTT_SIZE
);
175 pklm
->key
= cpu_to_be32(dev
->null_mkey
);
181 odp
= odp_lookup(offset
* MLX5_IMR_MTT_SIZE
,
182 nentries
* MLX5_IMR_MTT_SIZE
, mr
);
184 for (i
= 0; i
< nentries
; i
++, pklm
++) {
185 pklm
->bcount
= cpu_to_be32(MLX5_IMR_MTT_SIZE
);
186 va
= (offset
+ i
) * MLX5_IMR_MTT_SIZE
;
187 if (odp
&& odp
->umem
.address
== va
) {
188 struct mlx5_ib_mr
*mtt
= odp
->private;
190 pklm
->key
= cpu_to_be32(mtt
->ibmr
.lkey
);
193 pklm
->key
= cpu_to_be32(dev
->null_mkey
);
195 mlx5_ib_dbg(dev
, "[%d] va %lx key %x\n",
196 i
, va
, be32_to_cpu(pklm
->key
));
200 static void mr_leaf_free_action(struct work_struct
*work
)
202 struct ib_umem_odp
*odp
= container_of(work
, struct ib_umem_odp
, work
);
203 int idx
= ib_umem_start(&odp
->umem
) >> MLX5_IMR_MTT_SHIFT
;
204 struct mlx5_ib_mr
*mr
= odp
->private, *imr
= mr
->parent
;
207 synchronize_srcu(&mr
->dev
->mr_srcu
);
209 ib_umem_release(&odp
->umem
);
211 mlx5_ib_update_xlt(imr
, idx
, 1, 0,
212 MLX5_IB_UPD_XLT_INDIRECT
|
213 MLX5_IB_UPD_XLT_ATOMIC
);
214 mlx5_mr_cache_free(mr
->dev
, mr
);
216 if (atomic_dec_and_test(&imr
->num_leaf_free
))
217 wake_up(&imr
->q_leaf_free
);
220 void mlx5_ib_invalidate_range(struct ib_umem_odp
*umem_odp
, unsigned long start
,
223 struct mlx5_ib_mr
*mr
;
224 const u64 umr_block_mask
= (MLX5_UMR_MTT_ALIGNMENT
/
225 sizeof(struct mlx5_mtt
)) - 1;
226 u64 idx
= 0, blk_start_idx
= 0;
227 struct ib_umem
*umem
;
232 pr_err("invalidation called on NULL umem or non-ODP umem\n");
235 umem
= &umem_odp
->umem
;
237 mr
= umem_odp
->private;
239 if (!mr
|| !mr
->ibmr
.pd
)
242 start
= max_t(u64
, ib_umem_start(umem
), start
);
243 end
= min_t(u64
, ib_umem_end(umem
), end
);
246 * Iteration one - zap the HW's MTTs. The notifiers_count ensures that
247 * while we are doing the invalidation, no page fault will attempt to
248 * overwrite the same MTTs. Concurent invalidations might race us,
249 * but they will write 0s as well, so no difference in the end result.
252 for (addr
= start
; addr
< end
; addr
+= BIT(umem
->page_shift
)) {
253 idx
= (addr
- ib_umem_start(umem
)) >> umem
->page_shift
;
255 * Strive to write the MTTs in chunks, but avoid overwriting
256 * non-existing MTTs. The huristic here can be improved to
257 * estimate the cost of another UMR vs. the cost of bigger
260 if (umem_odp
->dma_list
[idx
] &
261 (ODP_READ_ALLOWED_BIT
| ODP_WRITE_ALLOWED_BIT
)) {
267 u64 umr_offset
= idx
& umr_block_mask
;
269 if (in_block
&& umr_offset
== 0) {
270 mlx5_ib_update_xlt(mr
, blk_start_idx
,
271 idx
- blk_start_idx
, 0,
272 MLX5_IB_UPD_XLT_ZAP
|
273 MLX5_IB_UPD_XLT_ATOMIC
);
279 mlx5_ib_update_xlt(mr
, blk_start_idx
,
280 idx
- blk_start_idx
+ 1, 0,
281 MLX5_IB_UPD_XLT_ZAP
|
282 MLX5_IB_UPD_XLT_ATOMIC
);
284 * We are now sure that the device will not access the
285 * memory. We can safely unmap it, and mark it as dirty if
289 ib_umem_odp_unmap_dma_pages(umem_odp
, start
, end
);
291 if (unlikely(!umem
->npages
&& mr
->parent
&&
293 WRITE_ONCE(umem_odp
->dying
, 1);
294 atomic_inc(&mr
->parent
->num_leaf_free
);
295 schedule_work(&umem_odp
->work
);
299 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev
*dev
)
301 struct ib_odp_caps
*caps
= &dev
->odp_caps
;
303 memset(caps
, 0, sizeof(*caps
));
305 if (!MLX5_CAP_GEN(dev
->mdev
, pg
))
308 caps
->general_caps
= IB_ODP_SUPPORT
;
310 if (MLX5_CAP_GEN(dev
->mdev
, umr_extended_translation_offset
))
311 dev
->odp_max_size
= U64_MAX
;
313 dev
->odp_max_size
= BIT_ULL(MLX5_MAX_UMR_SHIFT
+ PAGE_SHIFT
);
315 if (MLX5_CAP_ODP(dev
->mdev
, ud_odp_caps
.send
))
316 caps
->per_transport_caps
.ud_odp_caps
|= IB_ODP_SUPPORT_SEND
;
318 if (MLX5_CAP_ODP(dev
->mdev
, ud_odp_caps
.srq_receive
))
319 caps
->per_transport_caps
.ud_odp_caps
|= IB_ODP_SUPPORT_SRQ_RECV
;
321 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.send
))
322 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_SEND
;
324 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.receive
))
325 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_RECV
;
327 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.write
))
328 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_WRITE
;
330 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.read
))
331 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_READ
;
333 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.atomic
))
334 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_ATOMIC
;
336 if (MLX5_CAP_ODP(dev
->mdev
, rc_odp_caps
.srq_receive
))
337 caps
->per_transport_caps
.rc_odp_caps
|= IB_ODP_SUPPORT_SRQ_RECV
;
339 if (MLX5_CAP_ODP(dev
->mdev
, xrc_odp_caps
.send
))
340 caps
->per_transport_caps
.xrc_odp_caps
|= IB_ODP_SUPPORT_SEND
;
342 if (MLX5_CAP_ODP(dev
->mdev
, xrc_odp_caps
.receive
))
343 caps
->per_transport_caps
.xrc_odp_caps
|= IB_ODP_SUPPORT_RECV
;
345 if (MLX5_CAP_ODP(dev
->mdev
, xrc_odp_caps
.write
))
346 caps
->per_transport_caps
.xrc_odp_caps
|= IB_ODP_SUPPORT_WRITE
;
348 if (MLX5_CAP_ODP(dev
->mdev
, xrc_odp_caps
.read
))
349 caps
->per_transport_caps
.xrc_odp_caps
|= IB_ODP_SUPPORT_READ
;
351 if (MLX5_CAP_ODP(dev
->mdev
, xrc_odp_caps
.atomic
))
352 caps
->per_transport_caps
.xrc_odp_caps
|= IB_ODP_SUPPORT_ATOMIC
;
354 if (MLX5_CAP_ODP(dev
->mdev
, xrc_odp_caps
.srq_receive
))
355 caps
->per_transport_caps
.xrc_odp_caps
|= IB_ODP_SUPPORT_SRQ_RECV
;
357 if (MLX5_CAP_GEN(dev
->mdev
, fixed_buffer_size
) &&
358 MLX5_CAP_GEN(dev
->mdev
, null_mkey
) &&
359 MLX5_CAP_GEN(dev
->mdev
, umr_extended_translation_offset
))
360 caps
->general_caps
|= IB_ODP_SUPPORT_IMPLICIT
;
365 static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev
*dev
,
366 struct mlx5_pagefault
*pfault
,
369 int wq_num
= pfault
->event_subtype
== MLX5_PFAULT_SUBTYPE_WQE
?
370 pfault
->wqe
.wq_num
: pfault
->token
;
371 u32 out
[MLX5_ST_SZ_DW(page_fault_resume_out
)] = { };
372 u32 in
[MLX5_ST_SZ_DW(page_fault_resume_in
)] = { };
375 MLX5_SET(page_fault_resume_in
, in
, opcode
, MLX5_CMD_OP_PAGE_FAULT_RESUME
);
376 MLX5_SET(page_fault_resume_in
, in
, page_fault_type
, pfault
->type
);
377 MLX5_SET(page_fault_resume_in
, in
, token
, pfault
->token
);
378 MLX5_SET(page_fault_resume_in
, in
, wq_number
, wq_num
);
379 MLX5_SET(page_fault_resume_in
, in
, error
, !!error
);
381 err
= mlx5_cmd_exec(dev
->mdev
, in
, sizeof(in
), out
, sizeof(out
));
383 mlx5_ib_err(dev
, "Failed to resolve the page fault on WQ 0x%x err %d\n",
387 static struct mlx5_ib_mr
*implicit_mr_alloc(struct ib_pd
*pd
,
388 struct ib_umem
*umem
,
389 bool ksm
, int access_flags
)
391 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
392 struct mlx5_ib_mr
*mr
;
395 mr
= mlx5_mr_cache_alloc(dev
, ksm
? MLX5_IMR_KSM_CACHE_ENTRY
:
396 MLX5_IMR_MTT_CACHE_ENTRY
);
404 mr
->access_flags
= access_flags
;
409 err
= mlx5_ib_update_xlt(mr
, 0,
410 mlx5_imr_ksm_entries
,
412 MLX5_IB_UPD_XLT_INDIRECT
|
413 MLX5_IB_UPD_XLT_ZAP
|
414 MLX5_IB_UPD_XLT_ENABLE
);
417 err
= mlx5_ib_update_xlt(mr
, 0,
418 MLX5_IMR_MTT_ENTRIES
,
420 MLX5_IB_UPD_XLT_ZAP
|
421 MLX5_IB_UPD_XLT_ENABLE
|
422 MLX5_IB_UPD_XLT_ATOMIC
);
428 mr
->ibmr
.lkey
= mr
->mmkey
.key
;
429 mr
->ibmr
.rkey
= mr
->mmkey
.key
;
433 mlx5_ib_dbg(dev
, "key %x dev %p mr %p\n",
434 mr
->mmkey
.key
, dev
->mdev
, mr
);
439 mlx5_ib_err(dev
, "Failed to register MKEY %d\n", err
);
440 mlx5_mr_cache_free(dev
, mr
);
445 static struct ib_umem_odp
*implicit_mr_get_data(struct mlx5_ib_mr
*mr
,
446 u64 io_virt
, size_t bcnt
)
448 struct mlx5_ib_dev
*dev
= to_mdev(mr
->ibmr
.pd
->device
);
449 struct ib_umem_odp
*odp
, *result
= NULL
;
450 struct ib_umem_odp
*odp_mr
= to_ib_umem_odp(mr
->umem
);
451 u64 addr
= io_virt
& MLX5_IMR_MTT_MASK
;
452 int nentries
= 0, start_idx
= 0, ret
;
453 struct mlx5_ib_mr
*mtt
;
455 mutex_lock(&odp_mr
->umem_mutex
);
456 odp
= odp_lookup(addr
, 1, mr
);
458 mlx5_ib_dbg(dev
, "io_virt:%llx bcnt:%zx addr:%llx odp:%p\n",
459 io_virt
, bcnt
, addr
, odp
);
466 odp
= ib_alloc_odp_umem(odp_mr
, addr
,
469 mutex_unlock(&odp_mr
->umem_mutex
);
470 return ERR_CAST(odp
);
473 mtt
= implicit_mr_alloc(mr
->ibmr
.pd
, &odp
->umem
, 0,
476 mutex_unlock(&odp_mr
->umem_mutex
);
477 ib_umem_release(&odp
->umem
);
478 return ERR_CAST(mtt
);
482 mtt
->umem
= &odp
->umem
;
483 mtt
->mmkey
.iova
= addr
;
485 INIT_WORK(&odp
->work
, mr_leaf_free_action
);
488 start_idx
= addr
>> MLX5_IMR_MTT_SHIFT
;
492 /* Return first odp if region not covered by single one */
496 addr
+= MLX5_IMR_MTT_SIZE
;
497 if (unlikely(addr
< io_virt
+ bcnt
)) {
499 if (odp
&& odp
->umem
.address
!= addr
)
504 if (unlikely(nentries
)) {
505 ret
= mlx5_ib_update_xlt(mr
, start_idx
, nentries
, 0,
506 MLX5_IB_UPD_XLT_INDIRECT
|
507 MLX5_IB_UPD_XLT_ATOMIC
);
509 mlx5_ib_err(dev
, "Failed to update PAS\n");
510 result
= ERR_PTR(ret
);
514 mutex_unlock(&odp_mr
->umem_mutex
);
518 struct mlx5_ib_mr
*mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd
*pd
,
519 struct ib_udata
*udata
,
522 struct mlx5_ib_mr
*imr
;
523 struct ib_umem
*umem
;
525 umem
= ib_umem_get(udata
, 0, 0, access_flags
, 0);
527 return ERR_CAST(umem
);
529 imr
= implicit_mr_alloc(&pd
->ibpd
, umem
, 1, access_flags
);
531 ib_umem_release(umem
);
532 return ERR_CAST(imr
);
536 init_waitqueue_head(&imr
->q_leaf_free
);
537 atomic_set(&imr
->num_leaf_free
, 0);
538 atomic_set(&imr
->num_pending_prefetch
, 0);
543 static int mr_leaf_free(struct ib_umem_odp
*umem_odp
, u64 start
, u64 end
,
546 struct mlx5_ib_mr
*mr
= umem_odp
->private, *imr
= cookie
;
547 struct ib_umem
*umem
= &umem_odp
->umem
;
549 if (mr
->parent
!= imr
)
552 ib_umem_odp_unmap_dma_pages(umem_odp
, ib_umem_start(umem
),
558 WRITE_ONCE(umem_odp
->dying
, 1);
559 atomic_inc(&imr
->num_leaf_free
);
560 schedule_work(&umem_odp
->work
);
565 void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr
*imr
)
567 struct ib_ucontext_per_mm
*per_mm
= mr_to_per_mm(imr
);
569 down_read(&per_mm
->umem_rwsem
);
570 rbt_ib_umem_for_each_in_range(&per_mm
->umem_tree
, 0, ULLONG_MAX
,
571 mr_leaf_free
, true, imr
);
572 up_read(&per_mm
->umem_rwsem
);
574 wait_event(imr
->q_leaf_free
, !atomic_read(&imr
->num_leaf_free
));
577 #define MLX5_PF_FLAGS_PREFETCH BIT(0)
578 #define MLX5_PF_FLAGS_DOWNGRADE BIT(1)
579 static int pagefault_mr(struct mlx5_ib_dev
*dev
, struct mlx5_ib_mr
*mr
,
580 u64 io_virt
, size_t bcnt
, u32
*bytes_mapped
,
583 int npages
= 0, current_seq
, page_shift
, ret
, np
;
584 bool implicit
= false;
585 struct ib_umem_odp
*odp_mr
= to_ib_umem_odp(mr
->umem
);
586 bool downgrade
= flags
& MLX5_PF_FLAGS_DOWNGRADE
;
587 bool prefetch
= flags
& MLX5_PF_FLAGS_PREFETCH
;
589 u64 start_idx
, page_mask
;
590 struct ib_umem_odp
*odp
;
593 if (!odp_mr
->page_list
) {
594 odp
= implicit_mr_get_data(mr
, io_virt
, bcnt
);
605 size
= min_t(size_t, bcnt
, ib_umem_end(&odp
->umem
) - io_virt
);
607 page_shift
= mr
->umem
->page_shift
;
608 page_mask
= ~(BIT(page_shift
) - 1);
609 start_idx
= (io_virt
- (mr
->mmkey
.iova
& page_mask
)) >> page_shift
;
610 access_mask
= ODP_READ_ALLOWED_BIT
;
612 if (prefetch
&& !downgrade
&& !mr
->umem
->writable
) {
613 /* prefetch with write-access must
614 * be supported by the MR
620 if (mr
->umem
->writable
&& !downgrade
)
621 access_mask
|= ODP_WRITE_ALLOWED_BIT
;
623 current_seq
= READ_ONCE(odp
->notifiers_seq
);
625 * Ensure the sequence number is valid for some time before we call
630 ret
= ib_umem_odp_map_dma_pages(to_ib_umem_odp(mr
->umem
), io_virt
, size
,
631 access_mask
, current_seq
);
638 mutex_lock(&odp
->umem_mutex
);
639 if (!ib_umem_mmu_notifier_retry(to_ib_umem_odp(mr
->umem
),
642 * No need to check whether the MTTs really belong to
643 * this MR, since ib_umem_odp_map_dma_pages already
646 ret
= mlx5_ib_update_xlt(mr
, start_idx
, np
,
647 page_shift
, MLX5_IB_UPD_XLT_ATOMIC
);
651 mutex_unlock(&odp
->umem_mutex
);
655 mlx5_ib_err(dev
, "Failed to update mkey page tables\n");
660 u32 new_mappings
= (np
<< page_shift
) -
661 (io_virt
- round_down(io_virt
, 1 << page_shift
));
662 *bytes_mapped
+= min_t(u32
, new_mappings
, size
);
665 npages
+= np
<< (page_shift
- PAGE_SHIFT
);
668 if (unlikely(bcnt
)) {
669 struct ib_umem_odp
*next
;
672 next
= odp_next(odp
);
673 if (unlikely(!next
|| next
->umem
.address
!= io_virt
)) {
674 mlx5_ib_dbg(dev
, "next implicit leaf removed at 0x%llx. got %p\n",
686 if (ret
== -EAGAIN
) {
687 if (implicit
|| !odp
->dying
) {
688 unsigned long timeout
=
689 msecs_to_jiffies(MMU_NOTIFIER_TIMEOUT
);
691 if (!wait_for_completion_timeout(
692 &odp
->notifier_completion
,
694 mlx5_ib_warn(dev
, "timeout waiting for mmu notifier. seq %d against %d. notifiers_count=%d\n",
695 current_seq
, odp
->notifiers_seq
, odp
->notifiers_count
);
698 /* The MR is being killed, kill the QP as well. */
707 struct pf_frame
*next
;
714 static int get_indirect_num_descs(struct mlx5_core_mkey
*mmkey
)
716 struct mlx5_ib_mw
*mw
;
717 struct mlx5_ib_devx_mr
*devx_mr
;
719 if (mmkey
->type
== MLX5_MKEY_MW
) {
720 mw
= container_of(mmkey
, struct mlx5_ib_mw
, mmkey
);
724 devx_mr
= container_of(mmkey
, struct mlx5_ib_devx_mr
,
726 return devx_mr
->ndescs
;
730 * Handle a single data segment in a page-fault WQE or RDMA region.
732 * Returns number of OS pages retrieved on success. The caller may continue to
733 * the next data segment.
734 * Can return the following error codes:
735 * -EAGAIN to designate a temporary error. The caller will abort handling the
736 * page fault and resolve it.
737 * -EFAULT when there's an error mapping the requested pages. The caller will
738 * abort the page fault handling.
740 static int pagefault_single_data_segment(struct mlx5_ib_dev
*dev
,
741 struct ib_pd
*pd
, u32 key
,
742 u64 io_virt
, size_t bcnt
,
743 u32
*bytes_committed
,
744 u32
*bytes_mapped
, u32 flags
)
746 int npages
= 0, srcu_key
, ret
, i
, outlen
, cur_outlen
= 0, depth
= 0;
747 bool prefetch
= flags
& MLX5_PF_FLAGS_PREFETCH
;
748 struct pf_frame
*head
= NULL
, *frame
;
749 struct mlx5_core_mkey
*mmkey
;
750 struct mlx5_ib_mr
*mr
;
751 struct mlx5_klm
*pklm
;
756 srcu_key
= srcu_read_lock(&dev
->mr_srcu
);
758 io_virt
+= *bytes_committed
;
759 bcnt
-= *bytes_committed
;
762 mmkey
= __mlx5_mr_lookup(dev
->mdev
, mlx5_base_mkey(key
));
763 if (!mmkey
|| mmkey
->key
!= key
) {
764 mlx5_ib_dbg(dev
, "failed to find mkey %x\n", key
);
769 if (prefetch
&& mmkey
->type
!= MLX5_MKEY_MR
) {
770 mlx5_ib_dbg(dev
, "prefetch is allowed only for MR\n");
775 switch (mmkey
->type
) {
777 mr
= container_of(mmkey
, struct mlx5_ib_mr
, mmkey
);
778 if (!mr
->live
|| !mr
->ibmr
.pd
) {
779 mlx5_ib_dbg(dev
, "got dead MR\n");
785 if (!is_odp_mr(mr
) ||
787 mlx5_ib_dbg(dev
, "Invalid prefetch request: %s\n",
788 is_odp_mr(mr
) ? "MR is not ODP" :
789 "PD is not of the MR");
795 if (!is_odp_mr(mr
)) {
796 mlx5_ib_dbg(dev
, "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n",
799 *bytes_mapped
+= bcnt
;
804 ret
= pagefault_mr(dev
, mr
, io_virt
, bcnt
, bytes_mapped
, flags
);
813 case MLX5_MKEY_INDIRECT_DEVX
:
814 ndescs
= get_indirect_num_descs(mmkey
);
816 if (depth
>= MLX5_CAP_GEN(dev
->mdev
, max_indirection
)) {
817 mlx5_ib_dbg(dev
, "indirection level exceeded\n");
822 outlen
= MLX5_ST_SZ_BYTES(query_mkey_out
) +
823 sizeof(*pklm
) * (ndescs
- 2);
825 if (outlen
> cur_outlen
) {
827 out
= kzalloc(outlen
, GFP_KERNEL
);
835 pklm
= (struct mlx5_klm
*)MLX5_ADDR_OF(query_mkey_out
, out
,
836 bsf0_klm0_pas_mtt0_1
);
838 ret
= mlx5_core_query_mkey(dev
->mdev
, mmkey
, out
, outlen
);
842 offset
= io_virt
- MLX5_GET64(query_mkey_out
, out
,
843 memory_key_mkey_entry
.start_addr
);
845 for (i
= 0; bcnt
&& i
< ndescs
; i
++, pklm
++) {
846 if (offset
>= be32_to_cpu(pklm
->bcount
)) {
847 offset
-= be32_to_cpu(pklm
->bcount
);
851 frame
= kzalloc(sizeof(*frame
), GFP_KERNEL
);
857 frame
->key
= be32_to_cpu(pklm
->key
);
858 frame
->io_virt
= be64_to_cpu(pklm
->va
) + offset
;
859 frame
->bcnt
= min_t(size_t, bcnt
,
860 be32_to_cpu(pklm
->bcount
) - offset
);
861 frame
->depth
= depth
+ 1;
871 mlx5_ib_dbg(dev
, "wrong mkey type %d\n", mmkey
->type
);
881 io_virt
= frame
->io_virt
;
883 depth
= frame
->depth
;
897 srcu_read_unlock(&dev
->mr_srcu
, srcu_key
);
898 *bytes_committed
= 0;
899 return ret
? ret
: npages
;
903 * Parse a series of data segments for page fault handling.
905 * @pfault contains page fault information.
906 * @wqe points at the first data segment in the WQE.
907 * @wqe_end points after the end of the WQE.
908 * @bytes_mapped receives the number of bytes that the function was able to
909 * map. This allows the caller to decide intelligently whether
910 * enough memory was mapped to resolve the page fault
911 * successfully (e.g. enough for the next MTU, or the entire
913 * @total_wqe_bytes receives the total data size of this WQE in bytes (minus
914 * the committed bytes).
916 * Returns the number of pages loaded if positive, zero for an empty WQE, or a
917 * negative error code.
919 static int pagefault_data_segments(struct mlx5_ib_dev
*dev
,
920 struct mlx5_pagefault
*pfault
,
922 void *wqe_end
, u32
*bytes_mapped
,
923 u32
*total_wqe_bytes
, int receive_queue
)
925 int ret
= 0, npages
= 0;
935 *total_wqe_bytes
= 0;
937 while (wqe
< wqe_end
) {
938 struct mlx5_wqe_data_seg
*dseg
= wqe
;
940 io_virt
= be64_to_cpu(dseg
->addr
);
941 key
= be32_to_cpu(dseg
->lkey
);
942 byte_count
= be32_to_cpu(dseg
->byte_count
);
943 inline_segment
= !!(byte_count
& MLX5_INLINE_SEG
);
944 bcnt
= byte_count
& ~MLX5_INLINE_SEG
;
946 if (inline_segment
) {
947 bcnt
= bcnt
& MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK
;
948 wqe
+= ALIGN(sizeof(struct mlx5_wqe_inline_seg
) + bcnt
,
951 wqe
+= sizeof(*dseg
);
954 /* receive WQE end of sg list. */
955 if (receive_queue
&& bcnt
== 0 && key
== MLX5_INVALID_LKEY
&&
959 if (!inline_segment
&& total_wqe_bytes
) {
960 *total_wqe_bytes
+= bcnt
- min_t(size_t, bcnt
,
961 pfault
->bytes_committed
);
964 /* A zero length data segment designates a length of 2GB. */
968 if (inline_segment
|| bcnt
<= pfault
->bytes_committed
) {
969 pfault
->bytes_committed
-=
971 pfault
->bytes_committed
);
975 ret
= pagefault_single_data_segment(dev
, NULL
, key
,
977 &pfault
->bytes_committed
,
984 return ret
< 0 ? ret
: npages
;
987 static const u32 mlx5_ib_odp_opcode_cap
[] = {
988 [MLX5_OPCODE_SEND
] = IB_ODP_SUPPORT_SEND
,
989 [MLX5_OPCODE_SEND_IMM
] = IB_ODP_SUPPORT_SEND
,
990 [MLX5_OPCODE_SEND_INVAL
] = IB_ODP_SUPPORT_SEND
,
991 [MLX5_OPCODE_RDMA_WRITE
] = IB_ODP_SUPPORT_WRITE
,
992 [MLX5_OPCODE_RDMA_WRITE_IMM
] = IB_ODP_SUPPORT_WRITE
,
993 [MLX5_OPCODE_RDMA_READ
] = IB_ODP_SUPPORT_READ
,
994 [MLX5_OPCODE_ATOMIC_CS
] = IB_ODP_SUPPORT_ATOMIC
,
995 [MLX5_OPCODE_ATOMIC_FA
] = IB_ODP_SUPPORT_ATOMIC
,
999 * Parse initiator WQE. Advances the wqe pointer to point at the
1000 * scatter-gather list, and set wqe_end to the end of the WQE.
1002 static int mlx5_ib_mr_initiator_pfault_handler(
1003 struct mlx5_ib_dev
*dev
, struct mlx5_pagefault
*pfault
,
1004 struct mlx5_ib_qp
*qp
, void **wqe
, void **wqe_end
, int wqe_length
)
1006 struct mlx5_wqe_ctrl_seg
*ctrl
= *wqe
;
1007 u16 wqe_index
= pfault
->wqe
.wqe_index
;
1009 struct mlx5_base_av
*av
;
1010 unsigned ds
, opcode
;
1012 u32 ctrl_wqe_index
, ctrl_qpn
;
1014 u32 qpn
= qp
->trans_qp
.base
.mqp
.qpn
;
1016 ds
= be32_to_cpu(ctrl
->qpn_ds
) & MLX5_WQE_CTRL_DS_MASK
;
1017 if (ds
* MLX5_WQE_DS_UNITS
> wqe_length
) {
1018 mlx5_ib_err(dev
, "Unable to read the complete WQE. ds = 0x%x, ret = 0x%x\n",
1024 mlx5_ib_err(dev
, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n",
1030 ctrl_wqe_index
= (be32_to_cpu(ctrl
->opmod_idx_opcode
) &
1031 MLX5_WQE_CTRL_WQE_INDEX_MASK
) >>
1032 MLX5_WQE_CTRL_WQE_INDEX_SHIFT
;
1033 if (wqe_index
!= ctrl_wqe_index
) {
1034 mlx5_ib_err(dev
, "Got WQE with invalid wqe_index. wqe_index=0x%x, qpn=0x%x ctrl->wqe_index=0x%x\n",
1040 ctrl_qpn
= (be32_to_cpu(ctrl
->qpn_ds
) & MLX5_WQE_CTRL_QPN_MASK
) >>
1041 MLX5_WQE_CTRL_QPN_SHIFT
;
1042 if (qpn
!= ctrl_qpn
) {
1043 mlx5_ib_err(dev
, "Got WQE with incorrect QP number. wqe_index=0x%x, qpn=0x%x ctrl->qpn=0x%x\n",
1050 *wqe_end
= *wqe
+ ds
* MLX5_WQE_DS_UNITS
;
1051 *wqe
+= sizeof(*ctrl
);
1053 opcode
= be32_to_cpu(ctrl
->opmod_idx_opcode
) &
1054 MLX5_WQE_CTRL_OPCODE_MASK
;
1056 switch (qp
->ibqp
.qp_type
) {
1057 case IB_QPT_XRC_INI
:
1058 *wqe
+= sizeof(struct mlx5_wqe_xrc_seg
);
1059 transport_caps
= dev
->odp_caps
.per_transport_caps
.xrc_odp_caps
;
1062 transport_caps
= dev
->odp_caps
.per_transport_caps
.rc_odp_caps
;
1065 transport_caps
= dev
->odp_caps
.per_transport_caps
.ud_odp_caps
;
1068 mlx5_ib_err(dev
, "ODP fault on QP of an unsupported transport 0x%x\n",
1073 if (unlikely(opcode
>= ARRAY_SIZE(mlx5_ib_odp_opcode_cap
) ||
1074 !(transport_caps
& mlx5_ib_odp_opcode_cap
[opcode
]))) {
1075 mlx5_ib_err(dev
, "ODP fault on QP of an unsupported opcode 0x%x\n",
1080 if (qp
->ibqp
.qp_type
== IB_QPT_UD
) {
1082 if (av
->dqp_dct
& cpu_to_be32(MLX5_EXTENDED_UD_AV
))
1083 *wqe
+= sizeof(struct mlx5_av
);
1085 *wqe
+= sizeof(struct mlx5_base_av
);
1089 case MLX5_OPCODE_RDMA_WRITE
:
1090 case MLX5_OPCODE_RDMA_WRITE_IMM
:
1091 case MLX5_OPCODE_RDMA_READ
:
1092 *wqe
+= sizeof(struct mlx5_wqe_raddr_seg
);
1094 case MLX5_OPCODE_ATOMIC_CS
:
1095 case MLX5_OPCODE_ATOMIC_FA
:
1096 *wqe
+= sizeof(struct mlx5_wqe_raddr_seg
);
1097 *wqe
+= sizeof(struct mlx5_wqe_atomic_seg
);
1105 * Parse responder WQE and set wqe_end to the end of the WQE.
1107 static int mlx5_ib_mr_responder_pfault_handler_srq(struct mlx5_ib_dev
*dev
,
1108 struct mlx5_ib_srq
*srq
,
1109 void **wqe
, void **wqe_end
,
1112 int wqe_size
= 1 << srq
->msrq
.wqe_shift
;
1114 if (wqe_size
> wqe_length
) {
1115 mlx5_ib_err(dev
, "Couldn't read all of the receive WQE's content\n");
1119 *wqe_end
= *wqe
+ wqe_size
;
1120 *wqe
+= sizeof(struct mlx5_wqe_srq_next_seg
);
1125 static int mlx5_ib_mr_responder_pfault_handler_rq(struct mlx5_ib_dev
*dev
,
1126 struct mlx5_ib_qp
*qp
,
1127 void *wqe
, void **wqe_end
,
1130 struct mlx5_ib_wq
*wq
= &qp
->rq
;
1131 int wqe_size
= 1 << wq
->wqe_shift
;
1134 mlx5_ib_err(dev
, "ODP fault with WQE signatures is not supported\n");
1138 if (wqe_size
> wqe_length
) {
1139 mlx5_ib_err(dev
, "Couldn't read all of the receive WQE's content\n");
1143 switch (qp
->ibqp
.qp_type
) {
1145 if (!(dev
->odp_caps
.per_transport_caps
.rc_odp_caps
&
1146 IB_ODP_SUPPORT_RECV
))
1147 goto invalid_transport_or_opcode
;
1150 invalid_transport_or_opcode
:
1151 mlx5_ib_err(dev
, "ODP fault on QP of an unsupported transport. transport: 0x%x\n",
1156 *wqe_end
= wqe
+ wqe_size
;
1161 static inline struct mlx5_core_rsc_common
*odp_get_rsc(struct mlx5_ib_dev
*dev
,
1162 u32 wq_num
, int pf_type
)
1164 struct mlx5_core_rsc_common
*common
= NULL
;
1165 struct mlx5_core_srq
*srq
;
1168 case MLX5_WQE_PF_TYPE_RMP
:
1169 srq
= mlx5_cmd_get_srq(dev
, wq_num
);
1171 common
= &srq
->common
;
1173 case MLX5_WQE_PF_TYPE_REQ_SEND_OR_WRITE
:
1174 case MLX5_WQE_PF_TYPE_RESP
:
1175 case MLX5_WQE_PF_TYPE_REQ_READ_OR_ATOMIC
:
1176 common
= mlx5_core_res_hold(dev
->mdev
, wq_num
, MLX5_RES_QP
);
1185 static inline struct mlx5_ib_qp
*res_to_qp(struct mlx5_core_rsc_common
*res
)
1187 struct mlx5_core_qp
*mqp
= (struct mlx5_core_qp
*)res
;
1189 return to_mibqp(mqp
);
1192 static inline struct mlx5_ib_srq
*res_to_srq(struct mlx5_core_rsc_common
*res
)
1194 struct mlx5_core_srq
*msrq
=
1195 container_of(res
, struct mlx5_core_srq
, common
);
1197 return to_mibsrq(msrq
);
1200 static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev
*dev
,
1201 struct mlx5_pagefault
*pfault
)
1204 void *wqe
, *wqe_end
;
1205 u32 bytes_mapped
, total_wqe_bytes
;
1206 char *buffer
= NULL
;
1207 int resume_with_error
= 1;
1208 u16 wqe_index
= pfault
->wqe
.wqe_index
;
1209 int requestor
= pfault
->type
& MLX5_PFAULT_REQUESTOR
;
1210 struct mlx5_core_rsc_common
*res
= NULL
;
1211 struct mlx5_ib_qp
*qp
= NULL
;
1212 struct mlx5_ib_srq
*srq
= NULL
;
1213 size_t bytes_copied
;
1215 res
= odp_get_rsc(dev
, pfault
->wqe
.wq_num
, pfault
->type
);
1217 mlx5_ib_dbg(dev
, "wqe page fault for missing resource %d\n", pfault
->wqe
.wq_num
);
1223 qp
= res_to_qp(res
);
1227 srq
= res_to_srq(res
);
1230 mlx5_ib_err(dev
, "wqe page fault for unsupported type %d\n", pfault
->type
);
1231 goto resolve_page_fault
;
1234 buffer
= (char *)__get_free_page(GFP_KERNEL
);
1236 mlx5_ib_err(dev
, "Error allocating memory for IO page fault handling.\n");
1237 goto resolve_page_fault
;
1242 ret
= mlx5_ib_read_user_wqe_sq(qp
, wqe_index
,
1246 ret
= mlx5_ib_read_user_wqe_rq(qp
, wqe_index
,
1251 ret
= mlx5_ib_read_user_wqe_srq(srq
, wqe_index
,
1257 mlx5_ib_err(dev
, "Failed reading a WQE following page fault, error=%d, wqe_index=%x, qpn=%x\n",
1258 ret
, wqe_index
, pfault
->token
);
1259 goto resolve_page_fault
;
1264 ret
= mlx5_ib_mr_initiator_pfault_handler(dev
, pfault
, qp
,
1268 ret
= mlx5_ib_mr_responder_pfault_handler_rq(dev
, qp
,
1272 ret
= mlx5_ib_mr_responder_pfault_handler_srq(dev
, srq
,
1277 goto resolve_page_fault
;
1279 if (wqe
>= wqe_end
) {
1280 mlx5_ib_err(dev
, "ODP fault on invalid WQE.\n");
1281 goto resolve_page_fault
;
1284 ret
= pagefault_data_segments(dev
, pfault
, wqe
, wqe_end
,
1285 &bytes_mapped
, &total_wqe_bytes
,
1287 if (ret
== -EAGAIN
) {
1288 resume_with_error
= 0;
1289 goto resolve_page_fault
;
1290 } else if (ret
< 0 || total_wqe_bytes
> bytes_mapped
) {
1291 goto resolve_page_fault
;
1294 resume_with_error
= 0;
1296 mlx5_ib_page_fault_resume(dev
, pfault
, resume_with_error
);
1297 mlx5_ib_dbg(dev
, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n",
1298 pfault
->wqe
.wq_num
, resume_with_error
,
1300 mlx5_core_res_put(res
);
1301 free_page((unsigned long)buffer
);
1304 static int pages_in_range(u64 address
, u32 length
)
1306 return (ALIGN(address
+ length
, PAGE_SIZE
) -
1307 (address
& PAGE_MASK
)) >> PAGE_SHIFT
;
1310 static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev
*dev
,
1311 struct mlx5_pagefault
*pfault
)
1315 u32 prefetch_len
= pfault
->bytes_committed
;
1316 int prefetch_activated
= 0;
1317 u32 rkey
= pfault
->rdma
.r_key
;
1320 /* The RDMA responder handler handles the page fault in two parts.
1321 * First it brings the necessary pages for the current packet
1322 * (and uses the pfault context), and then (after resuming the QP)
1323 * prefetches more pages. The second operation cannot use the pfault
1324 * context and therefore uses the dummy_pfault context allocated on
1326 pfault
->rdma
.rdma_va
+= pfault
->bytes_committed
;
1327 pfault
->rdma
.rdma_op_len
-= min(pfault
->bytes_committed
,
1328 pfault
->rdma
.rdma_op_len
);
1329 pfault
->bytes_committed
= 0;
1331 address
= pfault
->rdma
.rdma_va
;
1332 length
= pfault
->rdma
.rdma_op_len
;
1334 /* For some operations, the hardware cannot tell the exact message
1335 * length, and in those cases it reports zero. Use prefetch
1338 prefetch_activated
= 1;
1339 length
= pfault
->rdma
.packet_size
;
1340 prefetch_len
= min(MAX_PREFETCH_LEN
, prefetch_len
);
1343 ret
= pagefault_single_data_segment(dev
, NULL
, rkey
, address
, length
,
1344 &pfault
->bytes_committed
, NULL
,
1346 if (ret
== -EAGAIN
) {
1347 /* We're racing with an invalidation, don't prefetch */
1348 prefetch_activated
= 0;
1349 } else if (ret
< 0 || pages_in_range(address
, length
) > ret
) {
1350 mlx5_ib_page_fault_resume(dev
, pfault
, 1);
1352 mlx5_ib_dbg(dev
, "PAGE FAULT error %d. QP 0x%x, type: 0x%x\n",
1353 ret
, pfault
->token
, pfault
->type
);
1357 mlx5_ib_page_fault_resume(dev
, pfault
, 0);
1358 mlx5_ib_dbg(dev
, "PAGE FAULT completed. QP 0x%x, type: 0x%x, prefetch_activated: %d\n",
1359 pfault
->token
, pfault
->type
,
1360 prefetch_activated
);
1362 /* At this point, there might be a new pagefault already arriving in
1363 * the eq, switch to the dummy pagefault for the rest of the
1364 * processing. We're still OK with the objects being alive as the
1365 * work-queue is being fenced. */
1367 if (prefetch_activated
) {
1368 u32 bytes_committed
= 0;
1370 ret
= pagefault_single_data_segment(dev
, NULL
, rkey
, address
,
1372 &bytes_committed
, NULL
,
1374 if (ret
< 0 && ret
!= -EAGAIN
) {
1375 mlx5_ib_dbg(dev
, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n",
1376 ret
, pfault
->token
, address
, prefetch_len
);
1381 static void mlx5_ib_pfault(struct mlx5_ib_dev
*dev
, struct mlx5_pagefault
*pfault
)
1383 u8 event_subtype
= pfault
->event_subtype
;
1385 switch (event_subtype
) {
1386 case MLX5_PFAULT_SUBTYPE_WQE
:
1387 mlx5_ib_mr_wqe_pfault_handler(dev
, pfault
);
1389 case MLX5_PFAULT_SUBTYPE_RDMA
:
1390 mlx5_ib_mr_rdma_pfault_handler(dev
, pfault
);
1393 mlx5_ib_err(dev
, "Invalid page fault event subtype: 0x%x\n",
1395 mlx5_ib_page_fault_resume(dev
, pfault
, 1);
1399 static void mlx5_ib_eqe_pf_action(struct work_struct
*work
)
1401 struct mlx5_pagefault
*pfault
= container_of(work
,
1402 struct mlx5_pagefault
,
1404 struct mlx5_ib_pf_eq
*eq
= pfault
->eq
;
1406 mlx5_ib_pfault(eq
->dev
, pfault
);
1407 mempool_free(pfault
, eq
->pool
);
1410 static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq
*eq
)
1412 struct mlx5_eqe_page_fault
*pf_eqe
;
1413 struct mlx5_pagefault
*pfault
;
1414 struct mlx5_eqe
*eqe
;
1417 while ((eqe
= mlx5_eq_get_eqe(eq
->core
, cc
))) {
1418 pfault
= mempool_alloc(eq
->pool
, GFP_ATOMIC
);
1420 schedule_work(&eq
->work
);
1424 pf_eqe
= &eqe
->data
.page_fault
;
1425 pfault
->event_subtype
= eqe
->sub_type
;
1426 pfault
->bytes_committed
= be32_to_cpu(pf_eqe
->bytes_committed
);
1428 mlx5_ib_dbg(eq
->dev
,
1429 "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x\n",
1430 eqe
->sub_type
, pfault
->bytes_committed
);
1432 switch (eqe
->sub_type
) {
1433 case MLX5_PFAULT_SUBTYPE_RDMA
:
1434 /* RDMA based event */
1436 be32_to_cpu(pf_eqe
->rdma
.pftype_token
) >> 24;
1438 be32_to_cpu(pf_eqe
->rdma
.pftype_token
) &
1440 pfault
->rdma
.r_key
=
1441 be32_to_cpu(pf_eqe
->rdma
.r_key
);
1442 pfault
->rdma
.packet_size
=
1443 be16_to_cpu(pf_eqe
->rdma
.packet_length
);
1444 pfault
->rdma
.rdma_op_len
=
1445 be32_to_cpu(pf_eqe
->rdma
.rdma_op_len
);
1446 pfault
->rdma
.rdma_va
=
1447 be64_to_cpu(pf_eqe
->rdma
.rdma_va
);
1448 mlx5_ib_dbg(eq
->dev
,
1449 "PAGE_FAULT: type:0x%x, token: 0x%06x, r_key: 0x%08x\n",
1450 pfault
->type
, pfault
->token
,
1451 pfault
->rdma
.r_key
);
1452 mlx5_ib_dbg(eq
->dev
,
1453 "PAGE_FAULT: rdma_op_len: 0x%08x, rdma_va: 0x%016llx\n",
1454 pfault
->rdma
.rdma_op_len
,
1455 pfault
->rdma
.rdma_va
);
1458 case MLX5_PFAULT_SUBTYPE_WQE
:
1459 /* WQE based event */
1461 (be32_to_cpu(pf_eqe
->wqe
.pftype_wq
) >> 24) & 0x7;
1463 be32_to_cpu(pf_eqe
->wqe
.token
);
1464 pfault
->wqe
.wq_num
=
1465 be32_to_cpu(pf_eqe
->wqe
.pftype_wq
) &
1467 pfault
->wqe
.wqe_index
=
1468 be16_to_cpu(pf_eqe
->wqe
.wqe_index
);
1469 pfault
->wqe
.packet_size
=
1470 be16_to_cpu(pf_eqe
->wqe
.packet_length
);
1471 mlx5_ib_dbg(eq
->dev
,
1472 "PAGE_FAULT: type:0x%x, token: 0x%06x, wq_num: 0x%06x, wqe_index: 0x%04x\n",
1473 pfault
->type
, pfault
->token
,
1475 pfault
->wqe
.wqe_index
);
1479 mlx5_ib_warn(eq
->dev
,
1480 "Unsupported page fault event sub-type: 0x%02hhx\n",
1482 /* Unsupported page faults should still be
1483 * resolved by the page fault handler
1488 INIT_WORK(&pfault
->work
, mlx5_ib_eqe_pf_action
);
1489 queue_work(eq
->wq
, &pfault
->work
);
1491 cc
= mlx5_eq_update_cc(eq
->core
, ++cc
);
1494 mlx5_eq_update_ci(eq
->core
, cc
, 1);
1497 static irqreturn_t
mlx5_ib_eq_pf_int(int irq
, void *eq_ptr
)
1499 struct mlx5_ib_pf_eq
*eq
= eq_ptr
;
1500 unsigned long flags
;
1502 if (spin_trylock_irqsave(&eq
->lock
, flags
)) {
1503 mlx5_ib_eq_pf_process(eq
);
1504 spin_unlock_irqrestore(&eq
->lock
, flags
);
1506 schedule_work(&eq
->work
);
1512 /* mempool_refill() was proposed but unfortunately wasn't accepted
1513 * http://lkml.iu.edu/hypermail/linux/kernel/1512.1/05073.html
1516 static void mempool_refill(mempool_t
*pool
)
1518 while (pool
->curr_nr
< pool
->min_nr
)
1519 mempool_free(mempool_alloc(pool
, GFP_KERNEL
), pool
);
1522 static void mlx5_ib_eq_pf_action(struct work_struct
*work
)
1524 struct mlx5_ib_pf_eq
*eq
=
1525 container_of(work
, struct mlx5_ib_pf_eq
, work
);
1527 mempool_refill(eq
->pool
);
1529 spin_lock_irq(&eq
->lock
);
1530 mlx5_ib_eq_pf_process(eq
);
1531 spin_unlock_irq(&eq
->lock
);
1535 MLX5_IB_NUM_PF_EQE
= 0x1000,
1536 MLX5_IB_NUM_PF_DRAIN
= 64,
1540 mlx5_ib_create_pf_eq(struct mlx5_ib_dev
*dev
, struct mlx5_ib_pf_eq
*eq
)
1542 struct mlx5_eq_param param
= {};
1545 INIT_WORK(&eq
->work
, mlx5_ib_eq_pf_action
);
1546 spin_lock_init(&eq
->lock
);
1549 eq
->pool
= mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN
,
1550 sizeof(struct mlx5_pagefault
));
1554 eq
->wq
= alloc_workqueue("mlx5_ib_page_fault",
1555 WQ_HIGHPRI
| WQ_UNBOUND
| WQ_MEM_RECLAIM
,
1562 param
= (struct mlx5_eq_param
) {
1563 .index
= MLX5_EQ_PFAULT_IDX
,
1564 .mask
= 1 << MLX5_EVENT_TYPE_PAGE_FAULT
,
1565 .nent
= MLX5_IB_NUM_PF_EQE
,
1567 .handler
= mlx5_ib_eq_pf_int
1569 eq
->core
= mlx5_eq_create_generic(dev
->mdev
, "mlx5_ib_page_fault_eq", ¶m
);
1570 if (IS_ERR(eq
->core
)) {
1571 err
= PTR_ERR(eq
->core
);
1577 destroy_workqueue(eq
->wq
);
1579 mempool_destroy(eq
->pool
);
1584 mlx5_ib_destroy_pf_eq(struct mlx5_ib_dev
*dev
, struct mlx5_ib_pf_eq
*eq
)
1588 err
= mlx5_eq_destroy_generic(dev
->mdev
, eq
->core
);
1589 cancel_work_sync(&eq
->work
);
1590 destroy_workqueue(eq
->wq
);
1591 mempool_destroy(eq
->pool
);
1596 void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent
*ent
)
1598 if (!(ent
->dev
->odp_caps
.general_caps
& IB_ODP_SUPPORT_IMPLICIT
))
1601 switch (ent
->order
- 2) {
1602 case MLX5_IMR_MTT_CACHE_ENTRY
:
1603 ent
->page
= PAGE_SHIFT
;
1604 ent
->xlt
= MLX5_IMR_MTT_ENTRIES
*
1605 sizeof(struct mlx5_mtt
) /
1606 MLX5_IB_UMR_OCTOWORD
;
1607 ent
->access_mode
= MLX5_MKC_ACCESS_MODE_MTT
;
1611 case MLX5_IMR_KSM_CACHE_ENTRY
:
1612 ent
->page
= MLX5_KSM_PAGE_SHIFT
;
1613 ent
->xlt
= mlx5_imr_ksm_entries
*
1614 sizeof(struct mlx5_klm
) /
1615 MLX5_IB_UMR_OCTOWORD
;
1616 ent
->access_mode
= MLX5_MKC_ACCESS_MODE_KSM
;
1622 static const struct ib_device_ops mlx5_ib_dev_odp_ops
= {
1623 .advise_mr
= mlx5_ib_advise_mr
,
1626 int mlx5_ib_odp_init_one(struct mlx5_ib_dev
*dev
)
1630 if (dev
->odp_caps
.general_caps
& IB_ODP_SUPPORT
)
1631 ib_set_device_ops(&dev
->ib_dev
, &mlx5_ib_dev_odp_ops
);
1633 if (dev
->odp_caps
.general_caps
& IB_ODP_SUPPORT_IMPLICIT
) {
1634 ret
= mlx5_cmd_null_mkey(dev
->mdev
, &dev
->null_mkey
);
1636 mlx5_ib_err(dev
, "Error getting null_mkey %d\n", ret
);
1641 if (!MLX5_CAP_GEN(dev
->mdev
, pg
))
1644 ret
= mlx5_ib_create_pf_eq(dev
, &dev
->odp_pf_eq
);
1649 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev
*dev
)
1651 if (!MLX5_CAP_GEN(dev
->mdev
, pg
))
1654 mlx5_ib_destroy_pf_eq(dev
, &dev
->odp_pf_eq
);
1657 int mlx5_ib_odp_init(void)
1659 mlx5_imr_ksm_entries
= BIT_ULL(get_order(TASK_SIZE
) -
1665 struct prefetch_mr_work
{
1666 struct work_struct work
;
1670 struct ib_sge sg_list
[0];
1673 static void num_pending_prefetch_dec(struct mlx5_ib_dev
*dev
,
1674 struct ib_sge
*sg_list
, u32 num_sge
,
1680 srcu_key
= srcu_read_lock(&dev
->mr_srcu
);
1682 for (i
= from
; i
< num_sge
; ++i
) {
1683 struct mlx5_core_mkey
*mmkey
;
1684 struct mlx5_ib_mr
*mr
;
1686 mmkey
= __mlx5_mr_lookup(dev
->mdev
,
1687 mlx5_base_mkey(sg_list
[i
].lkey
));
1688 mr
= container_of(mmkey
, struct mlx5_ib_mr
, mmkey
);
1689 atomic_dec(&mr
->num_pending_prefetch
);
1692 srcu_read_unlock(&dev
->mr_srcu
, srcu_key
);
1695 static bool num_pending_prefetch_inc(struct ib_pd
*pd
,
1696 struct ib_sge
*sg_list
, u32 num_sge
)
1698 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
1702 for (i
= 0; i
< num_sge
; ++i
) {
1703 struct mlx5_core_mkey
*mmkey
;
1704 struct mlx5_ib_mr
*mr
;
1706 mmkey
= __mlx5_mr_lookup(dev
->mdev
,
1707 mlx5_base_mkey(sg_list
[i
].lkey
));
1708 if (!mmkey
|| mmkey
->key
!= sg_list
[i
].lkey
) {
1713 if (mmkey
->type
!= MLX5_MKEY_MR
) {
1718 mr
= container_of(mmkey
, struct mlx5_ib_mr
, mmkey
);
1720 if (mr
->ibmr
.pd
!= pd
) {
1730 atomic_inc(&mr
->num_pending_prefetch
);
1734 num_pending_prefetch_dec(dev
, sg_list
, i
, 0);
1739 static int mlx5_ib_prefetch_sg_list(struct ib_pd
*pd
, u32 pf_flags
,
1740 struct ib_sge
*sg_list
, u32 num_sge
)
1744 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
1746 for (i
= 0; i
< num_sge
; ++i
) {
1747 struct ib_sge
*sg
= &sg_list
[i
];
1748 int bytes_committed
= 0;
1750 ret
= pagefault_single_data_segment(dev
, pd
, sg
->lkey
, sg
->addr
,
1752 &bytes_committed
, NULL
,
1758 return ret
< 0 ? ret
: 0;
1761 static void mlx5_ib_prefetch_mr_work(struct work_struct
*work
)
1763 struct prefetch_mr_work
*w
=
1764 container_of(work
, struct prefetch_mr_work
, work
);
1766 if (ib_device_try_get(w
->pd
->device
)) {
1767 mlx5_ib_prefetch_sg_list(w
->pd
, w
->pf_flags
, w
->sg_list
,
1769 ib_device_put(w
->pd
->device
);
1772 num_pending_prefetch_dec(to_mdev(w
->pd
->device
), w
->sg_list
,
1777 int mlx5_ib_advise_mr_prefetch(struct ib_pd
*pd
,
1778 enum ib_uverbs_advise_mr_advice advice
,
1779 u32 flags
, struct ib_sge
*sg_list
, u32 num_sge
)
1781 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
1782 u32 pf_flags
= MLX5_PF_FLAGS_PREFETCH
;
1783 struct prefetch_mr_work
*work
;
1787 if (advice
== IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH
)
1788 pf_flags
|= MLX5_PF_FLAGS_DOWNGRADE
;
1790 if (flags
& IB_UVERBS_ADVISE_MR_FLAG_FLUSH
)
1791 return mlx5_ib_prefetch_sg_list(pd
, pf_flags
, sg_list
,
1794 work
= kvzalloc(struct_size(work
, sg_list
, num_sge
), GFP_KERNEL
);
1798 memcpy(work
->sg_list
, sg_list
, num_sge
* sizeof(struct ib_sge
));
1800 /* It is guaranteed that the pd when work is executed is the pd when
1801 * work was queued since pd can't be destroyed while it holds MRs and
1802 * destroying a MR leads to flushing the workquque
1805 work
->pf_flags
= pf_flags
;
1806 work
->num_sge
= num_sge
;
1808 INIT_WORK(&work
->work
, mlx5_ib_prefetch_mr_work
);
1810 srcu_key
= srcu_read_lock(&dev
->mr_srcu
);
1812 valid_req
= num_pending_prefetch_inc(pd
, sg_list
, num_sge
);
1814 queue_work(system_unbound_wq
, &work
->work
);
1818 srcu_read_unlock(&dev
->mr_srcu
, srcu_key
);
1820 return valid_req
? 0 : -EINVAL
;