]> git.ipfire.org Git - thirdparty/linux.git/blob - drivers/infiniband/hw/mlx5/mr.c
projects / thirdparty / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
x86/fpu/xstate: Restore supervisor states for signal return
[thirdparty/linux.git] / drivers / infiniband / hw / mlx5 / mr.c
1 /*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33
34 #include <linux/kref.h>
35 #include <linux/random.h>
36 #include <linux/debugfs.h>
37 #include <linux/export.h>
38 #include <linux/delay.h>
39 #include <rdma/ib_umem.h>
40 #include <rdma/ib_umem_odp.h>
41 #include <rdma/ib_verbs.h>
42 #include "mlx5_ib.h"
43
44 enum {
45 MAX_PENDING_REG_MR = 8,
46 };
47
48 #define MLX5_UMR_ALIGN 2048
49
50 static void
51 create_mkey_callback(int status, struct mlx5_async_work *context);
52
53 static void
54 assign_mkey_variant(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey,
55 u32 *in)
56 {
57 u8 key = atomic_inc_return(&dev->mkey_var);
58 void *mkc;
59
60 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
61 MLX5_SET(mkc, mkc, mkey_7_0, key);
62 mkey->key = key;
63 }
64
65 static int
66 mlx5_ib_create_mkey(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey,
67 u32 *in, int inlen)
68 {
69 assign_mkey_variant(dev, mkey, in);
70 return mlx5_core_create_mkey(dev->mdev, mkey, in, inlen);
71 }
72
73 static int
74 mlx5_ib_create_mkey_cb(struct mlx5_ib_dev *dev,
75 struct mlx5_core_mkey *mkey,
76 struct mlx5_async_ctx *async_ctx,
77 u32 *in, int inlen, u32 *out, int outlen,
78 struct mlx5_async_work *context)
79 {
80 MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
81 assign_mkey_variant(dev, mkey, in);
82 return mlx5_cmd_exec_cb(async_ctx, in, inlen, out, outlen,
83 create_mkey_callback, context);
84 }
85
86 static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
87 static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
88 static int mr_cache_max_order(struct mlx5_ib_dev *dev);
89 static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent);
90
91 static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev)
92 {
93 return !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled);
94 }
95
96 static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
97 {
98 WARN_ON(xa_load(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)));
99
100 return mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
101 }
102
103 static bool use_umr_mtt_update(struct mlx5_ib_mr *mr, u64 start, u64 length)
104 {
105 return ((u64)1 << mr->order) * MLX5_ADAPTER_PAGE_SIZE >=
106 length + (start & (MLX5_ADAPTER_PAGE_SIZE - 1));
107 }
108
109 static void create_mkey_callback(int status, struct mlx5_async_work *context)
110 {
111 struct mlx5_ib_mr *mr =
112 container_of(context, struct mlx5_ib_mr, cb_work);
113 struct mlx5_ib_dev *dev = mr->dev;
114 struct mlx5_cache_ent *ent = mr->cache_ent;
115 unsigned long flags;
116
117 if (status) {
118 mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status);
119 kfree(mr);
120 spin_lock_irqsave(&ent->lock, flags);
121 ent->pending--;
122 WRITE_ONCE(dev->fill_delay, 1);
123 spin_unlock_irqrestore(&ent->lock, flags);
124 mod_timer(&dev->delay_timer, jiffies + HZ);
125 return;
126 }
127
128 mr->mmkey.type = MLX5_MKEY_MR;
129 mr->mmkey.key |= mlx5_idx_to_mkey(
130 MLX5_GET(create_mkey_out, mr->out, mkey_index));
131
132 WRITE_ONCE(dev->cache.last_add, jiffies);
133
134 spin_lock_irqsave(&ent->lock, flags);
135 list_add_tail(&mr->list, &ent->head);
136 ent->available_mrs++;
137 ent->total_mrs++;
138 /* If we are doing fill_to_high_water then keep going. */
139 queue_adjust_cache_locked(ent);
140 ent->pending--;
141 spin_unlock_irqrestore(&ent->lock, flags);
142 }
143
144 static struct mlx5_ib_mr *alloc_cache_mr(struct mlx5_cache_ent *ent, void *mkc)
145 {
146 struct mlx5_ib_mr *mr;
147
148 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
149 if (!mr)
150 return NULL;
151 mr->order = ent->order;
152 mr->cache_ent = ent;
153 mr->dev = ent->dev;
154
155 MLX5_SET(mkc, mkc, free, 1);
156 MLX5_SET(mkc, mkc, umr_en, 1);
157 MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3);
158 MLX5_SET(mkc, mkc, access_mode_4_2, (ent->access_mode >> 2) & 0x7);
159
160 MLX5_SET(mkc, mkc, qpn, 0xffffff);
161 MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt);
162 MLX5_SET(mkc, mkc, log_page_size, ent->page);
163 return mr;
164 }
165
166 /* Asynchronously schedule new MRs to be populated in the cache. */
167 static int add_keys(struct mlx5_cache_ent *ent, unsigned int num)
168 {
169 size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
170 struct mlx5_ib_mr *mr;
171 void *mkc;
172 u32 *in;
173 int err = 0;
174 int i;
175
176 in = kzalloc(inlen, GFP_KERNEL);
177 if (!in)
178 return -ENOMEM;
179
180 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
181 for (i = 0; i < num; i++) {
182 mr = alloc_cache_mr(ent, mkc);
183 if (!mr) {
184 err = -ENOMEM;
185 break;
186 }
187 spin_lock_irq(&ent->lock);
188 if (ent->pending >= MAX_PENDING_REG_MR) {
189 err = -EAGAIN;
190 spin_unlock_irq(&ent->lock);
191 kfree(mr);
192 break;
193 }
194 ent->pending++;
195 spin_unlock_irq(&ent->lock);
196 err = mlx5_ib_create_mkey_cb(ent->dev, &mr->mmkey,
197 &ent->dev->async_ctx, in, inlen,
198 mr->out, sizeof(mr->out),
199 &mr->cb_work);
200 if (err) {
201 spin_lock_irq(&ent->lock);
202 ent->pending--;
203 spin_unlock_irq(&ent->lock);
204 mlx5_ib_warn(ent->dev, "create mkey failed %d\n", err);
205 kfree(mr);
206 break;
207 }
208 }
209
210 kfree(in);
211 return err;
212 }
213
214 /* Synchronously create a MR in the cache */
215 static struct mlx5_ib_mr *create_cache_mr(struct mlx5_cache_ent *ent)
216 {
217 size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
218 struct mlx5_ib_mr *mr;
219 void *mkc;
220 u32 *in;
221 int err;
222
223 in = kzalloc(inlen, GFP_KERNEL);
224 if (!in)
225 return ERR_PTR(-ENOMEM);
226 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
227
228 mr = alloc_cache_mr(ent, mkc);
229 if (!mr) {
230 err = -ENOMEM;
231 goto free_in;
232 }
233
234 err = mlx5_core_create_mkey(ent->dev->mdev, &mr->mmkey, in, inlen);
235 if (err)
236 goto free_mr;
237
238 mr->mmkey.type = MLX5_MKEY_MR;
239 WRITE_ONCE(ent->dev->cache.last_add, jiffies);
240 spin_lock_irq(&ent->lock);
241 ent->total_mrs++;
242 spin_unlock_irq(&ent->lock);
243 kfree(in);
244 return mr;
245 free_mr:
246 kfree(mr);
247 free_in:
248 kfree(in);
249 return ERR_PTR(err);
250 }
251
252 static void remove_cache_mr_locked(struct mlx5_cache_ent *ent)
253 {
254 struct mlx5_ib_mr *mr;
255
256 lockdep_assert_held(&ent->lock);
257 if (list_empty(&ent->head))
258 return;
259 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
260 list_del(&mr->list);
261 ent->available_mrs--;
262 ent->total_mrs--;
263 spin_unlock_irq(&ent->lock);
264 mlx5_core_destroy_mkey(ent->dev->mdev, &mr->mmkey);
265 kfree(mr);
266 spin_lock_irq(&ent->lock);
267 }
268
269 static int resize_available_mrs(struct mlx5_cache_ent *ent, unsigned int target,
270 bool limit_fill)
271 {
272 int err;
273
274 lockdep_assert_held(&ent->lock);
275
276 while (true) {
277 if (limit_fill)
278 target = ent->limit * 2;
279 if (target == ent->available_mrs + ent->pending)
280 return 0;
281 if (target > ent->available_mrs + ent->pending) {
282 u32 todo = target - (ent->available_mrs + ent->pending);
283
284 spin_unlock_irq(&ent->lock);
285 err = add_keys(ent, todo);
286 if (err == -EAGAIN)
287 usleep_range(3000, 5000);
288 spin_lock_irq(&ent->lock);
289 if (err) {
290 if (err != -EAGAIN)
291 return err;
292 } else
293 return 0;
294 } else {
295 remove_cache_mr_locked(ent);
296 }
297 }
298 }
299
300 static ssize_t size_write(struct file *filp, const char __user *buf,
301 size_t count, loff_t *pos)
302 {
303 struct mlx5_cache_ent *ent = filp->private_data;
304 u32 target;
305 int err;
306
307 err = kstrtou32_from_user(buf, count, 0, &target);
308 if (err)
309 return err;
310
311 /*
312 * Target is the new value of total_mrs the user requests, however we
313 * cannot free MRs that are in use. Compute the target value for
314 * available_mrs.
315 */
316 spin_lock_irq(&ent->lock);
317 if (target < ent->total_mrs - ent->available_mrs) {
318 err = -EINVAL;
319 goto err_unlock;
320 }
321 target = target - (ent->total_mrs - ent->available_mrs);
322 if (target < ent->limit || target > ent->limit*2) {
323 err = -EINVAL;
324 goto err_unlock;
325 }
326 err = resize_available_mrs(ent, target, false);
327 if (err)
328 goto err_unlock;
329 spin_unlock_irq(&ent->lock);
330
331 return count;
332
333 err_unlock:
334 spin_unlock_irq(&ent->lock);
335 return err;
336 }
337
338 static ssize_t size_read(struct file *filp, char __user *buf, size_t count,
339 loff_t *pos)
340 {
341 struct mlx5_cache_ent *ent = filp->private_data;
342 char lbuf[20];
343 int err;
344
345 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->total_mrs);
346 if (err < 0)
347 return err;
348
349 return simple_read_from_buffer(buf, count, pos, lbuf, err);
350 }
351
352 static const struct file_operations size_fops = {
353 .owner = THIS_MODULE,
354 .open = simple_open,
355 .write = size_write,
356 .read = size_read,
357 };
358
359 static ssize_t limit_write(struct file *filp, const char __user *buf,
360 size_t count, loff_t *pos)
361 {
362 struct mlx5_cache_ent *ent = filp->private_data;
363 u32 var;
364 int err;
365
366 err = kstrtou32_from_user(buf, count, 0, &var);
367 if (err)
368 return err;
369
370 /*
371 * Upon set we immediately fill the cache to high water mark implied by
372 * the limit.
373 */
374 spin_lock_irq(&ent->lock);
375 ent->limit = var;
376 err = resize_available_mrs(ent, 0, true);
377 spin_unlock_irq(&ent->lock);
378 if (err)
379 return err;
380 return count;
381 }
382
383 static ssize_t limit_read(struct file *filp, char __user *buf, size_t count,
384 loff_t *pos)
385 {
386 struct mlx5_cache_ent *ent = filp->private_data;
387 char lbuf[20];
388 int err;
389
390 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit);
391 if (err < 0)
392 return err;
393
394 return simple_read_from_buffer(buf, count, pos, lbuf, err);
395 }
396
397 static const struct file_operations limit_fops = {
398 .owner = THIS_MODULE,
399 .open = simple_open,
400 .write = limit_write,
401 .read = limit_read,
402 };
403
404 static bool someone_adding(struct mlx5_mr_cache *cache)
405 {
406 unsigned int i;
407
408 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
409 struct mlx5_cache_ent *ent = &cache->ent[i];
410 bool ret;
411
412 spin_lock_irq(&ent->lock);
413 ret = ent->available_mrs < ent->limit;
414 spin_unlock_irq(&ent->lock);
415 if (ret)
416 return true;
417 }
418 return false;
419 }
420
421 /*
422 * Check if the bucket is outside the high/low water mark and schedule an async
423 * update. The cache refill has hysteresis, once the low water mark is hit it is
424 * refilled up to the high mark.
425 */
426 static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent)
427 {
428 lockdep_assert_held(&ent->lock);
429
430 if (ent->disabled || READ_ONCE(ent->dev->fill_delay))
431 return;
432 if (ent->available_mrs < ent->limit) {
433 ent->fill_to_high_water = true;
434 queue_work(ent->dev->cache.wq, &ent->work);
435 } else if (ent->fill_to_high_water &&
436 ent->available_mrs + ent->pending < 2 * ent->limit) {
437 /*
438 * Once we start populating due to hitting a low water mark
439 * continue until we pass the high water mark.
440 */
441 queue_work(ent->dev->cache.wq, &ent->work);
442 } else if (ent->available_mrs == 2 * ent->limit) {
443 ent->fill_to_high_water = false;
444 } else if (ent->available_mrs > 2 * ent->limit) {
445 /* Queue deletion of excess entries */
446 ent->fill_to_high_water = false;
447 if (ent->pending)
448 queue_delayed_work(ent->dev->cache.wq, &ent->dwork,
449 msecs_to_jiffies(1000));
450 else
451 queue_work(ent->dev->cache.wq, &ent->work);
452 }
453 }
454
455 static void __cache_work_func(struct mlx5_cache_ent *ent)
456 {
457 struct mlx5_ib_dev *dev = ent->dev;
458 struct mlx5_mr_cache *cache = &dev->cache;
459 int err;
460
461 spin_lock_irq(&ent->lock);
462 if (ent->disabled)
463 goto out;
464
465 if (ent->fill_to_high_water &&
466 ent->available_mrs + ent->pending < 2 * ent->limit &&
467 !READ_ONCE(dev->fill_delay)) {
468 spin_unlock_irq(&ent->lock);
469 err = add_keys(ent, 1);
470 spin_lock_irq(&ent->lock);
471 if (ent->disabled)
472 goto out;
473 if (err) {
474 /*
475 * EAGAIN only happens if pending is positive, so we
476 * will be rescheduled from reg_mr_callback(). The only
477 * failure path here is ENOMEM.
478 */
479 if (err != -EAGAIN) {
480 mlx5_ib_warn(
481 dev,
482 "command failed order %d, err %d\n",
483 ent->order, err);
484 queue_delayed_work(cache->wq, &ent->dwork,
485 msecs_to_jiffies(1000));
486 }
487 }
488 } else if (ent->available_mrs > 2 * ent->limit) {
489 bool need_delay;
490
491 /*
492 * The remove_cache_mr() logic is performed as garbage
493 * collection task. Such task is intended to be run when no
494 * other active processes are running.
495 *
496 * The need_resched() will return TRUE if there are user tasks
497 * to be activated in near future.
498 *
499 * In such case, we don't execute remove_cache_mr() and postpone
500 * the garbage collection work to try to run in next cycle, in
501 * order to free CPU resources to other tasks.
502 */
503 spin_unlock_irq(&ent->lock);
504 need_delay = need_resched() || someone_adding(cache) ||
505 time_after(jiffies,
506 READ_ONCE(cache->last_add) + 300 * HZ);
507 spin_lock_irq(&ent->lock);
508 if (ent->disabled)
509 goto out;
510 if (need_delay)
511 queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ);
512 remove_cache_mr_locked(ent);
513 queue_adjust_cache_locked(ent);
514 }
515 out:
516 spin_unlock_irq(&ent->lock);
517 }
518
519 static void delayed_cache_work_func(struct work_struct *work)
520 {
521 struct mlx5_cache_ent *ent;
522
523 ent = container_of(work, struct mlx5_cache_ent, dwork.work);
524 __cache_work_func(ent);
525 }
526
527 static void cache_work_func(struct work_struct *work)
528 {
529 struct mlx5_cache_ent *ent;
530
531 ent = container_of(work, struct mlx5_cache_ent, work);
532 __cache_work_func(ent);
533 }
534
535 /* Allocate a special entry from the cache */
536 struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
537 unsigned int entry)
538 {
539 struct mlx5_mr_cache *cache = &dev->cache;
540 struct mlx5_cache_ent *ent;
541 struct mlx5_ib_mr *mr;
542
543 if (WARN_ON(entry <= MR_CACHE_LAST_STD_ENTRY ||
544 entry >= ARRAY_SIZE(cache->ent)))
545 return ERR_PTR(-EINVAL);
546
547 ent = &cache->ent[entry];
548 spin_lock_irq(&ent->lock);
549 if (list_empty(&ent->head)) {
550 spin_unlock_irq(&ent->lock);
551 mr = create_cache_mr(ent);
552 if (IS_ERR(mr))
553 return mr;
554 } else {
555 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
556 list_del(&mr->list);
557 ent->available_mrs--;
558 queue_adjust_cache_locked(ent);
559 spin_unlock_irq(&ent->lock);
560 }
561 return mr;
562 }
563
564 /* Return a MR already available in the cache */
565 static struct mlx5_ib_mr *get_cache_mr(struct mlx5_cache_ent *req_ent)
566 {
567 struct mlx5_ib_dev *dev = req_ent->dev;
568 struct mlx5_ib_mr *mr = NULL;
569 struct mlx5_cache_ent *ent = req_ent;
570
571 /* Try larger MR pools from the cache to satisfy the allocation */
572 for (; ent != &dev->cache.ent[MR_CACHE_LAST_STD_ENTRY + 1]; ent++) {
573 mlx5_ib_dbg(dev, "order %u, cache index %zu\n", ent->order,
574 ent - dev->cache.ent);
575
576 spin_lock_irq(&ent->lock);
577 if (!list_empty(&ent->head)) {
578 mr = list_first_entry(&ent->head, struct mlx5_ib_mr,
579 list);
580 list_del(&mr->list);
581 ent->available_mrs--;
582 queue_adjust_cache_locked(ent);
583 spin_unlock_irq(&ent->lock);
584 break;
585 }
586 queue_adjust_cache_locked(ent);
587 spin_unlock_irq(&ent->lock);
588 }
589
590 if (!mr)
591 req_ent->miss++;
592
593 return mr;
594 }
595
596 static void detach_mr_from_cache(struct mlx5_ib_mr *mr)
597 {
598 struct mlx5_cache_ent *ent = mr->cache_ent;
599
600 mr->cache_ent = NULL;
601 spin_lock_irq(&ent->lock);
602 ent->total_mrs--;
603 spin_unlock_irq(&ent->lock);
604 }
605
606 void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
607 {
608 struct mlx5_cache_ent *ent = mr->cache_ent;
609
610 if (!ent)
611 return;
612
613 if (mlx5_mr_cache_invalidate(mr)) {
614 detach_mr_from_cache(mr);
615 destroy_mkey(dev, mr);
616 return;
617 }
618
619 spin_lock_irq(&ent->lock);
620 list_add_tail(&mr->list, &ent->head);
621 ent->available_mrs++;
622 queue_adjust_cache_locked(ent);
623 spin_unlock_irq(&ent->lock);
624 }
625
626 static void clean_keys(struct mlx5_ib_dev *dev, int c)
627 {
628 struct mlx5_mr_cache *cache = &dev->cache;
629 struct mlx5_cache_ent *ent = &cache->ent[c];
630 struct mlx5_ib_mr *tmp_mr;
631 struct mlx5_ib_mr *mr;
632 LIST_HEAD(del_list);
633
634 cancel_delayed_work(&ent->dwork);
635 while (1) {
636 spin_lock_irq(&ent->lock);
637 if (list_empty(&ent->head)) {
638 spin_unlock_irq(&ent->lock);
639 break;
640 }
641 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
642 list_move(&mr->list, &del_list);
643 ent->available_mrs--;
644 ent->total_mrs--;
645 spin_unlock_irq(&ent->lock);
646 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
647 }
648
649 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) {
650 list_del(&mr->list);
651 kfree(mr);
652 }
653 }
654
655 static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)
656 {
657 if (!mlx5_debugfs_root || dev->is_rep)
658 return;
659
660 debugfs_remove_recursive(dev->cache.root);
661 dev->cache.root = NULL;
662 }
663
664 static void mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev)
665 {
666 struct mlx5_mr_cache *cache = &dev->cache;
667 struct mlx5_cache_ent *ent;
668 struct dentry *dir;
669 int i;
670
671 if (!mlx5_debugfs_root || dev->is_rep)
672 return;
673
674 cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root);
675
676 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
677 ent = &cache->ent[i];
678 sprintf(ent->name, "%d", ent->order);
679 dir = debugfs_create_dir(ent->name, cache->root);
680 debugfs_create_file("size", 0600, dir, ent, &size_fops);
681 debugfs_create_file("limit", 0600, dir, ent, &limit_fops);
682 debugfs_create_u32("cur", 0400, dir, &ent->available_mrs);
683 debugfs_create_u32("miss", 0600, dir, &ent->miss);
684 }
685 }
686
687 static void delay_time_func(struct timer_list *t)
688 {
689 struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer);
690
691 WRITE_ONCE(dev->fill_delay, 0);
692 }
693
694 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
695 {
696 struct mlx5_mr_cache *cache = &dev->cache;
697 struct mlx5_cache_ent *ent;
698 int i;
699
700 mutex_init(&dev->slow_path_mutex);
701 cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);
702 if (!cache->wq) {
703 mlx5_ib_warn(dev, "failed to create work queue\n");
704 return -ENOMEM;
705 }
706
707 mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx);
708 timer_setup(&dev->delay_timer, delay_time_func, 0);
709 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
710 ent = &cache->ent[i];
711 INIT_LIST_HEAD(&ent->head);
712 spin_lock_init(&ent->lock);
713 ent->order = i + 2;
714 ent->dev = dev;
715 ent->limit = 0;
716
717 INIT_WORK(&ent->work, cache_work_func);
718 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
719
720 if (i > MR_CACHE_LAST_STD_ENTRY) {
721 mlx5_odp_init_mr_cache_entry(ent);
722 continue;
723 }
724
725 if (ent->order > mr_cache_max_order(dev))
726 continue;
727
728 ent->page = PAGE_SHIFT;
729 ent->xlt = (1 << ent->order) * sizeof(struct mlx5_mtt) /
730 MLX5_IB_UMR_OCTOWORD;
731 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
732 if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) &&
733 !dev->is_rep &&
734 mlx5_core_is_pf(dev->mdev))
735 ent->limit = dev->mdev->profile->mr_cache[i].limit;
736 else
737 ent->limit = 0;
738 spin_lock_irq(&ent->lock);
739 queue_adjust_cache_locked(ent);
740 spin_unlock_irq(&ent->lock);
741 }
742
743 mlx5_mr_cache_debugfs_init(dev);
744
745 return 0;
746 }
747
748 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
749 {
750 unsigned int i;
751
752 if (!dev->cache.wq)
753 return 0;
754
755 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
756 struct mlx5_cache_ent *ent = &dev->cache.ent[i];
757
758 spin_lock_irq(&ent->lock);
759 ent->disabled = true;
760 spin_unlock_irq(&ent->lock);
761 cancel_work_sync(&ent->work);
762 cancel_delayed_work_sync(&ent->dwork);
763 }
764
765 mlx5_mr_cache_debugfs_cleanup(dev);
766 mlx5_cmd_cleanup_async_ctx(&dev->async_ctx);
767
768 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++)
769 clean_keys(dev, i);
770
771 destroy_workqueue(dev->cache.wq);
772 del_timer_sync(&dev->delay_timer);
773
774 return 0;
775 }
776
777 static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr,
778 struct ib_pd *pd)
779 {
780 struct mlx5_ib_dev *dev = to_mdev(pd->device);
781
782 MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC));
783 MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE));
784 MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ));
785 MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE));
786 MLX5_SET(mkc, mkc, lr, 1);
787
788 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write))
789 MLX5_SET(mkc, mkc, relaxed_ordering_write,
790 !!(acc & IB_ACCESS_RELAXED_ORDERING));
791 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read))
792 MLX5_SET(mkc, mkc, relaxed_ordering_read,
793 !!(acc & IB_ACCESS_RELAXED_ORDERING));
794
795 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
796 MLX5_SET(mkc, mkc, qpn, 0xffffff);
797 MLX5_SET64(mkc, mkc, start_addr, start_addr);
798 }
799
800 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)
801 {
802 struct mlx5_ib_dev *dev = to_mdev(pd->device);
803 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
804 struct mlx5_ib_mr *mr;
805 void *mkc;
806 u32 *in;
807 int err;
808
809 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
810 if (!mr)
811 return ERR_PTR(-ENOMEM);
812
813 in = kzalloc(inlen, GFP_KERNEL);
814 if (!in) {
815 err = -ENOMEM;
816 goto err_free;
817 }
818
819 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
820
821 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
822 MLX5_SET(mkc, mkc, length64, 1);
823 set_mkc_access_pd_addr_fields(mkc, acc, 0, pd);
824
825 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
826 if (err)
827 goto err_in;
828
829 kfree(in);
830 mr->mmkey.type = MLX5_MKEY_MR;
831 mr->ibmr.lkey = mr->mmkey.key;
832 mr->ibmr.rkey = mr->mmkey.key;
833 mr->umem = NULL;
834
835 return &mr->ibmr;
836
837 err_in:
838 kfree(in);
839
840 err_free:
841 kfree(mr);
842
843 return ERR_PTR(err);
844 }
845
846 static int get_octo_len(u64 addr, u64 len, int page_shift)
847 {
848 u64 page_size = 1ULL << page_shift;
849 u64 offset;
850 int npages;
851
852 offset = addr & (page_size - 1);
853 npages = ALIGN(len + offset, page_size) >> page_shift;
854 return (npages + 1) / 2;
855 }
856
857 static int mr_cache_max_order(struct mlx5_ib_dev *dev)
858 {
859 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
860 return MR_CACHE_LAST_STD_ENTRY + 2;
861 return MLX5_MAX_UMR_SHIFT;
862 }
863
864 static int mr_umem_get(struct mlx5_ib_dev *dev, u64 start, u64 length,
865 int access_flags, struct ib_umem **umem, int *npages,
866 int *page_shift, int *ncont, int *order)
867 {
868 struct ib_umem *u;
869
870 *umem = NULL;
871
872 if (access_flags & IB_ACCESS_ON_DEMAND) {
873 struct ib_umem_odp *odp;
874
875 odp = ib_umem_odp_get(&dev->ib_dev, start, length, access_flags,
876 &mlx5_mn_ops);
877 if (IS_ERR(odp)) {
878 mlx5_ib_dbg(dev, "umem get failed (%ld)\n",
879 PTR_ERR(odp));
880 return PTR_ERR(odp);
881 }
882
883 u = &odp->umem;
884
885 *page_shift = odp->page_shift;
886 *ncont = ib_umem_odp_num_pages(odp);
887 *npages = *ncont << (*page_shift - PAGE_SHIFT);
888 if (order)
889 *order = ilog2(roundup_pow_of_two(*ncont));
890 } else {
891 u = ib_umem_get(&dev->ib_dev, start, length, access_flags);
892 if (IS_ERR(u)) {
893 mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(u));
894 return PTR_ERR(u);
895 }
896
897 mlx5_ib_cont_pages(u, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages,
898 page_shift, ncont, order);
899 }
900
901 if (!*npages) {
902 mlx5_ib_warn(dev, "avoid zero region\n");
903 ib_umem_release(u);
904 return -EINVAL;
905 }
906
907 *umem = u;
908
909 mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",
910 *npages, *ncont, *order, *page_shift);
911
912 return 0;
913 }
914
915 static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc)
916 {
917 struct mlx5_ib_umr_context *context =
918 container_of(wc->wr_cqe, struct mlx5_ib_umr_context, cqe);
919
920 context->status = wc->status;
921 complete(&context->done);
922 }
923
924 static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context)
925 {
926 context->cqe.done = mlx5_ib_umr_done;
927 context->status = -1;
928 init_completion(&context->done);
929 }
930
931 static int mlx5_ib_post_send_wait(struct mlx5_ib_dev *dev,
932 struct mlx5_umr_wr *umrwr)
933 {
934 struct umr_common *umrc = &dev->umrc;
935 const struct ib_send_wr *bad;
936 int err;
937 struct mlx5_ib_umr_context umr_context;
938
939 mlx5_ib_init_umr_context(&umr_context);
940 umrwr->wr.wr_cqe = &umr_context.cqe;
941
942 down(&umrc->sem);
943 err = ib_post_send(umrc->qp, &umrwr->wr, &bad);
944 if (err) {
945 mlx5_ib_warn(dev, "UMR post send failed, err %d\n", err);
946 } else {
947 wait_for_completion(&umr_context.done);
948 if (umr_context.status != IB_WC_SUCCESS) {
949 mlx5_ib_warn(dev, "reg umr failed (%u)\n",
950 umr_context.status);
951 err = -EFAULT;
952 }
953 }
954 up(&umrc->sem);
955 return err;
956 }
957
958 static struct mlx5_cache_ent *mr_cache_ent_from_order(struct mlx5_ib_dev *dev,
959 unsigned int order)
960 {
961 struct mlx5_mr_cache *cache = &dev->cache;
962
963 if (order < cache->ent[0].order)
964 return &cache->ent[0];
965 order = order - cache->ent[0].order;
966 if (order > MR_CACHE_LAST_STD_ENTRY)
967 return NULL;
968 return &cache->ent[order];
969 }
970
971 static struct mlx5_ib_mr *
972 alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr,
973 u64 len, int npages, int page_shift, unsigned int order,
974 int access_flags)
975 {
976 struct mlx5_ib_dev *dev = to_mdev(pd->device);
977 struct mlx5_cache_ent *ent = mr_cache_ent_from_order(dev, order);
978 struct mlx5_ib_mr *mr;
979
980 if (!ent)
981 return ERR_PTR(-E2BIG);
982 mr = get_cache_mr(ent);
983 if (!mr) {
984 mr = create_cache_mr(ent);
985 if (IS_ERR(mr))
986 return mr;
987 }
988
989 mr->ibmr.pd = pd;
990 mr->umem = umem;
991 mr->access_flags = access_flags;
992 mr->desc_size = sizeof(struct mlx5_mtt);
993 mr->mmkey.iova = virt_addr;
994 mr->mmkey.size = len;
995 mr->mmkey.pd = to_mpd(pd)->pdn;
996
997 return mr;
998 }
999
1000 #define MLX5_MAX_UMR_CHUNK ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - \
1001 MLX5_UMR_MTT_ALIGNMENT)
1002 #define MLX5_SPARE_UMR_CHUNK 0x10000
1003
1004 int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
1005 int page_shift, int flags)
1006 {
1007 struct mlx5_ib_dev *dev = mr->dev;
1008 struct device *ddev = dev->ib_dev.dev.parent;
1009 int size;
1010 void *xlt;
1011 dma_addr_t dma;
1012 struct mlx5_umr_wr wr;
1013 struct ib_sge sg;
1014 int err = 0;
1015 int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT)
1016 ? sizeof(struct mlx5_klm)
1017 : sizeof(struct mlx5_mtt);
1018 const int page_align = MLX5_UMR_MTT_ALIGNMENT / desc_size;
1019 const int page_mask = page_align - 1;
1020 size_t pages_mapped = 0;
1021 size_t pages_to_map = 0;
1022 size_t pages_iter = 0;
1023 size_t size_to_map = 0;
1024 gfp_t gfp;
1025 bool use_emergency_page = false;
1026
1027 if ((flags & MLX5_IB_UPD_XLT_INDIRECT) &&
1028 !umr_can_use_indirect_mkey(dev))
1029 return -EPERM;
1030
1031 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes,
1032 * so we need to align the offset and length accordingly
1033 */
1034 if (idx & page_mask) {
1035 npages += idx & page_mask;
1036 idx &= ~page_mask;
1037 }
1038
1039 gfp = flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC : GFP_KERNEL;
1040 gfp |= __GFP_ZERO | __GFP_NOWARN;
1041
1042 pages_to_map = ALIGN(npages, page_align);
1043 size = desc_size * pages_to_map;
1044 size = min_t(int, size, MLX5_MAX_UMR_CHUNK);
1045
1046 xlt = (void *)__get_free_pages(gfp, get_order(size));
1047 if (!xlt && size > MLX5_SPARE_UMR_CHUNK) {
1048 mlx5_ib_dbg(dev, "Failed to allocate %d bytes of order %d. fallback to spare UMR allocation od %d bytes\n",
1049 size, get_order(size), MLX5_SPARE_UMR_CHUNK);
1050
1051 size = MLX5_SPARE_UMR_CHUNK;
1052 xlt = (void *)__get_free_pages(gfp, get_order(size));
1053 }
1054
1055 if (!xlt) {
1056 mlx5_ib_warn(dev, "Using XLT emergency buffer\n");
1057 xlt = (void *)mlx5_ib_get_xlt_emergency_page();
1058 size = PAGE_SIZE;
1059 memset(xlt, 0, size);
1060 use_emergency_page = true;
1061 }
1062 pages_iter = size / desc_size;
1063 dma = dma_map_single(ddev, xlt, size, DMA_TO_DEVICE);
1064 if (dma_mapping_error(ddev, dma)) {
1065 mlx5_ib_err(dev, "unable to map DMA during XLT update.\n");
1066 err = -ENOMEM;
1067 goto free_xlt;
1068 }
1069
1070 if (mr->umem->is_odp) {
1071 if (!(flags & MLX5_IB_UPD_XLT_INDIRECT)) {
1072 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
1073 size_t max_pages = ib_umem_odp_num_pages(odp) - idx;
1074
1075 pages_to_map = min_t(size_t, pages_to_map, max_pages);
1076 }
1077 }
1078
1079 sg.addr = dma;
1080 sg.lkey = dev->umrc.pd->local_dma_lkey;
1081
1082 memset(&wr, 0, sizeof(wr));
1083 wr.wr.send_flags = MLX5_IB_SEND_UMR_UPDATE_XLT;
1084 if (!(flags & MLX5_IB_UPD_XLT_ENABLE))
1085 wr.wr.send_flags |= MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1086 wr.wr.sg_list = &sg;
1087 wr.wr.num_sge = 1;
1088 wr.wr.opcode = MLX5_IB_WR_UMR;
1089
1090 wr.pd = mr->ibmr.pd;
1091 wr.mkey = mr->mmkey.key;
1092 wr.length = mr->mmkey.size;
1093 wr.virt_addr = mr->mmkey.iova;
1094 wr.access_flags = mr->access_flags;
1095 wr.page_shift = page_shift;
1096
1097 for (pages_mapped = 0;
1098 pages_mapped < pages_to_map && !err;
1099 pages_mapped += pages_iter, idx += pages_iter) {
1100 npages = min_t(int, pages_iter, pages_to_map - pages_mapped);
1101 size_to_map = npages * desc_size;
1102 dma_sync_single_for_cpu(ddev, dma, size, DMA_TO_DEVICE);
1103 if (mr->umem->is_odp) {
1104 mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags);
1105 } else {
1106 __mlx5_ib_populate_pas(dev, mr->umem, page_shift, idx,
1107 npages, xlt,
1108 MLX5_IB_MTT_PRESENT);
1109 /* Clear padding after the pages
1110 * brought from the umem.
1111 */
1112 memset(xlt + size_to_map, 0, size - size_to_map);
1113 }
1114 dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE);
1115
1116 sg.length = ALIGN(size_to_map, MLX5_UMR_MTT_ALIGNMENT);
1117
1118 if (pages_mapped + pages_iter >= pages_to_map) {
1119 if (flags & MLX5_IB_UPD_XLT_ENABLE)
1120 wr.wr.send_flags |=
1121 MLX5_IB_SEND_UMR_ENABLE_MR |
1122 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS |
1123 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1124 if (flags & MLX5_IB_UPD_XLT_PD ||
1125 flags & MLX5_IB_UPD_XLT_ACCESS)
1126 wr.wr.send_flags |=
1127 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1128 if (flags & MLX5_IB_UPD_XLT_ADDR)
1129 wr.wr.send_flags |=
1130 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1131 }
1132
1133 wr.offset = idx * desc_size;
1134 wr.xlt_size = sg.length;
1135
1136 err = mlx5_ib_post_send_wait(dev, &wr);
1137 }
1138 dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);
1139
1140 free_xlt:
1141 if (use_emergency_page)
1142 mlx5_ib_put_xlt_emergency_page();
1143 else
1144 free_pages((unsigned long)xlt, get_order(size));
1145
1146 return err;
1147 }
1148
1149 /*
1150 * If ibmr is NULL it will be allocated by reg_create.
1151 * Else, the given ibmr will be used.
1152 */
1153 static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
1154 u64 virt_addr, u64 length,
1155 struct ib_umem *umem, int npages,
1156 int page_shift, int access_flags,
1157 bool populate)
1158 {
1159 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1160 struct mlx5_ib_mr *mr;
1161 __be64 *pas;
1162 void *mkc;
1163 int inlen;
1164 u32 *in;
1165 int err;
1166 bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg));
1167
1168 mr = ibmr ? to_mmr(ibmr) : kzalloc(sizeof(*mr), GFP_KERNEL);
1169 if (!mr)
1170 return ERR_PTR(-ENOMEM);
1171
1172 mr->ibmr.pd = pd;
1173 mr->access_flags = access_flags;
1174
1175 inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1176 if (populate)
1177 inlen += sizeof(*pas) * roundup(npages, 2);
1178 in = kvzalloc(inlen, GFP_KERNEL);
1179 if (!in) {
1180 err = -ENOMEM;
1181 goto err_1;
1182 }
1183 pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
1184 if (populate && !(access_flags & IB_ACCESS_ON_DEMAND))
1185 mlx5_ib_populate_pas(dev, umem, page_shift, pas,
1186 pg_cap ? MLX5_IB_MTT_PRESENT : 0);
1187
1188 /* The pg_access bit allows setting the access flags
1189 * in the page list submitted with the command. */
1190 MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap));
1191
1192 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1193 MLX5_SET(mkc, mkc, free, !populate);
1194 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
1195 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write))
1196 MLX5_SET(mkc, mkc, relaxed_ordering_write,
1197 !!(access_flags & IB_ACCESS_RELAXED_ORDERING));
1198 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read))
1199 MLX5_SET(mkc, mkc, relaxed_ordering_read,
1200 !!(access_flags & IB_ACCESS_RELAXED_ORDERING));
1201 MLX5_SET(mkc, mkc, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
1202 MLX5_SET(mkc, mkc, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
1203 MLX5_SET(mkc, mkc, rr, !!(access_flags & IB_ACCESS_REMOTE_READ));
1204 MLX5_SET(mkc, mkc, lw, !!(access_flags & IB_ACCESS_LOCAL_WRITE));
1205 MLX5_SET(mkc, mkc, lr, 1);
1206 MLX5_SET(mkc, mkc, umr_en, 1);
1207
1208 MLX5_SET64(mkc, mkc, start_addr, virt_addr);
1209 MLX5_SET64(mkc, mkc, len, length);
1210 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1211 MLX5_SET(mkc, mkc, bsf_octword_size, 0);
1212 MLX5_SET(mkc, mkc, translations_octword_size,
1213 get_octo_len(virt_addr, length, page_shift));
1214 MLX5_SET(mkc, mkc, log_page_size, page_shift);
1215 MLX5_SET(mkc, mkc, qpn, 0xffffff);
1216 if (populate) {
1217 MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
1218 get_octo_len(virt_addr, length, page_shift));
1219 }
1220
1221 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
1222 if (err) {
1223 mlx5_ib_warn(dev, "create mkey failed\n");
1224 goto err_2;
1225 }
1226 mr->mmkey.type = MLX5_MKEY_MR;
1227 mr->desc_size = sizeof(struct mlx5_mtt);
1228 mr->dev = dev;
1229 kvfree(in);
1230
1231 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key);
1232
1233 return mr;
1234
1235 err_2:
1236 kvfree(in);
1237
1238 err_1:
1239 if (!ibmr)
1240 kfree(mr);
1241
1242 return ERR_PTR(err);
1243 }
1244
1245 static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
1246 int npages, u64 length, int access_flags)
1247 {
1248 mr->npages = npages;
1249 atomic_add(npages, &dev->mdev->priv.reg_pages);
1250 mr->ibmr.lkey = mr->mmkey.key;
1251 mr->ibmr.rkey = mr->mmkey.key;
1252 mr->ibmr.length = length;
1253 mr->access_flags = access_flags;
1254 }
1255
1256 static struct ib_mr *mlx5_ib_get_dm_mr(struct ib_pd *pd, u64 start_addr,
1257 u64 length, int acc, int mode)
1258 {
1259 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1260 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1261 struct mlx5_ib_mr *mr;
1262 void *mkc;
1263 u32 *in;
1264 int err;
1265
1266 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1267 if (!mr)
1268 return ERR_PTR(-ENOMEM);
1269
1270 in = kzalloc(inlen, GFP_KERNEL);
1271 if (!in) {
1272 err = -ENOMEM;
1273 goto err_free;
1274 }
1275
1276 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1277
1278 MLX5_SET(mkc, mkc, access_mode_1_0, mode & 0x3);
1279 MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7);
1280 MLX5_SET64(mkc, mkc, len, length);
1281 set_mkc_access_pd_addr_fields(mkc, acc, start_addr, pd);
1282
1283 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
1284 if (err)
1285 goto err_in;
1286
1287 kfree(in);
1288
1289 mr->umem = NULL;
1290 set_mr_fields(dev, mr, 0, length, acc);
1291
1292 return &mr->ibmr;
1293
1294 err_in:
1295 kfree(in);
1296
1297 err_free:
1298 kfree(mr);
1299
1300 return ERR_PTR(err);
1301 }
1302
1303 int mlx5_ib_advise_mr(struct ib_pd *pd,
1304 enum ib_uverbs_advise_mr_advice advice,
1305 u32 flags,
1306 struct ib_sge *sg_list,
1307 u32 num_sge,
1308 struct uverbs_attr_bundle *attrs)
1309 {
1310 if (advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH &&
1311 advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE)
1312 return -EOPNOTSUPP;
1313
1314 return mlx5_ib_advise_mr_prefetch(pd, advice, flags,
1315 sg_list, num_sge);
1316 }
1317
1318 struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,
1319 struct ib_dm_mr_attr *attr,
1320 struct uverbs_attr_bundle *attrs)
1321 {
1322 struct mlx5_ib_dm *mdm = to_mdm(dm);
1323 struct mlx5_core_dev *dev = to_mdev(dm->device)->mdev;
1324 u64 start_addr = mdm->dev_addr + attr->offset;
1325 int mode;
1326
1327 switch (mdm->type) {
1328 case MLX5_IB_UAPI_DM_TYPE_MEMIC:
1329 if (attr->access_flags & ~MLX5_IB_DM_MEMIC_ALLOWED_ACCESS)
1330 return ERR_PTR(-EINVAL);
1331
1332 mode = MLX5_MKC_ACCESS_MODE_MEMIC;
1333 start_addr -= pci_resource_start(dev->pdev, 0);
1334 break;
1335 case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
1336 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
1337 if (attr->access_flags & ~MLX5_IB_DM_SW_ICM_ALLOWED_ACCESS)
1338 return ERR_PTR(-EINVAL);
1339
1340 mode = MLX5_MKC_ACCESS_MODE_SW_ICM;
1341 break;
1342 default:
1343 return ERR_PTR(-EINVAL);
1344 }
1345
1346 return mlx5_ib_get_dm_mr(pd, start_addr, attr->length,
1347 attr->access_flags, mode);
1348 }
1349
1350 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
1351 u64 virt_addr, int access_flags,
1352 struct ib_udata *udata)
1353 {
1354 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1355 struct mlx5_ib_mr *mr = NULL;
1356 bool use_umr;
1357 struct ib_umem *umem;
1358 int page_shift;
1359 int npages;
1360 int ncont;
1361 int order;
1362 int err;
1363
1364 if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM))
1365 return ERR_PTR(-EOPNOTSUPP);
1366
1367 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
1368 start, virt_addr, length, access_flags);
1369
1370 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && !start &&
1371 length == U64_MAX) {
1372 if (virt_addr != start)
1373 return ERR_PTR(-EINVAL);
1374 if (!(access_flags & IB_ACCESS_ON_DEMAND) ||
1375 !(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
1376 return ERR_PTR(-EINVAL);
1377
1378 mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), udata, access_flags);
1379 if (IS_ERR(mr))
1380 return ERR_CAST(mr);
1381 return &mr->ibmr;
1382 }
1383
1384 err = mr_umem_get(dev, start, length, access_flags, &umem,
1385 &npages, &page_shift, &ncont, &order);
1386
1387 if (err < 0)
1388 return ERR_PTR(err);
1389
1390 use_umr = mlx5_ib_can_use_umr(dev, true, access_flags);
1391
1392 if (order <= mr_cache_max_order(dev) && use_umr) {
1393 mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont,
1394 page_shift, order, access_flags);
1395 if (PTR_ERR(mr) == -EAGAIN) {
1396 mlx5_ib_dbg(dev, "cache empty for order %d\n", order);
1397 mr = NULL;
1398 }
1399 } else if (!MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) {
1400 if (access_flags & IB_ACCESS_ON_DEMAND) {
1401 err = -EINVAL;
1402 pr_err("Got MR registration for ODP MR > 512MB, not supported for Connect-IB\n");
1403 goto error;
1404 }
1405 use_umr = false;
1406 }
1407
1408 if (!mr) {
1409 mutex_lock(&dev->slow_path_mutex);
1410 mr = reg_create(NULL, pd, virt_addr, length, umem, ncont,
1411 page_shift, access_flags, !use_umr);
1412 mutex_unlock(&dev->slow_path_mutex);
1413 }
1414
1415 if (IS_ERR(mr)) {
1416 err = PTR_ERR(mr);
1417 goto error;
1418 }
1419
1420 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key);
1421
1422 mr->umem = umem;
1423 set_mr_fields(dev, mr, npages, length, access_flags);
1424
1425 if (use_umr) {
1426 int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;
1427
1428 if (access_flags & IB_ACCESS_ON_DEMAND)
1429 update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP;
1430
1431 err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,
1432 update_xlt_flags);
1433
1434 if (err) {
1435 dereg_mr(dev, mr);
1436 return ERR_PTR(err);
1437 }
1438 }
1439
1440 if (is_odp_mr(mr)) {
1441 to_ib_umem_odp(mr->umem)->private = mr;
1442 atomic_set(&mr->num_deferred_work, 0);
1443 err = xa_err(xa_store(&dev->odp_mkeys,
1444 mlx5_base_mkey(mr->mmkey.key), &mr->mmkey,
1445 GFP_KERNEL));
1446 if (err) {
1447 dereg_mr(dev, mr);
1448 return ERR_PTR(err);
1449 }
1450 }
1451
1452 return &mr->ibmr;
1453 error:
1454 ib_umem_release(umem);
1455 return ERR_PTR(err);
1456 }
1457
1458 /**
1459 * mlx5_mr_cache_invalidate - Fence all DMA on the MR
1460 * @mr: The MR to fence
1461 *
1462 * Upon return the NIC will not be doing any DMA to the pages under the MR,
1463 * and any DMA inprogress will be completed. Failure of this function
1464 * indicates the HW has failed catastrophically.
1465 */
1466 int mlx5_mr_cache_invalidate(struct mlx5_ib_mr *mr)
1467 {
1468 struct mlx5_umr_wr umrwr = {};
1469
1470 if (mr->dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
1471 return 0;
1472
1473 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_DISABLE_MR |
1474 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1475 umrwr.wr.opcode = MLX5_IB_WR_UMR;
1476 umrwr.pd = mr->dev->umrc.pd;
1477 umrwr.mkey = mr->mmkey.key;
1478 umrwr.ignore_free_state = 1;
1479
1480 return mlx5_ib_post_send_wait(mr->dev, &umrwr);
1481 }
1482
1483 static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1484 int access_flags, int flags)
1485 {
1486 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1487 struct mlx5_umr_wr umrwr = {};
1488 int err;
1489
1490 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1491
1492 umrwr.wr.opcode = MLX5_IB_WR_UMR;
1493 umrwr.mkey = mr->mmkey.key;
1494
1495 if (flags & IB_MR_REREG_PD || flags & IB_MR_REREG_ACCESS) {
1496 umrwr.pd = pd;
1497 umrwr.access_flags = access_flags;
1498 umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1499 }
1500
1501 err = mlx5_ib_post_send_wait(dev, &umrwr);
1502
1503 return err;
1504 }
1505
1506 int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
1507 u64 length, u64 virt_addr, int new_access_flags,
1508 struct ib_pd *new_pd, struct ib_udata *udata)
1509 {
1510 struct mlx5_ib_dev *dev = to_mdev(ib_mr->device);
1511 struct mlx5_ib_mr *mr = to_mmr(ib_mr);
1512 struct ib_pd *pd = (flags & IB_MR_REREG_PD) ? new_pd : ib_mr->pd;
1513 int access_flags = flags & IB_MR_REREG_ACCESS ?
1514 new_access_flags :
1515 mr->access_flags;
1516 int page_shift = 0;
1517 int upd_flags = 0;
1518 int npages = 0;
1519 int ncont = 0;
1520 int order = 0;
1521 u64 addr, len;
1522 int err;
1523
1524 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
1525 start, virt_addr, length, access_flags);
1526
1527 atomic_sub(mr->npages, &dev->mdev->priv.reg_pages);
1528
1529 if (!mr->umem)
1530 return -EINVAL;
1531
1532 if (is_odp_mr(mr))
1533 return -EOPNOTSUPP;
1534
1535 if (flags & IB_MR_REREG_TRANS) {
1536 addr = virt_addr;
1537 len = length;
1538 } else {
1539 addr = mr->umem->address;
1540 len = mr->umem->length;
1541 }
1542
1543 if (flags != IB_MR_REREG_PD) {
1544 /*
1545 * Replace umem. This needs to be done whether or not UMR is
1546 * used.
1547 */
1548 flags |= IB_MR_REREG_TRANS;
1549 ib_umem_release(mr->umem);
1550 mr->umem = NULL;
1551 err = mr_umem_get(dev, addr, len, access_flags, &mr->umem,
1552 &npages, &page_shift, &ncont, &order);
1553 if (err)
1554 goto err;
1555 }
1556
1557 if (!mlx5_ib_can_use_umr(dev, true, access_flags) ||
1558 (flags & IB_MR_REREG_TRANS && !use_umr_mtt_update(mr, addr, len))) {
1559 /*
1560 * UMR can't be used - MKey needs to be replaced.
1561 */
1562 if (mr->cache_ent)
1563 detach_mr_from_cache(mr);
1564 err = destroy_mkey(dev, mr);
1565 if (err)
1566 goto err;
1567
1568 mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont,
1569 page_shift, access_flags, true);
1570
1571 if (IS_ERR(mr)) {
1572 err = PTR_ERR(mr);
1573 mr = to_mmr(ib_mr);
1574 goto err;
1575 }
1576 } else {
1577 /*
1578 * Send a UMR WQE
1579 */
1580 mr->ibmr.pd = pd;
1581 mr->access_flags = access_flags;
1582 mr->mmkey.iova = addr;
1583 mr->mmkey.size = len;
1584 mr->mmkey.pd = to_mpd(pd)->pdn;
1585
1586 if (flags & IB_MR_REREG_TRANS) {
1587 upd_flags = MLX5_IB_UPD_XLT_ADDR;
1588 if (flags & IB_MR_REREG_PD)
1589 upd_flags |= MLX5_IB_UPD_XLT_PD;
1590 if (flags & IB_MR_REREG_ACCESS)
1591 upd_flags |= MLX5_IB_UPD_XLT_ACCESS;
1592 err = mlx5_ib_update_xlt(mr, 0, npages, page_shift,
1593 upd_flags);
1594 } else {
1595 err = rereg_umr(pd, mr, access_flags, flags);
1596 }
1597
1598 if (err)
1599 goto err;
1600 }
1601
1602 set_mr_fields(dev, mr, npages, len, access_flags);
1603
1604 return 0;
1605
1606 err:
1607 ib_umem_release(mr->umem);
1608 mr->umem = NULL;
1609
1610 clean_mr(dev, mr);
1611 return err;
1612 }
1613
1614 static int
1615 mlx5_alloc_priv_descs(struct ib_device *device,
1616 struct mlx5_ib_mr *mr,
1617 int ndescs,
1618 int desc_size)
1619 {
1620 int size = ndescs * desc_size;
1621 int add_size;
1622 int ret;
1623
1624 add_size = max_t(int, MLX5_UMR_ALIGN - ARCH_KMALLOC_MINALIGN, 0);
1625
1626 mr->descs_alloc = kzalloc(size + add_size, GFP_KERNEL);
1627 if (!mr->descs_alloc)
1628 return -ENOMEM;
1629
1630 mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN);
1631
1632 mr->desc_map = dma_map_single(device->dev.parent, mr->descs,
1633 size, DMA_TO_DEVICE);
1634 if (dma_mapping_error(device->dev.parent, mr->desc_map)) {
1635 ret = -ENOMEM;
1636 goto err;
1637 }
1638
1639 return 0;
1640 err:
1641 kfree(mr->descs_alloc);
1642
1643 return ret;
1644 }
1645
1646 static void
1647 mlx5_free_priv_descs(struct mlx5_ib_mr *mr)
1648 {
1649 if (mr->descs) {
1650 struct ib_device *device = mr->ibmr.device;
1651 int size = mr->max_descs * mr->desc_size;
1652
1653 dma_unmap_single(device->dev.parent, mr->desc_map,
1654 size, DMA_TO_DEVICE);
1655 kfree(mr->descs_alloc);
1656 mr->descs = NULL;
1657 }
1658 }
1659
1660 static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
1661 {
1662 if (mr->sig) {
1663 if (mlx5_core_destroy_psv(dev->mdev,
1664 mr->sig->psv_memory.psv_idx))
1665 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1666 mr->sig->psv_memory.psv_idx);
1667 if (mlx5_core_destroy_psv(dev->mdev,
1668 mr->sig->psv_wire.psv_idx))
1669 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1670 mr->sig->psv_wire.psv_idx);
1671 xa_erase(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key));
1672 kfree(mr->sig);
1673 mr->sig = NULL;
1674 }
1675
1676 if (!mr->cache_ent) {
1677 destroy_mkey(dev, mr);
1678 mlx5_free_priv_descs(mr);
1679 }
1680 }
1681
1682 static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
1683 {
1684 int npages = mr->npages;
1685 struct ib_umem *umem = mr->umem;
1686
1687 /* Stop all DMA */
1688 if (is_odp_mr(mr))
1689 mlx5_ib_fence_odp_mr(mr);
1690 else
1691 clean_mr(dev, mr);
1692
1693 if (mr->cache_ent)
1694 mlx5_mr_cache_free(dev, mr);
1695 else
1696 kfree(mr);
1697
1698 ib_umem_release(umem);
1699 atomic_sub(npages, &dev->mdev->priv.reg_pages);
1700
1701 }
1702
1703 int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
1704 {
1705 struct mlx5_ib_mr *mmr = to_mmr(ibmr);
1706
1707 if (ibmr->type == IB_MR_TYPE_INTEGRITY) {
1708 dereg_mr(to_mdev(mmr->mtt_mr->ibmr.device), mmr->mtt_mr);
1709 dereg_mr(to_mdev(mmr->klm_mr->ibmr.device), mmr->klm_mr);
1710 }
1711
1712 if (is_odp_mr(mmr) && to_ib_umem_odp(mmr->umem)->is_implicit_odp) {
1713 mlx5_ib_free_implicit_mr(mmr);
1714 return 0;
1715 }
1716
1717 dereg_mr(to_mdev(ibmr->device), mmr);
1718
1719 return 0;
1720 }
1721
1722 static void mlx5_set_umr_free_mkey(struct ib_pd *pd, u32 *in, int ndescs,
1723 int access_mode, int page_shift)
1724 {
1725 void *mkc;
1726
1727 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1728
1729 MLX5_SET(mkc, mkc, free, 1);
1730 MLX5_SET(mkc, mkc, qpn, 0xffffff);
1731 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1732 MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
1733 MLX5_SET(mkc, mkc, access_mode_1_0, access_mode & 0x3);
1734 MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode >> 2) & 0x7);
1735 MLX5_SET(mkc, mkc, umr_en, 1);
1736 MLX5_SET(mkc, mkc, log_page_size, page_shift);
1737 }
1738
1739 static int _mlx5_alloc_mkey_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1740 int ndescs, int desc_size, int page_shift,
1741 int access_mode, u32 *in, int inlen)
1742 {
1743 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1744 int err;
1745
1746 mr->access_mode = access_mode;
1747 mr->desc_size = desc_size;
1748 mr->max_descs = ndescs;
1749
1750 err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, desc_size);
1751 if (err)
1752 return err;
1753
1754 mlx5_set_umr_free_mkey(pd, in, ndescs, access_mode, page_shift);
1755
1756 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
1757 if (err)
1758 goto err_free_descs;
1759
1760 mr->mmkey.type = MLX5_MKEY_MR;
1761 mr->ibmr.lkey = mr->mmkey.key;
1762 mr->ibmr.rkey = mr->mmkey.key;
1763
1764 return 0;
1765
1766 err_free_descs:
1767 mlx5_free_priv_descs(mr);
1768 return err;
1769 }
1770
1771 static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd,
1772 u32 max_num_sg, u32 max_num_meta_sg,
1773 int desc_size, int access_mode)
1774 {
1775 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1776 int ndescs = ALIGN(max_num_sg + max_num_meta_sg, 4);
1777 int page_shift = 0;
1778 struct mlx5_ib_mr *mr;
1779 u32 *in;
1780 int err;
1781
1782 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1783 if (!mr)
1784 return ERR_PTR(-ENOMEM);
1785
1786 mr->ibmr.pd = pd;
1787 mr->ibmr.device = pd->device;
1788
1789 in = kzalloc(inlen, GFP_KERNEL);
1790 if (!in) {
1791 err = -ENOMEM;
1792 goto err_free;
1793 }
1794
1795 if (access_mode == MLX5_MKC_ACCESS_MODE_MTT)
1796 page_shift = PAGE_SHIFT;
1797
1798 err = _mlx5_alloc_mkey_descs(pd, mr, ndescs, desc_size, page_shift,
1799 access_mode, in, inlen);
1800 if (err)
1801 goto err_free_in;
1802
1803 mr->umem = NULL;
1804 kfree(in);
1805
1806 return mr;
1807
1808 err_free_in:
1809 kfree(in);
1810 err_free:
1811 kfree(mr);
1812 return ERR_PTR(err);
1813 }
1814
1815 static int mlx5_alloc_mem_reg_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1816 int ndescs, u32 *in, int inlen)
1817 {
1818 return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_mtt),
1819 PAGE_SHIFT, MLX5_MKC_ACCESS_MODE_MTT, in,
1820 inlen);
1821 }
1822
1823 static int mlx5_alloc_sg_gaps_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1824 int ndescs, u32 *in, int inlen)
1825 {
1826 return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_klm),
1827 0, MLX5_MKC_ACCESS_MODE_KLMS, in, inlen);
1828 }
1829
1830 static int mlx5_alloc_integrity_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1831 int max_num_sg, int max_num_meta_sg,
1832 u32 *in, int inlen)
1833 {
1834 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1835 u32 psv_index[2];
1836 void *mkc;
1837 int err;
1838
1839 mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL);
1840 if (!mr->sig)
1841 return -ENOMEM;
1842
1843 /* create mem & wire PSVs */
1844 err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 2, psv_index);
1845 if (err)
1846 goto err_free_sig;
1847
1848 mr->sig->psv_memory.psv_idx = psv_index[0];
1849 mr->sig->psv_wire.psv_idx = psv_index[1];
1850
1851 mr->sig->sig_status_checked = true;
1852 mr->sig->sig_err_exists = false;
1853 /* Next UMR, Arm SIGERR */
1854 ++mr->sig->sigerr_count;
1855 mr->klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg,
1856 sizeof(struct mlx5_klm),
1857 MLX5_MKC_ACCESS_MODE_KLMS);
1858 if (IS_ERR(mr->klm_mr)) {
1859 err = PTR_ERR(mr->klm_mr);
1860 goto err_destroy_psv;
1861 }
1862 mr->mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg,
1863 sizeof(struct mlx5_mtt),
1864 MLX5_MKC_ACCESS_MODE_MTT);
1865 if (IS_ERR(mr->mtt_mr)) {
1866 err = PTR_ERR(mr->mtt_mr);
1867 goto err_free_klm_mr;
1868 }
1869
1870 /* Set bsf descriptors for mkey */
1871 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1872 MLX5_SET(mkc, mkc, bsf_en, 1);
1873 MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE);
1874
1875 err = _mlx5_alloc_mkey_descs(pd, mr, 4, sizeof(struct mlx5_klm), 0,
1876 MLX5_MKC_ACCESS_MODE_KLMS, in, inlen);
1877 if (err)
1878 goto err_free_mtt_mr;
1879
1880 err = xa_err(xa_store(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key),
1881 mr->sig, GFP_KERNEL));
1882 if (err)
1883 goto err_free_descs;
1884 return 0;
1885
1886 err_free_descs:
1887 destroy_mkey(dev, mr);
1888 mlx5_free_priv_descs(mr);
1889 err_free_mtt_mr:
1890 dereg_mr(to_mdev(mr->mtt_mr->ibmr.device), mr->mtt_mr);
1891 mr->mtt_mr = NULL;
1892 err_free_klm_mr:
1893 dereg_mr(to_mdev(mr->klm_mr->ibmr.device), mr->klm_mr);
1894 mr->klm_mr = NULL;
1895 err_destroy_psv:
1896 if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_memory.psv_idx))
1897 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1898 mr->sig->psv_memory.psv_idx);
1899 if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx))
1900 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1901 mr->sig->psv_wire.psv_idx);
1902 err_free_sig:
1903 kfree(mr->sig);
1904
1905 return err;
1906 }
1907
1908 static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd,
1909 enum ib_mr_type mr_type, u32 max_num_sg,
1910 u32 max_num_meta_sg)
1911 {
1912 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1913 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1914 int ndescs = ALIGN(max_num_sg, 4);
1915 struct mlx5_ib_mr *mr;
1916 u32 *in;
1917 int err;
1918
1919 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1920 if (!mr)
1921 return ERR_PTR(-ENOMEM);
1922
1923 in = kzalloc(inlen, GFP_KERNEL);
1924 if (!in) {
1925 err = -ENOMEM;
1926 goto err_free;
1927 }
1928
1929 mr->ibmr.device = pd->device;
1930 mr->umem = NULL;
1931
1932 switch (mr_type) {
1933 case IB_MR_TYPE_MEM_REG:
1934 err = mlx5_alloc_mem_reg_descs(pd, mr, ndescs, in, inlen);
1935 break;
1936 case IB_MR_TYPE_SG_GAPS:
1937 err = mlx5_alloc_sg_gaps_descs(pd, mr, ndescs, in, inlen);
1938 break;
1939 case IB_MR_TYPE_INTEGRITY:
1940 err = mlx5_alloc_integrity_descs(pd, mr, max_num_sg,
1941 max_num_meta_sg, in, inlen);
1942 break;
1943 default:
1944 mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type);
1945 err = -EINVAL;
1946 }
1947
1948 if (err)
1949 goto err_free_in;
1950
1951 kfree(in);
1952
1953 return &mr->ibmr;
1954
1955 err_free_in:
1956 kfree(in);
1957 err_free:
1958 kfree(mr);
1959 return ERR_PTR(err);
1960 }
1961
1962 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1963 u32 max_num_sg, struct ib_udata *udata)
1964 {
1965 return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0);
1966 }
1967
1968 struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd,
1969 u32 max_num_sg, u32 max_num_meta_sg)
1970 {
1971 return __mlx5_ib_alloc_mr(pd, IB_MR_TYPE_INTEGRITY, max_num_sg,
1972 max_num_meta_sg);
1973 }
1974
1975 struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
1976 struct ib_udata *udata)
1977 {
1978 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1979 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1980 struct mlx5_ib_mw *mw = NULL;
1981 u32 *in = NULL;
1982 void *mkc;
1983 int ndescs;
1984 int err;
1985 struct mlx5_ib_alloc_mw req = {};
1986 struct {
1987 __u32 comp_mask;
1988 __u32 response_length;
1989 } resp = {};
1990
1991 err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
1992 if (err)
1993 return ERR_PTR(err);
1994
1995 if (req.comp_mask || req.reserved1 || req.reserved2)
1996 return ERR_PTR(-EOPNOTSUPP);
1997
1998 if (udata->inlen > sizeof(req) &&
1999 !ib_is_udata_cleared(udata, sizeof(req),
2000 udata->inlen - sizeof(req)))
2001 return ERR_PTR(-EOPNOTSUPP);
2002
2003 ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4);
2004
2005 mw = kzalloc(sizeof(*mw), GFP_KERNEL);
2006 in = kzalloc(inlen, GFP_KERNEL);
2007 if (!mw || !in) {
2008 err = -ENOMEM;
2009 goto free;
2010 }
2011
2012 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
2013
2014 MLX5_SET(mkc, mkc, free, 1);
2015 MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
2016 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
2017 MLX5_SET(mkc, mkc, umr_en, 1);
2018 MLX5_SET(mkc, mkc, lr, 1);
2019 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS);
2020 MLX5_SET(mkc, mkc, en_rinval, !!((type == IB_MW_TYPE_2)));
2021 MLX5_SET(mkc, mkc, qpn, 0xffffff);
2022
2023 err = mlx5_ib_create_mkey(dev, &mw->mmkey, in, inlen);
2024 if (err)
2025 goto free;
2026
2027 mw->mmkey.type = MLX5_MKEY_MW;
2028 mw->ibmw.rkey = mw->mmkey.key;
2029 mw->ndescs = ndescs;
2030
2031 resp.response_length = min(offsetof(typeof(resp), response_length) +
2032 sizeof(resp.response_length), udata->outlen);
2033 if (resp.response_length) {
2034 err = ib_copy_to_udata(udata, &resp, resp.response_length);
2035 if (err) {
2036 mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey);
2037 goto free;
2038 }
2039 }
2040
2041 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
2042 err = xa_err(xa_store(&dev->odp_mkeys,
2043 mlx5_base_mkey(mw->mmkey.key), &mw->mmkey,
2044 GFP_KERNEL));
2045 if (err)
2046 goto free_mkey;
2047 }
2048
2049 kfree(in);
2050 return &mw->ibmw;
2051
2052 free_mkey:
2053 mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey);
2054 free:
2055 kfree(mw);
2056 kfree(in);
2057 return ERR_PTR(err);
2058 }
2059
2060 int mlx5_ib_dealloc_mw(struct ib_mw *mw)
2061 {
2062 struct mlx5_ib_dev *dev = to_mdev(mw->device);
2063 struct mlx5_ib_mw *mmw = to_mmw(mw);
2064 int err;
2065
2066 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
2067 xa_erase(&dev->odp_mkeys, mlx5_base_mkey(mmw->mmkey.key));
2068 /*
2069 * pagefault_single_data_segment() may be accessing mmw under
2070 * SRCU if the user bound an ODP MR to this MW.
2071 */
2072 synchronize_srcu(&dev->odp_srcu);
2073 }
2074
2075 err = mlx5_core_destroy_mkey(dev->mdev, &mmw->mmkey);
2076 if (err)
2077 return err;
2078 kfree(mmw);
2079 return 0;
2080 }
2081
2082 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
2083 struct ib_mr_status *mr_status)
2084 {
2085 struct mlx5_ib_mr *mmr = to_mmr(ibmr);
2086 int ret = 0;
2087
2088 if (check_mask & ~IB_MR_CHECK_SIG_STATUS) {
2089 pr_err("Invalid status check mask\n");
2090 ret = -EINVAL;
2091 goto done;
2092 }
2093
2094 mr_status->fail_status = 0;
2095 if (check_mask & IB_MR_CHECK_SIG_STATUS) {
2096 if (!mmr->sig) {
2097 ret = -EINVAL;
2098 pr_err("signature status check requested on a non-signature enabled MR\n");
2099 goto done;
2100 }
2101
2102 mmr->sig->sig_status_checked = true;
2103 if (!mmr->sig->sig_err_exists)
2104 goto done;
2105
2106 if (ibmr->lkey == mmr->sig->err_item.key)
2107 memcpy(&mr_status->sig_err, &mmr->sig->err_item,
2108 sizeof(mr_status->sig_err));
2109 else {
2110 mr_status->sig_err.err_type = IB_SIG_BAD_GUARD;
2111 mr_status->sig_err.sig_err_offset = 0;
2112 mr_status->sig_err.key = mmr->sig->err_item.key;
2113 }
2114
2115 mmr->sig->sig_err_exists = false;
2116 mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS;
2117 }
2118
2119 done:
2120 return ret;
2121 }
2122
2123 static int
2124 mlx5_ib_map_pa_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
2125 int data_sg_nents, unsigned int *data_sg_offset,
2126 struct scatterlist *meta_sg, int meta_sg_nents,
2127 unsigned int *meta_sg_offset)
2128 {
2129 struct mlx5_ib_mr *mr = to_mmr(ibmr);
2130 unsigned int sg_offset = 0;
2131 int n = 0;
2132
2133 mr->meta_length = 0;
2134 if (data_sg_nents == 1) {
2135 n++;
2136 mr->ndescs = 1;
2137 if (data_sg_offset)
2138 sg_offset = *data_sg_offset;
2139 mr->data_length = sg_dma_len(data_sg) - sg_offset;
2140 mr->data_iova = sg_dma_address(data_sg) + sg_offset;
2141 if (meta_sg_nents == 1) {
2142 n++;
2143 mr->meta_ndescs = 1;
2144 if (meta_sg_offset)
2145 sg_offset = *meta_sg_offset;
2146 else
2147 sg_offset = 0;
2148 mr->meta_length = sg_dma_len(meta_sg) - sg_offset;
2149 mr->pi_iova = sg_dma_address(meta_sg) + sg_offset;
2150 }
2151 ibmr->length = mr->data_length + mr->meta_length;
2152 }
2153
2154 return n;
2155 }
2156
2157 static int
2158 mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
2159 struct scatterlist *sgl,
2160 unsigned short sg_nents,
2161 unsigned int *sg_offset_p,
2162 struct scatterlist *meta_sgl,
2163 unsigned short meta_sg_nents,
2164 unsigned int *meta_sg_offset_p)
2165 {
2166 struct scatterlist *sg = sgl;
2167 struct mlx5_klm *klms = mr->descs;
2168 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
2169 u32 lkey = mr->ibmr.pd->local_dma_lkey;
2170 int i, j = 0;
2171
2172 mr->ibmr.iova = sg_dma_address(sg) + sg_offset;
2173 mr->ibmr.length = 0;
2174
2175 for_each_sg(sgl, sg, sg_nents, i) {
2176 if (unlikely(i >= mr->max_descs))
2177 break;
2178 klms[i].va = cpu_to_be64(sg_dma_address(sg) + sg_offset);
2179 klms[i].bcount = cpu_to_be32(sg_dma_len(sg) - sg_offset);
2180 klms[i].key = cpu_to_be32(lkey);
2181 mr->ibmr.length += sg_dma_len(sg) - sg_offset;
2182
2183 sg_offset = 0;
2184 }
2185
2186 if (sg_offset_p)
2187 *sg_offset_p = sg_offset;
2188
2189 mr->ndescs = i;
2190 mr->data_length = mr->ibmr.length;
2191
2192 if (meta_sg_nents) {
2193 sg = meta_sgl;
2194 sg_offset = meta_sg_offset_p ? *meta_sg_offset_p : 0;
2195 for_each_sg(meta_sgl, sg, meta_sg_nents, j) {
2196 if (unlikely(i + j >= mr->max_descs))
2197 break;
2198 klms[i + j].va = cpu_to_be64(sg_dma_address(sg) +
2199 sg_offset);
2200 klms[i + j].bcount = cpu_to_be32(sg_dma_len(sg) -
2201 sg_offset);
2202 klms[i + j].key = cpu_to_be32(lkey);
2203 mr->ibmr.length += sg_dma_len(sg) - sg_offset;
2204
2205 sg_offset = 0;
2206 }
2207 if (meta_sg_offset_p)
2208 *meta_sg_offset_p = sg_offset;
2209
2210 mr->meta_ndescs = j;
2211 mr->meta_length = mr->ibmr.length - mr->data_length;
2212 }
2213
2214 return i + j;
2215 }
2216
2217 static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)
2218 {
2219 struct mlx5_ib_mr *mr = to_mmr(ibmr);
2220 __be64 *descs;
2221
2222 if (unlikely(mr->ndescs == mr->max_descs))
2223 return -ENOMEM;
2224
2225 descs = mr->descs;
2226 descs[mr->ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);
2227
2228 return 0;
2229 }
2230
2231 static int mlx5_set_page_pi(struct ib_mr *ibmr, u64 addr)
2232 {
2233 struct mlx5_ib_mr *mr = to_mmr(ibmr);
2234 __be64 *descs;
2235
2236 if (unlikely(mr->ndescs + mr->meta_ndescs == mr->max_descs))
2237 return -ENOMEM;
2238
2239 descs = mr->descs;
2240 descs[mr->ndescs + mr->meta_ndescs++] =
2241 cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);
2242
2243 return 0;
2244 }
2245
2246 static int
2247 mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
2248 int data_sg_nents, unsigned int *data_sg_offset,
2249 struct scatterlist *meta_sg, int meta_sg_nents,
2250 unsigned int *meta_sg_offset)
2251 {
2252 struct mlx5_ib_mr *mr = to_mmr(ibmr);
2253 struct mlx5_ib_mr *pi_mr = mr->mtt_mr;
2254 int n;
2255
2256 pi_mr->ndescs = 0;
2257 pi_mr->meta_ndescs = 0;
2258 pi_mr->meta_length = 0;
2259
2260 ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map,
2261 pi_mr->desc_size * pi_mr->max_descs,
2262 DMA_TO_DEVICE);
2263
2264 pi_mr->ibmr.page_size = ibmr->page_size;
2265 n = ib_sg_to_pages(&pi_mr->ibmr, data_sg, data_sg_nents, data_sg_offset,
2266 mlx5_set_page);
2267 if (n != data_sg_nents)
2268 return n;
2269
2270 pi_mr->data_iova = pi_mr->ibmr.iova;
2271 pi_mr->data_length = pi_mr->ibmr.length;
2272 pi_mr->ibmr.length = pi_mr->data_length;
2273 ibmr->length = pi_mr->data_length;
2274
2275 if (meta_sg_nents) {
2276 u64 page_mask = ~((u64)ibmr->page_size - 1);
2277 u64 iova = pi_mr->data_iova;
2278
2279 n += ib_sg_to_pages(&pi_mr->ibmr, meta_sg, meta_sg_nents,
2280 meta_sg_offset, mlx5_set_page_pi);
2281
2282 pi_mr->meta_length = pi_mr->ibmr.length;
2283 /*
2284 * PI address for the HW is the offset of the metadata address
2285 * relative to the first data page address.
2286 * It equals to first data page address + size of data pages +
2287 * metadata offset at the first metadata page
2288 */
2289 pi_mr->pi_iova = (iova & page_mask) +
2290 pi_mr->ndescs * ibmr->page_size +
2291 (pi_mr->ibmr.iova & ~page_mask);
2292 /*
2293 * In order to use one MTT MR for data and metadata, we register
2294 * also the gaps between the end of the data and the start of
2295 * the metadata (the sig MR will verify that the HW will access
2296 * to right addresses). This mapping is safe because we use
2297 * internal mkey for the registration.
2298 */
2299 pi_mr->ibmr.length = pi_mr->pi_iova + pi_mr->meta_length - iova;
2300 pi_mr->ibmr.iova = iova;
2301 ibmr->length += pi_mr->meta_length;
2302 }
2303
2304 ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map,
2305 pi_mr->desc_size * pi_mr->max_descs,
2306 DMA_TO_DEVICE);
2307
2308 return n;
2309 }
2310
2311 static int
2312 mlx5_ib_map_klm_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
2313 int data_sg_nents, unsigned int *data_sg_offset,
2314 struct scatterlist *meta_sg, int meta_sg_nents,
2315 unsigned int *meta_sg_offset)
2316 {
2317 struct mlx5_ib_mr *mr = to_mmr(ibmr);
2318 struct mlx5_ib_mr *pi_mr = mr->klm_mr;
2319 int n;
2320
2321 pi_mr->ndescs = 0;
2322 pi_mr->meta_ndescs = 0;
2323 pi_mr->meta_length = 0;
2324
2325 ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map,
2326 pi_mr->desc_size * pi_mr->max_descs,
2327 DMA_TO_DEVICE);
2328
2329 n = mlx5_ib_sg_to_klms(pi_mr, data_sg, data_sg_nents, data_sg_offset,
2330 meta_sg, meta_sg_nents, meta_sg_offset);
2331
2332 ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map,
2333 pi_mr->desc_size * pi_mr->max_descs,
2334 DMA_TO_DEVICE);
2335
2336 /* This is zero-based memory region */
2337 pi_mr->data_iova = 0;
2338 pi_mr->ibmr.iova = 0;
2339 pi_mr->pi_iova = pi_mr->data_length;
2340 ibmr->length = pi_mr->ibmr.length;
2341
2342 return n;
2343 }
2344
2345 int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
2346 int data_sg_nents, unsigned int *data_sg_offset,
2347 struct scatterlist *meta_sg, int meta_sg_nents,
2348 unsigned int *meta_sg_offset)
2349 {
2350 struct mlx5_ib_mr *mr = to_mmr(ibmr);
2351 struct mlx5_ib_mr *pi_mr = NULL;
2352 int n;
2353
2354 WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY);
2355
2356 mr->ndescs = 0;
2357 mr->data_length = 0;
2358 mr->data_iova = 0;
2359 mr->meta_ndescs = 0;
2360 mr->pi_iova = 0;
2361 /*
2362 * As a performance optimization, if possible, there is no need to
2363 * perform UMR operation to register the data/metadata buffers.
2364 * First try to map the sg lists to PA descriptors with local_dma_lkey.
2365 * Fallback to UMR only in case of a failure.
2366 */
2367 n = mlx5_ib_map_pa_mr_sg_pi(ibmr, data_sg, data_sg_nents,
2368 data_sg_offset, meta_sg, meta_sg_nents,
2369 meta_sg_offset);
2370 if (n == data_sg_nents + meta_sg_nents)
2371 goto out;
2372 /*
2373 * As a performance optimization, if possible, there is no need to map
2374 * the sg lists to KLM descriptors. First try to map the sg lists to MTT
2375 * descriptors and fallback to KLM only in case of a failure.
2376 * It's more efficient for the HW to work with MTT descriptors
2377 * (especially in high load).
2378 * Use KLM (indirect access) only if it's mandatory.
2379 */
2380 pi_mr = mr->mtt_mr;
2381 n = mlx5_ib_map_mtt_mr_sg_pi(ibmr, data_sg, data_sg_nents,
2382 data_sg_offset, meta_sg, meta_sg_nents,
2383 meta_sg_offset);
2384 if (n == data_sg_nents + meta_sg_nents)
2385 goto out;
2386
2387 pi_mr = mr->klm_mr;
2388 n = mlx5_ib_map_klm_mr_sg_pi(ibmr, data_sg, data_sg_nents,
2389 data_sg_offset, meta_sg, meta_sg_nents,
2390 meta_sg_offset);
2391 if (unlikely(n != data_sg_nents + meta_sg_nents))
2392 return -ENOMEM;
2393
2394 out:
2395 /* This is zero-based memory region */
2396 ibmr->iova = 0;
2397 mr->pi_mr = pi_mr;
2398 if (pi_mr)
2399 ibmr->sig_attrs->meta_length = pi_mr->meta_length;
2400 else
2401 ibmr->sig_attrs->meta_length = mr->meta_length;
2402
2403 return 0;
2404 }
2405
2406 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
2407 unsigned int *sg_offset)
2408 {
2409 struct mlx5_ib_mr *mr = to_mmr(ibmr);
2410 int n;
2411
2412 mr->ndescs = 0;
2413
2414 ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map,
2415 mr->desc_size * mr->max_descs,
2416 DMA_TO_DEVICE);
2417
2418 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
2419 n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset, NULL, 0,
2420 NULL);
2421 else
2422 n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset,
2423 mlx5_set_page);
2424
2425 ib_dma_sync_single_for_device(ibmr->device, mr->desc_map,
2426 mr->desc_size * mr->max_descs,
2427 DMA_TO_DEVICE);
2428
2429 return n;
2430 }
A mirror of Linus' kernel repository