]>
Commit | Line | Data |
---|---|---|
f931551b RC |
1 | /* |
2 | * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved. | |
3 | * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. | |
4 | * | |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the | |
9 | * OpenIB.org BSD license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or | |
12 | * without modification, are permitted provided that the following | |
13 | * conditions are met: | |
14 | * | |
15 | * - Redistributions of source code must retain the above | |
16 | * copyright notice, this list of conditions and the following | |
17 | * disclaimer. | |
18 | * | |
19 | * - Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials | |
22 | * provided with the distribution. | |
23 | * | |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
31 | * SOFTWARE. | |
32 | */ | |
33 | ||
34 | #include <rdma/ib_umem.h> | |
35 | #include <rdma/ib_smi.h> | |
36 | ||
37 | #include "qib.h" | |
38 | ||
39 | /* Fast memory region */ | |
40 | struct qib_fmr { | |
41 | struct ib_fmr ibfmr; | |
f931551b RC |
42 | struct qib_mregion mr; /* must be last */ |
43 | }; | |
44 | ||
45 | static inline struct qib_fmr *to_ifmr(struct ib_fmr *ibfmr) | |
46 | { | |
47 | return container_of(ibfmr, struct qib_fmr, ibfmr); | |
48 | } | |
49 | ||
6a82649f MM |
50 | static int init_qib_mregion(struct qib_mregion *mr, struct ib_pd *pd, |
51 | int count) | |
52 | { | |
53 | int m, i = 0; | |
54 | int rval = 0; | |
55 | ||
56 | m = (count + QIB_SEGSZ - 1) / QIB_SEGSZ; | |
57 | for (; i < m; i++) { | |
041af0bb | 58 | mr->map[i] = kzalloc(sizeof(*mr->map[0]), GFP_KERNEL); |
6a82649f MM |
59 | if (!mr->map[i]) |
60 | goto bail; | |
61 | } | |
62 | mr->mapsz = m; | |
63 | init_completion(&mr->comp); | |
64 | /* count returning the ptr to user */ | |
65 | atomic_set(&mr->refcount, 1); | |
66 | mr->pd = pd; | |
67 | mr->max_segs = count; | |
68 | out: | |
69 | return rval; | |
70 | bail: | |
71 | while (i) | |
72 | kfree(mr->map[--i]); | |
73 | rval = -ENOMEM; | |
74 | goto out; | |
75 | } | |
76 | ||
77 | static void deinit_qib_mregion(struct qib_mregion *mr) | |
78 | { | |
79 | int i = mr->mapsz; | |
80 | ||
81 | mr->mapsz = 0; | |
82 | while (i) | |
83 | kfree(mr->map[--i]); | |
84 | } | |
85 | ||
86 | ||
f931551b RC |
87 | /** |
88 | * qib_get_dma_mr - get a DMA memory region | |
89 | * @pd: protection domain for this memory region | |
90 | * @acc: access flags | |
91 | * | |
92 | * Returns the memory region on success, otherwise returns an errno. | |
93 | * Note that all DMA addresses should be created via the | |
94 | * struct ib_dma_mapping_ops functions (see qib_dma.c). | |
95 | */ | |
96 | struct ib_mr *qib_get_dma_mr(struct ib_pd *pd, int acc) | |
97 | { | |
6a82649f | 98 | struct qib_mr *mr = NULL; |
f931551b | 99 | struct ib_mr *ret; |
6a82649f | 100 | int rval; |
f931551b RC |
101 | |
102 | if (to_ipd(pd)->user) { | |
103 | ret = ERR_PTR(-EPERM); | |
104 | goto bail; | |
105 | } | |
106 | ||
041af0bb | 107 | mr = kzalloc(sizeof(*mr), GFP_KERNEL); |
f931551b RC |
108 | if (!mr) { |
109 | ret = ERR_PTR(-ENOMEM); | |
110 | goto bail; | |
111 | } | |
112 | ||
6a82649f MM |
113 | rval = init_qib_mregion(&mr->mr, pd, 0); |
114 | if (rval) { | |
115 | ret = ERR_PTR(rval); | |
116 | goto bail; | |
117 | } | |
f931551b | 118 | |
f931551b | 119 | |
6a82649f MM |
120 | rval = qib_alloc_lkey(&mr->mr, 1); |
121 | if (rval) { | |
122 | ret = ERR_PTR(rval); | |
123 | goto bail_mregion; | |
124 | } | |
125 | ||
126 | mr->mr.access_flags = acc; | |
f931551b | 127 | ret = &mr->ibmr; |
6a82649f MM |
128 | done: |
129 | return ret; | |
f931551b | 130 | |
6a82649f MM |
131 | bail_mregion: |
132 | deinit_qib_mregion(&mr->mr); | |
f931551b | 133 | bail: |
6a82649f MM |
134 | kfree(mr); |
135 | goto done; | |
f931551b RC |
136 | } |
137 | ||
6a82649f | 138 | static struct qib_mr *alloc_mr(int count, struct ib_pd *pd) |
f931551b RC |
139 | { |
140 | struct qib_mr *mr; | |
6a82649f MM |
141 | int rval = -ENOMEM; |
142 | int m; | |
f931551b RC |
143 | |
144 | /* Allocate struct plus pointers to first level page tables. */ | |
145 | m = (count + QIB_SEGSZ - 1) / QIB_SEGSZ; | |
041af0bb | 146 | mr = kzalloc(sizeof(*mr) + m * sizeof(mr->mr.map[0]), GFP_KERNEL); |
f931551b | 147 | if (!mr) |
6a82649f | 148 | goto bail; |
f931551b | 149 | |
6a82649f MM |
150 | rval = init_qib_mregion(&mr->mr, pd, count); |
151 | if (rval) | |
152 | goto bail; | |
f931551b RC |
153 | /* |
154 | * ib_reg_phys_mr() will initialize mr->ibmr except for | |
155 | * lkey and rkey. | |
156 | */ | |
6a82649f MM |
157 | rval = qib_alloc_lkey(&mr->mr, 0); |
158 | if (rval) | |
159 | goto bail_mregion; | |
f931551b RC |
160 | mr->ibmr.lkey = mr->mr.lkey; |
161 | mr->ibmr.rkey = mr->mr.lkey; | |
6a82649f MM |
162 | done: |
163 | return mr; | |
f931551b | 164 | |
6a82649f MM |
165 | bail_mregion: |
166 | deinit_qib_mregion(&mr->mr); | |
f931551b | 167 | bail: |
f931551b | 168 | kfree(mr); |
6a82649f MM |
169 | mr = ERR_PTR(rval); |
170 | goto done; | |
f931551b RC |
171 | } |
172 | ||
173 | /** | |
174 | * qib_reg_phys_mr - register a physical memory region | |
175 | * @pd: protection domain for this memory region | |
176 | * @buffer_list: pointer to the list of physical buffers to register | |
177 | * @num_phys_buf: the number of physical buffers to register | |
178 | * @iova_start: the starting address passed over IB which maps to this MR | |
179 | * | |
180 | * Returns the memory region on success, otherwise returns an errno. | |
181 | */ | |
182 | struct ib_mr *qib_reg_phys_mr(struct ib_pd *pd, | |
183 | struct ib_phys_buf *buffer_list, | |
184 | int num_phys_buf, int acc, u64 *iova_start) | |
185 | { | |
186 | struct qib_mr *mr; | |
187 | int n, m, i; | |
188 | struct ib_mr *ret; | |
189 | ||
6a82649f MM |
190 | mr = alloc_mr(num_phys_buf, pd); |
191 | if (IS_ERR(mr)) { | |
192 | ret = (struct ib_mr *)mr; | |
f931551b RC |
193 | goto bail; |
194 | } | |
195 | ||
f931551b RC |
196 | mr->mr.user_base = *iova_start; |
197 | mr->mr.iova = *iova_start; | |
f931551b | 198 | mr->mr.access_flags = acc; |
f931551b RC |
199 | |
200 | m = 0; | |
201 | n = 0; | |
202 | for (i = 0; i < num_phys_buf; i++) { | |
203 | mr->mr.map[m]->segs[n].vaddr = (void *) buffer_list[i].addr; | |
204 | mr->mr.map[m]->segs[n].length = buffer_list[i].size; | |
205 | mr->mr.length += buffer_list[i].size; | |
206 | n++; | |
207 | if (n == QIB_SEGSZ) { | |
208 | m++; | |
209 | n = 0; | |
210 | } | |
211 | } | |
212 | ||
213 | ret = &mr->ibmr; | |
214 | ||
215 | bail: | |
216 | return ret; | |
217 | } | |
218 | ||
219 | /** | |
220 | * qib_reg_user_mr - register a userspace memory region | |
221 | * @pd: protection domain for this memory region | |
222 | * @start: starting userspace address | |
223 | * @length: length of region to register | |
f931551b RC |
224 | * @mr_access_flags: access flags for this memory region |
225 | * @udata: unused by the QLogic_IB driver | |
226 | * | |
227 | * Returns the memory region on success, otherwise returns an errno. | |
228 | */ | |
229 | struct ib_mr *qib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, | |
230 | u64 virt_addr, int mr_access_flags, | |
231 | struct ib_udata *udata) | |
232 | { | |
233 | struct qib_mr *mr; | |
234 | struct ib_umem *umem; | |
eeb8461e YH |
235 | struct scatterlist *sg; |
236 | int n, m, entry; | |
f931551b RC |
237 | struct ib_mr *ret; |
238 | ||
239 | if (length == 0) { | |
240 | ret = ERR_PTR(-EINVAL); | |
241 | goto bail; | |
242 | } | |
243 | ||
244 | umem = ib_umem_get(pd->uobject->context, start, length, | |
245 | mr_access_flags, 0); | |
246 | if (IS_ERR(umem)) | |
247 | return (void *) umem; | |
248 | ||
eeb8461e | 249 | n = umem->nmap; |
f931551b | 250 | |
6a82649f MM |
251 | mr = alloc_mr(n, pd); |
252 | if (IS_ERR(mr)) { | |
253 | ret = (struct ib_mr *)mr; | |
f931551b RC |
254 | ib_umem_release(umem); |
255 | goto bail; | |
256 | } | |
257 | ||
f931551b RC |
258 | mr->mr.user_base = start; |
259 | mr->mr.iova = virt_addr; | |
260 | mr->mr.length = length; | |
406f9e5f | 261 | mr->mr.offset = ib_umem_offset(umem); |
f931551b RC |
262 | mr->mr.access_flags = mr_access_flags; |
263 | mr->umem = umem; | |
264 | ||
2a600f14 MM |
265 | if (is_power_of_2(umem->page_size)) |
266 | mr->mr.page_shift = ilog2(umem->page_size); | |
f931551b RC |
267 | m = 0; |
268 | n = 0; | |
eeb8461e | 269 | for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { |
f931551b RC |
270 | void *vaddr; |
271 | ||
eeb8461e | 272 | vaddr = page_address(sg_page(sg)); |
f931551b RC |
273 | if (!vaddr) { |
274 | ret = ERR_PTR(-EINVAL); | |
275 | goto bail; | |
276 | } | |
277 | mr->mr.map[m]->segs[n].vaddr = vaddr; | |
278 | mr->mr.map[m]->segs[n].length = umem->page_size; | |
279 | n++; | |
280 | if (n == QIB_SEGSZ) { | |
281 | m++; | |
282 | n = 0; | |
283 | } | |
f931551b RC |
284 | } |
285 | ret = &mr->ibmr; | |
286 | ||
287 | bail: | |
288 | return ret; | |
289 | } | |
290 | ||
291 | /** | |
292 | * qib_dereg_mr - unregister and free a memory region | |
293 | * @ibmr: the memory region to free | |
294 | * | |
295 | * Returns 0 on success. | |
296 | * | |
297 | * Note that this is called to free MRs created by qib_get_dma_mr() | |
298 | * or qib_reg_user_mr(). | |
299 | */ | |
300 | int qib_dereg_mr(struct ib_mr *ibmr) | |
301 | { | |
302 | struct qib_mr *mr = to_imr(ibmr); | |
6a82649f MM |
303 | int ret = 0; |
304 | unsigned long timeout; | |
305 | ||
306 | qib_free_lkey(&mr->mr); | |
307 | ||
308 | qib_put_mr(&mr->mr); /* will set completion if last */ | |
309 | timeout = wait_for_completion_timeout(&mr->mr.comp, | |
310 | 5 * HZ); | |
311 | if (!timeout) { | |
312 | qib_get_mr(&mr->mr); | |
313 | ret = -EBUSY; | |
314 | goto out; | |
315 | } | |
316 | deinit_qib_mregion(&mr->mr); | |
f931551b RC |
317 | if (mr->umem) |
318 | ib_umem_release(mr->umem); | |
319 | kfree(mr); | |
6a82649f MM |
320 | out: |
321 | return ret; | |
f931551b RC |
322 | } |
323 | ||
324 | /* | |
325 | * Allocate a memory region usable with the | |
326 | * IB_WR_FAST_REG_MR send work request. | |
327 | * | |
328 | * Return the memory region on success, otherwise return an errno. | |
329 | */ | |
1302f845 SG |
330 | struct ib_mr *qib_alloc_mr(struct ib_pd *pd, |
331 | enum ib_mr_type mr_type, | |
332 | u32 max_num_sg) | |
f931551b RC |
333 | { |
334 | struct qib_mr *mr; | |
335 | ||
1302f845 SG |
336 | if (mr_type != IB_MR_TYPE_MEM_REG) |
337 | return ERR_PTR(-EINVAL); | |
338 | ||
339 | mr = alloc_mr(max_num_sg, pd); | |
6a82649f MM |
340 | if (IS_ERR(mr)) |
341 | return (struct ib_mr *)mr; | |
f931551b RC |
342 | |
343 | return &mr->ibmr; | |
344 | } | |
345 | ||
346 | struct ib_fast_reg_page_list * | |
347 | qib_alloc_fast_reg_page_list(struct ib_device *ibdev, int page_list_len) | |
348 | { | |
349 | unsigned size = page_list_len * sizeof(u64); | |
350 | struct ib_fast_reg_page_list *pl; | |
351 | ||
352 | if (size > PAGE_SIZE) | |
353 | return ERR_PTR(-EINVAL); | |
354 | ||
041af0bb | 355 | pl = kzalloc(sizeof(*pl), GFP_KERNEL); |
f931551b RC |
356 | if (!pl) |
357 | return ERR_PTR(-ENOMEM); | |
358 | ||
6a82649f | 359 | pl->page_list = kzalloc(size, GFP_KERNEL); |
f931551b RC |
360 | if (!pl->page_list) |
361 | goto err_free; | |
362 | ||
363 | return pl; | |
364 | ||
365 | err_free: | |
366 | kfree(pl); | |
367 | return ERR_PTR(-ENOMEM); | |
368 | } | |
369 | ||
370 | void qib_free_fast_reg_page_list(struct ib_fast_reg_page_list *pl) | |
371 | { | |
372 | kfree(pl->page_list); | |
373 | kfree(pl); | |
374 | } | |
375 | ||
376 | /** | |
377 | * qib_alloc_fmr - allocate a fast memory region | |
378 | * @pd: the protection domain for this memory region | |
379 | * @mr_access_flags: access flags for this memory region | |
380 | * @fmr_attr: fast memory region attributes | |
381 | * | |
382 | * Returns the memory region on success, otherwise returns an errno. | |
383 | */ | |
384 | struct ib_fmr *qib_alloc_fmr(struct ib_pd *pd, int mr_access_flags, | |
385 | struct ib_fmr_attr *fmr_attr) | |
386 | { | |
387 | struct qib_fmr *fmr; | |
6a82649f | 388 | int m; |
f931551b | 389 | struct ib_fmr *ret; |
6a82649f | 390 | int rval = -ENOMEM; |
f931551b RC |
391 | |
392 | /* Allocate struct plus pointers to first level page tables. */ | |
393 | m = (fmr_attr->max_pages + QIB_SEGSZ - 1) / QIB_SEGSZ; | |
041af0bb | 394 | fmr = kzalloc(sizeof(*fmr) + m * sizeof(fmr->mr.map[0]), GFP_KERNEL); |
f931551b RC |
395 | if (!fmr) |
396 | goto bail; | |
397 | ||
6a82649f MM |
398 | rval = init_qib_mregion(&fmr->mr, pd, fmr_attr->max_pages); |
399 | if (rval) | |
400 | goto bail; | |
f931551b RC |
401 | |
402 | /* | |
403 | * ib_alloc_fmr() will initialize fmr->ibfmr except for lkey & | |
404 | * rkey. | |
405 | */ | |
6a82649f MM |
406 | rval = qib_alloc_lkey(&fmr->mr, 0); |
407 | if (rval) | |
408 | goto bail_mregion; | |
f931551b RC |
409 | fmr->ibfmr.rkey = fmr->mr.lkey; |
410 | fmr->ibfmr.lkey = fmr->mr.lkey; | |
411 | /* | |
412 | * Resources are allocated but no valid mapping (RKEY can't be | |
413 | * used). | |
414 | */ | |
f931551b RC |
415 | fmr->mr.access_flags = mr_access_flags; |
416 | fmr->mr.max_segs = fmr_attr->max_pages; | |
2a600f14 | 417 | fmr->mr.page_shift = fmr_attr->page_shift; |
f931551b | 418 | |
f931551b | 419 | ret = &fmr->ibfmr; |
6a82649f MM |
420 | done: |
421 | return ret; | |
f931551b | 422 | |
6a82649f MM |
423 | bail_mregion: |
424 | deinit_qib_mregion(&fmr->mr); | |
f931551b | 425 | bail: |
f931551b | 426 | kfree(fmr); |
6a82649f MM |
427 | ret = ERR_PTR(rval); |
428 | goto done; | |
f931551b RC |
429 | } |
430 | ||
431 | /** | |
432 | * qib_map_phys_fmr - set up a fast memory region | |
433 | * @ibmfr: the fast memory region to set up | |
434 | * @page_list: the list of pages to associate with the fast memory region | |
435 | * @list_len: the number of pages to associate with the fast memory region | |
436 | * @iova: the virtual address of the start of the fast memory region | |
437 | * | |
438 | * This may be called from interrupt context. | |
439 | */ | |
440 | ||
441 | int qib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, | |
442 | int list_len, u64 iova) | |
443 | { | |
444 | struct qib_fmr *fmr = to_ifmr(ibfmr); | |
445 | struct qib_lkey_table *rkt; | |
446 | unsigned long flags; | |
447 | int m, n, i; | |
448 | u32 ps; | |
449 | int ret; | |
450 | ||
6a82649f MM |
451 | i = atomic_read(&fmr->mr.refcount); |
452 | if (i > 2) | |
f931551b RC |
453 | return -EBUSY; |
454 | ||
455 | if (list_len > fmr->mr.max_segs) { | |
456 | ret = -EINVAL; | |
457 | goto bail; | |
458 | } | |
459 | rkt = &to_idev(ibfmr->device)->lk_table; | |
460 | spin_lock_irqsave(&rkt->lock, flags); | |
461 | fmr->mr.user_base = iova; | |
462 | fmr->mr.iova = iova; | |
2a600f14 | 463 | ps = 1 << fmr->mr.page_shift; |
f931551b RC |
464 | fmr->mr.length = list_len * ps; |
465 | m = 0; | |
466 | n = 0; | |
467 | for (i = 0; i < list_len; i++) { | |
468 | fmr->mr.map[m]->segs[n].vaddr = (void *) page_list[i]; | |
469 | fmr->mr.map[m]->segs[n].length = ps; | |
470 | if (++n == QIB_SEGSZ) { | |
471 | m++; | |
472 | n = 0; | |
473 | } | |
474 | } | |
475 | spin_unlock_irqrestore(&rkt->lock, flags); | |
476 | ret = 0; | |
477 | ||
478 | bail: | |
479 | return ret; | |
480 | } | |
481 | ||
482 | /** | |
483 | * qib_unmap_fmr - unmap fast memory regions | |
484 | * @fmr_list: the list of fast memory regions to unmap | |
485 | * | |
486 | * Returns 0 on success. | |
487 | */ | |
488 | int qib_unmap_fmr(struct list_head *fmr_list) | |
489 | { | |
490 | struct qib_fmr *fmr; | |
491 | struct qib_lkey_table *rkt; | |
492 | unsigned long flags; | |
493 | ||
494 | list_for_each_entry(fmr, fmr_list, ibfmr.list) { | |
495 | rkt = &to_idev(fmr->ibfmr.device)->lk_table; | |
496 | spin_lock_irqsave(&rkt->lock, flags); | |
497 | fmr->mr.user_base = 0; | |
498 | fmr->mr.iova = 0; | |
499 | fmr->mr.length = 0; | |
500 | spin_unlock_irqrestore(&rkt->lock, flags); | |
501 | } | |
502 | return 0; | |
503 | } | |
504 | ||
505 | /** | |
506 | * qib_dealloc_fmr - deallocate a fast memory region | |
507 | * @ibfmr: the fast memory region to deallocate | |
508 | * | |
509 | * Returns 0 on success. | |
510 | */ | |
511 | int qib_dealloc_fmr(struct ib_fmr *ibfmr) | |
512 | { | |
513 | struct qib_fmr *fmr = to_ifmr(ibfmr); | |
6a82649f MM |
514 | int ret = 0; |
515 | unsigned long timeout; | |
516 | ||
517 | qib_free_lkey(&fmr->mr); | |
518 | qib_put_mr(&fmr->mr); /* will set completion if last */ | |
519 | timeout = wait_for_completion_timeout(&fmr->mr.comp, | |
520 | 5 * HZ); | |
521 | if (!timeout) { | |
522 | qib_get_mr(&fmr->mr); | |
523 | ret = -EBUSY; | |
524 | goto out; | |
525 | } | |
526 | deinit_qib_mregion(&fmr->mr); | |
f931551b | 527 | kfree(fmr); |
6a82649f MM |
528 | out: |
529 | return ret; | |
f931551b | 530 | } |
8aac4cc3 MM |
531 | |
532 | void mr_rcu_callback(struct rcu_head *list) | |
533 | { | |
534 | struct qib_mregion *mr = container_of(list, struct qib_mregion, list); | |
535 | ||
536 | complete(&mr->comp); | |
537 | } |