]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
14fa93cd SJ |
2 | #include <linux/gfp.h> |
3 | #include <linux/workqueue.h> | |
4 | #include <crypto/internal/skcipher.h> | |
5 | ||
6 | #include "nitrox_dev.h" | |
7 | #include "nitrox_req.h" | |
8 | #include "nitrox_csr.h" | |
9 | #include "nitrox_req.h" | |
10 | ||
11 | /* SLC_STORE_INFO */ | |
12 | #define MIN_UDD_LEN 16 | |
13 | /* PKT_IN_HDR + SLC_STORE_INFO */ | |
14 | #define FDATA_SIZE 32 | |
15 | /* Base destination port for the solicited requests */ | |
16 | #define SOLICIT_BASE_DPORT 256 | |
17 | #define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL | |
18 | ||
19 | #define REQ_NOT_POSTED 1 | |
20 | #define REQ_BACKLOG 2 | |
21 | #define REQ_POSTED 3 | |
22 | ||
23 | /** | |
24 | * Response codes from SE microcode | |
25 | * 0x00 - Success | |
26 | * Completion with no error | |
27 | * 0x43 - ERR_GC_DATA_LEN_INVALID | |
28 | * Invalid Data length if Encryption Data length is | |
29 | * less than 16 bytes for AES-XTS and AES-CTS. | |
30 | * 0x45 - ERR_GC_CTX_LEN_INVALID | |
31 | * Invalid context length: CTXL != 23 words. | |
32 | * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID | |
33 | * DOCSIS support is enabled with other than | |
34 | * AES/DES-CBC mode encryption. | |
35 | * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID | |
36 | * Authentication offset is other than 0 with | |
37 | * Encryption IV source = 0. | |
38 | * Authentication offset is other than 8 (DES)/16 (AES) | |
39 | * with Encryption IV source = 1 | |
40 | * 0x51 - ERR_GC_CRC32_INVALID_SELECTION | |
41 | * CRC32 is enabled for other than DOCSIS encryption. | |
42 | * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID | |
43 | * Invalid flag options in AES-CCM IV. | |
44 | */ | |
45 | ||
46 | /** | |
47 | * dma_free_sglist - unmap and free the sg lists. | |
48 | * @ndev: N5 device | |
49 | * @sgtbl: SG table | |
50 | */ | |
51 | static void softreq_unmap_sgbufs(struct nitrox_softreq *sr) | |
52 | { | |
53 | struct nitrox_device *ndev = sr->ndev; | |
54 | struct device *dev = DEV(ndev); | |
55 | struct nitrox_sglist *sglist; | |
56 | ||
57 | /* unmap in sgbuf */ | |
58 | sglist = sr->in.sglist; | |
59 | if (!sglist) | |
60 | goto out_unmap; | |
61 | ||
62 | /* unmap iv */ | |
63 | dma_unmap_single(dev, sglist->dma, sglist->len, DMA_BIDIRECTIONAL); | |
64 | /* unmpa src sglist */ | |
65 | dma_unmap_sg(dev, sr->in.buf, (sr->in.map_bufs_cnt - 1), sr->in.dir); | |
66 | /* unamp gather component */ | |
67 | dma_unmap_single(dev, sr->in.dma, sr->in.len, DMA_TO_DEVICE); | |
68 | kfree(sr->in.sglist); | |
69 | kfree(sr->in.sgcomp); | |
70 | sr->in.sglist = NULL; | |
71 | sr->in.buf = NULL; | |
72 | sr->in.map_bufs_cnt = 0; | |
73 | ||
74 | out_unmap: | |
75 | /* unmap out sgbuf */ | |
76 | sglist = sr->out.sglist; | |
77 | if (!sglist) | |
78 | return; | |
79 | ||
80 | /* unmap orh */ | |
81 | dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir); | |
82 | ||
83 | /* unmap dst sglist */ | |
84 | if (!sr->inplace) { | |
85 | dma_unmap_sg(dev, sr->out.buf, (sr->out.map_bufs_cnt - 3), | |
86 | sr->out.dir); | |
87 | } | |
88 | /* unmap completion */ | |
89 | dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir); | |
90 | ||
91 | /* unmap scatter component */ | |
92 | dma_unmap_single(dev, sr->out.dma, sr->out.len, DMA_TO_DEVICE); | |
93 | kfree(sr->out.sglist); | |
94 | kfree(sr->out.sgcomp); | |
95 | sr->out.sglist = NULL; | |
96 | sr->out.buf = NULL; | |
97 | sr->out.map_bufs_cnt = 0; | |
98 | } | |
99 | ||
100 | static void softreq_destroy(struct nitrox_softreq *sr) | |
101 | { | |
102 | softreq_unmap_sgbufs(sr); | |
103 | kfree(sr); | |
104 | } | |
105 | ||
106 | /** | |
107 | * create_sg_component - create SG componets for N5 device. | |
108 | * @sr: Request structure | |
109 | * @sgtbl: SG table | |
110 | * @nr_comp: total number of components required | |
111 | * | |
112 | * Component structure | |
113 | * | |
114 | * 63 48 47 32 31 16 15 0 | |
115 | * -------------------------------------- | |
116 | * | LEN0 | LEN1 | LEN2 | LEN3 | | |
117 | * |------------------------------------- | |
118 | * | PTR0 | | |
119 | * -------------------------------------- | |
120 | * | PTR1 | | |
121 | * -------------------------------------- | |
122 | * | PTR2 | | |
123 | * -------------------------------------- | |
124 | * | PTR3 | | |
125 | * -------------------------------------- | |
126 | * | |
127 | * Returns 0 if success or a negative errno code on error. | |
128 | */ | |
129 | static int create_sg_component(struct nitrox_softreq *sr, | |
130 | struct nitrox_sgtable *sgtbl, int map_nents) | |
131 | { | |
132 | struct nitrox_device *ndev = sr->ndev; | |
133 | struct nitrox_sgcomp *sgcomp; | |
134 | struct nitrox_sglist *sglist; | |
135 | dma_addr_t dma; | |
136 | size_t sz_comp; | |
137 | int i, j, nr_sgcomp; | |
138 | ||
139 | nr_sgcomp = roundup(map_nents, 4) / 4; | |
140 | ||
141 | /* each component holds 4 dma pointers */ | |
142 | sz_comp = nr_sgcomp * sizeof(*sgcomp); | |
143 | sgcomp = kzalloc(sz_comp, sr->gfp); | |
144 | if (!sgcomp) | |
145 | return -ENOMEM; | |
146 | ||
147 | sgtbl->sgcomp = sgcomp; | |
148 | sgtbl->nr_sgcomp = nr_sgcomp; | |
149 | ||
150 | sglist = sgtbl->sglist; | |
151 | /* populate device sg component */ | |
152 | for (i = 0; i < nr_sgcomp; i++) { | |
153 | for (j = 0; j < 4; j++) { | |
154 | sgcomp->len[j] = cpu_to_be16(sglist->len); | |
155 | sgcomp->dma[j] = cpu_to_be64(sglist->dma); | |
156 | sglist++; | |
157 | } | |
158 | sgcomp++; | |
159 | } | |
160 | /* map the device sg component */ | |
161 | dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE); | |
162 | if (dma_mapping_error(DEV(ndev), dma)) { | |
163 | kfree(sgtbl->sgcomp); | |
164 | sgtbl->sgcomp = NULL; | |
165 | return -ENOMEM; | |
166 | } | |
167 | ||
168 | sgtbl->dma = dma; | |
169 | sgtbl->len = sz_comp; | |
170 | ||
171 | return 0; | |
172 | } | |
173 | ||
174 | /** | |
175 | * dma_map_inbufs - DMA map input sglist and creates sglist component | |
176 | * for N5 device. | |
177 | * @sr: Request structure | |
178 | * @req: Crypto request structre | |
179 | * | |
180 | * Returns 0 if successful or a negative errno code on error. | |
181 | */ | |
182 | static int dma_map_inbufs(struct nitrox_softreq *sr, | |
183 | struct se_crypto_request *req) | |
184 | { | |
185 | struct device *dev = DEV(sr->ndev); | |
186 | struct scatterlist *sg = req->src; | |
187 | struct nitrox_sglist *glist; | |
188 | int i, nents, ret = 0; | |
189 | dma_addr_t dma; | |
190 | size_t sz; | |
191 | ||
192 | nents = sg_nents(req->src); | |
193 | ||
194 | /* creater gather list IV and src entries */ | |
195 | sz = roundup((1 + nents), 4) * sizeof(*glist); | |
196 | glist = kzalloc(sz, sr->gfp); | |
197 | if (!glist) | |
198 | return -ENOMEM; | |
199 | ||
200 | sr->in.sglist = glist; | |
201 | /* map IV */ | |
202 | dma = dma_map_single(dev, &req->iv, req->ivsize, DMA_BIDIRECTIONAL); | |
f2339eb9 DC |
203 | if (dma_mapping_error(dev, dma)) { |
204 | ret = -EINVAL; | |
14fa93cd | 205 | goto iv_map_err; |
f2339eb9 | 206 | } |
14fa93cd SJ |
207 | |
208 | sr->in.dir = (req->src == req->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; | |
209 | /* map src entries */ | |
210 | nents = dma_map_sg(dev, req->src, nents, sr->in.dir); | |
211 | if (!nents) { | |
212 | ret = -EINVAL; | |
213 | goto src_map_err; | |
214 | } | |
215 | sr->in.buf = req->src; | |
216 | ||
217 | /* store the mappings */ | |
218 | glist->len = req->ivsize; | |
219 | glist->dma = dma; | |
220 | glist++; | |
221 | sr->in.total_bytes += req->ivsize; | |
222 | ||
223 | for_each_sg(req->src, sg, nents, i) { | |
224 | glist->len = sg_dma_len(sg); | |
225 | glist->dma = sg_dma_address(sg); | |
226 | sr->in.total_bytes += glist->len; | |
227 | glist++; | |
228 | } | |
229 | /* roundup map count to align with entires in sg component */ | |
230 | sr->in.map_bufs_cnt = (1 + nents); | |
231 | ||
232 | /* create NITROX gather component */ | |
233 | ret = create_sg_component(sr, &sr->in, sr->in.map_bufs_cnt); | |
234 | if (ret) | |
235 | goto incomp_err; | |
236 | ||
237 | return 0; | |
238 | ||
239 | incomp_err: | |
240 | dma_unmap_sg(dev, req->src, nents, sr->in.dir); | |
241 | sr->in.map_bufs_cnt = 0; | |
242 | src_map_err: | |
243 | dma_unmap_single(dev, dma, req->ivsize, DMA_BIDIRECTIONAL); | |
244 | iv_map_err: | |
245 | kfree(sr->in.sglist); | |
246 | sr->in.sglist = NULL; | |
247 | return ret; | |
248 | } | |
249 | ||
250 | static int dma_map_outbufs(struct nitrox_softreq *sr, | |
251 | struct se_crypto_request *req) | |
252 | { | |
253 | struct device *dev = DEV(sr->ndev); | |
254 | struct nitrox_sglist *glist = sr->in.sglist; | |
255 | struct nitrox_sglist *slist; | |
256 | struct scatterlist *sg; | |
257 | int i, nents, map_bufs_cnt, ret = 0; | |
258 | size_t sz; | |
259 | ||
260 | nents = sg_nents(req->dst); | |
261 | ||
262 | /* create scatter list ORH, IV, dst entries and Completion header */ | |
263 | sz = roundup((3 + nents), 4) * sizeof(*slist); | |
264 | slist = kzalloc(sz, sr->gfp); | |
265 | if (!slist) | |
266 | return -ENOMEM; | |
267 | ||
268 | sr->out.sglist = slist; | |
269 | sr->out.dir = DMA_BIDIRECTIONAL; | |
270 | /* map ORH */ | |
271 | sr->resp.orh_dma = dma_map_single(dev, &sr->resp.orh, ORH_HLEN, | |
272 | sr->out.dir); | |
f2339eb9 DC |
273 | if (dma_mapping_error(dev, sr->resp.orh_dma)) { |
274 | ret = -EINVAL; | |
14fa93cd | 275 | goto orh_map_err; |
f2339eb9 | 276 | } |
14fa93cd SJ |
277 | |
278 | /* map completion */ | |
279 | sr->resp.completion_dma = dma_map_single(dev, &sr->resp.completion, | |
280 | COMP_HLEN, sr->out.dir); | |
f2339eb9 DC |
281 | if (dma_mapping_error(dev, sr->resp.completion_dma)) { |
282 | ret = -EINVAL; | |
14fa93cd | 283 | goto compl_map_err; |
f2339eb9 | 284 | } |
14fa93cd SJ |
285 | |
286 | sr->inplace = (req->src == req->dst) ? true : false; | |
287 | /* out place */ | |
288 | if (!sr->inplace) { | |
289 | nents = dma_map_sg(dev, req->dst, nents, sr->out.dir); | |
290 | if (!nents) { | |
291 | ret = -EINVAL; | |
292 | goto dst_map_err; | |
293 | } | |
294 | } | |
295 | sr->out.buf = req->dst; | |
296 | ||
297 | /* store the mappings */ | |
298 | /* orh */ | |
299 | slist->len = ORH_HLEN; | |
300 | slist->dma = sr->resp.orh_dma; | |
301 | slist++; | |
302 | ||
303 | /* copy the glist mappings */ | |
304 | if (sr->inplace) { | |
305 | nents = sr->in.map_bufs_cnt - 1; | |
306 | map_bufs_cnt = sr->in.map_bufs_cnt; | |
307 | while (map_bufs_cnt--) { | |
308 | slist->len = glist->len; | |
309 | slist->dma = glist->dma; | |
310 | slist++; | |
311 | glist++; | |
312 | } | |
313 | } else { | |
314 | /* copy iv mapping */ | |
315 | slist->len = glist->len; | |
316 | slist->dma = glist->dma; | |
317 | slist++; | |
318 | /* copy remaining maps */ | |
319 | for_each_sg(req->dst, sg, nents, i) { | |
320 | slist->len = sg_dma_len(sg); | |
321 | slist->dma = sg_dma_address(sg); | |
322 | slist++; | |
323 | } | |
324 | } | |
325 | ||
326 | /* completion */ | |
327 | slist->len = COMP_HLEN; | |
328 | slist->dma = sr->resp.completion_dma; | |
329 | ||
330 | sr->out.map_bufs_cnt = (3 + nents); | |
331 | ||
332 | ret = create_sg_component(sr, &sr->out, sr->out.map_bufs_cnt); | |
333 | if (ret) | |
334 | goto outcomp_map_err; | |
335 | ||
336 | return 0; | |
337 | ||
338 | outcomp_map_err: | |
339 | if (!sr->inplace) | |
340 | dma_unmap_sg(dev, req->dst, nents, sr->out.dir); | |
341 | sr->out.map_bufs_cnt = 0; | |
342 | sr->out.buf = NULL; | |
343 | dst_map_err: | |
344 | dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir); | |
345 | sr->resp.completion_dma = 0; | |
346 | compl_map_err: | |
347 | dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir); | |
348 | sr->resp.orh_dma = 0; | |
349 | orh_map_err: | |
350 | kfree(sr->out.sglist); | |
351 | sr->out.sglist = NULL; | |
352 | return ret; | |
353 | } | |
354 | ||
355 | static inline int softreq_map_iobuf(struct nitrox_softreq *sr, | |
356 | struct se_crypto_request *creq) | |
357 | { | |
358 | int ret; | |
359 | ||
360 | ret = dma_map_inbufs(sr, creq); | |
361 | if (ret) | |
362 | return ret; | |
363 | ||
364 | ret = dma_map_outbufs(sr, creq); | |
365 | if (ret) | |
366 | softreq_unmap_sgbufs(sr); | |
367 | ||
368 | return ret; | |
369 | } | |
370 | ||
371 | static inline void backlog_list_add(struct nitrox_softreq *sr, | |
372 | struct nitrox_cmdq *cmdq) | |
373 | { | |
374 | INIT_LIST_HEAD(&sr->backlog); | |
375 | ||
376 | spin_lock_bh(&cmdq->backlog_lock); | |
377 | list_add_tail(&sr->backlog, &cmdq->backlog_head); | |
378 | atomic_inc(&cmdq->backlog_count); | |
379 | atomic_set(&sr->status, REQ_BACKLOG); | |
380 | spin_unlock_bh(&cmdq->backlog_lock); | |
381 | } | |
382 | ||
383 | static inline void response_list_add(struct nitrox_softreq *sr, | |
384 | struct nitrox_cmdq *cmdq) | |
385 | { | |
386 | INIT_LIST_HEAD(&sr->response); | |
387 | ||
388 | spin_lock_bh(&cmdq->response_lock); | |
389 | list_add_tail(&sr->response, &cmdq->response_head); | |
390 | spin_unlock_bh(&cmdq->response_lock); | |
391 | } | |
392 | ||
393 | static inline void response_list_del(struct nitrox_softreq *sr, | |
394 | struct nitrox_cmdq *cmdq) | |
395 | { | |
396 | spin_lock_bh(&cmdq->response_lock); | |
397 | list_del(&sr->response); | |
398 | spin_unlock_bh(&cmdq->response_lock); | |
399 | } | |
400 | ||
401 | static struct nitrox_softreq * | |
402 | get_first_response_entry(struct nitrox_cmdq *cmdq) | |
403 | { | |
404 | return list_first_entry_or_null(&cmdq->response_head, | |
405 | struct nitrox_softreq, response); | |
406 | } | |
407 | ||
408 | static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen) | |
409 | { | |
410 | if (atomic_inc_return(&cmdq->pending_count) > qlen) { | |
411 | atomic_dec(&cmdq->pending_count); | |
412 | /* sync with other cpus */ | |
413 | smp_mb__after_atomic(); | |
414 | return true; | |
415 | } | |
416 | return false; | |
417 | } | |
418 | ||
419 | /** | |
420 | * post_se_instr - Post SE instruction to Packet Input ring | |
421 | * @sr: Request structure | |
422 | * | |
423 | * Returns 0 if successful or a negative error code, | |
424 | * if no space in ring. | |
425 | */ | |
426 | static void post_se_instr(struct nitrox_softreq *sr, | |
427 | struct nitrox_cmdq *cmdq) | |
428 | { | |
429 | struct nitrox_device *ndev = sr->ndev; | |
430 | union nps_pkt_in_instr_baoff_dbell pkt_in_baoff_dbell; | |
431 | u64 offset; | |
432 | u8 *ent; | |
433 | ||
434 | spin_lock_bh(&cmdq->cmdq_lock); | |
435 | ||
436 | /* get the next write offset */ | |
437 | offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(cmdq->qno); | |
438 | pkt_in_baoff_dbell.value = nitrox_read_csr(ndev, offset); | |
439 | /* copy the instruction */ | |
440 | ent = cmdq->head + pkt_in_baoff_dbell.s.aoff; | |
441 | memcpy(ent, &sr->instr, cmdq->instr_size); | |
442 | /* flush the command queue updates */ | |
443 | dma_wmb(); | |
444 | ||
445 | sr->tstamp = jiffies; | |
446 | atomic_set(&sr->status, REQ_POSTED); | |
447 | response_list_add(sr, cmdq); | |
448 | ||
449 | /* Ring doorbell with count 1 */ | |
450 | writeq(1, cmdq->dbell_csr_addr); | |
451 | /* orders the doorbell rings */ | |
452 | mmiowb(); | |
453 | ||
454 | spin_unlock_bh(&cmdq->cmdq_lock); | |
455 | } | |
456 | ||
457 | static int post_backlog_cmds(struct nitrox_cmdq *cmdq) | |
458 | { | |
459 | struct nitrox_device *ndev = cmdq->ndev; | |
460 | struct nitrox_softreq *sr, *tmp; | |
461 | int ret = 0; | |
462 | ||
463 | spin_lock_bh(&cmdq->backlog_lock); | |
464 | ||
465 | list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) { | |
466 | struct skcipher_request *skreq; | |
467 | ||
468 | /* submit until space available */ | |
469 | if (unlikely(cmdq_full(cmdq, ndev->qlen))) { | |
470 | ret = -EBUSY; | |
471 | break; | |
472 | } | |
473 | /* delete from backlog list */ | |
474 | list_del(&sr->backlog); | |
475 | atomic_dec(&cmdq->backlog_count); | |
476 | /* sync with other cpus */ | |
477 | smp_mb__after_atomic(); | |
478 | ||
479 | skreq = sr->skreq; | |
480 | /* post the command */ | |
481 | post_se_instr(sr, cmdq); | |
482 | ||
483 | /* backlog requests are posted, wakeup with -EINPROGRESS */ | |
484 | skcipher_request_complete(skreq, -EINPROGRESS); | |
485 | } | |
486 | spin_unlock_bh(&cmdq->backlog_lock); | |
487 | ||
488 | return ret; | |
489 | } | |
490 | ||
491 | static int nitrox_enqueue_request(struct nitrox_softreq *sr) | |
492 | { | |
493 | struct nitrox_cmdq *cmdq = sr->cmdq; | |
494 | struct nitrox_device *ndev = sr->ndev; | |
495 | int ret = -EBUSY; | |
496 | ||
497 | if (unlikely(cmdq_full(cmdq, ndev->qlen))) { | |
498 | if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) | |
499 | return -EAGAIN; | |
500 | ||
501 | backlog_list_add(sr, cmdq); | |
502 | } else { | |
503 | ret = post_backlog_cmds(cmdq); | |
504 | if (ret) { | |
505 | backlog_list_add(sr, cmdq); | |
506 | return ret; | |
507 | } | |
508 | post_se_instr(sr, cmdq); | |
509 | ret = -EINPROGRESS; | |
510 | } | |
511 | return ret; | |
512 | } | |
513 | ||
514 | /** | |
515 | * nitrox_se_request - Send request to SE core | |
516 | * @ndev: NITROX device | |
517 | * @req: Crypto request | |
518 | * | |
519 | * Returns 0 on success, or a negative error code. | |
520 | */ | |
521 | int nitrox_process_se_request(struct nitrox_device *ndev, | |
522 | struct se_crypto_request *req, | |
523 | completion_t callback, | |
524 | struct skcipher_request *skreq) | |
525 | { | |
526 | struct nitrox_softreq *sr; | |
527 | dma_addr_t ctx_handle = 0; | |
528 | int qno, ret = 0; | |
529 | ||
530 | if (!nitrox_ready(ndev)) | |
531 | return -ENODEV; | |
532 | ||
533 | sr = kzalloc(sizeof(*sr), req->gfp); | |
534 | if (!sr) | |
535 | return -ENOMEM; | |
536 | ||
537 | sr->ndev = ndev; | |
538 | sr->flags = req->flags; | |
539 | sr->gfp = req->gfp; | |
540 | sr->callback = callback; | |
541 | sr->skreq = skreq; | |
542 | ||
543 | atomic_set(&sr->status, REQ_NOT_POSTED); | |
544 | ||
545 | WRITE_ONCE(sr->resp.orh, PENDING_SIG); | |
546 | WRITE_ONCE(sr->resp.completion, PENDING_SIG); | |
547 | ||
548 | ret = softreq_map_iobuf(sr, req); | |
549 | if (ret) { | |
550 | kfree(sr); | |
551 | return ret; | |
552 | } | |
553 | ||
554 | /* get the context handle */ | |
555 | if (req->ctx_handle) { | |
556 | struct ctx_hdr *hdr; | |
557 | u8 *ctx_ptr; | |
558 | ||
559 | ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle; | |
560 | hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr)); | |
561 | ctx_handle = hdr->ctx_dma; | |
562 | } | |
563 | ||
564 | /* select the queue */ | |
565 | qno = smp_processor_id() % ndev->nr_queues; | |
566 | ||
567 | sr->cmdq = &ndev->pkt_cmdqs[qno]; | |
568 | ||
569 | /* | |
570 | * 64-Byte Instruction Format | |
571 | * | |
572 | * ---------------------- | |
573 | * | DPTR0 | 8 bytes | |
574 | * ---------------------- | |
575 | * | PKT_IN_INSTR_HDR | 8 bytes | |
576 | * ---------------------- | |
577 | * | PKT_IN_HDR | 16 bytes | |
578 | * ---------------------- | |
579 | * | SLC_INFO | 16 bytes | |
580 | * ---------------------- | |
581 | * | Front data | 16 bytes | |
582 | * ---------------------- | |
583 | */ | |
584 | ||
585 | /* fill the packet instruction */ | |
586 | /* word 0 */ | |
587 | sr->instr.dptr0 = cpu_to_be64(sr->in.dma); | |
588 | ||
589 | /* word 1 */ | |
590 | sr->instr.ih.value = 0; | |
591 | sr->instr.ih.s.g = 1; | |
592 | sr->instr.ih.s.gsz = sr->in.map_bufs_cnt; | |
593 | sr->instr.ih.s.ssz = sr->out.map_bufs_cnt; | |
594 | sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr); | |
595 | sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes; | |
596 | sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value); | |
597 | ||
598 | /* word 2 */ | |
599 | sr->instr.irh.value[0] = 0; | |
600 | sr->instr.irh.s.uddl = MIN_UDD_LEN; | |
601 | /* context length in 64-bit words */ | |
602 | sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8); | |
603 | /* offset from solicit base port 256 */ | |
604 | sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno; | |
605 | sr->instr.irh.s.ctxc = req->ctrl.s.ctxc; | |
606 | sr->instr.irh.s.arg = req->ctrl.s.arg; | |
607 | sr->instr.irh.s.opcode = req->opcode; | |
608 | sr->instr.irh.value[0] = cpu_to_be64(sr->instr.irh.value[0]); | |
609 | ||
610 | /* word 3 */ | |
611 | sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle); | |
612 | ||
613 | /* word 4 */ | |
614 | sr->instr.slc.value[0] = 0; | |
615 | sr->instr.slc.s.ssz = sr->out.map_bufs_cnt; | |
616 | sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]); | |
617 | ||
618 | /* word 5 */ | |
619 | sr->instr.slc.s.rptr = cpu_to_be64(sr->out.dma); | |
620 | ||
621 | /* | |
622 | * No conversion for front data, | |
623 | * It goes into payload | |
624 | * put GP Header in front data | |
625 | */ | |
626 | sr->instr.fdata[0] = *((u64 *)&req->gph); | |
627 | sr->instr.fdata[1] = 0; | |
628 | /* flush the soft_req changes before posting the cmd */ | |
629 | wmb(); | |
630 | ||
631 | ret = nitrox_enqueue_request(sr); | |
632 | if (ret == -EAGAIN) | |
633 | goto send_fail; | |
634 | ||
635 | return ret; | |
636 | ||
637 | send_fail: | |
638 | softreq_destroy(sr); | |
639 | return ret; | |
640 | } | |
641 | ||
642 | static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout) | |
643 | { | |
644 | return time_after_eq(jiffies, (tstamp + timeout)); | |
645 | } | |
646 | ||
647 | void backlog_qflush_work(struct work_struct *work) | |
648 | { | |
649 | struct nitrox_cmdq *cmdq; | |
650 | ||
651 | cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush); | |
652 | post_backlog_cmds(cmdq); | |
653 | } | |
654 | ||
655 | /** | |
656 | * process_request_list - process completed requests | |
657 | * @ndev: N5 device | |
658 | * @qno: queue to operate | |
659 | * | |
660 | * Returns the number of responses processed. | |
661 | */ | |
662 | static void process_response_list(struct nitrox_cmdq *cmdq) | |
663 | { | |
664 | struct nitrox_device *ndev = cmdq->ndev; | |
665 | struct nitrox_softreq *sr; | |
666 | struct skcipher_request *skreq; | |
667 | completion_t callback; | |
668 | int req_completed = 0, err = 0, budget; | |
669 | ||
670 | /* check all pending requests */ | |
671 | budget = atomic_read(&cmdq->pending_count); | |
672 | ||
673 | while (req_completed < budget) { | |
674 | sr = get_first_response_entry(cmdq); | |
675 | if (!sr) | |
676 | break; | |
677 | ||
678 | if (atomic_read(&sr->status) != REQ_POSTED) | |
679 | break; | |
680 | ||
681 | /* check orh and completion bytes updates */ | |
682 | if (READ_ONCE(sr->resp.orh) == READ_ONCE(sr->resp.completion)) { | |
683 | /* request not completed, check for timeout */ | |
684 | if (!cmd_timeout(sr->tstamp, ndev->timeout)) | |
685 | break; | |
686 | dev_err_ratelimited(DEV(ndev), | |
687 | "Request timeout, orh 0x%016llx\n", | |
688 | READ_ONCE(sr->resp.orh)); | |
689 | } | |
690 | atomic_dec(&cmdq->pending_count); | |
691 | /* sync with other cpus */ | |
692 | smp_mb__after_atomic(); | |
693 | /* remove from response list */ | |
694 | response_list_del(sr, cmdq); | |
695 | ||
696 | callback = sr->callback; | |
697 | skreq = sr->skreq; | |
698 | ||
699 | /* ORH error code */ | |
700 | err = READ_ONCE(sr->resp.orh) & 0xff; | |
701 | softreq_destroy(sr); | |
702 | ||
703 | if (callback) | |
704 | callback(skreq, err); | |
705 | ||
706 | req_completed++; | |
707 | } | |
708 | } | |
709 | ||
710 | /** | |
711 | * pkt_slc_resp_handler - post processing of SE responses | |
712 | */ | |
713 | void pkt_slc_resp_handler(unsigned long data) | |
714 | { | |
715 | struct bh_data *bh = (void *)(uintptr_t)(data); | |
716 | struct nitrox_cmdq *cmdq = bh->cmdq; | |
717 | union nps_pkt_slc_cnts pkt_slc_cnts; | |
718 | ||
719 | /* read completion count */ | |
720 | pkt_slc_cnts.value = readq(bh->completion_cnt_csr_addr); | |
721 | /* resend the interrupt if more work to do */ | |
722 | pkt_slc_cnts.s.resend = 1; | |
723 | ||
724 | process_response_list(cmdq); | |
725 | ||
726 | /* | |
727 | * clear the interrupt with resend bit enabled, | |
728 | * MSI-X interrupt generates if Completion count > Threshold | |
729 | */ | |
730 | writeq(pkt_slc_cnts.value, bh->completion_cnt_csr_addr); | |
731 | /* order the writes */ | |
732 | mmiowb(); | |
733 | ||
734 | if (atomic_read(&cmdq->backlog_count)) | |
735 | schedule_work(&cmdq->backlog_qflush); | |
736 | } |