]> git.ipfire.org Git - thirdparty/linux.git/blob - drivers/dma/idxd/device.c
io_uring: reset -EBUSY error when io sq thread is waken up
[thirdparty/linux.git] / drivers / dma / idxd / device.c
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #include <linux/init.h>
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <linux/pci.h>
7 #include <linux/io-64-nonatomic-lo-hi.h>
8 #include <linux/dmaengine.h>
9 #include <uapi/linux/idxd.h>
10 #include "../dmaengine.h"
11 #include "idxd.h"
12 #include "registers.h"
13
14 static int idxd_cmd_wait(struct idxd_device *idxd, u32 *status, int timeout);
15 static int idxd_cmd_send(struct idxd_device *idxd, int cmd_code, u32 operand);
16
17 /* Interrupt control bits */
18 int idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id)
19 {
20 struct pci_dev *pdev = idxd->pdev;
21 int msixcnt = pci_msix_vec_count(pdev);
22 union msix_perm perm;
23 u32 offset;
24
25 if (vec_id < 0 || vec_id >= msixcnt)
26 return -EINVAL;
27
28 offset = idxd->msix_perm_offset + vec_id * 8;
29 perm.bits = ioread32(idxd->reg_base + offset);
30 perm.ignore = 1;
31 iowrite32(perm.bits, idxd->reg_base + offset);
32
33 return 0;
34 }
35
36 void idxd_mask_msix_vectors(struct idxd_device *idxd)
37 {
38 struct pci_dev *pdev = idxd->pdev;
39 int msixcnt = pci_msix_vec_count(pdev);
40 int i, rc;
41
42 for (i = 0; i < msixcnt; i++) {
43 rc = idxd_mask_msix_vector(idxd, i);
44 if (rc < 0)
45 dev_warn(&pdev->dev,
46 "Failed disabling msix vec %d\n", i);
47 }
48 }
49
50 int idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id)
51 {
52 struct pci_dev *pdev = idxd->pdev;
53 int msixcnt = pci_msix_vec_count(pdev);
54 union msix_perm perm;
55 u32 offset;
56
57 if (vec_id < 0 || vec_id >= msixcnt)
58 return -EINVAL;
59
60 offset = idxd->msix_perm_offset + vec_id * 8;
61 perm.bits = ioread32(idxd->reg_base + offset);
62 perm.ignore = 0;
63 iowrite32(perm.bits, idxd->reg_base + offset);
64
65 return 0;
66 }
67
68 void idxd_unmask_error_interrupts(struct idxd_device *idxd)
69 {
70 union genctrl_reg genctrl;
71
72 genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
73 genctrl.softerr_int_en = 1;
74 iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
75 }
76
77 void idxd_mask_error_interrupts(struct idxd_device *idxd)
78 {
79 union genctrl_reg genctrl;
80
81 genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
82 genctrl.softerr_int_en = 0;
83 iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
84 }
85
86 static void free_hw_descs(struct idxd_wq *wq)
87 {
88 int i;
89
90 for (i = 0; i < wq->num_descs; i++)
91 kfree(wq->hw_descs[i]);
92
93 kfree(wq->hw_descs);
94 }
95
96 static int alloc_hw_descs(struct idxd_wq *wq, int num)
97 {
98 struct device *dev = &wq->idxd->pdev->dev;
99 int i;
100 int node = dev_to_node(dev);
101
102 wq->hw_descs = kcalloc_node(num, sizeof(struct dsa_hw_desc *),
103 GFP_KERNEL, node);
104 if (!wq->hw_descs)
105 return -ENOMEM;
106
107 for (i = 0; i < num; i++) {
108 wq->hw_descs[i] = kzalloc_node(sizeof(*wq->hw_descs[i]),
109 GFP_KERNEL, node);
110 if (!wq->hw_descs[i]) {
111 free_hw_descs(wq);
112 return -ENOMEM;
113 }
114 }
115
116 return 0;
117 }
118
119 static void free_descs(struct idxd_wq *wq)
120 {
121 int i;
122
123 for (i = 0; i < wq->num_descs; i++)
124 kfree(wq->descs[i]);
125
126 kfree(wq->descs);
127 }
128
129 static int alloc_descs(struct idxd_wq *wq, int num)
130 {
131 struct device *dev = &wq->idxd->pdev->dev;
132 int i;
133 int node = dev_to_node(dev);
134
135 wq->descs = kcalloc_node(num, sizeof(struct idxd_desc *),
136 GFP_KERNEL, node);
137 if (!wq->descs)
138 return -ENOMEM;
139
140 for (i = 0; i < num; i++) {
141 wq->descs[i] = kzalloc_node(sizeof(*wq->descs[i]),
142 GFP_KERNEL, node);
143 if (!wq->descs[i]) {
144 free_descs(wq);
145 return -ENOMEM;
146 }
147 }
148
149 return 0;
150 }
151
152 /* WQ control bits */
153 int idxd_wq_alloc_resources(struct idxd_wq *wq)
154 {
155 struct idxd_device *idxd = wq->idxd;
156 struct idxd_group *group = wq->group;
157 struct device *dev = &idxd->pdev->dev;
158 int rc, num_descs, i;
159
160 if (wq->type != IDXD_WQT_KERNEL)
161 return 0;
162
163 num_descs = wq->size +
164 idxd->hw.gen_cap.max_descs_per_engine * group->num_engines;
165 wq->num_descs = num_descs;
166
167 rc = alloc_hw_descs(wq, num_descs);
168 if (rc < 0)
169 return rc;
170
171 wq->compls_size = num_descs * sizeof(struct dsa_completion_record);
172 wq->compls = dma_alloc_coherent(dev, wq->compls_size,
173 &wq->compls_addr, GFP_KERNEL);
174 if (!wq->compls) {
175 rc = -ENOMEM;
176 goto fail_alloc_compls;
177 }
178
179 rc = alloc_descs(wq, num_descs);
180 if (rc < 0)
181 goto fail_alloc_descs;
182
183 rc = sbitmap_init_node(&wq->sbmap, num_descs, -1, GFP_KERNEL,
184 dev_to_node(dev));
185 if (rc < 0)
186 goto fail_sbitmap_init;
187
188 for (i = 0; i < num_descs; i++) {
189 struct idxd_desc *desc = wq->descs[i];
190
191 desc->hw = wq->hw_descs[i];
192 desc->completion = &wq->compls[i];
193 desc->compl_dma = wq->compls_addr +
194 sizeof(struct dsa_completion_record) * i;
195 desc->id = i;
196 desc->wq = wq;
197
198 dma_async_tx_descriptor_init(&desc->txd, &wq->dma_chan);
199 desc->txd.tx_submit = idxd_dma_tx_submit;
200 }
201
202 return 0;
203
204 fail_sbitmap_init:
205 free_descs(wq);
206 fail_alloc_descs:
207 dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
208 fail_alloc_compls:
209 free_hw_descs(wq);
210 return rc;
211 }
212
213 void idxd_wq_free_resources(struct idxd_wq *wq)
214 {
215 struct device *dev = &wq->idxd->pdev->dev;
216
217 if (wq->type != IDXD_WQT_KERNEL)
218 return;
219
220 free_hw_descs(wq);
221 free_descs(wq);
222 dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
223 sbitmap_free(&wq->sbmap);
224 }
225
226 int idxd_wq_enable(struct idxd_wq *wq)
227 {
228 struct idxd_device *idxd = wq->idxd;
229 struct device *dev = &idxd->pdev->dev;
230 u32 status;
231 int rc;
232
233 lockdep_assert_held(&idxd->dev_lock);
234
235 if (wq->state == IDXD_WQ_ENABLED) {
236 dev_dbg(dev, "WQ %d already enabled\n", wq->id);
237 return -ENXIO;
238 }
239
240 rc = idxd_cmd_send(idxd, IDXD_CMD_ENABLE_WQ, wq->id);
241 if (rc < 0)
242 return rc;
243 rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
244 if (rc < 0)
245 return rc;
246
247 if (status != IDXD_CMDSTS_SUCCESS &&
248 status != IDXD_CMDSTS_ERR_WQ_ENABLED) {
249 dev_dbg(dev, "WQ enable failed: %#x\n", status);
250 return -ENXIO;
251 }
252
253 wq->state = IDXD_WQ_ENABLED;
254 dev_dbg(dev, "WQ %d enabled\n", wq->id);
255 return 0;
256 }
257
258 int idxd_wq_disable(struct idxd_wq *wq)
259 {
260 struct idxd_device *idxd = wq->idxd;
261 struct device *dev = &idxd->pdev->dev;
262 u32 status, operand;
263 int rc;
264
265 lockdep_assert_held(&idxd->dev_lock);
266 dev_dbg(dev, "Disabling WQ %d\n", wq->id);
267
268 if (wq->state != IDXD_WQ_ENABLED) {
269 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
270 return 0;
271 }
272
273 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
274 rc = idxd_cmd_send(idxd, IDXD_CMD_DISABLE_WQ, operand);
275 if (rc < 0)
276 return rc;
277 rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
278 if (rc < 0)
279 return rc;
280
281 if (status != IDXD_CMDSTS_SUCCESS) {
282 dev_dbg(dev, "WQ disable failed: %#x\n", status);
283 return -ENXIO;
284 }
285
286 wq->state = IDXD_WQ_DISABLED;
287 dev_dbg(dev, "WQ %d disabled\n", wq->id);
288 return 0;
289 }
290
291 int idxd_wq_map_portal(struct idxd_wq *wq)
292 {
293 struct idxd_device *idxd = wq->idxd;
294 struct pci_dev *pdev = idxd->pdev;
295 struct device *dev = &pdev->dev;
296 resource_size_t start;
297
298 start = pci_resource_start(pdev, IDXD_WQ_BAR);
299 start = start + wq->id * IDXD_PORTAL_SIZE;
300
301 wq->dportal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE);
302 if (!wq->dportal)
303 return -ENOMEM;
304 dev_dbg(dev, "wq %d portal mapped at %p\n", wq->id, wq->dportal);
305
306 return 0;
307 }
308
309 void idxd_wq_unmap_portal(struct idxd_wq *wq)
310 {
311 struct device *dev = &wq->idxd->pdev->dev;
312
313 devm_iounmap(dev, wq->dportal);
314 }
315
316 /* Device control bits */
317 static inline bool idxd_is_enabled(struct idxd_device *idxd)
318 {
319 union gensts_reg gensts;
320
321 gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
322
323 if (gensts.state == IDXD_DEVICE_STATE_ENABLED)
324 return true;
325 return false;
326 }
327
328 static int idxd_cmd_wait(struct idxd_device *idxd, u32 *status, int timeout)
329 {
330 u32 sts, to = timeout;
331
332 lockdep_assert_held(&idxd->dev_lock);
333 sts = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
334 while (sts & IDXD_CMDSTS_ACTIVE && --to) {
335 cpu_relax();
336 sts = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
337 }
338
339 if (to == 0 && sts & IDXD_CMDSTS_ACTIVE) {
340 dev_warn(&idxd->pdev->dev, "%s timed out!\n", __func__);
341 *status = 0;
342 return -EBUSY;
343 }
344
345 *status = sts;
346 return 0;
347 }
348
349 static int idxd_cmd_send(struct idxd_device *idxd, int cmd_code, u32 operand)
350 {
351 union idxd_command_reg cmd;
352 int rc;
353 u32 status;
354
355 lockdep_assert_held(&idxd->dev_lock);
356 rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
357 if (rc < 0)
358 return rc;
359
360 memset(&cmd, 0, sizeof(cmd));
361 cmd.cmd = cmd_code;
362 cmd.operand = operand;
363 dev_dbg(&idxd->pdev->dev, "%s: sending cmd: %#x op: %#x\n",
364 __func__, cmd_code, operand);
365 iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
366
367 return 0;
368 }
369
370 int idxd_device_enable(struct idxd_device *idxd)
371 {
372 struct device *dev = &idxd->pdev->dev;
373 int rc;
374 u32 status;
375
376 lockdep_assert_held(&idxd->dev_lock);
377 if (idxd_is_enabled(idxd)) {
378 dev_dbg(dev, "Device already enabled\n");
379 return -ENXIO;
380 }
381
382 rc = idxd_cmd_send(idxd, IDXD_CMD_ENABLE_DEVICE, 0);
383 if (rc < 0)
384 return rc;
385 rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
386 if (rc < 0)
387 return rc;
388
389 /* If the command is successful or if the device was enabled */
390 if (status != IDXD_CMDSTS_SUCCESS &&
391 status != IDXD_CMDSTS_ERR_DEV_ENABLED) {
392 dev_dbg(dev, "%s: err_code: %#x\n", __func__, status);
393 return -ENXIO;
394 }
395
396 idxd->state = IDXD_DEV_ENABLED;
397 return 0;
398 }
399
400 int idxd_device_disable(struct idxd_device *idxd)
401 {
402 struct device *dev = &idxd->pdev->dev;
403 int rc;
404 u32 status;
405
406 lockdep_assert_held(&idxd->dev_lock);
407 if (!idxd_is_enabled(idxd)) {
408 dev_dbg(dev, "Device is not enabled\n");
409 return 0;
410 }
411
412 rc = idxd_cmd_send(idxd, IDXD_CMD_DISABLE_DEVICE, 0);
413 if (rc < 0)
414 return rc;
415 rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
416 if (rc < 0)
417 return rc;
418
419 /* If the command is successful or if the device was disabled */
420 if (status != IDXD_CMDSTS_SUCCESS &&
421 !(status & IDXD_CMDSTS_ERR_DIS_DEV_EN)) {
422 dev_dbg(dev, "%s: err_code: %#x\n", __func__, status);
423 rc = -ENXIO;
424 return rc;
425 }
426
427 idxd->state = IDXD_DEV_CONF_READY;
428 return 0;
429 }
430
431 int __idxd_device_reset(struct idxd_device *idxd)
432 {
433 u32 status;
434 int rc;
435
436 rc = idxd_cmd_send(idxd, IDXD_CMD_RESET_DEVICE, 0);
437 if (rc < 0)
438 return rc;
439 rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
440 if (rc < 0)
441 return rc;
442
443 return 0;
444 }
445
446 int idxd_device_reset(struct idxd_device *idxd)
447 {
448 unsigned long flags;
449 int rc;
450
451 spin_lock_irqsave(&idxd->dev_lock, flags);
452 rc = __idxd_device_reset(idxd);
453 spin_unlock_irqrestore(&idxd->dev_lock, flags);
454 return rc;
455 }
456
457 /* Device configuration bits */
458 static void idxd_group_config_write(struct idxd_group *group)
459 {
460 struct idxd_device *idxd = group->idxd;
461 struct device *dev = &idxd->pdev->dev;
462 int i;
463 u32 grpcfg_offset;
464
465 dev_dbg(dev, "Writing group %d cfg registers\n", group->id);
466
467 /* setup GRPWQCFG */
468 for (i = 0; i < 4; i++) {
469 grpcfg_offset = idxd->grpcfg_offset +
470 group->id * 64 + i * sizeof(u64);
471 iowrite64(group->grpcfg.wqs[i],
472 idxd->reg_base + grpcfg_offset);
473 dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
474 group->id, i, grpcfg_offset,
475 ioread64(idxd->reg_base + grpcfg_offset));
476 }
477
478 /* setup GRPENGCFG */
479 grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 32;
480 iowrite64(group->grpcfg.engines, idxd->reg_base + grpcfg_offset);
481 dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id,
482 grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset));
483
484 /* setup GRPFLAGS */
485 grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 40;
486 iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset);
487 dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n",
488 group->id, grpcfg_offset,
489 ioread32(idxd->reg_base + grpcfg_offset));
490 }
491
492 static int idxd_groups_config_write(struct idxd_device *idxd)
493
494 {
495 union gencfg_reg reg;
496 int i;
497 struct device *dev = &idxd->pdev->dev;
498
499 /* Setup bandwidth token limit */
500 if (idxd->token_limit) {
501 reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
502 reg.token_limit = idxd->token_limit;
503 iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET);
504 }
505
506 dev_dbg(dev, "GENCFG(%#x): %#x\n", IDXD_GENCFG_OFFSET,
507 ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET));
508
509 for (i = 0; i < idxd->max_groups; i++) {
510 struct idxd_group *group = &idxd->groups[i];
511
512 idxd_group_config_write(group);
513 }
514
515 return 0;
516 }
517
518 static int idxd_wq_config_write(struct idxd_wq *wq)
519 {
520 struct idxd_device *idxd = wq->idxd;
521 struct device *dev = &idxd->pdev->dev;
522 u32 wq_offset;
523 int i;
524
525 if (!wq->group)
526 return 0;
527
528 memset(&wq->wqcfg, 0, sizeof(union wqcfg));
529
530 /* byte 0-3 */
531 wq->wqcfg.wq_size = wq->size;
532
533 if (wq->size == 0) {
534 dev_warn(dev, "Incorrect work queue size: 0\n");
535 return -EINVAL;
536 }
537
538 /* bytes 4-7 */
539 wq->wqcfg.wq_thresh = wq->threshold;
540
541 /* byte 8-11 */
542 wq->wqcfg.priv = !!(wq->type == IDXD_WQT_KERNEL);
543 wq->wqcfg.mode = 1;
544
545 wq->wqcfg.priority = wq->priority;
546
547 /* bytes 12-15 */
548 wq->wqcfg.max_xfer_shift = idxd->hw.gen_cap.max_xfer_shift;
549 wq->wqcfg.max_batch_shift = idxd->hw.gen_cap.max_batch_shift;
550
551 dev_dbg(dev, "WQ %d CFGs\n", wq->id);
552 for (i = 0; i < 8; i++) {
553 wq_offset = idxd->wqcfg_offset + wq->id * 32 + i * sizeof(u32);
554 iowrite32(wq->wqcfg.bits[i], idxd->reg_base + wq_offset);
555 dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n",
556 wq->id, i, wq_offset,
557 ioread32(idxd->reg_base + wq_offset));
558 }
559
560 return 0;
561 }
562
563 static int idxd_wqs_config_write(struct idxd_device *idxd)
564 {
565 int i, rc;
566
567 for (i = 0; i < idxd->max_wqs; i++) {
568 struct idxd_wq *wq = &idxd->wqs[i];
569
570 rc = idxd_wq_config_write(wq);
571 if (rc < 0)
572 return rc;
573 }
574
575 return 0;
576 }
577
578 static void idxd_group_flags_setup(struct idxd_device *idxd)
579 {
580 int i;
581
582 /* TC-A 0 and TC-B 1 should be defaults */
583 for (i = 0; i < idxd->max_groups; i++) {
584 struct idxd_group *group = &idxd->groups[i];
585
586 if (group->tc_a == -1)
587 group->tc_a = group->grpcfg.flags.tc_a = 0;
588 else
589 group->grpcfg.flags.tc_a = group->tc_a;
590 if (group->tc_b == -1)
591 group->tc_b = group->grpcfg.flags.tc_b = 1;
592 else
593 group->grpcfg.flags.tc_b = group->tc_b;
594 group->grpcfg.flags.use_token_limit = group->use_token_limit;
595 group->grpcfg.flags.tokens_reserved = group->tokens_reserved;
596 if (group->tokens_allowed)
597 group->grpcfg.flags.tokens_allowed =
598 group->tokens_allowed;
599 else
600 group->grpcfg.flags.tokens_allowed = idxd->max_tokens;
601 }
602 }
603
604 static int idxd_engines_setup(struct idxd_device *idxd)
605 {
606 int i, engines = 0;
607 struct idxd_engine *eng;
608 struct idxd_group *group;
609
610 for (i = 0; i < idxd->max_groups; i++) {
611 group = &idxd->groups[i];
612 group->grpcfg.engines = 0;
613 }
614
615 for (i = 0; i < idxd->max_engines; i++) {
616 eng = &idxd->engines[i];
617 group = eng->group;
618
619 if (!group)
620 continue;
621
622 group->grpcfg.engines |= BIT(eng->id);
623 engines++;
624 }
625
626 if (!engines)
627 return -EINVAL;
628
629 return 0;
630 }
631
632 static int idxd_wqs_setup(struct idxd_device *idxd)
633 {
634 struct idxd_wq *wq;
635 struct idxd_group *group;
636 int i, j, configured = 0;
637 struct device *dev = &idxd->pdev->dev;
638
639 for (i = 0; i < idxd->max_groups; i++) {
640 group = &idxd->groups[i];
641 for (j = 0; j < 4; j++)
642 group->grpcfg.wqs[j] = 0;
643 }
644
645 for (i = 0; i < idxd->max_wqs; i++) {
646 wq = &idxd->wqs[i];
647 group = wq->group;
648
649 if (!wq->group)
650 continue;
651 if (!wq->size)
652 continue;
653
654 if (!wq_dedicated(wq)) {
655 dev_warn(dev, "No shared workqueue support.\n");
656 return -EINVAL;
657 }
658
659 group->grpcfg.wqs[wq->id / 64] |= BIT(wq->id % 64);
660 configured++;
661 }
662
663 if (configured == 0)
664 return -EINVAL;
665
666 return 0;
667 }
668
669 int idxd_device_config(struct idxd_device *idxd)
670 {
671 int rc;
672
673 lockdep_assert_held(&idxd->dev_lock);
674 rc = idxd_wqs_setup(idxd);
675 if (rc < 0)
676 return rc;
677
678 rc = idxd_engines_setup(idxd);
679 if (rc < 0)
680 return rc;
681
682 idxd_group_flags_setup(idxd);
683
684 rc = idxd_wqs_config_write(idxd);
685 if (rc < 0)
686 return rc;
687
688 rc = idxd_groups_config_write(idxd);
689 if (rc < 0)
690 return rc;
691
692 return 0;
693 }