drivers/dma/idxd/device.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
   3 #include <linux/init.h>
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/pci.h>
   7 #include <linux/io-64-nonatomic-lo-hi.h>
   8 #include <linux/dmaengine.h>
   9 #include <uapi/linux/idxd.h>
  10 #include "../dmaengine.h"
  11 #include "idxd.h"
  12 #include "registers.h"
  13
  14 static int idxd_cmd_wait(struct idxd_device *idxd, u32 *status, int timeout);
  15 static int idxd_cmd_send(struct idxd_device *idxd, int cmd_code, u32 operand);
  16
  17 /* Interrupt control bits */
  18 int idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id)
  19 {
  20         struct pci_dev *pdev = idxd->pdev;
  21         int msixcnt = pci_msix_vec_count(pdev);
  22         union msix_perm perm;
  23         u32 offset;
  24
  25         if (vec_id < 0 || vec_id >= msixcnt)
  26                 return -EINVAL;
  27
  28         offset = idxd->msix_perm_offset + vec_id * 8;
  29         perm.bits = ioread32(idxd->reg_base + offset);
  30         perm.ignore = 1;
  31         iowrite32(perm.bits, idxd->reg_base + offset);
  32
  33         return 0;
  34 }
  35
  36 void idxd_mask_msix_vectors(struct idxd_device *idxd)
  37 {
  38         struct pci_dev *pdev = idxd->pdev;
  39         int msixcnt = pci_msix_vec_count(pdev);
  40         int i, rc;
  41
  42         for (i = 0; i < msixcnt; i++) {
  43                 rc = idxd_mask_msix_vector(idxd, i);
  44                 if (rc < 0)
  45                         dev_warn(&pdev->dev,
  46                                  "Failed disabling msix vec %d\n", i);
  47         }
  48 }
  49
  50 int idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id)
  51 {
  52         struct pci_dev *pdev = idxd->pdev;
  53         int msixcnt = pci_msix_vec_count(pdev);
  54         union msix_perm perm;
  55         u32 offset;
  56
  57         if (vec_id < 0 || vec_id >= msixcnt)
  58                 return -EINVAL;
  59
  60         offset = idxd->msix_perm_offset + vec_id * 8;
  61         perm.bits = ioread32(idxd->reg_base + offset);
  62         perm.ignore = 0;
  63         iowrite32(perm.bits, idxd->reg_base + offset);
  64
  65         return 0;
  66 }
  67
  68 void idxd_unmask_error_interrupts(struct idxd_device *idxd)
  69 {
  70         union genctrl_reg genctrl;
  71
  72         genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
  73         genctrl.softerr_int_en = 1;
  74         iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
  75 }
  76
  77 void idxd_mask_error_interrupts(struct idxd_device *idxd)
  78 {
  79         union genctrl_reg genctrl;
  80
  81         genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
  82         genctrl.softerr_int_en = 0;
  83         iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
  84 }
  85
  86 static void free_hw_descs(struct idxd_wq *wq)
  87 {
  88         int i;
  89
  90         for (i = 0; i < wq->num_descs; i++)
  91                 kfree(wq->hw_descs[i]);
  92
  93         kfree(wq->hw_descs);
  94 }
  95
  96 static int alloc_hw_descs(struct idxd_wq *wq, int num)
  97 {
  98         struct device *dev = &wq->idxd->pdev->dev;
  99         int i;
 100         int node = dev_to_node(dev);
 101
 102         wq->hw_descs = kcalloc_node(num, sizeof(struct dsa_hw_desc *),
 103                                     GFP_KERNEL, node);
 104         if (!wq->hw_descs)
 105                 return -ENOMEM;
 106
 107         for (i = 0; i < num; i++) {
 108                 wq->hw_descs[i] = kzalloc_node(sizeof(*wq->hw_descs[i]),
 109                                                GFP_KERNEL, node);
 110                 if (!wq->hw_descs[i]) {
 111                         free_hw_descs(wq);
 112                         return -ENOMEM;
 113                 }
 114         }
 115
 116         return 0;
 117 }
 118
 119 static void free_descs(struct idxd_wq *wq)
 120 {
 121         int i;
 122
 123         for (i = 0; i < wq->num_descs; i++)
 124                 kfree(wq->descs[i]);
 125
 126         kfree(wq->descs);
 127 }
 128
 129 static int alloc_descs(struct idxd_wq *wq, int num)
 130 {
 131         struct device *dev = &wq->idxd->pdev->dev;
 132         int i;
 133         int node = dev_to_node(dev);
 134
 135         wq->descs = kcalloc_node(num, sizeof(struct idxd_desc *),
 136                                  GFP_KERNEL, node);
 137         if (!wq->descs)
 138                 return -ENOMEM;
 139
 140         for (i = 0; i < num; i++) {
 141                 wq->descs[i] = kzalloc_node(sizeof(*wq->descs[i]),
 142                                             GFP_KERNEL, node);
 143                 if (!wq->descs[i]) {
 144                         free_descs(wq);
 145                         return -ENOMEM;
 146                 }
 147         }
 148
 149         return 0;
 150 }
 151
 152 /* WQ control bits */
 153 int idxd_wq_alloc_resources(struct idxd_wq *wq)
 154 {
 155         struct idxd_device *idxd = wq->idxd;
 156         struct idxd_group *group = wq->group;
 157         struct device *dev = &idxd->pdev->dev;
 158         int rc, num_descs, i;
 159
 160         if (wq->type != IDXD_WQT_KERNEL)
 161                 return 0;
 162
 163         num_descs = wq->size +
 164                 idxd->hw.gen_cap.max_descs_per_engine * group->num_engines;
 165         wq->num_descs = num_descs;
 166
 167         rc = alloc_hw_descs(wq, num_descs);
 168         if (rc < 0)
 169                 return rc;
 170
 171         wq->compls_size = num_descs * sizeof(struct dsa_completion_record);
 172         wq->compls = dma_alloc_coherent(dev, wq->compls_size,
 173                                         &wq->compls_addr, GFP_KERNEL);
 174         if (!wq->compls) {
 175                 rc = -ENOMEM;
 176                 goto fail_alloc_compls;
 177         }
 178
 179         rc = alloc_descs(wq, num_descs);
 180         if (rc < 0)
 181                 goto fail_alloc_descs;
 182
 183         rc = sbitmap_init_node(&wq->sbmap, num_descs, -1, GFP_KERNEL,
 184                                dev_to_node(dev));
 185         if (rc < 0)
 186                 goto fail_sbitmap_init;
 187
 188         for (i = 0; i < num_descs; i++) {
 189                 struct idxd_desc *desc = wq->descs[i];
 190
 191                 desc->hw = wq->hw_descs[i];
 192                 desc->completion = &wq->compls[i];
 193                 desc->compl_dma  = wq->compls_addr +
 194                         sizeof(struct dsa_completion_record) * i;
 195                 desc->id = i;
 196                 desc->wq = wq;
 197
 198                 dma_async_tx_descriptor_init(&desc->txd, &wq->dma_chan);
 199                 desc->txd.tx_submit = idxd_dma_tx_submit;
 200         }
 201
 202         return 0;
 203
 204  fail_sbitmap_init:
 205         free_descs(wq);
 206  fail_alloc_descs:
 207         dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
 208  fail_alloc_compls:
 209         free_hw_descs(wq);
 210         return rc;
 211 }
 212
 213 void idxd_wq_free_resources(struct idxd_wq *wq)
 214 {
 215         struct device *dev = &wq->idxd->pdev->dev;
 216
 217         if (wq->type != IDXD_WQT_KERNEL)
 218                 return;
 219
 220         free_hw_descs(wq);
 221         free_descs(wq);
 222         dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
 223         sbitmap_free(&wq->sbmap);
 224 }
 225
 226 int idxd_wq_enable(struct idxd_wq *wq)
 227 {
 228         struct idxd_device *idxd = wq->idxd;
 229         struct device *dev = &idxd->pdev->dev;
 230         u32 status;
 231         int rc;
 232
 233         lockdep_assert_held(&idxd->dev_lock);
 234
 235         if (wq->state == IDXD_WQ_ENABLED) {
 236                 dev_dbg(dev, "WQ %d already enabled\n", wq->id);
 237                 return -ENXIO;
 238         }
 239
 240         rc = idxd_cmd_send(idxd, IDXD_CMD_ENABLE_WQ, wq->id);
 241         if (rc < 0)
 242                 return rc;
 243         rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
 244         if (rc < 0)
 245                 return rc;
 246
 247         if (status != IDXD_CMDSTS_SUCCESS &&
 248             status != IDXD_CMDSTS_ERR_WQ_ENABLED) {
 249                 dev_dbg(dev, "WQ enable failed: %#x\n", status);
 250                 return -ENXIO;
 251         }
 252
 253         wq->state = IDXD_WQ_ENABLED;
 254         dev_dbg(dev, "WQ %d enabled\n", wq->id);
 255         return 0;
 256 }
 257
 258 int idxd_wq_disable(struct idxd_wq *wq)
 259 {
 260         struct idxd_device *idxd = wq->idxd;
 261         struct device *dev = &idxd->pdev->dev;
 262         u32 status, operand;
 263         int rc;
 264
 265         lockdep_assert_held(&idxd->dev_lock);
 266         dev_dbg(dev, "Disabling WQ %d\n", wq->id);
 267
 268         if (wq->state != IDXD_WQ_ENABLED) {
 269                 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
 270                 return 0;
 271         }
 272
 273         operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
 274         rc = idxd_cmd_send(idxd, IDXD_CMD_DISABLE_WQ, operand);
 275         if (rc < 0)
 276                 return rc;
 277         rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
 278         if (rc < 0)
 279                 return rc;
 280
 281         if (status != IDXD_CMDSTS_SUCCESS) {
 282                 dev_dbg(dev, "WQ disable failed: %#x\n", status);
 283                 return -ENXIO;
 284         }
 285
 286         wq->state = IDXD_WQ_DISABLED;
 287         dev_dbg(dev, "WQ %d disabled\n", wq->id);
 288         return 0;
 289 }
 290
 291 int idxd_wq_map_portal(struct idxd_wq *wq)
 292 {
 293         struct idxd_device *idxd = wq->idxd;
 294         struct pci_dev *pdev = idxd->pdev;
 295         struct device *dev = &pdev->dev;
 296         resource_size_t start;
 297
 298         start = pci_resource_start(pdev, IDXD_WQ_BAR);
 299         start = start + wq->id * IDXD_PORTAL_SIZE;
 300
 301         wq->dportal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE);
 302         if (!wq->dportal)
 303                 return -ENOMEM;
 304         dev_dbg(dev, "wq %d portal mapped at %p\n", wq->id, wq->dportal);
 305
 306         return 0;
 307 }
 308
 309 void idxd_wq_unmap_portal(struct idxd_wq *wq)
 310 {
 311         struct device *dev = &wq->idxd->pdev->dev;
 312
 313         devm_iounmap(dev, wq->dportal);
 314 }
 315
 316 /* Device control bits */
 317 static inline bool idxd_is_enabled(struct idxd_device *idxd)
 318 {
 319         union gensts_reg gensts;
 320
 321         gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
 322
 323         if (gensts.state == IDXD_DEVICE_STATE_ENABLED)
 324                 return true;
 325         return false;
 326 }
 327
 328 static int idxd_cmd_wait(struct idxd_device *idxd, u32 *status, int timeout)
 329 {
 330         u32 sts, to = timeout;
 331
 332         lockdep_assert_held(&idxd->dev_lock);
 333         sts = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
 334         while (sts & IDXD_CMDSTS_ACTIVE && --to) {
 335                 cpu_relax();
 336                 sts = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
 337         }
 338
 339         if (to == 0 && sts & IDXD_CMDSTS_ACTIVE) {
 340                 dev_warn(&idxd->pdev->dev, "%s timed out!\n", __func__);
 341                 *status = 0;
 342                 return -EBUSY;
 343         }
 344
 345         *status = sts;
 346         return 0;
 347 }
 348
 349 static int idxd_cmd_send(struct idxd_device *idxd, int cmd_code, u32 operand)
 350 {
 351         union idxd_command_reg cmd;
 352         int rc;
 353         u32 status;
 354
 355         lockdep_assert_held(&idxd->dev_lock);
 356         rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
 357         if (rc < 0)
 358                 return rc;
 359
 360         memset(&cmd, 0, sizeof(cmd));
 361         cmd.cmd = cmd_code;
 362         cmd.operand = operand;
 363         dev_dbg(&idxd->pdev->dev, "%s: sending cmd: %#x op: %#x\n",
 364                 __func__, cmd_code, operand);
 365         iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
 366
 367         return 0;
 368 }
 369
 370 int idxd_device_enable(struct idxd_device *idxd)
 371 {
 372         struct device *dev = &idxd->pdev->dev;
 373         int rc;
 374         u32 status;
 375
 376         lockdep_assert_held(&idxd->dev_lock);
 377         if (idxd_is_enabled(idxd)) {
 378                 dev_dbg(dev, "Device already enabled\n");
 379                 return -ENXIO;
 380         }
 381
 382         rc = idxd_cmd_send(idxd, IDXD_CMD_ENABLE_DEVICE, 0);
 383         if (rc < 0)
 384                 return rc;
 385         rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
 386         if (rc < 0)
 387                 return rc;
 388
 389         /* If the command is successful or if the device was enabled */
 390         if (status != IDXD_CMDSTS_SUCCESS &&
 391             status != IDXD_CMDSTS_ERR_DEV_ENABLED) {
 392                 dev_dbg(dev, "%s: err_code: %#x\n", __func__, status);
 393                 return -ENXIO;
 394         }
 395
 396         idxd->state = IDXD_DEV_ENABLED;
 397         return 0;
 398 }
 399
 400 int idxd_device_disable(struct idxd_device *idxd)
 401 {
 402         struct device *dev = &idxd->pdev->dev;
 403         int rc;
 404         u32 status;
 405
 406         lockdep_assert_held(&idxd->dev_lock);
 407         if (!idxd_is_enabled(idxd)) {
 408                 dev_dbg(dev, "Device is not enabled\n");
 409                 return 0;
 410         }
 411
 412         rc = idxd_cmd_send(idxd, IDXD_CMD_DISABLE_DEVICE, 0);
 413         if (rc < 0)
 414                 return rc;
 415         rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
 416         if (rc < 0)
 417                 return rc;
 418
 419         /* If the command is successful or if the device was disabled */
 420         if (status != IDXD_CMDSTS_SUCCESS &&
 421             !(status & IDXD_CMDSTS_ERR_DIS_DEV_EN)) {
 422                 dev_dbg(dev, "%s: err_code: %#x\n", __func__, status);
 423                 rc = -ENXIO;
 424                 return rc;
 425         }
 426
 427         idxd->state = IDXD_DEV_CONF_READY;
 428         return 0;
 429 }
 430
 431 int __idxd_device_reset(struct idxd_device *idxd)
 432 {
 433         u32 status;
 434         int rc;
 435
 436         rc = idxd_cmd_send(idxd, IDXD_CMD_RESET_DEVICE, 0);
 437         if (rc < 0)
 438                 return rc;
 439         rc = idxd_cmd_wait(idxd, &status, IDXD_REG_TIMEOUT);
 440         if (rc < 0)
 441                 return rc;
 442
 443         return 0;
 444 }
 445
 446 int idxd_device_reset(struct idxd_device *idxd)
 447 {
 448         unsigned long flags;
 449         int rc;
 450
 451         spin_lock_irqsave(&idxd->dev_lock, flags);
 452         rc = __idxd_device_reset(idxd);
 453         spin_unlock_irqrestore(&idxd->dev_lock, flags);
 454         return rc;
 455 }
 456
 457 /* Device configuration bits */
 458 static void idxd_group_config_write(struct idxd_group *group)
 459 {
 460         struct idxd_device *idxd = group->idxd;
 461         struct device *dev = &idxd->pdev->dev;
 462         int i;
 463         u32 grpcfg_offset;
 464
 465         dev_dbg(dev, "Writing group %d cfg registers\n", group->id);
 466
 467         /* setup GRPWQCFG */
 468         for (i = 0; i < 4; i++) {
 469                 grpcfg_offset = idxd->grpcfg_offset +
 470                         group->id * 64 + i * sizeof(u64);
 471                 iowrite64(group->grpcfg.wqs[i],
 472                           idxd->reg_base + grpcfg_offset);
 473                 dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
 474                         group->id, i, grpcfg_offset,
 475                         ioread64(idxd->reg_base + grpcfg_offset));
 476         }
 477
 478         /* setup GRPENGCFG */
 479         grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 32;
 480         iowrite64(group->grpcfg.engines, idxd->reg_base + grpcfg_offset);
 481         dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id,
 482                 grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset));
 483
 484         /* setup GRPFLAGS */
 485         grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 40;
 486         iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset);
 487         dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n",
 488                 group->id, grpcfg_offset,
 489                 ioread32(idxd->reg_base + grpcfg_offset));
 490 }
 491
 492 static int idxd_groups_config_write(struct idxd_device *idxd)
 493
 494 {
 495         union gencfg_reg reg;
 496         int i;
 497         struct device *dev = &idxd->pdev->dev;
 498
 499         /* Setup bandwidth token limit */
 500         if (idxd->token_limit) {
 501                 reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
 502                 reg.token_limit = idxd->token_limit;
 503                 iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET);
 504         }
 505
 506         dev_dbg(dev, "GENCFG(%#x): %#x\n", IDXD_GENCFG_OFFSET,
 507                 ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET));
 508
 509         for (i = 0; i < idxd->max_groups; i++) {
 510                 struct idxd_group *group = &idxd->groups[i];
 511
 512                 idxd_group_config_write(group);
 513         }
 514
 515         return 0;
 516 }
 517
 518 static int idxd_wq_config_write(struct idxd_wq *wq)
 519 {
 520         struct idxd_device *idxd = wq->idxd;
 521         struct device *dev = &idxd->pdev->dev;
 522         u32 wq_offset;
 523         int i;
 524
 525         if (!wq->group)
 526                 return 0;
 527
 528         memset(&wq->wqcfg, 0, sizeof(union wqcfg));
 529
 530         /* byte 0-3 */
 531         wq->wqcfg.wq_size = wq->size;
 532
 533         if (wq->size == 0) {
 534                 dev_warn(dev, "Incorrect work queue size: 0\n");
 535                 return -EINVAL;
 536         }
 537
 538         /* bytes 4-7 */
 539         wq->wqcfg.wq_thresh = wq->threshold;
 540
 541         /* byte 8-11 */
 542         wq->wqcfg.priv = !!(wq->type == IDXD_WQT_KERNEL);
 543         wq->wqcfg.mode = 1;
 544
 545         wq->wqcfg.priority = wq->priority;
 546
 547         /* bytes 12-15 */
 548         wq->wqcfg.max_xfer_shift = idxd->hw.gen_cap.max_xfer_shift;
 549         wq->wqcfg.max_batch_shift = idxd->hw.gen_cap.max_batch_shift;
 550
 551         dev_dbg(dev, "WQ %d CFGs\n", wq->id);
 552         for (i = 0; i < 8; i++) {
 553                 wq_offset = idxd->wqcfg_offset + wq->id * 32 + i * sizeof(u32);
 554                 iowrite32(wq->wqcfg.bits[i], idxd->reg_base + wq_offset);
 555                 dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n",
 556                         wq->id, i, wq_offset,
 557                         ioread32(idxd->reg_base + wq_offset));
 558         }
 559
 560         return 0;
 561 }
 562
 563 static int idxd_wqs_config_write(struct idxd_device *idxd)
 564 {
 565         int i, rc;
 566
 567         for (i = 0; i < idxd->max_wqs; i++) {
 568                 struct idxd_wq *wq = &idxd->wqs[i];
 569
 570                 rc = idxd_wq_config_write(wq);
 571                 if (rc < 0)
 572                         return rc;
 573         }
 574
 575         return 0;
 576 }
 577
 578 static void idxd_group_flags_setup(struct idxd_device *idxd)
 579 {
 580         int i;
 581
 582         /* TC-A 0 and TC-B 1 should be defaults */
 583         for (i = 0; i < idxd->max_groups; i++) {
 584                 struct idxd_group *group = &idxd->groups[i];
 585
 586                 if (group->tc_a == -1)
 587                         group->tc_a = group->grpcfg.flags.tc_a = 0;
 588                 else
 589                         group->grpcfg.flags.tc_a = group->tc_a;
 590                 if (group->tc_b == -1)
 591                         group->tc_b = group->grpcfg.flags.tc_b = 1;
 592                 else
 593                         group->grpcfg.flags.tc_b = group->tc_b;
 594                 group->grpcfg.flags.use_token_limit = group->use_token_limit;
 595                 group->grpcfg.flags.tokens_reserved = group->tokens_reserved;
 596                 if (group->tokens_allowed)
 597                         group->grpcfg.flags.tokens_allowed =
 598                                 group->tokens_allowed;
 599                 else
 600                         group->grpcfg.flags.tokens_allowed = idxd->max_tokens;
 601         }
 602 }
 603
 604 static int idxd_engines_setup(struct idxd_device *idxd)
 605 {
 606         int i, engines = 0;
 607         struct idxd_engine *eng;
 608         struct idxd_group *group;
 609
 610         for (i = 0; i < idxd->max_groups; i++) {
 611                 group = &idxd->groups[i];
 612                 group->grpcfg.engines = 0;
 613         }
 614
 615         for (i = 0; i < idxd->max_engines; i++) {
 616                 eng = &idxd->engines[i];
 617                 group = eng->group;
 618
 619                 if (!group)
 620                         continue;
 621
 622                 group->grpcfg.engines |= BIT(eng->id);
 623                 engines++;
 624         }
 625
 626         if (!engines)
 627                 return -EINVAL;
 628
 629         return 0;
 630 }
 631
 632 static int idxd_wqs_setup(struct idxd_device *idxd)
 633 {
 634         struct idxd_wq *wq;
 635         struct idxd_group *group;
 636         int i, j, configured = 0;
 637         struct device *dev = &idxd->pdev->dev;
 638
 639         for (i = 0; i < idxd->max_groups; i++) {
 640                 group = &idxd->groups[i];
 641                 for (j = 0; j < 4; j++)
 642                         group->grpcfg.wqs[j] = 0;
 643         }
 644
 645         for (i = 0; i < idxd->max_wqs; i++) {
 646                 wq = &idxd->wqs[i];
 647                 group = wq->group;
 648
 649                 if (!wq->group)
 650                         continue;
 651                 if (!wq->size)
 652                         continue;
 653
 654                 if (!wq_dedicated(wq)) {
 655                         dev_warn(dev, "No shared workqueue support.\n");
 656                         return -EINVAL;
 657                 }
 658
 659                 group->grpcfg.wqs[wq->id / 64] |= BIT(wq->id % 64);
 660                 configured++;
 661         }
 662
 663         if (configured == 0)
 664                 return -EINVAL;
 665
 666         return 0;
 667 }
 668
 669 int idxd_device_config(struct idxd_device *idxd)
 670 {
 671         int rc;
 672
 673         lockdep_assert_held(&idxd->dev_lock);
 674         rc = idxd_wqs_setup(idxd);
 675         if (rc < 0)
 676                 return rc;
 677
 678         rc = idxd_engines_setup(idxd);
 679         if (rc < 0)
 680                 return rc;
 681
 682         idxd_group_flags_setup(idxd);
 683
 684         rc = idxd_wqs_config_write(idxd);
 685         if (rc < 0)
 686                 return rc;
 687
 688         rc = idxd_groups_config_write(idxd);
 689         if (rc < 0)
 690                 return rc;
 691
 692         return 0;
 693 }