bhyve: implement NVMe device support

NVMe devices in bhyve are modeled this way:

 -s $pciaddr,nvme,devpath[,opts]

devpath can be a path to the image or the block device. It also can be
"ram=size_in_MiB", but this is not covered by this series.

There could be only a single device per PCI address.

Optional configuration options (such as max number of queues, concurrent
I/O requests, etc) are also not covered by this series.

Signed-off-by: Roman Bogorodskiy <bogorodskiy@gmail.com>
Reviewed-by: Michal Privoznik <mprivozn@redhat.com>

diff --git a/src/bhyve/bhyve_capabilities.c b/src/bhyve/bhyve_capabilities.c
index 04a5a4cf29036a157fc72cc8533e412243f9b871..aad757e80164b9e48bee5c0c3ea6cc248025bcd4 100644 (file)
--- a/src/bhyve/bhyve_capabilities.c
+++ b/src/bhyve/bhyve_capabilities.c
@@ -355,6 +355,17 @@ bhyveProbeCapsVirtioRnd(unsigned int *caps, char *binary)
 }
 
 
+static int
+bhyveProbeCapsNvme(unsigned int *caps, char *binary)
+{
+    return bhyveProbeCapsDeviceHelper(caps, binary,
+                                      "-s",
+                                      "0,nvme",
+                                      "pci slot 0:0: unknown device \"nvme\"",
+                                      BHYVE_CAP_NVME);
+}
+
+
 int
 virBhyveProbeCaps(unsigned int *caps)
 {
@@ -395,6 +406,9 @@ virBhyveProbeCaps(unsigned int *caps)
     if ((ret = bhyveProbeCapsVirtioRnd(caps, binary)))
         goto out;
 
+    if ((ret = bhyveProbeCapsNvme(caps, binary)))
+        goto out;
+
  out:
     VIR_FREE(binary);
     return ret;

diff --git a/src/bhyve/bhyve_capabilities.h b/src/bhyve/bhyve_capabilities.h
index 9b24241dc1009d4d682ede99efbc92601eeae34b..500b2353979a906498d9d6e5f128be79fd37aadb 100644 (file)
--- a/src/bhyve/bhyve_capabilities.h
+++ b/src/bhyve/bhyve_capabilities.h
@@ -54,6 +54,7 @@ typedef enum {
     BHYVE_CAP_VNC_PASSWORD = 1 << 8,
     BHYVE_CAP_VIRTIO_9P = 1 << 9,
     BHYVE_CAP_VIRTIO_RND = 1 << 10,
+    BHYVE_CAP_NVME = 1 << 11,
 } virBhyveCapsFlags;
 
 int virBhyveProbeGrubCaps(virBhyveGrubCapsFlags *caps);

diff --git a/src/bhyve/bhyve_command.c b/src/bhyve/bhyve_command.c
index ab6d6e92e4b288df35647d0283721f0b203dd14b..9f2d02b48476161b72cfa3336e078fb43bcb5354 100644 (file)
--- a/src/bhyve/bhyve_command.c
+++ b/src/bhyve/bhyve_command.c
@@ -328,6 +328,38 @@ bhyveBuildUSBControllerArgStr(const virDomainDef *def,
     return 0;
 }
 
+static int
+bhyveBuildNVMeControllerArgStr(const virDomainDef *def,
+                               virDomainControllerDef *controller,
+                               struct _bhyveConn *driver G_GNUC_UNUSED,
+                               virCommand *cmd)
+{
+    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
+    const char *disk_source;
+    size_t i;
+
+    for (i = 0; i < def->ndisks; i++) {
+        virDomainDiskDef *disk = def->disks[i];
+
+        if (disk->bus != VIR_DOMAIN_DISK_BUS_NVME)
+            continue;
+
+        if (disk->info.addr.drive.controller != controller->idx)
+            continue;
+
+        VIR_DEBUG("disk %zu controller %d", i, controller->idx);
+
+        disk_source = virDomainDiskGetSource(disk);
+
+        virCommandAddArg(cmd, "-s");
+        virCommandAddArgFormat(cmd, "%d:0,nvme,%s",
+                               controller->info.addr.pci.slot,
+                               disk_source);
+    }
+
+    return 0;
+}
+
 static int
 bhyveBuildVirtIODiskArgStr(const virDomainDef *def G_GNUC_UNUSED,
                            virDomainDiskDef *disk,
@@ -370,6 +402,9 @@ bhyveBuildDiskArgStr(const virDomainDef *def,
     case VIR_DOMAIN_DISK_BUS_SATA:
         /* Handled by bhyveBuildAHCIControllerArgStr() */
         break;
+    case VIR_DOMAIN_DISK_BUS_NVME:
+        /* Handled by bhyveBuildNVMeControllerArgStr() */
+        break;
     case VIR_DOMAIN_DISK_BUS_VIRTIO:
         if (bhyveBuildVirtIODiskArgStr(def, disk, cmd) < 0)
             return -1;
@@ -382,7 +417,6 @@ bhyveBuildDiskArgStr(const virDomainDef *def,
     case VIR_DOMAIN_DISK_BUS_USB:
     case VIR_DOMAIN_DISK_BUS_UML:
     case VIR_DOMAIN_DISK_BUS_SD:
-    case VIR_DOMAIN_DISK_BUS_NVME:
     case VIR_DOMAIN_DISK_BUS_LAST:
     default:
         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
@@ -432,13 +466,16 @@ bhyveBuildControllerArgStr(const virDomainDef *def,
         virCommandAddArgFormat(cmd, "%d:0,lpc",
                                 controller->info.addr.pci.slot);
         break;
+    case VIR_DOMAIN_CONTROLLER_TYPE_NVME:
+        if (bhyveBuildNVMeControllerArgStr(def, controller, driver, cmd) < 0)
+            return -1;
+        break;
     case VIR_DOMAIN_CONTROLLER_TYPE_IDE:
     case VIR_DOMAIN_CONTROLLER_TYPE_FDC:
     case VIR_DOMAIN_CONTROLLER_TYPE_SCSI:
     case VIR_DOMAIN_CONTROLLER_TYPE_VIRTIO_SERIAL:
     case VIR_DOMAIN_CONTROLLER_TYPE_CCID:
     case VIR_DOMAIN_CONTROLLER_TYPE_XENBUS:
-    case VIR_DOMAIN_CONTROLLER_TYPE_NVME:
     case VIR_DOMAIN_CONTROLLER_TYPE_LAST:
     default:
         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",

diff --git a/src/bhyve/bhyve_device.c b/src/bhyve/bhyve_device.c
index 68983d5bc49fad27612177facb3fd3a0ff7d1e58..49cfccaeba21f6d4baaf65a64ab4bf7c0145d824 100644 (file)
--- a/src/bhyve/bhyve_device.c
+++ b/src/bhyve/bhyve_device.c
@@ -114,6 +114,7 @@ bhyveAssignDevicePCISlots(virDomainDef *def,
     for (i = 0; i < def->ncontrollers; i++) {
         if ((def->controllers[i]->type == VIR_DOMAIN_CONTROLLER_TYPE_PCI) ||
             (def->controllers[i]->type == VIR_DOMAIN_CONTROLLER_TYPE_SATA) ||
+            (def->controllers[i]->type == VIR_DOMAIN_CONTROLLER_TYPE_NVME) ||
             ((def->controllers[i]->type == VIR_DOMAIN_CONTROLLER_TYPE_USB) &&
              (def->controllers[i]->model == VIR_DOMAIN_CONTROLLER_MODEL_USB_NEC_XHCI)) ||
             def->controllers[i]->type == VIR_DOMAIN_CONTROLLER_TYPE_ISA) {

diff --git a/src/bhyve/bhyve_domain.c b/src/bhyve/bhyve_domain.c
index 9dec300a99d0aa460309084aaf1cdcdd677d1294..79ac336430bcad0fd3ec6141aae452fbd242fd88 100644 (file)
--- a/src/bhyve/bhyve_domain.c
+++ b/src/bhyve/bhyve_domain.c
@@ -111,9 +111,10 @@ bhyveDomainDiskDefAssignAddress(struct _bhyveConn *driver,
                                 virDomainDiskDef *def,
                                 const virDomainDef *vmdef G_GNUC_UNUSED)
 {
-    int idx = virDiskNameToIndex(def->dst);
+    int idx = -1;
+    int nvme_ctrl = 0;
 
-    if (idx < 0) {
+    if (virDiskNameParse(def->dst, &nvme_ctrl, &idx, NULL) < 0) {
         virReportError(VIR_ERR_XML_ERROR,
                        _("Unknown disk name '%1$s' and no address specified"),
                        def->dst);
@@ -134,6 +135,15 @@ bhyveDomainDiskDefAssignAddress(struct _bhyveConn *driver,
 
         def->info.addr.drive.bus = 0;
         break;
+
+    case VIR_DOMAIN_DISK_BUS_NVME:
+        def->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_DRIVE;
+
+        def->info.addr.drive.controller = nvme_ctrl;
+        def->info.addr.drive.unit = 0;
+        def->info.addr.drive.bus = idx;
+        break;
+
     case VIR_DOMAIN_DISK_BUS_SCSI:
     case VIR_DOMAIN_DISK_BUS_IDE:
     case VIR_DOMAIN_DISK_BUS_FDC:
@@ -143,7 +153,6 @@ bhyveDomainDiskDefAssignAddress(struct _bhyveConn *driver,
     case VIR_DOMAIN_DISK_BUS_USB:
     case VIR_DOMAIN_DISK_BUS_UML:
     case VIR_DOMAIN_DISK_BUS_SD:
-    case VIR_DOMAIN_DISK_BUS_NVME:
     case VIR_DOMAIN_DISK_BUS_LAST:
     default:
         break;

diff --git a/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.args b/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.args
new file mode 100644 (file)
index 0000000..bd39db1
--- /dev/null
+++ b/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.args
@@ -0,0 +1,9 @@
+bhyve \
+-c 1 \
+-m 214 \
+-u \
+-H \
+-P \
+-s 0:0,hostbridge \
+-s 2:0,nvme,/tmp/freebsd.img \
+bhyve

diff --git a/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.ldargs b/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.ldargs
new file mode 100644 (file)
index 0000000..5905f4b
--- /dev/null
+++ b/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.ldargs
@@ -0,0 +1,4 @@
+bhyveload \
+-m 214 \
+-d /tmp/freebsd.img \
+bhyve

diff --git a/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.xml b/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.xml
new file mode 100644 (file)
index 0000000..8daaa11
--- /dev/null
+++ b/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.xml
@@ -0,0 +1,16 @@
+<domain type='bhyve'>
+  <name>bhyve</name>
+  <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+  <memory>219136</memory>
+  <vcpu>1</vcpu>
+  <os>
+    <type>hvm</type>
+  </os>
+  <devices>
+    <disk type='file'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd.img'/>
+      <target dev='nvme0n1' bus='nvme'/>
+    </disk>
+  </devices>
+</domain>

diff --git a/tests/bhyvexml2argvtest.c b/tests/bhyvexml2argvtest.c
index cc6b17233dd073c6f2085189ce1f9880b8a7a1eb..0a5202e4252ad2dd4b304f4a36c809f414a75c12 100644 (file)
--- a/tests/bhyvexml2argvtest.c
+++ b/tests/bhyvexml2argvtest.c
@@ -195,7 +195,8 @@ mymain(void)
                        BHYVE_CAP_NET_E1000 | BHYVE_CAP_LPC_BOOTROM | \
                        BHYVE_CAP_FBUF | BHYVE_CAP_XHCI | \
                        BHYVE_CAP_CPUTOPOLOGY | BHYVE_CAP_SOUND_HDA | \
-                       BHYVE_CAP_VNC_PASSWORD | BHYVE_CAP_VIRTIO_9P;
+                       BHYVE_CAP_VNC_PASSWORD | BHYVE_CAP_VIRTIO_9P | \
+                       BHYVE_CAP_NVME;
 
     DO_TEST("base");
     DO_TEST("wired");
@@ -259,6 +260,7 @@ mymain(void)
     DO_TEST("serial-tcp");
     DO_TEST("4-consoles");
     DO_TEST_FAILURE("serial-invalid-port");
+    DO_TEST("nvme");
 
     /* Address allocation tests */
     DO_TEST("addr-single-sata-disk");

diff --git a/tests/bhyvexml2xmloutdata/bhyvexml2xmlout-nvme.xml b/tests/bhyvexml2xmloutdata/bhyvexml2xmlout-nvme.xml
new file mode 100644 (file)
index 0000000..d4ab69b
--- /dev/null
+++ b/tests/bhyvexml2xmloutdata/bhyvexml2xmlout-nvme.xml
@@ -0,0 +1,27 @@
+<domain type='bhyve'>
+  <name>bhyve</name>
+  <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+  <memory unit='KiB'>219136</memory>
+  <currentMemory unit='KiB'>219136</currentMemory>
+  <vcpu placement='static'>1</vcpu>
+  <os>
+    <type arch='x86_64'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <disk type='file' device='disk'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd.img'/>
+      <target dev='nvme0n1' bus='nvme'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <controller type='pci' index='0' model='pci-root'/>
+    <controller type='nvme' index='0'>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
+    </controller>
+  </devices>
+</domain>

diff --git a/tests/bhyvexml2xmltest.c b/tests/bhyvexml2xmltest.c
index df093a553982be299946db827e22f204b0925278..be69e2148493980ea22a094e73251e09d9cbba40 100644 (file)
--- a/tests/bhyvexml2xmltest.c
+++ b/tests/bhyvexml2xmltest.c
@@ -117,6 +117,7 @@ mymain(void)
     DO_TEST_DIFFERENT("virtio-rnd");
     DO_TEST_DIFFERENT("serial-tcp");
     DO_TEST_DIFFERENT("4-consoles");
+    DO_TEST_DIFFERENT("nvme");
 
     /* Address allocation tests */
     DO_TEST_DIFFERENT("addr-single-sata-disk");