bhyve: add blkiotune support

FreeBSD supports resource limiting with the rctl(4) framework.
It supports various resource types, including I/O resources.
It allows to limit resources for users, processes, login classes,
and jails.

To apply blkiotune limits set limits for the bhyve process.

I/O related resources supported by rctl(4) are:

  readbps            filesystem reads, in bytes per second
  writebps           filesystem writes, in bytes per second
  readiops           filesystem reads, in operations per second
  writeiops          filesystem writes, in operations per second

Thus, the actual commands look like:

rctl -a process:$bhyvepid:writebps:throttle=10000000
rctl -a process:$bhyvepid:readbps:throttle=10000000
rctl -a process:$bhyvepid:writeiops:throttle=20000
rctl -a process:$bhyvepid:readiops:throttle=20000

This is different from the current blkiotune modeling in libvirt as
it requires specific device to apply limits to. To adapt this model
to per-domain I/O limits, update domain schema to specify "*" as a
device name.

The rctl(8) may be not available or not enabled, so add a capability
check for that.

Per process rules get removed when the process disappears, so no special
clean up is necessary.

Signed-off-by: Roman Bogorodskiy <bogorodskiy@gmail.com>
Reviewed-by: Peter Krempa <pkrempa@redhat.com>

diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
index b589fc942953a1c1d5614c8414e90775878ee110..078cd7aa84afafd5a8b61596308e325e29756388 100644 (file)
--- a/docs/formatdomain.rst
+++ b/docs/formatdomain.rst
@@ -1367,10 +1367,15 @@ Block I/O Tuning
    associated with each guest disk device (contrast this to the <iotune>
    element of a disk definition (See `Hard drives, floppy disks, CDROMs`_)
    which can applies to an individual disk).  Each ``device`` element has
-   two mandatory sub-elements, ``path`` describing the absolute path of the
-   device, and ``weight`` giving the relative weight of that device, in the
-   range [100, 1000]. After kernel 2.6.39, the value could be in the range [10,
-   1000]. :since:`Since 0.9.8`
+   a mandatory  ``path`` sub-element describing the absolute path of the
+   device.
+
+   A special value ``*`` can be used to throttle all domain
+   devices. :since:`Since 12.3.0, bhyve`
+
+   An optional ``weight`` sub-element specifies the relative
+   weight of the device, in the range [100, 1000]. After kernel 2.6.39,
+   the value could be in the range [10, 1000]. :since:`Since 0.9.8`
    Additionally, the following optional sub-elements can be used:
 
    ``read_bytes_sec``

diff --git a/src/bhyve/bhyve_capabilities.c b/src/bhyve/bhyve_capabilities.c
index c3fb88fe9f40bee2f325879a00fdf893fa4b79dc..1fe0a3ad77cb5fc3b482ec17e0a84a1715fd804b 100644 (file)
--- a/src/bhyve/bhyve_capabilities.c
+++ b/src/bhyve/bhyve_capabilities.c
@@ -24,6 +24,7 @@
 #include <config.h>
 #include <sys/utsname.h>
 #include <dirent.h>
+#include <sys/sysctl.h>
 #include <sys/types.h>
 
 #include "viralloc.h"
@@ -334,6 +335,27 @@ bhyveProbeCapsVNCPassword(unsigned int *caps, char *binary)
 }
 
 
+static void
+bhyveProbeCapsRctl(unsigned int *caps)
+{
+    bool racct_enable;
+    size_t racct_enable_len;
+    g_autofree char *rctl = NULL;
+
+    if (!(rctl = virFindFileInPath("rctl")))
+        return;
+
+    racct_enable_len = sizeof(racct_enable);
+    if (sysctlbyname("kern.racct.enable", &racct_enable,
+                     &racct_enable_len, NULL, 0) < 0)
+        return;
+
+    if (racct_enable)
+        *caps |= BHYVE_CAP_RCTL;
+
+    return;
+}
+
 int
 virBhyveProbeCaps(unsigned int *caps)
 {
@@ -356,6 +378,7 @@ virBhyveProbeCaps(unsigned int *caps)
     if ((ret = bhyveProbeCapsVNCPassword(caps, binary)))
         goto out;
 
+    bhyveProbeCapsRctl(caps);
 
  out:
     VIR_FREE(binary);

diff --git a/src/bhyve/bhyve_capabilities.h b/src/bhyve/bhyve_capabilities.h
index 31fd9ab86a22feabdc5bb89deb00a742782fe741..0302b68e2229d98b1dc326b6cabb8ece3c507d71 100644 (file)
--- a/src/bhyve/bhyve_capabilities.h
+++ b/src/bhyve/bhyve_capabilities.h
@@ -57,6 +57,7 @@ typedef enum {
     BHYVE_CAP_NVME = 1 << 11,
     BHYVE_CAP_ACPI = 1 << 12,
     BHYVE_CAP_NUMA = 1 << 13,
+    BHYVE_CAP_RCTL = 1 << 14,
 } virBhyveCapsFlags;
 
 int virBhyveProbeGrubCaps(virBhyveGrubCapsFlags *caps);

diff --git a/src/bhyve/bhyve_domain.c b/src/bhyve/bhyve_domain.c
index 4594d7673fb71edd978f9b916a058fa1bc916419..170f6edc56e62905503a687bc2f9c16798d655b3 100644 (file)
--- a/src/bhyve/bhyve_domain.c
+++ b/src/bhyve/bhyve_domain.c
@@ -464,6 +464,26 @@ bhyveDomainDefValidate(const virDomainDef *def,
         }
     }
 
+    if (def->blkio.ndevices > 1) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                       _("Per device I/O tuning is not supported"));
+        return -1;
+    } else if (def->blkio.ndevices == 1) {
+        virBlkioDevice *device = &def->blkio.devices[0];
+
+        if (STRNEQ(device->path, "*")) {
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                           _("Per device I/O tuning is not supported"));
+            return -1;
+        }
+
+        if (device->weight != 0) {
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                           _("The 'weight' I/O tuning setting does not make sense with '*'"));
+            return -1;
+        }
+    }
+
     if (!def->os.loader)
         return 0;
 

diff --git a/src/bhyve/bhyve_process.c b/src/bhyve/bhyve_process.c
index 1d436da609d43373616ac9a8aebaa1850fcd2ed6..26320200c5dd503f95c21e5d2a467caf38f2d8fc 100644 (file)
--- a/src/bhyve/bhyve_process.c
+++ b/src/bhyve/bhyve_process.c
@@ -34,6 +34,7 @@
 
 #include "bhyve_device.h"
 #include "bhyve_driver.h"
+#include "bhyve_capabilities.h"
 #include "bhyve_command.h"
 #include "bhyve_firmware.h"
 #include "bhyve_monitor.h"
@@ -132,6 +133,44 @@ bhyveProcessStopHook(struct _bhyveConn *driver,
                 VIR_HOOK_SUBOP_END, NULL, xml, NULL);
 }
 
+static int
+bhyveSetResourceLimits(struct _bhyveConn *driver, virDomainObj *vm)
+{
+    virBlkioDevice *device;
+
+    if (vm->def->blkio.ndevices != 1)
+        return 0;
+
+    if ((bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_RCTL) == 0) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                       _("Cannot set resource limits: RACCT/RCTL is either not supported or not enabled"));
+        return -1;
+    }
+
+    device = &vm->def->blkio.devices[0];
+
+#define BHYVE_APPLY_RCTL_RULE(field, type, format) \
+    do { \
+        if ((field)) { \
+            g_autofree char *rule = NULL; \
+            g_autoptr(virCommand) cmd = virCommandNewArgList("rctl", "-a", NULL); \
+            virCommandAddArgFormat(cmd, "process:%d:" type ":throttle=" format, \
+                                   vm->pid, (field)); \
+            if (virCommandRun(cmd, NULL) < 0) \
+                return -1; \
+         } \
+    } while (0)
+
+    BHYVE_APPLY_RCTL_RULE(device->riops, "readiops", "%u");
+    BHYVE_APPLY_RCTL_RULE(device->wiops, "writeiops", "%u");
+    BHYVE_APPLY_RCTL_RULE(device->rbps, "readbps", "%llu");
+    BHYVE_APPLY_RCTL_RULE(device->wbps, "writebps", "%llu");
+
+#undef BHYVE_APPLY_RCTL_RULE
+
+    return 0;
+}
+
 static int
 virBhyveProcessStartImpl(struct _bhyveConn *driver,
                          virDomainObj *vm,
@@ -258,6 +297,9 @@ virBhyveProcessStartImpl(struct _bhyveConn *driver,
                          BHYVE_STATE_DIR) < 0)
         goto cleanup;
 
+    if (bhyveSetResourceLimits(driver, vm) < 0)
+        goto cleanup;
+
     if (bhyveProcessStartHook(driver, vm, VIR_HOOK_BHYVE_OP_STARTED) < 0)
         goto cleanup;
 

diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
index db1dcd3bb70fd8562bb77c53047ed3823e069c91..c7f442a4c18145e07fbf35a774ec00f0b27ae6b3 100644 (file)
--- a/src/conf/schemas/domaincommon.rng
+++ b/src/conf/schemas/domaincommon.rng
@@ -1049,7 +1049,10 @@
           <element name="device">
             <interleave>
               <element name="path">
-                <ref name="absFilePath"/>
+                <choice>
+                  <ref name="absFilePath"/>
+                  <value>*</value>
+                </choice>
               </element>
               <optional>
                 <element name="weight">

diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-invalid-device.xml b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-invalid-device.xml
new file mode 100644 (file)
index 0000000..68d6871
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-invalid-device.xml
@@ -0,0 +1,32 @@
+<domain type='bhyve'>
+  <name>bhyve</name>
+  <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+  <memory>219136</memory>
+  <vcpu>1</vcpu>
+  <os>
+    <type>hvm</type>
+  </os>
+  <blkiotune>
+    <device>
+      <path>/dev/sda</path>
+      <read_bytes_sec>10000</read_bytes_sec>
+      <write_bytes_sec>10000</write_bytes_sec>
+      <read_iops_sec>20000</read_iops_sec>
+      <write_iops_sec>20000</write_iops_sec>
+    </device>
+  </blkiotune>
+  <devices>
+    <disk type='file'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd.img'/>
+      <target dev='hda' bus='sata'/>
+      <address type='drive' controller='0' bus='0' target='2' unit='0'/>
+    </disk>
+    <interface type='bridge'>
+      <mac address='52:54:00:b9:94:02'/>
+      <model type='virtio'/>
+      <source bridge="virbr0"/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </interface>
+  </devices>
+</domain>

diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml
new file mode 100644 (file)
index 0000000..a6b689a
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml
@@ -0,0 +1,39 @@
+<domain type='bhyve'>
+  <name>bhyve</name>
+  <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+  <memory>219136</memory>
+  <vcpu>1</vcpu>
+  <os>
+    <type>hvm</type>
+  </os>
+  <blkiotune>
+    <device>
+      <path>*</path>
+      <read_bytes_sec>10000</read_bytes_sec>
+      <write_bytes_sec>10000</write_bytes_sec>
+      <read_iops_sec>20000</read_iops_sec>
+      <write_iops_sec>20000</write_iops_sec>
+    </device>
+    <device>
+      <path>/dev/hda</path>
+      <read_bytes_sec>10000</read_bytes_sec>
+      <write_bytes_sec>10000</write_bytes_sec>
+      <read_iops_sec>20000</read_iops_sec>
+      <write_iops_sec>20000</write_iops_sec>
+    </device>
+  </blkiotune>
+  <devices>
+    <disk type='file'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd.img'/>
+      <target dev='hda' bus='sata'/>
+      <address type='drive' controller='0' bus='0' target='2' unit='0'/>
+    </disk>
+    <interface type='bridge'>
+      <mac address='52:54:00:b9:94:02'/>
+      <model type='virtio'/>
+      <source bridge="virbr0"/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </interface>
+  </devices>
+</domain>

diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-weight.xml b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-weight.xml
new file mode 100644 (file)
index 0000000..f760382
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-weight.xml
@@ -0,0 +1,33 @@
+<domain type='bhyve'>
+  <name>bhyve</name>
+  <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+  <memory>219136</memory>
+  <vcpu>1</vcpu>
+  <os>
+    <type>hvm</type>
+  </os>
+  <blkiotune>
+    <device>
+      <path>*</path>
+      <weight>100</weight>
+      <read_bytes_sec>10000</read_bytes_sec>
+      <write_bytes_sec>10000</write_bytes_sec>
+      <read_iops_sec>20000</read_iops_sec>
+      <write_iops_sec>20000</write_iops_sec>
+    </device>
+  </blkiotune>
+  <devices>
+    <disk type='file'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd.img'/>
+      <target dev='hda' bus='sata'/>
+      <address type='drive' controller='0' bus='0' target='2' unit='0'/>
+    </disk>
+    <interface type='bridge'>
+      <mac address='52:54:00:b9:94:02'/>
+      <model type='virtio'/>
+      <source bridge="virbr0"/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </interface>
+  </devices>
+</domain>

diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args
new file mode 100644 (file)
index 0000000..507e0be
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args
@@ -0,0 +1,10 @@
+bhyve \
+-c 1 \
+-m 214 \
+-u \
+-H \
+-P \
+-s 0:0,hostbridge \
+-s 2:0,ahci,hd:/tmp/freebsd.img \
+-s 3:0,virtio-net,faketapdev,mac=52:54:00:b9:94:02 \
+bhyve

diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs
new file mode 100644 (file)
index 0000000..5905f4b
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs
@@ -0,0 +1,4 @@
+bhyveload \
+-m 214 \
+-d /tmp/freebsd.img \
+bhyve

diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml
new file mode 100644 (file)
index 0000000..956d96c
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml
@@ -0,0 +1,32 @@
+<domain type='bhyve'>
+  <name>bhyve</name>
+  <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+  <memory>219136</memory>
+  <vcpu>1</vcpu>
+  <os>
+    <type>hvm</type>
+  </os>
+  <blkiotune>
+    <device>
+      <path>*</path>
+      <read_bytes_sec>10000</read_bytes_sec>
+      <write_bytes_sec>10000</write_bytes_sec>
+      <read_iops_sec>20000</read_iops_sec>
+      <write_iops_sec>20000</write_iops_sec>
+    </device>
+  </blkiotune>
+  <devices>
+    <disk type='file'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd.img'/>
+      <target dev='hda' bus='sata'/>
+      <address type='drive' controller='0' bus='0' target='2' unit='0'/>
+    </disk>
+    <interface type='bridge'>
+      <mac address='52:54:00:b9:94:02'/>
+      <model type='virtio'/>
+      <source bridge="virbr0"/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </interface>
+  </devices>
+</domain>

diff --git a/tests/bhyvexml2argvtest.c b/tests/bhyvexml2argvtest.c
index b7749fec6ff6d52bc81e096f8202e61fb816e86f..71c67ba2a2a2fd3d52d3ede26e3495516f91e186 100644 (file)
--- a/tests/bhyvexml2argvtest.c
+++ b/tests/bhyvexml2argvtest.c
@@ -289,6 +289,10 @@ mymain(void)
     DO_TEST_FAILURE("slirp-ip");
     DO_TEST("virtio-scsi");
     DO_TEST("vcpupin");
+    DO_TEST("blkiotune");
+    DO_TEST_FAILURE("blkiotune-invalid-device");
+    DO_TEST_FAILURE("blkiotune-multiple-devices");
+    DO_TEST_FAILURE("blkiotune-weight");
 
     /* Address allocation tests */
     DO_TEST("addr-single-sata-disk");

diff --git a/tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml b/tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml
new file mode 100644 (file)
index 0000000..4170303
--- /dev/null
+++ b/tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml
@@ -0,0 +1,42 @@
+<domain type='bhyve'>
+  <name>bhyve</name>
+  <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+  <memory unit='KiB'>219136</memory>
+  <currentMemory unit='KiB'>219136</currentMemory>
+  <blkiotune>
+    <device>
+      <path>*</path>
+      <read_iops_sec>20000</read_iops_sec>
+      <write_iops_sec>20000</write_iops_sec>
+      <read_bytes_sec>10000</read_bytes_sec>
+      <write_bytes_sec>10000</write_bytes_sec>
+    </device>
+  </blkiotune>
+  <vcpu placement='static'>1</vcpu>
+  <os>
+    <type arch='x86_64'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <disk type='file' device='disk'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd.img'/>
+      <target dev='hda' bus='sata'/>
+      <address type='drive' controller='0' bus='0' target='2' unit='0'/>
+    </disk>
+    <controller type='pci' index='0' model='pci-root'/>
+    <controller type='sata' index='0'>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
+    </controller>
+    <interface type='bridge'>
+      <mac address='52:54:00:b9:94:02'/>
+      <source bridge='virbr0'/>
+      <model type='virtio'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </interface>
+  </devices>
+</domain>

diff --git a/tests/bhyvexml2xmltest.c b/tests/bhyvexml2xmltest.c
index 950aaea67223d43bc36874b6649a1cef2f778213..9fa8a9765dd368ea362fc6f2b724ad2883b37684 100644 (file)
--- a/tests/bhyvexml2xmltest.c
+++ b/tests/bhyvexml2xmltest.c
@@ -133,6 +133,10 @@ mymain(void)
     DO_TEST_DIFFERENT("slirp");
     DO_TEST_DIFFERENT("virtio-scsi");
     DO_TEST_DIFFERENT("numa");
+    DO_TEST_DIFFERENT("blkiotune");
+    DO_TEST_FAILURE("blkiotune-invalid-device");
+    DO_TEST_FAILURE("blkiotune-weight");
+    DO_TEST_FAILURE("blkiotune-multiple-devices");
 
     /* Address allocation tests */
     DO_TEST_DIFFERENT("addr-single-sata-disk");