From: Roman Bogorodskiy Date: Sun, 5 Apr 2026 09:23:59 +0000 (+0200) Subject: bhyve: add blkiotune support X-Git-Tag: v12.3.0-rc1~19 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ddf09630e536394e28a3000e7f58e5d78d948303;p=thirdparty%2Flibvirt.git bhyve: add blkiotune support FreeBSD supports resource limiting with the rctl(4) framework. It supports various resource types, including I/O resources. It allows to limit resources for users, processes, login classes, and jails. To apply blkiotune limits set limits for the bhyve process. I/O related resources supported by rctl(4) are: readbps filesystem reads, in bytes per second writebps filesystem writes, in bytes per second readiops filesystem reads, in operations per second writeiops filesystem writes, in operations per second Thus, the actual commands look like: rctl -a process:$bhyvepid:writebps:throttle=10000000 rctl -a process:$bhyvepid:readbps:throttle=10000000 rctl -a process:$bhyvepid:writeiops:throttle=20000 rctl -a process:$bhyvepid:readiops:throttle=20000 This is different from the current blkiotune modeling in libvirt as it requires specific device to apply limits to. To adapt this model to per-domain I/O limits, update domain schema to specify "*" as a device name. The rctl(8) may be not available or not enabled, so add a capability check for that. Per process rules get removed when the process disappears, so no special clean up is necessary. Signed-off-by: Roman Bogorodskiy Reviewed-by: Peter Krempa --- diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst index b589fc9429..078cd7aa84 100644 --- a/docs/formatdomain.rst +++ b/docs/formatdomain.rst @@ -1367,10 +1367,15 @@ Block I/O Tuning associated with each guest disk device (contrast this to the element of a disk definition (See `Hard drives, floppy disks, CDROMs`_) which can applies to an individual disk). Each ``device`` element has - two mandatory sub-elements, ``path`` describing the absolute path of the - device, and ``weight`` giving the relative weight of that device, in the - range [100, 1000]. After kernel 2.6.39, the value could be in the range [10, - 1000]. :since:`Since 0.9.8` + a mandatory ``path`` sub-element describing the absolute path of the + device. + + A special value ``*`` can be used to throttle all domain + devices. :since:`Since 12.3.0, bhyve` + + An optional ``weight`` sub-element specifies the relative + weight of the device, in the range [100, 1000]. After kernel 2.6.39, + the value could be in the range [10, 1000]. :since:`Since 0.9.8` Additionally, the following optional sub-elements can be used: ``read_bytes_sec`` diff --git a/src/bhyve/bhyve_capabilities.c b/src/bhyve/bhyve_capabilities.c index c3fb88fe9f..1fe0a3ad77 100644 --- a/src/bhyve/bhyve_capabilities.c +++ b/src/bhyve/bhyve_capabilities.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include "viralloc.h" @@ -334,6 +335,27 @@ bhyveProbeCapsVNCPassword(unsigned int *caps, char *binary) } +static void +bhyveProbeCapsRctl(unsigned int *caps) +{ + bool racct_enable; + size_t racct_enable_len; + g_autofree char *rctl = NULL; + + if (!(rctl = virFindFileInPath("rctl"))) + return; + + racct_enable_len = sizeof(racct_enable); + if (sysctlbyname("kern.racct.enable", &racct_enable, + &racct_enable_len, NULL, 0) < 0) + return; + + if (racct_enable) + *caps |= BHYVE_CAP_RCTL; + + return; +} + int virBhyveProbeCaps(unsigned int *caps) { @@ -356,6 +378,7 @@ virBhyveProbeCaps(unsigned int *caps) if ((ret = bhyveProbeCapsVNCPassword(caps, binary))) goto out; + bhyveProbeCapsRctl(caps); out: VIR_FREE(binary); diff --git a/src/bhyve/bhyve_capabilities.h b/src/bhyve/bhyve_capabilities.h index 31fd9ab86a..0302b68e22 100644 --- a/src/bhyve/bhyve_capabilities.h +++ b/src/bhyve/bhyve_capabilities.h @@ -57,6 +57,7 @@ typedef enum { BHYVE_CAP_NVME = 1 << 11, BHYVE_CAP_ACPI = 1 << 12, BHYVE_CAP_NUMA = 1 << 13, + BHYVE_CAP_RCTL = 1 << 14, } virBhyveCapsFlags; int virBhyveProbeGrubCaps(virBhyveGrubCapsFlags *caps); diff --git a/src/bhyve/bhyve_domain.c b/src/bhyve/bhyve_domain.c index 4594d7673f..170f6edc56 100644 --- a/src/bhyve/bhyve_domain.c +++ b/src/bhyve/bhyve_domain.c @@ -464,6 +464,26 @@ bhyveDomainDefValidate(const virDomainDef *def, } } + if (def->blkio.ndevices > 1) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Per device I/O tuning is not supported")); + return -1; + } else if (def->blkio.ndevices == 1) { + virBlkioDevice *device = &def->blkio.devices[0]; + + if (STRNEQ(device->path, "*")) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Per device I/O tuning is not supported")); + return -1; + } + + if (device->weight != 0) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("The 'weight' I/O tuning setting does not make sense with '*'")); + return -1; + } + } + if (!def->os.loader) return 0; diff --git a/src/bhyve/bhyve_process.c b/src/bhyve/bhyve_process.c index 1d436da609..26320200c5 100644 --- a/src/bhyve/bhyve_process.c +++ b/src/bhyve/bhyve_process.c @@ -34,6 +34,7 @@ #include "bhyve_device.h" #include "bhyve_driver.h" +#include "bhyve_capabilities.h" #include "bhyve_command.h" #include "bhyve_firmware.h" #include "bhyve_monitor.h" @@ -132,6 +133,44 @@ bhyveProcessStopHook(struct _bhyveConn *driver, VIR_HOOK_SUBOP_END, NULL, xml, NULL); } +static int +bhyveSetResourceLimits(struct _bhyveConn *driver, virDomainObj *vm) +{ + virBlkioDevice *device; + + if (vm->def->blkio.ndevices != 1) + return 0; + + if ((bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_RCTL) == 0) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Cannot set resource limits: RACCT/RCTL is either not supported or not enabled")); + return -1; + } + + device = &vm->def->blkio.devices[0]; + +#define BHYVE_APPLY_RCTL_RULE(field, type, format) \ + do { \ + if ((field)) { \ + g_autofree char *rule = NULL; \ + g_autoptr(virCommand) cmd = virCommandNewArgList("rctl", "-a", NULL); \ + virCommandAddArgFormat(cmd, "process:%d:" type ":throttle=" format, \ + vm->pid, (field)); \ + if (virCommandRun(cmd, NULL) < 0) \ + return -1; \ + } \ + } while (0) + + BHYVE_APPLY_RCTL_RULE(device->riops, "readiops", "%u"); + BHYVE_APPLY_RCTL_RULE(device->wiops, "writeiops", "%u"); + BHYVE_APPLY_RCTL_RULE(device->rbps, "readbps", "%llu"); + BHYVE_APPLY_RCTL_RULE(device->wbps, "writebps", "%llu"); + +#undef BHYVE_APPLY_RCTL_RULE + + return 0; +} + static int virBhyveProcessStartImpl(struct _bhyveConn *driver, virDomainObj *vm, @@ -258,6 +297,9 @@ virBhyveProcessStartImpl(struct _bhyveConn *driver, BHYVE_STATE_DIR) < 0) goto cleanup; + if (bhyveSetResourceLimits(driver, vm) < 0) + goto cleanup; + if (bhyveProcessStartHook(driver, vm, VIR_HOOK_BHYVE_OP_STARTED) < 0) goto cleanup; diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng index db1dcd3bb7..c7f442a4c1 100644 --- a/src/conf/schemas/domaincommon.rng +++ b/src/conf/schemas/domaincommon.rng @@ -1049,7 +1049,10 @@ - + + + * + diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-invalid-device.xml b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-invalid-device.xml new file mode 100644 index 0000000000..68d6871e0c --- /dev/null +++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-invalid-device.xml @@ -0,0 +1,32 @@ + + bhyve + df3be7e7-a104-11e3-aeb0-50e5492bd3dc + 219136 + 1 + + hvm + + + + /dev/sda + 10000 + 10000 + 20000 + 20000 + + + + + + + +
+ + + + + +
+ + + diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml new file mode 100644 index 0000000000..a6b689ad9a --- /dev/null +++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml @@ -0,0 +1,39 @@ + + bhyve + df3be7e7-a104-11e3-aeb0-50e5492bd3dc + 219136 + 1 + + hvm + + + + * + 10000 + 10000 + 20000 + 20000 + + + /dev/hda + 10000 + 10000 + 20000 + 20000 + + + + + + + +
+ + + + + +
+ + + diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-weight.xml b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-weight.xml new file mode 100644 index 0000000000..f7603828e0 --- /dev/null +++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-weight.xml @@ -0,0 +1,33 @@ + + bhyve + df3be7e7-a104-11e3-aeb0-50e5492bd3dc + 219136 + 1 + + hvm + + + + * + 100 + 10000 + 10000 + 20000 + 20000 + + + + + + + +
+ + + + + +
+ + + diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args new file mode 100644 index 0000000000..507e0be668 --- /dev/null +++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args @@ -0,0 +1,10 @@ +bhyve \ +-c 1 \ +-m 214 \ +-u \ +-H \ +-P \ +-s 0:0,hostbridge \ +-s 2:0,ahci,hd:/tmp/freebsd.img \ +-s 3:0,virtio-net,faketapdev,mac=52:54:00:b9:94:02 \ +bhyve diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs new file mode 100644 index 0000000000..5905f4b3e6 --- /dev/null +++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs @@ -0,0 +1,4 @@ +bhyveload \ +-m 214 \ +-d /tmp/freebsd.img \ +bhyve diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml new file mode 100644 index 0000000000..956d96cf18 --- /dev/null +++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml @@ -0,0 +1,32 @@ + + bhyve + df3be7e7-a104-11e3-aeb0-50e5492bd3dc + 219136 + 1 + + hvm + + + + * + 10000 + 10000 + 20000 + 20000 + + + + + + + +
+ + + + + +
+ + + diff --git a/tests/bhyvexml2argvtest.c b/tests/bhyvexml2argvtest.c index b7749fec6f..71c67ba2a2 100644 --- a/tests/bhyvexml2argvtest.c +++ b/tests/bhyvexml2argvtest.c @@ -289,6 +289,10 @@ mymain(void) DO_TEST_FAILURE("slirp-ip"); DO_TEST("virtio-scsi"); DO_TEST("vcpupin"); + DO_TEST("blkiotune"); + DO_TEST_FAILURE("blkiotune-invalid-device"); + DO_TEST_FAILURE("blkiotune-multiple-devices"); + DO_TEST_FAILURE("blkiotune-weight"); /* Address allocation tests */ DO_TEST("addr-single-sata-disk"); diff --git a/tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml b/tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml new file mode 100644 index 0000000000..4170303a6e --- /dev/null +++ b/tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml @@ -0,0 +1,42 @@ + + bhyve + df3be7e7-a104-11e3-aeb0-50e5492bd3dc + 219136 + 219136 + + + * + 20000 + 20000 + 10000 + 10000 + + + 1 + + hvm + + + + destroy + restart + destroy + + + + + +
+ + + +
+ + + + + +
+ + + diff --git a/tests/bhyvexml2xmltest.c b/tests/bhyvexml2xmltest.c index 950aaea672..9fa8a9765d 100644 --- a/tests/bhyvexml2xmltest.c +++ b/tests/bhyvexml2xmltest.c @@ -133,6 +133,10 @@ mymain(void) DO_TEST_DIFFERENT("slirp"); DO_TEST_DIFFERENT("virtio-scsi"); DO_TEST_DIFFERENT("numa"); + DO_TEST_DIFFERENT("blkiotune"); + DO_TEST_FAILURE("blkiotune-invalid-device"); + DO_TEST_FAILURE("blkiotune-weight"); + DO_TEST_FAILURE("blkiotune-multiple-devices"); /* Address allocation tests */ DO_TEST_DIFFERENT("addr-single-sata-disk");