vmspawn-qmp: pipeline remove-fd after each blockdev-add

QEMU keeps a monitor-side fd alive until either an explicit remove-fd
arrives or the fdset's last duplicate is closed. Today vmspawn issues
add-fd but never the matching remove-fd, so each fdset stays around for
the lifetime of the VM even after the consuming blockdev is torn down.
Pipelining a remove-fd directly after the blockdev-add that consumed
the fd hands ownership entirely to the blockdev: the fdset
auto-disposes when raw_close runs at blockdev-del time. This is the
shape needed by hotplug, where blockdev-del must clean up everything
without further coordination.

Mechanically:
- qmp_fdset_add() takes a callback/userdata pair (so callers control
  failure handling) and an optional out-param for the numeric fdset id.
  All boot-time callers keep using on_qmp_complete with a label.
- A new qmp_fdset_remove() helper sends remove-fd with caller-supplied
  callback/userdata.
- qmp_setup_ephemeral_drive captures both fdset ids and fires remove-fd
  immediately after each base/overlay file blockdev-add.
- qmp_setup_regular_drive does the same for its single file blockdev-add.

Signed-off-by: Christian Brauner (Amutable) <brauner@kernel.org>

diff --git a/src/vmspawn/vmspawn-qmp.c b/src/vmspawn/vmspawn-qmp.c
index bb7c9e06a3b381a8ad5422f74d2da7862db825f7..cfa887e1223a3cd5b3dca46fae71be64e0513642 100644 (file)
--- a/src/vmspawn/vmspawn-qmp.c
+++ b/src/vmspawn/vmspawn-qmp.c
@@ -125,9 +125,15 @@ static int on_qmp_complete(
         return 0;
 }
 
-/* Send add-fd via SCM_RIGHTS; return /dev/fdset/N. Allocations run before invoke so a late
- * OOM cannot orphan an fdset on QEMU's side; *ret_path is only written on full success. */
-static int qmp_fdset_add(QmpClient *qmp, int fd_consume, char **ret_path) {
+/* Send add-fd via SCM_RIGHTS; return /dev/fdset/N and the numeric fdset id. */
+static int qmp_fdset_add(
+                QmpClient *qmp,
+                int fd_consume,
+                qmp_command_callback_t callback,
+                void *userdata,
+                char **ret_path,
+                uint64_t *ret_fdset_id) {
+
         _cleanup_(sd_json_variant_unrefp) sd_json_variant *args = NULL;
         _cleanup_close_ int fd = fd_consume;
         _cleanup_free_ char *path = NULL;
@@ -136,6 +142,7 @@ static int qmp_fdset_add(QmpClient *qmp, int fd_consume, char **ret_path) {
 
         assert(qmp);
         assert(fd_consume >= 0);
+        assert(callback);
         assert(ret_path);
 
         id = qmp_client_next_fdset_id(qmp);
@@ -148,14 +155,39 @@ static int qmp_fdset_add(QmpClient *qmp, int fd_consume, char **ret_path) {
                 return -ENOMEM;
 
         r = qmp_client_invoke(qmp, /* ret_slot= */ NULL, "add-fd", QMP_CLIENT_ARGS_FD(args, TAKE_FD(fd)),
-                              on_qmp_complete, (void*) "add-fd");
+                              callback, userdata);
         if (r < 0)
                 return r;
 
         *ret_path = TAKE_PTR(path);
+        if (ret_fdset_id)
+                *ret_fdset_id = id;
         return 0;
 }
 
+/* Issue remove-fd for an fdset whose dup is now held by a blockdev. The fdset
+ * persists until the dup is closed (in raw_close at blockdev-del time) — see
+ * QEMU's monitor/fds.c:177-181 on the fds/dup_fds split. */
+static int qmp_fdset_remove(
+                QmpClient *qmp,
+                uint64_t fdset_id,
+                qmp_command_callback_t callback,
+                void *userdata) {
+
+        _cleanup_(sd_json_variant_unrefp) sd_json_variant *args = NULL;
+        int r;
+
+        assert(qmp);
+        assert(callback);
+
+        r = sd_json_buildo(&args, SD_JSON_BUILD_PAIR_UNSIGNED("fdset-id", fdset_id));
+        if (r < 0)
+                return r;
+
+        return qmp_client_invoke(qmp, /* ret_slot= */ NULL, "remove-fd", QMP_CLIENT_ARGS(args),
+                                 callback, userdata);
+}
+
 typedef struct QmpFileNodeParams {
         const char *node_name;
         const char *filename;
@@ -399,12 +431,16 @@ static int qmp_setup_ephemeral_drive(VmspawnQmpBridge *bridge, QmpClient *qmp, D
 
         /* Step 1-2: Pass both fds to QEMU */
         _cleanup_free_ char *base_path = NULL;
-        r = qmp_fdset_add(qmp, TAKE_FD(drive->fd), &base_path);
+        uint64_t base_fdset_id;
+        r = qmp_fdset_add(qmp, TAKE_FD(drive->fd),
+                          on_qmp_complete, (void*) "add-fd", &base_path, &base_fdset_id);
         if (r < 0)
                 return log_error_errno(r, "Failed to send add-fd for base image '%s': %m", drive->path);
 
         _cleanup_free_ char *overlay_path = NULL;
-        r = qmp_fdset_add(qmp, TAKE_FD(drive->overlay_fd), &overlay_path);
+        uint64_t overlay_fdset_id;
+        r = qmp_fdset_add(qmp, TAKE_FD(drive->overlay_fd),
+                          on_qmp_complete, (void*) "add-fd", &overlay_path, &overlay_fdset_id);
         if (r < 0)
                 return log_error_errno(r, "Failed to send add-fd for overlay of '%s': %m", drive->path);
 
@@ -421,6 +457,12 @@ static int qmp_setup_ephemeral_drive(VmspawnQmpBridge *bridge, QmpClient *qmp, D
         if (r < 0)
                 return log_error_errno(r, "Failed to send blockdev-add for base file '%s': %m", drive->path);
 
+        /* The base file node now holds a dup of the fd; release the monitor's
+         * original so the fdset auto-frees when raw_close runs at teardown. */
+        r = qmp_fdset_remove(qmp, base_fdset_id, on_qmp_complete, (void*) "remove-fd");
+        if (r < 0)
+                return log_error_errno(r, "Failed to send remove-fd for base image '%s': %m", drive->path);
+
         /* Step 4: Base image format node (read-only) */
         QmpFormatNodeParams base_fmt_params = {
                 .node_name      = base_fmt_node,
@@ -453,6 +495,11 @@ static int qmp_setup_ephemeral_drive(VmspawnQmpBridge *bridge, QmpClient *qmp, D
         if (r < 0)
                 return log_error_errno(r, "Failed to send blockdev-add for overlay file '%s': %m", drive->path);
 
+        /* Same as for base: the overlay file node has the dup. */
+        r = qmp_fdset_remove(qmp, overlay_fdset_id, on_qmp_complete, (void*) "remove-fd");
+        if (r < 0)
+                return log_error_errno(r, "Failed to send remove-fd for overlay of '%s': %m", drive->path);
+
         /* Step 6: Fire blockdev-create to format the overlay */
         _cleanup_(sd_json_variant_unrefp) sd_json_variant *create_options = NULL;
         r = sd_json_buildo(&create_options,
@@ -533,7 +580,9 @@ static int qmp_setup_regular_drive(VmspawnQmpBridge *bridge, QmpClient *qmp, Dri
                 return log_oom();
 
         _cleanup_free_ char *fdset_path = NULL;
-        r = qmp_fdset_add(qmp, TAKE_FD(drive->fd), &fdset_path);
+        uint64_t fdset_id;
+        r = qmp_fdset_add(qmp, TAKE_FD(drive->fd),
+                          on_qmp_complete, (void*) "add-fd", &fdset_path, &fdset_id);
         if (r < 0)
                 return log_error_errno(r, "Failed to send add-fd for '%s': %m", drive->path);
 
@@ -549,6 +598,12 @@ static int qmp_setup_regular_drive(VmspawnQmpBridge *bridge, QmpClient *qmp, Dri
         if (r < 0)
                 return log_error_errno(r, "Failed to send blockdev-add for '%s': %m", drive->path);
 
+        /* The file node now holds a dup of the fd; release the monitor's
+         * original so the fdset auto-frees when raw_close runs at teardown. */
+        r = qmp_fdset_remove(qmp, fdset_id, on_qmp_complete, (void*) "remove-fd");
+        if (r < 0)
+                return log_error_errno(r, "Failed to send remove-fd for '%s': %m", drive->path);
+
         QmpFormatNodeParams fmt_params = {
                 .node_name      = drive->qmp_node_name,
                 .format         = drive->format,