vhost-scsi: Reduce mem use by moving upages to per queue

Each worker thread can process 1 command at a time so there's no need to
allocate a upages array per cmd. This patch moves it to per queue. Even a
small device with 128 cmds and 1 queue this brings mem use for the array
from

2 MB = 8 bytes per page pointer * 2048 pointers * 128 cmds

to

16K = 8 bytes per pointer * 2048 * 1 queue

Signed-off-by: Mike Christie <michael.christie@oracle.com>
Message-Id: <20241203191705.19431-2-michael.christie@oracle.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
Acked-by: Stefan Hajnoczi <stefanha@redhat.com>

diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c
index 7aeff435c1d873f0ff8e80ceae8f2ea8916561c5..c266171045c54af60b66371f0aa5b0397878ffc6 100644 (file)
--- a/drivers/vhost/scsi.c
+++ b/drivers/vhost/scsi.c
@@ -83,7 +83,6 @@ struct vhost_scsi_cmd {
        /* Pointer to the SGL formatted memory from virtio-scsi */
        struct scatterlist *tvc_sgl;
        struct scatterlist *tvc_prot_sgl;
-       struct page **tvc_upages;
        /* Pointer to response header iovec */
        struct iovec *tvc_resp_iov;
        /* Pointer to vhost_scsi for our device */
@@ -187,6 +186,7 @@ struct vhost_scsi_virtqueue {
        struct vhost_scsi_cmd *scsi_cmds;
        struct sbitmap scsi_tags;
        int max_cmds;
+       struct page **upages;
 
        struct vhost_work completion_work;
        struct llist_head completion_list;
@@ -619,7 +619,6 @@ vhost_scsi_get_cmd(struct vhost_virtqueue *vq, struct vhost_scsi_tpg *tpg,
        struct vhost_scsi_nexus *tv_nexus;
        struct scatterlist *sg, *prot_sg;
        struct iovec *tvc_resp_iov;
-       struct page **pages;
        int tag;
 
        tv_nexus = tpg->tpg_nexus;
@@ -637,12 +636,10 @@ vhost_scsi_get_cmd(struct vhost_virtqueue *vq, struct vhost_scsi_tpg *tpg,
        cmd = &svq->scsi_cmds[tag];
        sg = cmd->tvc_sgl;
        prot_sg = cmd->tvc_prot_sgl;
-       pages = cmd->tvc_upages;
        tvc_resp_iov = cmd->tvc_resp_iov;
        memset(cmd, 0, sizeof(*cmd));
        cmd->tvc_sgl = sg;
        cmd->tvc_prot_sgl = prot_sg;
-       cmd->tvc_upages = pages;
        cmd->tvc_se_cmd.map_tag = tag;
        cmd->tvc_tag = scsi_tag;
        cmd->tvc_lun = lun;
@@ -669,7 +666,9 @@ vhost_scsi_map_to_sgl(struct vhost_scsi_cmd *cmd,
                      struct scatterlist *sgl,
                      bool is_prot)
 {
-       struct page **pages = cmd->tvc_upages;
+       struct vhost_scsi_virtqueue *svq = container_of(cmd->tvc_vq,
+                                       struct vhost_scsi_virtqueue, vq);
+       struct page **pages = svq->upages;
        struct scatterlist *sg = sgl;
        ssize_t bytes, mapped_bytes;
        size_t offset, mapped_offset;
@@ -1598,11 +1597,11 @@ static void vhost_scsi_destroy_vq_cmds(struct vhost_virtqueue *vq)
 
                kfree(tv_cmd->tvc_sgl);
                kfree(tv_cmd->tvc_prot_sgl);
-               kfree(tv_cmd->tvc_upages);
                kfree(tv_cmd->tvc_resp_iov);
        }
 
        sbitmap_free(&svq->scsi_tags);
+       kfree(svq->upages);
        kfree(svq->scsi_cmds);
        svq->scsi_cmds = NULL;
 }
@@ -1628,6 +1627,11 @@ static int vhost_scsi_setup_vq_cmds(struct vhost_virtqueue *vq, int max_cmds)
                return -ENOMEM;
        }
 
+       svq->upages = kcalloc(VHOST_SCSI_PREALLOC_UPAGES, sizeof(struct page *),
+                             GFP_KERNEL);
+       if (!svq->upages)
+               goto out;
+
        for (i = 0; i < max_cmds; i++) {
                tv_cmd = &svq->scsi_cmds[i];
 
@@ -1639,14 +1643,6 @@ static int vhost_scsi_setup_vq_cmds(struct vhost_virtqueue *vq, int max_cmds)
                        goto out;
                }
 
-               tv_cmd->tvc_upages = kcalloc(VHOST_SCSI_PREALLOC_UPAGES,
-                                            sizeof(struct page *),
-                                            GFP_KERNEL);
-               if (!tv_cmd->tvc_upages) {
-                       pr_err("Unable to allocate tv_cmd->tvc_upages\n");
-                       goto out;
-               }
-
                tv_cmd->tvc_resp_iov = kcalloc(UIO_MAXIOV,
                                               sizeof(struct iovec),
                                               GFP_KERNEL);