Merge tag 'kvm-x86-generic-6.8' of https://github.com/kvm-x86/linux into HEAD

[thirdparty/kernel/stable.git] / fs / fuse / dax.c

// SPDX-License-Identifier: GPL-2.0
/*
 * dax: direct host memory access
 * Copyright (C) 2020 Red Hat, Inc.
 */

#include "fuse_i.h"

#include <linux/delay.h>
#include <linux/dax.h>
#include <linux/uio.h>
#include <linux/pagemap.h>
#include <linux/pfn_t.h>
#include <linux/iomap.h>
#include <linux/interval_tree.h>

/*
 * Default memory range size.  A power of 2 so it agrees with common FUSE_INIT
 * map_alignment values 4KB and 64KB.
 */
#define FUSE_DAX_SHIFT  21
#define FUSE_DAX_SZ     (1 << FUSE_DAX_SHIFT)
#define FUSE_DAX_PAGES  (FUSE_DAX_SZ / PAGE_SIZE)

/* Number of ranges reclaimer will try to free in one invocation */
#define FUSE_DAX_RECLAIM_CHUNK          (10)

/*
 * Dax memory reclaim threshold in percetage of total ranges. When free
 * number of free ranges drops below this threshold, reclaim can trigger
 * Default is 20%
 */
#define FUSE_DAX_RECLAIM_THRESHOLD      (20)

/** Translation information for file offsets to DAX window offsets */
struct fuse_dax_mapping {
        /* Pointer to inode where this memory range is mapped */
        struct inode *inode;

        /* Will connect in fcd->free_ranges to keep track of free memory */
        struct list_head list;

        /* For interval tree in file/inode */
        struct interval_tree_node itn;

        /* Will connect in fc->busy_ranges to keep track busy memory */
        struct list_head busy_list;

        /** Position in DAX window */
        u64 window_offset;

        /** Length of mapping, in bytes */
        loff_t length;

        /* Is this mapping read-only or read-write */
        bool writable;

        /* reference count when the mapping is used by dax iomap. */
        refcount_t refcnt;
};

/* Per-inode dax map */
struct fuse_inode_dax {
        /* Semaphore to protect modifications to the dmap tree */
        struct rw_semaphore sem;

        /* Sorted rb tree of struct fuse_dax_mapping elements */
        struct rb_root_cached tree;
        unsigned long nr;
};

struct fuse_conn_dax {
        /* DAX device */
        struct dax_device *dev;

        /* Lock protecting accessess to  members of this structure */
        spinlock_t lock;

        /* List of memory ranges which are busy */
        unsigned long nr_busy_ranges;
        struct list_head busy_ranges;

        /* Worker to free up memory ranges */
        struct delayed_work free_work;

        /* Wait queue for a dax range to become free */
        wait_queue_head_t range_waitq;

        /* DAX Window Free Ranges */
        long nr_free_ranges;
        struct list_head free_ranges;

        unsigned long nr_ranges;
};

static inline struct fuse_dax_mapping *
node_to_dmap(struct interval_tree_node *node)
{
        if (!node)
                return NULL;

        return container_of(node, struct fuse_dax_mapping, itn);
}

static struct fuse_dax_mapping *
alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode);

static void
__kick_dmap_free_worker(struct fuse_conn_dax *fcd, unsigned long delay_ms)
{
        unsigned long free_threshold;

        /* If number of free ranges are below threshold, start reclaim */
        free_threshold = max_t(unsigned long, fcd->nr_ranges * FUSE_DAX_RECLAIM_THRESHOLD / 100,
                             1);
        if (fcd->nr_free_ranges < free_threshold)
                queue_delayed_work(system_long_wq, &fcd->free_work,
                                   msecs_to_jiffies(delay_ms));
}

static void kick_dmap_free_worker(struct fuse_conn_dax *fcd,
                                  unsigned long delay_ms)
{
        spin_lock(&fcd->lock);
        __kick_dmap_free_worker(fcd, delay_ms);
        spin_unlock(&fcd->lock);
}

static struct fuse_dax_mapping *alloc_dax_mapping(struct fuse_conn_dax *fcd)
{
        struct fuse_dax_mapping *dmap;

        spin_lock(&fcd->lock);
        dmap = list_first_entry_or_null(&fcd->free_ranges,
                                        struct fuse_dax_mapping, list);
        if (dmap) {
                list_del_init(&dmap->list);
                WARN_ON(fcd->nr_free_ranges <= 0);
                fcd->nr_free_ranges--;
        }
        __kick_dmap_free_worker(fcd, 0);
        spin_unlock(&fcd->lock);

        return dmap;
}

/* This assumes fcd->lock is held */
static void __dmap_remove_busy_list(struct fuse_conn_dax *fcd,
                                    struct fuse_dax_mapping *dmap)
{
        list_del_init(&dmap->busy_list);
        WARN_ON(fcd->nr_busy_ranges == 0);
        fcd->nr_busy_ranges--;
}

static void dmap_remove_busy_list(struct fuse_conn_dax *fcd,
                                  struct fuse_dax_mapping *dmap)
{
        spin_lock(&fcd->lock);
        __dmap_remove_busy_list(fcd, dmap);
        spin_unlock(&fcd->lock);
}

/* This assumes fcd->lock is held */
static void __dmap_add_to_free_pool(struct fuse_conn_dax *fcd,
                                struct fuse_dax_mapping *dmap)
{
        list_add_tail(&dmap->list, &fcd->free_ranges);
        fcd->nr_free_ranges++;
        wake_up(&fcd->range_waitq);
}

static void dmap_add_to_free_pool(struct fuse_conn_dax *fcd,
                                struct fuse_dax_mapping *dmap)
{
        /* Return fuse_dax_mapping to free list */
        spin_lock(&fcd->lock);
        __dmap_add_to_free_pool(fcd, dmap);
        spin_unlock(&fcd->lock);
}

static int fuse_setup_one_mapping(struct inode *inode, unsigned long start_idx,
                                  struct fuse_dax_mapping *dmap, bool writable,
                                  bool upgrade)
{
        struct fuse_mount *fm = get_fuse_mount(inode);
        struct fuse_conn_dax *fcd = fm->fc->dax;
        struct fuse_inode *fi = get_fuse_inode(inode);
        struct fuse_setupmapping_in inarg;
        loff_t offset = start_idx << FUSE_DAX_SHIFT;
        FUSE_ARGS(args);
        ssize_t err;

        WARN_ON(fcd->nr_free_ranges < 0);

        /* Ask fuse daemon to setup mapping */
        memset(&inarg, 0, sizeof(inarg));
        inarg.foffset = offset;
        inarg.fh = -1;
        inarg.moffset = dmap->window_offset;
        inarg.len = FUSE_DAX_SZ;
        inarg.flags |= FUSE_SETUPMAPPING_FLAG_READ;
        if (writable)
                inarg.flags |= FUSE_SETUPMAPPING_FLAG_WRITE;
        args.opcode = FUSE_SETUPMAPPING;
        args.nodeid = fi->nodeid;
        args.in_numargs = 1;
        args.in_args[0].size = sizeof(inarg);
        args.in_args[0].value = &inarg;
        err = fuse_simple_request(fm, &args);
        if (err < 0)
                return err;
        dmap->writable = writable;
        if (!upgrade) {
                /*
                 * We don't take a reference on inode. inode is valid right now
                 * and when inode is going away, cleanup logic should first
                 * cleanup dmap entries.
                 */
                dmap->inode = inode;
                dmap->itn.start = dmap->itn.last = start_idx;
                /* Protected by fi->dax->sem */
                interval_tree_insert(&dmap->itn, &fi->dax->tree);
                fi->dax->nr++;
                spin_lock(&fcd->lock);
                list_add_tail(&dmap->busy_list, &fcd->busy_ranges);
                fcd->nr_busy_ranges++;
                spin_unlock(&fcd->lock);
        }
        return 0;
}

static int fuse_send_removemapping(struct inode *inode,
                                   struct fuse_removemapping_in *inargp,
                                   struct fuse_removemapping_one *remove_one)
{
        struct fuse_inode *fi = get_fuse_inode(inode);
        struct fuse_mount *fm = get_fuse_mount(inode);
        FUSE_ARGS(args);

        args.opcode = FUSE_REMOVEMAPPING;
        args.nodeid = fi->nodeid;
        args.in_numargs = 2;
        args.in_args[0].size = sizeof(*inargp);
        args.in_args[0].value = inargp;
        args.in_args[1].size = inargp->count * sizeof(*remove_one);
        args.in_args[1].value = remove_one;
        return fuse_simple_request(fm, &args);
}

static int dmap_removemapping_list(struct inode *inode, unsigned int num,
                                   struct list_head *to_remove)
{
        struct fuse_removemapping_one *remove_one, *ptr;
        struct fuse_removemapping_in inarg;
        struct fuse_dax_mapping *dmap;
        int ret, i = 0, nr_alloc;

        nr_alloc = min_t(unsigned int, num, FUSE_REMOVEMAPPING_MAX_ENTRY);
        remove_one = kmalloc_array(nr_alloc, sizeof(*remove_one), GFP_NOFS);
        if (!remove_one)
                return -ENOMEM;

        ptr = remove_one;
        list_for_each_entry(dmap, to_remove, list) {
                ptr->moffset = dmap->window_offset;
                ptr->len = dmap->length;
                ptr++;
                i++;
                num--;
                if (i >= nr_alloc || num == 0) {
                        memset(&inarg, 0, sizeof(inarg));
                        inarg.count = i;
                        ret = fuse_send_removemapping(inode, &inarg,
                                                      remove_one);
                        if (ret)
                                goto out;
                        ptr = remove_one;
                        i = 0;
                }
        }
out:
        kfree(remove_one);
        return ret;
}

/*
 * Cleanup dmap entry and add back to free list. This should be called with
 * fcd->lock held.
 */
static void dmap_reinit_add_to_free_pool(struct fuse_conn_dax *fcd,
                                            struct fuse_dax_mapping *dmap)
{
        pr_debug("fuse: freeing memory range start_idx=0x%lx end_idx=0x%lx window_offset=0x%llx length=0x%llx\n",
                 dmap->itn.start, dmap->itn.last, dmap->window_offset,
                 dmap->length);
        __dmap_remove_busy_list(fcd, dmap);
        dmap->inode = NULL;
        dmap->itn.start = dmap->itn.last = 0;
        __dmap_add_to_free_pool(fcd, dmap);
}

/*
 * Free inode dmap entries whose range falls inside [start, end].
 * Does not take any locks. At this point of time it should only be
 * called from evict_inode() path where we know all dmap entries can be
 * reclaimed.
 */
static void inode_reclaim_dmap_range(struct fuse_conn_dax *fcd,
                                     struct inode *inode,
                                     loff_t start, loff_t end)
{
        struct fuse_inode *fi = get_fuse_inode(inode);
        struct fuse_dax_mapping *dmap, *n;
        int err, num = 0;
        LIST_HEAD(to_remove);
        unsigned long start_idx = start >> FUSE_DAX_SHIFT;
        unsigned long end_idx = end >> FUSE_DAX_SHIFT;
        struct interval_tree_node *node;

        while (1) {
                node = interval_tree_iter_first(&fi->dax->tree, start_idx,
                                                end_idx);
                if (!node)
                        break;
                dmap = node_to_dmap(node);
                /* inode is going away. There should not be any users of dmap */
                WARN_ON(refcount_read(&dmap->refcnt) > 1);
                interval_tree_remove(&dmap->itn, &fi->dax->tree);
                num++;
                list_add(&dmap->list, &to_remove);
        }

        /* Nothing to remove */
        if (list_empty(&to_remove))
                return;

        WARN_ON(fi->dax->nr < num);
        fi->dax->nr -= num;
        err = dmap_removemapping_list(inode, num, &to_remove);
        if (err && err != -ENOTCONN) {
                pr_warn("Failed to removemappings. start=0x%llx end=0x%llx\n",
                        start, end);
        }
        spin_lock(&fcd->lock);
        list_for_each_entry_safe(dmap, n, &to_remove, list) {
                list_del_init(&dmap->list);
                dmap_reinit_add_to_free_pool(fcd, dmap);
        }
        spin_unlock(&fcd->lock);
}

static int dmap_removemapping_one(struct inode *inode,
                                  struct fuse_dax_mapping *dmap)
{
        struct fuse_removemapping_one forget_one;
        struct fuse_removemapping_in inarg;

        memset(&inarg, 0, sizeof(inarg));
        inarg.count = 1;
        memset(&forget_one, 0, sizeof(forget_one));
        forget_one.moffset = dmap->window_offset;
        forget_one.len = dmap->length;

        return fuse_send_removemapping(inode, &inarg, &forget_one);
}

/*
 * It is called from evict_inode() and by that time inode is going away. So
 * this function does not take any locks like fi->dax->sem for traversing
 * that fuse inode interval tree. If that lock is taken then lock validator
 * complains of deadlock situation w.r.t fs_reclaim lock.
 */
void fuse_dax_inode_cleanup(struct inode *inode)
{
        struct fuse_conn *fc = get_fuse_conn(inode);
        struct fuse_inode *fi = get_fuse_inode(inode);

        /*
         * fuse_evict_inode() has already called truncate_inode_pages_final()
         * before we arrive here. So we should not have to worry about any
         * pages/exception entries still associated with inode.
         */
        inode_reclaim_dmap_range(fc->dax, inode, 0, -1);
        WARN_ON(fi->dax->nr);
}

static void fuse_fill_iomap_hole(struct iomap *iomap, loff_t length)
{
        iomap->addr = IOMAP_NULL_ADDR;
        iomap->length = length;
        iomap->type = IOMAP_HOLE;
}

static void fuse_fill_iomap(struct inode *inode, loff_t pos, loff_t length,
                            struct iomap *iomap, struct fuse_dax_mapping *dmap,
                            unsigned int flags)
{
        loff_t offset, len;
        loff_t i_size = i_size_read(inode);

        offset = pos - (dmap->itn.start << FUSE_DAX_SHIFT);
        len = min(length, dmap->length - offset);

        /* If length is beyond end of file, truncate further */
        if (pos + len > i_size)
                len = i_size - pos;

        if (len > 0) {
                iomap->addr = dmap->window_offset + offset;
                iomap->length = len;
                if (flags & IOMAP_FAULT)
                        iomap->length = ALIGN(len, PAGE_SIZE);
                iomap->type = IOMAP_MAPPED;
                /*
                 * increace refcnt so that reclaim code knows this dmap is in
                 * use. This assumes fi->dax->sem mutex is held either
                 * shared/exclusive.
                 */
                refcount_inc(&dmap->refcnt);

                /* iomap->private should be NULL */
                WARN_ON_ONCE(iomap->private);
                iomap->private = dmap;
        } else {
                /* Mapping beyond end of file is hole */
                fuse_fill_iomap_hole(iomap, length);
        }
}

static int fuse_setup_new_dax_mapping(struct inode *inode, loff_t pos,
                                      loff_t length, unsigned int flags,
                                      struct iomap *iomap)
{
        struct fuse_inode *fi = get_fuse_inode(inode);
        struct fuse_conn *fc = get_fuse_conn(inode);
        struct fuse_conn_dax *fcd = fc->dax;
        struct fuse_dax_mapping *dmap, *alloc_dmap = NULL;
        int ret;
        bool writable = flags & IOMAP_WRITE;
        unsigned long start_idx = pos >> FUSE_DAX_SHIFT;
        struct interval_tree_node *node;

        /*
         * Can't do inline reclaim in fault path. We call
         * dax_layout_busy_page() before we free a range. And
         * fuse_wait_dax_page() drops mapping->invalidate_lock and requires it.
         * In fault path we enter with mapping->invalidate_lock held and can't
         * drop it. Also in fault path we hold mapping->invalidate_lock shared
         * and not exclusive, so that creates further issues with
         * fuse_wait_dax_page().  Hence return -EAGAIN and fuse_dax_fault()
         * will wait for a memory range to become free and retry.
         */
        if (flags & IOMAP_FAULT) {
                alloc_dmap = alloc_dax_mapping(fcd);
                if (!alloc_dmap)
                        return -EAGAIN;
        } else {
                alloc_dmap = alloc_dax_mapping_reclaim(fcd, inode);
                if (IS_ERR(alloc_dmap))
                        return PTR_ERR(alloc_dmap);
        }

        /* If we are here, we should have memory allocated */
        if (WARN_ON(!alloc_dmap))
                return -EIO;

        /*
         * Take write lock so that only one caller can try to setup mapping
         * and other waits.
         */
        down_write(&fi->dax->sem);
        /*
         * We dropped lock. Check again if somebody else setup
         * mapping already.
         */
        node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
        if (node) {
                dmap = node_to_dmap(node);
                fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
                dmap_add_to_free_pool(fcd, alloc_dmap);
                up_write(&fi->dax->sem);
                return 0;
        }

        /* Setup one mapping */
        ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, alloc_dmap,
                                     writable, false);
        if (ret < 0) {
                dmap_add_to_free_pool(fcd, alloc_dmap);
                up_write(&fi->dax->sem);
                return ret;
        }
        fuse_fill_iomap(inode, pos, length, iomap, alloc_dmap, flags);
        up_write(&fi->dax->sem);
        return 0;
}

static int fuse_upgrade_dax_mapping(struct inode *inode, loff_t pos,
                                    loff_t length, unsigned int flags,
                                    struct iomap *iomap)
{
        struct fuse_inode *fi = get_fuse_inode(inode);
        struct fuse_dax_mapping *dmap;
        int ret;
        unsigned long idx = pos >> FUSE_DAX_SHIFT;
        struct interval_tree_node *node;

        /*
         * Take exclusive lock so that only one caller can try to setup
         * mapping and others wait.
         */
        down_write(&fi->dax->sem);
        node = interval_tree_iter_first(&fi->dax->tree, idx, idx);

        /* We are holding either inode lock or invalidate_lock, and that should
         * ensure that dmap can't be truncated. We are holding a reference
         * on dmap and that should make sure it can't be reclaimed. So dmap
         * should still be there in tree despite the fact we dropped and
         * re-acquired the fi->dax->sem lock.
         */
        ret = -EIO;
        if (WARN_ON(!node))
                goto out_err;

        dmap = node_to_dmap(node);

        /* We took an extra reference on dmap to make sure its not reclaimd.
         * Now we hold fi->dax->sem lock and that reference is not needed
         * anymore. Drop it.
         */
        if (refcount_dec_and_test(&dmap->refcnt)) {
                /* refcount should not hit 0. This object only goes
                 * away when fuse connection goes away
                 */
                WARN_ON_ONCE(1);
        }

        /* Maybe another thread already upgraded mapping while we were not
         * holding lock.
         */
        if (dmap->writable) {
                ret = 0;
                goto out_fill_iomap;
        }

        ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, dmap, true,
                                     true);
        if (ret < 0)
                goto out_err;
out_fill_iomap:
        fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
out_err:
        up_write(&fi->dax->sem);
        return ret;
}

/* This is just for DAX and the mapping is ephemeral, do not use it for other
 * purposes since there is no block device with a permanent mapping.
 */
static int fuse_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
                            unsigned int flags, struct iomap *iomap,
                            struct iomap *srcmap)
{
        struct fuse_inode *fi = get_fuse_inode(inode);
        struct fuse_conn *fc = get_fuse_conn(inode);
        struct fuse_dax_mapping *dmap;
        bool writable = flags & IOMAP_WRITE;
        unsigned long start_idx = pos >> FUSE_DAX_SHIFT;
        struct interval_tree_node *node;

        /* We don't support FIEMAP */
        if (WARN_ON(flags & IOMAP_REPORT))
                return -EIO;

        iomap->offset = pos;
        iomap->flags = 0;
        iomap->bdev = NULL;
        iomap->dax_dev = fc->dax->dev;

        /*
         * Both read/write and mmap path can race here. So we need something
         * to make sure if we are setting up mapping, then other path waits
         *
         * For now, use a semaphore for this. It probably needs to be
         * optimized later.
         */
        down_read(&fi->dax->sem);
        node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
        if (node) {
                dmap = node_to_dmap(node);
                if (writable && !dmap->writable) {
                        /* Upgrade read-only mapping to read-write. This will
                         * require exclusive fi->dax->sem lock as we don't want
                         * two threads to be trying to this simultaneously
                         * for same dmap. So drop shared lock and acquire
                         * exclusive lock.
                         *
                         * Before dropping fi->dax->sem lock, take reference
                         * on dmap so that its not freed by range reclaim.
                         */
                        refcount_inc(&dmap->refcnt);
                        up_read(&fi->dax->sem);
                        pr_debug("%s: Upgrading mapping at offset 0x%llx length 0x%llx\n",
                                 __func__, pos, length);
                        return fuse_upgrade_dax_mapping(inode, pos, length,
                                                        flags, iomap);
                } else {
                        fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
                        up_read(&fi->dax->sem);
                        return 0;
                }
        } else {
                up_read(&fi->dax->sem);
                pr_debug("%s: no mapping at offset 0x%llx length 0x%llx\n",
                                __func__, pos, length);
                if (pos >= i_size_read(inode))
                        goto iomap_hole;

                return fuse_setup_new_dax_mapping(inode, pos, length, flags,
                                                  iomap);
        }

        /*
         * If read beyond end of file happens, fs code seems to return
         * it as hole
         */
iomap_hole:
        fuse_fill_iomap_hole(iomap, length);
        pr_debug("%s returning hole mapping. pos=0x%llx length_asked=0x%llx length_returned=0x%llx\n",
                 __func__, pos, length, iomap->length);
        return 0;
}

static int fuse_iomap_end(struct inode *inode, loff_t pos, loff_t length,
                          ssize_t written, unsigned int flags,
                          struct iomap *iomap)
{
        struct fuse_dax_mapping *dmap = iomap->private;

        if (dmap) {
                if (refcount_dec_and_test(&dmap->refcnt)) {
                        /* refcount should not hit 0. This object only goes
                         * away when fuse connection goes away
                         */
                        WARN_ON_ONCE(1);
                }
        }

        /* DAX writes beyond end-of-file aren't handled using iomap, so the
         * file size is unchanged and there is nothing to do here.
         */
        return 0;
}

static const struct iomap_ops fuse_iomap_ops = {
        .iomap_begin = fuse_iomap_begin,
        .iomap_end = fuse_iomap_end,
};

static void fuse_wait_dax_page(struct inode *inode)
{
        filemap_invalidate_unlock(inode->i_mapping);
        schedule();
        filemap_invalidate_lock(inode->i_mapping);
}

/* Should be called with mapping->invalidate_lock held exclusively */
static int __fuse_dax_break_layouts(struct inode *inode, bool *retry,
                                    loff_t start, loff_t end)
{
        struct page *page;

        page = dax_layout_busy_page_range(inode->i_mapping, start, end);
        if (!page)
                return 0;

        *retry = true;
        return ___wait_var_event(&page->_refcount,
                        atomic_read(&page->_refcount) == 1, TASK_INTERRUPTIBLE,
                        0, 0, fuse_wait_dax_page(inode));
}

/* dmap_end == 0 leads to unmapping of whole file */
int fuse_dax_break_layouts(struct inode *inode, u64 dmap_start,
                                  u64 dmap_end)
{
        bool    retry;
        int     ret;

        do {
                retry = false;
                ret = __fuse_dax_break_layouts(inode, &retry, dmap_start,
                                               dmap_end);
        } while (ret == 0 && retry);

        return ret;
}

ssize_t fuse_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)
{
        struct inode *inode = file_inode(iocb->ki_filp);
        ssize_t ret;

        if (iocb->ki_flags & IOCB_NOWAIT) {
                if (!inode_trylock_shared(inode))
                        return -EAGAIN;
        } else {
                inode_lock_shared(inode);
        }

        ret = dax_iomap_rw(iocb, to, &fuse_iomap_ops);
        inode_unlock_shared(inode);

        /* TODO file_accessed(iocb->f_filp) */
        return ret;
}

static bool file_extending_write(struct kiocb *iocb, struct iov_iter *from)
{
        struct inode *inode = file_inode(iocb->ki_filp);

        return (iov_iter_rw(from) == WRITE &&
                ((iocb->ki_pos) >= i_size_read(inode) ||
                  (iocb->ki_pos + iov_iter_count(from) > i_size_read(inode))));
}

static ssize_t fuse_dax_direct_write(struct kiocb *iocb, struct iov_iter *from)
{
        struct inode *inode = file_inode(iocb->ki_filp);
        struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
        ssize_t ret;

        ret = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE);

        fuse_write_update_attr(inode, iocb->ki_pos, ret);
        return ret;
}

ssize_t fuse_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)
{
        struct inode *inode = file_inode(iocb->ki_filp);
        ssize_t ret;

        if (iocb->ki_flags & IOCB_NOWAIT) {
                if (!inode_trylock(inode))
                        return -EAGAIN;
        } else {
                inode_lock(inode);
        }

        ret = generic_write_checks(iocb, from);
        if (ret <= 0)
                goto out;

        ret = file_remove_privs(iocb->ki_filp);
        if (ret)
                goto out;
        /* TODO file_update_time() but we don't want metadata I/O */

        /* Do not use dax for file extending writes as write and on
         * disk i_size increase are not atomic otherwise.
         */
        if (file_extending_write(iocb, from))
                ret = fuse_dax_direct_write(iocb, from);
        else
                ret = dax_iomap_rw(iocb, from, &fuse_iomap_ops);

out:
        inode_unlock(inode);

        if (ret > 0)
                ret = generic_write_sync(iocb, ret);
        return ret;
}

static int fuse_dax_writepages(struct address_space *mapping,
                               struct writeback_control *wbc)
{

        struct inode *inode = mapping->host;
        struct fuse_conn *fc = get_fuse_conn(inode);

        return dax_writeback_mapping_range(mapping, fc->dax->dev, wbc);
}

static vm_fault_t __fuse_dax_fault(struct vm_fault *vmf, unsigned int order,
                bool write)
{
        vm_fault_t ret;
        struct inode *inode = file_inode(vmf->vma->vm_file);
        struct super_block *sb = inode->i_sb;
        pfn_t pfn;
        int error = 0;
        struct fuse_conn *fc = get_fuse_conn(inode);
        struct fuse_conn_dax *fcd = fc->dax;
        bool retry = false;

        if (write)
                sb_start_pagefault(sb);
retry:
        if (retry && !(fcd->nr_free_ranges > 0))
                wait_event(fcd->range_waitq, (fcd->nr_free_ranges > 0));

        /*
         * We need to serialize against not only truncate but also against
         * fuse dax memory range reclaim. While a range is being reclaimed,
         * we do not want any read/write/mmap to make progress and try
         * to populate page cache or access memory we are trying to free.
         */
        filemap_invalidate_lock_shared(inode->i_mapping);
        ret = dax_iomap_fault(vmf, order, &pfn, &error, &fuse_iomap_ops);
        if ((ret & VM_FAULT_ERROR) && error == -EAGAIN) {
                error = 0;
                retry = true;
                filemap_invalidate_unlock_shared(inode->i_mapping);
                goto retry;
        }

        if (ret & VM_FAULT_NEEDDSYNC)
                ret = dax_finish_sync_fault(vmf, order, pfn);
        filemap_invalidate_unlock_shared(inode->i_mapping);

        if (write)
                sb_end_pagefault(sb);

        return ret;
}

static vm_fault_t fuse_dax_fault(struct vm_fault *vmf)
{
        return __fuse_dax_fault(vmf, 0, vmf->flags & FAULT_FLAG_WRITE);
}

static vm_fault_t fuse_dax_huge_fault(struct vm_fault *vmf, unsigned int order)
{
        return __fuse_dax_fault(vmf, order, vmf->flags & FAULT_FLAG_WRITE);
}

static vm_fault_t fuse_dax_page_mkwrite(struct vm_fault *vmf)
{
        return __fuse_dax_fault(vmf, 0, true);
}

static vm_fault_t fuse_dax_pfn_mkwrite(struct vm_fault *vmf)
{
        return __fuse_dax_fault(vmf, 0, true);
}

static const struct vm_operations_struct fuse_dax_vm_ops = {
        .fault          = fuse_dax_fault,
        .huge_fault     = fuse_dax_huge_fault,
        .page_mkwrite   = fuse_dax_page_mkwrite,
        .pfn_mkwrite    = fuse_dax_pfn_mkwrite,
};

int fuse_dax_mmap(struct file *file, struct vm_area_struct *vma)
{
        file_accessed(file);
        vma->vm_ops = &fuse_dax_vm_ops;
        vm_flags_set(vma, VM_MIXEDMAP | VM_HUGEPAGE);
        return 0;
}

static int dmap_writeback_invalidate(struct inode *inode,
                                     struct fuse_dax_mapping *dmap)
{
        int ret;
        loff_t start_pos = dmap->itn.start << FUSE_DAX_SHIFT;
        loff_t end_pos = (start_pos + FUSE_DAX_SZ - 1);

        ret = filemap_fdatawrite_range(inode->i_mapping, start_pos, end_pos);
        if (ret) {
                pr_debug("fuse: filemap_fdatawrite_range() failed. err=%d start_pos=0x%llx, end_pos=0x%llx\n",
                         ret, start_pos, end_pos);
                return ret;
        }

        ret = invalidate_inode_pages2_range(inode->i_mapping,
                                            start_pos >> PAGE_SHIFT,
                                            end_pos >> PAGE_SHIFT);
        if (ret)
                pr_debug("fuse: invalidate_inode_pages2_range() failed err=%d\n",
                         ret);

        return ret;
}

static int reclaim_one_dmap_locked(struct inode *inode,
                                   struct fuse_dax_mapping *dmap)
{
        int ret;
        struct fuse_inode *fi = get_fuse_inode(inode);

        /*
         * igrab() was done to make sure inode won't go under us, and this
         * further avoids the race with evict().
         */
        ret = dmap_writeback_invalidate(inode, dmap);
        if (ret)
                return ret;

        /* Remove dax mapping from inode interval tree now */
        interval_tree_remove(&dmap->itn, &fi->dax->tree);
        fi->dax->nr--;

        /* It is possible that umount/shutdown has killed the fuse connection
         * and worker thread is trying to reclaim memory in parallel.  Don't
         * warn in that case.
         */
        ret = dmap_removemapping_one(inode, dmap);
        if (ret && ret != -ENOTCONN) {
                pr_warn("Failed to remove mapping. offset=0x%llx len=0x%llx ret=%d\n",
                        dmap->window_offset, dmap->length, ret);
        }
        return 0;
}

/* Find first mapped dmap for an inode and return file offset. Caller needs
 * to hold fi->dax->sem lock either shared or exclusive.
 */
static struct fuse_dax_mapping *inode_lookup_first_dmap(struct inode *inode)
{
        struct fuse_inode *fi = get_fuse_inode(inode);
        struct fuse_dax_mapping *dmap;
        struct interval_tree_node *node;

        for (node = interval_tree_iter_first(&fi->dax->tree, 0, -1); node;
             node = interval_tree_iter_next(node, 0, -1)) {
                dmap = node_to_dmap(node);
                /* still in use. */
                if (refcount_read(&dmap->refcnt) > 1)
                        continue;

                return dmap;
        }

        return NULL;
}

/*
 * Find first mapping in the tree and free it and return it. Do not add
 * it back to free pool.
 */
static struct fuse_dax_mapping *
inode_inline_reclaim_one_dmap(struct fuse_conn_dax *fcd, struct inode *inode,
                              bool *retry)
{
        struct fuse_inode *fi = get_fuse_inode(inode);
        struct fuse_dax_mapping *dmap;
        u64 dmap_start, dmap_end;
        unsigned long start_idx;
        int ret;
        struct interval_tree_node *node;

        filemap_invalidate_lock(inode->i_mapping);

        /* Lookup a dmap and corresponding file offset to reclaim. */
        down_read(&fi->dax->sem);
        dmap = inode_lookup_first_dmap(inode);
        if (dmap) {
                start_idx = dmap->itn.start;
                dmap_start = start_idx << FUSE_DAX_SHIFT;
                dmap_end = dmap_start + FUSE_DAX_SZ - 1;
        }
        up_read(&fi->dax->sem);

        if (!dmap)
                goto out_mmap_sem;
        /*
         * Make sure there are no references to inode pages using
         * get_user_pages()
         */
        ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end);
        if (ret) {
                pr_debug("fuse: fuse_dax_break_layouts() failed. err=%d\n",
                         ret);
                dmap = ERR_PTR(ret);
                goto out_mmap_sem;
        }

        down_write(&fi->dax->sem);
        node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
        /* Range already got reclaimed by somebody else */
        if (!node) {
                if (retry)
                        *retry = true;
                goto out_write_dmap_sem;
        }

        dmap = node_to_dmap(node);
        /* still in use. */
        if (refcount_read(&dmap->refcnt) > 1) {
                dmap = NULL;
                if (retry)
                        *retry = true;
                goto out_write_dmap_sem;
        }

        ret = reclaim_one_dmap_locked(inode, dmap);
        if (ret < 0) {
                dmap = ERR_PTR(ret);
                goto out_write_dmap_sem;
        }

        /* Clean up dmap. Do not add back to free list */
        dmap_remove_busy_list(fcd, dmap);
        dmap->inode = NULL;
        dmap->itn.start = dmap->itn.last = 0;

        pr_debug("fuse: %s: inline reclaimed memory range. inode=%p, window_offset=0x%llx, length=0x%llx\n",
                 __func__, inode, dmap->window_offset, dmap->length);

out_write_dmap_sem:
        up_write(&fi->dax->sem);
out_mmap_sem:
        filemap_invalidate_unlock(inode->i_mapping);
        return dmap;
}

static struct fuse_dax_mapping *
alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode)
{
        struct fuse_dax_mapping *dmap;
        struct fuse_inode *fi = get_fuse_inode(inode);

        while (1) {
                bool retry = false;

                dmap = alloc_dax_mapping(fcd);
                if (dmap)
                        return dmap;

                dmap = inode_inline_reclaim_one_dmap(fcd, inode, &retry);
                /*
                 * Either we got a mapping or it is an error, return in both
                 * the cases.
                 */
                if (dmap)
                        return dmap;

                /* If we could not reclaim a mapping because it
                 * had a reference or some other temporary failure,
                 * Try again. We want to give up inline reclaim only
                 * if there is no range assigned to this node. Otherwise
                 * if a deadlock is possible if we sleep with
                 * mapping->invalidate_lock held and worker to free memory
                 * can't make progress due to unavailability of
                 * mapping->invalidate_lock.  So sleep only if fi->dax->nr=0
                 */
                if (retry)
                        continue;
                /*
                 * There are no mappings which can be reclaimed. Wait for one.
                 * We are not holding fi->dax->sem. So it is possible
                 * that range gets added now. But as we are not holding
                 * mapping->invalidate_lock, worker should still be able to
                 * free up a range and wake us up.
                 */
                if (!fi->dax->nr && !(fcd->nr_free_ranges > 0)) {
                        if (wait_event_killable_exclusive(fcd->range_waitq,
                                        (fcd->nr_free_ranges > 0))) {
                                return ERR_PTR(-EINTR);
                        }
                }
        }
}

static int lookup_and_reclaim_dmap_locked(struct fuse_conn_dax *fcd,
                                          struct inode *inode,
                                          unsigned long start_idx)
{
        int ret;
        struct fuse_inode *fi = get_fuse_inode(inode);
        struct fuse_dax_mapping *dmap;
        struct interval_tree_node *node;

        /* Find fuse dax mapping at file offset inode. */
        node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);

        /* Range already got cleaned up by somebody else */
        if (!node)
                return 0;
        dmap = node_to_dmap(node);

        /* still in use. */
        if (refcount_read(&dmap->refcnt) > 1)
                return 0;

        ret = reclaim_one_dmap_locked(inode, dmap);
        if (ret < 0)
                return ret;

        /* Cleanup dmap entry and add back to free list */
        spin_lock(&fcd->lock);
        dmap_reinit_add_to_free_pool(fcd, dmap);
        spin_unlock(&fcd->lock);
        return ret;
}

/*
 * Free a range of memory.
 * Locking:
 * 1. Take mapping->invalidate_lock to block dax faults.
 * 2. Take fi->dax->sem to protect interval tree and also to make sure
 *    read/write can not reuse a dmap which we might be freeing.
 */
static int lookup_and_reclaim_dmap(struct fuse_conn_dax *fcd,
                                   struct inode *inode,
                                   unsigned long start_idx,
                                   unsigned long end_idx)
{
        int ret;
        struct fuse_inode *fi = get_fuse_inode(inode);
        loff_t dmap_start = start_idx << FUSE_DAX_SHIFT;
        loff_t dmap_end = (dmap_start + FUSE_DAX_SZ) - 1;

        filemap_invalidate_lock(inode->i_mapping);
        ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end);
        if (ret) {
                pr_debug("virtio_fs: fuse_dax_break_layouts() failed. err=%d\n",
                         ret);
                goto out_mmap_sem;
        }

        down_write(&fi->dax->sem);
        ret = lookup_and_reclaim_dmap_locked(fcd, inode, start_idx);
        up_write(&fi->dax->sem);
out_mmap_sem:
        filemap_invalidate_unlock(inode->i_mapping);
        return ret;
}

static int try_to_free_dmap_chunks(struct fuse_conn_dax *fcd,
                                   unsigned long nr_to_free)
{
        struct fuse_dax_mapping *dmap, *pos, *temp;
        int ret, nr_freed = 0;
        unsigned long start_idx = 0, end_idx = 0;
        struct inode *inode = NULL;

        /* Pick first busy range and free it for now*/
        while (1) {
                if (nr_freed >= nr_to_free)
                        break;

                dmap = NULL;
                spin_lock(&fcd->lock);

                if (!fcd->nr_busy_ranges) {
                        spin_unlock(&fcd->lock);
                        return 0;
                }

                list_for_each_entry_safe(pos, temp, &fcd->busy_ranges,
                                                busy_list) {
                        /* skip this range if it's in use. */
                        if (refcount_read(&pos->refcnt) > 1)
                                continue;

                        inode = igrab(pos->inode);
                        /*
                         * This inode is going away. That will free
                         * up all the ranges anyway, continue to
                         * next range.
                         */
                        if (!inode)
                                continue;
                        /*
                         * Take this element off list and add it tail. If
                         * this element can't be freed, it will help with
                         * selecting new element in next iteration of loop.
                         */
                        dmap = pos;
                        list_move_tail(&dmap->busy_list, &fcd->busy_ranges);
                        start_idx = end_idx = dmap->itn.start;
                        break;
                }
                spin_unlock(&fcd->lock);
                if (!dmap)
                        return 0;

                ret = lookup_and_reclaim_dmap(fcd, inode, start_idx, end_idx);
                iput(inode);
                if (ret)
                        return ret;
                nr_freed++;
        }
        return 0;
}

static void fuse_dax_free_mem_worker(struct work_struct *work)
{
        int ret;
        struct fuse_conn_dax *fcd = container_of(work, struct fuse_conn_dax,
                                                 free_work.work);
        ret = try_to_free_dmap_chunks(fcd, FUSE_DAX_RECLAIM_CHUNK);
        if (ret) {
                pr_debug("fuse: try_to_free_dmap_chunks() failed with err=%d\n",
                         ret);
        }

        /* If number of free ranges are still below threshold, requeue */
        kick_dmap_free_worker(fcd, 1);
}

static void fuse_free_dax_mem_ranges(struct list_head *mem_list)
{
        struct fuse_dax_mapping *range, *temp;

        /* Free All allocated elements */
        list_for_each_entry_safe(range, temp, mem_list, list) {
                list_del(&range->list);
                if (!list_empty(&range->busy_list))
                        list_del(&range->busy_list);
                kfree(range);
        }
}

void fuse_dax_conn_free(struct fuse_conn *fc)
{
        if (fc->dax) {
                fuse_free_dax_mem_ranges(&fc->dax->free_ranges);
                kfree(fc->dax);
                fc->dax = NULL;
        }
}

static int fuse_dax_mem_range_init(struct fuse_conn_dax *fcd)
{
        long nr_pages, nr_ranges;
        struct fuse_dax_mapping *range;
        int ret, id;
        size_t dax_size = -1;
        unsigned long i;

        init_waitqueue_head(&fcd->range_waitq);
        INIT_LIST_HEAD(&fcd->free_ranges);
        INIT_LIST_HEAD(&fcd->busy_ranges);
        INIT_DELAYED_WORK(&fcd->free_work, fuse_dax_free_mem_worker);

        id = dax_read_lock();
        nr_pages = dax_direct_access(fcd->dev, 0, PHYS_PFN(dax_size),
                        DAX_ACCESS, NULL, NULL);
        dax_read_unlock(id);
        if (nr_pages < 0) {
                pr_debug("dax_direct_access() returned %ld\n", nr_pages);
                return nr_pages;
        }

        nr_ranges = nr_pages/FUSE_DAX_PAGES;
        pr_debug("%s: dax mapped %ld pages. nr_ranges=%ld\n",
                __func__, nr_pages, nr_ranges);

        for (i = 0; i < nr_ranges; i++) {
                range = kzalloc(sizeof(struct fuse_dax_mapping), GFP_KERNEL);
                ret = -ENOMEM;
                if (!range)
                        goto out_err;

                /* TODO: This offset only works if virtio-fs driver is not
                 * having some memory hidden at the beginning. This needs
                 * better handling
                 */
                range->window_offset = i * FUSE_DAX_SZ;
                range->length = FUSE_DAX_SZ;
                INIT_LIST_HEAD(&range->busy_list);
                refcount_set(&range->refcnt, 1);
                list_add_tail(&range->list, &fcd->free_ranges);
        }

        fcd->nr_free_ranges = nr_ranges;
        fcd->nr_ranges = nr_ranges;
        return 0;
out_err:
        /* Free All allocated elements */
        fuse_free_dax_mem_ranges(&fcd->free_ranges);
        return ret;
}

int fuse_dax_conn_alloc(struct fuse_conn *fc, enum fuse_dax_mode dax_mode,
                        struct dax_device *dax_dev)
{
        struct fuse_conn_dax *fcd;
        int err;

        fc->dax_mode = dax_mode;

        if (!dax_dev)
                return 0;

        fcd = kzalloc(sizeof(*fcd), GFP_KERNEL);
        if (!fcd)
                return -ENOMEM;

        spin_lock_init(&fcd->lock);
        fcd->dev = dax_dev;
        err = fuse_dax_mem_range_init(fcd);
        if (err) {
                kfree(fcd);
                return err;
        }

        fc->dax = fcd;
        return 0;
}

bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi)
{
        struct fuse_conn *fc = get_fuse_conn_super(sb);

        fi->dax = NULL;
        if (fc->dax) {
                fi->dax = kzalloc(sizeof(*fi->dax), GFP_KERNEL_ACCOUNT);
                if (!fi->dax)
                        return false;

                init_rwsem(&fi->dax->sem);
                fi->dax->tree = RB_ROOT_CACHED;
        }

        return true;
}

static const struct address_space_operations fuse_dax_file_aops  = {
        .writepages     = fuse_dax_writepages,
        .direct_IO      = noop_direct_IO,
        .dirty_folio    = noop_dirty_folio,
};

static bool fuse_should_enable_dax(struct inode *inode, unsigned int flags)
{
        struct fuse_conn *fc = get_fuse_conn(inode);
        enum fuse_dax_mode dax_mode = fc->dax_mode;

        if (dax_mode == FUSE_DAX_NEVER)
                return false;

        /*
         * fc->dax may be NULL in 'inode' mode when filesystem device doesn't
         * support DAX, in which case it will silently fallback to 'never' mode.
         */
        if (!fc->dax)
                return false;

        if (dax_mode == FUSE_DAX_ALWAYS)
                return true;

        /* dax_mode is FUSE_DAX_INODE* */
        return fc->inode_dax && (flags & FUSE_ATTR_DAX);
}

void fuse_dax_inode_init(struct inode *inode, unsigned int flags)
{
        if (!fuse_should_enable_dax(inode, flags))
                return;

        inode->i_flags |= S_DAX;
        inode->i_data.a_ops = &fuse_dax_file_aops;
}

void fuse_dax_dontcache(struct inode *inode, unsigned int flags)
{
        struct fuse_conn *fc = get_fuse_conn(inode);

        if (fuse_is_inode_dax_mode(fc->dax_mode) &&
            ((bool) IS_DAX(inode) != (bool) (flags & FUSE_ATTR_DAX)))
                d_mark_dontcache(inode);
}

bool fuse_dax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment)
{
        if (fc->dax && (map_alignment > FUSE_DAX_SHIFT)) {
                pr_warn("FUSE: map_alignment %u incompatible with dax mem range size %u\n",
                        map_alignment, FUSE_DAX_SZ);
                return false;
        }
        return true;
}

void fuse_dax_cancel_work(struct fuse_conn *fc)
{
        struct fuse_conn_dax *fcd = fc->dax;

        if (fcd)
                cancel_delayed_work_sync(&fcd->free_work);

}
EXPORT_SYMBOL_GPL(fuse_dax_cancel_work);