1 // SPDX-License-Identifier: GPL-2.0-only
5 * Write file data over NFS.
7 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
10 #include <linux/types.h>
11 #include <linux/slab.h>
13 #include <linux/pagemap.h>
14 #include <linux/file.h>
15 #include <linux/writeback.h>
16 #include <linux/swap.h>
17 #include <linux/migrate.h>
19 #include <linux/sunrpc/clnt.h>
20 #include <linux/nfs_fs.h>
21 #include <linux/nfs_mount.h>
22 #include <linux/nfs_page.h>
23 #include <linux/backing-dev.h>
24 #include <linux/export.h>
25 #include <linux/freezer.h>
26 #include <linux/wait.h>
27 #include <linux/iversion.h>
29 #include <linux/uaccess.h>
30 #include <linux/sched/mm.h>
32 #include "delegation.h"
41 #define NFSDBG_FACILITY NFSDBG_PAGECACHE
43 #define MIN_POOL_WRITE (32)
44 #define MIN_POOL_COMMIT (4)
46 struct nfs_io_completion
{
47 void (*complete
)(void *data
);
53 * Local function declarations
55 static void nfs_redirty_request(struct nfs_page
*req
);
56 static const struct rpc_call_ops nfs_commit_ops
;
57 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops
;
58 static const struct nfs_commit_completion_ops nfs_commit_completion_ops
;
59 static const struct nfs_rw_ops nfs_rw_write_ops
;
60 static void nfs_inode_remove_request(struct nfs_page
*req
);
61 static void nfs_clear_request_commit(struct nfs_page
*req
);
62 static void nfs_init_cinfo_from_inode(struct nfs_commit_info
*cinfo
,
64 static struct nfs_page
*
65 nfs_page_search_commits_for_head_request_locked(struct nfs_inode
*nfsi
,
68 static struct kmem_cache
*nfs_wdata_cachep
;
69 static mempool_t
*nfs_wdata_mempool
;
70 static struct kmem_cache
*nfs_cdata_cachep
;
71 static mempool_t
*nfs_commit_mempool
;
73 struct nfs_commit_data
*nfs_commitdata_alloc(void)
75 struct nfs_commit_data
*p
;
77 p
= kmem_cache_zalloc(nfs_cdata_cachep
, nfs_io_gfp_mask());
79 p
= mempool_alloc(nfs_commit_mempool
, GFP_NOWAIT
);
82 memset(p
, 0, sizeof(*p
));
84 INIT_LIST_HEAD(&p
->pages
);
87 EXPORT_SYMBOL_GPL(nfs_commitdata_alloc
);
89 void nfs_commit_free(struct nfs_commit_data
*p
)
91 mempool_free(p
, nfs_commit_mempool
);
93 EXPORT_SYMBOL_GPL(nfs_commit_free
);
95 static struct nfs_pgio_header
*nfs_writehdr_alloc(void)
97 struct nfs_pgio_header
*p
;
99 p
= kmem_cache_zalloc(nfs_wdata_cachep
, nfs_io_gfp_mask());
101 p
= mempool_alloc(nfs_wdata_mempool
, GFP_NOWAIT
);
104 memset(p
, 0, sizeof(*p
));
106 p
->rw_mode
= FMODE_WRITE
;
110 static void nfs_writehdr_free(struct nfs_pgio_header
*hdr
)
112 mempool_free(hdr
, nfs_wdata_mempool
);
115 static struct nfs_io_completion
*nfs_io_completion_alloc(gfp_t gfp_flags
)
117 return kmalloc(sizeof(struct nfs_io_completion
), gfp_flags
);
120 static void nfs_io_completion_init(struct nfs_io_completion
*ioc
,
121 void (*complete
)(void *), void *data
)
123 ioc
->complete
= complete
;
125 kref_init(&ioc
->refcount
);
128 static void nfs_io_completion_release(struct kref
*kref
)
130 struct nfs_io_completion
*ioc
= container_of(kref
,
131 struct nfs_io_completion
, refcount
);
132 ioc
->complete(ioc
->data
);
136 static void nfs_io_completion_get(struct nfs_io_completion
*ioc
)
139 kref_get(&ioc
->refcount
);
142 static void nfs_io_completion_put(struct nfs_io_completion
*ioc
)
145 kref_put(&ioc
->refcount
, nfs_io_completion_release
);
149 nfs_page_set_inode_ref(struct nfs_page
*req
, struct inode
*inode
)
151 if (!test_and_set_bit(PG_INODE_REF
, &req
->wb_flags
)) {
152 kref_get(&req
->wb_kref
);
153 atomic_long_inc(&NFS_I(inode
)->nrequests
);
158 nfs_cancel_remove_inode(struct nfs_page
*req
, struct inode
*inode
)
162 if (!test_bit(PG_REMOVE
, &req
->wb_flags
))
164 ret
= nfs_page_group_lock(req
);
167 if (test_and_clear_bit(PG_REMOVE
, &req
->wb_flags
))
168 nfs_page_set_inode_ref(req
, inode
);
169 nfs_page_group_unlock(req
);
173 static struct nfs_page
*
174 nfs_page_private_request(struct page
*page
)
176 if (!PagePrivate(page
))
178 return (struct nfs_page
*)page_private(page
);
182 * nfs_page_find_head_request_locked - find head request associated with @page
184 * must be called while holding the inode lock.
186 * returns matching head request with reference held, or NULL if not found.
188 static struct nfs_page
*
189 nfs_page_find_private_request(struct page
*page
)
191 struct address_space
*mapping
= page_file_mapping(page
);
192 struct nfs_page
*req
;
194 if (!PagePrivate(page
))
196 spin_lock(&mapping
->private_lock
);
197 req
= nfs_page_private_request(page
);
199 WARN_ON_ONCE(req
->wb_head
!= req
);
200 kref_get(&req
->wb_kref
);
202 spin_unlock(&mapping
->private_lock
);
206 static struct nfs_page
*
207 nfs_page_find_swap_request(struct page
*page
)
209 struct inode
*inode
= page_file_mapping(page
)->host
;
210 struct nfs_inode
*nfsi
= NFS_I(inode
);
211 struct nfs_page
*req
= NULL
;
212 if (!PageSwapCache(page
))
214 mutex_lock(&nfsi
->commit_mutex
);
215 if (PageSwapCache(page
)) {
216 req
= nfs_page_search_commits_for_head_request_locked(nfsi
,
219 WARN_ON_ONCE(req
->wb_head
!= req
);
220 kref_get(&req
->wb_kref
);
223 mutex_unlock(&nfsi
->commit_mutex
);
228 * nfs_page_find_head_request - find head request associated with @page
230 * returns matching head request with reference held, or NULL if not found.
232 static struct nfs_page
*nfs_page_find_head_request(struct page
*page
)
234 struct nfs_page
*req
;
236 req
= nfs_page_find_private_request(page
);
238 req
= nfs_page_find_swap_request(page
);
242 static struct nfs_page
*nfs_find_and_lock_page_request(struct page
*page
)
244 struct inode
*inode
= page_file_mapping(page
)->host
;
245 struct nfs_page
*req
, *head
;
249 req
= nfs_page_find_head_request(page
);
252 head
= nfs_page_group_lock_head(req
);
254 nfs_release_request(req
);
257 ret
= nfs_cancel_remove_inode(head
, inode
);
259 nfs_unlock_and_release_request(head
);
262 /* Ensure that nobody removed the request before we locked it */
263 if (head
== nfs_page_private_request(page
))
265 if (PageSwapCache(page
))
267 nfs_unlock_and_release_request(head
);
272 /* Adjust the file length if we're writing beyond the end */
273 static void nfs_grow_file(struct page
*page
, unsigned int offset
, unsigned int count
)
275 struct inode
*inode
= page_file_mapping(page
)->host
;
279 spin_lock(&inode
->i_lock
);
280 i_size
= i_size_read(inode
);
281 end_index
= (i_size
- 1) >> PAGE_SHIFT
;
282 if (i_size
> 0 && page_index(page
) < end_index
)
284 end
= page_file_offset(page
) + ((loff_t
)offset
+count
);
287 trace_nfs_size_grow(inode
, end
);
288 i_size_write(inode
, end
);
289 NFS_I(inode
)->cache_validity
&= ~NFS_INO_INVALID_SIZE
;
290 nfs_inc_stats(inode
, NFSIOS_EXTENDWRITE
);
292 spin_unlock(&inode
->i_lock
);
293 nfs_fscache_invalidate(inode
, 0);
296 /* A writeback failed: mark the page as bad, and invalidate the page cache */
297 static void nfs_set_pageerror(struct address_space
*mapping
)
299 struct inode
*inode
= mapping
->host
;
301 nfs_zap_mapping(mapping
->host
, mapping
);
302 /* Force file size revalidation */
303 spin_lock(&inode
->i_lock
);
304 nfs_set_cache_invalid(inode
, NFS_INO_REVAL_FORCED
|
305 NFS_INO_INVALID_CHANGE
|
306 NFS_INO_INVALID_SIZE
);
307 spin_unlock(&inode
->i_lock
);
310 static void nfs_mapping_set_error(struct page
*page
, int error
)
312 struct address_space
*mapping
= page_file_mapping(page
);
315 filemap_set_wb_err(mapping
, error
);
317 errseq_set(&mapping
->host
->i_sb
->s_wb_err
,
318 error
== -ENOSPC
? -ENOSPC
: -EIO
);
319 nfs_set_pageerror(mapping
);
323 * nfs_page_group_search_locked
324 * @head - head request of page group
325 * @page_offset - offset into page
327 * Search page group with head @head to find a request that contains the
328 * page offset @page_offset.
330 * Returns a pointer to the first matching nfs request, or NULL if no
333 * Must be called with the page group lock held
335 static struct nfs_page
*
336 nfs_page_group_search_locked(struct nfs_page
*head
, unsigned int page_offset
)
338 struct nfs_page
*req
;
342 if (page_offset
>= req
->wb_pgbase
&&
343 page_offset
< (req
->wb_pgbase
+ req
->wb_bytes
))
346 req
= req
->wb_this_page
;
347 } while (req
!= head
);
353 * nfs_page_group_covers_page
354 * @head - head request of page group
356 * Return true if the page group with head @head covers the whole page,
357 * returns false otherwise
359 static bool nfs_page_group_covers_page(struct nfs_page
*req
)
361 struct nfs_page
*tmp
;
362 unsigned int pos
= 0;
363 unsigned int len
= nfs_page_length(req
->wb_page
);
365 nfs_page_group_lock(req
);
368 tmp
= nfs_page_group_search_locked(req
->wb_head
, pos
);
371 pos
= tmp
->wb_pgbase
+ tmp
->wb_bytes
;
374 nfs_page_group_unlock(req
);
378 /* We can set the PG_uptodate flag if we see that a write request
379 * covers the full page.
381 static void nfs_mark_uptodate(struct nfs_page
*req
)
383 if (PageUptodate(req
->wb_page
))
385 if (!nfs_page_group_covers_page(req
))
387 SetPageUptodate(req
->wb_page
);
390 static int wb_priority(struct writeback_control
*wbc
)
394 if (wbc
->sync_mode
== WB_SYNC_ALL
)
395 ret
= FLUSH_COND_STABLE
;
400 * NFS congestion control
403 int nfs_congestion_kb
;
405 #define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10))
406 #define NFS_CONGESTION_OFF_THRESH \
407 (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))
409 static void nfs_set_page_writeback(struct page
*page
)
411 struct inode
*inode
= page_file_mapping(page
)->host
;
412 struct nfs_server
*nfss
= NFS_SERVER(inode
);
413 int ret
= test_set_page_writeback(page
);
415 WARN_ON_ONCE(ret
!= 0);
417 if (atomic_long_inc_return(&nfss
->writeback
) >
418 NFS_CONGESTION_ON_THRESH
)
419 nfss
->write_congested
= 1;
422 static void nfs_end_page_writeback(struct nfs_page
*req
)
424 struct inode
*inode
= nfs_page_to_inode(req
);
425 struct nfs_server
*nfss
= NFS_SERVER(inode
);
428 is_done
= nfs_page_group_sync_on_bit(req
, PG_WB_END
);
429 nfs_unlock_request(req
);
433 end_page_writeback(req
->wb_page
);
434 if (atomic_long_dec_return(&nfss
->writeback
) < NFS_CONGESTION_OFF_THRESH
)
435 nfss
->write_congested
= 0;
439 * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests
441 * @destroy_list - request list (using wb_this_page) terminated by @old_head
442 * @old_head - the old head of the list
444 * All subrequests must be locked and removed from all lists, so at this point
445 * they are only "active" in this function, and possibly in nfs_wait_on_request
446 * with a reference held by some other context.
449 nfs_destroy_unlinked_subrequests(struct nfs_page
*destroy_list
,
450 struct nfs_page
*old_head
,
453 while (destroy_list
) {
454 struct nfs_page
*subreq
= destroy_list
;
456 destroy_list
= (subreq
->wb_this_page
== old_head
) ?
457 NULL
: subreq
->wb_this_page
;
459 /* Note: lock subreq in order to change subreq->wb_head */
460 nfs_page_set_headlock(subreq
);
461 WARN_ON_ONCE(old_head
!= subreq
->wb_head
);
463 /* make sure old group is not used */
464 subreq
->wb_this_page
= subreq
;
465 subreq
->wb_head
= subreq
;
467 clear_bit(PG_REMOVE
, &subreq
->wb_flags
);
469 /* Note: races with nfs_page_group_destroy() */
470 if (!kref_read(&subreq
->wb_kref
)) {
471 /* Check if we raced with nfs_page_group_destroy() */
472 if (test_and_clear_bit(PG_TEARDOWN
, &subreq
->wb_flags
)) {
473 nfs_page_clear_headlock(subreq
);
474 nfs_free_request(subreq
);
476 nfs_page_clear_headlock(subreq
);
479 nfs_page_clear_headlock(subreq
);
481 nfs_release_request(old_head
);
483 if (test_and_clear_bit(PG_INODE_REF
, &subreq
->wb_flags
)) {
484 nfs_release_request(subreq
);
485 atomic_long_dec(&NFS_I(inode
)->nrequests
);
488 /* subreq is now totally disconnected from page group or any
489 * write / commit lists. last chance to wake any waiters */
490 nfs_unlock_and_release_request(subreq
);
495 * nfs_join_page_group - destroy subrequests of the head req
496 * @head: the page used to lookup the "page group" of nfs_page structures
497 * @inode: Inode to which the request belongs.
499 * This function joins all sub requests to the head request by first
500 * locking all requests in the group, cancelling any pending operations
501 * and finally updating the head request to cover the whole range covered by
502 * the (former) group. All subrequests are removed from any write or commit
503 * lists, unlinked from the group and destroyed.
506 nfs_join_page_group(struct nfs_page
*head
, struct inode
*inode
)
508 struct nfs_page
*subreq
;
509 struct nfs_page
*destroy_list
= NULL
;
510 unsigned int pgbase
, off
, bytes
;
512 pgbase
= head
->wb_pgbase
;
513 bytes
= head
->wb_bytes
;
514 off
= head
->wb_offset
;
515 for (subreq
= head
->wb_this_page
; subreq
!= head
;
516 subreq
= subreq
->wb_this_page
) {
517 /* Subrequests should always form a contiguous range */
518 if (pgbase
> subreq
->wb_pgbase
) {
519 off
-= pgbase
- subreq
->wb_pgbase
;
520 bytes
+= pgbase
- subreq
->wb_pgbase
;
521 pgbase
= subreq
->wb_pgbase
;
523 bytes
= max(subreq
->wb_pgbase
+ subreq
->wb_bytes
527 /* Set the head request's range to cover the former page group */
528 head
->wb_pgbase
= pgbase
;
529 head
->wb_bytes
= bytes
;
530 head
->wb_offset
= off
;
532 /* Now that all requests are locked, make sure they aren't on any list.
533 * Commit list removal accounting is done after locks are dropped */
536 nfs_clear_request_commit(subreq
);
537 subreq
= subreq
->wb_this_page
;
538 } while (subreq
!= head
);
540 /* unlink subrequests from head, destroy them later */
541 if (head
->wb_this_page
!= head
) {
542 /* destroy list will be terminated by head */
543 destroy_list
= head
->wb_this_page
;
544 head
->wb_this_page
= head
;
547 nfs_destroy_unlinked_subrequests(destroy_list
, head
, inode
);
551 * nfs_lock_and_join_requests - join all subreqs to the head req
552 * @page: the page used to lookup the "page group" of nfs_page structures
554 * This function joins all sub requests to the head request by first
555 * locking all requests in the group, cancelling any pending operations
556 * and finally updating the head request to cover the whole range covered by
557 * the (former) group. All subrequests are removed from any write or commit
558 * lists, unlinked from the group and destroyed.
560 * Returns a locked, referenced pointer to the head request - which after
561 * this call is guaranteed to be the only request associated with the page.
562 * Returns NULL if no requests are found for @page, or a ERR_PTR if an
563 * error was encountered.
565 static struct nfs_page
*
566 nfs_lock_and_join_requests(struct page
*page
)
568 struct inode
*inode
= page_file_mapping(page
)->host
;
569 struct nfs_page
*head
;
573 * A reference is taken only on the head request which acts as a
574 * reference to the whole page group - the group will not be destroyed
575 * until the head reference is released.
577 head
= nfs_find_and_lock_page_request(page
);
578 if (IS_ERR_OR_NULL(head
))
581 /* lock each request in the page group */
582 ret
= nfs_page_group_lock_subrequests(head
);
584 nfs_unlock_and_release_request(head
);
588 nfs_join_page_group(head
, inode
);
593 static void nfs_write_error(struct nfs_page
*req
, int error
)
595 trace_nfs_write_error(nfs_page_to_inode(req
), req
, error
);
596 nfs_mapping_set_error(req
->wb_page
, error
);
597 nfs_inode_remove_request(req
);
598 nfs_end_page_writeback(req
);
599 nfs_release_request(req
);
603 * Find an associated nfs write request, and prepare to flush it out
604 * May return an error if the user signalled nfs_wait_on_request().
606 static int nfs_page_async_flush(struct page
*page
,
607 struct writeback_control
*wbc
,
608 struct nfs_pageio_descriptor
*pgio
)
610 struct nfs_page
*req
;
613 req
= nfs_lock_and_join_requests(page
);
620 nfs_set_page_writeback(page
);
621 WARN_ON_ONCE(test_bit(PG_CLEAN
, &req
->wb_flags
));
623 /* If there is a fatal error that covers this write, just exit */
624 ret
= pgio
->pg_error
;
625 if (nfs_error_is_fatal_on_server(ret
))
629 if (!nfs_pageio_add_request(pgio
, req
)) {
630 ret
= pgio
->pg_error
;
632 * Remove the problematic req upon fatal errors on the server
634 if (nfs_error_is_fatal_on_server(ret
))
636 if (wbc
->sync_mode
== WB_SYNC_NONE
)
637 ret
= AOP_WRITEPAGE_ACTIVATE
;
638 redirty_page_for_writepage(wbc
, page
);
639 nfs_redirty_request(req
);
642 nfs_add_stats(page_file_mapping(page
)->host
,
643 NFSIOS_WRITEPAGES
, 1);
647 nfs_write_error(req
, ret
);
651 static int nfs_do_writepage(struct page
*page
, struct writeback_control
*wbc
,
652 struct nfs_pageio_descriptor
*pgio
)
654 nfs_pageio_cond_complete(pgio
, page_index(page
));
655 return nfs_page_async_flush(page
, wbc
, pgio
);
659 * Write an mmapped page to the server.
661 static int nfs_writepage_locked(struct page
*page
,
662 struct writeback_control
*wbc
)
664 struct nfs_pageio_descriptor pgio
;
665 struct inode
*inode
= page_file_mapping(page
)->host
;
668 if (wbc
->sync_mode
== WB_SYNC_NONE
&&
669 NFS_SERVER(inode
)->write_congested
)
670 return AOP_WRITEPAGE_ACTIVATE
;
672 nfs_inc_stats(inode
, NFSIOS_VFSWRITEPAGE
);
673 nfs_pageio_init_write(&pgio
, inode
, 0,
674 false, &nfs_async_write_completion_ops
);
675 err
= nfs_do_writepage(page
, wbc
, &pgio
);
677 nfs_pageio_complete(&pgio
);
681 int nfs_writepage(struct page
*page
, struct writeback_control
*wbc
)
685 ret
= nfs_writepage_locked(page
, wbc
);
686 if (ret
!= AOP_WRITEPAGE_ACTIVATE
)
691 static int nfs_writepages_callback(struct page
*page
, struct writeback_control
*wbc
, void *data
)
695 ret
= nfs_do_writepage(page
, wbc
, data
);
696 if (ret
!= AOP_WRITEPAGE_ACTIVATE
)
701 static void nfs_io_completion_commit(void *inode
)
703 nfs_commit_inode(inode
, 0);
706 int nfs_writepages(struct address_space
*mapping
, struct writeback_control
*wbc
)
708 struct inode
*inode
= mapping
->host
;
709 struct nfs_pageio_descriptor pgio
;
710 struct nfs_io_completion
*ioc
= NULL
;
711 unsigned int mntflags
= NFS_SERVER(inode
)->flags
;
715 if (wbc
->sync_mode
== WB_SYNC_NONE
&&
716 NFS_SERVER(inode
)->write_congested
)
719 nfs_inc_stats(inode
, NFSIOS_VFSWRITEPAGES
);
721 if (!(mntflags
& NFS_MOUNT_WRITE_EAGER
) || wbc
->for_kupdate
||
722 wbc
->for_background
|| wbc
->for_sync
|| wbc
->for_reclaim
) {
723 ioc
= nfs_io_completion_alloc(GFP_KERNEL
);
725 nfs_io_completion_init(ioc
, nfs_io_completion_commit
,
727 priority
= wb_priority(wbc
);
731 nfs_pageio_init_write(&pgio
, inode
, priority
, false,
732 &nfs_async_write_completion_ops
);
733 pgio
.pg_io_completion
= ioc
;
734 err
= write_cache_pages(mapping
, wbc
, nfs_writepages_callback
,
737 nfs_pageio_complete(&pgio
);
738 } while (err
< 0 && !nfs_error_is_fatal(err
));
739 nfs_io_completion_put(ioc
);
749 * Insert a write request into an inode
751 static void nfs_inode_add_request(struct inode
*inode
, struct nfs_page
*req
)
753 struct address_space
*mapping
= page_file_mapping(req
->wb_page
);
754 struct nfs_inode
*nfsi
= NFS_I(inode
);
756 WARN_ON_ONCE(req
->wb_this_page
!= req
);
758 /* Lock the request! */
759 nfs_lock_request(req
);
762 * Swap-space should not get truncated. Hence no need to plug the race
763 * with invalidate/truncate.
765 spin_lock(&mapping
->private_lock
);
766 if (likely(!PageSwapCache(req
->wb_page
))) {
767 set_bit(PG_MAPPED
, &req
->wb_flags
);
768 SetPagePrivate(req
->wb_page
);
769 set_page_private(req
->wb_page
, (unsigned long)req
);
771 spin_unlock(&mapping
->private_lock
);
772 atomic_long_inc(&nfsi
->nrequests
);
773 /* this a head request for a page group - mark it as having an
774 * extra reference so sub groups can follow suit.
775 * This flag also informs pgio layer when to bump nrequests when
776 * adding subrequests. */
777 WARN_ON(test_and_set_bit(PG_INODE_REF
, &req
->wb_flags
));
778 kref_get(&req
->wb_kref
);
782 * Remove a write request from an inode
784 static void nfs_inode_remove_request(struct nfs_page
*req
)
786 struct address_space
*mapping
= page_file_mapping(req
->wb_page
);
787 struct inode
*inode
= mapping
->host
;
788 struct nfs_inode
*nfsi
= NFS_I(inode
);
789 struct nfs_page
*head
;
791 if (nfs_page_group_sync_on_bit(req
, PG_REMOVE
)) {
794 spin_lock(&mapping
->private_lock
);
795 if (likely(head
->wb_page
&& !PageSwapCache(head
->wb_page
))) {
796 set_page_private(head
->wb_page
, 0);
797 ClearPagePrivate(head
->wb_page
);
798 clear_bit(PG_MAPPED
, &head
->wb_flags
);
800 spin_unlock(&mapping
->private_lock
);
803 if (test_and_clear_bit(PG_INODE_REF
, &req
->wb_flags
)) {
804 nfs_release_request(req
);
805 atomic_long_dec(&nfsi
->nrequests
);
810 nfs_mark_request_dirty(struct nfs_page
*req
)
813 __set_page_dirty_nobuffers(req
->wb_page
);
817 * nfs_page_search_commits_for_head_request_locked
819 * Search through commit lists on @inode for the head request for @page.
820 * Must be called while holding the inode (which is cinfo) lock.
822 * Returns the head request if found, or NULL if not found.
824 static struct nfs_page
*
825 nfs_page_search_commits_for_head_request_locked(struct nfs_inode
*nfsi
,
828 struct nfs_page
*freq
, *t
;
829 struct nfs_commit_info cinfo
;
830 struct inode
*inode
= &nfsi
->vfs_inode
;
832 nfs_init_cinfo_from_inode(&cinfo
, inode
);
834 /* search through pnfs commit lists */
835 freq
= pnfs_search_commit_reqs(inode
, &cinfo
, page
);
837 return freq
->wb_head
;
839 /* Linearly search the commit list for the correct request */
840 list_for_each_entry_safe(freq
, t
, &cinfo
.mds
->list
, wb_list
) {
841 if (freq
->wb_page
== page
)
842 return freq
->wb_head
;
849 * nfs_request_add_commit_list_locked - add request to a commit list
850 * @req: pointer to a struct nfs_page
851 * @dst: commit list head
852 * @cinfo: holds list lock and accounting info
854 * This sets the PG_CLEAN bit, updates the cinfo count of
855 * number of outstanding requests requiring a commit as well as
858 * The caller must hold NFS_I(cinfo->inode)->commit_mutex, and the
862 nfs_request_add_commit_list_locked(struct nfs_page
*req
, struct list_head
*dst
,
863 struct nfs_commit_info
*cinfo
)
865 set_bit(PG_CLEAN
, &req
->wb_flags
);
866 nfs_list_add_request(req
, dst
);
867 atomic_long_inc(&cinfo
->mds
->ncommit
);
869 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list_locked
);
872 * nfs_request_add_commit_list - add request to a commit list
873 * @req: pointer to a struct nfs_page
874 * @cinfo: holds list lock and accounting info
876 * This sets the PG_CLEAN bit, updates the cinfo count of
877 * number of outstanding requests requiring a commit as well as
880 * The caller must _not_ hold the cinfo->lock, but must be
881 * holding the nfs_page lock.
884 nfs_request_add_commit_list(struct nfs_page
*req
, struct nfs_commit_info
*cinfo
)
886 mutex_lock(&NFS_I(cinfo
->inode
)->commit_mutex
);
887 nfs_request_add_commit_list_locked(req
, &cinfo
->mds
->list
, cinfo
);
888 mutex_unlock(&NFS_I(cinfo
->inode
)->commit_mutex
);
890 nfs_mark_page_unstable(req
->wb_page
, cinfo
);
892 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list
);
895 * nfs_request_remove_commit_list - Remove request from a commit list
896 * @req: pointer to a nfs_page
897 * @cinfo: holds list lock and accounting info
899 * This clears the PG_CLEAN bit, and updates the cinfo's count of
900 * number of outstanding requests requiring a commit
901 * It does not update the MM page stats.
903 * The caller _must_ hold the cinfo->lock and the nfs_page lock.
906 nfs_request_remove_commit_list(struct nfs_page
*req
,
907 struct nfs_commit_info
*cinfo
)
909 if (!test_and_clear_bit(PG_CLEAN
, &(req
)->wb_flags
))
911 nfs_list_remove_request(req
);
912 atomic_long_dec(&cinfo
->mds
->ncommit
);
914 EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list
);
916 static void nfs_init_cinfo_from_inode(struct nfs_commit_info
*cinfo
,
919 cinfo
->inode
= inode
;
920 cinfo
->mds
= &NFS_I(inode
)->commit_info
;
921 cinfo
->ds
= pnfs_get_ds_info(inode
);
923 cinfo
->completion_ops
= &nfs_commit_completion_ops
;
926 void nfs_init_cinfo(struct nfs_commit_info
*cinfo
,
928 struct nfs_direct_req
*dreq
)
931 nfs_init_cinfo_from_dreq(cinfo
, dreq
);
933 nfs_init_cinfo_from_inode(cinfo
, inode
);
935 EXPORT_SYMBOL_GPL(nfs_init_cinfo
);
938 * Add a request to the inode's commit list.
941 nfs_mark_request_commit(struct nfs_page
*req
, struct pnfs_layout_segment
*lseg
,
942 struct nfs_commit_info
*cinfo
, u32 ds_commit_idx
)
944 if (pnfs_mark_request_commit(req
, lseg
, cinfo
, ds_commit_idx
))
946 nfs_request_add_commit_list(req
, cinfo
);
950 nfs_clear_page_commit(struct page
*page
)
952 dec_node_page_state(page
, NR_WRITEBACK
);
953 dec_wb_stat(&inode_to_bdi(page_file_mapping(page
)->host
)->wb
,
957 /* Called holding the request lock on @req */
959 nfs_clear_request_commit(struct nfs_page
*req
)
961 if (test_bit(PG_CLEAN
, &req
->wb_flags
)) {
962 struct nfs_open_context
*ctx
= nfs_req_openctx(req
);
963 struct inode
*inode
= d_inode(ctx
->dentry
);
964 struct nfs_commit_info cinfo
;
966 nfs_init_cinfo_from_inode(&cinfo
, inode
);
967 mutex_lock(&NFS_I(inode
)->commit_mutex
);
968 if (!pnfs_clear_request_commit(req
, &cinfo
)) {
969 nfs_request_remove_commit_list(req
, &cinfo
);
971 mutex_unlock(&NFS_I(inode
)->commit_mutex
);
972 nfs_clear_page_commit(req
->wb_page
);
976 int nfs_write_need_commit(struct nfs_pgio_header
*hdr
)
978 if (hdr
->verf
.committed
== NFS_DATA_SYNC
)
979 return hdr
->lseg
== NULL
;
980 return hdr
->verf
.committed
!= NFS_FILE_SYNC
;
983 static void nfs_async_write_init(struct nfs_pgio_header
*hdr
)
985 nfs_io_completion_get(hdr
->io_completion
);
988 static void nfs_write_completion(struct nfs_pgio_header
*hdr
)
990 struct nfs_commit_info cinfo
;
991 unsigned long bytes
= 0;
993 if (test_bit(NFS_IOHDR_REDO
, &hdr
->flags
))
995 nfs_init_cinfo_from_inode(&cinfo
, hdr
->inode
);
996 while (!list_empty(&hdr
->pages
)) {
997 struct nfs_page
*req
= nfs_list_entry(hdr
->pages
.next
);
999 bytes
+= req
->wb_bytes
;
1000 nfs_list_remove_request(req
);
1001 if (test_bit(NFS_IOHDR_ERROR
, &hdr
->flags
) &&
1002 (hdr
->good_bytes
< bytes
)) {
1003 trace_nfs_comp_error(hdr
->inode
, req
, hdr
->error
);
1004 nfs_mapping_set_error(req
->wb_page
, hdr
->error
);
1007 if (nfs_write_need_commit(hdr
)) {
1008 /* Reset wb_nio, since the write was successful. */
1010 memcpy(&req
->wb_verf
, &hdr
->verf
.verifier
, sizeof(req
->wb_verf
));
1011 nfs_mark_request_commit(req
, hdr
->lseg
, &cinfo
,
1012 hdr
->pgio_mirror_idx
);
1016 nfs_inode_remove_request(req
);
1018 nfs_end_page_writeback(req
);
1019 nfs_release_request(req
);
1022 nfs_io_completion_put(hdr
->io_completion
);
1027 nfs_reqs_to_commit(struct nfs_commit_info
*cinfo
)
1029 return atomic_long_read(&cinfo
->mds
->ncommit
);
1032 /* NFS_I(cinfo->inode)->commit_mutex held by caller */
1034 nfs_scan_commit_list(struct list_head
*src
, struct list_head
*dst
,
1035 struct nfs_commit_info
*cinfo
, int max
)
1037 struct nfs_page
*req
, *tmp
;
1040 list_for_each_entry_safe(req
, tmp
, src
, wb_list
) {
1041 kref_get(&req
->wb_kref
);
1042 if (!nfs_lock_request(req
)) {
1043 nfs_release_request(req
);
1046 nfs_request_remove_commit_list(req
, cinfo
);
1047 clear_bit(PG_COMMIT_TO_DS
, &req
->wb_flags
);
1048 nfs_list_add_request(req
, dst
);
1050 if ((ret
== max
) && !cinfo
->dreq
)
1056 EXPORT_SYMBOL_GPL(nfs_scan_commit_list
);
1059 * nfs_scan_commit - Scan an inode for commit requests
1060 * @inode: NFS inode to scan
1061 * @dst: mds destination list
1062 * @cinfo: mds and ds lists of reqs ready to commit
1064 * Moves requests from the inode's 'commit' request list.
1065 * The requests are *not* checked to ensure that they form a contiguous set.
1068 nfs_scan_commit(struct inode
*inode
, struct list_head
*dst
,
1069 struct nfs_commit_info
*cinfo
)
1073 if (!atomic_long_read(&cinfo
->mds
->ncommit
))
1075 mutex_lock(&NFS_I(cinfo
->inode
)->commit_mutex
);
1076 if (atomic_long_read(&cinfo
->mds
->ncommit
) > 0) {
1077 const int max
= INT_MAX
;
1079 ret
= nfs_scan_commit_list(&cinfo
->mds
->list
, dst
,
1081 ret
+= pnfs_scan_commit_lists(inode
, cinfo
, max
- ret
);
1083 mutex_unlock(&NFS_I(cinfo
->inode
)->commit_mutex
);
1088 * Search for an existing write request, and attempt to update
1089 * it to reflect a new dirty region on a given page.
1091 * If the attempt fails, then the existing request is flushed out
1094 static struct nfs_page
*nfs_try_to_update_request(struct inode
*inode
,
1096 unsigned int offset
,
1099 struct nfs_page
*req
;
1104 end
= offset
+ bytes
;
1106 req
= nfs_lock_and_join_requests(page
);
1107 if (IS_ERR_OR_NULL(req
))
1110 rqend
= req
->wb_offset
+ req
->wb_bytes
;
1112 * Tell the caller to flush out the request if
1113 * the offsets are non-contiguous.
1114 * Note: nfs_flush_incompatible() will already
1115 * have flushed out requests having wrong owners.
1117 if (offset
> rqend
|| end
< req
->wb_offset
)
1120 /* Okay, the request matches. Update the region */
1121 if (offset
< req
->wb_offset
) {
1122 req
->wb_offset
= offset
;
1123 req
->wb_pgbase
= offset
;
1126 req
->wb_bytes
= end
- req
->wb_offset
;
1128 req
->wb_bytes
= rqend
- req
->wb_offset
;
1133 * Note: we mark the request dirty here because
1134 * nfs_lock_and_join_requests() cannot preserve
1135 * commit flags, so we have to replay the write.
1137 nfs_mark_request_dirty(req
);
1138 nfs_unlock_and_release_request(req
);
1139 error
= nfs_wb_page(inode
, page
);
1140 return (error
< 0) ? ERR_PTR(error
) : NULL
;
1144 * Try to update an existing write request, or create one if there is none.
1146 * Note: Should always be called with the Page Lock held to prevent races
1147 * if we have to add a new request. Also assumes that the caller has
1148 * already called nfs_flush_incompatible() if necessary.
1150 static struct nfs_page
* nfs_setup_write_request(struct nfs_open_context
* ctx
,
1151 struct page
*page
, unsigned int offset
, unsigned int bytes
)
1153 struct inode
*inode
= page_file_mapping(page
)->host
;
1154 struct nfs_page
*req
;
1156 req
= nfs_try_to_update_request(inode
, page
, offset
, bytes
);
1159 req
= nfs_create_request(ctx
, page
, offset
, bytes
);
1162 nfs_inode_add_request(inode
, req
);
1167 static int nfs_writepage_setup(struct nfs_open_context
*ctx
, struct page
*page
,
1168 unsigned int offset
, unsigned int count
)
1170 struct nfs_page
*req
;
1172 req
= nfs_setup_write_request(ctx
, page
, offset
, count
);
1174 return PTR_ERR(req
);
1175 /* Update file length */
1176 nfs_grow_file(page
, offset
, count
);
1177 nfs_mark_uptodate(req
);
1178 nfs_mark_request_dirty(req
);
1179 nfs_unlock_and_release_request(req
);
1183 int nfs_flush_incompatible(struct file
*file
, struct page
*page
)
1185 struct nfs_open_context
*ctx
= nfs_file_open_context(file
);
1186 struct nfs_lock_context
*l_ctx
;
1187 struct file_lock_context
*flctx
= locks_inode_context(file_inode(file
));
1188 struct nfs_page
*req
;
1189 int do_flush
, status
;
1191 * Look for a request corresponding to this page. If there
1192 * is one, and it belongs to another file, we flush it out
1193 * before we try to copy anything into the page. Do this
1194 * due to the lack of an ACCESS-type call in NFSv2.
1195 * Also do the same if we find a request from an existing
1199 req
= nfs_page_find_head_request(page
);
1202 l_ctx
= req
->wb_lock_context
;
1203 do_flush
= req
->wb_page
!= page
||
1204 !nfs_match_open_context(nfs_req_openctx(req
), ctx
);
1205 if (l_ctx
&& flctx
&&
1206 !(list_empty_careful(&flctx
->flc_posix
) &&
1207 list_empty_careful(&flctx
->flc_flock
))) {
1208 do_flush
|= l_ctx
->lockowner
!= current
->files
;
1210 nfs_release_request(req
);
1213 status
= nfs_wb_page(page_file_mapping(page
)->host
, page
);
1214 } while (status
== 0);
1219 * Avoid buffered writes when a open context credential's key would
1222 * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL.
1224 * Return 0 and set a credential flag which triggers the inode to flush
1225 * and performs NFS_FILE_SYNC writes if the key will expired within
1226 * RPC_KEY_EXPIRE_TIMEO.
1229 nfs_key_timeout_notify(struct file
*filp
, struct inode
*inode
)
1231 struct nfs_open_context
*ctx
= nfs_file_open_context(filp
);
1233 if (nfs_ctx_key_to_expire(ctx
, inode
) &&
1234 !rcu_access_pointer(ctx
->ll_cred
))
1235 /* Already expired! */
1241 * Test if the open context credential key is marked to expire soon.
1243 bool nfs_ctx_key_to_expire(struct nfs_open_context
*ctx
, struct inode
*inode
)
1245 struct rpc_auth
*auth
= NFS_SERVER(inode
)->client
->cl_auth
;
1246 struct rpc_cred
*cred
, *new, *old
= NULL
;
1247 struct auth_cred acred
= {
1253 cred
= rcu_dereference(ctx
->ll_cred
);
1254 if (cred
&& !(cred
->cr_ops
->crkey_timeout
&&
1255 cred
->cr_ops
->crkey_timeout(cred
)))
1259 new = auth
->au_ops
->lookup_cred(auth
, &acred
, 0);
1264 if (IS_ERR_OR_NULL(new)) {
1267 } else if (new->cr_ops
->crkey_timeout
&&
1268 new->cr_ops
->crkey_timeout(new))
1272 old
= rcu_dereference_protected(xchg(&ctx
->ll_cred
,
1273 RCU_INITIALIZER(new)), 1);
1281 * If the page cache is marked as unsafe or invalid, then we can't rely on
1282 * the PageUptodate() flag. In this case, we will need to turn off
1283 * write optimisations that depend on the page contents being correct.
1285 static bool nfs_write_pageuptodate(struct page
*page
, struct inode
*inode
,
1286 unsigned int pagelen
)
1288 struct nfs_inode
*nfsi
= NFS_I(inode
);
1290 if (nfs_have_delegated_attributes(inode
))
1292 if (nfsi
->cache_validity
&
1293 (NFS_INO_INVALID_CHANGE
| NFS_INO_INVALID_SIZE
))
1296 if (test_bit(NFS_INO_INVALIDATING
, &nfsi
->flags
) && pagelen
!= 0)
1299 if (nfsi
->cache_validity
& NFS_INO_INVALID_DATA
&& pagelen
!= 0)
1301 return PageUptodate(page
) != 0;
1305 is_whole_file_wrlock(struct file_lock
*fl
)
1307 return fl
->fl_start
== 0 && fl
->fl_end
== OFFSET_MAX
&&
1308 fl
->fl_type
== F_WRLCK
;
1311 /* If we know the page is up to date, and we're not using byte range locks (or
1312 * if we have the whole file locked for writing), it may be more efficient to
1313 * extend the write to cover the entire page in order to avoid fragmentation
1316 * If the file is opened for synchronous writes then we can just skip the rest
1319 static int nfs_can_extend_write(struct file
*file
, struct page
*page
,
1320 struct inode
*inode
, unsigned int pagelen
)
1323 struct file_lock_context
*flctx
= locks_inode_context(inode
);
1324 struct file_lock
*fl
;
1326 if (file
->f_flags
& O_DSYNC
)
1328 if (!nfs_write_pageuptodate(page
, inode
, pagelen
))
1330 if (NFS_PROTO(inode
)->have_delegation(inode
, FMODE_WRITE
))
1332 if (!flctx
|| (list_empty_careful(&flctx
->flc_flock
) &&
1333 list_empty_careful(&flctx
->flc_posix
)))
1336 /* Check to see if there are whole file write locks */
1338 spin_lock(&flctx
->flc_lock
);
1339 if (!list_empty(&flctx
->flc_posix
)) {
1340 fl
= list_first_entry(&flctx
->flc_posix
, struct file_lock
,
1342 if (is_whole_file_wrlock(fl
))
1344 } else if (!list_empty(&flctx
->flc_flock
)) {
1345 fl
= list_first_entry(&flctx
->flc_flock
, struct file_lock
,
1347 if (fl
->fl_type
== F_WRLCK
)
1350 spin_unlock(&flctx
->flc_lock
);
1355 * Update and possibly write a cached page of an NFS file.
1357 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
1358 * things with a page scheduled for an RPC call (e.g. invalidate it).
1360 int nfs_updatepage(struct file
*file
, struct page
*page
,
1361 unsigned int offset
, unsigned int count
)
1363 struct nfs_open_context
*ctx
= nfs_file_open_context(file
);
1364 struct address_space
*mapping
= page_file_mapping(page
);
1365 struct inode
*inode
= mapping
->host
;
1366 unsigned int pagelen
= nfs_page_length(page
);
1369 nfs_inc_stats(inode
, NFSIOS_VFSUPDATEPAGE
);
1371 dprintk("NFS: nfs_updatepage(%pD2 %d@%lld)\n",
1372 file
, count
, (long long)(page_file_offset(page
) + offset
));
1377 if (nfs_can_extend_write(file
, page
, inode
, pagelen
)) {
1378 count
= max(count
+ offset
, pagelen
);
1382 status
= nfs_writepage_setup(ctx
, page
, offset
, count
);
1384 nfs_set_pageerror(mapping
);
1386 dprintk("NFS: nfs_updatepage returns %d (isize %lld)\n",
1387 status
, (long long)i_size_read(inode
));
1391 static int flush_task_priority(int how
)
1393 switch (how
& (FLUSH_HIGHPRI
|FLUSH_LOWPRI
)) {
1395 return RPC_PRIORITY_HIGH
;
1397 return RPC_PRIORITY_LOW
;
1399 return RPC_PRIORITY_NORMAL
;
1402 static void nfs_initiate_write(struct nfs_pgio_header
*hdr
,
1403 struct rpc_message
*msg
,
1404 const struct nfs_rpc_ops
*rpc_ops
,
1405 struct rpc_task_setup
*task_setup_data
, int how
)
1407 int priority
= flush_task_priority(how
);
1409 if (IS_SWAPFILE(hdr
->inode
))
1410 task_setup_data
->flags
|= RPC_TASK_SWAPPER
;
1411 task_setup_data
->priority
= priority
;
1412 rpc_ops
->write_setup(hdr
, msg
, &task_setup_data
->rpc_client
);
1413 trace_nfs_initiate_write(hdr
);
1416 /* If a nfs_flush_* function fails, it should remove reqs from @head and
1417 * call this on each, which will prepare them to be retried on next
1418 * writeback using standard nfs.
1420 static void nfs_redirty_request(struct nfs_page
*req
)
1422 struct nfs_inode
*nfsi
= NFS_I(nfs_page_to_inode(req
));
1424 /* Bump the transmission count */
1426 nfs_mark_request_dirty(req
);
1427 atomic_long_inc(&nfsi
->redirtied_pages
);
1428 nfs_end_page_writeback(req
);
1429 nfs_release_request(req
);
1432 static void nfs_async_write_error(struct list_head
*head
, int error
)
1434 struct nfs_page
*req
;
1436 while (!list_empty(head
)) {
1437 req
= nfs_list_entry(head
->next
);
1438 nfs_list_remove_request(req
);
1439 if (nfs_error_is_fatal_on_server(error
))
1440 nfs_write_error(req
, error
);
1442 nfs_redirty_request(req
);
1446 static void nfs_async_write_reschedule_io(struct nfs_pgio_header
*hdr
)
1448 nfs_async_write_error(&hdr
->pages
, 0);
1451 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops
= {
1452 .init_hdr
= nfs_async_write_init
,
1453 .error_cleanup
= nfs_async_write_error
,
1454 .completion
= nfs_write_completion
,
1455 .reschedule_io
= nfs_async_write_reschedule_io
,
1458 void nfs_pageio_init_write(struct nfs_pageio_descriptor
*pgio
,
1459 struct inode
*inode
, int ioflags
, bool force_mds
,
1460 const struct nfs_pgio_completion_ops
*compl_ops
)
1462 struct nfs_server
*server
= NFS_SERVER(inode
);
1463 const struct nfs_pageio_ops
*pg_ops
= &nfs_pgio_rw_ops
;
1465 #ifdef CONFIG_NFS_V4_1
1466 if (server
->pnfs_curr_ld
&& !force_mds
)
1467 pg_ops
= server
->pnfs_curr_ld
->pg_write_ops
;
1469 nfs_pageio_init(pgio
, inode
, pg_ops
, compl_ops
, &nfs_rw_write_ops
,
1470 server
->wsize
, ioflags
);
1472 EXPORT_SYMBOL_GPL(nfs_pageio_init_write
);
1474 void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor
*pgio
)
1476 struct nfs_pgio_mirror
*mirror
;
1478 if (pgio
->pg_ops
&& pgio
->pg_ops
->pg_cleanup
)
1479 pgio
->pg_ops
->pg_cleanup(pgio
);
1481 pgio
->pg_ops
= &nfs_pgio_rw_ops
;
1483 nfs_pageio_stop_mirroring(pgio
);
1485 mirror
= &pgio
->pg_mirrors
[0];
1486 mirror
->pg_bsize
= NFS_SERVER(pgio
->pg_inode
)->wsize
;
1488 EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds
);
1491 void nfs_commit_prepare(struct rpc_task
*task
, void *calldata
)
1493 struct nfs_commit_data
*data
= calldata
;
1495 NFS_PROTO(data
->inode
)->commit_rpc_prepare(task
, data
);
1498 static void nfs_writeback_check_extend(struct nfs_pgio_header
*hdr
,
1499 struct nfs_fattr
*fattr
)
1501 struct nfs_pgio_args
*argp
= &hdr
->args
;
1502 struct nfs_pgio_res
*resp
= &hdr
->res
;
1503 u64 size
= argp
->offset
+ resp
->count
;
1505 if (!(fattr
->valid
& NFS_ATTR_FATTR_SIZE
))
1507 if (nfs_size_to_loff_t(fattr
->size
) < i_size_read(hdr
->inode
)) {
1508 fattr
->valid
&= ~NFS_ATTR_FATTR_SIZE
;
1511 if (size
!= fattr
->size
)
1513 /* Set attribute barrier */
1514 nfs_fattr_set_barrier(fattr
);
1515 /* ...and update size */
1516 fattr
->valid
|= NFS_ATTR_FATTR_SIZE
;
1519 void nfs_writeback_update_inode(struct nfs_pgio_header
*hdr
)
1521 struct nfs_fattr
*fattr
= &hdr
->fattr
;
1522 struct inode
*inode
= hdr
->inode
;
1524 spin_lock(&inode
->i_lock
);
1525 nfs_writeback_check_extend(hdr
, fattr
);
1526 nfs_post_op_update_inode_force_wcc_locked(inode
, fattr
);
1527 spin_unlock(&inode
->i_lock
);
1529 EXPORT_SYMBOL_GPL(nfs_writeback_update_inode
);
1532 * This function is called when the WRITE call is complete.
1534 static int nfs_writeback_done(struct rpc_task
*task
,
1535 struct nfs_pgio_header
*hdr
,
1536 struct inode
*inode
)
1541 * ->write_done will attempt to use post-op attributes to detect
1542 * conflicting writes by other clients. A strict interpretation
1543 * of close-to-open would allow us to continue caching even if
1544 * another writer had changed the file, but some applications
1545 * depend on tighter cache coherency when writing.
1547 status
= NFS_PROTO(inode
)->write_done(task
, hdr
);
1551 nfs_add_stats(inode
, NFSIOS_SERVERWRITTENBYTES
, hdr
->res
.count
);
1552 trace_nfs_writeback_done(task
, hdr
);
1554 if (task
->tk_status
>= 0) {
1555 enum nfs3_stable_how committed
= hdr
->res
.verf
->committed
;
1557 if (committed
== NFS_UNSTABLE
) {
1559 * We have some uncommitted data on the server at
1560 * this point, so ensure that we keep track of that
1561 * fact irrespective of what later writes do.
1563 set_bit(NFS_IOHDR_UNSTABLE_WRITES
, &hdr
->flags
);
1566 if (committed
< hdr
->args
.stable
) {
1567 /* We tried a write call, but the server did not
1568 * commit data to stable storage even though we
1570 * Note: There is a known bug in Tru64 < 5.0 in which
1571 * the server reports NFS_DATA_SYNC, but performs
1572 * NFS_FILE_SYNC. We therefore implement this checking
1573 * as a dprintk() in order to avoid filling syslog.
1575 static unsigned long complain
;
1577 /* Note this will print the MDS for a DS write */
1578 if (time_before(complain
, jiffies
)) {
1579 dprintk("NFS: faulty NFS server %s:"
1580 " (committed = %d) != (stable = %d)\n",
1581 NFS_SERVER(inode
)->nfs_client
->cl_hostname
,
1582 committed
, hdr
->args
.stable
);
1583 complain
= jiffies
+ 300 * HZ
;
1588 /* Deal with the suid/sgid bit corner case */
1589 if (nfs_should_remove_suid(inode
)) {
1590 spin_lock(&inode
->i_lock
);
1591 nfs_set_cache_invalid(inode
, NFS_INO_INVALID_MODE
);
1592 spin_unlock(&inode
->i_lock
);
1598 * This function is called when the WRITE call is complete.
1600 static void nfs_writeback_result(struct rpc_task
*task
,
1601 struct nfs_pgio_header
*hdr
)
1603 struct nfs_pgio_args
*argp
= &hdr
->args
;
1604 struct nfs_pgio_res
*resp
= &hdr
->res
;
1606 if (resp
->count
< argp
->count
) {
1607 static unsigned long complain
;
1609 /* This a short write! */
1610 nfs_inc_stats(hdr
->inode
, NFSIOS_SHORTWRITE
);
1612 /* Has the server at least made some progress? */
1613 if (resp
->count
== 0) {
1614 if (time_before(complain
, jiffies
)) {
1616 "NFS: Server wrote zero bytes, expected %u.\n",
1618 complain
= jiffies
+ 300 * HZ
;
1620 nfs_set_pgio_error(hdr
, -EIO
, argp
->offset
);
1621 task
->tk_status
= -EIO
;
1625 /* For non rpc-based layout drivers, retry-through-MDS */
1626 if (!task
->tk_ops
) {
1627 hdr
->pnfs_error
= -EAGAIN
;
1631 /* Was this an NFSv2 write or an NFSv3 stable write? */
1632 if (resp
->verf
->committed
!= NFS_UNSTABLE
) {
1633 /* Resend from where the server left off */
1634 hdr
->mds_offset
+= resp
->count
;
1635 argp
->offset
+= resp
->count
;
1636 argp
->pgbase
+= resp
->count
;
1637 argp
->count
-= resp
->count
;
1639 /* Resend as a stable write in order to avoid
1640 * headaches in the case of a server crash.
1642 argp
->stable
= NFS_FILE_SYNC
;
1645 resp
->verf
->committed
= 0;
1646 rpc_restart_call_prepare(task
);
1650 static int wait_on_commit(struct nfs_mds_commit_info
*cinfo
)
1652 return wait_var_event_killable(&cinfo
->rpcs_out
,
1653 !atomic_read(&cinfo
->rpcs_out
));
1656 static void nfs_commit_begin(struct nfs_mds_commit_info
*cinfo
)
1658 atomic_inc(&cinfo
->rpcs_out
);
1661 bool nfs_commit_end(struct nfs_mds_commit_info
*cinfo
)
1663 if (atomic_dec_and_test(&cinfo
->rpcs_out
)) {
1664 wake_up_var(&cinfo
->rpcs_out
);
1670 void nfs_commitdata_release(struct nfs_commit_data
*data
)
1672 put_nfs_open_context(data
->context
);
1673 nfs_commit_free(data
);
1675 EXPORT_SYMBOL_GPL(nfs_commitdata_release
);
1677 int nfs_initiate_commit(struct rpc_clnt
*clnt
, struct nfs_commit_data
*data
,
1678 const struct nfs_rpc_ops
*nfs_ops
,
1679 const struct rpc_call_ops
*call_ops
,
1682 struct rpc_task
*task
;
1683 int priority
= flush_task_priority(how
);
1684 struct rpc_message msg
= {
1685 .rpc_argp
= &data
->args
,
1686 .rpc_resp
= &data
->res
,
1687 .rpc_cred
= data
->cred
,
1689 struct rpc_task_setup task_setup_data
= {
1690 .task
= &data
->task
,
1692 .rpc_message
= &msg
,
1693 .callback_ops
= call_ops
,
1694 .callback_data
= data
,
1695 .workqueue
= nfsiod_workqueue
,
1696 .flags
= RPC_TASK_ASYNC
| flags
,
1697 .priority
= priority
,
1700 if (nfs_server_capable(data
->inode
, NFS_CAP_MOVEABLE
))
1701 task_setup_data
.flags
|= RPC_TASK_MOVEABLE
;
1703 /* Set up the initial task struct. */
1704 nfs_ops
->commit_setup(data
, &msg
, &task_setup_data
.rpc_client
);
1705 trace_nfs_initiate_commit(data
);
1707 dprintk("NFS: initiated commit call\n");
1709 task
= rpc_run_task(&task_setup_data
);
1711 return PTR_ERR(task
);
1712 if (how
& FLUSH_SYNC
)
1713 rpc_wait_for_completion_task(task
);
1717 EXPORT_SYMBOL_GPL(nfs_initiate_commit
);
1719 static loff_t
nfs_get_lwb(struct list_head
*head
)
1722 struct nfs_page
*req
;
1724 list_for_each_entry(req
, head
, wb_list
)
1725 if (lwb
< (req_offset(req
) + req
->wb_bytes
))
1726 lwb
= req_offset(req
) + req
->wb_bytes
;
1732 * Set up the argument/result storage required for the RPC call.
1734 void nfs_init_commit(struct nfs_commit_data
*data
,
1735 struct list_head
*head
,
1736 struct pnfs_layout_segment
*lseg
,
1737 struct nfs_commit_info
*cinfo
)
1739 struct nfs_page
*first
;
1740 struct nfs_open_context
*ctx
;
1741 struct inode
*inode
;
1743 /* Set up the RPC argument and reply structs
1744 * NB: take care not to mess about with data->commit et al. */
1747 list_splice_init(head
, &data
->pages
);
1749 first
= nfs_list_entry(data
->pages
.next
);
1750 ctx
= nfs_req_openctx(first
);
1751 inode
= d_inode(ctx
->dentry
);
1753 data
->inode
= inode
;
1754 data
->cred
= ctx
->cred
;
1755 data
->lseg
= lseg
; /* reference transferred */
1756 /* only set lwb for pnfs commit */
1758 data
->lwb
= nfs_get_lwb(&data
->pages
);
1759 data
->mds_ops
= &nfs_commit_ops
;
1760 data
->completion_ops
= cinfo
->completion_ops
;
1761 data
->dreq
= cinfo
->dreq
;
1763 data
->args
.fh
= NFS_FH(data
->inode
);
1764 /* Note: we always request a commit of the entire inode */
1765 data
->args
.offset
= 0;
1766 data
->args
.count
= 0;
1767 data
->context
= get_nfs_open_context(ctx
);
1768 data
->res
.fattr
= &data
->fattr
;
1769 data
->res
.verf
= &data
->verf
;
1770 nfs_fattr_init(&data
->fattr
);
1771 nfs_commit_begin(cinfo
->mds
);
1773 EXPORT_SYMBOL_GPL(nfs_init_commit
);
1775 void nfs_retry_commit(struct list_head
*page_list
,
1776 struct pnfs_layout_segment
*lseg
,
1777 struct nfs_commit_info
*cinfo
,
1780 struct nfs_page
*req
;
1782 while (!list_empty(page_list
)) {
1783 req
= nfs_list_entry(page_list
->next
);
1784 nfs_list_remove_request(req
);
1785 nfs_mark_request_commit(req
, lseg
, cinfo
, ds_commit_idx
);
1787 nfs_clear_page_commit(req
->wb_page
);
1788 nfs_unlock_and_release_request(req
);
1791 EXPORT_SYMBOL_GPL(nfs_retry_commit
);
1794 nfs_commit_resched_write(struct nfs_commit_info
*cinfo
,
1795 struct nfs_page
*req
)
1797 __set_page_dirty_nobuffers(req
->wb_page
);
1801 * Commit dirty pages
1804 nfs_commit_list(struct inode
*inode
, struct list_head
*head
, int how
,
1805 struct nfs_commit_info
*cinfo
)
1807 struct nfs_commit_data
*data
;
1808 unsigned short task_flags
= 0;
1810 /* another commit raced with us */
1811 if (list_empty(head
))
1814 data
= nfs_commitdata_alloc();
1816 nfs_retry_commit(head
, NULL
, cinfo
, -1);
1820 /* Set up the argument struct */
1821 nfs_init_commit(data
, head
, NULL
, cinfo
);
1822 if (NFS_SERVER(inode
)->nfs_client
->cl_minorversion
)
1823 task_flags
= RPC_TASK_MOVEABLE
;
1824 return nfs_initiate_commit(NFS_CLIENT(inode
), data
, NFS_PROTO(inode
),
1826 RPC_TASK_CRED_NOREF
| task_flags
);
1830 * COMMIT call returned
1832 static void nfs_commit_done(struct rpc_task
*task
, void *calldata
)
1834 struct nfs_commit_data
*data
= calldata
;
1836 /* Call the NFS version-specific code */
1837 NFS_PROTO(data
->inode
)->commit_done(task
, data
);
1838 trace_nfs_commit_done(task
, data
);
1841 static void nfs_commit_release_pages(struct nfs_commit_data
*data
)
1843 const struct nfs_writeverf
*verf
= data
->res
.verf
;
1844 struct nfs_page
*req
;
1845 int status
= data
->task
.tk_status
;
1846 struct nfs_commit_info cinfo
;
1847 struct nfs_server
*nfss
;
1849 while (!list_empty(&data
->pages
)) {
1850 req
= nfs_list_entry(data
->pages
.next
);
1851 nfs_list_remove_request(req
);
1853 nfs_clear_page_commit(req
->wb_page
);
1855 dprintk("NFS: commit (%s/%llu %d@%lld)",
1856 nfs_req_openctx(req
)->dentry
->d_sb
->s_id
,
1857 (unsigned long long)NFS_FILEID(d_inode(nfs_req_openctx(req
)->dentry
)),
1859 (long long)req_offset(req
));
1862 trace_nfs_commit_error(data
->inode
, req
,
1864 nfs_mapping_set_error(req
->wb_page
, status
);
1865 nfs_inode_remove_request(req
);
1867 dprintk_cont(", error = %d\n", status
);
1871 /* Okay, COMMIT succeeded, apparently. Check the verifier
1872 * returned by the server against all stored verfs. */
1873 if (nfs_write_match_verf(verf
, req
)) {
1874 /* We have a match */
1876 nfs_inode_remove_request(req
);
1877 dprintk_cont(" OK\n");
1880 /* We have a mismatch. Write the page again */
1881 dprintk_cont(" mismatch\n");
1882 nfs_mark_request_dirty(req
);
1883 atomic_long_inc(&NFS_I(data
->inode
)->redirtied_pages
);
1885 nfs_unlock_and_release_request(req
);
1886 /* Latency breaker */
1889 nfss
= NFS_SERVER(data
->inode
);
1890 if (atomic_long_read(&nfss
->writeback
) < NFS_CONGESTION_OFF_THRESH
)
1891 nfss
->write_congested
= 0;
1893 nfs_init_cinfo(&cinfo
, data
->inode
, data
->dreq
);
1894 nfs_commit_end(cinfo
.mds
);
1897 static void nfs_commit_release(void *calldata
)
1899 struct nfs_commit_data
*data
= calldata
;
1901 data
->completion_ops
->completion(data
);
1902 nfs_commitdata_release(calldata
);
1905 static const struct rpc_call_ops nfs_commit_ops
= {
1906 .rpc_call_prepare
= nfs_commit_prepare
,
1907 .rpc_call_done
= nfs_commit_done
,
1908 .rpc_release
= nfs_commit_release
,
1911 static const struct nfs_commit_completion_ops nfs_commit_completion_ops
= {
1912 .completion
= nfs_commit_release_pages
,
1913 .resched_write
= nfs_commit_resched_write
,
1916 int nfs_generic_commit_list(struct inode
*inode
, struct list_head
*head
,
1917 int how
, struct nfs_commit_info
*cinfo
)
1921 status
= pnfs_commit_list(inode
, head
, how
, cinfo
);
1922 if (status
== PNFS_NOT_ATTEMPTED
)
1923 status
= nfs_commit_list(inode
, head
, how
, cinfo
);
1927 static int __nfs_commit_inode(struct inode
*inode
, int how
,
1928 struct writeback_control
*wbc
)
1931 struct nfs_commit_info cinfo
;
1932 int may_wait
= how
& FLUSH_SYNC
;
1936 nfs_init_cinfo_from_inode(&cinfo
, inode
);
1937 nfs_commit_begin(cinfo
.mds
);
1939 ret
= nscan
= nfs_scan_commit(inode
, &head
, &cinfo
);
1942 ret
= nfs_generic_commit_list(inode
, &head
, how
, &cinfo
);
1946 if (wbc
&& wbc
->sync_mode
== WB_SYNC_NONE
) {
1947 if (nscan
< wbc
->nr_to_write
)
1948 wbc
->nr_to_write
-= nscan
;
1950 wbc
->nr_to_write
= 0;
1952 if (nscan
< INT_MAX
)
1956 nfs_commit_end(cinfo
.mds
);
1957 if (ret
|| !may_wait
)
1959 return wait_on_commit(cinfo
.mds
);
1962 int nfs_commit_inode(struct inode
*inode
, int how
)
1964 return __nfs_commit_inode(inode
, how
, NULL
);
1966 EXPORT_SYMBOL_GPL(nfs_commit_inode
);
1968 int nfs_write_inode(struct inode
*inode
, struct writeback_control
*wbc
)
1970 struct nfs_inode
*nfsi
= NFS_I(inode
);
1971 int flags
= FLUSH_SYNC
;
1974 if (wbc
->sync_mode
== WB_SYNC_NONE
) {
1975 /* no commits means nothing needs to be done */
1976 if (!atomic_long_read(&nfsi
->commit_info
.ncommit
))
1977 goto check_requests_outstanding
;
1979 /* Don't commit yet if this is a non-blocking flush and there
1980 * are a lot of outstanding writes for this mapping.
1982 if (mapping_tagged(inode
->i_mapping
, PAGECACHE_TAG_WRITEBACK
))
1983 goto out_mark_dirty
;
1985 /* don't wait for the COMMIT response */
1989 ret
= __nfs_commit_inode(inode
, flags
, wbc
);
1991 if (flags
& FLUSH_SYNC
)
1993 } else if (atomic_long_read(&nfsi
->commit_info
.ncommit
))
1994 goto out_mark_dirty
;
1996 check_requests_outstanding
:
1997 if (!atomic_read(&nfsi
->commit_info
.rpcs_out
))
2000 __mark_inode_dirty(inode
, I_DIRTY_DATASYNC
);
2003 EXPORT_SYMBOL_GPL(nfs_write_inode
);
2006 * Wrapper for filemap_write_and_wait_range()
2008 * Needed for pNFS in order to ensure data becomes visible to the
2011 int nfs_filemap_write_and_wait_range(struct address_space
*mapping
,
2012 loff_t lstart
, loff_t lend
)
2016 ret
= filemap_write_and_wait_range(mapping
, lstart
, lend
);
2018 ret
= pnfs_sync_inode(mapping
->host
, true);
2021 EXPORT_SYMBOL_GPL(nfs_filemap_write_and_wait_range
);
2024 * flush the inode to disk.
2026 int nfs_wb_all(struct inode
*inode
)
2030 trace_nfs_writeback_inode_enter(inode
);
2032 ret
= filemap_write_and_wait(inode
->i_mapping
);
2035 ret
= nfs_commit_inode(inode
, FLUSH_SYNC
);
2038 pnfs_sync_inode(inode
, true);
2042 trace_nfs_writeback_inode_exit(inode
, ret
);
2045 EXPORT_SYMBOL_GPL(nfs_wb_all
);
2047 int nfs_wb_folio_cancel(struct inode
*inode
, struct folio
*folio
)
2049 struct nfs_page
*req
;
2052 folio_wait_writeback(folio
);
2054 /* blocking call to cancel all requests and join to a single (head)
2056 req
= nfs_lock_and_join_requests(&folio
->page
);
2061 /* all requests from this folio have been cancelled by
2062 * nfs_lock_and_join_requests, so just remove the head
2063 * request from the inode / page_private pointer and
2065 nfs_inode_remove_request(req
);
2066 nfs_unlock_and_release_request(req
);
2073 * nfs_wb_folio - Write back all requests on one page
2074 * @inode: pointer to page
2075 * @folio: pointer to folio
2077 * Assumes that the folio has been locked by the caller, and will
2080 int nfs_wb_folio(struct inode
*inode
, struct folio
*folio
)
2082 loff_t range_start
= folio_file_pos(folio
);
2083 loff_t range_end
= range_start
+ (loff_t
)folio_size(folio
) - 1;
2084 struct writeback_control wbc
= {
2085 .sync_mode
= WB_SYNC_ALL
,
2087 .range_start
= range_start
,
2088 .range_end
= range_end
,
2092 trace_nfs_writeback_page_enter(inode
);
2095 folio_wait_writeback(folio
);
2096 if (folio_clear_dirty_for_io(folio
)) {
2097 ret
= nfs_writepage_locked(&folio
->page
, &wbc
);
2103 if (!folio_test_private(folio
))
2105 ret
= nfs_commit_inode(inode
, FLUSH_SYNC
);
2110 trace_nfs_writeback_page_exit(inode
, ret
);
2114 int nfs_wb_page(struct inode
*inode
, struct page
*page
)
2116 return nfs_wb_folio(inode
, page_folio(page
));
2119 #ifdef CONFIG_MIGRATION
2120 int nfs_migrate_folio(struct address_space
*mapping
, struct folio
*dst
,
2121 struct folio
*src
, enum migrate_mode mode
)
2124 * If the private flag is set, the folio is currently associated with
2125 * an in-progress read or write request. Don't try to migrate it.
2127 * FIXME: we could do this in principle, but we'll need a way to ensure
2128 * that we can safely release the inode reference while holding
2131 if (folio_test_private(src
))
2134 if (folio_test_fscache(src
)) {
2135 if (mode
== MIGRATE_ASYNC
)
2137 folio_wait_fscache(src
);
2140 return migrate_folio(mapping
, dst
, src
, mode
);
2144 int __init
nfs_init_writepagecache(void)
2146 nfs_wdata_cachep
= kmem_cache_create("nfs_write_data",
2147 sizeof(struct nfs_pgio_header
),
2148 0, SLAB_HWCACHE_ALIGN
,
2150 if (nfs_wdata_cachep
== NULL
)
2153 nfs_wdata_mempool
= mempool_create_slab_pool(MIN_POOL_WRITE
,
2155 if (nfs_wdata_mempool
== NULL
)
2156 goto out_destroy_write_cache
;
2158 nfs_cdata_cachep
= kmem_cache_create("nfs_commit_data",
2159 sizeof(struct nfs_commit_data
),
2160 0, SLAB_HWCACHE_ALIGN
,
2162 if (nfs_cdata_cachep
== NULL
)
2163 goto out_destroy_write_mempool
;
2165 nfs_commit_mempool
= mempool_create_slab_pool(MIN_POOL_COMMIT
,
2167 if (nfs_commit_mempool
== NULL
)
2168 goto out_destroy_commit_cache
;
2171 * NFS congestion size, scale with available memory.
2183 * This allows larger machines to have larger/more transfers.
2184 * Limit the default to 256M
2186 nfs_congestion_kb
= (16*int_sqrt(totalram_pages())) << (PAGE_SHIFT
-10);
2187 if (nfs_congestion_kb
> 256*1024)
2188 nfs_congestion_kb
= 256*1024;
2192 out_destroy_commit_cache
:
2193 kmem_cache_destroy(nfs_cdata_cachep
);
2194 out_destroy_write_mempool
:
2195 mempool_destroy(nfs_wdata_mempool
);
2196 out_destroy_write_cache
:
2197 kmem_cache_destroy(nfs_wdata_cachep
);
2201 void nfs_destroy_writepagecache(void)
2203 mempool_destroy(nfs_commit_mempool
);
2204 kmem_cache_destroy(nfs_cdata_cachep
);
2205 mempool_destroy(nfs_wdata_mempool
);
2206 kmem_cache_destroy(nfs_wdata_cachep
);
2209 static const struct nfs_rw_ops nfs_rw_write_ops
= {
2210 .rw_alloc_header
= nfs_writehdr_alloc
,
2211 .rw_free_header
= nfs_writehdr_free
,
2212 .rw_done
= nfs_writeback_done
,
2213 .rw_result
= nfs_writeback_result
,
2214 .rw_initiate
= nfs_initiate_write
,