1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/slab.h>
3 #include <linux/stat.h>
4 #include <linux/sched/xacct.h>
5 #include <linux/fcntl.h>
6 #include <linux/file.h>
8 #include <linux/fsnotify.h>
9 #include <linux/security.h>
10 #include <linux/export.h>
11 #include <linux/syscalls.h>
12 #include <linux/pagemap.h>
13 #include <linux/splice.h>
14 #include <linux/compat.h>
15 #include <linux/mount.h>
17 #include <linux/dax.h>
20 #include <linux/uaccess.h>
21 #include <asm/unistd.h>
24 * Performs necessary checks before doing a clone.
26 * Can adjust amount of bytes to clone via @req_count argument.
27 * Returns appropriate error code that caller should return or
28 * zero in case the clone should be allowed.
30 static int generic_remap_checks(struct file
*file_in
, loff_t pos_in
,
31 struct file
*file_out
, loff_t pos_out
,
32 loff_t
*req_count
, unsigned int remap_flags
)
34 struct inode
*inode_in
= file_in
->f_mapping
->host
;
35 struct inode
*inode_out
= file_out
->f_mapping
->host
;
36 uint64_t count
= *req_count
;
38 loff_t size_in
, size_out
;
39 loff_t bs
= inode_out
->i_sb
->s_blocksize
;
42 /* The start of both ranges must be aligned to an fs block. */
43 if (!IS_ALIGNED(pos_in
, bs
) || !IS_ALIGNED(pos_out
, bs
))
46 /* Ensure offsets don't wrap. */
47 if (pos_in
+ count
< pos_in
|| pos_out
+ count
< pos_out
)
50 size_in
= i_size_read(inode_in
);
51 size_out
= i_size_read(inode_out
);
53 /* Dedupe requires both ranges to be within EOF. */
54 if ((remap_flags
& REMAP_FILE_DEDUP
) &&
55 (pos_in
>= size_in
|| pos_in
+ count
> size_in
||
56 pos_out
>= size_out
|| pos_out
+ count
> size_out
))
59 /* Ensure the infile range is within the infile. */
60 if (pos_in
>= size_in
)
62 count
= min(count
, size_in
- (uint64_t)pos_in
);
64 ret
= generic_write_check_limits(file_out
, pos_out
, &count
);
69 * If the user wanted us to link to the infile's EOF, round up to the
70 * next block boundary for this check.
72 * Otherwise, make sure the count is also block-aligned, having
73 * already confirmed the starting offsets' block alignment.
75 if (pos_in
+ count
== size_in
&&
76 (!(remap_flags
& REMAP_FILE_DEDUP
) || pos_out
+ count
== size_out
)) {
77 bcount
= ALIGN(size_in
, bs
) - pos_in
;
79 if (!IS_ALIGNED(count
, bs
))
80 count
= ALIGN_DOWN(count
, bs
);
84 /* Don't allow overlapped cloning within the same file. */
85 if (inode_in
== inode_out
&&
86 pos_out
+ bcount
> pos_in
&&
87 pos_out
< pos_in
+ bcount
)
91 * We shortened the request but the caller can't deal with that, so
92 * bounce the request back to userspace.
94 if (*req_count
!= count
&& !(remap_flags
& REMAP_FILE_CAN_SHORTEN
))
101 static int remap_verify_area(struct file
*file
, loff_t pos
, loff_t len
,
104 if (unlikely(pos
< 0 || len
< 0))
107 if (unlikely((loff_t
) (pos
+ len
) < 0))
110 return security_file_permission(file
, write
? MAY_WRITE
: MAY_READ
);
114 * Ensure that we don't remap a partial EOF block in the middle of something
115 * else. Assume that the offsets have already been checked for block
118 * For clone we only link a partial EOF block above or at the destination file's
119 * EOF. For deduplication we accept a partial EOF block only if it ends at the
120 * destination file's EOF (can not link it into the middle of a file).
122 * Shorten the request if possible.
124 static int generic_remap_check_len(struct inode
*inode_in
,
125 struct inode
*inode_out
,
128 unsigned int remap_flags
)
130 u64 blkmask
= i_blocksize(inode_in
) - 1;
131 loff_t new_len
= *len
;
133 if ((*len
& blkmask
) == 0)
136 if (pos_out
+ *len
< i_size_read(inode_out
))
142 if (remap_flags
& REMAP_FILE_CAN_SHORTEN
) {
147 return (remap_flags
& REMAP_FILE_DEDUP
) ? -EBADE
: -EINVAL
;
150 /* Read a page's worth of file data into the page cache. */
151 static struct folio
*vfs_dedupe_get_folio(struct file
*file
, loff_t pos
)
153 return read_mapping_folio(file
->f_mapping
, pos
>> PAGE_SHIFT
, file
);
157 * Lock two folios, ensuring that we lock in offset order if the folios
158 * are from the same file.
160 static void vfs_lock_two_folios(struct folio
*folio1
, struct folio
*folio2
)
162 /* Always lock in order of increasing index. */
163 if (folio1
->index
> folio2
->index
)
164 swap(folio1
, folio2
);
167 if (folio1
!= folio2
)
171 /* Unlock two folios, being careful not to unlock the same folio twice. */
172 static void vfs_unlock_two_folios(struct folio
*folio1
, struct folio
*folio2
)
174 folio_unlock(folio1
);
175 if (folio1
!= folio2
)
176 folio_unlock(folio2
);
180 * Compare extents of two files to see if they are the same.
181 * Caller must have locked both inodes to prevent write races.
183 static int vfs_dedupe_file_range_compare(struct file
*src
, loff_t srcoff
,
184 struct file
*dest
, loff_t dstoff
,
185 loff_t len
, bool *is_same
)
191 struct folio
*src_folio
, *dst_folio
;
192 void *src_addr
, *dst_addr
;
193 loff_t cmp_len
= min(PAGE_SIZE
- offset_in_page(srcoff
),
194 PAGE_SIZE
- offset_in_page(dstoff
));
196 cmp_len
= min(cmp_len
, len
);
200 src_folio
= vfs_dedupe_get_folio(src
, srcoff
);
201 if (IS_ERR(src_folio
)) {
202 error
= PTR_ERR(src_folio
);
205 dst_folio
= vfs_dedupe_get_folio(dest
, dstoff
);
206 if (IS_ERR(dst_folio
)) {
207 error
= PTR_ERR(dst_folio
);
208 folio_put(src_folio
);
212 vfs_lock_two_folios(src_folio
, dst_folio
);
215 * Now that we've locked both folios, make sure they're still
216 * mapped to the file data we're interested in. If not,
217 * someone is invalidating pages on us and we lose.
219 if (!folio_test_uptodate(src_folio
) || !folio_test_uptodate(dst_folio
) ||
220 src_folio
->mapping
!= src
->f_mapping
||
221 dst_folio
->mapping
!= dest
->f_mapping
) {
226 src_addr
= kmap_local_folio(src_folio
,
227 offset_in_folio(src_folio
, srcoff
));
228 dst_addr
= kmap_local_folio(dst_folio
,
229 offset_in_folio(dst_folio
, dstoff
));
231 flush_dcache_folio(src_folio
);
232 flush_dcache_folio(dst_folio
);
234 if (memcmp(src_addr
, dst_addr
, cmp_len
))
237 kunmap_local(dst_addr
);
238 kunmap_local(src_addr
);
240 vfs_unlock_two_folios(src_folio
, dst_folio
);
241 folio_put(dst_folio
);
242 folio_put(src_folio
);
260 * Check that the two inodes are eligible for cloning, the ranges make
261 * sense, and then flush all dirty data. Caller must ensure that the
262 * inodes have been locked against any other modifications.
264 * If there's an error, then the usual negative error code is returned.
265 * Otherwise returns 0 with *len set to the request length.
268 __generic_remap_file_range_prep(struct file
*file_in
, loff_t pos_in
,
269 struct file
*file_out
, loff_t pos_out
,
270 loff_t
*len
, unsigned int remap_flags
,
271 const struct iomap_ops
*dax_read_ops
)
273 struct inode
*inode_in
= file_inode(file_in
);
274 struct inode
*inode_out
= file_inode(file_out
);
275 bool same_inode
= (inode_in
== inode_out
);
278 /* Don't touch certain kinds of inodes */
279 if (IS_IMMUTABLE(inode_out
))
282 if (IS_SWAPFILE(inode_in
) || IS_SWAPFILE(inode_out
))
285 /* Don't reflink dirs, pipes, sockets... */
286 if (S_ISDIR(inode_in
->i_mode
) || S_ISDIR(inode_out
->i_mode
))
288 if (!S_ISREG(inode_in
->i_mode
) || !S_ISREG(inode_out
->i_mode
))
291 /* Zero length dedupe exits immediately; reflink goes to EOF. */
293 loff_t isize
= i_size_read(inode_in
);
295 if ((remap_flags
& REMAP_FILE_DEDUP
) || pos_in
== isize
)
299 *len
= isize
- pos_in
;
304 /* Check that we don't violate system file offset limits. */
305 ret
= generic_remap_checks(file_in
, pos_in
, file_out
, pos_out
, len
,
307 if (ret
|| *len
== 0)
310 /* Wait for the completion of any pending IOs on both files */
311 inode_dio_wait(inode_in
);
313 inode_dio_wait(inode_out
);
315 ret
= filemap_write_and_wait_range(inode_in
->i_mapping
,
316 pos_in
, pos_in
+ *len
- 1);
320 ret
= filemap_write_and_wait_range(inode_out
->i_mapping
,
321 pos_out
, pos_out
+ *len
- 1);
326 * Check that the extents are the same.
328 if (remap_flags
& REMAP_FILE_DEDUP
) {
329 bool is_same
= false;
331 if (!IS_DAX(inode_in
))
332 ret
= vfs_dedupe_file_range_compare(file_in
, pos_in
,
333 file_out
, pos_out
, *len
, &is_same
);
334 else if (dax_read_ops
)
335 ret
= dax_dedupe_file_range_compare(inode_in
, pos_in
,
336 inode_out
, pos_out
, *len
, &is_same
,
346 ret
= generic_remap_check_len(inode_in
, inode_out
, pos_out
, len
,
348 if (ret
|| *len
== 0)
351 /* If can't alter the file contents, we're done. */
352 if (!(remap_flags
& REMAP_FILE_DEDUP
))
353 ret
= file_modified(file_out
);
358 int generic_remap_file_range_prep(struct file
*file_in
, loff_t pos_in
,
359 struct file
*file_out
, loff_t pos_out
,
360 loff_t
*len
, unsigned int remap_flags
)
362 return __generic_remap_file_range_prep(file_in
, pos_in
, file_out
,
363 pos_out
, len
, remap_flags
, NULL
);
365 EXPORT_SYMBOL(generic_remap_file_range_prep
);
367 loff_t
do_clone_file_range(struct file
*file_in
, loff_t pos_in
,
368 struct file
*file_out
, loff_t pos_out
,
369 loff_t len
, unsigned int remap_flags
)
373 WARN_ON_ONCE(remap_flags
& REMAP_FILE_DEDUP
);
375 if (file_inode(file_in
)->i_sb
!= file_inode(file_out
)->i_sb
)
378 ret
= generic_file_rw_checks(file_in
, file_out
);
382 if (!file_in
->f_op
->remap_file_range
)
385 ret
= remap_verify_area(file_in
, pos_in
, len
, false);
389 ret
= remap_verify_area(file_out
, pos_out
, len
, true);
393 ret
= file_in
->f_op
->remap_file_range(file_in
, pos_in
,
394 file_out
, pos_out
, len
, remap_flags
);
398 fsnotify_access(file_in
);
399 fsnotify_modify(file_out
);
402 EXPORT_SYMBOL(do_clone_file_range
);
404 loff_t
vfs_clone_file_range(struct file
*file_in
, loff_t pos_in
,
405 struct file
*file_out
, loff_t pos_out
,
406 loff_t len
, unsigned int remap_flags
)
410 file_start_write(file_out
);
411 ret
= do_clone_file_range(file_in
, pos_in
, file_out
, pos_out
, len
,
413 file_end_write(file_out
);
417 EXPORT_SYMBOL(vfs_clone_file_range
);
419 /* Check whether we are allowed to dedupe the destination file */
420 static bool allow_file_dedupe(struct file
*file
)
422 struct mnt_idmap
*idmap
= file_mnt_idmap(file
);
423 struct inode
*inode
= file_inode(file
);
425 if (capable(CAP_SYS_ADMIN
))
427 if (file
->f_mode
& FMODE_WRITE
)
429 if (vfsuid_eq_kuid(i_uid_into_vfsuid(idmap
, inode
), current_fsuid()))
431 if (!inode_permission(idmap
, inode
, MAY_WRITE
))
436 loff_t
vfs_dedupe_file_range_one(struct file
*src_file
, loff_t src_pos
,
437 struct file
*dst_file
, loff_t dst_pos
,
438 loff_t len
, unsigned int remap_flags
)
442 WARN_ON_ONCE(remap_flags
& ~(REMAP_FILE_DEDUP
|
443 REMAP_FILE_CAN_SHORTEN
));
445 ret
= mnt_want_write_file(dst_file
);
450 * This is redundant if called from vfs_dedupe_file_range(), but other
451 * callers need it and it's not performance sesitive...
453 ret
= remap_verify_area(src_file
, src_pos
, len
, false);
457 ret
= remap_verify_area(dst_file
, dst_pos
, len
, true);
462 if (!allow_file_dedupe(dst_file
))
466 if (file_inode(src_file
)->i_sb
!= file_inode(dst_file
)->i_sb
)
470 if (S_ISDIR(file_inode(dst_file
)->i_mode
))
474 if (!dst_file
->f_op
->remap_file_range
)
482 ret
= dst_file
->f_op
->remap_file_range(src_file
, src_pos
, dst_file
,
483 dst_pos
, len
, remap_flags
| REMAP_FILE_DEDUP
);
485 mnt_drop_write_file(dst_file
);
489 EXPORT_SYMBOL(vfs_dedupe_file_range_one
);
491 int vfs_dedupe_file_range(struct file
*file
, struct file_dedupe_range
*same
)
493 struct file_dedupe_range_info
*info
;
494 struct inode
*src
= file_inode(file
);
499 u16 count
= same
->dest_count
;
502 if (!(file
->f_mode
& FMODE_READ
))
505 if (same
->reserved1
|| same
->reserved2
)
508 off
= same
->src_offset
;
509 len
= same
->src_length
;
511 if (S_ISDIR(src
->i_mode
))
514 if (!S_ISREG(src
->i_mode
))
517 if (!file
->f_op
->remap_file_range
)
520 ret
= remap_verify_area(file
, off
, len
, false);
525 if (off
+ len
> i_size_read(src
))
528 /* Arbitrary 1G limit on a single dedupe request, can be raised. */
529 len
= min_t(u64
, len
, 1 << 30);
531 /* pre-format output fields to sane values */
532 for (i
= 0; i
< count
; i
++) {
533 same
->info
[i
].bytes_deduped
= 0ULL;
534 same
->info
[i
].status
= FILE_DEDUPE_RANGE_SAME
;
537 for (i
= 0, info
= same
->info
; i
< count
; i
++, info
++) {
538 struct fd dst_fd
= fdget(info
->dest_fd
);
539 struct file
*dst_file
= dst_fd
.file
;
542 info
->status
= -EBADF
;
546 if (info
->reserved
) {
547 info
->status
= -EINVAL
;
551 deduped
= vfs_dedupe_file_range_one(file
, off
, dst_file
,
552 info
->dest_offset
, len
,
553 REMAP_FILE_CAN_SHORTEN
);
554 if (deduped
== -EBADE
)
555 info
->status
= FILE_DEDUPE_RANGE_DIFFERS
;
556 else if (deduped
< 0)
557 info
->status
= deduped
;
559 info
->bytes_deduped
= len
;
564 if (fatal_signal_pending(current
))
569 EXPORT_SYMBOL(vfs_dedupe_file_range
);