fs/read_write.c

   1 /*
   2  *  linux/fs/read_write.c
   3  *
   4  *  Copyright (C) 1991, 1992  Linus Torvalds
   5  */
   6
   7 #include <linux/slab.h>
   8 #include <linux/stat.h>
   9 #include <linux/fcntl.h>
  10 #include <linux/file.h>
  11 #include <linux/uio.h>
  12 #include <linux/smp_lock.h>
  13 #include <linux/fsnotify.h>
  14 #include <linux/security.h>
  15 #include <linux/module.h>
  16 #include <linux/syscalls.h>
  17 #include <linux/pagemap.h>
  18 #include <linux/splice.h>
  19 #include "read_write.h"
  20
  21 #include <asm/uaccess.h>
  22 #include <asm/unistd.h>
  23
  24 const struct file_operations generic_ro_fops = {
  25         .llseek         = generic_file_llseek,
  26         .read           = do_sync_read,
  27         .aio_read       = generic_file_aio_read,
  28         .mmap           = generic_file_readonly_mmap,
  29         .splice_read    = generic_file_splice_read,
  30 };
  31
  32 EXPORT_SYMBOL(generic_ro_fops);
  33
  34 /**
  35  * generic_file_llseek_unlocked - lockless generic llseek implementation
  36  * @file:       file structure to seek on
  37  * @offset:     file offset to seek to
  38  * @origin:     type of seek
  39  *
  40  * Updates the file offset to the value specified by @offset and @origin.
  41  * Locking must be provided by the caller.
  42  */
  43 loff_t
  44 generic_file_llseek_unlocked(struct file *file, loff_t offset, int origin)
  45 {
  46         struct inode *inode = file->f_mapping->host;
  47
  48         switch (origin) {
  49         case SEEK_END:
  50                 offset += inode->i_size;
  51                 break;
  52         case SEEK_CUR:
  53                 /*
  54                  * Here we special-case the lseek(fd, 0, SEEK_CUR)
  55                  * position-querying operation.  Avoid rewriting the "same"
  56                  * f_pos value back to the file because a concurrent read(),
  57                  * write() or lseek() might have altered it
  58                  */
  59                 if (offset == 0)
  60                         return file->f_pos;
  61                 offset += file->f_pos;
  62                 break;
  63         }
  64
  65         if (offset < 0 || offset > inode->i_sb->s_maxbytes)
  66                 return -EINVAL;
  67
  68         /* Special lock needed here? */
  69         if (offset != file->f_pos) {
  70                 file->f_pos = offset;
  71                 file->f_version = 0;
  72         }
  73
  74         return offset;
  75 }
  76 EXPORT_SYMBOL(generic_file_llseek_unlocked);
  77
  78 /**
  79  * generic_file_llseek - generic llseek implementation for regular files
  80  * @file:       file structure to seek on
  81  * @offset:     file offset to seek to
  82  * @origin:     type of seek
  83  *
  84  * This is a generic implemenation of ->llseek useable for all normal local
  85  * filesystems.  It just updates the file offset to the value specified by
  86  * @offset and @origin under i_mutex.
  87  */
  88 loff_t generic_file_llseek(struct file *file, loff_t offset, int origin)
  89 {
  90         loff_t rval;
  91
  92         mutex_lock(&file->f_dentry->d_inode->i_mutex);
  93         rval = generic_file_llseek_unlocked(file, offset, origin);
  94         mutex_unlock(&file->f_dentry->d_inode->i_mutex);
  95
  96         return rval;
  97 }
  98 EXPORT_SYMBOL(generic_file_llseek);
  99
 100 loff_t no_llseek(struct file *file, loff_t offset, int origin)
 101 {
 102         return -ESPIPE;
 103 }
 104 EXPORT_SYMBOL(no_llseek);
 105
 106 loff_t default_llseek(struct file *file, loff_t offset, int origin)
 107 {
 108         loff_t retval;
 109
 110         lock_kernel();
 111         switch (origin) {
 112                 case SEEK_END:
 113                         offset += i_size_read(file->f_path.dentry->d_inode);
 114                         break;
 115                 case SEEK_CUR:
 116                         if (offset == 0) {
 117                                 retval = file->f_pos;
 118                                 goto out;
 119                         }
 120                         offset += file->f_pos;
 121         }
 122         retval = -EINVAL;
 123         if (offset >= 0) {
 124                 if (offset != file->f_pos) {
 125                         file->f_pos = offset;
 126                         file->f_version = 0;
 127                 }
 128                 retval = offset;
 129         }
 130 out:
 131         unlock_kernel();
 132         return retval;
 133 }
 134 EXPORT_SYMBOL(default_llseek);
 135
 136 loff_t vfs_llseek(struct file *file, loff_t offset, int origin)
 137 {
 138         loff_t (*fn)(struct file *, loff_t, int);
 139
 140         fn = no_llseek;
 141         if (file->f_mode & FMODE_LSEEK) {
 142                 fn = default_llseek;
 143                 if (file->f_op && file->f_op->llseek)
 144                         fn = file->f_op->llseek;
 145         }
 146         return fn(file, offset, origin);
 147 }
 148 EXPORT_SYMBOL(vfs_llseek);
 149
 150 asmlinkage long sys_lseek(unsigned int fd, off_t offset, unsigned int origin)
 151 {
 152         off_t retval;
 153         struct file * file;
 154         int fput_needed;
 155
 156         retval = -EBADF;
 157         file = fget_light(fd, &fput_needed);
 158         if (!file)
 159                 goto bad;
 160
 161         retval = -EINVAL;
 162         if (origin <= SEEK_MAX) {
 163                 loff_t res = vfs_llseek(file, offset, origin);
 164                 retval = res;
 165                 if (res != (loff_t)retval)
 166                         retval = -EOVERFLOW;    /* LFS: should only happen on 32 bit platforms */
 167         }
 168         fput_light(file, fput_needed);
 169 bad:
 170         return retval;
 171 }
 172
 173 #ifdef __ARCH_WANT_SYS_LLSEEK
 174 asmlinkage long sys_llseek(unsigned int fd, unsigned long offset_high,
 175                            unsigned long offset_low, loff_t __user * result,
 176                            unsigned int origin)
 177 {
 178         int retval;
 179         struct file * file;
 180         loff_t offset;
 181         int fput_needed;
 182
 183         retval = -EBADF;
 184         file = fget_light(fd, &fput_needed);
 185         if (!file)
 186                 goto bad;
 187
 188         retval = -EINVAL;
 189         if (origin > SEEK_MAX)
 190                 goto out_putf;
 191
 192         offset = vfs_llseek(file, ((loff_t) offset_high << 32) | offset_low,
 193                         origin);
 194
 195         retval = (int)offset;
 196         if (offset >= 0) {
 197                 retval = -EFAULT;
 198                 if (!copy_to_user(result, &offset, sizeof(offset)))
 199                         retval = 0;
 200         }
 201 out_putf:
 202         fput_light(file, fput_needed);
 203 bad:
 204         return retval;
 205 }
 206 #endif
 207
 208 /*
 209  * rw_verify_area doesn't like huge counts. We limit
 210  * them to something that fits in "int" so that others
 211  * won't have to do range checks all the time.
 212  */
 213 #define MAX_RW_COUNT (INT_MAX & PAGE_CACHE_MASK)
 214
 215 int rw_verify_area(int read_write, struct file *file, loff_t *ppos, size_t count)
 216 {
 217         struct inode *inode;
 218         loff_t pos;
 219         int retval = -EINVAL;
 220
 221         inode = file->f_path.dentry->d_inode;
 222         if (unlikely((ssize_t) count < 0))
 223                 return retval;
 224         pos = *ppos;
 225         if (unlikely((pos < 0) || (loff_t) (pos + count) < 0))
 226                 return retval;
 227
 228         if (unlikely(inode->i_flock && mandatory_lock(inode))) {
 229                 retval = locks_mandatory_area(
 230                         read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
 231                         inode, file, pos, count);
 232                 if (retval < 0)
 233                         return retval;
 234         }
 235         retval = security_file_permission(file,
 236                                 read_write == READ ? MAY_READ : MAY_WRITE);
 237         if (retval)
 238                 return retval;
 239         return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
 240 }
 241
 242 static void wait_on_retry_sync_kiocb(struct kiocb *iocb)
 243 {
 244         set_current_state(TASK_UNINTERRUPTIBLE);
 245         if (!kiocbIsKicked(iocb))
 246                 schedule();
 247         else
 248                 kiocbClearKicked(iocb);
 249         __set_current_state(TASK_RUNNING);
 250 }
 251
 252 ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
 253 {
 254         struct iovec iov = { .iov_base = buf, .iov_len = len };
 255         struct kiocb kiocb;
 256         ssize_t ret;
 257
 258         init_sync_kiocb(&kiocb, filp);
 259         kiocb.ki_pos = *ppos;
 260         kiocb.ki_left = len;
 261
 262         for (;;) {
 263                 ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos);
 264                 if (ret != -EIOCBRETRY)
 265                         break;
 266                 wait_on_retry_sync_kiocb(&kiocb);
 267         }
 268
 269         if (-EIOCBQUEUED == ret)
 270                 ret = wait_on_sync_kiocb(&kiocb);
 271         *ppos = kiocb.ki_pos;
 272         return ret;
 273 }
 274
 275 EXPORT_SYMBOL(do_sync_read);
 276
 277 ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
 278 {
 279         ssize_t ret;
 280
 281         if (!(file->f_mode & FMODE_READ))
 282                 return -EBADF;
 283         if (!file->f_op || (!file->f_op->read && !file->f_op->aio_read))
 284                 return -EINVAL;
 285         if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
 286                 return -EFAULT;
 287
 288         ret = rw_verify_area(READ, file, pos, count);
 289         if (ret >= 0) {
 290                 count = ret;
 291                 if (file->f_op->read)
 292                         ret = file->f_op->read(file, buf, count, pos);
 293                 else
 294                         ret = do_sync_read(file, buf, count, pos);
 295                 if (ret > 0) {
 296                         fsnotify_access(file->f_path.dentry);
 297                         add_rchar(current, ret);
 298                 }
 299                 inc_syscr(current);
 300         }
 301
 302         return ret;
 303 }
 304
 305 EXPORT_SYMBOL(vfs_read);
 306
 307 ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
 308 {
 309         struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
 310         struct kiocb kiocb;
 311         ssize_t ret;
 312
 313         init_sync_kiocb(&kiocb, filp);
 314         kiocb.ki_pos = *ppos;
 315         kiocb.ki_left = len;
 316
 317         for (;;) {
 318                 ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos);
 319                 if (ret != -EIOCBRETRY)
 320                         break;
 321                 wait_on_retry_sync_kiocb(&kiocb);
 322         }
 323
 324         if (-EIOCBQUEUED == ret)
 325                 ret = wait_on_sync_kiocb(&kiocb);
 326         *ppos = kiocb.ki_pos;
 327         return ret;
 328 }
 329
 330 EXPORT_SYMBOL(do_sync_write);
 331
 332 ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
 333 {
 334         ssize_t ret;
 335
 336         if (!(file->f_mode & FMODE_WRITE))
 337                 return -EBADF;
 338         if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
 339                 return -EINVAL;
 340         if (unlikely(!access_ok(VERIFY_READ, buf, count)))
 341                 return -EFAULT;
 342
 343         ret = rw_verify_area(WRITE, file, pos, count);
 344         if (ret >= 0) {
 345                 count = ret;
 346                 if (file->f_op->write)
 347                         ret = file->f_op->write(file, buf, count, pos);
 348                 else
 349                         ret = do_sync_write(file, buf, count, pos);
 350                 if (ret > 0) {
 351                         fsnotify_modify(file->f_path.dentry);
 352                         add_wchar(current, ret);
 353                 }
 354                 inc_syscw(current);
 355         }
 356
 357         return ret;
 358 }
 359
 360 EXPORT_SYMBOL(vfs_write);
 361
 362 static inline loff_t file_pos_read(struct file *file)
 363 {
 364         return file->f_pos;
 365 }
 366
 367 static inline void file_pos_write(struct file *file, loff_t pos)
 368 {
 369         file->f_pos = pos;
 370 }
 371
 372 asmlinkage long sys_read(unsigned int fd, char __user * buf, size_t count)
 373 {
 374         struct file *file;
 375         ssize_t ret = -EBADF;
 376         int fput_needed;
 377
 378         file = fget_light(fd, &fput_needed);
 379         if (file) {
 380                 loff_t pos = file_pos_read(file);
 381                 ret = vfs_read(file, buf, count, &pos);
 382                 file_pos_write(file, pos);
 383                 fput_light(file, fput_needed);
 384         }
 385
 386         return ret;
 387 }
 388
 389 asmlinkage long sys_write(unsigned int fd, const char __user * buf, size_t count)
 390 {
 391         struct file *file;
 392         ssize_t ret = -EBADF;
 393         int fput_needed;
 394
 395         file = fget_light(fd, &fput_needed);
 396         if (file) {
 397                 loff_t pos = file_pos_read(file);
 398                 ret = vfs_write(file, buf, count, &pos);
 399                 file_pos_write(file, pos);
 400                 fput_light(file, fput_needed);
 401         }
 402
 403         return ret;
 404 }
 405
 406 SYSCALL_DEFINE(pread64)(unsigned int fd, char __user *buf,
 407                         size_t count, loff_t pos)
 408 {
 409         struct file *file;
 410         ssize_t ret = -EBADF;
 411         int fput_needed;
 412
 413         if (pos < 0)
 414                 return -EINVAL;
 415
 416         file = fget_light(fd, &fput_needed);
 417         if (file) {
 418                 ret = -ESPIPE;
 419                 if (file->f_mode & FMODE_PREAD)
 420                         ret = vfs_read(file, buf, count, &pos);
 421                 fput_light(file, fput_needed);
 422         }
 423
 424         return ret;
 425 }
 426 #ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
 427 asmlinkage long SyS_pread64(long fd, long buf, long count, loff_t pos)
 428 {
 429         return SYSC_pread64((unsigned int) fd, (char __user *) buf,
 430                             (size_t) count, pos);
 431 }
 432 SYSCALL_ALIAS(sys_pread64, SyS_pread64);
 433 #endif
 434
 435 SYSCALL_DEFINE(pwrite64)(unsigned int fd, const char __user *buf,
 436                          size_t count, loff_t pos)
 437 {
 438         struct file *file;
 439         ssize_t ret = -EBADF;
 440         int fput_needed;
 441
 442         if (pos < 0)
 443                 return -EINVAL;
 444
 445         file = fget_light(fd, &fput_needed);
 446         if (file) {
 447                 ret = -ESPIPE;
 448                 if (file->f_mode & FMODE_PWRITE)
 449                         ret = vfs_write(file, buf, count, &pos);
 450                 fput_light(file, fput_needed);
 451         }
 452
 453         return ret;
 454 }
 455 #ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
 456 asmlinkage long SyS_pwrite64(long fd, long buf, long count, loff_t pos)
 457 {
 458         return SYSC_pwrite64((unsigned int) fd, (const char __user *) buf,
 459                              (size_t) count, pos);
 460 }
 461 SYSCALL_ALIAS(sys_pwrite64, SyS_pwrite64);
 462 #endif
 463
 464 /*
 465  * Reduce an iovec's length in-place.  Return the resulting number of segments
 466  */
 467 unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
 468 {
 469         unsigned long seg = 0;
 470         size_t len = 0;
 471
 472         while (seg < nr_segs) {
 473                 seg++;
 474                 if (len + iov->iov_len >= to) {
 475                         iov->iov_len = to - len;
 476                         break;
 477                 }
 478                 len += iov->iov_len;
 479                 iov++;
 480         }
 481         return seg;
 482 }
 483 EXPORT_SYMBOL(iov_shorten);
 484
 485 ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
 486                 unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
 487 {
 488         struct kiocb kiocb;
 489         ssize_t ret;
 490
 491         init_sync_kiocb(&kiocb, filp);
 492         kiocb.ki_pos = *ppos;
 493         kiocb.ki_left = len;
 494         kiocb.ki_nbytes = len;
 495
 496         for (;;) {
 497                 ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos);
 498                 if (ret != -EIOCBRETRY)
 499                         break;
 500                 wait_on_retry_sync_kiocb(&kiocb);
 501         }
 502
 503         if (ret == -EIOCBQUEUED)
 504                 ret = wait_on_sync_kiocb(&kiocb);
 505         *ppos = kiocb.ki_pos;
 506         return ret;
 507 }
 508
 509 /* Do it by hand, with file-ops */
 510 ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
 511                 unsigned long nr_segs, loff_t *ppos, io_fn_t fn)
 512 {
 513         struct iovec *vector = iov;
 514         ssize_t ret = 0;
 515
 516         while (nr_segs > 0) {
 517                 void __user *base;
 518                 size_t len;
 519                 ssize_t nr;
 520
 521                 base = vector->iov_base;
 522                 len = vector->iov_len;
 523                 vector++;
 524                 nr_segs--;
 525
 526                 nr = fn(filp, base, len, ppos);
 527
 528                 if (nr < 0) {
 529                         if (!ret)
 530                                 ret = nr;
 531                         break;
 532                 }
 533                 ret += nr;
 534                 if (nr != len)
 535                         break;
 536         }
 537
 538         return ret;
 539 }
 540
 541 /* A write operation does a read from user space and vice versa */
 542 #define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
 543
 544 ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
 545                               unsigned long nr_segs, unsigned long fast_segs,
 546                               struct iovec *fast_pointer,
 547                               struct iovec **ret_pointer)
 548   {
 549         unsigned long seg;
 550         ssize_t ret;
 551         struct iovec *iov = fast_pointer;
 552
 553         /*
 554          * SuS says "The readv() function *may* fail if the iovcnt argument
 555          * was less than or equal to 0, or greater than {IOV_MAX}.  Linux has
 556          * traditionally returned zero for zero segments, so...
 557          */
 558         if (nr_segs == 0) {
 559                 ret = 0;
 560                 goto out;
 561         }
 562
 563         /*
 564          * First get the "struct iovec" from user memory and
 565          * verify all the pointers
 566          */
 567         if (nr_segs > UIO_MAXIOV) {
 568                 ret = -EINVAL;
 569                 goto out;
 570         }
 571         if (nr_segs > fast_segs) {
 572                 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
 573                 if (iov == NULL) {
 574                         ret = -ENOMEM;
 575                         goto out;
 576                 }
 577         }
 578         if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
 579                 ret = -EFAULT;
 580                 goto out;
 581         }
 582
 583         /*
 584          * According to the Single Unix Specification we should return EINVAL
 585          * if an element length is < 0 when cast to ssize_t or if the
 586          * total length would overflow the ssize_t return value of the
 587          * system call.
 588          */
 589         ret = 0;
 590         for (seg = 0; seg < nr_segs; seg++) {
 591                 void __user *buf = iov[seg].iov_base;
 592                 ssize_t len = (ssize_t)iov[seg].iov_len;
 593
 594                 /* see if we we're about to use an invalid len or if
 595                  * it's about to overflow ssize_t */
 596                 if (len < 0 || (ret + len < ret)) {
 597                         ret = -EINVAL;
 598                         goto out;
 599                 }
 600                 if (unlikely(!access_ok(vrfy_dir(type), buf, len))) {
 601                         ret = -EFAULT;
 602                         goto out;
 603                 }
 604
 605                 ret += len;
 606         }
 607 out:
 608         *ret_pointer = iov;
 609         return ret;
 610 }
 611
 612 static ssize_t do_readv_writev(int type, struct file *file,
 613                                const struct iovec __user * uvector,
 614                                unsigned long nr_segs, loff_t *pos)
 615 {
 616         size_t tot_len;
 617         struct iovec iovstack[UIO_FASTIOV];
 618         struct iovec *iov = iovstack;
 619         ssize_t ret;
 620         io_fn_t fn;
 621         iov_fn_t fnv;
 622
 623         if (!file->f_op) {
 624                 ret = -EINVAL;
 625                 goto out;
 626         }
 627
 628         ret = rw_copy_check_uvector(type, uvector, nr_segs,
 629                         ARRAY_SIZE(iovstack), iovstack, &iov);
 630         if (ret <= 0)
 631                 goto out;
 632
 633         tot_len = ret;
 634         ret = rw_verify_area(type, file, pos, tot_len);
 635         if (ret < 0)
 636                 goto out;
 637
 638         fnv = NULL;
 639         if (type == READ) {
 640                 fn = file->f_op->read;
 641                 fnv = file->f_op->aio_read;
 642         } else {
 643                 fn = (io_fn_t)file->f_op->write;
 644                 fnv = file->f_op->aio_write;
 645         }
 646
 647         if (fnv)
 648                 ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
 649                                                 pos, fnv);
 650         else
 651                 ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
 652
 653 out:
 654         if (iov != iovstack)
 655                 kfree(iov);
 656         if ((ret + (type == READ)) > 0) {
 657                 if (type == READ)
 658                         fsnotify_access(file->f_path.dentry);
 659                 else
 660                         fsnotify_modify(file->f_path.dentry);
 661         }
 662         return ret;
 663 }
 664
 665 ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
 666                   unsigned long vlen, loff_t *pos)
 667 {
 668         if (!(file->f_mode & FMODE_READ))
 669                 return -EBADF;
 670         if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read))
 671                 return -EINVAL;
 672
 673         return do_readv_writev(READ, file, vec, vlen, pos);
 674 }
 675
 676 EXPORT_SYMBOL(vfs_readv);
 677
 678 ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
 679                    unsigned long vlen, loff_t *pos)
 680 {
 681         if (!(file->f_mode & FMODE_WRITE))
 682                 return -EBADF;
 683         if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write))
 684                 return -EINVAL;
 685
 686         return do_readv_writev(WRITE, file, vec, vlen, pos);
 687 }
 688
 689 EXPORT_SYMBOL(vfs_writev);
 690
 691 asmlinkage long
 692 sys_readv(unsigned long fd, const struct iovec __user *vec, unsigned long vlen)
 693 {
 694         struct file *file;
 695         ssize_t ret = -EBADF;
 696         int fput_needed;
 697
 698         file = fget_light(fd, &fput_needed);
 699         if (file) {
 700                 loff_t pos = file_pos_read(file);
 701                 ret = vfs_readv(file, vec, vlen, &pos);
 702                 file_pos_write(file, pos);
 703                 fput_light(file, fput_needed);
 704         }
 705
 706         if (ret > 0)
 707                 add_rchar(current, ret);
 708         inc_syscr(current);
 709         return ret;
 710 }
 711
 712 asmlinkage long
 713 sys_writev(unsigned long fd, const struct iovec __user *vec, unsigned long vlen)
 714 {
 715         struct file *file;
 716         ssize_t ret = -EBADF;
 717         int fput_needed;
 718
 719         file = fget_light(fd, &fput_needed);
 720         if (file) {
 721                 loff_t pos = file_pos_read(file);
 722                 ret = vfs_writev(file, vec, vlen, &pos);
 723                 file_pos_write(file, pos);
 724                 fput_light(file, fput_needed);
 725         }
 726
 727         if (ret > 0)
 728                 add_wchar(current, ret);
 729         inc_syscw(current);
 730         return ret;
 731 }
 732
 733 static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
 734                            size_t count, loff_t max)
 735 {
 736         struct file * in_file, * out_file;
 737         struct inode * in_inode, * out_inode;
 738         loff_t pos;
 739         ssize_t retval;
 740         int fput_needed_in, fput_needed_out, fl;
 741
 742         /*
 743          * Get input file, and verify that it is ok..
 744          */
 745         retval = -EBADF;
 746         in_file = fget_light(in_fd, &fput_needed_in);
 747         if (!in_file)
 748                 goto out;
 749         if (!(in_file->f_mode & FMODE_READ))
 750                 goto fput_in;
 751         retval = -EINVAL;
 752         in_inode = in_file->f_path.dentry->d_inode;
 753         if (!in_inode)
 754                 goto fput_in;
 755         if (!in_file->f_op || !in_file->f_op->splice_read)
 756                 goto fput_in;
 757         retval = -ESPIPE;
 758         if (!ppos)
 759                 ppos = &in_file->f_pos;
 760         else
 761                 if (!(in_file->f_mode & FMODE_PREAD))
 762                         goto fput_in;
 763         retval = rw_verify_area(READ, in_file, ppos, count);
 764         if (retval < 0)
 765                 goto fput_in;
 766         count = retval;
 767
 768         /*
 769          * Get output file, and verify that it is ok..
 770          */
 771         retval = -EBADF;
 772         out_file = fget_light(out_fd, &fput_needed_out);
 773         if (!out_file)
 774                 goto fput_in;
 775         if (!(out_file->f_mode & FMODE_WRITE))
 776                 goto fput_out;
 777         retval = -EINVAL;
 778         if (!out_file->f_op || !out_file->f_op->sendpage)
 779                 goto fput_out;
 780         out_inode = out_file->f_path.dentry->d_inode;
 781         retval = rw_verify_area(WRITE, out_file, &out_file->f_pos, count);
 782         if (retval < 0)
 783                 goto fput_out;
 784         count = retval;
 785
 786         if (!max)
 787                 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
 788
 789         pos = *ppos;
 790         retval = -EINVAL;
 791         if (unlikely(pos < 0))
 792                 goto fput_out;
 793         if (unlikely(pos + count > max)) {
 794                 retval = -EOVERFLOW;
 795                 if (pos >= max)
 796                         goto fput_out;
 797                 count = max - pos;
 798         }
 799
 800         fl = 0;
 801 #if 0
 802         /*
 803          * We need to debate whether we can enable this or not. The
 804          * man page documents EAGAIN return for the output at least,
 805          * and the application is arguably buggy if it doesn't expect
 806          * EAGAIN on a non-blocking file descriptor.
 807          */
 808         if (in_file->f_flags & O_NONBLOCK)
 809                 fl = SPLICE_F_NONBLOCK;
 810 #endif
 811         retval = do_splice_direct(in_file, ppos, out_file, count, fl);
 812
 813         if (retval > 0) {
 814                 add_rchar(current, retval);
 815                 add_wchar(current, retval);
 816         }
 817
 818         inc_syscr(current);
 819         inc_syscw(current);
 820         if (*ppos > max)
 821                 retval = -EOVERFLOW;
 822
 823 fput_out:
 824         fput_light(out_file, fput_needed_out);
 825 fput_in:
 826         fput_light(in_file, fput_needed_in);
 827 out:
 828         return retval;
 829 }
 830
 831 SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
 832 {
 833         loff_t pos;
 834         off_t off;
 835         ssize_t ret;
 836
 837         if (offset) {
 838                 if (unlikely(get_user(off, offset)))
 839                         return -EFAULT;
 840                 pos = off;
 841                 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
 842                 if (unlikely(put_user(pos, offset)))
 843                         return -EFAULT;
 844                 return ret;
 845         }
 846
 847         return do_sendfile(out_fd, in_fd, NULL, count, 0);
 848 }
 849
 850 SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
 851 {
 852         loff_t pos;
 853         ssize_t ret;
 854
 855         if (offset) {
 856                 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
 857                         return -EFAULT;
 858                 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
 859                 if (unlikely(put_user(pos, offset)))
 860                         return -EFAULT;
 861                 return ret;
 862         }
 863
 864         return do_sendfile(out_fd, in_fd, NULL, count, 0);
 865 }