ext2fs: Add Direct I/O support to the ext2fs library

This adds the basic support for Direct I/O to unix_io.c, and adds a
new flag EXT_FLAG_DIRECT_IO which can be passed to ext2fs_open() or
ext2fs_open2() to request Direct I/O support.

Note that device mapper devices in Linux don't support Direct I/O, and
in some circumstances using Direct I/O can actually make performance
*worse*!

Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>

diff --git a/lib/ext2fs/closefs.c b/lib/ext2fs/closefs.c
index db6b8a6a82e759d4bf0845777ce382284ff504d3..99430d7bfe1000f30237bef52208054000bbf7fa 100644 (file)
--- a/lib/ext2fs/closefs.c
+++ b/lib/ext2fs/closefs.c
@@ -145,6 +145,7 @@ static errcode_t write_primary_superblock(ext2_filsys fs,
        errcode_t       retval;
 
        if (!fs->io->manager->write_byte || !fs->orig_super) {
+       fallback:
                io_channel_set_blksize(fs->io, SUPERBLOCK_OFFSET);
                retval = io_channel_write_blk(fs->io, 1, -SUPERBLOCK_SIZE,
                                              super);
@@ -170,6 +171,8 @@ static errcode_t write_primary_superblock(ext2_filsys fs,
                retval = io_channel_write_byte(fs->io,
                               SUPERBLOCK_OFFSET + (2 * write_idx), size,
                                               new_super + write_idx);
+               if (retval == EXT2_ET_UNIMPLEMENTED)
+                       goto fallback;
                if (retval)
                        return retval;
        }

diff --git a/lib/ext2fs/ext2_io.h b/lib/ext2fs/ext2_io.h
index 5bebbfb3b82b972ab488956b9d5e8adddf5b039c..b917559cb2abf7f94cd08216ea6e577090b81321 100644 (file)
--- a/lib/ext2fs/ext2_io.h
+++ b/lib/ext2fs/ext2_io.h
@@ -88,6 +88,7 @@ struct struct_io_manager {
 
 #define IO_FLAG_RW             0x0001
 #define IO_FLAG_EXCLUSIVE      0x0002
+#define IO_FLAG_DIRECT_IO      0x0004
 
 /*
  * Convenience functions....

diff --git a/lib/ext2fs/ext2fs.h b/lib/ext2fs/ext2fs.h
index bc0d7cef9bdb7b5d2611d9f4d893d45b012771c4..8dac89a9b1cb07ffffed267cf0ee97ed75c7550f 100644 (file)
--- a/lib/ext2fs/ext2fs.h
+++ b/lib/ext2fs/ext2fs.h
@@ -173,6 +173,7 @@ typedef struct ext2_file *ext2_file_t;
 #define EXT2_FLAG_EXCLUSIVE            0x4000
 #define EXT2_FLAG_SOFTSUPP_FEATURES    0x8000
 #define EXT2_FLAG_NOFREE_ON_ERROR      0x10000
+#define EXT2_FLAG_DIRECT_IO            0x20000
 
 /*
  * Special flag in the ext2 inode i_flag field that means that this is

diff --git a/lib/ext2fs/openfs.c b/lib/ext2fs/openfs.c
index a6b08c358946a1114f886af14f4573155f26951e..52157d850165d09c24c646efddb16bd720e2e760 100644 (file)
--- a/lib/ext2fs/openfs.c
+++ b/lib/ext2fs/openfs.c
@@ -121,6 +121,8 @@ errcode_t ext2fs_open2(const char *name, const char *io_options,
                io_flags |= IO_FLAG_RW;
        if (flags & EXT2_FLAG_EXCLUSIVE)
                io_flags |= IO_FLAG_EXCLUSIVE;
+       if (flags & EXT2_FLAG_DIRECT_IO)
+               io_flags |= IO_FLAG_DIRECT_IO;
        retval = manager->open(fs->device_name, io_flags, &fs->io);
        if (retval)
                goto cleanup;

diff --git a/lib/ext2fs/unix_io.c b/lib/ext2fs/unix_io.c
index 78a1d1cbab8cf2088d3ee1b8679d0f3851b73cc3..1df1fddc249d32a849ec2aaf44ab37f644d5409b 100644 (file)
--- a/lib/ext2fs/unix_io.c
+++ b/lib/ext2fs/unix_io.c
@@ -17,6 +17,7 @@
 
 #define _LARGEFILE_SOURCE
 #define _LARGEFILE64_SOURCE
+#define _GNU_SOURCE
 
 #include <stdio.h>
 #include <string.h>
@@ -47,10 +48,16 @@
 #include <sys/resource.h>
 #endif
 
-#if defined(__linux__) && defined(_IO) && !defined(BLKGETSIZE)
+#if defined(__linux__) && defined(_IO) && !defined(BLKROGET)
 #define BLKROGET   _IO(0x12, 94) /* Get read-only status (0 = read_write).  */
 #endif
 
+#if defined(__linux__) && defined(_IO) && !defined(BLKSSZGET)
+#define BLKSSZGET  _IO(0x12,104)/* get block device sector size */
+#endif
+
+#undef ALIGN_DEBUG
+
 #include "ext2_fs.h"
 #include "ext2fs.h"
 
@@ -77,12 +84,17 @@ struct unix_private_data {
        int     magic;
        int     dev;
        int     flags;
+       int     align;
        int     access_time;
        ext2_loff_t offset;
        struct unix_cache cache[CACHE_SIZE];
+       void    *bounce;
        struct struct_io_stats io_stats;
 };
 
+#define IS_ALIGNED(n, align) ((((unsigned long) n) & \
+                              ((unsigned long) ((align)-1))) == 0)
+
 static errcode_t unix_open(const char *name, int flags, io_channel *channel);
 static errcode_t unix_close(io_channel channel);
 static errcode_t unix_set_blksize(io_channel channel, int blksize);
@@ -104,14 +116,6 @@ static errcode_t unix_read_blk64(io_channel channel, unsigned long long block,
 static errcode_t unix_write_blk64(io_channel channel, unsigned long long block,
                                int count, const void *data);
 
-/* __FreeBSD_kernel__ is defined by GNU/kFreeBSD - the FreeBSD kernel
- * does not know buffered block devices - everything is raw. */
-#if defined(__CYGWIN__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
-#define NEED_BOUNCE_BUFFER
-#else
-#undef NEED_BOUNCE_BUFFER
-#endif
-
 static struct struct_io_manager struct_unix_manager = {
        EXT2_ET_MAGIC_IO_MANAGER,
        "Unix I/O Manager",
@@ -121,11 +125,7 @@ static struct struct_io_manager struct_unix_manager = {
        unix_read_blk,
        unix_write_blk,
        unix_flush,
-#ifdef NEED_BOUNCE_BUFFER
-       0,
-#else
        unix_write_byte,
-#endif
        unix_set_option,
        unix_get_stats,
        unix_read_blk64,
@@ -153,7 +153,6 @@ static errcode_t unix_get_stats(io_channel channel, io_stats *stats)
 /*
  * Here are the raw I/O functions
  */
-#ifndef NEED_BOUNCE_BUFFER
 static errcode_t raw_read_blk(io_channel channel,
                              struct unix_private_data *data,
                              unsigned long long block,
@@ -171,77 +170,48 @@ static errcode_t raw_read_blk(io_channel channel,
                retval = errno ? errno : EXT2_ET_LLSEEK_FAILED;
                goto error_out;
        }
-       actual = read(data->dev, buf, size);
-       if (actual != size) {
-               if (actual < 0)
-                       actual = 0;
-               retval = EXT2_ET_SHORT_READ;
-               goto error_out;
+       if ((data->align == 0) ||
+           ((IS_ALIGNED(buf, data->align)) && IS_ALIGNED(size, data->align))) {
+               actual = read(data->dev, buf, size);
+               if (actual != size) {
+               short_read:
+                       if (actual < 0)
+                               actual = 0;
+                       retval = EXT2_ET_SHORT_READ;
+                       goto error_out;
+               }
+               return 0;
        }
-       return 0;
-
-error_out:
-       memset((char *) buf+actual, 0, size-actual);
-       if (channel->read_error)
-               retval = (channel->read_error)(channel, block, count, buf,
-                                              size, actual, retval);
-       return retval;
-}
-#else /* NEED_BOUNCE_BUFFER */
-/*
- * Windows and FreeBSD block devices only allow sector alignment IO in offset and size
- */
-static errcode_t raw_read_blk(io_channel channel,
-                             struct unix_private_data *data,
-                             unsigned long block,
-                             int count, void *buf)
-{
-       errcode_t       retval;
-       size_t          size, alignsize, fragment;
-       ext2_loff_t     location;
-       int             total = 0, actual;
-#define BLOCKALIGN 512
-       char            sector[BLOCKALIGN];
 
-       size = (count < 0) ? -count : count * channel->block_size;
-       data->io_stats.bytes_read += size;
-       location = ((ext2_loff_t) block * channel->block_size) + data->offset;
-#ifdef DEBUG
-       printf("count=%d, size=%d, block=%lu, blk_size=%d, location=%llx\n",
-                       count, size, block, channel->block_size, (long long)location);
+#ifdef ALIGN_DEBUG
+       printf("raw_read_blk: O_DIRECT fallback: %p %lu\n", buf,
+              (unsigned long) size);
 #endif
-       if (ext2fs_llseek(data->dev, location, SEEK_SET) != location) {
-               retval = errno ? errno : EXT2_ET_LLSEEK_FAILED;
-               goto error_out;
-       }
-       fragment = size % BLOCKALIGN;
-       alignsize = size - fragment;
-       if (alignsize) {
-               actual = read(data->dev, buf, alignsize);
-               if (actual != alignsize)
-                       goto short_read;
-       }
-       if (fragment) {
-               actual = read(data->dev, sector, BLOCKALIGN);
-               if (actual != BLOCKALIGN)
+
+       /*
+        * The buffer or size which we're trying to read isn't aligned
+        * to the O_DIRECT rules, so we need to do this the hard way...
+        */
+       while (size > 0) {
+               actual = read(data->dev, data->bounce, channel->block_size);
+               if (actual != channel->block_size)
                        goto short_read;
-               memcpy(buf+alignsize, sector, fragment);
+               actual = size;
+               if (size > channel->block_size)
+                       actual = channel->block_size;
+               memcpy(buf, data->bounce, actual);
+               size -= actual;
+               buf += actual;
        }
        return 0;
 
-short_read:
-       if (actual>0)
-               total += actual;
-       retval = EXT2_ET_SHORT_READ;
-
 error_out:
-       memset((char *) buf+total, 0, size-actual);
+       memset((char *) buf+actual, 0, size-actual);
        if (channel->read_error)
                retval = (channel->read_error)(channel, block, count, buf,
                                               size, actual, retval);
        return retval;
 }
-#endif
 
 static errcode_t raw_write_blk(io_channel channel,
                               struct unix_private_data *data,
@@ -269,10 +239,43 @@ static errcode_t raw_write_blk(io_channel channel,
                goto error_out;
        }
 
-       actual = write(data->dev, buf, size);
-       if (actual != size) {
-               retval = EXT2_ET_SHORT_WRITE;
-               goto error_out;
+       if ((data->align == 0) ||
+           ((IS_ALIGNED(buf, data->align)) && IS_ALIGNED(size, data->align))) {
+               actual = write(data->dev, buf, size);
+               if (actual != size) {
+               short_write:
+                       retval = EXT2_ET_SHORT_WRITE;
+                       goto error_out;
+               }
+               return 0;
+       }
+
+#ifdef ALIGN_DEBUG
+       printf("raw_write_blk: O_DIRECT fallback: %p %lu\n", buf,
+              (unsigned long) size);
+#endif
+       /*
+        * The buffer or size which we're trying to write isn't aligned
+        * to the O_DIRECT rules, so we need to do this the hard way...
+        */
+       while (size > 0) {
+               if (size < channel->block_size) {
+                       actual = read(data->dev, data->bounce,
+                                     channel->block_size);
+                       if (actual != channel->block_size) {
+                               retval = EXT2_ET_SHORT_READ;
+                               goto error_out;
+                       }
+               }
+               actual = size;
+               if (size > channel->block_size)
+                       actual = channel->block_size;
+               memcpy(data->bounce, buf, actual);
+               actual = write(data->dev, data->bounce, channel->block_size);
+               if (actual != channel->block_size)
+                       goto short_write;
+               size -= actual;
+               buf += actual;
        }
        return 0;
 
@@ -304,11 +307,18 @@ static errcode_t alloc_cache(io_channel channel,
                cache->in_use = 0;
                if (cache->buf)
                        ext2fs_free_mem(&cache->buf);
-               if ((retval = ext2fs_get_mem(channel->block_size,
-                                            &cache->buf)))
+               retval = ext2fs_get_memalign(channel->block_size,
+                                            data->align, &cache->buf);
+               if (retval)
                        return retval;
        }
-       return 0;
+       if (data->align) {
+               if (data->bounce)
+                       ext2fs_free_mem(&data->bounce);
+               retval = ext2fs_get_memalign(channel->block_size, data->align,
+                                            &data->bounce);
+       }
+       return retval;
 }
 
 /* Free the cache buffers */
@@ -325,8 +335,9 @@ static void free_cache(struct unix_private_data *data)
                cache->in_use = 0;
                if (cache->buf)
                        ext2fs_free_mem(&cache->buf);
-               cache->buf = 0;
        }
+       if (data->bounce)
+               ext2fs_free_mem(&data->bounce);
 }
 
 #ifndef NO_IO_CACHE
@@ -449,12 +460,13 @@ static errcode_t unix_open(const char *name, int flags, io_channel *channel)
        data->magic = EXT2_ET_MAGIC_UNIX_IO_CHANNEL;
        data->io_stats.num_fields = 2;
 
-       if ((retval = alloc_cache(io, data)))
-               goto cleanup;
-
        open_flags = (flags & IO_FLAG_RW) ? O_RDWR : O_RDONLY;
        if (flags & IO_FLAG_EXCLUSIVE)
                open_flags |= O_EXCL;
+       if (flags & IO_FLAG_DIRECT_IO)
+               open_flags |= O_DIRECT;
+       data->flags = flags;
+
 #ifdef HAVE_OPEN64
        data->dev = open64(io->name, open_flags);
 #else
@@ -465,6 +477,25 @@ static errcode_t unix_open(const char *name, int flags, io_channel *channel)
                goto cleanup;
        }
 
+#ifdef BLKSSZGET
+       if (flags & IO_FLAG_DIRECT_IO) {
+               if (ioctl(data->dev, BLKSSZGET, &data->align) != 0)
+                       data->align = io->block_size;
+       }
+#endif
+
+#if defined(__CYGWIN__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
+       /*
+        * Some operating systems require that the buffers be aligned,
+        * regardless of O_DIRECT
+        */
+       data->align = 512;
+#endif
+
+
+       if ((retval = alloc_cache(io, data)))
+               goto cleanup;
+
 #ifdef BLKROGET
        if (flags & IO_FLAG_RW) {
                int error;
@@ -617,6 +648,22 @@ static errcode_t unix_read_blk64(io_channel channel, unsigned long long block,
                        cp += channel->block_size;
                        continue;
                }
+               if (count == 1) {
+                       /*
+                        * Special case where we read directly into the
+                        * cache buffer; important in the O_DIRECT case
+                        */
+                       cache = reuse[0];
+                       reuse_cache(channel, data, cache, block);
+                       if ((retval = raw_read_blk(channel, data, block, 1,
+                                                  cache->buf))) {
+                               cache->in_use = 0;
+                               return retval;
+                       }
+                       memcpy(cp, cache->buf, channel->block_size);
+                       return 0;
+               }
+
                /*
                 * Find the number of uncached blocks so we can do a
                 * single read request
@@ -718,6 +765,13 @@ static errcode_t unix_write_byte(io_channel channel, unsigned long offset,
        data = (struct unix_private_data *) channel->private_data;
        EXT2_CHECK_MAGIC(data, EXT2_ET_MAGIC_UNIX_IO_CHANNEL);
 
+       if (data->align != 0) {
+#ifdef ALIGN_DEBUG
+               printf("unix_write_byte: O_DIRECT fallback\n");
+#endif
+               return EXT2_ET_UNIMPLEMENTED;
+       }
+
 #ifndef NO_IO_CACHE
        /*
         * Flush out the cache completely