From: Timo Sirainen Date: Mon, 21 Jan 2013 12:43:09 +0000 (+0200) Subject: lib-fs: Added "metawrap" wrapper to implement metadata as headers in the file content. X-Git-Tag: 2.2.beta2~195 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=84669c712403b742cc07ae70229725c486ef1235;p=thirdparty%2Fdovecot%2Fcore.git lib-fs: Added "metawrap" wrapper to implement metadata as headers in the file content. --- diff --git a/src/lib-fs/Makefile.am b/src/lib-fs/Makefile.am index eb8d0107f5..4c944f9c25 100644 --- a/src/lib-fs/Makefile.am +++ b/src/lib-fs/Makefile.am @@ -8,16 +8,19 @@ AM_CPPFLAGS = \ libfs_la_SOURCES = \ fs-api.c \ + fs-metawrap.c \ fs-posix.c \ fs-sis.c \ fs-sis-common.c \ fs-sis-queue.c \ + istream-metawrap.c \ ostream-cmp.c headers = \ fs-api.h \ fs-api-private.h \ fs-sis-common.h \ + istream-metawrap.h \ ostream-cmp.h fs_test_SOURCES = fs-test.c diff --git a/src/lib-fs/fs-api-private.h b/src/lib-fs/fs-api-private.h index 06e3497d9d..7c806aeda5 100644 --- a/src/lib-fs/fs-api-private.h +++ b/src/lib-fs/fs-api-private.h @@ -69,6 +69,13 @@ struct fs_file { struct istream *pending_read_input; bool write_pending; + + pool_t metadata_pool; + ARRAY_TYPE(fs_metadata) metadata; + + struct fs_file *copy_src; + struct istream *copy_input; + struct ostream *copy_output; }; struct fs_lock { @@ -81,6 +88,7 @@ struct fs_iter { }; extern const struct fs fs_class_posix; +extern const struct fs fs_class_metawrap; extern const struct fs fs_class_sis; extern const struct fs fs_class_sis_queue; @@ -89,4 +97,11 @@ void fs_set_critical(struct fs *fs, const char *fmt, ...) ATTR_FORMAT(2, 3); void fs_set_error_async(struct fs *fs); +ssize_t fs_read_via_stream(struct fs_file *file, void *buf, size_t size); +int fs_write_via_stream(struct fs_file *file, const void *data, size_t size); +void fs_metadata_init(struct fs_file *file); +void fs_default_set_metadata(struct fs_file *file, + const char *key, const char *value); +int fs_default_copy(struct fs_file *src, struct fs_file *dest); + #endif diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c index 4294719e49..64aacde95e 100644 --- a/src/lib-fs/fs-api.c +++ b/src/lib-fs/fs-api.c @@ -46,6 +46,7 @@ static void fs_classes_init(void) { i_array_init(&fs_classes, 8); fs_class_register(&fs_class_posix); + fs_class_register(&fs_class_metawrap); fs_class_register(&fs_class_sis); fs_class_register(&fs_class_sis_queue); } @@ -146,6 +147,7 @@ struct fs_file *fs_file_init(struct fs *fs, const char *path, int mode_flags) void fs_file_deinit(struct fs_file **_file) { struct fs_file *file = *_file; + pool_t metadata_pool = file->metadata_pool; i_assert(file->fs->files_open_count > 0); @@ -153,6 +155,9 @@ void fs_file_deinit(struct fs_file **_file) file->fs->files_open_count--; file->fs->v.file_deinit(file); + + if (metadata_pool != NULL) + pool_unref(&metadata_pool); } enum fs_properties fs_get_properties(struct fs *fs) @@ -160,6 +165,25 @@ enum fs_properties fs_get_properties(struct fs *fs) return fs->v.get_properties(fs); } +void fs_metadata_init(struct fs_file *file) +{ + if (file->metadata_pool == NULL) { + file->metadata_pool = pool_alloconly_create("fs metadata", 1024); + p_array_init(&file->metadata, file->metadata_pool, 8); + } +} + +void fs_default_set_metadata(struct fs_file *file, + const char *key, const char *value) +{ + struct fs_metadata *metadata; + + fs_metadata_init(file); + metadata = array_append_space(&file->metadata); + metadata->key = p_strdup(file->metadata_pool, key); + metadata->value = p_strdup(file->metadata_pool, value); +} + void fs_set_metadata(struct fs_file *file, const char *key, const char *value) { if (file->fs->v.set_metadata != NULL) @@ -199,17 +223,14 @@ bool fs_prefetch(struct fs_file *file, uoff_t length) return file->fs->v.prefetch(file, length); } -ssize_t fs_read(struct fs_file *file, void *buf, size_t size) +ssize_t fs_read_via_stream(struct fs_file *file, void *buf, size_t size) { const unsigned char *data; size_t data_size; ssize_t ret; - if (file->fs->v.read != NULL) - return file->fs->v.read(file, buf, size); + i_assert(size > 0); - /* backend didn't bother to implement read(), but we can do it with - streams. */ if (file->pending_read_input == NULL) file->pending_read_input = fs_read_stream(file, size+1); ret = i_stream_read_data(file->pending_read_input, @@ -229,22 +250,27 @@ ssize_t fs_read(struct fs_file *file, void *buf, size_t size) return ret; } +ssize_t fs_read(struct fs_file *file, void *buf, size_t size) +{ + if (file->fs->v.read != NULL) + return file->fs->v.read(file, buf, size); + + /* backend didn't bother to implement read(), but we can do it with + streams. */ + return fs_read_via_stream(file, buf, size); +} + struct istream *fs_read_stream(struct fs_file *file, size_t max_buffer_size) { return file->fs->v.read_stream(file, max_buffer_size); } -int fs_write(struct fs_file *file, const void *data, size_t size) +int fs_write_via_stream(struct fs_file *file, const void *data, size_t size) { struct ostream *output; ssize_t ret; int err; - if (file->fs->v.write != NULL) - return file->fs->v.write(file, data, size); - - /* backend didn't bother to implement write(), but we can do it with - streams. */ if (!file->write_pending) { output = fs_write_stream(file); if ((ret = o_stream_send(output, data, size)) < 0) { @@ -269,6 +295,16 @@ int fs_write(struct fs_file *file, const void *data, size_t size) return ret < 0 ? -1 : 0; } +int fs_write(struct fs_file *file, const void *data, size_t size) +{ + if (file->fs->v.write != NULL) + return file->fs->v.write(file, data, size); + + /* backend didn't bother to implement write(), but we can do it with + streams. */ + return fs_write_via_stream(file, data, size); +} + struct ostream *fs_write_stream(struct fs_file *file) { file->fs->v.write_stream(file); @@ -338,6 +374,44 @@ int fs_stat(struct fs_file *file, struct stat *st_r) return file->fs->v.stat(file, st_r); } +int fs_default_copy(struct fs_file *src, struct fs_file *dest) +{ + if (dest->copy_src != NULL) { + i_assert(src == NULL || src == dest->copy_src); + if (dest->copy_output == NULL) { + i_assert(dest->copy_input == NULL); + if (fs_write_stream_finish_async(dest) < 0) + return -1; + dest->copy_src = NULL; + return 0; + } + } else { + dest->copy_src = src; + dest->copy_input = fs_read_stream(src, IO_BLOCK_SIZE); + dest->copy_output = fs_write_stream(dest); + } + while (o_stream_send_istream(dest->copy_output, dest->copy_input) > 0) ; + if (dest->copy_input->stream_errno != 0) { + fs_set_error(dest->fs, "read(%s) failed: %m", + i_stream_get_name(dest->copy_input)); + return -1; + } + if (dest->copy_output->stream_errno != 0) { + fs_set_error(dest->fs, "write(%s) failed: %m", + o_stream_get_name(dest->copy_output)); + return -1; + } + if (!dest->copy_input->eof) { + fs_set_error_async(dest->fs); + return -1; + } + i_stream_unref(&dest->copy_input); + if (fs_write_stream_finish(dest, &dest->copy_output) < 0) + return -1; + dest->copy_src = NULL; + return 0; +} + int fs_copy(struct fs_file *src, struct fs_file *dest) { i_assert(src->fs == dest->fs); diff --git a/src/lib-fs/fs-metawrap.c b/src/lib-fs/fs-metawrap.c new file mode 100644 index 0000000000..faa78ab756 --- /dev/null +++ b/src/lib-fs/fs-metawrap.c @@ -0,0 +1,429 @@ +/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "strescape.h" +#include "istream.h" +#include "istream-private.h" +#include "istream-metawrap.h" +#include "ostream.h" +#include "fs-api-private.h" + +struct metawrap_fs { + struct fs fs; + struct fs *super; + bool wrap_metadata; +}; + +struct metawrap_fs_file { + struct fs_file file; + struct metawrap_fs *fs; + struct fs_file *super; + enum fs_open_mode open_mode; +}; + +static void fs_metawrap_copy_error(struct metawrap_fs *fs) +{ + fs_set_error(&fs->fs, "%s", fs_last_error(fs->super)); +} + +static void fs_metawrap_file_copy_error(struct metawrap_fs_file *file) +{ + struct metawrap_fs *fs = (struct metawrap_fs *)file->file.fs; + + fs_metawrap_copy_error(fs); +} + +static struct fs *fs_metawrap_alloc(void) +{ + struct metawrap_fs *fs; + + fs = i_new(struct metawrap_fs, 1); + fs->fs = fs_class_metawrap; + return &fs->fs; +} + +static int +fs_metawrap_init(struct fs *_fs, const char *args, const + struct fs_settings *set) +{ + struct metawrap_fs *fs = (struct metawrap_fs *)_fs; + const char *parent_name, *parent_args, *error; + + if (*args == '\0') { + fs_set_error(_fs, "Parent filesystem not given as parameter"); + return -1; + } + + parent_args = strchr(args, ':'); + if (parent_args == NULL) { + parent_name = args; + parent_args = ""; + } else { + parent_name = t_strdup_until(args, parent_args); + parent_args++; + } + if (fs_init(parent_name, parent_args, set, &fs->super, &error) < 0) { + fs_set_error(_fs, "%s: %s", parent_name, error); + return -1; + } + if ((fs_get_properties(fs->super) & FS_PROPERTY_METADATA) == 0) + fs->wrap_metadata = TRUE; + return 0; +} + +static void fs_metawrap_deinit(struct fs *_fs) +{ + struct metawrap_fs *fs = (struct metawrap_fs *)_fs; + + if (fs->super != NULL) + fs_deinit(&fs->super); + i_free(fs); +} + +static enum fs_properties fs_metawrap_get_properties(struct fs *_fs) +{ + const struct metawrap_fs *fs = (const struct metawrap_fs *)_fs; + enum fs_properties props; + + props = fs_get_properties(fs->super); + if (fs->wrap_metadata) { + /* we don't have a quick stat() to see the file's size, + because of the metadata header */ + props &= ~FS_PROPERTY_STAT; + } + return props; +} + +static struct fs_file * +fs_metawrap_file_init(struct fs *_fs, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags) +{ + struct metawrap_fs *fs = (struct metawrap_fs *)_fs; + struct metawrap_fs_file *file; + + file = i_new(struct metawrap_fs_file, 1); + file->file.fs = _fs; + file->file.path = i_strdup(path); + file->fs = fs; + file->open_mode = mode; + + file->super = fs_file_init(fs->super, path, mode | flags); + i_array_init(&file->file.metadata, 8); + return &file->file; +} + +static void fs_metawrap_file_deinit(struct fs_file *_file) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + + fs_file_deinit(&file->super); + array_free(&file->file.metadata); + i_free(file->file.path); + i_free(file); +} + +static const char *fs_metawrap_file_get_path(struct fs_file *_file) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + + return fs_file_path(file->super); +} + +static void +fs_metawrap_set_async_callback(struct fs_file *_file, + fs_file_async_callback_t *callback, + void *context) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + + fs_file_set_async_callback(file->super, callback, context); +} + +static int fs_metawrap_wait_async(struct fs *_fs) +{ + struct metawrap_fs *fs = (struct metawrap_fs *)_fs; + + return fs_wait_async(fs->super); +} + +static void +fs_metawrap_set_metadata(struct fs_file *_file, const char *key, + const char *value) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + + if (!file->fs->wrap_metadata) + fs_set_metadata(file->super, key, value); + else + fs_default_set_metadata(_file, key, value); +} + +static int +fs_metawrap_get_metadata(struct fs_file *_file, + const ARRAY_TYPE(fs_metadata) **metadata_r) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + char c; + + if (!file->fs->wrap_metadata) + return fs_get_metadata(file->super, metadata_r); + + if (fs_read(_file, &c, 1) < 0) + return -1; + *metadata_r = &_file->metadata; + return 0; +} + +static bool fs_metawrap_prefetch(struct fs_file *_file, uoff_t length) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + + return fs_prefetch(file->super, length); +} + +static ssize_t fs_metawrap_read(struct fs_file *_file, void *buf, size_t size) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + ssize_t ret; + + if (!file->fs->wrap_metadata) { + if ((ret = fs_read(file->super, buf, size)) < 0) + fs_metawrap_file_copy_error(file); + return ret; + } + return fs_read_via_stream(_file, buf, size); +} + +static void +fs_metawrap_callback(const char *key, const char *value, void *context) +{ + struct metawrap_fs_file *file = context; + + T_BEGIN { + key = str_tabunescape(t_strdup_noconst(key)); + value = str_tabunescape(t_strdup_noconst(value)); + fs_default_set_metadata(&file->file, key, value); + } T_END; +} + +static struct istream * +fs_metawrap_read_stream(struct fs_file *_file, size_t max_buffer_size) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + struct istream *input, *input2; + + if (!file->fs->wrap_metadata) + return fs_read_stream(file->super, max_buffer_size); + + input = fs_read_stream(file->super, max_buffer_size); + input2 = i_stream_create_metawrap(input, fs_metawrap_callback, file); + i_stream_unref(&input); + return input2; +} + +static int fs_metawrap_write(struct fs_file *_file, const void *data, size_t size) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + + if (!file->fs->wrap_metadata) { + if (fs_write(file->super, data, size) < 0) { + fs_metawrap_file_copy_error(file); + return -1; + } + return 0; + } + return fs_write_via_stream(_file, data, size); +} + +static void fs_metawrap_write_stream(struct fs_file *_file) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + + i_assert(_file->output == NULL); + + _file->output = fs_write_stream(file->super); + if (file->fs->wrap_metadata) T_BEGIN { + const struct fs_metadata *metadata; + string_t *str = t_str_new(256); + ssize_t ret; + + /* FIXME: if fs_set_metadata() is called later the changes are + ignored. we'd need to write via temporary file then. */ + array_foreach(&_file->metadata, metadata) { + str_append_tabescaped(str, metadata->key); + str_append_c(str, ':'); + str_append_tabescaped(str, metadata->value); + str_append_c(str, '\n'); + } + str_append_c(str, '\n'); + ret = o_stream_send(_file->output, str_data(str), str_len(str)); + if (ret < 0) { + int err = _file->output->stream_errno; + fs_write_stream_abort(file->super, &_file->output); + _file->output = o_stream_create_error(err); + } else { + i_assert((size_t)ret == str_len(str)); + } + } T_END; +} + +static int fs_metawrap_write_stream_finish(struct fs_file *_file, bool success) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + int ret; + + if (!success) { + fs_write_stream_abort(file->super, &_file->output); + fs_metawrap_file_copy_error(file); + return -1; + } + + if ((ret = fs_write_stream_finish(file->super, &_file->output)) < 0) { + fs_metawrap_file_copy_error(file); + return -1; + } + return ret; +} + +static int +fs_metawrap_lock(struct fs_file *_file, unsigned int secs, struct fs_lock **lock_r) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + + if (fs_lock(file->super, secs, lock_r) < 0) { + fs_metawrap_file_copy_error(file); + return -1; + } + return 0; +} + +static void fs_metawrap_unlock(struct fs_lock *_lock ATTR_UNUSED) +{ + i_unreached(); +} + +static int fs_metawrap_exists(struct fs_file *_file) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + + if (fs_exists(file->super) < 0) { + fs_metawrap_copy_error(file->fs); + return -1; + } + return 0; +} + +static int fs_metawrap_stat(struct fs_file *_file, struct stat *st_r) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + struct istream *input; + uoff_t content_offset; + ssize_t ret; + + if (!file->fs->wrap_metadata) { + if (fs_stat(file->super, st_r) < 0) { + fs_metawrap_copy_error(file->fs); + return -1; + } + return 0; + } + input = fs_read_stream(_file, IO_BLOCK_SIZE); + ret = i_stream_read(input); + content_offset = input->real_stream->parent->v_offset; + i_stream_unref(&input); + if (ret <= 0) { + if (ret == 0) + fs_set_error_async(_file->fs); + return -1; + } + + if (fs_stat(file->super, st_r) < 0) { + fs_metawrap_copy_error(file->fs); + return -1; + } + i_assert((uoff_t)st_r->st_size >= content_offset); + st_r->st_size -= content_offset; + return 0; +} + +static int fs_metawrap_copy(struct fs_file *_src, struct fs_file *_dest) +{ + struct metawrap_fs_file *src = (struct metawrap_fs_file *)_src; + struct metawrap_fs_file *dest = (struct metawrap_fs_file *)_dest; + + if (!dest->fs->wrap_metadata) { + if (fs_copy(src->super, dest->super) < 0) { + fs_metawrap_copy_error(src->fs); + return -1; + } + return 0; + } + return fs_default_copy(_src, _dest); +} + +static int fs_metawrap_rename(struct fs_file *_src, struct fs_file *_dest) +{ + struct metawrap_fs_file *src = (struct metawrap_fs_file *)_src; + struct metawrap_fs_file *dest = (struct metawrap_fs_file *)_dest; + + if (fs_rename(src->super, dest->super) < 0) { + fs_metawrap_copy_error(src->fs); + return -1; + } + return 0; +} + +static int fs_metawrap_delete(struct fs_file *_file) +{ + struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + + if (fs_delete(file->super) < 0) { + fs_metawrap_copy_error(file->fs); + return -1; + } + return 0; +} + +static struct fs_iter * +fs_metawrap_iter_init(struct fs *_fs, const char *path, + enum fs_iter_flags flags) +{ + struct metawrap_fs *fs = (struct metawrap_fs *)_fs; + + return fs_iter_init(fs->super, path, flags); +} + +const struct fs fs_class_metawrap = { + .name = "metawrap", + .v = { + fs_metawrap_alloc, + fs_metawrap_init, + fs_metawrap_deinit, + fs_metawrap_get_properties, + fs_metawrap_file_init, + fs_metawrap_file_deinit, + fs_metawrap_file_get_path, + fs_metawrap_set_async_callback, + fs_metawrap_wait_async, + fs_metawrap_set_metadata, + fs_metawrap_get_metadata, + fs_metawrap_prefetch, + fs_metawrap_read, + fs_metawrap_read_stream, + fs_metawrap_write, + fs_metawrap_write_stream, + fs_metawrap_write_stream_finish, + fs_metawrap_lock, + fs_metawrap_unlock, + fs_metawrap_exists, + fs_metawrap_stat, + fs_metawrap_copy, + fs_metawrap_rename, + fs_metawrap_delete, + fs_metawrap_iter_init, + NULL, + NULL + } +}; diff --git a/src/lib-fs/istream-metawrap.c b/src/lib-fs/istream-metawrap.c new file mode 100644 index 0000000000..87466e5f94 --- /dev/null +++ b/src/lib-fs/istream-metawrap.c @@ -0,0 +1,79 @@ +/* Copyright (c) 2007-2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream-private.h" +#include "istream-metawrap.h" + +struct metawrap_istream { + struct istream_private istream; + metawrap_callback_t *callback; + void *context; + + uoff_t start_offset; + bool in_metadata; +}; + +static int metadata_header_read(struct metawrap_istream *mstream) +{ + char *line, *p; + + while ((line = i_stream_read_next_line(mstream->istream.parent)) != NULL) { + if (*line == '\0') + return 1; + p = strchr(line, ':'); + if (p == NULL) { + mstream->istream.istream.stream_errno = EINVAL; + return -1; + } + *p++ = '\0'; + mstream->callback(line, p, mstream->context); + } + if (mstream->istream.parent->eof) { + mstream->istream.istream.stream_errno = + mstream->istream.parent->stream_errno; + mstream->istream.istream.eof = TRUE; + return -1; + } + return 0; +} + +static ssize_t i_stream_metawrap_read(struct istream_private *stream) +{ + struct metawrap_istream *mstream = (struct metawrap_istream *)stream; + int ret; + + i_stream_seek(stream->parent, mstream->start_offset + + stream->istream.v_offset); + + if (mstream->in_metadata) { + ret = metadata_header_read(mstream); + i_assert(stream->istream.v_offset == 0); + mstream->start_offset = stream->parent->v_offset; + if (ret <= 0) + return ret; + mstream->in_metadata = FALSE; + } + /* after metadata header it's all just passthrough */ + return i_stream_read_copy_from_parent(&stream->istream); +} + +struct istream * +i_stream_create_metawrap(struct istream *input, + metawrap_callback_t *callback, void *context) +{ + struct metawrap_istream *mstream; + + mstream = i_new(struct metawrap_istream, 1); + mstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + + mstream->istream.read = i_stream_metawrap_read; + + mstream->istream.istream.readable_fd = FALSE; + mstream->istream.istream.blocking = input->blocking; + mstream->istream.istream.seekable = FALSE; + mstream->in_metadata = TRUE; + mstream->callback = callback; + mstream->context = context; + return i_stream_create(&mstream->istream, input, + i_stream_get_fd(input)); +} diff --git a/src/lib-fs/istream-metawrap.h b/src/lib-fs/istream-metawrap.h new file mode 100644 index 0000000000..90ca2d3378 --- /dev/null +++ b/src/lib-fs/istream-metawrap.h @@ -0,0 +1,14 @@ +#ifndef ISTREAM_METAWRAP_H +#define ISTREAM_METAWRAP_H + +typedef void +metawrap_callback_t(const char *key, const char *value, void *context); + +/* Input stream is in format "key:value\nkey2:value2\n...\n\ncontents. + The given callback is called for each key/value metadata pair, and the + returned stream will skip over the metadata and return only the contents. */ +struct istream * +i_stream_create_metawrap(struct istream *input, + metawrap_callback_t *callback, void *context); + +#endif