From: Timo Sirainen Date: Wed, 29 Apr 2009 01:05:00 +0000 (-0400) Subject: struct mail_private now contains all kinds of statistics about file accesses. X-Git-Tag: 2.0.alpha1~889 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=b0a901f1dbe9e05ac1c92a0974af6bce0274f31a;p=thirdparty%2Fdovecot%2Fcore.git struct mail_private now contains all kinds of statistics about file accesses. --HG-- branch : HEAD --- diff --git a/src/lib-storage/index/Makefile.am b/src/lib-storage/index/Makefile.am index 883a3210e9..d3903aa417 100644 --- a/src/lib-storage/index/Makefile.am +++ b/src/lib-storage/index/Makefile.am @@ -10,6 +10,7 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib-storage libstorage_index_la_SOURCES = \ + istream-mail-stats.c \ index-fetch.c \ index-mail.c \ index-mail-headers.c \ @@ -32,6 +33,7 @@ libstorage_index_la_LIBADD = @LINKED_STORAGE_LIBS@ libstorage_index_la_DEPENDENCIES = @LINKED_STORAGE_LIBS@ headers = \ + istream-mail-stats.h \ index-mail.h \ index-search-result.h \ index-sort.h \ diff --git a/src/lib-storage/index/cydir/cydir-mail.c b/src/lib-storage/index/cydir/cydir-mail.c index 9e938e09cc..bb5f7c9f86 100644 --- a/src/lib-storage/index/cydir/cydir-mail.c +++ b/src/lib-storage/index/cydir/cydir-mail.c @@ -20,11 +20,13 @@ static const char *cydir_mail_get_path(struct mail *mail) static int cydir_mail_stat(struct mail *mail, struct stat *st_r) { + struct mail_private *p = (struct mail_private *)mail; const char *path; if (mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE) return mail_set_aborted(mail); + p->stats_dentry_lookup_count++; path = cydir_mail_get_path(mail); if (stat(path, st_r) < 0) { if (errno == ENOENT) @@ -98,6 +100,7 @@ cydir_mail_get_stream(struct mail *_mail, struct message_size *hdr_size, int fd; if (mail->data.stream == NULL) { + mail->mail.stats_dentry_lookup_count++; path = cydir_mail_get_path(_mail); fd = open(path, O_RDONLY); if (fd == -1) { diff --git a/src/lib-storage/index/dbox/dbox-file.h b/src/lib-storage/index/dbox/dbox-file.h index d0c18637c4..6bba0b498e 100644 --- a/src/lib-storage/index/dbox/dbox-file.h +++ b/src/lib-storage/index/dbox/dbox-file.h @@ -119,6 +119,8 @@ struct dbox_file { unsigned int corrupted:1; }; +#define dbox_file_is_open(file) ((file)->input != NULL) + struct dbox_file * dbox_file_init_single(struct dbox_mailbox *mbox, uint32_t uid); struct dbox_file * diff --git a/src/lib-storage/index/dbox/dbox-mail.c b/src/lib-storage/index/dbox/dbox-mail.c index 2cf76873a5..27e17bf5c1 100644 --- a/src/lib-storage/index/dbox/dbox-mail.c +++ b/src/lib-storage/index/dbox/dbox-mail.c @@ -158,6 +158,8 @@ static int dbox_mail_open(struct dbox_mail *mail, return -1; } + if (!dbox_file_is_open(mail->open_file)) + mail->imail.mail.stats_dentry_lookup_count++; if (dbox_file_open(mail->open_file, &deleted) <= 0) return -1; if (deleted) { @@ -235,6 +237,7 @@ static int dbox_mail_get_save_date(struct mail *_mail, time_t *date_r) if (data->save_date == 0) { /* missing / corrupted save time - use the file's ctime */ i_assert(file->fd != -1); + mail->imail.mail.stats_attr_lookup_count++; if (fstat(file->fd, &st) < 0) { mail_storage_set_critical(_mail->box->storage, "fstat(%s) failed: %m", file->current_path); diff --git a/src/lib-storage/index/index-mail-headers.c b/src/lib-storage/index/index-mail-headers.c index a7b19c5fc9..bdbdd44396 100644 --- a/src/lib-storage/index/index-mail-headers.c +++ b/src/lib-storage/index/index-mail-headers.c @@ -471,8 +471,8 @@ int index_mail_headers_get_envelope(struct index_mail *mail) string_t *str; str = str_new(mail->data_pool, 256); - if (mail_cache_lookup_field(mail->trans->cache_view, str, - mail->data.seq, cache_field_envelope) > 0) { + if (index_mail_cache_lookup_field(mail, str, + cache_field_envelope) > 0) { mail->data.envelope = str_c(str); return 0; } @@ -637,6 +637,7 @@ index_mail_get_raw_headers(struct index_mail *mail, const char *field, *value_r = index_mail_get_parsed_header(mail, field_idx); return 0; } + mail->mail.stats_cache_hit_count++; data = buffer_get_modifiable_data(dest, &len); if (len == 0) { @@ -819,6 +820,7 @@ int index_mail_get_header_stream(struct mail *_mail, if (mail_cache_lookup_headers(mail->trans->cache_view, dest, mail->data.seq, headers->idx, headers->count) > 0) { + mail->mail.stats_cache_hit_count++; if (mail->data.filter_stream != NULL) i_stream_destroy(&mail->data.filter_stream); mail->data.filter_stream = diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c index 620aad42f3..2d65ec2805 100644 --- a/src/lib-storage/index/index-mail.c +++ b/src/lib-storage/index/index-mail.c @@ -15,6 +15,7 @@ #include "mail-cache.h" #include "mail-index-modseq.h" #include "index-storage.h" +#include "istream-mail-stats.h" #include "index-mail.h" struct mail_cache_field global_cache_fields[MAIL_INDEX_CACHE_FIELD_COUNT] = { @@ -40,6 +41,18 @@ struct mail_cache_field global_cache_fields[MAIL_INDEX_CACHE_FIELD_COUNT] = { static int index_mail_parse_body(struct index_mail *mail, enum index_cache_field field); +int index_mail_cache_lookup_field(struct index_mail *mail, buffer_t *buf, + unsigned int field_idx) +{ + int ret; + + ret = mail_cache_lookup_field(mail->trans->cache_view, buf, + mail->data.seq, field_idx); + if (ret > 0) + mail->mail.stats_cache_hit_count++; + return ret; +} + static struct message_part *get_unserialized_parts(struct index_mail *mail) { unsigned int field_idx = @@ -50,8 +63,7 @@ static struct message_part *get_unserialized_parts(struct index_mail *mail) int ret; part_buf = buffer_create_dynamic(pool_datastack_create(), 128); - ret = mail_cache_lookup_field(mail->trans->cache_view, part_buf, - mail->data.seq, field_idx); + ret = index_mail_cache_lookup_field(mail, part_buf, field_idx); if (ret <= 0) return NULL; @@ -100,8 +112,7 @@ static bool index_mail_get_fixed_field(struct index_mail *mail, buf = buffer_create_data(pool_datastack_create(), data, data_size); - if (mail_cache_lookup_field(mail->trans->cache_view, buf, - mail->data.seq, field_idx) <= 0) + if (index_mail_cache_lookup_field(mail, buf, field_idx) <= 0) ret = FALSE; else { i_assert(buf->used == data_size); @@ -803,8 +814,17 @@ int index_mail_init_stream(struct index_mail *mail, struct istream **stream_r) { struct index_mail_data *data = &mail->data; + struct istream *input; int ret; + if (!data->initialized_wrapper_stream && mail->mail.stats_track) { + input = i_stream_create_mail_stats_counter(&mail->mail, + data->stream); + i_stream_unref(&data->stream); + data->stream = input; + data->initialized_wrapper_stream = TRUE; + } + i_stream_set_destroy_callback(data->stream, index_mail_stream_destroy_callback, mail); @@ -954,11 +974,10 @@ int index_mail_get_special(struct mail *_mail, get_cached_parts(mail)) { index_mail_get_plain_bodystructure(mail, str, FALSE); data->body = str_c(str); - } else if (mail_cache_lookup_field(mail->trans->cache_view, str, - mail->data.seq, body_cache_field) > 0) + } else if (index_mail_cache_lookup_field(mail, str, + body_cache_field) > 0) data->body = str_c(str); - else if (mail_cache_lookup_field(mail->trans->cache_view, str, - mail->data.seq, + else if (index_mail_cache_lookup_field(mail, str, bodystructure_cache_field) > 0) { data->bodystructure = p_strdup(mail->data_pool, str_c(str)); @@ -999,8 +1018,7 @@ int index_mail_get_special(struct mail *_mail, get_cached_parts(mail)) { index_mail_get_plain_bodystructure(mail, str, TRUE); data->bodystructure = str_c(str); - } else if (mail_cache_lookup_field(mail->trans->cache_view, str, - mail->data.seq, + } else if (index_mail_cache_lookup_field(mail, str, bodystructure_cache_field) > 0) { data->bodystructure = str_c(str); } else { diff --git a/src/lib-storage/index/index-mail.h b/src/lib-storage/index/index-mail.h index 212cc617c6..2d16728e0d 100644 --- a/src/lib-storage/index/index-mail.h +++ b/src/lib-storage/index/index-mail.h @@ -109,6 +109,7 @@ struct index_mail_data { unsigned int no_caching:1; unsigned int forced_no_caching:1; unsigned int destroying_stream:1; + unsigned int initialized_wrapper_stream:1; }; struct index_mail { @@ -217,4 +218,7 @@ void index_mail_cache_parse_continue(struct mail *mail); void index_mail_cache_parse_deinit(struct mail *mail, time_t received_date, bool success); +int index_mail_cache_lookup_field(struct index_mail *mail, buffer_t *buf, + unsigned int field_idx); + #endif diff --git a/src/lib-storage/index/istream-mail-stats.c b/src/lib-storage/index/istream-mail-stats.c new file mode 100644 index 0000000000..9eb65cf7de --- /dev/null +++ b/src/lib-storage/index/istream-mail-stats.c @@ -0,0 +1,117 @@ +/* Copyright (c) 2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mail-storage-private.h" +#include "istream-internal.h" +#include "istream-mail-stats.h" + +struct mail_stats_istream { + struct istream_private istream; + + struct mail_private *mail; + unsigned int files_read_increased:1; +}; + +static void i_stream_mail_stats_destroy(struct iostream_private *stream) +{ + struct mail_stats_istream *mstream = + (struct mail_stats_istream *)stream; + + i_stream_unref(&mstream->istream.parent); +} + +static void +i_stream_mail_stats_set_max_buffer_size(struct iostream_private *stream, + size_t max_size) +{ + struct mail_stats_istream *mstream = + (struct mail_stats_istream *)stream; + + mstream->istream.max_buffer_size = max_size; + i_stream_set_max_buffer_size(mstream->istream.parent, max_size); +} + +static ssize_t +i_stream_mail_stats_read_mail_stats(struct istream_private *stream) +{ + struct mail_stats_istream *mstream = + (struct mail_stats_istream *)stream; + size_t pos; + ssize_t ret; + + if (stream->parent->v_offset != + stream->parent_start_offset + stream->istream.v_offset) { + i_stream_seek(stream->parent, stream->parent_start_offset + + stream->istream.v_offset); + } + + stream->buffer = i_stream_get_data(stream->parent, &pos); + if (pos <= stream->pos) { + if ((ret = i_stream_read(stream->parent)) == -2) + return -2; + + if (ret > 0) { + mstream->mail->stats_files_read_bytes+= ret; + if (!mstream->files_read_increased) { + mstream->files_read_increased = TRUE; + mstream->mail->stats_files_read_count++; + } + } + + stream->istream.stream_errno = stream->parent->stream_errno; + stream->istream.eof = stream->parent->eof; + stream->buffer = i_stream_get_data(stream->parent, &pos); + } else { + ret = 0; + } + + stream->pos -= stream->skip; + stream->skip = 0; + + ret = pos > stream->pos ? (ssize_t)(pos - stream->pos) : + (ret == 0 ? 0 : -1); + stream->pos = pos; + i_assert(ret != -1 || stream->istream.eof || + stream->istream.stream_errno != 0); + return ret; +} + +static void +i_stream_mail_stats_seek(struct istream_private *stream, + uoff_t v_offset, bool mark ATTR_UNUSED) +{ + stream->istream.v_offset = v_offset; + stream->skip = stream->pos = 0; +} + +static const struct stat * +i_stream_mail_stats_stat(struct istream_private *stream, bool exact) +{ + return i_stream_stat(stream->parent, exact); +} + +struct istream *i_stream_create_mail_stats_counter(struct mail_private *mail, + struct istream *input) +{ + struct mail_stats_istream *mstream; + + i_stream_ref(input); + + mstream = i_new(struct mail_stats_istream, 1); + mstream->mail = mail; + mstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + + mstream->istream.iostream.destroy = i_stream_mail_stats_destroy; + mstream->istream.iostream.set_max_buffer_size = + i_stream_mail_stats_set_max_buffer_size; + + mstream->istream.parent = input; + mstream->istream.read = i_stream_mail_stats_read_mail_stats; + mstream->istream.seek = i_stream_mail_stats_seek; + mstream->istream.stat = i_stream_mail_stats_stat; + + mstream->istream.istream.blocking = input->blocking; + mstream->istream.istream.seekable = input->seekable; + return i_stream_create(&mstream->istream, input, + i_stream_get_fd(input)); +} diff --git a/src/lib-storage/index/istream-mail-stats.h b/src/lib-storage/index/istream-mail-stats.h new file mode 100644 index 0000000000..2b5e8356f9 --- /dev/null +++ b/src/lib-storage/index/istream-mail-stats.h @@ -0,0 +1,7 @@ +#ifndef ISTREAM_MAIL_STATS_H +#define ISTREAM_MAIL_STATS_H + +struct istream *i_stream_create_mail_stats_counter(struct mail_private *mail, + struct istream *input); + +#endif diff --git a/src/lib-storage/index/maildir/maildir-mail.c b/src/lib-storage/index/maildir/maildir-mail.c index 35561293d0..87200d03ee 100644 --- a/src/lib-storage/index/maildir/maildir-mail.c +++ b/src/lib-storage/index/maildir/maildir-mail.c @@ -44,11 +44,13 @@ static struct istream * maildir_open_mail(struct maildir_mailbox *mbox, struct mail *mail, bool *deleted_r) { + struct mail_private *p = (struct mail_private *)mail; const char *path; int fd = -1; *deleted_r = FALSE; + p->stats_dentry_lookup_count++; if (mail->uid != 0) { if (maildir_file_do(mbox, mail->uid, do_open, &fd) < 0) return NULL; @@ -69,30 +71,32 @@ maildir_open_mail(struct maildir_mailbox *mbox, struct mail *mail, static int maildir_mail_stat(struct mail *mail, struct stat *st) { struct maildir_mailbox *mbox = (struct maildir_mailbox *)mail->box; - struct index_mail_data *data = &((struct index_mail *)mail)->data; + struct index_mail *imail = (struct index_mail *)mail; const char *path; int fd, ret; if (mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE) return mail_set_aborted(mail); - if (data->access_part != 0 && data->stream == NULL) { + if (imail->data.access_part != 0 && imail->data.stream == NULL) { /* we're going to open the mail anyway */ struct istream *input; (void)mail_get_stream(mail, NULL, NULL, &input); } - if (data->stream != NULL) { - fd = i_stream_get_fd(data->stream); + if (imail->data.stream != NULL) { + fd = i_stream_get_fd(imail->data.stream); i_assert(fd != -1); + imail->mail.stats_attr_lookup_count++; if (fstat(fd, st) < 0) { mail_storage_set_critical(&mbox->storage->storage, "fstat(maildir) failed: %m"); return -1; } } else if (mail->uid != 0) { + imail->mail.stats_dentry_lookup_count++; ret = maildir_file_do(mbox, mail->uid, do_stat, st); if (ret <= 0) { if (ret == 0) @@ -100,6 +104,7 @@ static int maildir_mail_stat(struct mail *mail, struct stat *st) return -1; } } else { + imail->mail.stats_dentry_lookup_count++; path = maildir_save_file_get_path(mail->transaction, mail->seq); if (stat(path, st) < 0) { mail_storage_set_critical(mail->box->storage, diff --git a/src/lib-storage/index/raw/raw-mail.c b/src/lib-storage/index/raw/raw-mail.c index 9fc00f17b1..162bec57b3 100644 --- a/src/lib-storage/index/raw/raw-mail.c +++ b/src/lib-storage/index/raw/raw-mail.c @@ -12,11 +12,13 @@ static int raw_mail_stat(struct mail *mail) { struct raw_mailbox *mbox = (struct raw_mailbox *)mail->box; + struct mail_private *p = (struct mail_private *)mail; const struct stat *st; if (mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE) return mail_set_aborted(mail); + p->stats_attr_lookup_count++; st = i_stream_stat(mbox->input, TRUE); if (st == NULL) { mail_storage_set_critical(mail->box->storage, diff --git a/src/lib-storage/mail-storage-private.h b/src/lib-storage/mail-storage-private.h index bcde88c8d4..c71bc25302 100644 --- a/src/lib-storage/mail-storage-private.h +++ b/src/lib-storage/mail-storage-private.h @@ -291,6 +291,22 @@ struct mail_private { pool_t pool; ARRAY_DEFINE(module_contexts, union mail_module_context *); + + /* these statistics are never reset by mail-storage API: */ + + /* open(), stat(), .. */ + unsigned long stats_dentry_lookup_count; + /* fstat() mostly */ + unsigned long stats_attr_lookup_count; + /* number of files we've opened and read */ + unsigned long stats_files_read_count; + /* number of bytes we've had to read from files */ + unsigned long long stats_files_read_bytes; + /* number of cache lookup hits */ + unsigned long stats_cache_hit_count; + + /* Set to TRUE to update stats_* fields */ + unsigned int stats_track:1; }; struct mailbox_list_context {