From: Timo Sirainen Date: Tue, 14 Jun 2011 13:28:55 +0000 (+0300) Subject: Merged changes from v2.0 tree. X-Git-Tag: 2.1.alpha1~246 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=737561538a2dcdcda948a1da2830a612d8263a23;p=thirdparty%2Fdovecot%2Fcore.git Merged changes from v2.0 tree. --- 737561538a2dcdcda948a1da2830a612d8263a23 diff --cc NEWS index 4d733a159f,a8753a9085..cf636436f8 --- a/NEWS +++ b/NEWS @@@ -1,19 -1,22 +1,41 @@@ +v2.1.UNSTABLE 2011-xx-xx Timo Sirainen + + * Plugins now use UTF-8 mailbox names rather than mUTF-7: + acl, autocreate, expire, trash, virtual + * auth_username_format default changed to %Lu. If you really want + case sensitive usernames, set it back to empty. + + + imapc (= IMAP client) storage allows using a remote IMAP server to + be used as storage. This allows using Dovecot as a smart (caching) + proxy or using dsync to do migration from remote IMAP server. + + Autocreate plugin creates/subscribes mailboxes physically only when + the mailbox is opened for the first time. Mailbox listing shows the + autocreated mailboxes even if they don't physically exist. + + Mailbox list indexes + - listescape plugin works perfectly now + ++ - Fixed potential crashes and other problems when parsing header names ++ that contained NUL characters. ++ + v2.0.13 2011-05-11 Timo Sirainen + + + Added "doveadm index" command to add unindexed messages into + index/cache. If full text search is enabled, it also adds unindexed + messages to the fts database. + + added "doveadm director dump" command. + + pop3: Added support for showing messages in "POP3 order", which can + be different from IMAP message order. This can be useful for + migrations from other servers. Implemented it for Maildir as 'O' + field in dovecot-uidlist. + - doveconf: Fixed a wrong "subsection has ssl=yes" warning. + - mdbox purge: Fixed wrong warning about corrupted extrefs. + - sdbox: INBOX GUID changed when INBOX was autocreated, leading to + trouble with dsync. + - script-login binary wasn't actually dropping privileges to the + user/group/chroot specified by its service settings. + - Fixed potential crashes and other problems when parsing header names + that contained NUL characters. + v2.0.12 2011-04-12 Timo Sirainen + doveadm: Added "move" command for moving mails between mailboxes. diff --cc src/doveadm/doveadm-mail-index.c index 0000000000,1ff0757fe1..409797c4e8 mode 000000,100644..100644 --- a/src/doveadm/doveadm-mail-index.c +++ b/src/doveadm/doveadm-mail-index.c @@@ -1,0 -1,247 +1,253 @@@ + /* Copyright (c) 2010-2011 Dovecot authors, see the included COPYING file */ + + #include "lib.h" + #include "mail-namespace.h" + #include "mail-storage.h" + #include "mail-search-build.h" + #include "doveadm-mail.h" + + enum cache_mask { + CACHE_HDR = 0x01, + CACHE_BODY = 0x02, + CACHE_RECEIVED_DATE = 0x04, + CACHE_SAVE_DATE = 0x08, + CACHE_VIRTUAL_SIZE = 0x10, + CACHE_PHYSICAL_SIZE = 0x20, + CACHE_POP3_UIDL = 0x40, + CACHE_GUID = 0x80 + }; + + static bool fts_is_enabled = FALSE; + -static enum cache_mask cache_fields_get(const struct mailbox_status *status) ++static enum cache_mask ++cache_fields_get(const struct mailbox_metadata *metadata) + { + const char *const *cache_fields; + unsigned int i, count; + enum cache_mask cache = 0; + - cache_fields = array_get(status->cache_fields, &count); ++ cache_fields = array_get(metadata->cache_fields, &count); + for (i = 0; i < count; i++) { + if (strncmp(cache_fields[i], "hdr.", 4) == 0 || + strcmp(cache_fields[i], "date.sent") == 0 || + strcmp(cache_fields[i], "imap.envelope") == 0) + cache |= CACHE_HDR; + else if (strcmp(cache_fields[i], "mime.parts") == 0 || + strcmp(cache_fields[i], "imap.body") == 0 || + strcmp(cache_fields[i], "imap.bodystructure") == 0) + cache |= CACHE_BODY; + else if (strcmp(cache_fields[i], "date.received") == 0) + cache |= CACHE_RECEIVED_DATE; + else if (strcmp(cache_fields[i], "date.save") == 0) + cache |= CACHE_SAVE_DATE; + else if (strcmp(cache_fields[i], "size.virtual") == 0) + cache |= CACHE_VIRTUAL_SIZE; + else if (strcmp(cache_fields[i], "size.physical") == 0) + cache |= CACHE_PHYSICAL_SIZE; + else if (strcmp(cache_fields[i], "pop3.uidl") == 0) + cache |= CACHE_POP3_UIDL; + else if (strcmp(cache_fields[i], "guid") == 0) + cache |= CACHE_GUID; + else if (doveadm_debug) { + i_debug("Ignoring unknown cache field: %s", + cache_fields[i]); + } + } + return cache; + } + -static int cache_add(struct mailbox *box, const struct mailbox_status *status, - enum cache_mask cache) ++static int cache_add(struct mailbox *box, enum cache_mask cache) + { ++ struct mailbox_status status; + struct mailbox_transaction_context *trans; + struct mail *mail; + uint32_t seq; + time_t date; + uoff_t size; + const char *str; + + if (doveadm_debug) { + i_debug("%s: Nothing in mailbox cache, skipping", + mailbox_get_vname(box)); + return 0; + } + + /* find the first message we need to index */ ++ mailbox_get_open_status(box, STATUS_MESSAGES, &status); + trans = mailbox_transaction_begin(box, 0); + mail = mail_alloc(trans, 0, NULL); - for (seq = status->messages; seq > 0; seq--) { ++ for (seq = status.messages; seq > 0; seq--) { + mail_set_seq(mail, seq); + if (mail_is_cached(mail)) + break; + } + seq++; + + if (doveadm_debug) { - if (seq > status->messages) { ++ if (seq > status.messages) { + i_debug("%s: Cache is already up to date", + mailbox_get_vname(box)); + } else { + i_debug("%s: Caching mails seq=%u..%u cache=0x%x", + mailbox_get_vname(box), - seq, status->messages, cache); ++ seq, status.messages, cache); + } + } + - for (; seq <= status->messages; seq++) { ++ for (; seq <= status.messages; seq++) { + mail_set_seq(mail, seq); + + if ((cache & (CACHE_HDR | CACHE_BODY)) != 0) + mail_parse(mail, (cache & CACHE_BODY) != 0); + if ((cache & CACHE_RECEIVED_DATE) != 0) + (void)mail_get_received_date(mail, &date); + if ((cache & CACHE_SAVE_DATE) != 0) + (void)mail_get_save_date(mail, &date); + if ((cache & CACHE_VIRTUAL_SIZE) != 0) + (void)mail_get_virtual_size(mail, &size); + if ((cache & CACHE_PHYSICAL_SIZE) != 0) + (void)mail_get_physical_size(mail, &size); + if ((cache & CACHE_POP3_UIDL) != 0) { + (void)mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, + &str); + } + if ((cache & CACHE_GUID) != 0) + (void)mail_get_special(mail, MAIL_FETCH_GUID, &str); + } + mail_free(&mail); + if (mailbox_transaction_commit(&trans) < 0) { + i_error("Commiting mailbox %s failed: %s", + mailbox_get_vname(box), + mail_storage_get_last_error(mailbox_get_storage(box), NULL)); + return -1; + } + return 0; + } + -static int fts_update(struct mailbox *box, const struct mailbox_status *status) ++static int fts_update(struct mailbox *box) + { ++ struct mailbox_status status; + struct mailbox_transaction_context *t; + struct mail_search_args *search_args; + struct mail_search_arg *arg; + struct mail_search_context *ctx; + struct mail *mail; + int ret; + + if (!fts_is_enabled) + return 0; + ++ mailbox_get_open_status(box, STATUS_MESSAGES, &status); ++ + /* a bit kludgy way to trigger the full text search update: + search for a string in the last message */ + t = mailbox_transaction_begin(box, 0); + search_args = mail_search_build_init(); - search_args->charset = "UTF-8"; + mail_search_build_add_seqset(search_args, - status->messages, status->messages); ++ status.messages, status.messages); + arg = mail_search_build_add(search_args, SEARCH_BODY_FAST); + arg->value.str = "xyzzy"; + - ctx = mailbox_search_init(t, search_args, NULL); ++ ctx = mailbox_search_init(t, search_args, NULL, 0, NULL); + mail_search_args_unref(&search_args); + - mail = mail_alloc(t, 0, NULL); - while (mailbox_search_next(ctx, mail)) { ++ while (mailbox_search_next(ctx, &mail)) { + } - mail_free(&mail); + + ret = mailbox_search_deinit(&ctx); + if (mailbox_transaction_commit(&t) < 0) + ret = -1; + return ret; + } + + static int + cmd_index_box(const struct mailbox_info *info) + { + struct mailbox *box; - struct mailbox_status status; - const char *storage_name; ++ struct mailbox_metadata metadata; + enum cache_mask cache; + int ret = 0; + - storage_name = mail_namespace_get_storage_name(info->ns, info->name); - box = mailbox_alloc(info->ns->list, storage_name, ++ box = mailbox_alloc(info->ns->list, info->name, + MAILBOX_FLAG_KEEP_RECENT | + MAILBOX_FLAG_IGNORE_ACLS); + + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { + i_error("Syncing mailbox %s failed: %s", info->name, + mail_storage_get_last_error(mailbox_get_storage(box), NULL)); + mailbox_free(&box); + return -1; + } - mailbox_get_status(box, STATUS_MESSAGES | STATUS_CACHE_FIELDS, &status); ++ if (mailbox_get_metadata(box, MAILBOX_METADATA_CACHE_FIELDS, ++ &metadata) < 0) { ++ i_error("Metadata lookup from mailbox %s failed: %s", info->name, ++ mail_storage_get_last_error(mailbox_get_storage(box), NULL)); ++ mailbox_free(&box); ++ return -1; ++ } + - cache = cache_fields_get(&status); - ret = cache_add(box, &status, cache); ++ cache = cache_fields_get(&metadata); ++ ret = cache_add(box, cache); + - if (fts_update(box, &status) < 0) ++ if (fts_update(box) < 0) + ret = -1; + + mailbox_free(&box); + return ret; + } + + static void + cmd_index_run(struct doveadm_mail_cmd_context *ctx, struct mail_user *user) + { + const enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_RAW_LIST | - MAILBOX_LIST_ITER_NO_AUTO_INBOX | ++ MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS | + MAILBOX_LIST_ITER_STAR_WITHIN_NS; + const enum namespace_type ns_mask = + NAMESPACE_PRIVATE | NAMESPACE_SHARED | NAMESPACE_PUBLIC; + struct mailbox_list_iterate_context *iter; + const struct mailbox_info *info; + + if (mail_user_plugin_getenv(user, "fts") != NULL) T_BEGIN { + const char *const *plugins; + + plugins = t_strsplit(user->set->mail_plugins, " "); + for (; *plugins != NULL; plugins++) { + if (strncmp(*plugins, "fts", 3) == 0) + fts_is_enabled = TRUE; + } + } T_END; + + iter = mailbox_list_iter_init_namespaces(user->namespaces, ctx->args, + ns_mask, iter_flags); + while ((info = mailbox_list_iter_next(iter)) != NULL) { + if ((info->flags & (MAILBOX_NOSELECT | + MAILBOX_NONEXISTENT)) == 0) T_BEGIN { + (void)cmd_index_box(info); + } T_END; + } + if (mailbox_list_iter_deinit(&iter) < 0) + i_error("Listing mailboxes failed"); + } + + static void cmd_index_init(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED, + const char *const args[]) + { + if (args[0] == NULL) + doveadm_mail_help_name("index"); + } + + static struct doveadm_mail_cmd_context *cmd_index_alloc(void) + { + struct doveadm_mail_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context); + ctx->v.init = cmd_index_init; + ctx->v.run = cmd_index_run; + return ctx; + } + + struct doveadm_mail_cmd cmd_index = { + cmd_index_alloc, "index", "" + }; diff --cc src/doveadm/doveadm-mail.c index 9574238f70,23d7143a87..858c87baca --- a/src/doveadm/doveadm-mail.c +++ b/src/doveadm/doveadm-mail.c @@@ -111,8 -108,9 +111,9 @@@ int doveadm_mailbox_find_and_sync(struc return -1; if (mailbox_sync(*box_r, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { i_error("Syncing mailbox %s failed: %s", mailbox, - mail_storage_get_last_error(mailbox_get_storage(*box_r), - NULL)); + mailbox_get_last_error(*box_r, NULL)); + mailbox_free(box_r); + mailbox_free(box_r); return -1; } return 0; diff --cc src/imap/cmd-append.c index 06a1fe5a54,d1fecbe862..04b8aabefa --- a/src/imap/cmd-append.c +++ b/src/imap/cmd-append.c @@@ -468,7 -527,9 +486,8 @@@ bool cmd_append(struct client_command_c ctx = p_new(cmd->pool, struct cmd_append_context, 1); ctx->cmd = cmd; ctx->client = client; - ctx->box = get_mailbox(cmd, mailbox); + ctx->started = ioloop_time; - if (ctx->box == NULL) + if (client_open_save_dest_box(cmd, mailbox, &ctx->box) < 0) ctx->failed = TRUE; else { ctx->storage = mailbox_get_storage(ctx->box); diff --cc src/lib-storage/index/imapc/imapc-mail.c index 8db89e66f3,0000000000..1a945cd77f mode 100644,000000..100644 --- a/src/lib-storage/index/imapc/imapc-mail.c +++ b/src/lib-storage/index/imapc/imapc-mail.c @@@ -1,248 -1,0 +1,249 @@@ +/* Copyright (c) 2011 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "istream.h" +#include "imap-envelope.h" +#include "imapc-seqmap.h" +#include "imapc-mail.h" +#include "imapc-client.h" +#include "imapc-storage.h" + +struct mail * +imapc_mail_alloc(struct mailbox_transaction_context *t, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers) +{ + struct imapc_mail *mail; + pool_t pool; + + pool = pool_alloconly_create("mail", 2048); + mail = p_new(pool, struct imapc_mail, 1); + mail->imail.mail.pool = pool; + + index_mail_init(&mail->imail, t, wanted_fields, wanted_headers); + return &mail->imail.mail.mail; +} + +static void imapc_mail_free(struct mail *_mail) +{ + struct imapc_mail *mail = (struct imapc_mail *)_mail; + + if (mail->body != NULL) + buffer_free(&mail->body); + index_mail_free(_mail); +} + +static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = (struct index_mail *)_mail; + struct index_mail_data *data = &mail->data; + + if (index_mail_get_received_date(_mail, date_r) == 0) + return 0; + + if (data->received_date == (time_t)-1) { + if (imapc_mail_fetch(_mail, MAIL_FETCH_RECEIVED_DATE) < 0) + return -1; + if (data->received_date == (time_t)-1) { + mail_storage_set_critical(_mail->box->storage, + "imapc: Remote server didn't send INTERNALDATE"); + return -1; + } + } + *date_r = data->received_date; + return 0; +} + +static int imapc_mail_get_save_date(struct mail *_mail, time_t *date_r) +{ + struct index_mail *mail = (struct index_mail *)_mail; + struct index_mail_data *data = &mail->data; + + if (data->save_date == (time_t)-1) { + /* FIXME */ + return -1; + } + *date_r = data->save_date; + return 0; +} + +static int imapc_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) +{ + struct index_mail *mail = (struct index_mail *)_mail; + struct index_mail_data *data = &mail->data; + struct istream *input; + uoff_t old_offset; + int ret; + + if (data->physical_size == (uoff_t)-1) + (void)index_mail_get_physical_size(_mail, size_r); + if (data->physical_size == (uoff_t)-1) { + old_offset = data->stream == NULL ? 0 : data->stream->v_offset; + if (mail_get_stream(_mail, NULL, NULL, &input) < 0) + return -1; + i_stream_seek(data->stream, old_offset); + + ret = i_stream_get_size(data->stream, TRUE, + &data->physical_size); + if (ret <= 0) { + i_assert(ret != 0); + mail_storage_set_critical(_mail->box->storage, + "imapc: stat(%s) failed: %m", + i_stream_get_name(data->stream)); + return -1; + } + } + *size_r = data->physical_size; + return 0; +} + +static bool imapc_mail_is_expunged(struct mail *_mail) +{ + struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box; + struct imapc_seqmap *seqmap; + uint32_t lseq; + + /* first we'll need to convert the mail's sequence to sync_view's + sequence. if there's no sync_view, then no mails have been + expunged. */ + if (mbox->sync_view == NULL) + return FALSE; + + if (!mail_index_lookup_seq(mbox->sync_view, _mail->uid, &lseq)) + return TRUE; + + seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box); + return imapc_seqmap_lseq_to_rseq(seqmap, lseq) == 0; +} + +static int +imapc_mail_get_stream(struct mail *_mail, struct message_size *hdr_size, + struct message_size *body_size, struct istream **stream_r) +{ + struct index_mail *mail = (struct index_mail *)_mail; + struct index_mail_data *data = &mail->data; + enum mail_fetch_field fetch_field; + + if (data->stream == NULL) { + if (!mail->data.initialized) { + /* coming here from mail_set_seq() */ + return mail_set_aborted(_mail); + } + fetch_field = body_size != NULL || + (mail->wanted_fields & MAIL_FETCH_STREAM_BODY) != 0 ? + MAIL_FETCH_STREAM_BODY : MAIL_FETCH_STREAM_HEADER; + if (imapc_mail_fetch(_mail, fetch_field) < 0) + return -1; + + if (data->stream == NULL) { + if (_mail->expunged || imapc_mail_is_expunged(_mail)) + mail_set_expunged(_mail); + else { + mail_storage_set_critical(_mail->box->storage, + "imapc: Remote server didn't send BODY[]"); + } + return -1; + } + } + + return index_mail_init_stream(mail, hdr_size, body_size, stream_r); +} + +static bool +imapc_mail_has_headers_in_cache(struct index_mail *mail, + struct mailbox_header_lookup_ctx *headers) +{ + struct mail *_mail = &mail->mail.mail; + unsigned int i; + + for (i = 0; i < headers->count; i++) { + if (mail_cache_field_exists(_mail->transaction->cache_view, + _mail->seq, headers->idx[i]) <= 0) + return FALSE; + } + return TRUE; +} + +static void imapc_mail_set_seq(struct mail *_mail, uint32_t seq) +{ + struct imapc_mail *imail = (struct imapc_mail *)_mail; + struct index_mail *mail = &imail->imail; + struct mailbox_header_lookup_ctx *header_ctx; + time_t date; + uoff_t size; + + index_mail_set_seq(_mail, seq); + + if ((mail->wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0) + (void)index_mail_get_received_date(_mail, &date); + if ((mail->wanted_fields & MAIL_FETCH_PHYSICAL_SIZE) != 0) { + if (index_mail_get_physical_size(_mail, &size) < 0) + mail->data.access_part |= READ_HDR | READ_BODY; + } + + if (mail->data.access_part == 0 && mail->wanted_headers != NULL) { + /* see if all wanted headers exist in cache */ + if (!imapc_mail_has_headers_in_cache(mail, mail->wanted_headers)) + mail->data.access_part |= PARSE_HDR; + } + if (mail->data.access_part == 0 && + (mail->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE) != 0) { + /* the common code already checked this partially, + but we need a guaranteed correct answer */ + header_ctx = mailbox_header_lookup_init(_mail->box, + imap_envelope_headers); + if (!imapc_mail_has_headers_in_cache(mail, header_ctx)) + mail->data.access_part |= PARSE_HDR; + mailbox_header_lookup_unref(&header_ctx); + } + /* searching code handles prefetching internally, + elsewhere we want to do it immediately */ + if (!mail->search_mail) + (void)imapc_mail_prefetch(_mail); +} + +static void imapc_mail_close(struct mail *_mail) +{ + struct imapc_mail *imail = (struct imapc_mail *)_mail; + struct imapc_storage *storage = + (struct imapc_storage *)_mail->box->storage; + + while (imail->fetch_count > 0) + imapc_client_run(storage->client); + index_mail_close(_mail); +} + +struct mail_vfuncs imapc_mail_vfuncs = { + imapc_mail_close, + imapc_mail_free, + imapc_mail_set_seq, + index_mail_set_uid, + index_mail_set_uid_cache_updates, + imapc_mail_prefetch, + + index_mail_get_flags, + index_mail_get_keywords, + index_mail_get_keyword_indexes, + index_mail_get_modseq, + index_mail_get_parts, + index_mail_get_date, + imapc_mail_get_received_date, + imapc_mail_get_save_date, + index_mail_get_virtual_size, + imapc_mail_get_physical_size, + index_mail_get_first_header, + index_mail_get_headers, + index_mail_get_header_stream, + imapc_mail_get_stream, + index_mail_get_special, + index_mail_get_real_mail, + index_mail_update_flags, + index_mail_update_keywords, + index_mail_update_modseq, + NULL, + index_mail_expunge, ++ index_mail_parse, + index_mail_set_cache_corrupted, + index_mail_opened +}; diff --cc src/lib-storage/list/mailbox-list-none.c index d965b85ec3,8254bcdb3d..6807560b75 --- a/src/lib-storage/list/mailbox-list-none.c +++ b/src/lib-storage/list/mailbox-list-none.c @@@ -137,14 -138,16 +137,15 @@@ none_list_iter_init(struct mailbox_lis ctx = i_new(struct noop_list_iterate_context, 1); ctx->ctx.list = list; ctx->ctx.flags = flags; - if ((list->ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0) T_BEGIN { + ctx->ctx.glob = imap_match_init_multiple(default_pool, patterns, TRUE, + mail_namespace_get_sep(list->ns)); + array_create(&ctx->ctx.module_contexts, default_pool, sizeof(void *), 5); - if ((list->ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0) { ++ if ((list->ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0 && ++ imap_match(ctx->ctx.glob, "INBOX") == IMAP_MATCH_YES) { + ctx->list_inbox = TRUE; ctx->inbox_info.ns = list->ns; ctx->inbox_info.name = "INBOX"; - - glob = imap_match_init_multiple(pool_datastack_create(), - patterns, TRUE, - list->hierarchy_sep); - if (imap_match(glob, "INBOX") == IMAP_MATCH_YES) - ctx->list_inbox = TRUE; - } T_END; + } return &ctx->ctx; } diff --cc src/lib-storage/list/mailbox-list-subscriptions.c index 12c28ec1f7,453f5f3e38..b186e97e4c --- a/src/lib-storage/list/mailbox-list-subscriptions.c +++ b/src/lib-storage/list/mailbox-list-subscriptions.c @@@ -9,165 -6,26 +9,169 @@@ #include "mailbox-list-private.h" #include "mailbox-list-subscriptions.h" +#include + +struct subscriptions_mailbox_list_iterate_context { + struct mailbox_list_iterate_context ctx; + struct mailbox_tree_context *tree; + struct mailbox_tree_iterate_context *iter; + struct mailbox_info info; +}; + static int -mailbox_list_subscriptions_fill_real(struct mailbox_list_iterate_context *ctx, - struct mailbox_tree_context *tree_ctx, - struct imap_match_glob *glob, - bool update_only) +mailbox_list_subscription_fill_one(struct mailbox_list *list, + const char *name) { - struct mail_namespace *default_ns = ctx->list->ns; + struct mail_namespace *ns, *default_ns = list->ns; struct mail_namespace *namespaces = default_ns->user->namespaces; - struct mailbox_list_iter_update_context update_ctx; + struct mailbox_node *node; + const char *vname; + unsigned int len; + bool created; + + /* default_ns is whatever namespace we're currently listing. + if we have e.g. prefix="" and prefix=pub/ namespaces with + pub/ namespace having subscriptions=no, we want to: + + 1) when listing "" namespace we want to skip over any names + that begin with pub/. */ + ns = mail_namespace_find_unsubscribable(namespaces, name); + if (ns != NULL && ns != default_ns) + return 0; + + /* 2) when listing pub/ namespace, skip over entries that don't + begin with pub/. */ + if (ns == NULL && + (default_ns->flags & NAMESPACE_FLAG_SUBSCRIPTIONS) == 0) + return 0; + + /* When listing shared namespace's subscriptions, we need to + autocreate all the visible child namespaces and use the + child namespace. */ + if (ns != NULL && ns->type == NAMESPACE_SHARED && + (ns->flags & NAMESPACE_FLAG_AUTOCREATED) == 0) { + /* we'll need to get the namespace autocreated. + one easy way is to just ask if a mailbox name under + it is valid, and it gets created */ + (void)mailbox_list_is_valid_existing_name(list, name); + ns = mail_namespace_find_unsubscribable(namespaces, name); + i_assert(ns != NULL && + (ns->flags & NAMESPACE_FLAG_AUTOCREATED) != 0); + } + + /* When listing pub/ namespace, skip over the namespace + prefix in the name. the rest of the name is storage_name. */ + if (ns != NULL) { + i_assert(strncmp(name, ns->prefix, ns->prefix_len) == 0); + name += ns->prefix_len; + } else { + ns = default_ns; + } + + len = strlen(name); + if (len > 0 && name[len-1] == mail_namespace_get_sep(ns)) { + /* entry ends with hierarchy separator, remove it. + this exists mainly for backwards compatibility with old + Dovecot versions and non-Dovecot software that added them */ + name = t_strndup(name, len-1); + } + + if (!mailbox_list_is_valid_existing_name(list, name)) { + /* we'll only get into trouble if we show this */ + return -1; + } else { + vname = mailbox_list_get_vname(list, name); + node = mailbox_tree_get(list->subscriptions, vname, &created); + node->flags = MAILBOX_SUBSCRIBED; + } + return 0; +} + +int mailbox_list_subscriptions_refresh(struct mailbox_list *src_list, + struct mailbox_list *dest_list) +{ struct subsfile_list_context *subsfile_ctx; - struct mail_namespace *ns; - const char *path, *name, *name2, *full_name, *orig_name; - string_t *vname; + struct stat st; + const char *path, *name; + char sep; + + i_assert((src_list->ns->flags & NAMESPACE_FLAG_SUBSCRIPTIONS) != 0); + + if (dest_list->subscriptions == NULL) { + sep = mail_namespace_get_sep(src_list->ns); + dest_list->subscriptions = mailbox_tree_init(sep); + } + path = t_strconcat(src_list->set.control_dir != NULL ? + src_list->set.control_dir : src_list->set.root_dir, + "/", src_list->set.subscription_fname, NULL); + if (stat(path, &st) < 0) { + if (errno == ENOENT) { + /* no subscriptions */ + mailbox_tree_clear(dest_list->subscriptions); + dest_list->subscriptions_mtime = 0; + return 0; + } + mailbox_list_set_critical(dest_list, "stat(%s) failed: %m", + path); + return -1; + } + if (st.st_mtime == dest_list->subscriptions_mtime && + st.st_mtime < dest_list->subscriptions_read_time-1) { + /* we're up to date */ + return 0; + } + + mailbox_tree_clear(dest_list->subscriptions); + dest_list->subscriptions_read_time = ioloop_time; + + subsfile_ctx = subsfile_list_init(dest_list, path); + if (subsfile_list_fstat(subsfile_ctx, &st) == 0) + dest_list->subscriptions_mtime = st.st_mtime; + while ((name = subsfile_list_next(subsfile_ctx)) != NULL) T_BEGIN { + if (mailbox_list_subscription_fill_one(dest_list, name) < 0) { + i_warning("Subscriptions file %s: " - "Ignoring invalid entry: %s", ++ "Removing invalid entry: %s", + path, name); ++ (void)subsfile_set_subscribed(dest_list, path, ++ mailbox_list_get_temp_prefix(dest_list), ++ name, FALSE); ++ + } + } T_END; + + if (subsfile_list_deinit(&subsfile_ctx) < 0) { + dest_list->subscriptions_mtime = (time_t)-1; + return -1; + } + return 0; +} + +void mailbox_list_set_subscription_flags(struct mailbox_list *list, + const char *vname, + enum mailbox_info_flags *flags) +{ + struct mailbox_node *node; - vname = str_new(default_pool, 256); - path = t_strconcat(ctx->list->set.control_dir != NULL ? - ctx->list->set.control_dir : - ctx->list->set.root_dir, - "/", ctx->list->set.subscription_fname, NULL); - subsfile_ctx = subsfile_list_init(ctx->list, path); + *flags &= ~(MAILBOX_SUBSCRIBED | MAILBOX_CHILD_SUBSCRIBED); + + node = mailbox_tree_lookup(list->subscriptions, vname); + if (node != NULL) { + *flags |= node->flags & MAILBOX_SUBSCRIBED; + + /* the only reason why node might have a child is if one of + them is subscribed */ + if (node->children != NULL) + *flags |= MAILBOX_CHILD_SUBSCRIBED; + } +} + +void mailbox_list_subscriptions_fill(struct mailbox_list_iterate_context *ctx, + struct mailbox_tree_context *tree) +{ + struct mailbox_list_iter_update_context update_ctx; + struct mailbox_tree_iterate_context *iter; + struct mailbox_node *node; + const char *name; memset(&update_ctx, 0, sizeof(update_ctx)); update_ctx.iter_ctx = ctx; diff --cc src/login-common/login-proxy.c index b85d94aab5,6e05bfc676..4fa62bb02f --- a/src/login-common/login-proxy.c +++ b/src/login-common/login-proxy.c @@@ -520,6 -576,123 +576,123 @@@ void login_proxy_kill_idle(void } } + static void + login_proxy_cmd_kick(struct ipc_cmd *cmd, const char *const *args) + { + struct login_proxy *proxy, *next; + unsigned int count = 0; + + if (args[0] == NULL) { + ipc_cmd_fail(&cmd, "Missing parameter"); + return; + } + + for (proxy = login_proxies; proxy != NULL; proxy = next) { + next = proxy->next; + + if (strcmp(proxy->client->virtual_user, args[0]) == 0) { + login_proxy_free_reason(&proxy, KILLED_BY_ADMIN_REASON); + count++; + } + } + for (proxy = login_proxies_pending; proxy != NULL; proxy = next) { + next = proxy->next; + + if (strcmp(proxy->client->virtual_user, args[0]) == 0) { + client_destroy(proxy->client, "Connection kicked"); + count++; + } + } + ipc_cmd_success_reply(&cmd, t_strdup_printf("%u", count)); + } + + static unsigned int director_username_hash(const char *username) + { + /* NOTE: If you modify this, modify also + user_directory_get_username_hash() in director/user-director.c */ + unsigned char md5[MD5_RESULTLEN]; + unsigned int i, hash = 0; + + md5_get_digest(username, strlen(username), md5); + for (i = 0; i < sizeof(hash); i++) + hash = (hash << CHAR_BIT) | md5[i]; + return hash; + } + + static void + login_proxy_cmd_kick_director_hash(struct ipc_cmd *cmd, const char *const *args) + { + struct login_proxy *proxy, *next; + unsigned int hash, count = 0; + + if (args[0] == NULL || str_to_uint(args[0], &hash) < 0) { + ipc_cmd_fail(&cmd, "Invalid parameters"); + return; + } + + for (proxy = login_proxies; proxy != NULL; proxy = next) { + next = proxy->next; + + if (director_username_hash(proxy->client->virtual_user) == hash) { + login_proxy_free_reason(&proxy, KILLED_BY_ADMIN_REASON); + count++; + } + } + for (proxy = login_proxies_pending; proxy != NULL; proxy = next) { + next = proxy->next; + + if (director_username_hash(proxy->client->virtual_user) == hash) { + client_destroy(proxy->client, "Connection kicked"); + count++; + } + } + ipc_cmd_success_reply(&cmd, t_strdup_printf("%u", count)); + } + + static void + login_proxy_cmd_list_reply(struct ipc_cmd *cmd, + struct login_proxy *proxy) + { + T_BEGIN { + const char *reply; + + reply = t_strdup_printf("%s\t%s\t%s\t%s\t%u", + proxy->client->virtual_user, - login_binary.protocol, ++ login_binary->protocol, + net_ip2addr(&proxy->client->ip), + net_ip2addr(&proxy->ip), proxy->port); + ipc_cmd_send(cmd, reply); + } T_END; + } + + static void + login_proxy_cmd_list(struct ipc_cmd *cmd, const char *const *args ATTR_UNUSED) + { + struct login_proxy *proxy; + + for (proxy = login_proxies; proxy != NULL; proxy = proxy->next) + login_proxy_cmd_list_reply(cmd, proxy); + for (proxy = login_proxies_pending; proxy != NULL; proxy = proxy->next) + login_proxy_cmd_list_reply(cmd, proxy); + ipc_cmd_success(&cmd); + } + + static void login_proxy_ipc_cmd(struct ipc_cmd *cmd, const char *line) + { + const char *const *args = t_strsplit(line, "\t"); + const char *name = args[0]; + + args++; + if (strcmp(name, "KICK") == 0) + login_proxy_cmd_kick(cmd, args); + else if (strcmp(name, "KICK-DIRECTOR-HASH") == 0) + login_proxy_cmd_kick_director_hash(cmd, args); + else if (strcmp(name, "LIST") == 0) + login_proxy_cmd_list(cmd, args); + else + ipc_cmd_fail(&cmd, "Unknown command"); + } + void login_proxy_init(const char *proxy_notify_pipe_path) { proxy_state = login_proxy_state_init(proxy_notify_pipe_path); diff --cc src/plugins/acl/doveadm-acl.c index a1bcad4cd5,a9370232a9..71ff738e6b --- a/src/plugins/acl/doveadm-acl.c +++ b/src/plugins/acl/doveadm-acl.c @@@ -351,6 -353,51 +351,50 @@@ cmd_acl_delete_alloc(void return ctx; } + static int + cmd_acl_debug_mailbox_open(struct mail_user *user, const char *mailbox, + struct mailbox **box_r) + { + struct acl_user *auser = ACL_USER_CONTEXT(user); + struct mail_namespace *ns; + struct mailbox *box; - const char *storage_name, *path, *errstr; ++ const char *path, *errstr; + enum mail_error error; + - storage_name = mailbox; - ns = mail_namespace_find(user->namespaces, &storage_name); ++ ns = mail_namespace_find(user->namespaces, mailbox); + if (ns == NULL) { + i_error("No namespace found for mailbox %s", mailbox); + return -1; + } - box = mailbox_alloc(ns->list, storage_name, ++ box = mailbox_alloc(ns->list, mailbox, + MAILBOX_FLAG_READONLY | MAILBOX_FLAG_KEEP_RECENT | + MAILBOX_FLAG_IGNORE_ACLS); + if (mailbox_open(box) < 0) { - path = mailbox_list_get_path(ns->list, storage_name, ++ path = mailbox_list_get_path(ns->list, box->name, + MAILBOX_LIST_PATH_TYPE_MAILBOX); + errstr = mail_storage_get_last_error(box->storage, &error); + if (error != MAIL_ERROR_NOTFOUND || + path == NULL || *path == '\0') + i_error("Can't open mailbox %s: %s", mailbox, errstr); + else { + i_error("Mailbox '%s' doesn't exist in %s", + mailbox, path); + } + mailbox_free(&box); + return -1; + } + + if (auser == NULL) { + i_info("ACL not enabled for user %s, mailbox can be accessed", + user->username); + mailbox_free(&box); + return -1; + } + + *box_r = box; + return 0; + } + static bool cmd_acl_debug_mailbox(struct mailbox *box, bool *retry_r) { struct mail_namespace *ns = mailbox_get_namespace(box); diff --cc src/pop3/pop3-client.c index 8cd430a72b,74d8ff0981..c4d6d54029 --- a/src/pop3/pop3-client.c +++ b/src/pop3/pop3-client.c @@@ -122,15 -153,16 +151,16 @@@ static int read_mailbox(struct client * search_args = mail_search_build_init(); mail_search_build_add_all(search_args); - ctx = mailbox_search_init(t, search_args, NULL, - ctx = mailbox_search_init(t, search_args, pop3_sort_program); ++ ctx = mailbox_search_init(t, search_args, pop3_sort_program, + MAIL_FETCH_VIRTUAL_SIZE, NULL); mail_search_args_unref(&search_args); - client->last_seen = 0; + client->last_seen_pop3_msn = 0; client->total_size = 0; i_array_init(&message_sizes, client->messages_count); + msgnum = 0; - mail = mail_alloc(t, MAIL_FETCH_VIRTUAL_SIZE, NULL); - while (mailbox_search_next(ctx, mail)) { + while (mailbox_search_next(ctx, &mail)) { if (pop3_mail_get_size(client, mail, &size) < 0) { ret = mail->expunged ? 0 : -1; *failed_uid_r = mail->uid; @@@ -142,7 -175,9 +173,8 @@@ client->total_size += size; array_append(&message_sizes, &size, 1); + msgnum++; } - mail_free(&mail); if (mailbox_search_deinit(&ctx) < 0) ret = -1; @@@ -268,9 -316,10 +309,9 @@@ struct client *client_create(int fd_in pop3_client_count++; DLLIST_PREPEND(&pop3_clients, client); - inbox = "INBOX"; - ns = mail_namespace_find(user->namespaces, &inbox); + ns = mail_namespace_find(user->namespaces, "INBOX"); if (ns == NULL) { - client_send_line(client, "-ERR No INBOX namespace for user."); + client_send_line(client, "-ERR [IN-USE] No INBOX namespace for user."); client_destroy(client, "No INBOX namespace for user."); return NULL; } diff --cc src/pop3/pop3-commands.c index 977044a9cf,f2e752dccd..ae5240a61d --- a/src/pop3/pop3-commands.c +++ b/src/pop3/pop3-commands.c @@@ -210,24 -237,35 +237,33 @@@ bool client_update_mails(struct client } search_args = pop3_search_build(client, 0); - ctx = mailbox_search_init(client->trans, search_args, NULL, 0, NULL); + ctx = mailbox_search_init(client->trans, search_args, - pop3_sort_program); ++ pop3_sort_program, 0, NULL); mail_search_args_unref(&search_args); + msgnum = 0; - mail = mail_alloc(client->trans, 0, NULL); - while (mailbox_search_next(ctx, mail)) { + while (mailbox_search_next(ctx, &mail)) { - idx = mail->seq - 1; - bit = 1 << (idx % CHAR_BIT); + if (client_verify_ordering(client, mail, msgnum) < 0) { + ret = FALSE; + break; + } + + bit = 1 << (msgnum % CHAR_BIT); if (client->deleted_bitmask != NULL && - (client->deleted_bitmask[idx / CHAR_BIT] & bit) != 0) { + (client->deleted_bitmask[msgnum / CHAR_BIT] & bit) != 0) { mail_expunge(mail); client->expunged_count++; } else if (client->seen_bitmask != NULL && - (client->seen_bitmask[idx / CHAR_BIT] & bit) != 0) { + (client->seen_bitmask[msgnum / CHAR_BIT] & bit) != 0) { mail_update_flags(mail, MODIFY_ADD, MAIL_SEEN); } + msgnum++; } - mail_free(&mail); client->seen_change_count = 0; - return mailbox_search_deinit(&ctx) == 0; + if (mailbox_search_deinit(&ctx) < 0) + ret = FALSE; + return ret; } static int cmd_quit(struct client *client, const char *args ATTR_UNUSED) @@@ -269,7 -309,7 +307,6 @@@ struct fetch_context static void fetch_deinit(struct fetch_context *ctx) { - (void)mailbox_search_deinit(&ctx->search_ctx); - mail_free(&ctx->mail); i_free(ctx); } @@@ -384,22 -423,21 +420,18 @@@ static int client_reply_msg_expunged(st return 1; } - static int fetch(struct client *client, unsigned int msgnum, uoff_t body_lines) + static int fetch(struct client *client, unsigned int msgnum, uoff_t body_lines, + uoff_t *byte_counter) { struct fetch_context *ctx; - struct mail_search_args *search_args; int ret; - search_args = pop3_search_build(client, msgnum+1); - ctx = i_new(struct fetch_context, 1); - ctx->search_ctx = mailbox_search_init(client->trans, search_args, NULL, - MAIL_FETCH_STREAM_HEADER | - MAIL_FETCH_STREAM_BODY, NULL); - mail_search_args_unref(&search_args); + ctx->byte_counter = byte_counter; + ctx->byte_counter_offset = client->output->offset; - - ctx->mail = mail_alloc(client->trans, MAIL_FETCH_STREAM_HEADER | - MAIL_FETCH_STREAM_BODY, NULL); + mail_set_seq(ctx->mail, msgnum_to_seq(client, msgnum)); - if (!mailbox_search_next(ctx->search_ctx, &ctx->mail) || - mail_get_stream(ctx->mail, NULL, NULL, &ctx->stream) < 0) { + if (mail_get_stream(ctx->mail, NULL, NULL, &ctx->stream) < 0) { ret = client_reply_msg_expunged(client, msgnum); fetch_deinit(ctx); return ret; @@@ -596,12 -633,14 +625,14 @@@ static bool list_uids_iter(struct clien } str = t_str_new(128); - while (mailbox_search_next(ctx->search_ctx, ctx->mail)) { + while (mailbox_search_next(ctx->search_ctx, &ctx->mail)) { - uint32_t idx = ctx->mail->seq - 1; + uint32_t msgnum = ctx->msgnum++; + if (client_verify_ordering(client, ctx->mail, msgnum) < 0) + i_fatal("Can't finish POP3 UIDL command"); if (client->deleted) { - if (client->deleted_bitmask[idx / CHAR_BIT] & - (1 << (idx % CHAR_BIT))) + if (client->deleted_bitmask[msgnum / CHAR_BIT] & + (1 << (msgnum % CHAR_BIT))) continue; } found = TRUE; @@@ -665,11 -704,12 +695,12 @@@ cmd_uidl_init(struct client *client, ui if ((client->uidl_keymask & UIDL_MD5) != 0) wanted_fields |= MAIL_FETCH_HEADER_MD5; - ctx->search_ctx = mailbox_search_init(client->trans, search_args, NULL, + ctx->search_ctx = mailbox_search_init(client->trans, search_args, - pop3_sort_program); ++ pop3_sort_program, + wanted_fields, NULL); mail_search_args_unref(&search_args); - if (message == 0) { - ctx->mail = mail_alloc(client->trans, wanted_fields, NULL); + if (seq == 0) { client->cmd = cmd_uidl_callback; client->cmd_context = ctx; }