mail-index-map.c \
mail-index-modseq.c \
mail-index-transaction.c \
+ mail-index-transaction-export.c \
+ mail-index-transaction-finish.c \
+ mail-index-transaction-sort-appends.c \
mail-index-transaction-view.c \
mail-index-strmap.c \
mail-index-sync.c \
--- /dev/null
+/* Copyright (c) 2003-2009 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "mail-index-private.h"
+#include "mail-index-modseq.h"
+#include "mail-transaction-log-private.h"
+#include "mail-index-transaction-private.h"
+
+struct mail_index_export_context {
+ struct mail_index_transaction *trans;
+ struct mail_transaction_log_append_ctx *append_ctx;
+};
+
+static void log_append_buffer(struct mail_index_export_context *ctx,
+ const buffer_t *buf, const buffer_t *hdr_buf,
+ enum mail_transaction_type type)
+{
+ buffer_t *output = ctx->append_ctx->output;
+ struct mail_transaction_header hdr;
+
+ i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
+ i_assert((buf->used % 4) == 0);
+ i_assert(hdr_buf == NULL || (hdr_buf->used % 4) == 0);
+
+ if (buf->used == 0)
+ return;
+
+ memset(&hdr, 0, sizeof(hdr));
+ hdr.type = type;
+ if (type == MAIL_TRANSACTION_EXPUNGE)
+ hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT;
+ if ((ctx->trans->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
+ hdr.type |= MAIL_TRANSACTION_EXTERNAL;
+ hdr.size = sizeof(hdr) + buf->used +
+ (hdr_buf == NULL ? 0 : hdr_buf->used);
+ hdr.size = mail_index_uint32_to_offset(hdr.size);
+
+ buffer_append(output, &hdr, sizeof(hdr));
+ if (hdr_buf != NULL)
+ buffer_append(output, hdr_buf->data, hdr_buf->used);
+ buffer_append(output, buf->data, buf->used);
+
+ if (mail_transaction_header_has_modseq(buf->data,
+ CONST_PTR_OFFSET(buf->data, sizeof(hdr)),
+ ctx->append_ctx->new_highest_modseq))
+ ctx->append_ctx->new_highest_modseq++;
+}
+
+static const buffer_t *
+log_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend)
+{
+ buffer_t *buf;
+ const unsigned char *data, *mask;
+ struct mail_transaction_header_update u;
+ uint16_t offset;
+ int state = 0;
+
+ memset(&u, 0, sizeof(u));
+
+ data = prepend ? t->pre_hdr_change : t->post_hdr_change;
+ mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask;
+
+ buf = buffer_create_dynamic(pool_datastack_create(), 256);
+ for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) {
+ if (offset < sizeof(t->pre_hdr_change) && mask[offset]) {
+ if (state == 0) {
+ u.offset = offset;
+ state++;
+ }
+ } else {
+ if (state > 0) {
+ u.size = offset - u.offset;
+ buffer_append(buf, &u, sizeof(u));
+ buffer_append(buf, data + u.offset, u.size);
+ state = 0;
+ }
+ }
+ }
+ return buf;
+}
+
+static void log_append_ext_intro(struct mail_index_export_context *ctx,
+ uint32_t ext_id, uint32_t reset_id)
+{
+ struct mail_index_transaction *t = ctx->trans;
+ const struct mail_index_registered_ext *rext;
+ struct mail_transaction_ext_intro *intro, *resizes;
+ buffer_t *buf;
+ uint32_t idx;
+ unsigned int count;
+
+ i_assert(ext_id != (uint32_t)-1);
+
+ if (t->reset ||
+ !mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
+ /* new extension */
+ idx = (uint32_t)-1;
+ }
+
+ rext = array_idx(&t->view->index->extensions, ext_id);
+ if (!array_is_created(&t->ext_resizes)) {
+ resizes = NULL;
+ count = 0;
+ } else {
+ resizes = array_get_modifiable(&t->ext_resizes, &count);
+ }
+
+ buf = buffer_create_dynamic(pool_datastack_create(), 128);
+ if (ext_id < count && resizes[ext_id].name_size != 0) {
+ /* we're resizing the extension. use the resize struct. */
+ intro = &resizes[ext_id];
+
+ i_assert(intro->ext_id == idx);
+ intro->name_size = idx != (uint32_t)-1 ? 0 :
+ strlen(rext->name);
+ buffer_append(buf, intro, sizeof(*intro));
+ } else {
+ /* generate a new intro structure */
+ intro = buffer_append_space_unsafe(buf, sizeof(*intro));
+ intro->ext_id = idx;
+ intro->hdr_size = rext->hdr_size;
+ intro->record_size = rext->record_size;
+ intro->record_align = rext->record_align;
+ intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK;
+ intro->name_size = idx != (uint32_t)-1 ? 0 :
+ strlen(rext->name);
+ }
+ if (reset_id != 0) {
+ /* we're going to reset this extension in this transaction */
+ intro->reset_id = reset_id;
+ } else if (idx != (uint32_t)-1) {
+ /* use the existing reset_id */
+ const struct mail_index_ext *map_ext =
+ array_idx(&t->view->index->map->extensions, idx);
+ intro->reset_id = map_ext->reset_id;
+ } else {
+ /* new extension, reset_id defaults to 0 */
+ }
+ buffer_append(buf, rext->name, intro->name_size);
+ if ((buf->used % 4) != 0)
+ buffer_append_zero(buf, 4 - (buf->used % 4));
+
+ if (ctx->append_ctx->new_highest_modseq == 0 &&
+ strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) {
+ /* modseq tracking started */
+ ctx->append_ctx->new_highest_modseq = 1;
+ }
+
+ log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_INTRO);
+}
+
+static void
+log_append_ext_hdr_update(struct mail_index_export_context *ctx,
+ const struct mail_index_transaction_ext_hdr_update *hdr)
+{
+ buffer_t *buf;
+ const unsigned char *data, *mask;
+ struct mail_transaction_ext_hdr_update u;
+ uint16_t offset;
+ bool started = FALSE;
+
+ memset(&u, 0, sizeof(u));
+
+ data = hdr->data;
+ mask = hdr->mask;
+
+ buf = buffer_create_dynamic(pool_datastack_create(), 256);
+ for (offset = 0; offset <= hdr->alloc_size; offset++) {
+ if (offset < hdr->alloc_size && mask[offset] != 0) {
+ if (!started) {
+ u.offset = offset;
+ started = TRUE;
+ }
+ } else {
+ if (started) {
+ u.size = offset - u.offset;
+ buffer_append(buf, &u, sizeof(u));
+ buffer_append(buf, data + u.offset, u.size);
+ started = FALSE;
+ }
+ }
+ }
+ if (buf->used % 4 != 0)
+ buffer_append_zero(buf, 4 - buf->used % 4);
+ log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_HDR_UPDATE);
+}
+
+static void
+mail_transaction_log_append_ext_intros(struct mail_index_export_context *ctx)
+{
+ struct mail_index_transaction *t = ctx->trans;
+ const struct mail_transaction_ext_intro *resize;
+ const struct mail_index_transaction_ext_hdr_update *hdrs;
+ struct mail_transaction_ext_reset ext_reset;
+ unsigned int resize_count, ext_count = 0;
+ unsigned int hdrs_count, reset_id_count, reset_count;
+ uint32_t ext_id, reset_id;
+ const struct mail_transaction_ext_reset *reset;
+ const uint32_t *reset_ids;
+ buffer_t *reset_buf;
+
+ if (!array_is_created(&t->ext_resizes)) {
+ resize = NULL;
+ resize_count = 0;
+ } else {
+ resize = array_get(&t->ext_resizes, &resize_count);
+ if (ext_count < resize_count)
+ ext_count = resize_count;
+ }
+
+ if (!array_is_created(&t->ext_reset_ids)) {
+ reset_ids = NULL;
+ reset_id_count = 0;
+ } else {
+ reset_ids = array_get(&t->ext_reset_ids, &reset_id_count);
+ }
+
+ if (!array_is_created(&t->ext_resets)) {
+ reset = NULL;
+ reset_count = 0;
+ } else {
+ reset = array_get(&t->ext_resets, &reset_count);
+ if (ext_count < reset_count)
+ ext_count = reset_count;
+ }
+
+ if (!array_is_created(&t->ext_hdr_updates)) {
+ hdrs = NULL;
+ hdrs_count = 0;
+ } else {
+ hdrs = array_get(&t->ext_hdr_updates, &hdrs_count);
+ if (ext_count < hdrs_count)
+ ext_count = hdrs_count;
+ }
+
+ memset(&ext_reset, 0, sizeof(ext_reset));
+ reset_buf = buffer_create_data(pool_datastack_create(),
+ &ext_reset, sizeof(ext_reset));
+ buffer_set_used_size(reset_buf, sizeof(ext_reset));
+
+ for (ext_id = 0; ext_id < ext_count; ext_id++) {
+ if (ext_id < reset_count)
+ ext_reset = reset[ext_id];
+ else
+ ext_reset.new_reset_id = 0;
+ if ((ext_id < resize_count && resize[ext_id].name_size) ||
+ ext_reset.new_reset_id != 0 ||
+ (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0)) {
+ if (ext_reset.new_reset_id != 0) {
+ /* we're going to reset this extension
+ immediately after the intro */
+ reset_id = 0;
+ } else {
+ reset_id = ext_id < reset_id_count ?
+ reset_ids[ext_id] : 0;
+ }
+ log_append_ext_intro(ctx, ext_id, reset_id);
+ }
+ if (ext_reset.new_reset_id != 0) {
+ i_assert(ext_id < reset_id_count &&
+ ext_reset.new_reset_id == reset_ids[ext_id]);
+ log_append_buffer(ctx, reset_buf, NULL,
+ MAIL_TRANSACTION_EXT_RESET);
+ }
+ if (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0) {
+ T_BEGIN {
+ log_append_ext_hdr_update(ctx, &hdrs[ext_id]);
+ } T_END;
+ }
+ }
+}
+
+static void log_append_ext_recs(struct mail_index_export_context *ctx,
+ const ARRAY_TYPE(seq_array_array) *arr,
+ enum mail_transaction_type type)
+{
+ struct mail_index_transaction *t = ctx->trans;
+ const ARRAY_TYPE(seq_array) *updates;
+ const uint32_t *reset_ids;
+ unsigned int ext_id, count, reset_id_count;
+ uint32_t reset_id;
+
+ if (!array_is_created(&t->ext_reset_ids)) {
+ reset_ids = NULL;
+ reset_id_count = 0;
+ } else {
+ reset_ids = array_get_modifiable(&t->ext_reset_ids,
+ &reset_id_count);
+ }
+
+ updates = array_get(arr, &count);
+ for (ext_id = 0; ext_id < count; ext_id++) {
+ if (!array_is_created(&updates[ext_id]))
+ continue;
+
+ reset_id = ext_id < reset_id_count ? reset_ids[ext_id] : 0;
+ log_append_ext_intro(ctx, ext_id, reset_id);
+
+ log_append_buffer(ctx, updates[ext_id].arr.buffer, NULL, type);
+ }
+}
+
+static void
+log_append_keyword_update(struct mail_index_export_context *ctx,
+ buffer_t *hdr_buf, enum modify_type modify_type,
+ const char *keyword, const buffer_t *buffer)
+{
+ struct mail_transaction_keyword_update kt_hdr;
+
+ memset(&kt_hdr, 0, sizeof(kt_hdr));
+ kt_hdr.modify_type = modify_type;
+ kt_hdr.name_size = strlen(keyword);
+
+ buffer_set_used_size(hdr_buf, 0);
+ buffer_append(hdr_buf, &kt_hdr, sizeof(kt_hdr));
+ buffer_append(hdr_buf, keyword, kt_hdr.name_size);
+ if ((hdr_buf->used % 4) != 0)
+ buffer_append_zero(hdr_buf, 4 - (hdr_buf->used % 4));
+
+ log_append_buffer(ctx, buffer, hdr_buf,
+ MAIL_TRANSACTION_KEYWORD_UPDATE);
+}
+
+static enum mail_index_sync_type
+log_append_keyword_updates(struct mail_index_export_context *ctx)
+{
+ const struct mail_index_transaction_keyword_update *updates;
+ const char *const *keywords;
+ buffer_t *hdr_buf;
+ enum mail_index_sync_type change_mask = 0;
+ unsigned int i, count, keywords_count;
+
+ hdr_buf = buffer_create_dynamic(pool_datastack_create(), 64);
+
+ keywords = array_get_modifiable(&ctx->trans->view->index->keywords,
+ &keywords_count);
+ updates = array_get_modifiable(&ctx->trans->keyword_updates, &count);
+ i_assert(count <= keywords_count);
+
+ for (i = 0; i < count; i++) {
+ if (array_is_created(&updates[i].add_seq)) {
+ change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD;
+ log_append_keyword_update(ctx, hdr_buf,
+ MODIFY_ADD, keywords[i],
+ updates[i].add_seq.arr.buffer);
+ }
+ if (array_is_created(&updates[i].remove_seq)) {
+ change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE;
+ log_append_keyword_update(ctx, hdr_buf,
+ MODIFY_REMOVE, keywords[i],
+ updates[i].remove_seq.arr.buffer);
+ }
+ }
+ return change_mask;
+}
+
+void mail_index_transaction_export(struct mail_index_transaction *t,
+ struct mail_transaction_log_append_ctx *append_ctx)
+{
+ enum mail_index_sync_type change_mask = 0;
+ struct mail_index_export_context ctx;
+
+ memset(&ctx, 0, sizeof(ctx));
+ ctx.trans = t;
+ ctx.append_ctx = append_ctx;
+
+ /* send all extension introductions and resizes before appends
+ to avoid resize overhead as much as possible */
+ mail_transaction_log_append_ext_intros(&ctx);
+
+ if (t->pre_hdr_changed) {
+ log_append_buffer(&ctx,
+ log_get_hdr_update_buffer(t, TRUE),
+ NULL, MAIL_TRANSACTION_HEADER_UPDATE);
+ }
+ if (array_is_created(&t->appends)) {
+ change_mask |= MAIL_INDEX_SYNC_TYPE_APPEND;
+ log_append_buffer(&ctx, t->appends.arr.buffer, NULL,
+ MAIL_TRANSACTION_APPEND);
+ }
+ if (array_is_created(&t->updates)) {
+ change_mask |= MAIL_INDEX_SYNC_TYPE_FLAGS;
+ log_append_buffer(&ctx, t->updates.arr.buffer, NULL,
+ MAIL_TRANSACTION_FLAG_UPDATE);
+ }
+
+ if (array_is_created(&t->ext_rec_updates)) {
+ log_append_ext_recs(&ctx, &t->ext_rec_updates,
+ MAIL_TRANSACTION_EXT_REC_UPDATE);
+ }
+ if (array_is_created(&t->ext_rec_atomics)) {
+ log_append_ext_recs(&ctx, &t->ext_rec_atomics,
+ MAIL_TRANSACTION_EXT_ATOMIC_INC);
+ }
+
+ /* keyword resets before updates */
+ if (array_is_created(&t->keyword_resets)) {
+ change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
+ log_append_buffer(&ctx, t->keyword_resets.arr.buffer,
+ NULL, MAIL_TRANSACTION_KEYWORD_RESET);
+ }
+ if (array_is_created(&t->keyword_updates))
+ change_mask |= log_append_keyword_updates(&ctx);
+
+ if (array_is_created(&t->expunges)) {
+ /* non-external expunges are only requests, ignore them when
+ checking fsync_mask */
+ if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
+ change_mask |= MAIL_INDEX_SYNC_TYPE_EXPUNGE;
+ log_append_buffer(&ctx, t->expunges.arr.buffer, NULL,
+ MAIL_TRANSACTION_EXPUNGE);
+ }
+
+ if (t->post_hdr_changed) {
+ log_append_buffer(&ctx, log_get_hdr_update_buffer(t, FALSE),
+ NULL, MAIL_TRANSACTION_HEADER_UPDATE);
+ }
+
+ /* Update the tail offsets only when committing the sync transaction.
+ Other transactions may not know the latest tail offset and might
+ end up shrinking it. (Alternatively the shrinking tail offsets could
+ just be ignored, which would probably work fine too.) */
+ append_ctx->append_sync_offset = t->sync_transaction;
+
+ append_ctx->want_fsync =
+ (t->view->index->fsync_mask & change_mask) != 0 ||
+ (t->flags & MAIL_INDEX_TRANSACTION_FLAG_FSYNC) != 0;
+}
--- /dev/null
+/* Copyright (c) 2003-2009 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "mail-index-private.h"
+#include "mail-index-modseq.h"
+#include "mail-index-transaction-private.h"
+
+static void
+ext_reset_update_atomic(struct mail_index_transaction *t,
+ uint32_t ext_id, uint32_t expected_reset_id)
+{
+ const struct mail_index_ext *map_ext;
+ struct mail_transaction_ext_reset *reset;
+ uint32_t idx, reset_id;
+
+ if (!mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
+ /* new extension */
+ reset_id = 1;
+ } else {
+ map_ext = array_idx(&t->view->index->map->extensions, idx);
+ reset_id = map_ext->reset_id + 1;
+ }
+ if (reset_id != expected_reset_id) {
+ /* ignore this extension update */
+ mail_index_ext_set_reset_id(t, ext_id, 0);
+ return;
+ }
+
+ if (reset_id == 0)
+ reset_id++;
+
+ array_idx_set(&t->ext_reset_ids, ext_id, &reset_id);
+
+ /* reseting existing data is optional */
+ if (array_is_created(&t->ext_resets)) {
+ reset = array_idx_modifiable(&t->ext_resets, ext_id);
+ if (reset->new_reset_id == (uint32_t)-1)
+ reset->new_reset_id = reset_id;
+ }
+}
+
+static void
+transaction_update_atomic_reset_ids(struct mail_index_transaction *t)
+{
+ const uint32_t *expected_reset_ids;
+ unsigned int ext_id, count;
+
+ if (!array_is_created(&t->ext_reset_atomic))
+ return;
+
+ expected_reset_ids = array_get(&t->ext_reset_atomic, &count);
+ for (ext_id = 0; ext_id < count; ext_id++) {
+ if (expected_reset_ids[ext_id] != 0) {
+ ext_reset_update_atomic(t, ext_id,
+ expected_reset_ids[ext_id]);
+ }
+ }
+}
+
+static bool
+mail_index_update_cancel_array(ARRAY_TYPE(seq_range) *array, uint32_t seq)
+{
+ if (array_is_created(array)) {
+ if (seq_range_array_remove(array, seq)) {
+ if (array_count(array) == 0)
+ array_free(array);
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+static bool
+mail_index_update_cancel(struct mail_index_transaction *t, uint32_t seq)
+{
+ struct mail_index_transaction_keyword_update *kw;
+ struct mail_transaction_flag_update *updates, tmp_update;
+ unsigned int i, count;
+ bool ret, have_kw_changes = FALSE;
+
+ ret = mail_index_update_cancel_array(&t->keyword_resets, seq);
+ if (array_is_created(&t->keyword_updates)) {
+ kw = array_get_modifiable(&t->keyword_updates, &count);
+ for (i = 0; i < count; i++) {
+ if (mail_index_update_cancel_array(&kw[i].add_seq, seq))
+ ret = TRUE;
+ if (mail_index_update_cancel_array(&kw[i].remove_seq,
+ seq))
+ ret = TRUE;
+ if (array_is_created(&kw[i].add_seq) ||
+ array_is_created(&kw[i].remove_seq))
+ have_kw_changes = TRUE;
+ }
+ if (!have_kw_changes)
+ array_free(&t->keyword_updates);
+ }
+
+ if (!array_is_created(&t->updates))
+ return ret;
+
+ updates = array_get_modifiable(&t->updates, &count);
+ i = mail_index_transaction_get_flag_update_pos(t, 0, count, seq);
+ if (i < count && updates[i].uid1 <= seq && updates[i].uid2 >= seq) {
+ /* exists */
+ ret = TRUE;
+ if (updates[i].uid1 == seq && updates[i].uid2 == seq) {
+ if (count > 1)
+ array_delete(&t->updates, i, 1);
+ else
+ array_free(&t->updates);
+ } else if (updates[i].uid1 == seq)
+ updates[i].uid1++;
+ else if (updates[i].uid2 == seq)
+ updates[i].uid2--;
+ else {
+ /* need to split it in two */
+ tmp_update = updates[i];
+ tmp_update.uid1 = seq+1;
+ updates[i].uid2 = seq-1;
+ array_insert(&t->updates, i + 1, &tmp_update, 1);
+ }
+ }
+ return ret;
+}
+
+static void
+mail_index_transaction_check_conflicts(struct mail_index_transaction *t)
+{
+ uint32_t seq;
+
+ i_assert(t->max_modseq != 0);
+ i_assert(t->conflict_seqs != NULL);
+
+ if (t->max_modseq == mail_index_modseq_get_highest(t->view)) {
+ /* no conflicts possible */
+ return;
+ }
+ if (t->min_flagupdate_seq == 0) {
+ /* no flag updates */
+ return;
+ }
+
+ for (seq = t->min_flagupdate_seq; seq <= t->max_flagupdate_seq; seq++) {
+ if (mail_index_modseq_lookup(t->view, seq) > t->max_modseq) {
+ if (mail_index_update_cancel(t, seq))
+ seq_range_array_add(t->conflict_seqs, 0, seq);
+ }
+ }
+ mail_index_transaction_set_log_updates(t);
+}
+
+static uint32_t
+mail_index_transaction_get_uid(struct mail_index_transaction *t, uint32_t seq)
+{
+ const struct mail_index_record *rec;
+
+ i_assert(seq > 0);
+
+ if (seq >= t->first_new_seq)
+ rec = mail_index_transaction_lookup(t, seq);
+ else {
+ i_assert(seq <= t->view->map->hdr.messages_count);
+ rec = MAIL_INDEX_MAP_IDX(t->view->map, seq - 1);
+ }
+ i_assert(rec->uid != 0);
+ return rec->uid;
+}
+
+static void
+mail_index_convert_to_uids(struct mail_index_transaction *t,
+ ARRAY_TYPE(seq_array) *array)
+{
+ uint32_t *seq;
+ unsigned int i, count;
+
+ if (!array_is_created(array))
+ return;
+
+ count = array_count(array);
+ for (i = 0; i < count; i++) {
+ seq = array_idx_modifiable(array, i);
+ *seq = mail_index_transaction_get_uid(t, *seq);
+ }
+}
+
+static uint32_t
+get_nonexpunged_uid2(struct mail_index_transaction *t,
+ uint32_t uid1, uint32_t seq1)
+{
+ seq1++;
+
+ while (mail_index_transaction_get_uid(t, seq1) == uid1 + 1) {
+ seq1++;
+ uid1++;
+ }
+ return uid1;
+}
+
+static void
+mail_index_convert_to_uid_ranges(struct mail_index_transaction *t,
+ ARRAY_TYPE(seq_range) *array)
+{
+ struct seq_range *range, *new_range;
+ unsigned int i, count;
+ uint32_t uid1, uid2;
+
+ if (!array_is_created(array))
+ return;
+
+ count = array_count(array);
+ for (i = 0; i < count; i++) {
+ range = array_idx_modifiable(array, i);
+
+ uid1 = mail_index_transaction_get_uid(t, range->seq1);
+ uid2 = mail_index_transaction_get_uid(t, range->seq2);
+ if (uid2 - uid1 == range->seq2 - range->seq1) {
+ /* simple conversion */
+ range->seq1 = uid1;
+ range->seq2 = uid2;
+ } else {
+ /* remove expunged UIDs */
+ new_range = array_insert_space(array, i);
+ range = array_idx_modifiable(array, i + 1);
+ count++;
+
+ memcpy(new_range, range, array->arr.element_size);
+ new_range->seq1 = uid1;
+ new_range->seq2 = get_nonexpunged_uid2(t, uid1,
+ range->seq1);
+
+ /* continue the range without the inserted seqs */
+ range->seq1 += new_range->seq2 - new_range->seq1 + 1;
+ }
+ }
+}
+
+static void keyword_updates_convert_to_uids(struct mail_index_transaction *t)
+{
+ struct mail_index_transaction_keyword_update *updates;
+ unsigned int i, count;
+
+ if (!array_is_created(&t->keyword_updates))
+ return;
+
+ updates = array_get_modifiable(&t->keyword_updates, &count);
+ for (i = 0; i < count; i++) {
+ mail_index_convert_to_uid_ranges(t, &updates[i].add_seq);
+ mail_index_convert_to_uid_ranges(t, &updates[i].remove_seq);
+ }
+}
+
+static void
+mail_index_transaction_convert_to_uids(struct mail_index_transaction *t)
+{
+ ARRAY_TYPE(seq_array) *updates;
+ unsigned int i, count;
+
+ if (array_is_created(&t->ext_rec_updates)) {
+ updates = array_get_modifiable(&t->ext_rec_updates, &count);
+ for (i = 0; i < count; i++)
+ mail_index_convert_to_uids(t, (void *)&updates[i]);
+ }
+ if (array_is_created(&t->ext_rec_atomics)) {
+ updates = array_get_modifiable(&t->ext_rec_atomics, &count);
+ for (i = 0; i < count; i++)
+ mail_index_convert_to_uids(t, (void *)&updates[i]);
+ }
+
+ keyword_updates_convert_to_uids(t);
+
+ mail_index_convert_to_uid_ranges(t, &t->expunges);
+ mail_index_convert_to_uid_ranges(t, (void *)&t->updates);
+ mail_index_convert_to_uid_ranges(t, &t->keyword_resets);
+}
+
+int mail_index_transaction_finish(struct mail_index_transaction *t)
+{
+ mail_index_transaction_sort_appends(t);
+
+ if (array_is_created(&t->ext_reset_atomic) || t->max_modseq != 0) {
+ if (mail_index_map(t->view->index,
+ MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0)
+ return -1;
+ }
+ if (array_is_created(&t->ext_reset_atomic))
+ transaction_update_atomic_reset_ids(t);
+ if (t->max_modseq != 0)
+ mail_index_transaction_check_conflicts(t);
+ /* finally convert all sequences to UIDs before we write them,
+ but after we've checked and removed conflicts */
+ mail_index_transaction_convert_to_uids(t);
+ return 0;
+}
unsigned int log_ext_updates:1;
};
+#define MAIL_INDEX_TRANSACTION_HAS_CHANGES(t) \
+ ((t)->log_updates || (t)->log_ext_updates || \
+ (array_is_created(&(t)->updates) && array_count(&(t)->updates) > 0))
+
extern void (*hook_mail_index_transaction_created)
(struct mail_index_transaction *t);
void mail_index_transaction_sort_appends(struct mail_index_transaction *t);
uint32_t mail_index_transaction_get_next_uid(struct mail_index_transaction *t);
-void mail_index_transaction_convert_to_uids(struct mail_index_transaction *t);
-void mail_index_transaction_check_conflicts(struct mail_index_transaction *t);
+void mail_index_transaction_set_log_updates(struct mail_index_transaction *t);
unsigned int
mail_index_transaction_get_flag_update_pos(struct mail_index_transaction *t,
bool mail_index_seq_array_lookup(const ARRAY_TYPE(seq_array) *array,
uint32_t seq, unsigned int *idx_r);
+bool mail_index_seq_array_add(ARRAY_TYPE(seq_array) *array, uint32_t seq,
+ const void *record, size_t record_size,
+ void *old_record);
+
+int mail_index_transaction_finish(struct mail_index_transaction *t);
+void mail_index_transaction_export(struct mail_index_transaction *t,
+ struct mail_transaction_log_append_ctx *append_ctx);
#endif
--- /dev/null
+/* Copyright (c) 2003-2009 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "seq-range-array.h"
+#include "mail-index-private.h"
+#include "mail-index-transaction-private.h"
+
+#include <stdlib.h>
+
+struct uid_map {
+ uint32_t idx;
+ uint32_t uid;
+};
+
+static int uid_map_cmp(const void *p1, const void *p2)
+{
+ const struct uid_map *m1 = p1, *m2 = p2;
+
+ return m1->uid < m2->uid ? -1 :
+ (m1->uid > m2->uid ? 1 : 0);
+}
+
+static void
+mail_index_transaction_sort_appends_ext(ARRAY_TYPE(seq_array_array) *updates,
+ uint32_t first_new_seq,
+ const uint32_t *old_to_newseq_map)
+{
+ ARRAY_TYPE(seq_array) *ext_rec_arrays;
+ ARRAY_TYPE(seq_array) *old_array;
+ ARRAY_TYPE(seq_array) new_array;
+ unsigned int ext_count;
+ const uint32_t *ext_rec;
+ uint32_t seq;
+ unsigned int i, j, count;
+
+ if (!array_is_created(updates))
+ return;
+
+ ext_rec_arrays = array_get_modifiable(updates, &count);
+ for (j = 0; j < count; j++) {
+ old_array = &ext_rec_arrays[j];
+ if (!array_is_created(old_array))
+ continue;
+
+ ext_count = array_count(old_array);
+ array_create(&new_array, default_pool,
+ old_array->arr.element_size, ext_count);
+ for (i = 0; i < ext_count; i++) {
+ ext_rec = array_idx(old_array, i);
+
+ seq = *ext_rec < first_new_seq ? *ext_rec :
+ old_to_newseq_map[*ext_rec - first_new_seq];
+ mail_index_seq_array_add(&new_array, seq, ext_rec+1,
+ old_array->arr.element_size -
+ sizeof(*ext_rec), NULL);
+ }
+ array_free(old_array);
+ ext_rec_arrays[j] = new_array;
+ }
+}
+
+static void
+sort_appends_seq_range(ARRAY_TYPE(seq_range) *array, uint32_t first_new_seq,
+ const uint32_t *old_to_newseq_map)
+{
+ struct seq_range *range, temp_range;
+ ARRAY_TYPE(seq_range) old_seqs;
+ uint32_t idx, idx1, idx2;
+ unsigned int i, count;
+
+ range = array_get_modifiable(array, &count);
+ for (i = 0; i < count; i++) {
+ if (range[i].seq2 >= first_new_seq)
+ break;
+ }
+ if (i == count) {
+ /* nothing to do */
+ return;
+ }
+
+ i_array_init(&old_seqs, count - i);
+ if (range[i].seq1 < first_new_seq) {
+ temp_range.seq1 = first_new_seq;
+ temp_range.seq2 = range[i].seq2;
+ array_append(&old_seqs, &temp_range, 1);
+ range[i].seq2 = first_new_seq - 1;
+ i++;
+ }
+ array_append(&old_seqs, &range[i], count - i);
+ array_delete(array, i, count - i);
+
+ range = array_get_modifiable(&old_seqs, &count);
+ for (i = 0; i < count; i++) {
+ idx1 = range[i].seq1 - first_new_seq;
+ idx2 = range[i].seq2 - first_new_seq;
+ for (idx = idx1; idx <= idx2; idx++)
+ seq_range_array_add(array, 0, old_to_newseq_map[idx]);
+ }
+ array_free(&old_seqs);
+}
+
+static void
+mail_index_transaction_sort_appends_keywords(struct mail_index_transaction *t,
+ const uint32_t *old_to_newseq_map)
+{
+ struct mail_index_transaction_keyword_update *updates;
+ unsigned int i, count;
+
+ if (array_is_created(&t->keyword_updates)) {
+ updates = array_get_modifiable(&t->keyword_updates, &count);
+ for (i = 0; i < count; i++) {
+ if (array_is_created(&updates[i].add_seq)) {
+ sort_appends_seq_range(&updates[i].add_seq,
+ t->first_new_seq,
+ old_to_newseq_map);
+ }
+ if (array_is_created(&updates[i].remove_seq)) {
+ sort_appends_seq_range(&updates[i].remove_seq,
+ t->first_new_seq,
+ old_to_newseq_map);
+ }
+ }
+ }
+
+ if (array_is_created(&t->keyword_resets)) {
+ sort_appends_seq_range(&t->keyword_resets, t->first_new_seq,
+ old_to_newseq_map);
+ }
+}
+
+void mail_index_transaction_sort_appends(struct mail_index_transaction *t)
+{
+ struct mail_index_record *recs, *sorted_recs;
+ struct uid_map *new_uid_map;
+ uint32_t *old_to_newseq_map;
+ unsigned int i, count;
+
+ if (!t->appends_nonsorted || !array_is_created(&t->appends))
+ return;
+
+ /* first make a copy of the UIDs and map them to sequences */
+ recs = array_get_modifiable(&t->appends, &count);
+ i_assert(count > 0);
+
+ new_uid_map = i_new(struct uid_map, count);
+ for (i = 0; i < count; i++) {
+ new_uid_map[i].idx = i;
+ new_uid_map[i].uid = recs[i].uid;
+ }
+
+ /* now sort the UID map */
+ qsort(new_uid_map, count, sizeof(*new_uid_map), uid_map_cmp);
+
+ /* sort mail records */
+ sorted_recs = i_new(struct mail_index_record, count);
+ sorted_recs[0] = recs[new_uid_map[0].idx];
+ for (i = 1; i < count; i++) {
+ sorted_recs[i] = recs[new_uid_map[i].idx];
+ if (sorted_recs[i].uid == sorted_recs[i-1].uid)
+ i_panic("Duplicate UIDs added in transaction");
+ }
+ buffer_write(t->appends.arr.buffer, 0, sorted_recs,
+ sizeof(*sorted_recs) * count);
+ i_free(sorted_recs);
+
+ old_to_newseq_map = i_new(uint32_t, count);
+ for (i = 0; i < count; i++)
+ old_to_newseq_map[new_uid_map[i].idx] = i + t->first_new_seq;
+ i_free(new_uid_map);
+
+ mail_index_transaction_sort_appends_ext(&t->ext_rec_updates,
+ t->first_new_seq,
+ old_to_newseq_map);
+ mail_index_transaction_sort_appends_ext(&t->ext_rec_atomics,
+ t->first_new_seq,
+ old_to_newseq_map);
+ mail_index_transaction_sort_appends_keywords(t, old_to_newseq_map);
+ i_free(old_to_newseq_map);
+
+ t->appends_nonsorted = FALSE;
+}
#include "ioloop.h"
#include "array.h"
#include "bsearch-insert-pos.h"
-#include "seq-range-array.h"
-#include "mail-index-view-private.h"
-#include "mail-index-modseq.h"
-#include "mail-transaction-log.h"
-#include "mail-cache-private.h"
+#include "mail-index-private.h"
+#include "mail-transaction-log-private.h"
+#include "mail-cache.h"
#include "mail-index-transaction-private.h"
-#include <stddef.h>
-#include <stdlib.h>
-#include <time.h>
-
void (*hook_mail_index_transaction_created)
(struct mail_index_transaction *t) = NULL;
t->log_ext_updates = FALSE;
}
-static bool mail_index_transaction_has_changes(struct mail_index_transaction *t)
+void mail_index_transaction_set_log_updates(struct mail_index_transaction *t)
{
/* flag updates aren't included in log_updates */
- return array_is_created(&t->appends) ||
+ t->log_updates = array_is_created(&t->appends) ||
array_is_created(&t->expunges) ||
array_is_created(&t->keyword_resets) ||
array_is_created(&t->keyword_updates) ||
mail_index_seq_record_cmp, idx_r);
}
-static bool mail_index_seq_array_add(ARRAY_TYPE(seq_array) *array, uint32_t seq,
- const void *record, size_t record_size,
- void *old_record)
+bool mail_index_seq_array_add(ARRAY_TYPE(seq_array) *array, uint32_t seq,
+ const void *record, size_t record_size,
+ void *old_record)
{
void *p;
unsigned int idx, aligned_record_size;
}
}
-static uint32_t
-mail_index_transaction_get_uid(struct mail_index_transaction *t, uint32_t seq)
+uint32_t mail_index_transaction_get_next_uid(struct mail_index_transaction *t)
{
- const struct mail_index_record *rec;
-
- i_assert(seq > 0);
+ const struct mail_index_header *head_hdr, *hdr;
+ unsigned int offset;
+ uint32_t next_uid;
- if (seq >= t->first_new_seq)
- rec = mail_index_transaction_lookup(t, seq);
- else {
- i_assert(seq <= t->view->map->hdr.messages_count);
- rec = MAIL_INDEX_MAP_IDX(t->view->map, seq - 1);
+ head_hdr = &t->view->index->map->hdr;
+ hdr = &t->view->map->hdr;
+ next_uid = t->reset || head_hdr->uid_validity != hdr->uid_validity ?
+ 1 : hdr->next_uid;
+ if (array_is_created(&t->appends) && t->highest_append_uid != 0) {
+ /* get next_uid from appends if they have UIDs */
+ i_assert(next_uid <= t->highest_append_uid);
+ next_uid = t->highest_append_uid + 1;
}
- i_assert(rec->uid != 0);
- return rec->uid;
-}
-static void
-mail_index_convert_to_uids(struct mail_index_transaction *t,
- ARRAY_TYPE(seq_array) *array)
-{
- uint32_t *seq;
- unsigned int i, count;
-
- if (!array_is_created(array))
- return;
-
- count = array_count(array);
- for (i = 0; i < count; i++) {
- seq = array_idx_modifiable(array, i);
- *seq = mail_index_transaction_get_uid(t, *seq);
+ /* see if it's been updated in pre/post header changes */
+ offset = offsetof(struct mail_index_header, next_uid);
+ if (t->post_hdr_mask[offset] != 0) {
+ hdr = (const void *)t->post_hdr_change;
+ if (hdr->next_uid > next_uid)
+ next_uid = hdr->next_uid;
}
-}
-
-static uint32_t
-get_nonexpunged_uid2(struct mail_index_transaction *t,
- uint32_t uid1, uint32_t seq1)
-{
- seq1++;
-
- while (mail_index_transaction_get_uid(t, seq1) == uid1 + 1) {
- seq1++;
- uid1++;
+ if (t->pre_hdr_mask[offset] != 0) {
+ hdr = (const void *)t->pre_hdr_change;
+ if (hdr->next_uid > next_uid)
+ next_uid = hdr->next_uid;
}
- return uid1;
+ return next_uid;
}
-static void
-mail_index_convert_to_uid_ranges(struct mail_index_transaction *t,
- ARRAY_TYPE(seq_range) *array)
+static int
+mail_transaction_log_file_refresh(struct mail_index_transaction *t,
+ struct mail_transaction_log_append_ctx *ctx)
{
- struct seq_range *range, *new_range;
- unsigned int i, count;
- uint32_t uid1, uid2;
-
- if (!array_is_created(array))
- return;
-
- count = array_count(array);
- for (i = 0; i < count; i++) {
- range = array_idx_modifiable(array, i);
-
- uid1 = mail_index_transaction_get_uid(t, range->seq1);
- uid2 = mail_index_transaction_get_uid(t, range->seq2);
- if (uid2 - uid1 == range->seq2 - range->seq1) {
- /* simple conversion */
- range->seq1 = uid1;
- range->seq2 = uid2;
- } else {
- /* remove expunged UIDs */
- new_range = array_insert_space(array, i);
- range = array_idx_modifiable(array, i + 1);
- count++;
-
- memcpy(new_range, range, array->arr.element_size);
- new_range->seq1 = uid1;
- new_range->seq2 = get_nonexpunged_uid2(t, uid1,
- range->seq1);
-
- /* continue the range without the inserted seqs */
- range->seq1 += new_range->seq2 - new_range->seq1 + 1;
+ struct mail_transaction_log_file *file;
+
+ if (t->reset) {
+ /* Reset the whole index, preserving only indexid. Begin by
+ rotating the log. We don't care if we skip some non-synced
+ transactions. */
+ if (mail_transaction_log_rotate(t->view->index->log, TRUE) < 0)
+ return -1;
+
+ if (!MAIL_INDEX_TRANSACTION_HAS_CHANGES(t)) {
+ /* we only wanted to reset */
+ return 0;
}
}
-}
+ file = t->view->index->log->head;
-static void keyword_updates_convert_to_uids(struct mail_index_transaction *t)
-{
- struct mail_index_transaction_keyword_update *updates;
- unsigned int i, count;
-
- if (!array_is_created(&t->keyword_updates))
- return;
-
- updates = array_get_modifiable(&t->keyword_updates, &count);
- for (i = 0; i < count; i++) {
- mail_index_convert_to_uid_ranges(t, &updates[i].add_seq);
- mail_index_convert_to_uid_ranges(t, &updates[i].remove_seq);
+ if (!t->view->index->log_locked) {
+ /* update sync_offset */
+ if (mail_transaction_log_file_map(file, file->sync_offset,
+ (uoff_t)-1) <= 0)
+ return -1;
}
+ i_assert(file->sync_offset >= file->buffer_offset);
+ ctx->new_highest_modseq = file->sync_highest_modseq;
+ return 1;
}
-void mail_index_transaction_convert_to_uids(struct mail_index_transaction *t)
+static int mail_index_transaction_commit_real(struct mail_index_transaction *t)
{
- ARRAY_TYPE(seq_array) *updates;
- unsigned int i, count;
+ struct mail_transaction_log *log = t->view->index->log;
+ struct mail_transaction_log_append_ctx *ctx;
+ uint32_t log_seq1, log_seq2;
+ uoff_t log_offset1, log_offset2;
+ int ret;
- if (array_is_created(&t->ext_rec_updates)) {
- updates = array_get_modifiable(&t->ext_rec_updates, &count);
- for (i = 0; i < count; i++)
- mail_index_convert_to_uids(t, (void *)&updates[i]);
- }
- if (array_is_created(&t->ext_rec_atomics)) {
- updates = array_get_modifiable(&t->ext_rec_atomics, &count);
- for (i = 0; i < count; i++)
- mail_index_convert_to_uids(t, (void *)&updates[i]);
+ if (mail_transaction_log_append_begin(t, &ctx) < 0)
+ return -1;
+ ret = mail_transaction_log_file_refresh(t, ctx);
+ if (ret > 0) {
+ ret = mail_index_transaction_finish(t);
+ if (ret == 0)
+ mail_index_transaction_export(t, ctx);
}
- keyword_updates_convert_to_uids(t);
-
- mail_index_convert_to_uid_ranges(t, &t->expunges);
- mail_index_convert_to_uid_ranges(t, (void *)&t->updates);
- mail_index_convert_to_uid_ranges(t, &t->keyword_resets);
-}
-
-struct uid_map {
- uint32_t idx;
- uint32_t uid;
-};
-
-static int uid_map_cmp(const void *p1, const void *p2)
-{
- const struct uid_map *m1 = p1, *m2 = p2;
+ mail_transaction_log_get_head(log, &log_seq1, &log_offset1);
+ if (mail_transaction_log_append_commit(&ctx) < 0 || ret < 0)
+ return -1;
+ mail_transaction_log_get_head(log, &log_seq2, &log_offset2);
+ i_assert(log_seq1 == log_seq2);
- return m1->uid < m2->uid ? -1 :
- (m1->uid > m2->uid ? 1 : 0);
+ if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_HIDE) != 0 &&
+ log_offset1 != log_offset2) {
+ /* mark the area covered by this transaction hidden */
+ mail_index_view_add_hidden_transaction(t->view, log_seq1,
+ log_offset1, log_offset2 - log_offset1);
+ }
+ return 0;
}
static void
hdr.day_first_uid, sizeof(hdr.day_first_uid), FALSE);
}
-static void
-mail_index_transaction_sort_appends_ext(struct mail_index_transaction *t,
- ARRAY_TYPE(seq_array_array) *updates,
- const uint32_t *old_to_newseq_map)
-{
- ARRAY_TYPE(seq_array) *ext_rec_arrays;
- ARRAY_TYPE(seq_array) *old_array;
- ARRAY_TYPE(seq_array) new_array;
- unsigned int ext_count;
- const uint32_t *ext_rec;
- uint32_t seq;
- unsigned int i, j, count;
-
- if (!array_is_created(updates))
- return;
-
- ext_rec_arrays = array_get_modifiable(updates, &count);
- for (j = 0; j < count; j++) {
- old_array = &ext_rec_arrays[j];
- if (!array_is_created(old_array))
- continue;
-
- ext_count = array_count(old_array);
- array_create(&new_array, default_pool,
- old_array->arr.element_size, ext_count);
- for (i = 0; i < ext_count; i++) {
- ext_rec = array_idx(old_array, i);
-
- seq = *ext_rec < t->first_new_seq ? *ext_rec :
- old_to_newseq_map[*ext_rec - t->first_new_seq];
- mail_index_seq_array_add(&new_array, seq, ext_rec+1,
- old_array->arr.element_size -
- sizeof(*ext_rec), NULL);
- }
- array_free(old_array);
- ext_rec_arrays[j] = new_array;
- }
-}
-
-static void
-sort_appends_seq_range(struct mail_index_transaction *t,
- ARRAY_TYPE(seq_range) *array,
- const uint32_t *old_to_newseq_map)
-{
- struct seq_range *range, temp_range;
- ARRAY_TYPE(seq_range) old_seqs;
- uint32_t idx, idx1, idx2;
- unsigned int i, count;
-
- range = array_get_modifiable(array, &count);
- for (i = 0; i < count; i++) {
- if (range[i].seq2 >= t->first_new_seq)
- break;
- }
- if (i == count) {
- /* nothing to do */
- return;
- }
-
- i_array_init(&old_seqs, count - i);
- if (range[i].seq1 < t->first_new_seq) {
- temp_range.seq1 = t->first_new_seq;
- temp_range.seq2 = range[i].seq2;
- array_append(&old_seqs, &temp_range, 1);
- range[i].seq2 = t->first_new_seq - 1;
- i++;
- }
- array_append(&old_seqs, &range[i], count - i);
- array_delete(array, i, count - i);
-
- range = array_get_modifiable(&old_seqs, &count);
- for (i = 0; i < count; i++) {
- idx1 = range[i].seq1 - t->first_new_seq;
- idx2 = range[i].seq2 - t->first_new_seq;
- for (idx = idx1; idx <= idx2; idx++)
- seq_range_array_add(array, 0, old_to_newseq_map[idx]);
- }
- array_free(&old_seqs);
-}
-
-static void
-mail_index_transaction_sort_appends_keywords(struct mail_index_transaction *t,
- const uint32_t *old_to_newseq_map)
-{
- struct mail_index_transaction_keyword_update *updates;
- unsigned int i, count;
-
- if (array_is_created(&t->keyword_updates)) {
- updates = array_get_modifiable(&t->keyword_updates, &count);
- for (i = 0; i < count; i++) {
- if (array_is_created(&updates[i].add_seq)) {
- sort_appends_seq_range(t, &updates[i].add_seq,
- old_to_newseq_map);
- }
- if (array_is_created(&updates[i].remove_seq)) {
- sort_appends_seq_range(t,
- &updates[i].remove_seq,
- old_to_newseq_map);
- }
- }
- }
-
- if (array_is_created(&t->keyword_resets)) {
- sort_appends_seq_range(t, &t->keyword_resets,
- old_to_newseq_map);
- }
-}
-
-void mail_index_transaction_sort_appends(struct mail_index_transaction *t)
-{
- struct mail_index_record *recs, *sorted_recs;
- struct uid_map *new_uid_map;
- uint32_t *old_to_newseq_map;
- unsigned int i, count;
-
- if (!t->appends_nonsorted)
- return;
-
- /* first make a copy of the UIDs and map them to sequences */
- recs = array_get_modifiable(&t->appends, &count);
- i_assert(count > 0);
-
- new_uid_map = i_new(struct uid_map, count);
- for (i = 0; i < count; i++) {
- new_uid_map[i].idx = i;
- new_uid_map[i].uid = recs[i].uid;
- }
-
- /* now sort the UID map */
- qsort(new_uid_map, count, sizeof(*new_uid_map), uid_map_cmp);
-
- /* sort mail records */
- sorted_recs = i_new(struct mail_index_record, count);
- sorted_recs[0] = recs[new_uid_map[0].idx];
- for (i = 1; i < count; i++) {
- sorted_recs[i] = recs[new_uid_map[i].idx];
- if (sorted_recs[i].uid == sorted_recs[i-1].uid)
- i_panic("Duplicate UIDs added in transaction");
- }
- buffer_write(t->appends.arr.buffer, 0, sorted_recs,
- sizeof(*sorted_recs) * count);
- i_free(sorted_recs);
-
- old_to_newseq_map = i_new(uint32_t, count);
- for (i = 0; i < count; i++)
- old_to_newseq_map[new_uid_map[i].idx] = i + t->first_new_seq;
- i_free(new_uid_map);
-
- mail_index_transaction_sort_appends_ext(t, &t->ext_rec_updates,
- old_to_newseq_map);
- mail_index_transaction_sort_appends_ext(t, &t->ext_rec_atomics,
- old_to_newseq_map);
- mail_index_transaction_sort_appends_keywords(t, old_to_newseq_map);
- i_free(old_to_newseq_map);
-
- t->appends_nonsorted = FALSE;
-}
-
-uint32_t mail_index_transaction_get_next_uid(struct mail_index_transaction *t)
-{
- const struct mail_index_header *head_hdr, *hdr;
- unsigned int offset;
- uint32_t next_uid;
-
- head_hdr = &t->view->index->map->hdr;
- hdr = &t->view->map->hdr;
- next_uid = t->reset || head_hdr->uid_validity != hdr->uid_validity ?
- 1 : hdr->next_uid;
- if (array_is_created(&t->appends) && t->highest_append_uid != 0) {
- /* get next_uid from appends if they have UIDs */
- i_assert(next_uid <= t->highest_append_uid);
- next_uid = t->highest_append_uid + 1;
- }
-
- /* see if it's been updated in pre/post header changes */
- offset = offsetof(struct mail_index_header, next_uid);
- if (t->post_hdr_mask[offset] != 0) {
- hdr = (const void *)t->post_hdr_change;
- if (hdr->next_uid > next_uid)
- next_uid = hdr->next_uid;
- }
- if (t->pre_hdr_mask[offset] != 0) {
- hdr = (const void *)t->pre_hdr_change;
- if (hdr->next_uid > next_uid)
- next_uid = hdr->next_uid;
- }
- return next_uid;
-}
-
static int mail_index_transaction_commit_v(struct mail_index_transaction *t,
uint32_t *log_file_seq_r,
uoff_t *log_file_offset_r)
{
+ struct mail_index *index = t->view->index;
int ret;
i_assert(t->first_new_seq >
if (t->cache_trans_ctx != NULL)
mail_cache_transaction_commit(&t->cache_trans_ctx);
- if (array_is_created(&t->appends)) {
- mail_index_transaction_sort_appends(t);
+ if (array_is_created(&t->appends))
mail_index_update_day_headers(t);
- }
- ret = mail_transaction_log_append(t, log_file_seq_r, log_file_offset_r);
+ if (!MAIL_INDEX_TRANSACTION_HAS_CHANGES(t) && !t->reset) {
+ /* nothing to append */
+ ret = 0;
+ } else {
+ ret = mail_index_transaction_commit_real(t);
+ }
+ mail_transaction_log_get_head(index->log, log_file_seq_r,
+ log_file_offset_r);
- if (ret == 0 && !t->view->index->syncing) {
+ if (ret == 0 && !index->syncing) {
/* if we're committing a normal transaction, we want to
have those changes in the index mapping immediately. this
is especially important when committing cache offset
be done later as MAIL_INDEX_SYNC_HANDLER_FILE so that
expunge handlers get run for the newly expunged messages
(and sync handlers that require HANDLER_FILE as well). */
- (void)mail_index_refresh(t->view->index);
+ (void)mail_index_refresh(index);
}
mail_index_transaction_unref(&t);
t->appends_nonsorted = FALSE;
array_free(&t->appends);
}
- t->log_updates = mail_index_transaction_has_changes(t);
+ mail_index_transaction_set_log_updates(t);
}
void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq)
if (array_is_created(&t->ext_hdr_updates)) {
const struct mail_index_transaction_ext_hdr_update *hdr;
- hdr =\v array_get(&t->ext_hdr_updates, &count);
+ hdr = array_get(&t->ext_hdr_updates, &count);
for (i = 0; i < count; i++) {
if (hdr[i].alloc_size > 0)
return TRUE;
t->conflict_seqs = seqs;
}
-static bool
-mail_index_update_cancel_array(ARRAY_TYPE(seq_range) *array, uint32_t seq)
-{
- if (array_is_created(array)) {
- if (seq_range_array_remove(array, seq)) {
- if (array_count(array) == 0)
- array_free(array);
- return TRUE;
- }
- }
- return FALSE;
-}
-
-static bool
-mail_index_update_cancel(struct mail_index_transaction *t, uint32_t seq)
-{
- struct mail_index_transaction_keyword_update *kw;
- struct mail_transaction_flag_update *updates, tmp_update;
- unsigned int i, count;
- bool ret, have_kw_changes = FALSE;
-
- ret = mail_index_update_cancel_array(&t->keyword_resets, seq);
- if (array_is_created(&t->keyword_updates)) {
- kw = array_get_modifiable(&t->keyword_updates, &count);
- for (i = 0; i < count; i++) {
- if (mail_index_update_cancel_array(&kw[i].add_seq, seq))
- ret = TRUE;
- if (mail_index_update_cancel_array(&kw[i].remove_seq,
- seq))
- ret = TRUE;
- if (array_is_created(&kw[i].add_seq) ||
- array_is_created(&kw[i].remove_seq))
- have_kw_changes = TRUE;
- }
- if (!have_kw_changes)
- array_free(&t->keyword_updates);
- }
-
- if (!array_is_created(&t->updates))
- return ret;
-
- updates = array_get_modifiable(&t->updates, &count);
- i = mail_index_transaction_get_flag_update_pos(t, 0, count, seq);
- if (i < count && updates[i].uid1 <= seq && updates[i].uid2 >= seq) {
- /* exists */
- ret = TRUE;
- if (updates[i].uid1 == seq && updates[i].uid2 == seq) {
- if (count > 1)
- array_delete(&t->updates, i, 1);
- else
- array_free(&t->updates);
- } else if (updates[i].uid1 == seq)
- updates[i].uid1++;
- else if (updates[i].uid2 == seq)
- updates[i].uid2--;
- else {
- /* need to split it in two */
- tmp_update = updates[i];
- tmp_update.uid1 = seq+1;
- updates[i].uid2 = seq-1;
- array_insert(&t->updates, i + 1, &tmp_update, 1);
- }
- }
- return ret;
-}
-
-void mail_index_transaction_check_conflicts(struct mail_index_transaction *t)
-{
- uint32_t seq;
-
- i_assert(t->max_modseq != 0);
- i_assert(t->conflict_seqs != NULL);
-
- if (t->max_modseq == mail_index_modseq_get_highest(t->view)) {
- /* no conflicts possible */
- return;
- }
- if (t->min_flagupdate_seq == 0) {
- /* no flag updates */
- return;
- }
-
- for (seq = t->min_flagupdate_seq; seq <= t->max_flagupdate_seq; seq++) {
- if (mail_index_modseq_lookup(t->view, seq) > t->max_modseq) {
- if (mail_index_update_cancel(t, seq))
- seq_range_array_add(t->conflict_seqs, 0, seq);
- }
- }
- t->log_updates = mail_index_transaction_has_changes(t);
-}
-
static struct mail_index_transaction_vfuncs trans_vfuncs = {
mail_index_transaction_commit_v,
mail_index_transaction_rollback_v
#include "lib.h"
#include "array.h"
-#include "buffer.h"
#include "write-full.h"
#include "mail-index-private.h"
-#include "mail-index-view-private.h"
-#include "mail-index-modseq.h"
-#include "mail-index-transaction-private.h"
#include "mail-transaction-log-private.h"
-struct log_append_context {
- struct mail_transaction_log_file *file;
- struct mail_index_transaction *trans;
- buffer_t *output;
-
- uint64_t modseq;
- uint32_t first_append_size;
- bool sync_includes_this;
-};
-
-static void log_append_buffer(struct log_append_context *ctx,
- const buffer_t *buf, const buffer_t *hdr_buf,
- enum mail_transaction_type type)
-{
- struct mail_transaction_header hdr;
- uint32_t hdr_size;
- size_t hdr_pos;
-
- i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
- i_assert((buf->used % 4) == 0);
- i_assert(hdr_buf == NULL || (hdr_buf->used % 4) == 0);
-
- if (buf->used == 0)
- return;
-
- memset(&hdr, 0, sizeof(hdr));
- hdr.type = type;
- if (type == MAIL_TRANSACTION_EXPUNGE)
- hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT;
- if ((ctx->trans->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
- hdr.type |= MAIL_TRANSACTION_EXTERNAL;
- hdr.size = sizeof(hdr) + buf->used +
- (hdr_buf == NULL ? 0 : hdr_buf->used);
-
- hdr_pos = ctx->output->used;
- buffer_append(ctx->output, &hdr, sizeof(hdr));
- if (hdr_buf != NULL)
- buffer_append(ctx->output, hdr_buf->data, hdr_buf->used);
- buffer_append(ctx->output, buf->data, buf->used);
-
- if (mail_transaction_header_has_modseq(buf->data,
- CONST_PTR_OFFSET(buf->data, sizeof(hdr)), ctx->modseq))
- ctx->modseq++;
-
- /* update the size */
- hdr_size = mail_index_uint32_to_offset(hdr.size);
- if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(ctx->file) &&
- ctx->first_append_size == 0) {
- /* size will be written later once everything
- is in disk */
- ctx->first_append_size = hdr_size;
- hdr.size = 0;
- } else {
- hdr.size = hdr_size;
- }
- buffer_write(ctx->output, hdr_pos, &hdr, sizeof(hdr));
-}
-
-static int log_buffer_move_to_memory(struct log_append_context *ctx)
+static int
+log_buffer_move_to_memory(struct mail_transaction_log_append_ctx *ctx)
{
- struct mail_transaction_log_file *file = ctx->file;
+ struct mail_transaction_log_file *file = ctx->log->head;
/* first we need to truncate this latest write so that log syncing
doesn't break */
i_assert(file->buffer_offset + file->buffer->used ==
file->sync_offset);
buffer_append_buf(file->buffer, ctx->output, 0, (size_t)-1);
- buffer_write(file->buffer, file->sync_offset - file->buffer_offset,
- &ctx->first_append_size, sizeof(uint32_t));
file->sync_offset = file->buffer_offset + file->buffer->used;
return 0;
}
-static int log_buffer_write(struct log_append_context *ctx, bool want_fsync)
+static int log_buffer_write(struct mail_transaction_log_append_ctx *ctx)
{
- struct mail_transaction_log_file *file = ctx->file;
+ struct mail_transaction_log_file *file = ctx->log->head;
+ struct mail_transaction_header *hdr;
+ uint32_t first_size;
if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
if (file->buffer == NULL) {
return 0;
}
- i_assert(ctx->first_append_size != 0);
+ /* size will be written later once everything is in disk */
+ hdr = buffer_get_space_unsafe(ctx->output, 0, sizeof(*hdr));
+ first_size = hdr->size;
+ i_assert(first_size != 0);
+ hdr->size = 0;
+
if (pwrite_full(file->fd, ctx->output->data, ctx->output->used,
file->sync_offset) < 0) {
/* write failure, fallback to in-memory indexes. */
+ hdr->size = first_size;
mail_index_file_set_syscall_error(file->log->index,
file->filepath,
"pwrite_full()");
/* now that the whole transaction has been written, rewrite the first
record's size so the transaction becomes visible */
- if (pwrite_full(file->fd, &ctx->first_append_size,
- sizeof(uint32_t), file->sync_offset) < 0) {
+ if (pwrite_full(file->fd, &first_size, sizeof(uint32_t),
+ file->sync_offset +
+ offsetof(struct mail_transaction_header, size)) < 0) {
mail_index_file_set_syscall_error(file->log->index,
file->filepath,
"pwrite_full()");
return log_buffer_move_to_memory(ctx);
}
- if ((want_fsync && !file->log->index->fsync_disable) ||
+ if ((ctx->want_fsync && !file->log->index->fsync_disable) ||
file->log->index->nfs_flush) {
if (fdatasync(file->fd) < 0) {
mail_index_file_set_syscall_error(file->log->index,
return 0;
}
-static const buffer_t *
-log_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend)
-{
- buffer_t *buf;
- const unsigned char *data, *mask;
- struct mail_transaction_header_update u;
- uint16_t offset;
- int state = 0;
-
- memset(&u, 0, sizeof(u));
-
- data = prepend ? t->pre_hdr_change : t->post_hdr_change;
- mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask;
-
- buf = buffer_create_dynamic(pool_datastack_create(), 256);
- for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) {
- if (offset < sizeof(t->pre_hdr_change) && mask[offset]) {
- if (state == 0) {
- u.offset = offset;
- state++;
- }
- } else {
- if (state > 0) {
- u.size = offset - u.offset;
- buffer_append(buf, &u, sizeof(u));
- buffer_append(buf, data + u.offset, u.size);
- state = 0;
- }
- }
- }
- return buf;
-}
-
-static void
-ext_reset_update_atomic(struct mail_index_transaction *t,
- uint32_t ext_id, uint32_t expected_reset_id)
-{
- const struct mail_index_ext *map_ext;
- struct mail_transaction_ext_reset *reset;
- uint32_t idx, reset_id;
-
- if (!mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
- /* new extension */
- reset_id = 1;
- } else {
- map_ext = array_idx(&t->view->index->map->extensions, idx);
- reset_id = map_ext->reset_id + 1;
- }
- if (reset_id != expected_reset_id) {
- /* ignore this extension update */
- mail_index_ext_set_reset_id(t, ext_id, 0);
- return;
- }
-
- if (reset_id == 0)
- reset_id++;
-
- array_idx_set(&t->ext_reset_ids, ext_id, &reset_id);
-
- /* reseting existing data is optional */
- if (array_is_created(&t->ext_resets)) {
- reset = array_idx_modifiable(&t->ext_resets, ext_id);
- if (reset->new_reset_id == (uint32_t)-1)
- reset->new_reset_id = reset_id;
- }
-}
-
-static void
-transaction_update_atomic_reset_ids(struct mail_index_transaction *t)
-{
- const uint32_t *expected_reset_ids;
- unsigned int ext_id, count;
-
- if (!array_is_created(&t->ext_reset_atomic))
- return;
-
- expected_reset_ids = array_get(&t->ext_reset_atomic, &count);
- for (ext_id = 0; ext_id < count; ext_id++) {
- if (expected_reset_ids[ext_id] != 0) {
- ext_reset_update_atomic(t, ext_id,
- expected_reset_ids[ext_id]);
- }
- }
-}
-
-static void log_append_ext_intro(struct log_append_context *ctx,
- uint32_t ext_id, uint32_t reset_id)
-{
- struct mail_index_transaction *t = ctx->trans;
- const struct mail_index_registered_ext *rext;
- struct mail_transaction_ext_intro *intro, *resizes;
- buffer_t *buf;
- uint32_t idx;
- unsigned int count;
-
- i_assert(ext_id != (uint32_t)-1);
-
- if (t->reset ||
- !mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
- /* new extension */
- idx = (uint32_t)-1;
- }
-
- rext = array_idx(&t->view->index->extensions, ext_id);
- if (!array_is_created(&t->ext_resizes)) {
- resizes = NULL;
- count = 0;
- } else {
- resizes = array_get_modifiable(&t->ext_resizes, &count);
- }
-
- buf = buffer_create_dynamic(pool_datastack_create(), 128);
- if (ext_id < count && resizes[ext_id].name_size != 0) {
- /* we're resizing the extension. use the resize struct. */
- intro = &resizes[ext_id];
-
- i_assert(intro->ext_id == idx);
- intro->name_size = idx != (uint32_t)-1 ? 0 :
- strlen(rext->name);
- buffer_append(buf, intro, sizeof(*intro));
- } else {
- /* generate a new intro structure */
- intro = buffer_append_space_unsafe(buf, sizeof(*intro));
- intro->ext_id = idx;
- intro->hdr_size = rext->hdr_size;
- intro->record_size = rext->record_size;
- intro->record_align = rext->record_align;
- intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK;
- intro->name_size = idx != (uint32_t)-1 ? 0 :
- strlen(rext->name);
- }
- if (reset_id != 0) {
- /* we're going to reset this extension in this transaction */
- intro->reset_id = reset_id;
- } else if (idx != (uint32_t)-1) {
- /* use the existing reset_id */
- const struct mail_index_ext *map_ext =
- array_idx(&t->view->index->map->extensions, idx);
- intro->reset_id = map_ext->reset_id;
- } else {
- /* new extension, reset_id defaults to 0 */
- }
- buffer_append(buf, rext->name, intro->name_size);
- if ((buf->used % 4) != 0)
- buffer_append_zero(buf, 4 - (buf->used % 4));
-
- if (ctx->file->sync_highest_modseq == 0 &&
- strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) {
- /* modseq tracking started */
- ctx->file->sync_highest_modseq = 1;
- }
-
- log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_INTRO);
-}
-
-static void
-log_append_ext_hdr_update(struct log_append_context *ctx,
- const struct mail_index_transaction_ext_hdr_update *hdr)
-{
- buffer_t *buf;
- const unsigned char *data, *mask;
- struct mail_transaction_ext_hdr_update u;
- uint16_t offset;
- bool started = FALSE;
-
- memset(&u, 0, sizeof(u));
-
- data = hdr->data;
- mask = hdr->mask;
-
- buf = buffer_create_dynamic(pool_datastack_create(), 256);
- for (offset = 0; offset <= hdr->alloc_size; offset++) {
- if (offset < hdr->alloc_size && mask[offset] != 0) {
- if (!started) {
- u.offset = offset;
- started = TRUE;
- }
- } else {
- if (started) {
- u.size = offset - u.offset;
- buffer_append(buf, &u, sizeof(u));
- buffer_append(buf, data + u.offset, u.size);
- started = FALSE;
- }
- }
- }
- if (buf->used % 4 != 0)
- buffer_append_zero(buf, 4 - buf->used % 4);
- log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_HDR_UPDATE);
-}
-
-static void
-mail_transaction_log_append_ext_intros(struct log_append_context *ctx)
-{
- struct mail_index_transaction *t = ctx->trans;
- const struct mail_transaction_ext_intro *resize;
- const struct mail_index_transaction_ext_hdr_update *hdrs;
- struct mail_transaction_ext_reset ext_reset;
- unsigned int resize_count, ext_count = 0;
- unsigned int hdrs_count, reset_id_count, reset_count;
- uint32_t ext_id, reset_id;
- const struct mail_transaction_ext_reset *reset;
- const uint32_t *reset_ids;
- buffer_t *reset_buf;
-
- if (!array_is_created(&t->ext_resizes)) {
- resize = NULL;
- resize_count = 0;
- } else {
- resize = array_get(&t->ext_resizes, &resize_count);
- if (ext_count < resize_count)
- ext_count = resize_count;
- }
-
- if (!array_is_created(&t->ext_reset_ids)) {
- reset_ids = NULL;
- reset_id_count = 0;
- } else {
- reset_ids = array_get(&t->ext_reset_ids, &reset_id_count);
- }
-
- if (!array_is_created(&t->ext_resets)) {
- reset = NULL;
- reset_count = 0;
- } else {
- reset = array_get(&t->ext_resets, &reset_count);
- if (ext_count < reset_count)
- ext_count = reset_count;
- }
-
- if (!array_is_created(&t->ext_hdr_updates)) {
- hdrs = NULL;
- hdrs_count = 0;
- } else {
- hdrs = array_get(&t->ext_hdr_updates, &hdrs_count);
- if (ext_count < hdrs_count)
- ext_count = hdrs_count;
- }
-
- memset(&ext_reset, 0, sizeof(ext_reset));
- reset_buf = buffer_create_data(pool_datastack_create(),
- &ext_reset, sizeof(ext_reset));
- buffer_set_used_size(reset_buf, sizeof(ext_reset));
-
- for (ext_id = 0; ext_id < ext_count; ext_id++) {
- if (ext_id < reset_count)
- ext_reset = reset[ext_id];
- else
- ext_reset.new_reset_id = 0;
- if ((ext_id < resize_count && resize[ext_id].name_size) ||
- ext_reset.new_reset_id != 0 ||
- (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0)) {
- if (ext_reset.new_reset_id != 0) {
- /* we're going to reset this extension
- immediately after the intro */
- reset_id = 0;
- } else {
- reset_id = ext_id < reset_id_count ?
- reset_ids[ext_id] : 0;
- }
- log_append_ext_intro(ctx, ext_id, reset_id);
- }
- if (ext_reset.new_reset_id != 0) {
- i_assert(ext_id < reset_id_count &&
- ext_reset.new_reset_id == reset_ids[ext_id]);
- log_append_buffer(ctx, reset_buf, NULL,
- MAIL_TRANSACTION_EXT_RESET);
- }
- if (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0) {
- T_BEGIN {
- log_append_ext_hdr_update(ctx, &hdrs[ext_id]);
- } T_END;
- }
- }
-}
-
-static void log_append_ext_recs(struct log_append_context *ctx,
- const ARRAY_TYPE(seq_array_array) *arr,
- enum mail_transaction_type type)
-{
- struct mail_index_transaction *t = ctx->trans;
- const ARRAY_TYPE(seq_array) *updates;
- const uint32_t *reset_ids;
- unsigned int ext_id, count, reset_id_count;
- uint32_t reset_id;
-
- if (!array_is_created(&t->ext_reset_ids)) {
- reset_ids = NULL;
- reset_id_count = 0;
- } else {
- reset_ids = array_get_modifiable(&t->ext_reset_ids,
- &reset_id_count);
- }
-
- updates = array_get(arr, &count);
- for (ext_id = 0; ext_id < count; ext_id++) {
- if (!array_is_created(&updates[ext_id]))
- continue;
-
- reset_id = ext_id < reset_id_count ? reset_ids[ext_id] : 0;
- log_append_ext_intro(ctx, ext_id, reset_id);
-
- log_append_buffer(ctx, updates[ext_id].arr.buffer, NULL, type);
- }
-}
-
static void
-log_append_keyword_update(struct log_append_context *ctx,
- buffer_t *hdr_buf, enum modify_type modify_type,
- const char *keyword, const buffer_t *buffer)
-{
- struct mail_transaction_keyword_update kt_hdr;
-
- memset(&kt_hdr, 0, sizeof(kt_hdr));
- kt_hdr.modify_type = modify_type;
- kt_hdr.name_size = strlen(keyword);
-
- buffer_set_used_size(hdr_buf, 0);
- buffer_append(hdr_buf, &kt_hdr, sizeof(kt_hdr));
- buffer_append(hdr_buf, keyword, kt_hdr.name_size);
- if ((hdr_buf->used % 4) != 0)
- buffer_append_zero(hdr_buf, 4 - (hdr_buf->used % 4));
-
- log_append_buffer(ctx, buffer, hdr_buf,
- MAIL_TRANSACTION_KEYWORD_UPDATE);
-}
-
-static enum mail_index_sync_type
-log_append_keyword_updates(struct log_append_context *ctx)
-{
- const struct mail_index_transaction_keyword_update *updates;
- const char *const *keywords;
- buffer_t *hdr_buf;
- enum mail_index_sync_type change_mask = 0;
- unsigned int i, count, keywords_count;
-
- hdr_buf = buffer_create_dynamic(pool_datastack_create(), 64);
-
- keywords = array_get_modifiable(&ctx->trans->view->index->keywords,
- &keywords_count);
- updates = array_get_modifiable(&ctx->trans->keyword_updates, &count);
- i_assert(count <= keywords_count);
-
- for (i = 0; i < count; i++) {
- if (array_is_created(&updates[i].add_seq)) {
- change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD;
- log_append_keyword_update(ctx, hdr_buf,
- MODIFY_ADD, keywords[i],
- updates[i].add_seq.arr.buffer);
- }
- if (array_is_created(&updates[i].remove_seq)) {
- change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE;
- log_append_keyword_update(ctx, hdr_buf,
- MODIFY_REMOVE, keywords[i],
- updates[i].remove_seq.arr.buffer);
- }
- }
- return change_mask;
-}
-
-static void log_append_sync_offset_if_needed(struct log_append_context *ctx)
+log_append_sync_offset_if_needed(struct mail_transaction_log_append_ctx *ctx)
{
+ struct mail_transaction_log_file *file = ctx->log->head;
struct mail_transaction_header_update *u;
- buffer_t *buf;
+ struct mail_transaction_header *hdr;
uint32_t offset;
- /* Update the tail offsets only when committing the sync transaction.
- Other transactions may not know the latest tail offset and might
- end up shrinking it. (Alternatively the shrinking tail offsets could
- just be ignored, which would probably work fine too.) */
- if (!ctx->trans->sync_transaction)
- return;
-
- if (ctx->file->max_tail_offset == ctx->file->sync_offset) {
+ if (file->max_tail_offset == file->sync_offset) {
/* FIXME: when we remove exclusive log locking, we
can't rely on this. then write non-changed offset + check
real offset + rewrite the new offset if other transactions
weren't written in the middle */
- ctx->file->max_tail_offset += ctx->output->used +
- sizeof(struct mail_transaction_header) +
- sizeof(*u) + sizeof(offset);
+ file->max_tail_offset += ctx->output->used +
+ sizeof(*hdr) + sizeof(*u) + sizeof(offset);
ctx->sync_includes_this = TRUE;
}
- offset = ctx->file->max_tail_offset;
+ offset = file->max_tail_offset;
- if (ctx->file->saved_tail_offset == offset)
+ if (file->saved_tail_offset == offset)
return;
- i_assert(offset > ctx->file->saved_tail_offset);
+ i_assert(offset > file->saved_tail_offset);
+
+ hdr = buffer_append_space_unsafe(ctx->output, sizeof(*hdr));
+ hdr->type = MAIL_TRANSACTION_HEADER_UPDATE | MAIL_TRANSACTION_EXTERNAL;
+ hdr->size = sizeof(*hdr) + sizeof(*u) + sizeof(uint32_t);
+ hdr->size = mail_index_uint32_to_offset(hdr->size);
- buf = buffer_create_static_hard(pool_datastack_create(),
- sizeof(*u) + sizeof(offset));
- u = buffer_append_space_unsafe(buf, sizeof(*u));
+ u = buffer_append_space_unsafe(ctx->output, sizeof(*u));
u->offset = offsetof(struct mail_index_header, log_file_tail_offset);
u->size = sizeof(offset);
- buffer_append(buf, &offset, sizeof(offset));
-
- log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_HEADER_UPDATE);
+ buffer_append(ctx->output, &offset, sizeof(offset));
}
-#define TRANSACTION_HAS_CHANGES(t) \
- ((t)->log_updates || (t)->log_ext_updates || \
- (array_is_created(&(t)->updates) && array_count(&(t)->updates) > 0))
-
static int
-mail_transaction_log_append_locked(struct mail_index_transaction *t,
- uint32_t *log_file_seq_r,
- uoff_t *log_file_offset_r)
+mail_transaction_log_append_locked(struct mail_transaction_log_append_ctx *ctx)
{
- enum mail_index_sync_type change_mask = 0;
- struct mail_index_view *view = t->view;
- struct mail_index *index;
- struct mail_transaction_log *log;
- struct mail_transaction_log_file *file;
- struct log_append_context ctx;
- uoff_t append_offset;
- bool want_fsync;
-
- index = mail_index_view_get_index(view);
- log = index->log;
-
- if (t->reset) {
- /* Reset the whole index, preserving only indexid. Begin by
- rotating the log. We don't care if we skip some non-synced
- transactions. */
- if (mail_transaction_log_rotate(log, TRUE) < 0)
- return -1;
-
- if (!TRANSACTION_HAS_CHANGES(t)) {
- /* we only wanted to reset */
- return 0;
- }
- }
-
- if (!index->log_locked) {
- /* update sync_offset */
- if (mail_transaction_log_file_map(log->head,
- log->head->sync_offset,
- (uoff_t)-1) <= 0)
- return -1;
- }
-
- if (array_is_created(&t->ext_reset_atomic) || t->max_modseq != 0) {
- if (mail_index_map(t->view->index,
- MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0)
- return -1;
- }
- if (array_is_created(&t->ext_reset_atomic))
- transaction_update_atomic_reset_ids(t);
- if (t->max_modseq != 0)
- mail_index_transaction_check_conflicts(t);
- if (!TRANSACTION_HAS_CHANGES(t)) {
- /* we aborted all changes, nothing else to do */
- return 0;
- }
- /* finally convert all sequences to UIDs before we write them,
- but after we've checked and removed conflicts */
- mail_index_transaction_convert_to_uids(t);
-
- file = log->head;
-
- i_assert(file->sync_offset >= file->buffer_offset);
-
- memset(&ctx, 0, sizeof(ctx));
- ctx.file = file;
- ctx.trans = t;
- ctx.output = buffer_create_dynamic(default_pool, 1024);
- ctx.modseq = file->sync_highest_modseq;
-
- /* send all extension introductions and resizes before appends
- to avoid resize overhead as much as possible */
- mail_transaction_log_append_ext_intros(&ctx);
-
- if (t->pre_hdr_changed) {
- log_append_buffer(&ctx,
- log_get_hdr_update_buffer(t, TRUE),
- NULL, MAIL_TRANSACTION_HEADER_UPDATE);
- }
- if (array_is_created(&t->appends)) {
- change_mask |= MAIL_INDEX_SYNC_TYPE_APPEND;
- log_append_buffer(&ctx, t->appends.arr.buffer, NULL,
- MAIL_TRANSACTION_APPEND);
- }
- if (array_is_created(&t->updates)) {
- change_mask |= MAIL_INDEX_SYNC_TYPE_FLAGS;
- log_append_buffer(&ctx, t->updates.arr.buffer, NULL,
- MAIL_TRANSACTION_FLAG_UPDATE);
- }
-
- if (array_is_created(&t->ext_rec_updates)) {
- log_append_ext_recs(&ctx, &t->ext_rec_updates,
- MAIL_TRANSACTION_EXT_REC_UPDATE);
- }
- if (array_is_created(&t->ext_rec_atomics)) {
- log_append_ext_recs(&ctx, &t->ext_rec_atomics,
- MAIL_TRANSACTION_EXT_ATOMIC_INC);
- }
-
- /* keyword resets before updates */
- if (array_is_created(&t->keyword_resets)) {
- change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
- log_append_buffer(&ctx, t->keyword_resets.arr.buffer,
- NULL, MAIL_TRANSACTION_KEYWORD_RESET);
- }
- if (array_is_created(&t->keyword_updates))
- change_mask |= log_append_keyword_updates(&ctx);
-
- if (array_is_created(&t->expunges)) {
- /* non-external expunges are only requests, ignore them when
- checking fsync_mask */
- if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
- change_mask |= MAIL_INDEX_SYNC_TYPE_EXPUNGE;
- log_append_buffer(&ctx, t->expunges.arr.buffer, NULL,
- MAIL_TRANSACTION_EXPUNGE);
- }
-
- if (t->post_hdr_changed) {
- log_append_buffer(&ctx, log_get_hdr_update_buffer(t, FALSE),
- NULL, MAIL_TRANSACTION_HEADER_UPDATE);
- }
-
- /* NOTE: mailbox sync offset update must be the last change.
- it may update the sync offset to include this transaction, so it
- needs to know this transaction's size */
- if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
- log_append_sync_offset_if_needed(&ctx);
+ struct mail_transaction_log_file *file = ctx->log->head;
if (file->sync_offset < file->last_size) {
/* there is some garbage at the end of the transaction log
file->sync_offset - file->buffer_offset);
if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
if (ftruncate(file->fd, file->sync_offset) < 0) {
- mail_index_file_set_syscall_error(index,
+ mail_index_file_set_syscall_error(ctx->log->index,
file->filepath, "ftruncate()");
}
}
}
- want_fsync = (view->index->fsync_mask & change_mask) != 0 ||
- (t->flags & MAIL_INDEX_TRANSACTION_FLAG_FSYNC) != 0;
- append_offset = file->sync_offset;
- if (log_buffer_write(&ctx, want_fsync) < 0) {
- buffer_free(&ctx.output);
- return -1;
- }
- file->sync_highest_modseq = ctx.modseq;
- buffer_free(&ctx.output);
-
- if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_HIDE) != 0) {
- /* mark the area covered by this transaction hidden */
- mail_index_view_add_hidden_transaction(view, file->hdr.file_seq,
- append_offset, file->sync_offset - append_offset);
- }
+ if (ctx->append_sync_offset)
+ log_append_sync_offset_if_needed(ctx);
- *log_file_seq_r = file->hdr.file_seq;
- *log_file_offset_r = file->sync_offset;
+ if (log_buffer_write(ctx) < 0)
+ return -1;
+ file->sync_highest_modseq = ctx->new_highest_modseq;
return 0;
}
-int mail_transaction_log_append(struct mail_index_transaction *t,
- uint32_t *log_file_seq_r,
- uoff_t *log_file_offset_r)
+int mail_transaction_log_append_begin(struct mail_index_transaction *t,
+ struct mail_transaction_log_append_ctx **ctx_r)
{
+ struct mail_transaction_log_append_ctx *ctx;
struct mail_index *index;
- int ret;
-
- *log_file_seq_r = 0;
- *log_file_offset_r = 0;
-
- if (!TRANSACTION_HAS_CHANGES(t) && !t->reset) {
- /* nothing to append */
- return 0;
- }
index = mail_index_view_get_index(t->view);
if (!index->log_locked) {
if (mail_transaction_log_lock_head(index->log) < 0)
return -1;
}
+ ctx = i_new(struct mail_transaction_log_append_ctx, 1);
+ ctx->log = index->log;
+ ctx->output = buffer_create_dynamic(default_pool, 1024);
+
+ *ctx_r = ctx;
+ return 0;
+}
+
+int mail_transaction_log_append_commit(struct mail_transaction_log_append_ctx **_ctx)
+{
+ struct mail_transaction_log_append_ctx *ctx = *_ctx;
+ struct mail_index *index = ctx->log->index;
+ int ret = 0;
- ret = mail_transaction_log_append_locked(t, log_file_seq_r,
- log_file_offset_r);
+ *_ctx = NULL;
+
+ if (ctx->output->used > 0)
+ ret = mail_transaction_log_append_locked(ctx);
if (!index->log_locked)
mail_transaction_log_file_unlock(index->log->head);
+
+ buffer_free(&ctx->output);
+ i_free(ctx);
return ret;
}
int32_t diff;
};
+struct mail_transaction_log_append_ctx {
+ struct mail_transaction_log *log;
+ buffer_t *output;
+
+ uint64_t new_highest_modseq;
+ unsigned int append_sync_offset:1;
+ unsigned int sync_includes_this:1;
+ unsigned int want_fsync:1;
+};
+
#define LOG_IS_BEFORE(seq1, offset1, seq2, offset2) \
(((offset1) < (offset2) && (seq1) == (seq2)) || (seq1) < (seq2))
void mail_transaction_log_views_close(struct mail_transaction_log *log);
-/* Write data to transaction log. This is atomic operation. Sequences in
- updates[] and expunges[] are relative to given view, they're modified
- to real ones. If nothing is written, log_file_seq_r and log_file_offset_r
- will be set to 0. */
-int mail_transaction_log_append(struct mail_index_transaction *t,
- uint32_t *log_file_seq_r,
- uoff_t *log_file_offset_r);
+int mail_transaction_log_append_begin(struct mail_index_transaction *t,
+ struct mail_transaction_log_append_ctx **ctx_r);
+int mail_transaction_log_append_commit(struct mail_transaction_log_append_ctx **ctx);
/* Lock transaction log for index synchronization. Log cannot be read or
written to while it's locked. Returns end offset. */
projects / thirdparty / dovecot / core.git / commitdiff
