From: Timo Sirainen Date: Thu, 29 May 2008 01:47:53 +0000 (+0300) Subject: Added mail_index_ext_reset_inc() to atomically increase extension's X-Git-Tag: 1.1.rc6~12 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=6e07b4251bf6a3cf34019c351a32a65c08392e58;p=thirdparty%2Fdovecot%2Fcore.git Added mail_index_ext_reset_inc() to atomically increase extension's reset_id. Added clear_data parameter to mail_index_ext_reset*(). --HG-- branch : HEAD --- diff --git a/src/lib-index/mail-cache-compress.c b/src/lib-index/mail-cache-compress.c index ece39327fc..d8879649fd 100644 --- a/src/lib-index/mail-cache-compress.c +++ b/src/lib-index/mail-cache-compress.c @@ -414,7 +414,7 @@ static int mail_cache_compress_locked(struct mail_cache *cache, /* once we're sure that the compression was successful, update the offsets */ - mail_index_ext_reset(trans, cache->ext_id, file_seq); + mail_index_ext_reset(trans, cache->ext_id, file_seq, TRUE); offsets = array_get(&ext_offsets, &count); for (i = 0; i < count; i++) { if (offsets[i] != 0) { diff --git a/src/lib-index/mail-index-sync-ext.c b/src/lib-index/mail-index-sync-ext.c index ca9f86abfe..7bea292905 100644 --- a/src/lib-index/mail-index-sync-ext.c +++ b/src/lib-index/mail-index-sync-ext.c @@ -424,7 +424,8 @@ int mail_index_sync_ext_intro(struct mail_index_sync_map_ctx *ctx, ctx->cur_ext_ignore = FALSE; } else { /* extension was reset and this transaction hadn't - yet seen it. ignore this update. */ + yet seen it. ignore this update (except for + resets). */ ctx->cur_ext_ignore = TRUE; } @@ -476,23 +477,39 @@ int mail_index_sync_ext_intro(struct mail_index_sync_map_ctx *ctx, return 1; } +static void mail_index_sync_ext_clear(struct mail_index_view *view, + struct mail_index_map *map, + struct mail_index_ext *ext) +{ + struct mail_index_record *rec; + uint32_t i; + + memset(buffer_get_space_unsafe(map->hdr_copy_buf, ext->hdr_offset, + ext->hdr_size), 0, ext->hdr_size); + map->hdr_base = map->hdr_copy_buf->data; + + for (i = 0; i < view->map->rec_map->records_count; i++) { + rec = MAIL_INDEX_MAP_IDX(view->map, i); + memset(PTR_OFFSET(rec, ext->record_offset), 0, + ext->record_size); + } + map->rec_map->write_seq_first = 1; + map->rec_map->write_seq_last = view->map->rec_map->records_count; +} + int mail_index_sync_ext_reset(struct mail_index_sync_map_ctx *ctx, const struct mail_transaction_ext_reset *u) { - struct mail_index_view *view = ctx->view; - struct mail_index_map *map = view->map; + struct mail_index_map *map = ctx->view->map; struct mail_index_ext_header *ext_hdr; struct mail_index_ext *ext; - struct mail_index_record *rec; - uint32_t i; if (ctx->cur_ext_map_idx == (uint32_t)-1) { mail_index_sync_set_corrupted(ctx, "Extension reset without intro prefix"); return -1; } - if (ctx->cur_ext_ignore) - return 1; + /* since we're resetting the extension, don't check cur_ext_ignore */ /* a new index file will be created, so the old data won't be accidentally used by other processes. */ @@ -501,21 +518,11 @@ int mail_index_sync_ext_reset(struct mail_index_sync_map_ctx *ctx, ext = array_idx_modifiable(&map->extensions, ctx->cur_ext_map_idx); ext->reset_id = u->new_reset_id; - memset(buffer_get_space_unsafe(map->hdr_copy_buf, ext->hdr_offset, - ext->hdr_size), 0, ext->hdr_size); - map->hdr_base = map->hdr_copy_buf->data; - - for (i = 0; i < view->map->rec_map->records_count; i++) { - rec = MAIL_INDEX_MAP_IDX(view->map, i); - memset(PTR_OFFSET(rec, ext->record_offset), 0, - ext->record_size); - } - map->rec_map->write_seq_first = 1; - map->rec_map->write_seq_last = view->map->rec_map->records_count; + if (!u->preserve_data) + mail_index_sync_ext_clear(ctx->view, map, ext); ext_hdr = get_ext_header(map, ext); ext_hdr->reset_id = u->new_reset_id; - return 1; } diff --git a/src/lib-index/mail-index-sync-update.c b/src/lib-index/mail-index-sync-update.c index 71480d6802..d6fe2a93a4 100644 --- a/src/lib-index/mail-index-sync-update.c +++ b/src/lib-index/mail-index-sync-update.c @@ -532,15 +532,17 @@ int mail_index_sync_record(struct mail_index_sync_map_ctx *ctx, break; } case MAIL_TRANSACTION_EXT_RESET: { - const struct mail_transaction_ext_reset *rec = data; + struct mail_transaction_ext_reset rec; - if (hdr->size != sizeof(*rec)) { + /* old versions have only new_reset_id */ + if (hdr->size < sizeof(uint32_t)) { mail_index_sync_set_corrupted(ctx, "ext reset: invalid record size"); ret = -1; break; } - ret = mail_index_sync_ext_reset(ctx, rec); + memcpy(&rec, data, I_MIN(hdr->size, sizeof(rec))); + ret = mail_index_sync_ext_reset(ctx, &rec); break; } case MAIL_TRANSACTION_EXT_HDR_UPDATE: { @@ -781,6 +783,9 @@ int mail_index_sync_map(struct mail_index_map **_map, map->hdr_base = map->hdr_copy_buf->data; } + mail_transaction_log_view_get_prev_pos(view->log_view, + &prev_seq, &prev_offset); + mail_index_sync_map_init(&sync_map_ctx, view, type); if (reset) { /* Reset the entire index. Leave only indexid and diff --git a/src/lib-index/mail-index-transaction-private.h b/src/lib-index/mail-index-transaction-private.h index b490777ce1..c6b47e3dfe 100644 --- a/src/lib-index/mail-index-transaction-private.h +++ b/src/lib-index/mail-index-transaction-private.h @@ -51,8 +51,9 @@ struct mail_index_transaction { struct mail_index_transaction_ext_hdr_update *); ARRAY_DEFINE(ext_rec_updates, ARRAY_TYPE(seq_array)); ARRAY_DEFINE(ext_resizes, struct mail_transaction_ext_intro); - ARRAY_DEFINE(ext_resets, uint32_t); + ARRAY_DEFINE(ext_resets, struct mail_transaction_ext_reset); ARRAY_DEFINE(ext_reset_ids, uint32_t); + ARRAY_DEFINE(ext_reset_atomic, uint32_t); ARRAY_DEFINE(keyword_updates, struct mail_index_transaction_keyword_update); diff --git a/src/lib-index/mail-index-transaction.c b/src/lib-index/mail-index-transaction.c index dd514b9587..0768b1f75b 100644 --- a/src/lib-index/mail-index-transaction.c +++ b/src/lib-index/mail-index-transaction.c @@ -75,6 +75,8 @@ void mail_index_transaction_reset(struct mail_index_transaction *t) array_free(&t->ext_resets); if (array_is_created(&t->ext_reset_ids)) array_free(&t->ext_reset_ids); + if (array_is_created(&t->ext_reset_atomic)) + array_free(&t->ext_reset_atomic); t->first_new_seq = mail_index_view_get_messages_count(t->view)+1; t->last_new_seq = 0; @@ -1087,18 +1089,36 @@ void mail_index_ext_resize(struct mail_index_transaction *t, uint32_t ext_id, } void mail_index_ext_reset(struct mail_index_transaction *t, uint32_t ext_id, - uint32_t reset_id) + uint32_t reset_id, bool clear_data) { + struct mail_transaction_ext_reset reset; + i_assert(reset_id != 0); + memset(&reset, 0, sizeof(reset)); + reset.new_reset_id = reset_id; + reset.preserve_data = !clear_data; + mail_index_ext_set_reset_id(t, ext_id, reset_id); if (!array_is_created(&t->ext_resets)) i_array_init(&t->ext_resets, ext_id + 2); - array_idx_set(&t->ext_resets, ext_id, &reset_id); + array_idx_set(&t->ext_resets, ext_id, &reset); t->log_ext_updates = TRUE; } +void mail_index_ext_reset_inc(struct mail_index_transaction *t, uint32_t ext_id, + uint32_t prev_reset_id, bool clear_data) +{ + uint32_t expected_reset_id = prev_reset_id + 1; + + mail_index_ext_reset(t, ext_id, (uint32_t)-1, clear_data); + + if (!array_is_created(&t->ext_reset_atomic)) + i_array_init(&t->ext_reset_atomic, ext_id + 2); + array_idx_set(&t->ext_reset_atomic, ext_id, &expected_reset_id); +} + static bool mail_index_transaction_has_ext_changes(struct mail_index_transaction *t) { @@ -1123,11 +1143,11 @@ mail_index_transaction_has_ext_changes(struct mail_index_transaction *t) } } if (array_is_created(&t->ext_resets)) { - const uint32_t *ids; + const struct mail_transaction_ext_reset *resets; - ids = array_get(&t->ext_resets, &count); + resets = array_get(&t->ext_resets, &count); for (i = 0; i < count; i++) { - if (ids[i] != 0) + if (resets[i].new_reset_id != 0) return TRUE; } } diff --git a/src/lib-index/mail-index.h b/src/lib-index/mail-index.h index ef5a1fef46..e116c615b6 100644 --- a/src/lib-index/mail-index.h +++ b/src/lib-index/mail-index.h @@ -428,14 +428,21 @@ void mail_index_ext_resize(struct mail_index_transaction *t, uint32_t ext_id, uint32_t hdr_size, uint16_t record_size, uint16_t record_align); -/* Reset extension records and header. Any updates for this extension which - were issued before the writer had seen this reset are discarded. reset_id is - used to figure this out, so it must be different every time. */ +/* Reset extension. Any updates for this extension which were issued before the + writer had seen this reset are discarded. reset_id is used to figure this + out, so it must be different every time. If clear_data=TRUE, records and + header is zeroed. */ void mail_index_ext_reset(struct mail_index_transaction *t, uint32_t ext_id, - uint32_t reset_id); -/* Discard existing extension updates and write new updates using the given - reset_id. The difference to mail_index_ext_reset() is that this doesn't - clear any existing record or header data. */ + uint32_t reset_id, bool clear_data); +/* Like mail_index_ext_reset(), but increase extension's reset_id atomically + when the transaction is being committed. If prev_reset_id doesn't match the + latest reset_id, the reset_id isn't increased and all extension changes are + ignored. */ +void mail_index_ext_reset_inc(struct mail_index_transaction *t, uint32_t ext_id, + uint32_t prev_reset_id, bool clear_data); +/* Discard existing extension updates in this transaction and write new updates + using the given reset_id. The difference to mail_index_ext_reset() is that + this doesn't clear any existing record or header data. */ void mail_index_ext_set_reset_id(struct mail_index_transaction *t, uint32_t ext_id, uint32_t reset_id); /* Get the current reset_id for given extension. Returns TRUE if it exists. */ diff --git a/src/lib-index/mail-transaction-log-append.c b/src/lib-index/mail-transaction-log-append.c index 3e8973a71f..f45f08f926 100644 --- a/src/lib-index/mail-transaction-log-append.c +++ b/src/lib-index/mail-transaction-log-append.c @@ -166,6 +166,58 @@ log_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend) return buf; } +static void +ext_reset_update_atomic(struct mail_index_transaction *t, + uint32_t ext_id, uint32_t expected_reset_id) +{ + const struct mail_index_ext *map_ext; + struct mail_transaction_ext_reset *reset; + uint32_t idx, reset_id; + + if (!mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) { + /* new extension */ + reset_id = 1; + } else { + map_ext = array_idx(&t->view->index->map->extensions, idx); + reset_id = map_ext->reset_id + 1; + } + if (reset_id != expected_reset_id) { + /* ignore this extension update */ + mail_index_ext_set_reset_id(t, ext_id, 0); + return; + } + + if (reset_id == 0) + reset_id++; + + array_idx_set(&t->ext_reset_ids, ext_id, &reset_id); + + /* reseting existing data is optional */ + if (array_is_created(&t->ext_resets)) { + reset = array_idx_modifiable(&t->ext_resets, ext_id); + if (reset->new_reset_id == (uint32_t)-1) + reset->new_reset_id = reset_id; + } +} + +static void +transaction_update_atomic_reset_ids(struct mail_index_transaction *t) +{ + const uint32_t *expected_reset_ids; + unsigned int ext_id, count; + + if (!array_is_created(&t->ext_reset_atomic)) + return; + + expected_reset_ids = array_get(&t->ext_reset_atomic, &count); + for (ext_id = 0; ext_id < count; ext_id++) { + if (expected_reset_ids[ext_id] != 0) { + ext_reset_update_atomic(t, ext_id, + expected_reset_ids[ext_id]); + } + } +} + static void log_append_ext_intro(struct log_append_context *ctx, uint32_t ext_id, uint32_t reset_id) { @@ -257,7 +309,8 @@ mail_transaction_log_append_ext_intros(struct log_append_context *ctx) unsigned int update_count, resize_count, ext_count = 0; unsigned int hdrs_count, reset_id_count, reset_count; uint32_t ext_id, reset_id; - const uint32_t *reset_ids, *reset; + const struct mail_transaction_ext_reset *reset; + const uint32_t *reset_ids; const ARRAY_TYPE(seq_array) *update; buffer_t *buf; @@ -304,15 +357,15 @@ mail_transaction_log_append_ext_intros(struct log_append_context *ctx) } memset(&ext_reset, 0, sizeof(ext_reset)); - buf = buffer_create_data(pool_datastack_create(), &ext_reset, sizeof(ext_reset)); buffer_set_used_size(buf, sizeof(ext_reset)); for (ext_id = 0; ext_id < ext_count; ext_id++) { - ext_reset.new_reset_id = - ext_id < reset_count && reset[ext_id] != 0 ? - reset[ext_id] : 0; + if (ext_id < reset_count) + ext_reset = reset[ext_id]; + else + ext_reset.new_reset_id = 0; if ((ext_id < resize_count && resize[ext_id].name_size) || (ext_id < update_count && array_is_created(&update[ext_id])) || @@ -499,6 +552,18 @@ mail_transaction_log_append_locked(struct mail_index_transaction *t, return -1; } + if (array_is_created(&t->ext_reset_atomic)) { + if (mail_index_map(t->view->index, + MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0) + return -1; + transaction_update_atomic_reset_ids(t); + + if (!TRANSACTION_HAS_CHANGES(t)) { + /* we aborted the ext changes, nothing else to do */ + return 0; + } + } + file = log->head; if (file->sync_offset < file->buffer_offset) diff --git a/src/lib-index/mail-transaction-log.h b/src/lib-index/mail-transaction-log.h index d5c2565bc8..b3445fa1ea 100644 --- a/src/lib-index/mail-transaction-log.h +++ b/src/lib-index/mail-transaction-log.h @@ -95,6 +95,8 @@ struct mail_transaction_ext_intro { struct mail_transaction_ext_reset { uint32_t new_reset_id; + uint8_t preserve_data; + uint8_t unused_padding[3]; }; /* these are set for the last ext_intro */ diff --git a/src/lib-storage/index/dbox/dbox-sync-rebuild.c b/src/lib-storage/index/dbox/dbox-sync-rebuild.c index ab593e33fd..ef80590242 100644 --- a/src/lib-storage/index/dbox/dbox-sync-rebuild.c +++ b/src/lib-storage/index/dbox/dbox-sync-rebuild.c @@ -70,7 +70,7 @@ dbox_sync_index_copy_cache(struct dbox_sync_rebuild_context *ctx, ctx->cache_used = TRUE; ctx->cache_reset_id = reset_id; mail_index_ext_reset(ctx->trans, ctx->cache_ext_id, - ctx->cache_reset_id); + ctx->cache_reset_id, TRUE); } if (ctx->cache_reset_id == reset_id) { mail_index_update_ext(ctx->trans, new_seq,