From: Timo Sirainen Date: Mon, 14 Jun 2004 02:07:33 +0000 (+0300) Subject: Modifying extra_records should work now. X-Git-Tag: 1.1.alpha1~3991 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1175f27441385a7011629f295f42708f9a3a4ffc;p=thirdparty%2Fdovecot%2Fcore.git Modifying extra_records should work now. --HG-- branch : HEAD --- diff --git a/src/lib-index/mail-index-private.h b/src/lib-index/mail-index-private.h index fb13a8319a..f84d45c004 100644 --- a/src/lib-index/mail-index-private.h +++ b/src/lib-index/mail-index-private.h @@ -6,6 +6,12 @@ struct mail_transaction_header; +/* Maximum number of extra record data items we allowed. Currently maximum + would be 32767 because of how transaction log is implemented. Raising this + limit only means it takes a few bytes more memory, but 32 should be enough + for a long time, right? :) */ +#define MAIL_INDEX_MAX_EXTRA_RECORDS 32 + /* Index file is grown exponentially when we're adding less than this many records. */ #define MAIL_INDEX_MAX_POWER_GROW (1024*1024 / sizeof(struct mail_index_record)) @@ -49,6 +55,10 @@ struct mail_index { mode_t mode; gid_t gid; + + uint16_t extra_record_offsets[MAIL_INDEX_MAX_EXTRA_RECORDS]; + uint16_t extra_record_sizes[MAIL_INDEX_MAX_EXTRA_RECORDS]; + unsigned int extra_records_count; unsigned int record_size; char *filepath; diff --git a/src/lib-index/mail-index-sync-update.c b/src/lib-index/mail-index-sync-update.c index cd475aadee..f3841c20fb 100644 --- a/src/lib-index/mail-index-sync-update.c +++ b/src/lib-index/mail-index-sync-update.c @@ -214,6 +214,31 @@ static int sync_header_update(const struct mail_transaction_header_update *u, return 1; } +static int +sync_extra_rec_update(const struct mail_transaction_extra_rec_header *hdr, + const struct mail_transaction_extra_rec_update *u, + void *context) +{ + struct mail_index_view *view = context; + struct mail_index_record *rec; + uint32_t seq; + uint16_t offset, size; + int ret; + + ret = mail_index_lookup_uid_range(view, u->uid, u->uid, + &seq, &seq); + i_assert(ret == 0); + + if (seq != 0) { + offset = view->index->extra_record_offsets[hdr->idx]; + size = view->index->extra_record_sizes[hdr->idx]; + + rec = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1); + memcpy(PTR_OFFSET(rec, offset), u->data, size); + } + return 1; +} + static int mail_index_grow(struct mail_index *index, struct mail_index_map *map, unsigned int count) { @@ -354,5 +379,5 @@ int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx) struct mail_transaction_map_functions mail_index_map_sync_funcs = { sync_expunge, sync_append, sync_flag_update, - sync_cache_update, sync_header_update + sync_cache_update, sync_header_update, sync_extra_rec_update }; diff --git a/src/lib-index/mail-index-transaction-private.h b/src/lib-index/mail-index-transaction-private.h index 4b29e5846a..f6be129e19 100644 --- a/src/lib-index/mail-index-transaction-private.h +++ b/src/lib-index/mail-index-transaction-private.h @@ -16,6 +16,8 @@ struct mail_index_transaction { unsigned char hdr_change[sizeof(struct mail_index_header)]; unsigned char hdr_mask[sizeof(struct mail_index_header)]; + buffer_t *extra_rec_updates[MAIL_INDEX_MAX_EXTRA_RECORDS]; + buffer_t *cache_updates; unsigned int hide_transaction:1; unsigned int hdr_changed:1; diff --git a/src/lib-index/mail-index-transaction.c b/src/lib-index/mail-index-transaction.c index ea9d85e447..7ed4fecf49 100644 --- a/src/lib-index/mail-index-transaction.c +++ b/src/lib-index/mail-index-transaction.c @@ -28,13 +28,23 @@ mail_index_transaction_begin(struct mail_index_view *view, int hide) static void mail_index_transaction_free(struct mail_index_transaction *t) { + unsigned int i; + mail_index_view_transaction_unref(t->view); + + for (i = 0; i < t->view->index->extra_records_count; i++) { + if (t->extra_rec_updates[i] != NULL) + buffer_free(t->extra_rec_updates[i]); + } + if (t->appends != NULL) buffer_free(t->appends); if (t->expunges != NULL) buffer_free(t->expunges); if (t->updates != NULL) buffer_free(t->updates); + if (t->cache_updates != NULL) + buffer_free(t->cache_updates); i_free(t); } @@ -66,15 +76,25 @@ mail_index_buffer_convert_to_uids(struct mail_index_view *view, static int mail_index_transaction_convert_to_uids(struct mail_index_transaction *t) { + struct mail_index *index = t->view->index; + unsigned int i; + if (mail_index_view_lock(t->view) < 0) return -1; + for (i = 0; i < index->extra_records_count; i++) { + mail_index_buffer_convert_to_uids(t->view, + t->extra_rec_updates[i], + index->extra_record_sizes[i], + FALSE); + } + mail_index_buffer_convert_to_uids(t->view, t->expunges, sizeof(struct mail_transaction_expunge), TRUE); mail_index_buffer_convert_to_uids(t->view, t->updates, sizeof(struct mail_transaction_flag_update), TRUE); mail_index_buffer_convert_to_uids(t->view, t->cache_updates, - sizeof(struct mail_transaction_cache_update), TRUE); + sizeof(struct mail_transaction_cache_update), FALSE); return 0; } @@ -391,45 +411,65 @@ static void mail_index_transaction_add_last(struct mail_index_transaction *t) &update, sizeof(update)); } -void mail_index_update_cache(struct mail_index_transaction *t, - uint32_t seq, uint32_t offset) +static void mail_index_update_seq_buffer(buffer_t **buffer, uint32_t seq, + const void *record, size_t record_size) { - struct mail_transaction_cache_update *data, update; unsigned int idx, left_idx, right_idx; + void *data; + uint32_t full_record_size, *seq_p; size_t size; - if (t->cache_updates == NULL) { - t->cache_updates = buffer_create_dynamic(default_pool, - 1024, (size_t)-1); - } + full_record_size = record_size + sizeof(uint32_t); - data = buffer_get_modifyable_data(t->cache_updates, &size); - size /= sizeof(*data); + if (*buffer == NULL) + *buffer = buffer_create_dynamic(default_pool, 1024, (size_t)-1); + data = buffer_get_modifyable_data(*buffer, &size); /* we're probably appending it, check */ - if (size == 0 || data[size-1].uid < seq) - idx = size; + if (size == 0) + idx = 0; + else if (*((uint32_t *)PTR_OFFSET(data, size-full_record_size)) < seq) + idx = size / full_record_size; else { - idx = 0; left_idx = 0; right_idx = size; + idx = 0; left_idx = 0; right_idx = size / full_record_size; while (left_idx < right_idx) { idx = (left_idx + right_idx) / 2; - if (data[idx].uid < seq) + seq_p = PTR_OFFSET(data, idx * full_record_size); + if (*seq_p < seq) left_idx = idx+1; - else if (data[idx].uid > seq) + else if (*seq_p > seq) right_idx = idx; else { /* already there, update */ - data[idx].cache_offset = offset; + memcpy(seq_p+1, record, record_size); return; } } } - update.uid = seq; - update.cache_offset = offset; - buffer_insert(t->updates, idx * sizeof(update), - &update, sizeof(update)); + idx *= full_record_size; + buffer_copy(*buffer, idx + full_record_size, *buffer, idx, (size_t)-1); + seq_p = buffer_get_space_unsafe(*buffer, idx, full_record_size); + + *seq_p = seq; + memcpy(seq_p+1, record, record_size); +} + +void mail_index_update_cache(struct mail_index_transaction *t, + uint32_t seq, uint32_t offset) +{ + mail_index_update_seq_buffer(&t->cache_updates, seq, + &offset, sizeof(offset)); +} + +void mail_index_update_extra_rec(struct mail_index_transaction *t, + uint32_t seq, uint32_t idx, const void *data) +{ + i_assert(idx < t->view->index->extra_records_count); + + mail_index_update_seq_buffer(&t->extra_rec_updates[idx], seq, data, + t->view->index->extra_record_sizes[idx]); } void mail_index_update_header(struct mail_index_transaction *t, diff --git a/src/lib-index/mail-index.c b/src/lib-index/mail-index.c index 31ebbc948c..6a80485580 100644 --- a/src/lib-index/mail-index.c +++ b/src/lib-index/mail-index.c @@ -41,14 +41,29 @@ void mail_index_free(struct mail_index *index) i_free(index); } -uint16_t mail_index_register_record_extra(struct mail_index *index, - uint16_t size) +uint32_t mail_index_register_record_extra(struct mail_index *index, + uint16_t size, uint32_t *offset_r) { + uint16_t offset; + i_assert(!index->opened); i_assert(index->record_size + size <= 65535); + i_assert(size % 4 == 0); + + if (index->extra_records_count >= MAIL_INDEX_MAX_EXTRA_RECORDS) { + i_panic("Maximum extra record count reached, " + "you'll need to recompile with larger limit. " + "MAIL_INDEX_MAX_EXTRA_RECORDS = %d", + MAIL_INDEX_MAX_EXTRA_RECORDS); + } + offset = index->record_size; index->record_size += size; - return index->record_size - size; + *offset_r = offset; + + index->extra_record_offsets[index->extra_records_count] = offset; + index->extra_record_sizes[index->extra_records_count] = size; + return index->extra_records_count++; } static int mail_index_check_header(struct mail_index *index, diff --git a/src/lib-index/mail-index.h b/src/lib-index/mail-index.h index dd2113daae..a38c138ce7 100644 --- a/src/lib-index/mail-index.h +++ b/src/lib-index/mail-index.h @@ -138,9 +138,11 @@ void mail_index_free(struct mail_index *index); /* register extra data to be used in mail_index_record. calls to this function must remain in same order as long as the index exists, or it breaks. - returns the relative offset in mail_index_record for the data. */ -uint16_t mail_index_register_record_extra(struct mail_index *index, - uint16_t size); + + returns the index number, and sets *offset_r to relative offset in + mail_index_record for the data. */ +uint32_t mail_index_register_record_extra(struct mail_index *index, + uint16_t size, uint32_t *offset_r); int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags); void mail_index_close(struct mail_index *index); @@ -267,6 +269,8 @@ void mail_index_update_flags(struct mail_index_transaction *t, uint32_t seq, /* Update field in header. */ void mail_index_update_header(struct mail_index_transaction *t, size_t offset, const void *data, size_t size); +void mail_index_update_extra_rec(struct mail_index_transaction *t, + uint32_t seq, uint32_t idx, const void *data); /* Returns the last error code. */ enum mail_index_error mail_index_get_last_error(struct mail_index *index); diff --git a/src/lib-index/mail-transaction-log-view.c b/src/lib-index/mail-transaction-log-view.c index 38055341e9..b185f36239 100644 --- a/src/lib-index/mail-transaction-log-view.c +++ b/src/lib-index/mail-transaction-log-view.c @@ -262,6 +262,7 @@ static int log_view_get_next(struct mail_transaction_log_view *view, hdr = CONST_PTR_OFFSET(data, view->cur_offset - file->buffer_offset); view->cur_offset += sizeof(*hdr); + data = CONST_PTR_OFFSET(hdr, sizeof(*hdr)); if (file_size - view->cur_offset < hdr->size) { mail_transaction_log_file_set_corrupted(file, @@ -296,6 +297,16 @@ static int log_view_get_next(struct mail_transaction_log_view *view, "extra bits in header type: 0x%x", hdr->type & MAIL_TRANSACTION_TYPE_MASK); return -1; + } else if (hdr->type == MAIL_TRANSACTION_EXTRA_REC_UPDATE) { + const struct mail_transaction_extra_rec_header *ehdr = data; + + if (ehdr->idx >= view->log->index->extra_records_count) { + mail_transaction_log_file_set_corrupted(file, + "extra record update out of range (%u > %u)", + ehdr->idx, + view->log->index->extra_records_count); + return -1; + } } if (hdr->size % record_size != 0) { @@ -308,8 +319,7 @@ static int log_view_get_next(struct mail_transaction_log_view *view, } *hdr_r = hdr; - *data_r = CONST_PTR_OFFSET(data, view->cur_offset - - file->buffer_offset); + *data_r = data; view->cur_offset += hdr->size; return 1; } diff --git a/src/lib-index/mail-transaction-log.c b/src/lib-index/mail-transaction-log.c index 8de3ffd0e7..a505a523b1 100644 --- a/src/lib-index/mail-transaction-log.c +++ b/src/lib-index/mail-transaction-log.c @@ -936,9 +936,9 @@ static int mail_transaction_log_fix_appends(struct mail_transaction_log *log, return ret; } -static int -log_append_buffer(struct mail_transaction_log_file *file, const buffer_t *buf, - enum mail_transaction_type type, int external) +static int log_append_buffer(struct mail_transaction_log_file *file, + const buffer_t *buf, const buffer_t *hdr_buf, + enum mail_transaction_type type, int external) { struct mail_transaction_header hdr; const void *data; @@ -946,15 +946,9 @@ log_append_buffer(struct mail_transaction_log_file *file, const buffer_t *buf, i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0); - if (buf != NULL) { - data = buffer_get_data(buf, &size); - if (size == 0) - return 0; - } else { - /* write only the header */ - data = NULL; - size = 0; - } + data = buffer_get_data(buf, &size); + if (size == 0) + return 0; hdr.type = type; if (type == MAIL_TRANSACTION_EXPUNGE) @@ -967,11 +961,22 @@ log_append_buffer(struct mail_transaction_log_file *file, const buffer_t *buf, return -1; file->hdr.used_size += sizeof(hdr); - if (size != 0) { - if (pwrite_full(file->fd, data, size, file->hdr.used_size) < 0) - return -1; - file->hdr.used_size += size; + if (hdr_buf != NULL) { + const void *hdr_data; + size_t hdr_size; + + hdr_data = buffer_get_data(buf, &hdr_size); + if (hdr_size > 0) { + if (pwrite_full(file->fd, hdr_data, hdr_size, + file->hdr.used_size) < 0) + return -1; + file->hdr.used_size += hdr_size; + } } + + if (pwrite_full(file->fd, data, size, file->hdr.used_size) < 0) + return -1; + file->hdr.used_size += size; return 0; } @@ -1009,12 +1014,15 @@ int mail_transaction_log_append(struct mail_index_transaction *t, uint32_t *log_file_seq_r, uoff_t *log_file_offset_r) { + struct mail_transaction_extra_rec_header extra_rec_hdr; struct mail_index_view *view = t->view; struct mail_index *index; struct mail_transaction_log *log; struct mail_transaction_log_file *file; size_t offset; uoff_t append_offset; + buffer_t *hdr_buf; + unsigned int i; int ret; index = mail_index_view_get_index(view); @@ -1070,28 +1078,41 @@ int mail_transaction_log_append(struct mail_index_transaction *t, ret = 0; if (t->appends != NULL) { - ret = log_append_buffer(file, t->appends, + ret = log_append_buffer(file, t->appends, NULL, MAIL_TRANSACTION_APPEND, view->external); } if (t->updates != NULL && ret == 0) { - ret = log_append_buffer(file, t->updates, + ret = log_append_buffer(file, t->updates, NULL, MAIL_TRANSACTION_FLAG_UPDATE, view->external); } if (t->cache_updates != NULL && ret == 0) { - ret = log_append_buffer(file, t->cache_updates, + ret = log_append_buffer(file, t->cache_updates, NULL, MAIL_TRANSACTION_CACHE_UPDATE, view->external); } + + hdr_buf = buffer_create_data(pool_datastack_create(), + &extra_rec_hdr, sizeof(extra_rec_hdr)); + for (i = 0; i < view->index->extra_records_count; i++) { + if (t->extra_rec_updates[i] == NULL || ret != 0) + continue; + + extra_rec_hdr.idx = i; + ret = log_append_buffer(file, t->extra_rec_updates[i], hdr_buf, + MAIL_TRANSACTION_EXTRA_REC_UPDATE, + view->external); + } + if (t->expunges != NULL && ret == 0) { - ret = log_append_buffer(file, t->expunges, + ret = log_append_buffer(file, t->expunges, NULL, MAIL_TRANSACTION_EXPUNGE, view->external); } if (t->hdr_changed && ret == 0) { ret = log_append_buffer(file, log_get_hdr_update_buffer(t), - MAIL_TRANSACTION_HEADER_UPDATE, + NULL, MAIL_TRANSACTION_HEADER_UPDATE, view->external); } diff --git a/src/lib-index/mail-transaction-log.h b/src/lib-index/mail-transaction-log.h index 14ce27a055..cb6745bde9 100644 --- a/src/lib-index/mail-transaction-log.h +++ b/src/lib-index/mail-transaction-log.h @@ -14,22 +14,23 @@ struct mail_transaction_log_header { }; enum mail_transaction_type { - MAIL_TRANSACTION_EXPUNGE = 0x00000001, - MAIL_TRANSACTION_APPEND = 0x00000002, - MAIL_TRANSACTION_FLAG_UPDATE = 0x00000004, - MAIL_TRANSACTION_CACHE_UPDATE = 0x00000008, - MAIL_TRANSACTION_HEADER_UPDATE = 0x00000010, + MAIL_TRANSACTION_EXPUNGE = 0x00000001, + MAIL_TRANSACTION_APPEND = 0x00000002, + MAIL_TRANSACTION_FLAG_UPDATE = 0x00000004, + MAIL_TRANSACTION_CACHE_UPDATE = 0x00000008, + MAIL_TRANSACTION_HEADER_UPDATE = 0x00000010, + MAIL_TRANSACTION_EXTRA_REC_UPDATE = 0x00000020, - MAIL_TRANSACTION_TYPE_MASK = 0x0000ffff, + MAIL_TRANSACTION_TYPE_MASK = 0x0000ffff, /* since we'll expunge mails based on data read from transaction log, try to avoid the possibility of corrupted transaction log expunging messages. this value is ORed to the actual MAIL_TRANSACTION_EXPUNGE flag. if it's not present, assume corrupted log. */ - MAIL_TRANSACTION_EXPUNGE_PROT = 0x0000cd90, + MAIL_TRANSACTION_EXPUNGE_PROT = 0x0000cd90, /* Mailbox synchronization noticed this change. */ - MAIL_TRANSACTION_EXTERNAL = 0x10000000 + MAIL_TRANSACTION_EXTERNAL = 0x10000000 }; struct mail_transaction_header { @@ -60,6 +61,15 @@ struct mail_transaction_header_update { unsigned char data[1]; /* variable size */ }; +struct mail_transaction_extra_rec_header { + uint32_t idx; +}; + +struct mail_transaction_extra_rec_update { + uint32_t uid; + unsigned char data[1]; /* variable size */ +}; + struct mail_transaction_log * mail_transaction_log_open_or_create(struct mail_index *index); void mail_transaction_log_close(struct mail_transaction_log *log); diff --git a/src/lib-index/mail-transaction-util.c b/src/lib-index/mail-transaction-util.c index b5b090ab06..9bf6afd544 100644 --- a/src/lib-index/mail-transaction-util.c +++ b/src/lib-index/mail-transaction-util.c @@ -23,6 +23,7 @@ const struct mail_transaction_type_map mail_transaction_type_map[] = { { MAIL_TRANSACTION_CACHE_UPDATE, 0, sizeof(struct mail_transaction_cache_update) }, { MAIL_TRANSACTION_HEADER_UPDATE, 0, 1 }, /* variable size, use 1 */ + { MAIL_TRANSACTION_EXTRA_REC_UPDATE, 0, 1 }, { 0, 0, 0 } }; @@ -57,6 +58,7 @@ int mail_transaction_map(struct mail_index *index, struct mail_transaction_map_functions *map, void *context) { + int ret = 0; switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { @@ -136,6 +138,28 @@ int mail_transaction_map(struct mail_index *index, } break; } + case MAIL_TRANSACTION_EXTRA_REC_UPDATE: { + const struct mail_transaction_extra_rec_header *ehdr; + const struct mail_transaction_extra_rec_update *rec; + unsigned int i, record_size; + + if (map->extra_rec_update == NULL) + break; + + ehdr = data; + i_assert(ehdr->idx < index->extra_records_count); + record_size = index->extra_record_sizes[ehdr->idx]; + + rec = CONST_PTR_OFFSET(data, sizeof(*hdr)); + for (i = 0; i < hdr->size; ) { + ret = map->extra_rec_update(ehdr, rec, context); + if (ret <= 0) + break; + + rec = CONST_PTR_OFFSET(rec, record_size); + } + break; + } default: i_unreached(); } diff --git a/src/lib-index/mail-transaction-util.h b/src/lib-index/mail-transaction-util.h index b90d55d20b..8bf6dd83be 100644 --- a/src/lib-index/mail-transaction-util.h +++ b/src/lib-index/mail-transaction-util.h @@ -17,6 +17,10 @@ struct mail_transaction_map_functions { void *context); int (*header_update)(const struct mail_transaction_header_update *u, void *context); + int (*extra_rec_update) + (const struct mail_transaction_extra_rec_header *hdr, + const struct mail_transaction_extra_rec_update *u, + void *context); }; const struct mail_transaction_type_map *