{
va_list args;
+ if (file->single_mbox == NULL)
+ file->storage->sync_rebuild = TRUE;
+
va_start(args, reason);
mail_storage_set_critical(&file->storage->storage,
"Corrupted dbox file %s (around offset=%"PRIuUOFF_T"): %s",
return;
/* don't cache metadata seeks while file isn't being referenced */
- file->metadata_read_offset = 0;
+ file->metadata_read_offset = (uoff_t)-1;
if (file->file_id != 0) {
files = array_get(&file->storage->open_files, &count);
return dbox_file_open(file, deleted_r);
}
-int dbox_file_open_if_needed(struct dbox_file *file)
-{
- const char *path;
- int ret;
-
- if (file->fd != -1)
- return 0;
-
- T_BEGIN {
- ret = dbox_file_open_fd(file);
- } T_END;
- if (ret == 0) {
- path = dbox_file_get_primary_path(file);
- mail_storage_set_critical(&file->storage->storage,
- "open(%s) failed: %m", path);
- }
- return ret <= 0 ? -1 : 0;
-}
-
void dbox_file_close(struct dbox_file *file)
{
dbox_file_unlock(file);
i_assert(o_stream_get_buffer_used_size(file->output) == 0);
if (fstat(file->fd, &st) == 0 &&
(uoff_t)st.st_size != file->output->offset)
- i_panic("dbox file modified while locked");
+ i_fatal("dbox file modified while locked");
o_stream_unref(&file->output);
}
file_unlock(&file->lock);
memcpy(&hdr, data, I_MIN(sizeof(hdr), file->msg_header_size));
if (memcmp(hdr.magic_pre, DBOX_MAGIC_PRE, sizeof(hdr.magic_pre)) != 0) {
/* probably broken offset */
- dbox_file_set_corrupted(file, "bad magic value");
+ dbox_file_set_corrupted(file, "msg header has bad magic value");
return 0;
}
{
uoff_t size;
bool first = *offset == 0;
- bool deleted;
+ bool expunged;
int ret;
+ /* FIXME: see if we can get rid of this function. only rebuild needs it. */
*last_r = FALSE;
- ret = dbox_file_get_mail_stream(file, *offset, &size, NULL,
- &deleted);
+ ret = dbox_file_get_mail_stream(file, *offset, &size, NULL, &expunged);
if (ret <= 0)
return ret;
- if (deleted) {
+ if (expunged) {
*last_r = TRUE;
return 0;
}
}
static int
-dbox_file_seek_append_pos(struct dbox_file *file,
- uoff_t last_msg_offset, uoff_t last_msg_size)
+dbox_file_seek_append_pos(struct dbox_file *file, uoff_t append_offset)
{
struct stat st;
+ i_assert(append_offset != 0);
+
if (file->file_version != DBOX_VERSION ||
file->msg_header_size != sizeof(struct dbox_message_header)) {
/* created by an incompatible version, can't append */
return 0;
}
- if (last_msg_size > 0) {
- /* if last_msg_offset + last_msg_size points to EOF,
- we can safely use it without reading the last message. */
- if (fstat(file->fd, &st) < 0) {
- dbox_file_set_syscall_error(file, "fstat()");
- return -1;
- }
- } else {
- st.st_size = -1;
- }
- if (st.st_size != (off_t)(last_msg_offset + last_msg_size)) {
+ if (fstat(file->fd, &st) < 0) {
+ dbox_file_set_syscall_error(file, "fstat()");
+ return -1;
+ }
+ if ((uoff_t)st.st_size != append_offset) {
/* not end of file? either previous write crashed or map index
got broken. play it safe and resync everything. */
+ dbox_file_set_corrupted(file, "file size (%"PRIuUOFF_T
+ ") not expected (%"PRIuUOFF_T")",
+ st.st_size, append_offset);
return 0;
}
file->output = o_stream_create_fd_file(file->fd, 0, FALSE);
- o_stream_seek(file->output, last_msg_offset + last_msg_size);
+ o_stream_seek(file->output, append_offset);
return 1;
}
int dbox_file_get_append_stream(struct dbox_file *file, uoff_t last_msg_offset,
uoff_t last_msg_size, struct ostream **stream_r)
{
- bool deleted;
int ret;
if (file->fd == -1) {
/* creating a new file */
i_assert(file->output == NULL);
+ i_assert(file->file_id == 0 && file->uid == 0);
- ret = dbox_file_open_or_create(file, &deleted);
- if (ret <= 0)
- return ret;
- if (deleted)
- return 0;
+ if (dbox_file_create(file) < 0)
+ return -1;
if (file->single_mbox == NULL) {
/* creating a new multi-file. even though we don't
return -1;
}
}
- }
-
- if (file->output == NULL) {
+ i_assert(file->output != NULL);
+ } else if (file->output == NULL) {
i_assert(file->lock != NULL || file->single_mbox != NULL);
- ret = dbox_file_seek_append_pos(file, last_msg_offset,
+ ret = dbox_file_seek_append_pos(file, last_msg_offset +
last_msg_size);
if (ret <= 0)
return ret;
return 0;
}
-static uoff_t
-dbox_file_get_metadata_offset(struct dbox_file *file, uoff_t offset,
- uoff_t physical_size)
-{
- if (offset == 0) {
- if (file->maildir_file)
- return 0;
-
- i_assert(file->file_header_size != 0);
- offset = file->file_header_size;
- }
- return offset + sizeof(struct dbox_message_header) + physical_size;
-}
-
static int dbox_file_metadata_skip_header(struct dbox_file *file)
{
struct dbox_metadata_header metadata_hdr;
if (ret <= 0) {
if (file->input->stream_errno == 0) {
/* EOF, broken offset */
+ dbox_file_set_corrupted(file,
+ "Unexpected EOF while reading metadata header");
return 0;
}
dbox_file_set_syscall_error(file, "read()");
if (memcmp(metadata_hdr.magic_post, DBOX_MAGIC_POST,
sizeof(metadata_hdr.magic_post)) != 0) {
/* probably broken offset */
+ dbox_file_set_corrupted(file,
+ "metadata header has bad magic value");
return 0;
}
i_stream_skip(file->input, sizeof(metadata_hdr));
return 1;
}
-int dbox_file_metadata_seek(struct dbox_file *file, uoff_t metadata_offset,
- bool *expunged_r)
+static int
+dbox_file_metadata_read_at(struct dbox_file *file, uoff_t metadata_offset)
{
const char *line;
- uoff_t metadata_data_offset, prev_offset;
- bool deleted;
int ret;
- *expunged_r = FALSE;
-
if (file->metadata_pool != NULL)
p_clear(file->metadata_pool);
else {
file->metadata_pool =
pool_alloconly_create("dbox metadata", 1024);
}
- file->metadata_read_offset = 0;
p_array_init(&file->metadata, file->metadata_pool, 16);
- if (file->metadata_read_offset == metadata_offset)
- return 1;
- i_assert(!file->maildir_file); /* previous check should catch this */
-
- if (file->input == NULL) {
- if ((ret = dbox_file_open(file, &deleted)) <= 0)
- return ret;
- if (deleted) {
- *expunged_r = TRUE;
- return 1;
- }
- }
-
i_stream_seek(file->input, metadata_offset);
if ((ret = dbox_file_metadata_skip_header(file)) <= 0)
return ret;
- metadata_data_offset = file->input->v_offset;
-
- *expunged_r = TRUE;
- for (;;) {
- prev_offset = file->input->v_offset;
- if ((line = i_stream_read_next_line(file->input)) == NULL)
- break;
+ ret = 0;
+ while ((line = i_stream_read_next_line(file->input)) != NULL) {
if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') {
/* end of metadata */
- *expunged_r = FALSE;
+ ret = 1;
break;
}
line = p_strdup(file->metadata_pool, line);
array_append(&file->metadata, &line, 1);
}
- file->metadata_read_offset = metadata_offset;
- return 1;
+ if (ret == 0)
+ dbox_file_set_corrupted(file, "missing end-of-metadata line");
+ return ret;
}
-int dbox_file_metadata_seek_mail_offset(struct dbox_file *file, uoff_t offset,
- bool *expunged_r)
+int dbox_file_metadata_read(struct dbox_file *file, uoff_t offset,
+ bool *expunged_r)
{
uoff_t physical_size, metadata_offset;
int ret;
+ if (file->metadata_read_offset == offset || file->maildir_file)
+ return 1;
+ i_assert(offset != 0);
+
ret = dbox_file_get_mail_stream(file, offset, &physical_size,
NULL, expunged_r);
if (ret <= 0 || *expunged_r)
return ret;
- metadata_offset =
- dbox_file_get_metadata_offset(file, offset, physical_size);
- return dbox_file_metadata_seek(file, metadata_offset, expunged_r);
+ metadata_offset = offset + file->msg_header_size + physical_size;
+ ret = dbox_file_metadata_read_at(file, metadata_offset);
+ if (ret <= 0)
+ return ret;
+
+ file->metadata_read_offset = offset;
+ return 1;
}
const char *dbox_file_metadata_get(struct dbox_file *file,
return NULL;
}
-int dbox_file_metadata_write_to(struct dbox_file *file, struct ostream *output)
-{
- struct dbox_metadata_header metadata_hdr;
- const char *const *metadata;
- unsigned int i, count;
-
- memset(&metadata_hdr, 0, sizeof(metadata_hdr));
- memcpy(metadata_hdr.magic_post, DBOX_MAGIC_POST,
- sizeof(metadata_hdr.magic_post));
- if (o_stream_send(output, &metadata_hdr, sizeof(metadata_hdr)) < 0)
- return -1;
-
- metadata = array_get(&file->metadata, &count);
- for (i = 0; i < count; i++) {
- if (o_stream_send_str(output, metadata[i]) < 0 ||
- o_stream_send(output, "\n", 1) < 0)
- return -1;
- }
-
- if (o_stream_send(output, "\n", 1) < 0)
- return -1;
- return 0;
-}
-
int dbox_file_move(struct dbox_file *file, bool alt_path)
{
struct ostream *output;
int out_fd, ret = 0;
i_assert(file->input != NULL);
+ i_assert(file->lock != NULL);
if (file->alt_path == alt_path)
return 0;
if (stat(file->current_path, &st) < 0 && errno == ENOENT) {
- /* already expunged by another session */
+ /* already expunged/moved by another session */
+ dbox_file_unlock(file);
return 0;
}
ok, 0 if file header is corrupted, -1 if error. If file is deleted,
deleted_r=TRUE and 1 is returned. */
int dbox_file_open_or_create(struct dbox_file *file, bool *deleted_r);
-/* Open the file's fd if it's currently closed. Assumes that the file exists. */
-int dbox_file_open_if_needed(struct dbox_file *file);
/* Close the file handle from the file, but don't free it. */
void dbox_file_close(struct dbox_file *file);
/* Flush writes to dbox file. */
int dbox_file_flush_append(struct dbox_file *file);
-/* Seek to given metadata block. Returns 1 if ok, 0 if file/offset is
+/* Read to given message's metadata. Returns 1 if ok, 0 if file/offset is
corrupted, -1 if I/O error. If message has already been expunged,
expunged_r=TRUE and 1 is returned. */
-int dbox_file_metadata_seek(struct dbox_file *file, uoff_t metadata_offset,
+int dbox_file_metadata_read(struct dbox_file *file, uoff_t offset,
bool *expunged_r);
-/* Like dbox_file_metadata_seek(), but the offset points to beginning of the
- message. The function internally reads the message header to find the
- metadata offset. */
-int dbox_file_metadata_seek_mail_offset(struct dbox_file *file, uoff_t offset,
- bool *expunged_r);
/* Return wanted metadata value, or NULL if not found. */
const char *dbox_file_metadata_get(struct dbox_file *file,
enum dbox_metadata_key key);
-/* Write all metadata to output stream. Returns 0 if ok, -1 if I/O error. */
-int dbox_file_metadata_write_to(struct dbox_file *file, struct ostream *output);
/* Move the file to alt path or back. */
int dbox_file_move(struct dbox_file *file, bool alt_path);
ret = dbox_map_lookup(mbox->storage->map, map_uid,
&file_id, &mail->offset);
if (ret <= 0) {
- // FIXME: ret=0 case - should we resync?
if (ret == 0)
mail_set_expunged(_mail);
return -1;
}
static int
-dbox_mail_metadata_seek(struct dbox_mail *mail, struct dbox_file **file_r)
+dbox_mail_metadata_read(struct dbox_mail *mail, struct dbox_file **file_r)
{
struct mail *_mail = &mail->imail.mail.mail;
uoff_t offset;
bool expunged;
- int ret;
if (dbox_mail_open(mail, &offset, file_r) < 0)
return -1;
- ret = dbox_file_metadata_seek_mail_offset(*file_r, offset, &expunged);
- if (ret <= 0) {
- if (ret < 0)
- return -1;
- /* FIXME */
- mail_storage_set_critical(mail->imail.ibox->storage,
- "Broken metadata in dbox file %s offset %"PRIuUOFF_T
- " (uid=%u)",
- (*file_r)->current_path, offset, _mail->uid);
+ if (dbox_file_metadata_read(*file_r, offset, &expunged) <= 0)
return -1;
- }
if (expunged) {
mail_set_expunged(_mail);
return -1;
if (index_mail_get_received_date(_mail, date_r) == 0)
return 0;
- if (dbox_mail_metadata_seek(mail, &file) < 0)
+ if (dbox_mail_metadata_read(mail, &file) < 0)
return -1;
value = dbox_file_metadata_get(file, DBOX_METADATA_RECEIVED_TIME);
if (index_mail_get_save_date(_mail, date_r) == 0)
return 0;
- if (dbox_mail_metadata_seek(mail, &file) < 0)
+ if (dbox_mail_metadata_read(mail, &file) < 0)
return -1;
value = dbox_file_metadata_get(file, DBOX_METADATA_SAVE_TIME);
if (index_mail_get_cached_virtual_size(&mail->imail, size_r))
return 0;
- if (dbox_mail_metadata_seek(mail, &file) < 0)
+ if (dbox_mail_metadata_read(mail, &file) < 0)
return -1;
value = dbox_file_metadata_get(file, DBOX_METADATA_VIRTUAL_SIZE);
return 0;
}
- if (dbox_mail_metadata_seek(mail, &file) < 0)
+ if (dbox_mail_metadata_read(mail, &file) < 0)
return -1;
value = dbox_file_metadata_get(file, key);
ret = dbox_file_get_mail_stream(mail->open_file, offset,
&size, &input, &expunged);
- if (ret < 0)
- return -1;
- if (ret > 0 && expunged) {
- mail_set_expunged(_mail);
+ if (ret <= 0) {
+ if (ret < 0)
+ return -1;
+ dbox_file_set_corrupted(mail->open_file,
+ "uid=%u points to broken data at offset="
+ "%"PRIuUOFF_T, _mail->uid, offset);
return -1;
}
- if (ret == 0) {
- /* FIXME: broken file/offset */
- if (ret > 0)
- i_stream_unref(&input);
- mail_storage_set_critical(_mail->box->storage,
- "broken pointer to dbox file %s "
- "offset %"PRIuUOFF_T" (uid=%u)",
- mail->open_file->current_path,
- offset, _mail->uid);
+ if (expunged) {
+ mail_set_expunged(_mail);
return -1;
}
data->physical_size = size;
unsigned int failed:1;
};
+void dbox_map_set_corrupted(struct dbox_map *map, const char *format, ...)
+{
+ va_list args;
+
+ va_start(args, format);
+ mail_storage_set_critical(&map->storage->storage,
+ "dbox map %s corrupted: %s",
+ map->index->filepath,
+ t_strdup_vprintf(format, args));
+ va_end(args);
+}
+
struct dbox_map *dbox_map_init(struct dbox_storage *storage)
{
struct dbox_map *map;
i_free(map);
}
-static int dbox_map_open(struct dbox_map *map, bool create)
+static int dbox_map_open(struct dbox_map *map)
{
struct mail_storage *storage = &map->storage->storage;
enum mail_index_open_flags open_flags;
if (map->view != NULL) {
/* already opened */
- return 1;
+ return 0;
}
open_flags = index_storage_get_index_open_flags(storage);
- if (create)
- open_flags |= MAIL_INDEX_OPEN_FLAG_CREATE;
-
- ret = mail_index_open(map->index, open_flags, storage->lock_method);
- if (ret <= 0) {
+ ret = mail_index_open_or_create(map->index, open_flags,
+ storage->lock_method);
+ if (ret < 0) {
mail_storage_set_internal_error(storage);
mail_index_reset_error(map->index);
- return ret;
+ return -1;
}
map->view = mail_index_view_open(map->index);
- return 1;
+ return 0;
}
static int dbox_map_refresh(struct dbox_map *map)
{
const struct dbox_mail_index_map_record *rec;
const void *data;
+ uint32_t uid;
bool expunged;
mail_index_lookup_ext(map->view, seq, map->map_ext_id,
rec = data;
if (rec == NULL || rec->file_id == 0) {
- /* corrupted */
- mail_storage_set_critical(&map->storage->storage,
- "dbox map %s corrupted: file_id=0 for seq=%u",
- map->index->filepath, seq);
+ mail_index_lookup_uid(map->view, seq, &uid);
+ dbox_map_set_corrupted(map, "file_id=0 for map_uid=%u", uid);
return -1;
}
static int
dbox_map_get_seq(struct dbox_map *map, uint32_t map_uid, uint32_t *seq_r)
{
- int ret;
-
- if ((ret = dbox_map_open(map, FALSE)) <= 0) {
- /* map doesn't exist or is broken */
- return ret;
- }
-
if (!mail_index_lookup_seq(map->view, map_uid, seq_r)) {
/* not found - try again after a refresh */
if (dbox_map_refresh(map) < 0)
uoff_t size;
int ret;
+ if (dbox_map_open(map) < 0)
+ return -1;
+
if ((ret = dbox_map_get_seq(map, map_uid, &seq)) <= 0)
return ret;
if (dbox_map_lookup_seq(map, seq, file_id_r, offset_r, &size) < 0)
- return 0;
+ return -1;
return 1;
}
const void *data;
uint32_t seq;
bool expunged;
- int ret;
if (array_is_created(&map->ref0_file_ids))
array_clear(&map->ref0_file_ids);
else
i_array_init(&map->ref0_file_ids, 64);
- if ((ret = dbox_map_open(map, FALSE)) <= 0) {
- /* map doesn't exist or is broken */
+ if (dbox_map_open(map) < 0) {
+ /* some internal error */
return &map->ref0_file_ids;
}
(void)dbox_map_refresh(map);
dbox_map_append_begin_storage(struct dbox_storage *storage)
{
struct dbox_map_append_context *ctx;
- int ret;
ctx = i_new(struct dbox_map_append_context, 1);
ctx->map = storage->map;
i_array_init(&ctx->files, 64);
i_array_init(&ctx->appends, 128);
- if ((ret = dbox_map_open(ctx->map, TRUE)) <= 0) {
- i_assert(ret != 0);
+ if (dbox_map_open(ctx->map) < 0)
ctx->failed = TRUE;
- }
+
/* refresh the map so we can try appending to the latest files */
(void)dbox_map_refresh(ctx->map);
return ctx;
mail_index_get_header_ext(view, map->map_ext_id, &data, &data_size);
if (data_size != sizeof(*hdr)) {
if (data_size != 0) {
- mail_storage_set_critical(&map->storage->storage,
- "dbox map %s corrupted: hdr size=%u",
- map->index->filepath, data_size);
+ dbox_map_set_corrupted(map, "hdr size=%u", data_size);
return -1;
}
/* first file */
struct dbox_map *dbox_map_init(struct dbox_storage *storage);
void dbox_map_deinit(struct dbox_map **map);
+/* Look up file_id and offset for given map UID. Returns 1 if ok, 0 if UID
+ is already expunged, -1 if error. */
int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
uint32_t *file_id_r, uoff_t *offset_r);
void dbox_map_append_commit(struct dbox_map_append_context **ctx);
void dbox_map_append_rollback(struct dbox_map_append_context **ctx);
+void dbox_map_set_corrupted(struct dbox_map *map, const char *format, ...)
+ ATTR_FORMAT(2, 3);
+
#endif
unsigned int rotate_days;
unsigned int max_open_files;
ARRAY_DEFINE(open_files, struct dbox_file *);
+
+ unsigned int sync_rebuild:1;
};
struct dbox_mail_index_record {
if (entry->move_to_alt != file->alt_path) {
/* move the file. if it fails, nothing broke so
don't worry about it. */
- (void)dbox_file_move(file, !file->alt_path);
+ if (dbox_file_try_lock(file) > 0) {
+ (void)dbox_file_move(file, !file->alt_path);
+ dbox_file_unlock(file);
+ }
}
}
return 0;
}
- ret = dbox_file_metadata_seek_mail_offset(file, *offset, &expunged);
+ ret = dbox_file_metadata_read(file, *offset, &expunged);
if (ret <= 0) {
if (ret < 0)
return -1;
maildir_keywords_sync_deinit(&ctx.maildir_sync_keywords);
if (ctx.mk != NULL)
maildir_keywords_deinit(&ctx.mk);
+ if (ret == 0)
+ mbox->storage->sync_rebuild = FALSE;
return ret;
}
ret = dbox_map_lookup(ctx->mbox->storage->map, map_uid,
&lookup_entry.file_id, &offset);
if (ret <= 0) {
- // FIXME: ret=0 case - should we resync?
- return -1;
+ if (ret < 0)
+ return -1;
+ /* mailbox is locked while syncing, so if ret=0 the
+ message got expunged from storage before it was
+ expunged from mailbox. that shouldn't happen. */
+ dbox_map_set_corrupted(ctx->mbox->storage->map,
+ "unexpectedly lost map_uid=%u", map_uid);
+ return 0;
}
}
else
entry->move_from_alt = TRUE;
}
- return 0;
+ return 1;
}
static int dbox_sync_add(struct dbox_sync_context *ctx,
const struct mail_index_sync_rec *sync_rec)
{
uint32_t seq, seq1, seq2;
+ int ret;
if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) {
/* we're interested */
/* we care only about alt flag changes */
if ((sync_rec->add_flags & DBOX_INDEX_FLAG_ALT) == 0 &&
(sync_rec->remove_flags & DBOX_INDEX_FLAG_ALT) == 0)
- return 0;
+ return 1;
} else {
/* not interested */
- return 0;
+ return 1;
}
if (!mail_index_lookup_seq_range(ctx->sync_view,
sync_rec->uid1, sync_rec->uid2,
&seq1, &seq2)) {
/* already expunged everything. nothing to do. */
- return 0;
+ return 1;
}
for (seq = seq1; seq <= seq2; seq++) {
- if (dbox_sync_add_seq(ctx, sync_rec, seq) < 0)
- return -1;
+ if ((ret = dbox_sync_add_seq(ctx, sync_rec, seq)) <= 0)
+ return ret;
}
- return 0;
+ return 1;
}
static int dbox_sync_index(struct dbox_sync_context *ctx)
for (;;) {
if (!mail_index_sync_next(ctx->index_sync_ctx, &sync_rec))
break;
- if (dbox_sync_add(ctx, &sync_rec) < 0) {
- ret = 0;
+ if ((ret = dbox_sync_add(ctx, &sync_rec)) <= 0)
break;
- }
}
if (ret > 0) {
hdr = data;
mbox->highest_maildir_uid = hdr->highest_maildir_uid;
+ if (mbox->storage->sync_rebuild)
+ return -1;
return 0;
}
return ret;
}
- if (rebuild && dbox_refresh_header(mbox) < 0) {
+ if (rebuild && dbox_refresh_header(mbox) == 0) {
/* another process rebuilt it already */
rebuild = FALSE;
}
projects / thirdparty / dovecot / core.git / commitdiff
