mdbox_altmove setting specifies how old files should be moved.
--HG--
branch : HEAD
# Maximum dbox file age until it's rotated. Typically in days. Day begins
# from midnight, so 1d = today, 2d = yesterday, etc. 0 = check disabled.
#mdbox_rotate_interval = 1d
+
+# When purging, move unchanged files to alt storage after this much time.
+#mdbox_altmove = 1w
return ret;
}
-static int dbox_file_open_fd(struct dbox_file *file)
+static int dbox_file_open_fd(struct dbox_file *file, bool try_altpath)
{
const char *path;
bool alt = FALSE;
return -1;
}
- if (file->alt_path == NULL || alt) {
+ if (file->alt_path == NULL || alt || !try_altpath) {
/* not found */
return 0;
}
return 1;
}
-int dbox_file_open(struct dbox_file *file, bool *deleted_r)
+static int dbox_file_open_full(struct dbox_file *file, bool try_altpath,
+ bool *notfound_r)
{
int ret;
- *deleted_r = FALSE;
+ *notfound_r = FALSE;
if (file->input != NULL)
return 1;
if (file->fd == -1) {
T_BEGIN {
- ret = dbox_file_open_fd(file);
+ ret = dbox_file_open_fd(file, try_altpath);
} T_END;
if (ret <= 0) {
if (ret < 0)
return -1;
- *deleted_r = TRUE;
+ *notfound_r = TRUE;
return 1;
}
}
return dbox_file_read_header(file);
}
+int dbox_file_open(struct dbox_file *file, bool *deleted_r)
+{
+ return dbox_file_open_full(file, TRUE, deleted_r);
+}
+
+int dbox_file_open_primary(struct dbox_file *file, bool *notfound_r)
+{
+ return dbox_file_open_full(file, FALSE, notfound_r);
+}
+
int dbox_file_header_write(struct dbox_file *file, struct ostream *output)
{
string_t *hdr;
/* Open the file. Returns 1 if ok, 0 if file header is corrupted, -1 if error.
If file is deleted, deleted_r=TRUE and 1 is returned. */
int dbox_file_open(struct dbox_file *file, bool *deleted_r);
+/* Try to open file only from primary path. */
+int dbox_file_open_primary(struct dbox_file *file, bool *notfound_r);
/* Close the file handle from the file, but don't free it. */
void dbox_file_close(struct dbox_file *file);
ARRAY_TYPE(uint32_t) copied_map_uids;
unsigned int i, count;
uoff_t offset, physical_size, msg_size;
+ enum dbox_map_append_flags append_flags = 0;
int ret;
if ((ret = dbox_file_try_lock(file)) <= 0)
return -1;
}
+ if (dstorage->set->mdbox_altmove > 0 &&
+ st.st_mtime + dstorage->set->mdbox_altmove < ioloop_time &&
+ dstorage->alt_storage_dir != NULL)
+ append_flags |= DBOX_MAP_APPEND_FLAG_ALT;
+
i_array_init(&msgs_arr, 128);
if (dbox_map_get_file_msgs(dstorage->map,
((struct mdbox_file *)file)->file_id,
array_sort(&msgs_arr, mdbox_map_file_msg_offset_cmp);
msgs = array_get(&msgs_arr, &count);
- append_ctx = dbox_map_append_begin(dstorage->map);
+ append_ctx = dbox_map_append_begin(dstorage->map, append_flags);
i_array_init(&copied_map_uids, I_MIN(count, 1));
i_array_init(&expunged_map_uids, I_MIN(count, 1));
offset = file->file_header_size;
static int mdbox_file_create(struct dbox_file *file)
{
+ bool create_parents;
int ret;
+ create_parents = dbox_file_is_in_alt(file);
file->fd = file->storage->v.
- file_create_fd(file, file->primary_path, FALSE);
+ file_create_fd(file, file->cur_path, create_parents);
/* even though we don't need it locked while writing to it, by the
time we rename() it it needs to be locked. so we might as well do
return 0;
}
-struct dbox_file *
-mdbox_file_init(struct mdbox_storage *storage, uint32_t file_id)
+static struct dbox_file *
+mdbox_file_init_full(struct mdbox_storage *storage,
+ uint32_t file_id, bool alt_dir)
{
struct mdbox_file *file;
const char *fname;
t_strdup_printf(MDBOX_MAIL_FILE_FORMAT, file_id);
mdbox_file_init_paths(file, fname);
dbox_file_init(&file->file);
+ if (alt_dir)
+ file->file.cur_path = file->file.alt_path;
if (file_id != 0)
array_append(&storage->open_files, &file, 1);
return &file->file;
}
+struct dbox_file *
+mdbox_file_init(struct mdbox_storage *storage, uint32_t file_id)
+{
+ return mdbox_file_init_full(storage, file_id, FALSE);
+}
+
+struct dbox_file *
+mdbox_file_init_new_alt(struct mdbox_storage *storage)
+{
+ return mdbox_file_init_full(storage, 0, TRUE);
+}
+
int mdbox_file_assign_file_id(struct mdbox_file *file, uint32_t file_id)
{
const char *old_path;
- const char *new_fname, *new_path;
+ const char *new_dir, *new_fname, *new_path;
i_assert(file->file_id == 0);
i_assert(file_id != 0);
old_path = file->file.cur_path;
new_fname = t_strdup_printf(MDBOX_MAIL_FILE_FORMAT, file_id);
- new_path = t_strdup_printf("%s/%s", file->storage->storage_dir,
- new_fname);
+ new_dir = !dbox_file_is_in_alt(&file->file) ?
+ file->storage->storage_dir : file->storage->alt_storage_dir;
+ new_path = t_strdup_printf("%s/%s", new_dir, new_fname);
if (rename(old_path, new_path) < 0) {
mail_storage_set_critical(&file->storage->storage.storage,
"rename(%s, %s) failed: %m",
struct dbox_file *
mdbox_file_init(struct mdbox_storage *storage, uint32_t file_id);
+struct dbox_file *
+mdbox_file_init_new_alt(struct mdbox_storage *storage);
/* Assign file ID for a newly created file. */
int mdbox_file_assign_file_id(struct mdbox_file *file, uint32_t file_id);
uint32_t created_uid_validity;
uint32_t map_ext_id, ref_ext_id;
- ARRAY_TYPE(seq_range) ref0_file_ids;
mode_t create_mode, create_dir_mode;
gid_t create_gid;
struct dbox_map_append_context {
struct dbox_map *map;
+ enum dbox_map_append_flags flags;
struct mail_index_sync_ctx *sync_ctx;
struct mail_index_view *sync_view;
ARRAY_DEFINE(appends, struct dbox_map_append);
uint32_t first_new_file_id;
- uint32_t orig_next_uid;
unsigned int files_nonappendable_count;
#include "mdbox-file.h"
#include "mdbox-map-private.h"
+#include <stdlib.h>
+#include <dirent.h>
+
#define MAX_BACKWARDS_LOOKUPS 10
#define DBOX_FORCE_PURGE_MIN_BYTES (1024*1024*10)
*_map = NULL;
- if (array_is_created(&map->ref0_file_ids))
- array_free(&map->ref0_file_ids);
if (map->view != NULL)
mail_index_view_close(&map->view);
mail_index_free(&map->index);
return 0;
}
-struct dbox_file_size {
- uoff_t file_size;
- uoff_t ref0_size;
-};
-
-const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map)
+int dbox_map_get_zero_ref_files(struct dbox_map *map,
+ ARRAY_TYPE(seq_range) *file_ids_r)
{
const struct mail_index_header *hdr;
const struct dbox_map_mail_index_record *rec;
uint32_t seq;
bool expunged;
- if (array_is_created(&map->ref0_file_ids))
- array_clear(&map->ref0_file_ids);
- else
- i_array_init(&map->ref0_file_ids, 64);
-
if (dbox_map_open(map, FALSE) < 0) {
/* some internal error */
- return &map->ref0_file_ids;
+ return -1;
}
- (void)dbox_map_refresh(map);
+ if (dbox_map_refresh(map) < 0)
+ return -1;
hdr = mail_index_get_header(map->view);
for (seq = 1; seq <= hdr->messages_count; seq++) {
&data, &expunged);
if (data != NULL && !expunged) {
rec = data;
- seq_range_array_add(&map->ref0_file_ids, 0,
- rec->file_id);
+ seq_range_array_add(file_ids_r, 0, rec->file_id);
}
}
- return &map->ref0_file_ids;
+ return 0;
}
struct dbox_map_transaction_context *
}
struct dbox_map_append_context *
-dbox_map_append_begin(struct dbox_map *map)
+dbox_map_append_begin(struct dbox_map *map, enum dbox_map_append_flags flags)
{
struct dbox_map_append_context *ctx;
ctx = i_new(struct dbox_map_append_context, 1);
ctx->map = map;
+ ctx->flags = flags;
ctx->first_new_file_id = (uint32_t)-1;
i_array_init(&ctx->file_appends, 64);
i_array_init(&ctx->files, 64);
return stamp - (interval - unit);
}
+static bool dbox_try_open(struct dbox_map_append_context *ctx,
+ struct dbox_file *file)
+{
+ bool want_altpath, notfound;
+
+ want_altpath = (ctx->flags & DBOX_MAP_APPEND_FLAG_ALT) != 0;
+ if (want_altpath) {
+ if (dbox_file_open(file, ¬found) <= 0)
+ return FALSE;
+ } else {
+ if (dbox_file_open_primary(file, ¬found) <= 0)
+ return FALSE;
+ }
+ if (notfound)
+ return FALSE;
+
+ if (file->lock != NULL) {
+ /* already locked, we're possibly in the middle of purging it
+ in which case we really don't want to write there. */
+ return FALSE;
+ }
+ if (dbox_file_is_in_alt(file) != want_altpath) {
+ /* different alt location than what we want, can't use it */
+ return FALSE;
+ }
+ return TRUE;
+}
+
static bool
dbox_map_file_try_append(struct dbox_map_append_context *ctx,
uint32_t file_id, time_t stamp, uoff_t mail_size,
struct dbox_file *file;
struct dbox_file_append_context *file_append;
struct stat st;
- bool deleted, file_too_old = FALSE;
+ bool file_too_old = FALSE;
int ret;
*file_append_r = NULL;
*retry_later_r = FALSE;
file = mdbox_file_init(storage, file_id);
- if (dbox_file_open(file, &deleted) <= 0 || deleted) {
- dbox_file_unref(&file);
- return TRUE;
- }
- if (file->lock != NULL) {
- /* already locked, we're possibly in the middle of purging it
- in which case we really don't want to write there. */
+ if (!dbox_try_open(ctx, file)) {
dbox_file_unref(&file);
return TRUE;
}
return FALSE;
}
-static int
-dbox_map_find_appendable_file(struct dbox_map_append_context *ctx,
- uoff_t mail_size,
- struct dbox_file_append_context **file_append_r,
- struct ostream **output_r, bool *existing_r)
+static struct dbox_file_append_context *
+dbox_map_find_existing_append(struct dbox_map_append_context *ctx,
+ uoff_t mail_size, struct ostream **output_r)
{
struct dbox_map *map = ctx->map;
- ARRAY_TYPE(seq_range) checked_file_ids;
struct dbox_file_append_context *const *file_appends;
- const struct mail_index_header *hdr;
- unsigned int i, count, backwards_lookup_count;
- uint32_t seq, seq1, uid, file_id;
- uoff_t offset, append_offset, size;
- time_t stamp;
- bool retry_later;
-
- *existing_r = FALSE;
+ unsigned int i, count;
+ uoff_t append_offset;
if (mail_size >= map->set->mdbox_rotate_size)
- return 0;
+ return NULL;
/* first try to use files already used in this append */
file_appends = array_get(&ctx->file_appends, &count);
for (i = count; i > ctx->files_nonappendable_count; i--) {
append_offset = file_appends[i-1]->output->offset;
if (append_offset + mail_size <= map->set->mdbox_rotate_size &&
- dbox_file_get_append_stream(file_appends[i-1], output_r) > 0) {
- *file_append_r = file_appends[i-1];
- *existing_r = TRUE;
- return 1;
- }
+ dbox_file_get_append_stream(file_appends[i-1], output_r) > 0)
+ return file_appends[i-1];
+
/* can't append to this file anymore. we could close it
otherwise, except that would also lose our lock too early. */
}
ctx->files_nonappendable_count = count;
+ return NULL;
+}
+
+static int
+dbox_map_find_first_alt(struct dbox_map_append_context *ctx,
+ uint32_t *min_file_id_r, uint32_t *seq_r)
+{
+ struct mdbox_storage *dstorage = ctx->map->storage;
+ struct mail_storage *storage = &dstorage->storage.storage;
+ DIR *dir;
+ struct dirent *d;
+ const struct mail_index_header *hdr;
+ uint32_t seq, file_id, min_file_id = -1U;
+ uoff_t offset, size;
+ int ret = 0;
+
+ /* we want to quickly find the latest alt file, but we also want to
+ avoid accessing the alt storage as much as possible. so we'll do
+ this by finding the lowest numbered file (n) from primary storage.
+ hopefully one of n-[1..m] is appendable in alt storage. */
+ dir = opendir(dstorage->storage_dir);
+ if (dir == NULL) {
+ mail_storage_set_critical(storage,
+ "opendir(%s) failed: %m", dstorage->storage_dir);
+ return -1;
+ }
+
+ for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) {
+ if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX,
+ strlen(MDBOX_MAIL_FILE_PREFIX)) != 0)
+ continue;
+
+ file_id = strtoul(d->d_name + strlen(MDBOX_MAIL_FILE_PREFIX),
+ NULL, 10);
+ if (min_file_id > file_id)
+ min_file_id = file_id;
+ }
+ if (errno != 0) {
+ mail_storage_set_critical(storage,
+ "readdir(%s) failed: %m", dstorage->storage_dir);
+ ret = -1;
+ }
+ if (closedir(dir) < 0) {
+ mail_storage_set_critical(storage,
+ "closedir(%s) failed: %m", dstorage->storage_dir);
+ ret = -1;
+ }
+ if (ret < 0)
+ return -1;
+
+ /* find the newest message in alt storage from map view */
+ hdr = mail_index_get_header(ctx->map->view);
+ for (seq = hdr->messages_count; seq > 0; seq--) {
+ if (dbox_map_lookup_seq(ctx->map, seq, &file_id,
+ &offset, &size) < 0)
+ return -1;
+
+ if (file_id < min_file_id)
+ break;
+ }
+
+ *min_file_id_r = min_file_id;
+ *seq_r = seq;
+ return 0;
+}
+
+static int
+dbox_map_find_appendable_file(struct dbox_map_append_context *ctx,
+ uoff_t mail_size,
+ struct dbox_file_append_context **file_append_r,
+ struct ostream **output_r)
+{
+ struct dbox_map *map = ctx->map;
+ ARRAY_TYPE(seq_range) checked_file_ids;
+ const struct mail_index_header *hdr;
+ unsigned int backwards_lookup_count;
+ uint32_t seq, seq1, uid, file_id, min_file_id;
+ uoff_t offset, size;
+ time_t stamp;
+ bool retry_later;
+
+ if (mail_size >= map->set->mdbox_rotate_size)
+ return 0;
/* try to find an existing appendable file */
stamp = day_begin_stamp(map->set->mdbox_rotate_interval);
hdr = mail_index_get_header(map->view);
- ctx->orig_next_uid = hdr->next_uid;
backwards_lookup_count = 0;
t_array_init(&checked_file_ids, 16);
- for (seq = hdr->messages_count; seq > 0; seq--) {
+
+ if ((ctx->flags & DBOX_MAP_APPEND_FLAG_ALT) == 0)
+ seq = hdr->messages_count;
+ else {
+ /* we want to save to alt storage. */
+ if (dbox_map_find_first_alt(ctx, &min_file_id, &seq) < 0)
+ return -1;
+ seq_range_array_add_range(&checked_file_ids,
+ min_file_id, (uint32_t)-1);
+ }
+
+ for (; seq > 0; seq--) {
if (dbox_map_lookup_seq(map, seq, &file_id, &offset, &size) < 0)
return -1;
if (ctx->failed)
return -1;
- ret = dbox_map_find_appendable_file(ctx, mail_size, &file_append,
- output_r, &existing);
+ file_append = dbox_map_find_existing_append(ctx, mail_size, output_r);
+ if (file_append != NULL) {
+ ret = 1;
+ existing = TRUE;
+ } else {
+ ret = dbox_map_find_appendable_file(ctx, mail_size,
+ &file_append, output_r);
+ existing = FALSE;
+ }
if (ret > 0)
file = file_append->file;
else if (ret < 0)
return -1;
else {
/* create a new file */
- file = mdbox_file_init(ctx->map->storage, 0);
+ file = (ctx->flags & DBOX_MAP_APPEND_FLAG_ALT) == 0 ?
+ mdbox_file_init(ctx->map->storage, 0) :
+ mdbox_file_init_new_alt(ctx->map->storage);
file_append = dbox_file_append_init(file);
ret = dbox_file_get_append_stream(file_append, output_r);
struct dbox_file_append_context;
struct mdbox_storage;
+enum dbox_map_append_flags {
+ DBOX_MAP_APPEND_FLAG_ALT = 0x01
+};
+
struct dbox_map_mail_index_header {
uint32_t highest_file_id;
};
int dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id);
/* Return all files containing messages with zero refcount. */
-const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map);
+int dbox_map_get_zero_ref_files(struct dbox_map *map,
+ ARRAY_TYPE(seq_range) *file_ids_r);
struct dbox_map_append_context *
-dbox_map_append_begin(struct dbox_map *map);
+dbox_map_append_begin(struct dbox_map *map, enum dbox_map_append_flags flags);
/* Request file for saving a new message with given size (if available). If an
existing file can be used, the record is locked and updated in index.
Returns 0 if ok, -1 if error. */
ctx->ctx.ctx.transaction = t;
ctx->ctx.trans = it->trans;
ctx->mbox = mbox;
- ctx->append_ctx = dbox_map_append_begin(mbox->storage->map);
+ ctx->append_ctx = dbox_map_append_begin(mbox->storage->map, 0);
i_array_init(&ctx->mails, 32);
t->save_ctx = &ctx->ctx.ctx;
return t->save_ctx;
static const struct setting_define mdbox_setting_defines[] = {
DEF(SET_SIZE, mdbox_rotate_size),
DEF(SET_TIME, mdbox_rotate_interval),
+ DEF(SET_TIME, mdbox_altmove),
DEF(SET_UINT, mdbox_max_open_files),
SETTING_DEFINE_LIST_END
static const struct mdbox_settings mdbox_default_settings = {
.mdbox_rotate_size = 2*1024*1024,
.mdbox_rotate_interval = 0,
+ .mdbox_altmove = 3600*24*7,
.mdbox_max_open_files = 64
};
struct mdbox_settings {
uoff_t mdbox_rotate_size;
unsigned int mdbox_rotate_interval;
+ unsigned int mdbox_altmove;
unsigned int mdbox_max_open_files;
};
#include "mdbox-file.h"
#include "mdbox-sync.h"
+#include <stdlib.h>
+#include <dirent.h>
+
#define DBOX_REBUILD_COUNT 3
static int
return index_mailbox_sync_init(box, flags, ret < 0);
}
+static int mdbox_sync_altmove_add_files(struct mdbox_storage *dstorage,
+ ARRAY_TYPE(seq_range) *file_ids)
+{
+ struct mail_storage *storage = &dstorage->storage.storage;
+ DIR *dir;
+ struct dirent *d;
+ struct stat st;
+ time_t altmove_mtime;
+ string_t *path;
+ unsigned int file_id, dir_len;
+ int ret = 0;
+
+ if (dstorage->set->mdbox_altmove == 0 ||
+ dstorage->alt_storage_dir == NULL)
+ return 0;
+
+ altmove_mtime = ioloop_time - dstorage->set->mdbox_altmove;
+
+ /* we want to quickly find the latest alt file, but we also want to
+ avoid accessing the alt storage as much as possible. so we'll do
+ this by finding the lowest numbered file (n) from primary storage.
+ hopefully one of n-[1..m] is appendable in alt storage. */
+ dir = opendir(dstorage->storage_dir);
+ if (dir == NULL) {
+ mail_storage_set_critical(storage,
+ "opendir(%s) failed: %m", dstorage->storage_dir);
+ return -1;
+ }
+
+ path = t_str_new(256);
+ str_append(path, dstorage->storage_dir);
+ str_append_c(path, '/');
+ dir_len = str_len(path);
+
+ for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) {
+ if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX,
+ strlen(MDBOX_MAIL_FILE_PREFIX)) != 0)
+ continue;
+
+ str_truncate(path, dir_len);
+ str_append(path, d->d_name);
+
+ file_id = strtoul(d->d_name + strlen(MDBOX_MAIL_FILE_PREFIX),
+ NULL, 10);
+
+ if (stat(str_c(path), &st) < 0) {
+ mail_storage_set_critical(storage,
+ "stat(%s) failed: %m", str_c(path));
+ } else if (st.st_mtime < altmove_mtime) {
+ seq_range_array_add(file_ids, 0, file_id);
+ }
+ }
+ if (errno != 0) {
+ mail_storage_set_critical(storage,
+ "readdir(%s) failed: %m", dstorage->storage_dir);
+ ret = -1;
+ }
+ if (closedir(dir) < 0) {
+ mail_storage_set_critical(storage,
+ "closedir(%s) failed: %m", dstorage->storage_dir);
+ ret = -1;
+ }
+ return ret;
+}
+
int mdbox_sync_purge(struct mail_storage *_storage)
{
struct mdbox_storage *storage = (struct mdbox_storage *)_storage;
- const ARRAY_TYPE(seq_range) *ref0_file_ids;
+ ARRAY_TYPE(seq_range) ref0_file_ids;
struct dbox_file *file;
struct seq_range_iter iter;
unsigned int i = 0;
bool deleted;
int ret = 0;
- ref0_file_ids = dbox_map_get_zero_ref_files(storage->map);
- seq_range_array_iter_init(&iter, ref0_file_ids); i = 0;
+ i_array_init(&ref0_file_ids, 64);
+ if (dbox_map_get_zero_ref_files(storage->map, &ref0_file_ids) < 0)
+ ret = -1;
+
+ /* add also files that can be altmoved */
+ if (mdbox_sync_altmove_add_files(storage, &ref0_file_ids) < 0)
+ ret = -1;
+
+ seq_range_array_iter_init(&iter, &ref0_file_ids); i = 0;
while (seq_range_array_iter_nth(&iter, i++, &file_id)) T_BEGIN {
file = mdbox_file_init(storage, file_id);
if (dbox_file_open(file, &deleted) > 0 && !deleted) {
}
dbox_file_unref(&file);
} T_END;
+ array_free(&ref0_file_ids);
return ret;
}
projects / thirdparty / dovecot / core.git / commitdiff
