From: Marco Bettini Date: Wed, 2 Feb 2022 15:09:37 +0000 (+0100) Subject: fts-squat: Remove the plugin X-Git-Tag: 2.4.0~4393 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=b54256099a08a2c2dc181bf6040568eedc5ed826;p=thirdparty%2Fdovecot%2Fcore.git fts-squat: Remove the plugin --- diff --git a/.gitignore b/.gitignore index 6a92599495..fd6d81d9e3 100644 --- a/.gitignore +++ b/.gitignore @@ -147,7 +147,6 @@ src/lmtp/lmtp src/master/dovecot src/ssl-params/ssl-params src/stats/stats -src/plugins/fts-squat/squat-test src/pop3-login/pop3-login src/pop3/pop3 src/replication/replicator/replicator diff --git a/configure.ac b/configure.ac index a325cf65e1..bea4efb75f 100644 --- a/configure.ac +++ b/configure.ac @@ -742,7 +742,7 @@ dnl dnl ** Full text search dnl -fts=" squat" +fts="" not_fts="" DOVECOT_WANT_SOLR @@ -892,7 +892,6 @@ src/plugins/imap-acl/Makefile src/plugins/fs-compress/Makefile src/plugins/fts/Makefile src/plugins/fts-solr/Makefile -src/plugins/fts-squat/Makefile src/plugins/last-login/Makefile src/plugins/lazy-expunge/Makefile src/plugins/listescape/Makefile diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am index 590d753dc8..1f30f16526 100644 --- a/src/plugins/Makefile.am +++ b/src/plugins/Makefile.am @@ -18,7 +18,6 @@ SUBDIRS = \ acl \ imap-acl \ fts \ - fts-squat \ last-login \ lazy-expunge \ listescape \ diff --git a/src/plugins/fts-squat/Makefile.am b/src/plugins/fts-squat/Makefile.am deleted file mode 100644 index 6c8181cef7..0000000000 --- a/src/plugins/fts-squat/Makefile.am +++ /dev/null @@ -1,47 +0,0 @@ -AM_CPPFLAGS = \ - -I$(top_srcdir)/src/lib \ - -I$(top_srcdir)/src/lib-mail \ - -I$(top_srcdir)/src/lib-index \ - -I$(top_srcdir)/src/lib-storage \ - -I$(top_srcdir)/src/plugins/fts - -NOPLUGIN_LDFLAGS = -lib21_fts_squat_plugin_la_LDFLAGS = -module -avoid-version - -module_LTLIBRARIES = \ - lib21_fts_squat_plugin.la - -if DOVECOT_PLUGIN_DEPS -lib21_fts_squat_plugin_la_LIBADD = \ - ../fts/lib20_fts_plugin.la -endif - -lib21_fts_squat_plugin_la_SOURCES = \ - fts-squat-plugin.c \ - fts-backend-squat.c \ - squat-trie.c \ - squat-uidlist.c - -noinst_HEADERS = \ - fts-squat-plugin.h \ - squat-trie.h \ - squat-trie-private.h \ - squat-uidlist.h - -noinst_PROGRAMS = squat-test - -squat_test_SOURCES = \ - squat-test.c - -common_objects = \ - squat-trie.lo \ - squat-uidlist.lo - -squat_test_LDADD = \ - $(common_objects) \ - $(LIBDOVECOT_STORAGE) \ - $(LIBDOVECOT) -squat_test_DEPENDENCIES = \ - $(common_objects) \ - $(LIBDOVECOT_STORAGE_DEPS) \ - $(LIBDOVECOT_DEPS) diff --git a/src/plugins/fts-squat/fts-backend-squat.c b/src/plugins/fts-squat/fts-backend-squat.c deleted file mode 100644 index e87eaad3aa..0000000000 --- a/src/plugins/fts-squat/fts-backend-squat.c +++ /dev/null @@ -1,494 +0,0 @@ -/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "str.h" -#include "unichar.h" -#include "mail-user.h" -#include "mail-namespace.h" -#include "mail-storage-private.h" -#include "mail-search-build.h" -#include "squat-trie.h" -#include "fts-squat-plugin.h" - - -#define SQUAT_FILE_PREFIX "dovecot.index.search" - -struct squat_fts_backend { - struct fts_backend backend; - - struct mailbox *box; - struct squat_trie *trie; - - unsigned int partial_len, full_len; - bool refresh; -}; - -struct squat_fts_backend_update_context { - struct fts_backend_update_context ctx; - struct squat_trie_build_context *build_ctx; - - enum squat_index_type squat_type; - uint32_t uid; - string_t *hdr; - - bool failed; -}; - -static struct fts_backend *fts_backend_squat_alloc(void) -{ - struct squat_fts_backend *backend; - - backend = i_new(struct squat_fts_backend, 1); - backend->backend = fts_backend_squat; - return &backend->backend; -} - -static int -fts_backend_squat_init(struct fts_backend *_backend, const char **error_r) -{ - struct squat_fts_backend *backend = - (struct squat_fts_backend *)_backend; - const char *const *tmp, *env; - unsigned int len; - - env = mail_user_plugin_getenv(_backend->ns->user, "fts_squat"); - if (env == NULL) - return 0; - - for (tmp = t_strsplit_spaces(env, " "); *tmp != NULL; tmp++) { - if (str_begins(*tmp, "partial=")) { - if (str_to_uint(*tmp + 8, &len) < 0 || len == 0) { - *error_r = t_strdup_printf( - "Invalid partial length: %s", *tmp + 8); - return -1; - } - backend->partial_len = len; - } else if (str_begins(*tmp, "full=")) { - if (str_to_uint(*tmp + 5, &len) < 0 || len == 0) { - *error_r = t_strdup_printf( - "Invalid full length: %s", *tmp + 5); - return -1; - } - backend->full_len = len; - } else { - *error_r = t_strdup_printf("Invalid setting: %s", *tmp); - return -1; - } - } - return 0; -} - -static void -fts_backend_squat_unset_box(struct squat_fts_backend *backend) -{ - if (backend->trie != NULL) - squat_trie_deinit(&backend->trie); - backend->box = NULL; -} - -static void fts_backend_squat_deinit(struct fts_backend *_backend) -{ - struct squat_fts_backend *backend = - (struct squat_fts_backend *)_backend; - - fts_backend_squat_unset_box(backend); - i_free(backend); -} - -static int -fts_backend_squat_set_box(struct squat_fts_backend *backend, - struct mailbox *box) -{ - const struct mailbox_permissions *perm; - struct mail_storage *storage; - struct mailbox_status status; - const char *path; - enum squat_index_flags flags = 0; - int ret; - - if (backend->box == box) - { - if (backend->refresh) { - ret = squat_trie_refresh(backend->trie); - if (ret < 0) - return ret; - backend->refresh = FALSE; - } - return 0; - } - fts_backend_squat_unset_box(backend); - backend->refresh = FALSE; - if (box == NULL) - return 0; - - perm = mailbox_get_permissions(box); - storage = mailbox_get_storage(box); - if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &path) <= 0) - i_unreached(); /* fts already checked this */ - - mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status); - if (storage->set->mmap_disable) - flags |= SQUAT_INDEX_FLAG_MMAP_DISABLE; - if (storage->set->mail_nfs_index) - flags |= SQUAT_INDEX_FLAG_NFS_FLUSH; - if (storage->set->dotlock_use_excl) - flags |= SQUAT_INDEX_FLAG_DOTLOCK_USE_EXCL; - - backend->trie = - squat_trie_init(t_strconcat(path, "/"SQUAT_FILE_PREFIX, NULL), - status.uidvalidity, - storage->set->parsed_lock_method, - flags, perm->file_create_mode, - perm->file_create_gid); - - if (backend->partial_len != 0) - squat_trie_set_partial_len(backend->trie, backend->partial_len); - if (backend->full_len != 0) - squat_trie_set_full_len(backend->trie, backend->full_len); - backend->box = box; - return squat_trie_open(backend->trie); -} - -static int -fts_backend_squat_get_last_uid(struct fts_backend *_backend, - struct mailbox *box, uint32_t *last_uid_r) -{ - struct squat_fts_backend *backend = - (struct squat_fts_backend *)_backend; - - int ret = fts_backend_squat_set_box(backend, box); - if (ret < 0) - return -1; - return squat_trie_get_last_uid(backend->trie, last_uid_r); -} - -static struct fts_backend_update_context * -fts_backend_squat_update_init(struct fts_backend *_backend) -{ - struct squat_fts_backend_update_context *ctx; - - ctx = i_new(struct squat_fts_backend_update_context, 1); - ctx->ctx.backend = _backend; - ctx->hdr = str_new(default_pool, 1024*32); - return &ctx->ctx; -} - -static int get_all_msg_uids(struct mailbox *box, ARRAY_TYPE(seq_range) *uids) -{ - struct mailbox_transaction_context *t; - struct mail_search_context *search_ctx; - struct mail_search_args *search_args; - struct mail *mail; - int ret; - - t = mailbox_transaction_begin(box, 0, __func__); - - search_args = mail_search_build_init(); - mail_search_build_add_all(search_args); - search_ctx = mailbox_search_init(t, search_args, NULL, 0, NULL); - mail_search_args_unref(&search_args); - - while (mailbox_search_next(search_ctx, &mail)) { - /* *2 because even/odd is for body/header */ - seq_range_array_add_range(uids, mail->uid * 2, - mail->uid * 2 + 1); - } - ret = mailbox_search_deinit(&search_ctx); - (void)mailbox_transaction_commit(&t); - return ret; -} - -static int -fts_backend_squat_update_uid_changed(struct squat_fts_backend_update_context *ctx) -{ - int ret = 0; - - if (ctx->uid == 0) - return 0; - - if (squat_trie_build_more(ctx->build_ctx, ctx->uid, - SQUAT_INDEX_TYPE_HEADER, - str_data(ctx->hdr), str_len(ctx->hdr)) < 0) - ret = -1; - str_truncate(ctx->hdr, 0); - return ret; -} - -static int -fts_backend_squat_build_deinit(struct squat_fts_backend_update_context *ctx) -{ - struct squat_fts_backend *backend = - (struct squat_fts_backend *)ctx->ctx.backend; - ARRAY_TYPE(seq_range) uids; - int ret = 0; - - if (ctx->build_ctx == NULL) - return 0; - - if (fts_backend_squat_update_uid_changed(ctx) < 0) - ret = -1; - - i_array_init(&uids, 1024); - if (get_all_msg_uids(backend->box, &uids) < 0) { - (void)squat_trie_build_deinit(&ctx->build_ctx, NULL); - ret = -1; - } else { - seq_range_array_invert(&uids, 2, (uint32_t)-2); - if (squat_trie_build_deinit(&ctx->build_ctx, &uids) < 0) - ret = -1; - } - array_free(&uids); - return ret; -} - -static int -fts_backend_squat_update_deinit(struct fts_backend_update_context *_ctx) -{ - struct squat_fts_backend_update_context *ctx = - (struct squat_fts_backend_update_context *)_ctx; - int ret = ctx->failed ? -1 : 0; - - if (fts_backend_squat_build_deinit(ctx) < 0) - ret = -1; - str_free(&ctx->hdr); - i_free(ctx); - return ret; -} - -static void -fts_backend_squat_update_set_mailbox(struct fts_backend_update_context *_ctx, - struct mailbox *box) -{ - struct squat_fts_backend_update_context *ctx = - (struct squat_fts_backend_update_context *)_ctx; - struct squat_fts_backend *backend = - (struct squat_fts_backend *)ctx->ctx.backend; - - if (fts_backend_squat_build_deinit(ctx) < 0) - ctx->failed = TRUE; - if (fts_backend_squat_set_box(backend, box) < 0) - ctx->failed = TRUE; - else if (box != NULL) { - if (squat_trie_build_init(backend->trie, &ctx->build_ctx) < 0) - ctx->failed = TRUE; - } -} - -static void -fts_backend_squat_update_expunge(struct fts_backend_update_context *_ctx ATTR_UNUSED, - uint32_t last_uid ATTR_UNUSED) -{ - /* FIXME */ -} - -static bool -fts_backend_squat_update_set_build_key(struct fts_backend_update_context *_ctx, - const struct fts_backend_build_key *key) -{ - struct squat_fts_backend_update_context *ctx = - (struct squat_fts_backend_update_context *)_ctx; - - if (ctx->failed) - return FALSE; - - if (key->uid != ctx->uid) { - if (fts_backend_squat_update_uid_changed(ctx) < 0) - ctx->failed = TRUE; - } - - switch (key->type) { - case FTS_BACKEND_BUILD_KEY_HDR: - case FTS_BACKEND_BUILD_KEY_MIME_HDR: - str_printfa(ctx->hdr, "%s: ", key->hdr_name); - ctx->squat_type = SQUAT_INDEX_TYPE_HEADER; - break; - case FTS_BACKEND_BUILD_KEY_BODY_PART: - ctx->squat_type = SQUAT_INDEX_TYPE_BODY; - break; - case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY: - i_unreached(); - } - ctx->uid = key->uid; - return TRUE; -} - -static void -fts_backend_squat_update_unset_build_key(struct fts_backend_update_context *_ctx) -{ - struct squat_fts_backend_update_context *ctx = - (struct squat_fts_backend_update_context *)_ctx; - - if (ctx->squat_type == SQUAT_INDEX_TYPE_HEADER) - str_append_c(ctx->hdr, '\n'); -} - -static int -fts_backend_squat_update_build_more(struct fts_backend_update_context *_ctx, - const unsigned char *data, size_t size) -{ - struct squat_fts_backend_update_context *ctx = - (struct squat_fts_backend_update_context *)_ctx; - - if (ctx->squat_type == SQUAT_INDEX_TYPE_HEADER) { - str_append_data(ctx->hdr, data, size); - return 0; - } - return squat_trie_build_more(ctx->build_ctx, ctx->uid, ctx->squat_type, - data, size); -} - -static int fts_backend_squat_refresh(struct fts_backend *_backend) -{ - struct squat_fts_backend *backend = - (struct squat_fts_backend *)_backend; - - backend->refresh = TRUE; - return 0; -} - -static int fts_backend_squat_optimize(struct fts_backend *_backend ATTR_UNUSED) -{ - /* FIXME: drop expunged messages */ - return 0; -} - -static int squat_lookup_arg(struct squat_fts_backend *backend, - const struct mail_search_arg *arg, bool and_args, - ARRAY_TYPE(seq_range) *definite_uids, - ARRAY_TYPE(seq_range) *maybe_uids) -{ - enum squat_index_type squat_type; - ARRAY_TYPE(seq_range) tmp_definite_uids, tmp_maybe_uids; - string_t *dtc; - uint32_t last_uid; - int ret; - - switch (arg->type) { - case SEARCH_TEXT: - squat_type = SQUAT_INDEX_TYPE_HEADER | - SQUAT_INDEX_TYPE_BODY; - break; - case SEARCH_BODY: - squat_type = SQUAT_INDEX_TYPE_BODY; - break; - case SEARCH_HEADER: - case SEARCH_HEADER_ADDRESS: - case SEARCH_HEADER_COMPRESS_LWSP: - squat_type = SQUAT_INDEX_TYPE_HEADER; - break; - default: - return 0; - } - - i_array_init(&tmp_definite_uids, 128); - i_array_init(&tmp_maybe_uids, 128); - - dtc = t_str_new(128); - if (backend->backend.ns->user-> - default_normalizer(arg->value.str, strlen(arg->value.str), dtc) < 0) - i_panic("squat: search key not utf8"); - - ret = squat_trie_lookup(backend->trie, str_c(dtc), squat_type, - &tmp_definite_uids, &tmp_maybe_uids); - if (arg->match_not) { - /* definite -> non-match - maybe -> maybe - non-match -> maybe */ - array_clear(&tmp_maybe_uids); - - if (squat_trie_get_last_uid(backend->trie, &last_uid) < 0) - i_unreached(); - seq_range_array_add_range(&tmp_maybe_uids, 1, last_uid); - seq_range_array_remove_seq_range(&tmp_maybe_uids, - &tmp_definite_uids); - array_clear(&tmp_definite_uids); - } - - if (and_args) { - /* AND: - definite && definite -> definite - definite && maybe -> maybe - maybe && maybe -> maybe */ - - /* put definites among maybies, so they can be intersected */ - seq_range_array_merge(maybe_uids, definite_uids); - seq_range_array_merge(&tmp_maybe_uids, &tmp_definite_uids); - - seq_range_array_intersect(maybe_uids, &tmp_maybe_uids); - seq_range_array_intersect(definite_uids, &tmp_definite_uids); - /* remove duplicate maybies that are also definites */ - seq_range_array_remove_seq_range(maybe_uids, definite_uids); - } else { - /* OR: - definite || definite -> definite - definite || maybe -> definite - maybe || maybe -> maybe */ - - /* remove maybies that are now definites */ - seq_range_array_remove_seq_range(&tmp_maybe_uids, - definite_uids); - seq_range_array_remove_seq_range(maybe_uids, - &tmp_definite_uids); - - seq_range_array_merge(definite_uids, &tmp_definite_uids); - seq_range_array_merge(maybe_uids, &tmp_maybe_uids); - } - - array_free(&tmp_definite_uids); - array_free(&tmp_maybe_uids); - return ret < 0 ? -1 : 1; -} - -static int -fts_backend_squat_lookup(struct fts_backend *_backend, struct mailbox *box, - struct mail_search_arg *args, - enum fts_lookup_flags flags, - struct fts_result *result) -{ - struct squat_fts_backend *backend = - (struct squat_fts_backend *)_backend; - bool and_args = (flags & FTS_LOOKUP_FLAG_AND_ARGS) != 0; - bool first = TRUE; - int ret; - - ret = fts_backend_squat_set_box(backend, box); - if (ret < 0) - return -1; - - for (; args != NULL; args = args->next) { - ret = squat_lookup_arg(backend, args, first ? FALSE : and_args, - &result->definite_uids, - &result->maybe_uids); - if (ret < 0) - return -1; - if (ret > 0) { - args->match_always = TRUE; - first = FALSE; - } - } - return 0; -} - -struct fts_backend fts_backend_squat = { - .name = "squat", - .flags = FTS_BACKEND_FLAG_NORMALIZE_INPUT, - - { - .alloc = fts_backend_squat_alloc, - .init = fts_backend_squat_init, - .deinit = fts_backend_squat_deinit, - .get_last_uid = fts_backend_squat_get_last_uid, - .update_init = fts_backend_squat_update_init, - .update_deinit = fts_backend_squat_update_deinit, - .update_set_mailbox = fts_backend_squat_update_set_mailbox, - .update_expunge = fts_backend_squat_update_expunge, - .update_set_build_key = fts_backend_squat_update_set_build_key, - .update_unset_build_key = fts_backend_squat_update_unset_build_key, - .update_build_more = fts_backend_squat_update_build_more, - .refresh = fts_backend_squat_refresh, - .optimize = fts_backend_squat_optimize, - .can_lookup = fts_backend_default_can_lookup, - .lookup = fts_backend_squat_lookup, - } -}; diff --git a/src/plugins/fts-squat/fts-squat-plugin.c b/src/plugins/fts-squat/fts-squat-plugin.c deleted file mode 100644 index 59d938331d..0000000000 --- a/src/plugins/fts-squat/fts-squat-plugin.c +++ /dev/null @@ -1,18 +0,0 @@ -/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "fts-squat-plugin.h" - -const char *fts_squat_plugin_version = DOVECOT_ABI_VERSION; - -void fts_squat_plugin_init(struct module *module ATTR_UNUSED) -{ - fts_backend_register(&fts_backend_squat); -} - -void fts_squat_plugin_deinit(void) -{ - fts_backend_unregister(fts_backend_squat.name); -} - -const char *fts_squat_plugin_dependencies[] = { "fts", NULL }; diff --git a/src/plugins/fts-squat/fts-squat-plugin.h b/src/plugins/fts-squat/fts-squat-plugin.h deleted file mode 100644 index 0d6bfcbfe6..0000000000 --- a/src/plugins/fts-squat/fts-squat-plugin.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef FTS_SQUAT_PLUGIN_H -#define FTS_SQUAT_PLUGIN_H - -#include "fts-api-private.h" - -struct module; - -extern const char *fts_squat_plugin_dependencies[]; -extern struct fts_backend fts_backend_squat; - -void fts_squat_plugin_init(struct module *module); -void fts_squat_plugin_deinit(void); - -#endif diff --git a/src/plugins/fts-squat/squat-test.c b/src/plugins/fts-squat/squat-test.c deleted file mode 100644 index b55646c9e9..0000000000 --- a/src/plugins/fts-squat/squat-test.c +++ /dev/null @@ -1,197 +0,0 @@ -/* Copyright (c) 2006-2018 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "file-lock.h" -#include "istream.h" -#include "time-util.h" -#include "unichar.h" -#include "squat-trie.h" -#include "squat-uidlist.h" - -#include -#include -#include -#include -#include - -static void result_print(ARRAY_TYPE(seq_range) *result) -{ - const struct seq_range *range; - unsigned int i, count; - - range = array_get(result, &count); - for (i = 0; i < count; i++) { - if (i != 0) - printf(","); - printf("%u", range[i].seq1); - if (range[i].seq1 != range[i].seq2) - printf("-%u", range[i].seq2); - } - printf("\n"); -} - -int main(int argc ATTR_UNUSED, char *argv[]) -{ - const char *trie_path = "/tmp/squat-test-index.search"; - const char *uidlist_path = "/tmp/squat-test-index.search.uids"; - struct squat_trie *trie; - struct squat_trie_build_context *build_ctx; - struct istream *input; - struct stat trie_st, uidlist_st; - ARRAY_TYPE(seq_range) definite_uids, maybe_uids; - char *line, *str, buf[4096]; - buffer_t *valid; - int ret, fd; - unsigned int last = 0, seq = 1, node_count, uidlist_count; - size_t len; - enum squat_index_type index_type; - bool data_header = TRUE, first = TRUE, skip_body = FALSE; - bool mime_header = TRUE; - size_t trie_mem, uidlist_mem; - clock_t clock_start, clock_end; - struct timeval tv_start, tv_end; - double cputime; - - lib_init(); - i_unlink_if_exists(trie_path); - i_unlink_if_exists(uidlist_path); - trie = squat_trie_init(trie_path, time(NULL), - FILE_LOCK_METHOD_FCNTL, 0, 0600, (gid_t)-1); - - clock_start = clock(); - i_gettimeofday(&tv_start); - - fd = open(argv[1], O_RDONLY); - if (fd == -1) - return 1; - - if (squat_trie_build_init(trie, &build_ctx) < 0) - return 1; - - valid = buffer_create_dynamic(default_pool, 4096); - input = i_stream_create_fd(fd, SIZE_MAX); - ret = 0; - while (ret == 0 && (line = i_stream_read_next_line(input)) != NULL) { - if (last != input->v_offset/(1024*100)) { - fprintf(stderr, "\r%ukB", (unsigned)(input->v_offset/1024)); - fflush(stderr); - last = input->v_offset/(1024*100); - } - if (str_begins(line, "From ")) { - if (!first) - seq++; - data_header = TRUE; - skip_body = FALSE; - mime_header = TRUE; - continue; - } - first = FALSE; - - if (str_begins(line, "--")) { - skip_body = FALSE; - mime_header = TRUE; - } - - if (mime_header) { - if (*line == '\0') { - data_header = FALSE; - mime_header = FALSE; - continue; - } - - if (strncasecmp(line, "Content-Type:", 13) == 0 && - strncasecmp(line, "Content-Type: text/", 19) != 0 && - strncasecmp(line, "Content-Type: message/", 22) != 0) - skip_body = TRUE; - else if (strncasecmp(line, "Content-Transfer-Encoding: base64", 33) == 0) - skip_body = TRUE; - } else if (skip_body) - continue; - if (*line == '\0') - continue; - - /* we're actually indexing here headers as bodies and bodies - as headers. it doesn't really matter in this test, and - fixing it would require storing headers temporarily - elsewhere and index them only after the body */ - index_type = !data_header ? SQUAT_INDEX_TYPE_HEADER : - SQUAT_INDEX_TYPE_BODY; - - buffer_set_used_size(valid, 0); - len = strlen(line); - if (uni_utf8_get_valid_data((const unsigned char *)line, - len, valid)) { - ret = squat_trie_build_more(build_ctx, seq, index_type, - (const void *)line, len); - } else if (valid->used > 0) { - ret = squat_trie_build_more(build_ctx, seq, index_type, - valid->data, valid->used); - } - } - buffer_free(&valid); - if (squat_trie_build_deinit(&build_ctx, NULL) < 0) - ret = -1; - if (ret < 0) { - printf("build broken\n"); - return 1; - } - - clock_end = clock(); - i_gettimeofday(&tv_end); - - cputime = (double)(clock_end - clock_start) / CLOCKS_PER_SEC; - fprintf(stderr, "\n - Index time: %.2f CPU seconds, " - "%.2f real seconds (%.02fMB/CPUs)\n", cputime, - timeval_diff_msecs(&tv_end, &tv_start)/1000.0, - input->v_offset / cputime / (1024*1024)); - - if (stat(trie_path, &trie_st) < 0) - i_error("stat(%s) failed: %m", trie_path); - if (stat(uidlist_path, &uidlist_st) < 0) - i_error("stat(%s) failed: %m", uidlist_path); - - trie_mem = squat_trie_mem_used(trie, &node_count); - uidlist_mem = squat_uidlist_mem_used(squat_trie_get_uidlist(trie), - &uidlist_count); - fprintf(stderr, " - memory: %uk for trie, %uk for uidlist\n", - (unsigned)(trie_mem/1024), (unsigned)(uidlist_mem/1024)); - fprintf(stderr, " - %"PRIuUOFF_T" bytes in %u nodes (%.02f%%)\n", - trie_st.st_size, node_count, - trie_st.st_size / (float)input->v_offset * 100.0); - fprintf(stderr, " - %"PRIuUOFF_T" bytes in %u UID lists (%.02f%%)\n", - uidlist_st.st_size, uidlist_count, - uidlist_st.st_size / (float)input->v_offset * 100.0); - fprintf(stderr, " - %"PRIuUOFF_T" bytes total of %" - PRIuUOFF_T" (%.02f%%)\n", - (trie_st.st_size + uidlist_st.st_size), input->v_offset, - (trie_st.st_size + uidlist_st.st_size) / - (float)input->v_offset * 100.0); - - i_stream_unref(&input); - i_close_fd(&fd); - - i_array_init(&definite_uids, 128); - i_array_init(&maybe_uids, 128); - while ((str = fgets(buf, sizeof(buf), stdin)) != NULL) { - ret = strlen(str)-1; - str[ret] = 0; - - i_gettimeofday(&tv_start); - ret = squat_trie_lookup(trie, str, SQUAT_INDEX_TYPE_HEADER | - SQUAT_INDEX_TYPE_BODY, - &definite_uids, &maybe_uids); - if (ret < 0) - printf("error\n"); - else { - i_gettimeofday(&tv_end); - printf(" - Search took %.05f CPU seconds\n", - timeval_diff_usecs(&tv_end, &tv_start)/1000000.0); - printf(" - definite uids: "); - result_print(&definite_uids); - printf(" - maybe uids: "); - result_print(&maybe_uids); - } - } - return 0; -} diff --git a/src/plugins/fts-squat/squat-trie-private.h b/src/plugins/fts-squat/squat-trie-private.h deleted file mode 100644 index b079554916..0000000000 --- a/src/plugins/fts-squat/squat-trie-private.h +++ /dev/null @@ -1,192 +0,0 @@ -#ifndef SQUAT_TRIE_PRIVATE_H -#define SQUAT_TRIE_PRIVATE_H - -#include "file-dotlock.h" -#include "squat-trie.h" - -#define SQUAT_TRIE_VERSION 2 -#define SQUAT_TRIE_LOCK_TIMEOUT 60 -#define SQUAT_TRIE_DOTLOCK_STALE_TIMEOUT (15*60) - -struct squat_file_header { - uint8_t version; - uint8_t unused[3]; - - uint32_t indexid; - uint32_t uidvalidity; - uint32_t used_file_size; - uint32_t deleted_space; - uint32_t node_count; - - uint32_t root_offset; - uint32_t root_unused_uids; - uint32_t root_next_uid; - uint32_t root_uidlist_idx; - - uint8_t partial_len; - uint8_t full_len; - uint8_t normalize_map[256]; -}; - -/* - node file: FIXME: no up-to-date - - struct squat_file_header; - - // children are written before their parents - node[] { - uint8_t child_count; - unsigned char chars[child_count]; - packed neg_diff_to_first_child_offset; // relative to node - packed diff_to_prev_offset[child_count-1]; - packed[child_count] { - // unused_uids_count == uid if have_uid_offset bit is zero - (unused_uids_count << 1) | (have_uid_offset); - [diff_to_prev_uid_offset;] // first one is relative to zero - } - } -*/ - -struct squat_node { - unsigned int child_count:8; - - /* children.leaf_string contains this many bytes */ - unsigned int leaf_string_length:16; - - /* TRUE = children.data contains our children. - FALSE = children.offset contains offset to our children in the - index file. */ - bool children_not_mapped:1; - /* When allocating our children, use a sequential array. */ - bool want_sequential:1; - /* This node's children are in a sequential array, meaning that the - first SEQUENTIAL_COUNT children have chars[n] = n. */ - bool have_sequential:1; - - /* Number of UIDs that exists in parent node but not in this one - (i.e. number of UIDs [0..next_uid-1] not in this node's uidlist). - This is mainly used when adding new UIDs to our children to set - the UID to be relative to this node's UID list. */ - uint32_t unused_uids; - - /* next_uid=0 means there are no UIDs in this node, otherwise - next_uid-1 is the last UID added to this node. */ - uint32_t next_uid; - uint32_t uid_list_idx; - - /* - struct { - unsigned char chars[child_count]; - struct squat_node[child_count]; - } *children; - */ - union { - /* children_not_mapped determines if data or offset should - be used. */ - void *data; - unsigned char *leaf_string; - unsigned char static_leaf_string[sizeof(void *)]; - uint32_t offset; - } children; -}; -/* Return pointer to node.children.chars[] */ -#define NODE_CHILDREN_CHARS(node) \ - ((unsigned char *)(node)->children.data) -/* Return pointer to node.children.node[] */ -#define NODE_CHILDREN_NODES(_node) \ - ((struct squat_node *)(NODE_CHILDREN_CHARS(_node) + \ - MEM_ALIGN((_node)->child_count))) -/* Return number of bytes allocated in node.children.data */ -#define NODE_CHILDREN_ALLOC_SIZE(child_count) \ - (MEM_ALIGN(child_count) + \ - ((child_count) / 8 + 1) * 8 * sizeof(struct squat_node)) -/* Return TRUE if children.leaf_string is set. */ -#define NODE_IS_DYNAMIC_LEAF(node) \ - ((node)->leaf_string_length > \ - sizeof((node)->children.static_leaf_string)) -/* Return node's leaf string. Assumes that it is set. */ -#define NODE_LEAF_STRING(node) \ - (NODE_IS_DYNAMIC_LEAF(node) ? \ - (node)->children.leaf_string : (node)->children.static_leaf_string) -struct squat_trie { - struct squat_node root; - struct squat_uidlist *uidlist; - - struct squat_file_header hdr; - size_t node_alloc_size; - unsigned int unmapped_child_count; - - enum squat_index_flags flags; - enum file_lock_method lock_method; - mode_t create_mode; - gid_t create_gid; - uint32_t uidvalidity; - - char *path; - int fd; - struct file_cache *file_cache; - struct dotlock_settings dotlock_set; - - uoff_t locked_file_size; - const void *data; - size_t data_size; - - void *mmap_base; - size_t mmap_size; - - unsigned char default_normalize_map[256]; - unsigned int default_partial_len; - unsigned int default_full_len; - - bool corrupted:1; -}; - -#define SQUAT_PACK_MAX_SIZE ((sizeof(uint32_t) * 8 + 7) / 7) - -static inline void squat_pack_num(uint8_t **p, uint32_t num) -{ - /* number continues as long as the highest bit is set */ - while (num >= 0x80) { - **p = (num & 0x7f) | 0x80; - *p += 1; - num >>= 7; - } - - **p = num; - *p += 1; -} - -static inline uint32_t squat_unpack_num(const uint8_t **p, const uint8_t *end) -{ - const uint8_t *c = *p; - uint32_t value = 0; - unsigned int bits = 0; - - for (;;) { - if (unlikely(c == end)) { - /* we should never see EOF */ - return 0; - } - - value |= (*c & 0x7f) << bits; - if (*c < 0x80) - break; - - bits += 7; - c++; - } - - if (unlikely(bits >= 32)) { - /* broken input */ - *p = end; - return 0; - } - - *p = c + 1; - return value; -} - -int squat_trie_create_fd(struct squat_trie *trie, const char *path, int flags); -void squat_trie_delete(struct squat_trie *trie); - -#endif diff --git a/src/plugins/fts-squat/squat-trie.c b/src/plugins/fts-squat/squat-trie.c deleted file mode 100644 index d006817364..0000000000 --- a/src/plugins/fts-squat/squat-trie.c +++ /dev/null @@ -1,2096 +0,0 @@ -/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "str.h" -#include "read-full.h" -#include "istream.h" -#include "ostream.h" -#include "unichar.h" -#include "nfs-workarounds.h" -#include "file-cache.h" -#include "seq-range-array.h" -#include "squat-uidlist.h" -#include "squat-trie-private.h" - -#include -#include -#include - -#define DEFAULT_NORMALIZE_MAP_CHARS \ - "EOTIRSACDNLMVUGPHBFWYXKJQZ0123456789@.-+#$%_&" -#define DEFAULT_PARTIAL_LEN 4 -#define DEFAULT_FULL_LEN 4 - -#define MAX_FAST_LEVEL 3 -#define SEQUENTIAL_COUNT 46 - -#define TRIE_BYTES_LEFT(n) \ - ((n) * SQUAT_PACK_MAX_SIZE) -#define TRIE_READAHEAD_SIZE \ - I_MAX(4096, 1 + 256 + TRIE_BYTES_LEFT(256)) - -struct squat_trie_build_context { - struct squat_trie *trie; - struct ostream *output; - struct squat_uidlist_build_context *uidlist_build_ctx; - - struct file_lock *file_lock; - struct dotlock *dotlock; - - uint32_t first_uid; - bool compress_nodes:1; -}; - -struct squat_trie_iterate_node { - struct squat_node *node; - ARRAY_TYPE(seq_range) shifts; - unsigned int idx; -}; - -struct squat_trie_iterate_context { - struct squat_trie *trie; - struct squat_trie_iterate_node cur; - ARRAY(struct squat_trie_iterate_node) parents; - bool failed; -}; - -static int squat_trie_map(struct squat_trie *trie, bool building); - -void squat_trie_delete(struct squat_trie *trie) -{ - i_unlink_if_exists(trie->path); - squat_uidlist_delete(trie->uidlist); -} - -static void squat_trie_set_corrupted(struct squat_trie *trie) -{ - trie->corrupted = TRUE; - i_error("Corrupted file %s", trie->path); - squat_trie_delete(trie); -} - -static void squat_trie_normalize_map_build(struct squat_trie *trie) -{ - static unsigned char valid_chars[] = - DEFAULT_NORMALIZE_MAP_CHARS; - unsigned int i, j; - - memset(trie->default_normalize_map, 0, - sizeof(trie->default_normalize_map)); - -#if 1 - for (i = 0, j = 1; i < sizeof(valid_chars)-1; i++) { - unsigned char chr = valid_chars[i]; - - if (chr >= 'A' && chr <= 'Z') - trie->default_normalize_map[chr-'A'+'a'] = j; - trie->default_normalize_map[chr] = j++; - } - i_assert(j <= SEQUENTIAL_COUNT); - - for (i = 128; i < 256; i++) - trie->default_normalize_map[i] = j++; -#else - for (i = 0; i < sizeof(valid_chars)-1; i++) { - unsigned char chr = valid_chars[i]; - - if (chr >= 'A' && chr <= 'Z') - trie->default_normalize_map[chr-'A'+'a'] = chr; - trie->default_normalize_map[chr] = chr; - } - for (i = 128; i < 256; i++) - trie->default_normalize_map[i] = i_toupper(i); -#endif -} - -static void node_free(struct squat_trie *trie, struct squat_node *node) -{ - struct squat_node *children; - unsigned int i; - - if (node->leaf_string_length > 0) { - if (NODE_IS_DYNAMIC_LEAF(node)) - i_free(node->children.leaf_string); - } else if (!node->children_not_mapped) { - children = NODE_CHILDREN_NODES(node); - - trie->node_alloc_size -= - NODE_CHILDREN_ALLOC_SIZE(node->child_count); - for (i = 0; i < node->child_count; i++) - node_free(trie, &children[i]); - - i_free(node->children.data); - } -} - -struct squat_trie * -squat_trie_init(const char *path, uint32_t uidvalidity, - enum file_lock_method lock_method, enum squat_index_flags flags, - mode_t mode, gid_t gid) -{ - struct squat_trie *trie; - - trie = i_new(struct squat_trie, 1); - trie->path = i_strdup(path); - trie->uidlist = squat_uidlist_init(trie); - trie->fd = -1; - trie->lock_method = lock_method; - trie->uidvalidity = uidvalidity; - trie->flags = flags; - trie->create_mode = mode; - trie->create_gid = gid; - squat_trie_normalize_map_build(trie); - - trie->dotlock_set.use_excl_lock = - (flags & SQUAT_INDEX_FLAG_DOTLOCK_USE_EXCL) != 0; - trie->dotlock_set.nfs_flush = (flags & SQUAT_INDEX_FLAG_NFS_FLUSH) != 0; - trie->dotlock_set.timeout = SQUAT_TRIE_LOCK_TIMEOUT; - trie->dotlock_set.stale_timeout = SQUAT_TRIE_DOTLOCK_STALE_TIMEOUT; - trie->default_partial_len = DEFAULT_PARTIAL_LEN; - trie->default_full_len = DEFAULT_FULL_LEN; - return trie; -} - -static void squat_trie_close_fd(struct squat_trie *trie) -{ - trie->data = NULL; - trie->data_size = 0; - - if (trie->mmap_size != 0) { - if (munmap(trie->mmap_base, trie->mmap_size) < 0) - i_error("munmap(%s) failed: %m", trie->path); - trie->mmap_base = NULL; - trie->mmap_size = 0; - } - i_close_fd_path(&trie->fd, trie->path); -} - -static void squat_trie_close(struct squat_trie *trie) -{ - trie->corrupted = FALSE; - node_free(trie, &trie->root); - i_zero(&trie->root); - i_zero(&trie->hdr); - - squat_trie_close_fd(trie); - if (trie->file_cache != NULL) - file_cache_free(&trie->file_cache); - trie->locked_file_size = 0; -} - -void squat_trie_deinit(struct squat_trie **_trie) -{ - struct squat_trie *trie = *_trie; - - *_trie = NULL; - squat_trie_close(trie); - squat_uidlist_deinit(trie->uidlist); - i_free(trie->path); - i_free(trie); -} - -void squat_trie_set_partial_len(struct squat_trie *trie, unsigned int len) -{ - trie->default_partial_len = len; -} - -void squat_trie_set_full_len(struct squat_trie *trie, unsigned int len) -{ - trie->default_full_len = len; -} - -static void squat_trie_header_init(struct squat_trie *trie) -{ - i_zero(&trie->hdr); - trie->hdr.version = SQUAT_TRIE_VERSION; - trie->hdr.indexid = time(NULL); - trie->hdr.uidvalidity = trie->uidvalidity; - trie->hdr.partial_len = trie->default_partial_len; - trie->hdr.full_len = trie->default_full_len; - - i_assert(sizeof(trie->hdr.normalize_map) == - sizeof(trie->default_normalize_map)); - memcpy(trie->hdr.normalize_map, trie->default_normalize_map, - sizeof(trie->hdr.normalize_map)); -} - -static int squat_trie_open_fd(struct squat_trie *trie) -{ - trie->fd = open(trie->path, O_RDWR); - if (trie->fd == -1) { - if (errno == ENOENT) { - squat_trie_header_init(trie); - return 0; - } - i_error("open(%s) failed: %m", trie->path); - return -1; - } - if (trie->file_cache != NULL) - file_cache_set_fd(trie->file_cache, trie->fd); - return 0; -} - -int squat_trie_open(struct squat_trie *trie) -{ - squat_trie_close(trie); - - if (squat_trie_open_fd(trie) < 0) - return -1; - return squat_trie_map(trie, FALSE); -} - -static int squat_trie_is_file_stale(struct squat_trie *trie) -{ - struct stat st, st2; - - if ((trie->flags & SQUAT_INDEX_FLAG_NFS_FLUSH) != 0) - nfs_flush_file_handle_cache(trie->path); - if (nfs_safe_stat(trie->path, &st) < 0) { - if (errno == ENOENT) - return 1; - - i_error("stat(%s) failed: %m", trie->path); - return -1; - } - if (fstat(trie->fd, &st2) < 0) { - if (errno == ESTALE) - return 1; - i_error("fstat(%s) failed: %m", trie->path); - return -1; - } - trie->locked_file_size = st2.st_size; - - if (st.st_ino == st2.st_ino && CMP_DEV_T(st.st_dev, st2.st_dev)) { - i_assert(trie->locked_file_size >= trie->data_size); - return 0; - } - return 1; -} - -int squat_trie_refresh(struct squat_trie *trie) -{ - int ret; - - ret = squat_trie_is_file_stale(trie); - if (ret > 0) - ret = squat_trie_open(trie); - return ret; -} - -static int squat_trie_lock(struct squat_trie *trie, int lock_type, - struct file_lock **file_lock_r, - struct dotlock **dotlock_r) -{ - const char *error; - int ret; - - i_assert(trie->fd != -1); - - *file_lock_r = NULL; - *dotlock_r = NULL; - - for (;;) { - if (trie->lock_method != FILE_LOCK_METHOD_DOTLOCK) { - struct file_lock_settings lock_set = { - .lock_method = trie->lock_method, - }; - ret = file_wait_lock(trie->fd, trie->path, lock_type, - &lock_set, SQUAT_TRIE_LOCK_TIMEOUT, - file_lock_r, &error); - if (ret < 0) { - i_error("squat trie %s: %s", - trie->path, error); - } - } else { - ret = file_dotlock_create(&trie->dotlock_set, - trie->path, 0, dotlock_r); - } - if (ret == 0) { - i_error("squat trie %s: Locking timed out", trie->path); - return 0; - } - if (ret < 0) - return -1; - - /* if the trie has been compressed, we need to reopen the - file and try to lock again */ - ret = squat_trie_is_file_stale(trie); - if (ret == 0) - break; - - if (*file_lock_r != NULL) - file_unlock(file_lock_r); - else - file_dotlock_delete(dotlock_r); - if (ret < 0) - return -1; - - squat_trie_close(trie); - if (squat_trie_open_fd(trie) < 0) - return -1; - if (trie->fd == -1) - return 0; - } - - if ((trie->flags & SQUAT_INDEX_FLAG_NFS_FLUSH) != 0) - nfs_flush_read_cache_locked(trie->path, trie->fd); - return 1; -} - -static void -node_make_sequential(struct squat_trie *trie, struct squat_node *node, int level) -{ - const unsigned int alloc_size = - NODE_CHILDREN_ALLOC_SIZE(SEQUENTIAL_COUNT); - struct squat_node *children; - unsigned char *chars; - unsigned int i; - - i_assert(node->child_count == 0); - - trie->node_alloc_size += alloc_size; - - node->want_sequential = FALSE; - node->have_sequential = TRUE; - - node->child_count = SEQUENTIAL_COUNT; - node->children.data = i_malloc(alloc_size); - - chars = NODE_CHILDREN_CHARS(node); - for (i = 0; i < SEQUENTIAL_COUNT; i++) - chars[i] = i; - - if (level < MAX_FAST_LEVEL) { - children = NODE_CHILDREN_NODES(node); - for (i = 0; i < SEQUENTIAL_COUNT; i++) - children[i].want_sequential = TRUE; - } -} - -static unsigned int -node_add_child(struct squat_trie *trie, struct squat_node *node, - unsigned char chr, int level) -{ - unsigned int old_child_count = node->child_count; - struct squat_node *children, *old_children; - unsigned char *chars; - size_t old_size, new_size; - - i_assert(node->leaf_string_length == 0); - - if (node->want_sequential) { - node_make_sequential(trie, node, level); - - if (chr < SEQUENTIAL_COUNT) - return chr; - old_child_count = SEQUENTIAL_COUNT; - } - - node->child_count++; - new_size = NODE_CHILDREN_ALLOC_SIZE(node->child_count); - - if (old_child_count == 0) { - /* first child */ - node->children.data = i_malloc(new_size); - trie->node_alloc_size += new_size; - } else { - old_size = NODE_CHILDREN_ALLOC_SIZE(old_child_count); - if (old_size != new_size) { - trie->node_alloc_size += new_size - old_size; - node->children.data = i_realloc(node->children.data, - old_size, new_size); - } - - children = NODE_CHILDREN_NODES(node); - old_children = (void *)(NODE_CHILDREN_CHARS(node) + - MEM_ALIGN(old_child_count)); - if (children != old_children) { - memmove(children, old_children, - old_child_count * sizeof(struct squat_node)); - } - } - - chars = NODE_CHILDREN_CHARS(node); - i_assert(chars != NULL); - chars[node->child_count - 1] = chr; - return node->child_count - 1; -} - -static int -trie_file_cache_read(struct squat_trie *trie, size_t offset, size_t size) -{ - if (trie->file_cache == NULL) - return 0; - - if (file_cache_read(trie->file_cache, offset, size) < 0) { - i_error("read(%s) failed: %m", trie->path); - return -1; - } - trie->data = file_cache_get_map(trie->file_cache, &trie->data_size); - return 0; -} - -static int -node_read_children(struct squat_trie *trie, struct squat_node *node, int level) -{ - const uint8_t *data, *end; - const unsigned char *child_chars; - struct squat_node *child, *children = NULL; - uoff_t node_offset; - unsigned int i, child_idx, child_count; - uoff_t base_offset; - uint32_t num; - - i_assert(node->children_not_mapped); - i_assert(!node->have_sequential); - i_assert(trie->unmapped_child_count > 0); - i_assert(trie->data_size <= trie->locked_file_size); - - trie->unmapped_child_count--; - node_offset = node->children.offset; - node->children_not_mapped = FALSE; - node->children.data = NULL; - - if (trie_file_cache_read(trie, node_offset, TRIE_READAHEAD_SIZE) < 0) - return -1; - if (unlikely(node_offset >= trie->data_size)) { - squat_trie_set_corrupted(trie); - return -1; - } - - data = CONST_PTR_OFFSET(trie->data, node_offset); - end = CONST_PTR_OFFSET(trie->data, trie->data_size); - child_count = *data++; - if (unlikely(node_offset + child_count >= trie->data_size)) { - squat_trie_set_corrupted(trie); - return -1; - } - - if (child_count == 0) - return 0; - - child_chars = data; - data += child_count; - - /* get child offsets */ - base_offset = node_offset; - for (i = 0; i < child_count; i++) { - /* we always start with !have_sequential, so at i=0 this - check always goes to add the first child */ - if (node->have_sequential && child_chars[i] < SEQUENTIAL_COUNT) - child_idx = child_chars[i]; - else { - child_idx = node_add_child(trie, node, child_chars[i], - level); - children = NODE_CHILDREN_NODES(node); - } - - i_assert(children != NULL); - - child = &children[child_idx]; - - /* 1) child offset */ - num = squat_unpack_num(&data, end); - if (num == 0) { - /* no children */ - } else { - if ((num & 1) != 0) { - base_offset += num >> 1; - } else { - base_offset -= num >> 1; - } - if (base_offset >= trie->locked_file_size) { - squat_trie_set_corrupted(trie); - return -1; - } - trie->unmapped_child_count++; - child->children_not_mapped = TRUE; - child->children.offset = base_offset; - } - - /* 2) uidlist */ - child->uid_list_idx = squat_unpack_num(&data, end); - if (child->uid_list_idx == 0) { - /* we don't write nodes with empty uidlists */ - squat_trie_set_corrupted(trie); - return -1; - } - if (!UIDLIST_IS_SINGLETON(child->uid_list_idx)) { - /* 3) next uid */ - child->next_uid = squat_unpack_num(&data, end) + 1; - } else { - uint32_t idx = child->uid_list_idx; - - child->next_uid = 1 + - squat_uidlist_singleton_last_uid(idx); - } - - /* 4) unused uids + leaf string flag */ - num = squat_unpack_num(&data, end); - child->unused_uids = num >> 1; - if ((num & 1) != 0) { - /* leaf string */ - unsigned int len; - unsigned char *dest; - - /* 5) leaf string length */ - len = child->leaf_string_length = - squat_unpack_num(&data, end) + 1; - if (!NODE_IS_DYNAMIC_LEAF(child)) - dest = child->children.static_leaf_string; - else { - dest = child->children.leaf_string = - i_malloc(len); - } - - if (trie->file_cache != NULL) { - /* the string may be long - - recalculate the end pos */ - size_t offset, size; - - offset = (const char *)data - - (const char *)trie->data; - size = len + TRIE_BYTES_LEFT(child_count - i); - - if (trie_file_cache_read(trie, offset, - size) < 0) - return -1; - data = CONST_PTR_OFFSET(trie->data, offset); - end = CONST_PTR_OFFSET(trie->data, - trie->data_size); - child_chars = CONST_PTR_OFFSET(trie->data, - node_offset + 1); - } - - if ((size_t)(end - data) < len) { - squat_trie_set_corrupted(trie); - return -1; - } - memcpy(dest, data, len); - data += len; - } - } - if (unlikely(data == end)) { - /* we should never get this far */ - squat_trie_set_corrupted(trie); - return -1; - } - return 0; -} - -static void -node_write_children(struct squat_trie_build_context *ctx, - struct squat_node *node, const uoff_t *node_offsets) -{ - struct squat_node *children; - const unsigned char *chars; - uint8_t child_count, buf[SQUAT_PACK_MAX_SIZE * 5], *bufp; - uoff_t base_offset; - unsigned int i; - - chars = NODE_CHILDREN_CHARS(node); - children = NODE_CHILDREN_NODES(node); - - base_offset = ctx->output->offset; - child_count = node->child_count; - o_stream_nsend(ctx->output, &child_count, 1); - o_stream_nsend(ctx->output, chars, child_count); - - for (i = 0; i < child_count; i++) { - bufp = buf; - /* 1) child offset */ - if (node_offsets[i] == 0) - *bufp++ = 0; - else if (node_offsets[i] >= base_offset) { - squat_pack_num(&bufp, - ((node_offsets[i] - base_offset) << 1) | 1); - base_offset = node_offsets[i]; - } else { - squat_pack_num(&bufp, - (base_offset - node_offsets[i]) << 1); - base_offset = node_offsets[i]; - } - - /* 2) uidlist */ - squat_pack_num(&bufp, children[i].uid_list_idx); - if (!UIDLIST_IS_SINGLETON(children[i].uid_list_idx)) { - /* 3) next uid */ - squat_pack_num(&bufp, children[i].next_uid - 1); - } - - if (children[i].leaf_string_length == 0) { - /* 4a) unused uids */ - squat_pack_num(&bufp, children[i].unused_uids << 1); - o_stream_nsend(ctx->output, buf, bufp - buf); - } else { - i_assert(node_offsets[i] == 0); - /* 4b) unused uids + flag */ - squat_pack_num(&bufp, (children[i].unused_uids << 1) | 1); - /* 5) leaf string length */ - squat_pack_num(&bufp, children[i].leaf_string_length - 1); - o_stream_nsend(ctx->output, buf, bufp - buf); - o_stream_nsend(ctx->output, - NODE_LEAF_STRING(&children[i]), - children[i].leaf_string_length); - } - } -} - -static inline void -node_add_uid(struct squat_trie_build_context *ctx, uint32_t uid, - struct squat_node *node) -{ - if (uid < node->next_uid) { - /* duplicate */ - return; - } - node->unused_uids += uid - node->next_uid; - node->next_uid = uid + 1; - - node->uid_list_idx = - squat_uidlist_build_add_uid(ctx->uidlist_build_ctx, - node->uid_list_idx, uid); -} - -static void -node_split_string(struct squat_trie_build_context *ctx, struct squat_node *node) -{ - struct squat_node *child; - unsigned char *str; - unsigned int uid, idx, leafstr_len = node->leaf_string_length; - - i_assert(leafstr_len > 0); - - /* make a copy of the leaf string and convert to normal node by - removing it. */ - str = t_malloc_no0(leafstr_len); - if (!NODE_IS_DYNAMIC_LEAF(node)) - memcpy(str, node->children.static_leaf_string, leafstr_len); - else { - memcpy(str, node->children.leaf_string, leafstr_len); - i_free(node->children.leaf_string); - } - node->leaf_string_length = 0; - - /* create a new child node for the rest of the string */ - idx = node_add_child(ctx->trie, node, str[0], MAX_FAST_LEVEL); - child = NODE_CHILDREN_NODES(node) + idx; - - /* update uidlist to contain all of parent's UIDs */ - child->next_uid = node->next_uid - node->unused_uids; - for (uid = 0; uid < child->next_uid; uid++) { - child->uid_list_idx = - squat_uidlist_build_add_uid(ctx->uidlist_build_ctx, - child->uid_list_idx, uid); - } - - i_assert(!child->have_sequential && child->children.data == NULL); - if (leafstr_len > 1) { - /* make the child a leaf string */ - leafstr_len--; - child->leaf_string_length = leafstr_len; - if (!NODE_IS_DYNAMIC_LEAF(child)) { - memcpy(child->children.static_leaf_string, - str + 1, leafstr_len); - } else { - child->children.leaf_string = i_malloc(leafstr_len); - memcpy(child->children.leaf_string, - str + 1, leafstr_len); - } - } -} - -static bool -node_leaf_string_add_or_split(struct squat_trie_build_context *ctx, - struct squat_node *node, - const unsigned char *data, unsigned int data_len) -{ - const unsigned char *str = NODE_LEAF_STRING(node); - const unsigned int leafstr_len = node->leaf_string_length; - unsigned int i; - - if (data_len != leafstr_len) { - /* different lengths, can't match */ - T_BEGIN { - node_split_string(ctx, node); - } T_END; - return FALSE; - } - - for (i = 0; i < data_len; i++) { - if (data[i] != str[i]) { - /* non-match */ - T_BEGIN { - node_split_string(ctx, node); - } T_END; - return FALSE; - } - } - return TRUE; -} - -static int squat_build_add(struct squat_trie_build_context *ctx, uint32_t uid, - const unsigned char *data, unsigned int size) -{ - struct squat_trie *trie = ctx->trie; - struct squat_node *node = &trie->root; - const unsigned char *end = data + size; - unsigned char *chars; - unsigned int idx; - int level = 0; - - for (;;) { - if (node->children_not_mapped) { - if (unlikely(node_read_children(trie, node, level) < 0)) - return -1; - } - - if (node->leaf_string_length != 0) { - /* the whole string must match or we need to split - the node */ - if (node_leaf_string_add_or_split(ctx, node, data, - end - data)) { - node_add_uid(ctx, uid, node); - return 0; - } - } - - node_add_uid(ctx, uid, node); - - if (unlikely(uid < node->unused_uids)) { - squat_trie_set_corrupted(trie); - return -1; - } - /* child node's UIDs are relative to ours. so for example if - we're adding UID 4 and this node now has [2,4] UIDs, - unused_uids=3 and so the child node will be adding - UID 4-3 = 1. */ - uid -= node->unused_uids; - - if (data == end) - return 0; - level++; - - if (node->have_sequential) { - i_assert(node->child_count >= SEQUENTIAL_COUNT); - if (*data < SEQUENTIAL_COUNT) { - idx = *data; - goto found; - } - idx = SEQUENTIAL_COUNT; - } else { - idx = 0; - } - chars = NODE_CHILDREN_CHARS(node); - for (; idx < node->child_count; idx++) { - if (chars[idx] == *data) - goto found; - } - break; - found: - data++; - node = NODE_CHILDREN_NODES(node) + idx; - } - - /* create new children */ - i_assert(node->leaf_string_length == 0); - - for (;;) { - idx = node_add_child(trie, node, *data, - size - (end - data) + 1); - node = NODE_CHILDREN_NODES(node) + idx; - - node_add_uid(ctx, uid, node); - uid = 0; - - if (++data == end) - break; - - if (!node->have_sequential) { - /* convert the node into a leaf string */ - unsigned int len = end - data; - - i_assert(node->children.data == NULL); - node->leaf_string_length = len; - if (!NODE_IS_DYNAMIC_LEAF(node)) { - memcpy(node->children.static_leaf_string, - data, len); - } else { - node->children.leaf_string = i_malloc(len); - memcpy(node->children.leaf_string, data, len); - } - break; - } - } - return 0; -} - -static int -squat_build_word_bytes(struct squat_trie_build_context *ctx, uint32_t uid, - const unsigned char *data, unsigned int size) -{ - struct squat_trie *trie = ctx->trie; - unsigned int i; - - if (trie->hdr.full_len <= trie->hdr.partial_len) - i = 0; - else { - /* the first word is longer than others */ - if (squat_build_add(ctx, uid, data, - I_MIN(size, trie->hdr.full_len)) < 0) - return -1; - i = 1; - } - - for (; i < size; i++) { - if (squat_build_add(ctx, uid, data + i, - I_MIN(trie->hdr.partial_len, size-i)) < 0) - return -1; - } - return 0; -} - -static int -squat_build_word(struct squat_trie_build_context *ctx, uint32_t uid, - const unsigned char *data, const uint8_t *char_lengths, - unsigned int size) -{ - struct squat_trie *trie = ctx->trie; - unsigned int i, j, bytelen; - - if (char_lengths == NULL) { - /* optimization path: all characters are bytes */ - return squat_build_word_bytes(ctx, uid, data, size); - } - - if (trie->hdr.full_len <= trie->hdr.partial_len) - i = 0; - else { - /* the first word is longer than others */ - bytelen = 0; - for (j = 0; j < trie->hdr.full_len && bytelen < size; j++) - bytelen += char_lengths[bytelen]; - i_assert(bytelen <= size); - - if (squat_build_add(ctx, uid, data, bytelen) < 0) - return -1; - i = char_lengths[0]; - } - - for (; i < size; i += char_lengths[i]) { - bytelen = 0; - for (j = 0; j < trie->hdr.partial_len && i+bytelen < size; j++) - bytelen += char_lengths[i + bytelen]; - i_assert(i + bytelen <= size); - - if (squat_build_add(ctx, uid, data + i, bytelen) < 0) - return -1; - } - return 0; -} - -static unsigned char * -squat_data_normalize(struct squat_trie *trie, const unsigned char *data, - unsigned int size) -{ - static const unsigned char replacement_utf8[] = { 0xef, 0xbf, 0xbd }; - unsigned char *dest; - unsigned int i; - - dest = t_malloc_no0(size); - for (i = 0; i < size; i++) { - if (data[i] == replacement_utf8[0] && i + 2 < size && - data[i+1] == replacement_utf8[1] && - data[i+2] == replacement_utf8[2]) { - /* Don't index replacement character */ - dest[i++] = 0; - dest[i++] = 0; - dest[i] = 0; - } else { - dest[i] = trie->hdr.normalize_map[data[i]]; - } - } - return dest; -} - -static int -squat_trie_build_more_real(struct squat_trie_build_context *ctx, - uint32_t uid, enum squat_index_type type, - const unsigned char *input, unsigned int size) -{ - struct squat_trie *trie = ctx->trie; - const unsigned char *data; - uint8_t *char_lengths; - unsigned int i, start = 0; - bool multibyte_chars = FALSE; - int ret = 0; - - uid = uid * 2 + (type == SQUAT_INDEX_TYPE_HEADER ? 1 : 0); - - char_lengths = t_malloc_no0(size); - data = squat_data_normalize(trie, input, size); - for (i = 0; i < size; i++) { - char_lengths[i] = uni_utf8_char_bytes(input[i]); - if (char_lengths[i] != 1) - multibyte_chars = TRUE; - if (data[i] != '\0') - continue; - - while (start < i && data[start] == '\0') - start++; - if (i != start) { - if (squat_build_word(ctx, uid, data + start, - !multibyte_chars ? NULL : - char_lengths + start, - i - start) < 0) { - ret = -1; - start = i; - break; - } - } - start = i + 1; - } - while (start < i && data[start] == '\0') - start++; - if (i != start) { - if (squat_build_word(ctx, uid, data + start, - !multibyte_chars ? NULL : - char_lengths + start, i - start) < 0) - ret = -1; - } - return ret; -} - -int squat_trie_build_more(struct squat_trie_build_context *ctx, - uint32_t uid, enum squat_index_type type, - const unsigned char *input, unsigned int size) -{ - int ret = 0; - - if (size != 0) T_BEGIN { - ret = squat_trie_build_more_real(ctx, uid, type, input, size); - } T_END; - return ret; -} - -static void -node_drop_unused_children(struct squat_trie *trie, struct squat_node *node) -{ - unsigned char *chars; - struct squat_node *children_src, *children_dest; - unsigned int i, j, orig_child_count = node->child_count; - - chars = NODE_CHILDREN_CHARS(node); - children_src = NODE_CHILDREN_NODES(node); - - /* move chars */ - for (i = j = 0; i < orig_child_count; i++) { - if (children_src[i].next_uid != 0) - chars[j++] = chars[i]; - } - node->child_count = j; - - /* move children. note that children_dest may point to different - location than children_src, although they both point to the - same node. */ - children_dest = NODE_CHILDREN_NODES(node); - for (i = j = 0; i < orig_child_count; i++) { - if (children_src[i].next_uid != 0) - children_dest[j++] = children_src[i]; - else - node_free(trie, &children_src[i]); - } -} - -static int -squat_write_node(struct squat_trie_build_context *ctx, struct squat_node *node, - uoff_t *node_offset_r, int level) -{ - struct squat_trie *trie = ctx->trie; - struct squat_node *children; - unsigned int i; - uoff_t *node_offsets; - uint8_t child_count; - int ret; - - i_assert(node->next_uid != 0); - - if (node->children_not_mapped && ctx->compress_nodes) { - if (node_read_children(trie, node, MAX_FAST_LEVEL) < 0) - return -1; - } - - node->have_sequential = FALSE; - node_drop_unused_children(trie, node); - - child_count = node->child_count; - if (child_count == 0) { - i_assert(!node->children_not_mapped || - node->leaf_string_length == 0); - *node_offset_r = !node->children_not_mapped ? 0 : - node->children.offset; - return 0; - } - i_assert(!node->children_not_mapped); - - trie->hdr.node_count++; - - children = NODE_CHILDREN_NODES(node); - node_offsets = t_new(uoff_t, child_count); - for (i = 0; i < child_count; i++) { - T_BEGIN { - ret = squat_write_node(ctx, &children[i], - &node_offsets[i], level + 1); - } T_END; - if (ret < 0) - return -1; - } - - *node_offset_r = ctx->output->offset; - node_write_children(ctx, node, node_offsets); - return 0; -} - -static int squat_write_nodes(struct squat_trie_build_context *ctx) -{ - struct squat_trie *trie = ctx->trie; - uoff_t node_offset; - int ret; - - if (ctx->trie->root.next_uid == 0) - return 0; - - T_BEGIN { - ret = squat_write_node(ctx, &ctx->trie->root, &node_offset, 0); - } T_END; - if (ret < 0) - return -1; - - trie->hdr.root_offset = node_offset; - trie->hdr.root_unused_uids = trie->root.unused_uids; - trie->hdr.root_next_uid = trie->root.next_uid; - trie->hdr.root_uidlist_idx = trie->root.uid_list_idx; - return 0; -} - -static struct squat_trie_iterate_context * -squat_trie_iterate_init(struct squat_trie *trie) -{ - struct squat_trie_iterate_context *ctx; - - ctx = i_new(struct squat_trie_iterate_context, 1); - ctx->trie = trie; - ctx->cur.node = &trie->root; - i_array_init(&ctx->parents, trie->hdr.partial_len*2); - return ctx; -} - -static int -squat_trie_iterate_deinit(struct squat_trie_iterate_context *ctx) -{ - struct squat_trie_iterate_node *node; - int ret = ctx->failed ? -1 : 0; - - if (array_is_created(&ctx->cur.shifts)) { - array_foreach_modifiable(&ctx->parents, node) - array_free(&node->shifts); - array_free(&ctx->cur.shifts); - } - array_free(&ctx->parents); - i_free(ctx); - return ret; -} - -static struct squat_node * -squat_trie_iterate_first(struct squat_trie_iterate_context *ctx) -{ - if (ctx->cur.node->children_not_mapped) { - if (node_read_children(ctx->trie, ctx->cur.node, 1) < 0) { - ctx->failed = TRUE; - return NULL; - } - } - return ctx->cur.node; -} - -static struct squat_node * -squat_trie_iterate_next(struct squat_trie_iterate_context *ctx, - ARRAY_TYPE(seq_range) *shifts_r) -{ - struct squat_trie_iterate_node *iter_nodes; - struct squat_node *children; - unsigned int count, shift_count = 0; - - while (ctx->cur.idx == ctx->cur.node->child_count || - ctx->cur.node->uid_list_idx == 0) - { - iter_nodes = array_get_modifiable(&ctx->parents, &count); - if (count == 0) - return NULL; - - if (array_is_created(&ctx->cur.shifts)) - array_free(&ctx->cur.shifts); - ctx->cur = iter_nodes[count-1]; - array_delete(&ctx->parents, count-1, 1); - } - - *shifts_r = ctx->cur.shifts; - if (array_is_created(&ctx->cur.shifts)) - shift_count = array_count(&ctx->cur.shifts); - - children = NODE_CHILDREN_NODES(ctx->cur.node); - while (children[ctx->cur.idx++].uid_list_idx == 0) { - if (ctx->cur.idx == ctx->cur.node->child_count) { - /* no more non-empty children in this node */ - return squat_trie_iterate_next(ctx, shifts_r); - } - } - array_push_back(&ctx->parents, &ctx->cur); - ctx->cur.node = &children[ctx->cur.idx-1]; - ctx->cur.idx = 0; - if (shift_count != 0) - i_array_init(&ctx->cur.shifts, shift_count); - else - i_zero(&ctx->cur.shifts); - return squat_trie_iterate_first(ctx); -} - -static void -squat_uidlist_update_expunged_uids(const ARRAY_TYPE(seq_range) *shifts_arr, - ARRAY_TYPE(seq_range) *child_shifts, - ARRAY_TYPE(seq_range) *uids_arr, - struct squat_trie *trie, - struct squat_node *node, bool do_shifts) -{ - const struct seq_range *shifts; - struct seq_range *uids, shift; - unsigned int i, uid_idx, uid_count, shift_count; - uint32_t child_shift_seq1, child_shift_count, seq_high; - unsigned int shift_sum = 0, child_sum = 0; - - if (!array_is_created(shifts_arr)) { - i_assert(node->uid_list_idx != 0 || node->child_count == 0); - return; - } - - /* we'll recalculate this */ - node->unused_uids = 0; - - uids = array_get_modifiable(uids_arr, &uid_count); - shifts = array_get(shifts_arr, &shift_count); - for (i = 0, uid_idx = 0, seq_high = 0;; ) { - /* skip UID ranges until we skip/overlap shifts */ - while (uid_idx < uid_count && - (i == shift_count || - I_MAX(shifts[i].seq1, seq_high) > uids[uid_idx].seq2)) - { - i_assert(uids[uid_idx].seq1 >= shift_sum); - uids[uid_idx].seq1 -= shift_sum; - uids[uid_idx].seq2 -= shift_sum; - child_sum += uids[uid_idx].seq2 - - uids[uid_idx].seq1 + 1; - - if (uid_idx > 0 && - uids[uid_idx-1].seq2 >= uids[uid_idx].seq1 - 1) { - /* we can merge this and the previous range */ - i_assert(uids[uid_idx-1].seq2 == - uids[uid_idx].seq1 - 1); - uids[uid_idx-1].seq2 = uids[uid_idx].seq2; - array_delete(uids_arr, uid_idx, 1); - uids = array_get_modifiable(uids_arr, - &uid_count); - } else { - if (uid_idx == 0) - node->unused_uids += uids[0].seq1; - else { - node->unused_uids += - uids[uid_idx].seq1 - - uids[uid_idx-1].seq2 - 1; - } - uid_idx++; - } - } - if (uid_idx == uid_count) - break; - - shift.seq1 = I_MAX(shifts[i].seq1, seq_high); - shift.seq2 = shifts[i].seq2; - if (shift.seq2 < uids[uid_idx].seq1) { - /* shift is entirely before UID range */ - shift_sum += shift.seq2 - shift.seq1 + 1; - i++; - } else { - /* handle shifts before UID range */ - if (shift.seq1 < uids[uid_idx].seq1) { - shift_sum += uids[uid_idx].seq1 - shift.seq1; - shift.seq1 = uids[uid_idx].seq1; - } - /* update child shifts */ - child_shift_seq1 = child_sum + - shift.seq1 - uids[uid_idx].seq1; - child_shift_count = - I_MIN(shift.seq2, uids[uid_idx].seq2) - - shift.seq1 + 1; - seq_range_array_add_range(child_shifts, - child_shift_seq1, - child_shift_seq1 + - child_shift_count - 1); - child_sum += child_shift_count; - - /* if the shifts continue after the UID range, - treat it in the next loop iteration */ - if (shift.seq2 <= uids[uid_idx].seq2) - i++; - else - seq_high = uids[uid_idx].seq2 + 1; - - /* update UIDs - no matter where within the UID range - the shifts happened, the result is the same: - shift number of UIDs are removed, and the rest - are decreased by shift_sum. - - 123 uids child_shifts - a) s -> 12 1 - b) s -> 12 2 - c) s -> 12 3 - */ - if (uids[uid_idx].seq1 + - child_shift_count > uids[uid_idx].seq2) { - /* removed completely */ - array_delete(uids_arr, uid_idx, 1); - uids = array_get_modifiable(uids_arr, - &uid_count); - } else if (do_shifts) { - /* the next loop iteration fixes the UIDs */ - uids[uid_idx].seq1 += child_shift_count; - } else { - seq_range_array_remove_range(uids_arr, - shift.seq1, - I_MIN(shift.seq2, uids[uid_idx].seq2)); - uids = array_get_modifiable(uids_arr, - &uid_count); - } - shift_sum += child_shift_count; - } - if (!do_shifts) { - /* root node - UIDs are only removed, not shifted */ - shift_sum = 0; - } - } - - if (uid_count == 0) { - /* no UIDs left, delete the node's children and mark it - unused */ - if (!NODE_IS_DYNAMIC_LEAF(node)) - node_free(trie, node); - - node->child_count = 0; - node->have_sequential = FALSE; - node->next_uid = 0; - } else { - if (do_shifts) - node->next_uid = uids[uid_count-1].seq2 + 1; - else { - node->unused_uids += (node->next_uid - 1) - - uids[uid_count-1].seq2; - } - } -} - -static int -squat_trie_expunge_uidlists(struct squat_trie_build_context *ctx, - struct squat_uidlist_rebuild_context *rebuild_ctx, - struct squat_trie_iterate_context *iter, - const ARRAY_TYPE(seq_range) *expunged_uids) -{ - struct squat_node *node; - ARRAY_TYPE(seq_range) uid_range, root_shifts, shifts; - bool shift = FALSE; - int ret = 0; - - node = squat_trie_iterate_first(iter); - if (node->uid_list_idx == 0) - return 0; - - i_array_init(&uid_range, 1024); - i_array_init(&root_shifts, array_count(expunged_uids)); - array_append_array(&root_shifts, expunged_uids); - - if (array_count(expunged_uids) > 0) - i_array_init(&iter->cur.shifts, array_count(expunged_uids)); - - shifts = root_shifts; - do { - i_assert(node->uid_list_idx != 0); - array_clear(&uid_range); - if (squat_uidlist_get_seqrange(ctx->trie->uidlist, - node->uid_list_idx, - &uid_range) < 0) { - ret = -1; - break; - } - squat_uidlist_update_expunged_uids(&shifts, &iter->cur.shifts, - &uid_range, ctx->trie, node, - shift); - node->uid_list_idx = - squat_uidlist_rebuild_nextu(rebuild_ctx, &uid_range); - i_assert(node->uid_list_idx != 0 || node->next_uid == 0); - - node = squat_trie_iterate_next(iter, &shifts); - shift = TRUE; - } while (node != NULL); - array_free(&uid_range); - array_free(&root_shifts); - return ret; -} - -static int -squat_trie_renumber_uidlists2(struct squat_trie_build_context *ctx, - struct squat_uidlist_rebuild_context *rebuild_ctx, - struct squat_trie_iterate_context *iter) -{ - struct squat_node *node; - ARRAY_TYPE(seq_range) shifts; - ARRAY_TYPE(uint32_t) uids; - int ret = 0; - - node = squat_trie_iterate_first(iter); - if (node->uid_list_idx == 0) - return 0; - - i_array_init(&uids, 1024); - while (node != NULL) { - i_assert(node->uid_list_idx != 0); - if (!UIDLIST_IS_SINGLETON(node->uid_list_idx)) { - /* rebuild the uidlist */ - array_clear(&uids); - if (squat_uidlist_get(ctx->trie->uidlist, - node->uid_list_idx, &uids) < 0) { - ret = -1; - break; - } - node->uid_list_idx = - squat_uidlist_rebuild_next(rebuild_ctx, &uids); - } - node = squat_trie_iterate_next(iter, &shifts); - } - array_free(&uids); - return ret; -} - -static int -squat_trie_renumber_uidlists(struct squat_trie_build_context *ctx, - const ARRAY_TYPE(seq_range) *expunged_uids, - bool compress) -{ - struct squat_trie_iterate_context *iter; - struct squat_uidlist_rebuild_context *rebuild_ctx; - time_t now; - int ret = 0; - - if ((ret = squat_uidlist_rebuild_init(ctx->uidlist_build_ctx, - compress, &rebuild_ctx)) <= 0) - return ret; - - now = time(NULL); - ctx->trie->hdr.indexid = - I_MAX((unsigned int)now, ctx->trie->hdr.indexid + 1); - - iter = squat_trie_iterate_init(ctx->trie); - if (expunged_uids != NULL) { - ret = squat_trie_expunge_uidlists(ctx, rebuild_ctx, iter, - expunged_uids); - } else { - ret = squat_trie_renumber_uidlists2(ctx, rebuild_ctx, iter); - } - if (squat_trie_iterate_deinit(iter) < 0) - ret = -1; - - /* lock the trie before we rename uidlist */ - i_assert(ctx->file_lock == NULL && ctx->dotlock == NULL); - if (squat_trie_lock(ctx->trie, F_WRLCK, - &ctx->file_lock, &ctx->dotlock) <= 0) - ret = -1; - return squat_uidlist_rebuild_finish(rebuild_ctx, ret < 0); -} - -static bool squat_trie_check_header(struct squat_trie *trie) -{ - if (trie->hdr.version != SQUAT_TRIE_VERSION || - trie->hdr.uidvalidity != trie->uidvalidity) - return FALSE; - - if (trie->hdr.partial_len > trie->hdr.full_len) { - i_error("Corrupted %s: partial len > full len", trie->path); - return FALSE; - } - if (trie->hdr.full_len == 0) { - i_error("Corrupted %s: full len=0", trie->path); - return FALSE; - } - return TRUE; -} - -static int squat_trie_map_header(struct squat_trie *trie) -{ - int ret; - - if (trie->locked_file_size == 0) { - /* newly created file */ - squat_trie_header_init(trie); - return 1; - } - i_assert(trie->fd != -1); - - if ((trie->flags & SQUAT_INDEX_FLAG_MMAP_DISABLE) != 0) { - ret = pread_full(trie->fd, &trie->hdr, sizeof(trie->hdr), 0); - if (ret <= 0) { - if (ret < 0) { - i_error("pread(%s) failed: %m", trie->path); - return -1; - } - i_error("Corrupted %s: File too small", trie->path); - return 0; - } - trie->data = NULL; - trie->data_size = 0; - } else { - if (trie->locked_file_size < sizeof(trie->hdr)) { - i_error("Corrupted %s: File too small", trie->path); - return 0; - } - if (trie->mmap_size != 0) { - if (munmap(trie->mmap_base, trie->mmap_size) < 0) - i_error("munmap(%s) failed: %m", trie->path); - } - - trie->mmap_size = trie->locked_file_size; - trie->mmap_base = mmap(NULL, trie->mmap_size, - PROT_READ | PROT_WRITE, - MAP_SHARED, trie->fd, 0); - if (trie->mmap_base == MAP_FAILED) { - trie->data = trie->mmap_base = NULL; - trie->data_size = trie->mmap_size = 0; - i_error("mmap(%s) failed: %m", trie->path); - return -1; - } - memcpy(&trie->hdr, trie->mmap_base, sizeof(trie->hdr)); - trie->data = trie->mmap_base; - trie->data_size = trie->mmap_size; - } - - return squat_trie_check_header(trie) ? 1 : 0; -} - -static int squat_trie_map(struct squat_trie *trie, bool building) -{ - struct file_lock *file_lock = NULL; - struct dotlock *dotlock = NULL; - bool changed; - int ret; - - if (trie->fd != -1) { - if (squat_trie_lock(trie, F_RDLCK, &file_lock, &dotlock) <= 0) - return -1; - if ((trie->flags & SQUAT_INDEX_FLAG_MMAP_DISABLE) != 0 && - trie->file_cache == NULL) - trie->file_cache = file_cache_new_path(trie->fd, trie->path); - } - - ret = squat_trie_map_header(trie); - if (ret == 0) { - if (file_lock != NULL) - file_unlock(&file_lock); - else - file_dotlock_delete(&dotlock); - squat_trie_delete(trie); - squat_trie_close(trie); - squat_trie_header_init(trie); - } - changed = trie->root.children.offset != trie->hdr.root_offset; - - if (changed || trie->hdr.root_offset == 0) { - node_free(trie, &trie->root); - i_zero(&trie->root); - trie->root.want_sequential = TRUE; - trie->root.unused_uids = trie->hdr.root_unused_uids; - trie->root.next_uid = trie->hdr.root_next_uid; - trie->root.uid_list_idx = trie->hdr.root_uidlist_idx; - trie->root.children.offset = trie->hdr.root_offset; - - if (trie->hdr.root_offset == 0) { - trie->unmapped_child_count = 0; - trie->root.children_not_mapped = FALSE; - } else { - trie->unmapped_child_count = 1; - trie->root.children_not_mapped = TRUE; - } - } - - if (ret >= 0 && !building) { - /* do this while we're still locked */ - ret = squat_uidlist_refresh(trie->uidlist); - } - - if (file_lock != NULL) - file_unlock(&file_lock); - if (dotlock != NULL) - file_dotlock_delete(&dotlock); - if (ret < 0) - return -1; - - return trie->hdr.root_offset == 0 || !changed ? 0 : - node_read_children(trie, &trie->root, 1); -} - -int squat_trie_create_fd(struct squat_trie *trie, const char *path, int flags) -{ - mode_t old_mask; - int fd; - - old_mask = umask(0); - fd = open(path, O_RDWR | O_CREAT | flags, trie->create_mode); - umask(old_mask); - if (fd == -1) { - i_error("creat(%s) failed: %m", path); - return -1; - } - if (trie->create_gid != (gid_t)-1) { - if (fchown(fd, (uid_t)-1, trie->create_gid) < 0) { - i_error("fchown(%s, -1, %ld) failed: %m", - path, (long)trie->create_gid); - i_close_fd(&fd); - return -1; - } - } - return fd; -} - -int squat_trie_build_init(struct squat_trie *trie, - struct squat_trie_build_context **ctx_r) -{ - struct squat_trie_build_context *ctx; - struct squat_uidlist_build_context *uidlist_build_ctx; - - if (trie->fd == -1) { - trie->fd = squat_trie_create_fd(trie, trie->path, 0); - if (trie->fd == -1) - return -1; - - if (trie->file_cache != NULL) - file_cache_set_fd(trie->file_cache, trie->fd); - i_assert(trie->locked_file_size == 0); - } - - /* uidlist locks building */ - if (squat_uidlist_build_init(trie->uidlist, &uidlist_build_ctx) < 0) - return -1; - - if (squat_trie_map(trie, TRUE) < 0) { - squat_uidlist_build_deinit(&uidlist_build_ctx); - return -1; - } - - ctx = i_new(struct squat_trie_build_context, 1); - ctx->trie = trie; - ctx->uidlist_build_ctx = uidlist_build_ctx; - ctx->first_uid = trie->root.next_uid; - - *ctx_r = ctx; - return 0; -} - -static int squat_trie_write_lock(struct squat_trie_build_context *ctx) -{ - if (ctx->file_lock != NULL || ctx->dotlock != NULL) - return 0; - - if (squat_trie_lock(ctx->trie, F_WRLCK, - &ctx->file_lock, &ctx->dotlock) <= 0) - return -1; - return 0; -} - -static int squat_trie_write(struct squat_trie_build_context *ctx) -{ - struct squat_trie *trie = ctx->trie; - struct file_lock *file_lock = NULL; - struct ostream *output; - const char *path, *error; - int fd = -1, ret = 0; - - if ((trie->hdr.used_file_size > sizeof(trie->hdr) && - trie->unmapped_child_count < trie->hdr.node_count/4) || 1) { - /* we might as well recreate the file */ - ctx->compress_nodes = TRUE; - - path = t_strconcat(trie->path, ".tmp", NULL); - fd = squat_trie_create_fd(trie, path, O_TRUNC); - if (fd == -1) - return -1; - - if (trie->lock_method != FILE_LOCK_METHOD_DOTLOCK) { - struct file_lock_settings lock_set = { - .lock_method = trie->lock_method, - }; - ret = file_wait_lock(fd, path, F_WRLCK, &lock_set, - SQUAT_TRIE_LOCK_TIMEOUT, - &file_lock, &error); - if (ret <= 0) { - i_error("file_wait_lock(%s) failed: %s", - path, error); - i_close_fd(&fd); - return -1; - } - } - - output = o_stream_create_fd(fd, 0); - o_stream_cork(output); - o_stream_nsend(output, &trie->hdr, sizeof(trie->hdr)); - } else { - /* we need to lock only while header is being written */ - path = trie->path; - ctx->compress_nodes = - trie->hdr.used_file_size == sizeof(trie->hdr); - - if (trie->hdr.used_file_size == 0) { - /* lock before opening the file, in case we reopen it */ - if (squat_trie_write_lock(ctx) < 0) - return -1; - } - output = o_stream_create_fd(trie->fd, 0); - o_stream_cork(output); - - if (trie->hdr.used_file_size != 0) - (void)o_stream_seek(output, trie->hdr.used_file_size); - else - o_stream_nsend(output, &trie->hdr, sizeof(trie->hdr)); - } - - ctx->output = output; - ret = squat_write_nodes(ctx); - ctx->output = NULL; - - /* write 1 byte guard at the end of file, so that we can verify broken - squat_unpack_num() input by checking if data==end */ - o_stream_nsend(output, "", 1); - - if (trie->corrupted) - ret = -1; - if (ret == 0) - ret = squat_trie_write_lock(ctx); - if (ret == 0) { - trie->hdr.used_file_size = output->offset; - (void)o_stream_seek(output, 0); - o_stream_nsend(output, &trie->hdr, sizeof(trie->hdr)); - } - if (o_stream_finish(output) < 0) { - i_error("write(%s) failed: %s", path, - o_stream_get_error(output)); - ret = -1; - } - o_stream_destroy(&output); - - if (fd == -1) { - /* appended to the existing file */ - i_assert(file_lock == NULL); - return ret; - } - - /* recreating the trie file */ - if (ret < 0) { - if (close(fd) < 0) - i_error("close(%s) failed: %m", path); - fd = -1; - } else if (rename(path, trie->path) < 0) { - i_error("rename(%s, %s) failed: %m", path, trie->path); - ret = -1; - } - - if (ret < 0) { - i_unlink_if_exists(path); - file_lock_free(&file_lock); - } else { - squat_trie_close_fd(trie); - trie->fd = fd; - trie->locked_file_size = trie->hdr.used_file_size; - if (trie->file_cache != NULL) - file_cache_set_fd(trie->file_cache, trie->fd); - - file_lock_free(&ctx->file_lock); - ctx->file_lock = file_lock; - } - return ret; -} - -int squat_trie_build_deinit(struct squat_trie_build_context **_ctx, - const ARRAY_TYPE(seq_range) *expunged_uids) -{ - struct squat_trie_build_context *ctx = *_ctx; - bool compress, unlock = TRUE; - int ret; - - *_ctx = NULL; - - compress = (ctx->trie->root.next_uid - ctx->first_uid) > 10; - - /* keep trie locked while header is being written and when files are - being renamed, so that while trie is read locked, uidlist can't - change under. */ - squat_uidlist_build_flush(ctx->uidlist_build_ctx); - ret = squat_trie_renumber_uidlists(ctx, expunged_uids, compress); - if (ret == 0) { - ret = squat_trie_write(ctx); - if (ret < 0) - unlock = FALSE; - } - - if (ret == 0) - ret = squat_uidlist_build_finish(ctx->uidlist_build_ctx); - if (ctx->file_lock != NULL) { - if (unlock) - file_unlock(&ctx->file_lock); - else - file_lock_free(&ctx->file_lock); - } - if (ctx->dotlock != NULL) - file_dotlock_delete(&ctx->dotlock); - squat_uidlist_build_deinit(&ctx->uidlist_build_ctx); - - i_free(ctx); - return ret; -} - -int squat_trie_get_last_uid(struct squat_trie *trie, uint32_t *last_uid_r) -{ - if (trie->fd == -1) { - if (squat_trie_open(trie) < 0) - return -1; - } - - *last_uid_r = I_MAX((trie->root.next_uid+1)/2, 1) - 1; - return 0; -} - -static int -squat_trie_lookup_data(struct squat_trie *trie, const unsigned char *data, - unsigned int size, ARRAY_TYPE(seq_range) *uids) -{ - struct squat_node *node = &trie->root; - unsigned char *chars; - unsigned int idx; - int level = 0; - - array_clear(uids); - - for (;;) { - if (node->children_not_mapped) { - if (node_read_children(trie, node, level) < 0) - return -1; - } - if (node->leaf_string_length != 0) { - unsigned int len = node->leaf_string_length; - const unsigned char *str; - - if (len > sizeof(node->children.static_leaf_string)) - str = node->children.leaf_string; - else - str = node->children.static_leaf_string; - - if (size > len || memcmp(data, str, size) != 0) - return 0; - - /* match */ - break; - } - - if (size == 0) - break; - level++; - - if (node->have_sequential) { - if (*data < SEQUENTIAL_COUNT) { - idx = *data; - goto found; - } - idx = SEQUENTIAL_COUNT; - } else { - idx = 0; - } - chars = NODE_CHILDREN_CHARS(node); - for (; idx < node->child_count; idx++) { - if (chars[idx] == *data) - goto found; - } - return 0; - found: - /* follow to children */ - if (level == 1) { - /* root level, add all UIDs */ - if (squat_uidlist_get_seqrange(trie->uidlist, - node->uid_list_idx, - uids) < 0) - return -1; - } else { - if (squat_uidlist_filter(trie->uidlist, - node->uid_list_idx, uids) < 0) - return -1; - } - data++; - size--; - node = NODE_CHILDREN_NODES(node) + idx; - } - - if (squat_uidlist_filter(trie->uidlist, node->uid_list_idx, uids) < 0) - return -1; - return 1; -} - -static void -squat_trie_filter_type(enum squat_index_type type, - const ARRAY_TYPE(seq_range) *src, - ARRAY_TYPE(seq_range) *dest) -{ - const struct seq_range *src_range; - struct seq_range new_range; - unsigned int i, count, mask; - uint32_t next_seq, uid; - - array_clear(dest); - src_range = array_get(src, &count); - if (count == 0) - return; - - if ((type & SQUAT_INDEX_TYPE_HEADER) != 0 && - (type & SQUAT_INDEX_TYPE_BODY) != 0) { - /* everything is fine, just fix the UIDs */ - new_range.seq1 = src_range[0].seq1 / 2; - new_range.seq2 = src_range[0].seq2 / 2; - for (i = 1; i < count; i++) { - next_seq = src_range[i].seq1 / 2; - if (next_seq == new_range.seq2 + 1) { - /* we can continue the previous range */ - } else { - array_push_back(dest, &new_range); - new_range.seq1 = src_range[i].seq1 / 2; - } - new_range.seq2 = src_range[i].seq2 / 2; - } - array_push_back(dest, &new_range); - return; - } - - /* we'll have to drop either header or body UIDs */ - mask = (type & SQUAT_INDEX_TYPE_HEADER) != 0 ? 1 : 0; - for (i = 0; i < count; i++) { - for (uid = src_range[i].seq1; uid <= src_range[i].seq2; uid++) { - if ((uid & 1) == mask) - seq_range_array_add(dest, uid/2); - } - } -} - -struct squat_trie_lookup_context { - struct squat_trie *trie; - enum squat_index_type type; - - ARRAY_TYPE(seq_range) *definite_uids, *maybe_uids; - ARRAY_TYPE(seq_range) tmp_uids, tmp_uids2; - bool first; -}; - -static int -squat_trie_lookup_partial(struct squat_trie_lookup_context *ctx, - const unsigned char *data, uint8_t *char_lengths, - unsigned int size) -{ - const unsigned int partial_len = ctx->trie->hdr.partial_len; - unsigned int char_idx, max_chars, i, j, bytelen; - int ret; - - for (i = 0, max_chars = 0; i < size; max_chars++) - i += char_lengths[i]; - i_assert(max_chars > 0); - - i = 0; char_idx = 0; - do { - bytelen = 0; - for (j = 0; j < partial_len && i+bytelen < size; j++) - bytelen += char_lengths[i + bytelen]; - - ret = squat_trie_lookup_data(ctx->trie, data + i, bytelen, - &ctx->tmp_uids); - if (ret <= 0) { - array_clear(ctx->maybe_uids); - return ret; - } - - if (ctx->first) { - squat_trie_filter_type(ctx->type, &ctx->tmp_uids, - ctx->maybe_uids); - ctx->first = FALSE; - } else { - squat_trie_filter_type(ctx->type, &ctx->tmp_uids, - &ctx->tmp_uids2); - seq_range_array_intersect(ctx->maybe_uids, - &ctx->tmp_uids2); - } - i += char_lengths[i]; - char_idx++; - } while (max_chars - char_idx >= partial_len); - return 1; -} - -static void squat_trie_add_unknown(struct squat_trie *trie, - ARRAY_TYPE(seq_range) *maybe_uids) -{ - struct seq_range *range, new_range; - unsigned int count; - uint32_t last_uid; - - last_uid = I_MAX((trie->root.next_uid+1)/2, 1) - 1; - - range = array_get_modifiable(maybe_uids, &count); - if (count > 0 && range[count-1].seq2 == last_uid) { - /* increase the range */ - range[count-1].seq2 = (uint32_t)-1; - } else { - new_range.seq1 = last_uid + 1; - new_range.seq2 = (uint32_t)-1; - array_push_back(maybe_uids, &new_range); - } -} - -static int -squat_trie_lookup_real(struct squat_trie *trie, const char *str, - enum squat_index_type type, - ARRAY_TYPE(seq_range) *definite_uids, - ARRAY_TYPE(seq_range) *maybe_uids) -{ - struct squat_trie_lookup_context ctx; - unsigned char *data; - uint8_t *char_lengths; - unsigned int i, start, bytes, str_bytelen, str_charlen; - bool searched = FALSE; - int ret = 0; - - array_clear(definite_uids); - array_clear(maybe_uids); - - i_zero(&ctx); - ctx.trie = trie; - ctx.type = type; - ctx.definite_uids = definite_uids; - ctx.maybe_uids = maybe_uids; - i_array_init(&ctx.tmp_uids, 128); - i_array_init(&ctx.tmp_uids2, 128); - ctx.first = TRUE; - - str_bytelen = strlen(str); - char_lengths = str_bytelen == 0 ? NULL : t_malloc0(str_bytelen); - for (i = 0, str_charlen = 0; i < str_bytelen; str_charlen++) { - bytes = uni_utf8_char_bytes(str[i]); - char_lengths[i] = bytes; - i += bytes; - } - data = squat_data_normalize(trie, (const unsigned char *)str, - str_bytelen); - - for (i = start = 0; i < str_bytelen && ret >= 0; i += char_lengths[i]) { - if (data[i] != '\0') - continue; - - /* string has nonindexed characters. - search it in parts. */ - if (i != start) { - ret = squat_trie_lookup_partial(&ctx, data + start, - char_lengths + start, - i - start); - searched = TRUE; - } - start = i + char_lengths[i]; - } - - if (start == 0) { - if (str_charlen <= trie->hdr.partial_len || - trie->hdr.full_len > trie->hdr.partial_len) { - ret = squat_trie_lookup_data(trie, data, str_bytelen, - &ctx.tmp_uids); - if (ret > 0) { - squat_trie_filter_type(type, &ctx.tmp_uids, - definite_uids); - } - } else { - array_clear(definite_uids); - } - - if (str_charlen <= trie->hdr.partial_len || - trie->hdr.partial_len == 0) { - /* we have the result */ - array_clear(maybe_uids); - } else { - ret = squat_trie_lookup_partial(&ctx, data + start, - char_lengths + start, - i - start); - } - } else if (str_bytelen > 0) { - /* string has nonindexed characters. finish the search. */ - array_clear(definite_uids); - if (i != start && ret >= 0) { - ret = squat_trie_lookup_partial(&ctx, data + start, - char_lengths + start, - i - start); - } else if (!searched) { - /* string has only nonindexed chars, - list all root UIDs as maybes */ - ret = squat_uidlist_get_seqrange(trie->uidlist, - trie->root.uid_list_idx, - &ctx.tmp_uids); - squat_trie_filter_type(type, &ctx.tmp_uids, - maybe_uids); - } - } else { - /* zero string length - list all root UIDs as definite - answers */ -#if 0 /* FIXME: this code is never actually reached now. */ - ret = squat_uidlist_get_seqrange(trie->uidlist, - trie->root.uid_list_idx, - &ctx.tmp_uids); - squat_trie_filter_type(type, &ctx.tmp_uids, - definite_uids); -#else - i_unreached(); -#endif - } - seq_range_array_remove_seq_range(maybe_uids, definite_uids); - squat_trie_add_unknown(trie, maybe_uids); - array_free(&ctx.tmp_uids); - array_free(&ctx.tmp_uids2); - return ret < 0 ? -1 : 0; -} - -int squat_trie_lookup(struct squat_trie *trie, const char *str, - enum squat_index_type type, - ARRAY_TYPE(seq_range) *definite_uids, - ARRAY_TYPE(seq_range) *maybe_uids) -{ - int ret; - - T_BEGIN { - ret = squat_trie_lookup_real(trie, str, type, - definite_uids, maybe_uids); - } T_END; - return ret; -} - -struct squat_uidlist *squat_trie_get_uidlist(struct squat_trie *trie) -{ - return trie->uidlist; -} - -size_t squat_trie_mem_used(struct squat_trie *trie, unsigned int *count_r) -{ - *count_r = trie->hdr.node_count; - return trie->node_alloc_size; -} diff --git a/src/plugins/fts-squat/squat-trie.h b/src/plugins/fts-squat/squat-trie.h deleted file mode 100644 index 91530b8659..0000000000 --- a/src/plugins/fts-squat/squat-trie.h +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef SQUAT_TRIE_H -#define SQUAT_TRIE_H - -#include "file-lock.h" -#include "seq-range-array.h" - -enum squat_index_flags { - SQUAT_INDEX_FLAG_MMAP_DISABLE = 0x01, - SQUAT_INDEX_FLAG_NFS_FLUSH = 0x02, - SQUAT_INDEX_FLAG_DOTLOCK_USE_EXCL = 0x04 -}; - -enum squat_index_type { - SQUAT_INDEX_TYPE_HEADER = 0x01, - SQUAT_INDEX_TYPE_BODY = 0x02 -}; - -struct squat_trie_build_context; - -struct squat_trie * -squat_trie_init(const char *path, uint32_t uidvalidity, - enum file_lock_method lock_method, - enum squat_index_flags flags, mode_t mode, gid_t gid); -void squat_trie_deinit(struct squat_trie **trie); - -void squat_trie_set_partial_len(struct squat_trie *trie, unsigned int len); -void squat_trie_set_full_len(struct squat_trie *trie, unsigned int len); - -int squat_trie_open(struct squat_trie *trie); -int squat_trie_refresh(struct squat_trie *trie); - -int squat_trie_build_init(struct squat_trie *trie, - struct squat_trie_build_context **ctx_r); -/* bodies must be added before headers */ -int squat_trie_build_more(struct squat_trie_build_context *ctx, - uint32_t uid, enum squat_index_type type, - const unsigned char *data, unsigned int size); -/* if expunged_uids is non-NULL, they may be removed from the index if they - still exist. */ -int squat_trie_build_deinit(struct squat_trie_build_context **ctx, - const ARRAY_TYPE(seq_range) *expunged_uids) - ATTR_NULL(2); - -int squat_trie_get_last_uid(struct squat_trie *trie, uint32_t *last_uid_r); -/* type specifies if we're looking at header, body or both */ -int squat_trie_lookup(struct squat_trie *trie, const char *str, - enum squat_index_type type, - ARRAY_TYPE(seq_range) *definite_uids, - ARRAY_TYPE(seq_range) *maybe_uids); - -struct squat_uidlist *squat_trie_get_uidlist(struct squat_trie *trie); -size_t squat_trie_mem_used(struct squat_trie *trie, unsigned int *count_r); - -#endif diff --git a/src/plugins/fts-squat/squat-uidlist.c b/src/plugins/fts-squat/squat-uidlist.c deleted file mode 100644 index b1ee8cd70f..0000000000 --- a/src/plugins/fts-squat/squat-uidlist.c +++ /dev/null @@ -1,1623 +0,0 @@ -/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "sort.h" -#include "bsearch-insert-pos.h" -#include "file-cache.h" -#include "file-lock.h" -#include "read-full.h" -#include "write-full.h" -#include "ostream.h" -#include "mmap-util.h" -#include "squat-trie-private.h" -#include "squat-uidlist.h" - -#include -#include - -#define UIDLIST_LIST_SIZE 31 -#define UIDLIST_BLOCK_LIST_COUNT 100 -#define UID_LIST_MASK_RANGE 0x80000000U - -/* set = points to uidlist index number, unset = points to uidlist offset */ -#define UID_LIST_POINTER_MASK_LIST_IDX 0x80000000U - -#define UIDLIST_PACKED_FLAG_BITMASK 1 -#define UIDLIST_PACKED_FLAG_BEGINS_WITH_POINTER 2 - -struct uidlist_list { - unsigned int uid_count:31; - bool uid_begins_with_pointer:1; - uint32_t uid_list[UIDLIST_LIST_SIZE]; -}; - -struct squat_uidlist { - struct squat_trie *trie; - - char *path; - int fd; - struct file_cache *file_cache; - - struct file_lock *file_lock; - struct dotlock *dotlock; - uoff_t locked_file_size; - - void *mmap_base; - size_t mmap_size; - struct squat_uidlist_file_header hdr; - - const void *data; - size_t data_size; - - unsigned int cur_block_count; - const uint32_t *cur_block_offsets; - const uint32_t *cur_block_end_indexes; - - size_t max_size; - bool corrupted:1; - bool building:1; -}; - -struct squat_uidlist_build_context { - struct squat_uidlist *uidlist; - struct ostream *output; - - ARRAY_TYPE(uint32_t) block_offsets; - ARRAY_TYPE(uint32_t) block_end_indexes; - - ARRAY(struct uidlist_list) lists; - uint32_t list_start_idx; - - struct squat_uidlist_file_header build_hdr; - bool need_reopen:1; -}; - -struct squat_uidlist_rebuild_context { - struct squat_uidlist *uidlist; - struct squat_uidlist_build_context *build_ctx; - - int fd; - struct ostream *output; - - ARRAY_TYPE(uint32_t) new_block_offsets, new_block_end_indexes; - uoff_t cur_block_start_offset; - - uint32_t list_sizes[UIDLIST_BLOCK_LIST_COUNT]; - uint32_t next_uid_list_idx; - unsigned int list_idx; - unsigned int new_count; -}; - -static void squat_uidlist_close(struct squat_uidlist *uidlist); - -void squat_uidlist_delete(struct squat_uidlist *uidlist) -{ - i_unlink_if_exists(uidlist->path); -} - -static void squat_uidlist_set_corrupted(struct squat_uidlist *uidlist, - const char *reason) -{ - if (uidlist->corrupted) - return; - uidlist->corrupted = TRUE; - - i_error("Corrupted squat uidlist file %s: %s", uidlist->path, reason); - squat_trie_delete(uidlist->trie); -} - -static int -uidlist_write_array(struct ostream *output, const uint32_t *uid_list, - unsigned int uid_count, uint32_t packed_flags, - uint32_t offset, bool write_size, uint32_t *size_r) -{ - uint8_t *uidbuf, *bufp, sizebuf[SQUAT_PACK_MAX_SIZE], *sizebufp; - uint8_t listbuf[SQUAT_PACK_MAX_SIZE], *listbufp = listbuf; - uint32_t uid, uid2, prev, base_uid, size_value; - unsigned int i, bitmask_len, uid_list_len; - unsigned int idx, max_idx, mask; - bool datastack; - int num; - - if ((packed_flags & UIDLIST_PACKED_FLAG_BEGINS_WITH_POINTER) != 0) - squat_pack_num(&listbufp, offset); - - /* @UNSAFE */ - base_uid = uid_list[0] & ~UID_LIST_MASK_RANGE; - datastack = uid_count < 1024*8/SQUAT_PACK_MAX_SIZE; - if (datastack) - uidbuf = t_malloc_no0(SQUAT_PACK_MAX_SIZE * uid_count); - else - uidbuf = i_malloc(SQUAT_PACK_MAX_SIZE * uid_count); - bufp = uidbuf; - squat_pack_num(&bufp, base_uid); - - bitmask_len = (uid_list[uid_count-1] - base_uid + 7) / 8 + - (bufp - uidbuf); - if (bitmask_len < uid_count) { - bitmask_build: - i_assert(bitmask_len < SQUAT_PACK_MAX_SIZE*uid_count); - - memset(bufp, 0, bitmask_len - (bufp - uidbuf)); - if ((uid_list[0] & UID_LIST_MASK_RANGE) == 0) { - i = 1; - uid = i == uid_count ? 0 : uid_list[i]; - } else { - i = 0; - uid = uid_list[0] + 1; - } - base_uid++; - - for (; i < uid_count; i++) { - i_assert((uid & ~UID_LIST_MASK_RANGE) >= base_uid); - if ((uid & UID_LIST_MASK_RANGE) == 0) { - uid -= base_uid; - uid2 = uid; - } else { - uid &= ~UID_LIST_MASK_RANGE; - uid -= base_uid; - uid2 = uid_list[i+1] - base_uid; - i++; - } - - if (uid2 - uid < 3*8) { - for (; uid <= uid2; uid++) - bufp[uid / 8] |= 1 << (uid % 8); - } else { - /* first byte */ - idx = uid / 8; - num = uid % 8; - if (num != 0) { - uid += 8 - num; - for (mask = 0; num < 8; num++) - mask |= 1 << num; - bufp[idx++] |= mask; - } - - /* middle bytes */ - num = uid2 % 8; - max_idx = idx + (uid2 - num - uid)/8; - for (; idx < max_idx; idx++, uid += 8) - bufp[idx] = 0xff; - - /* last byte */ - for (mask = 0; num >= 0; num--) - mask |= 1 << num; - bufp[idx] |= mask; - } - uid = i+1 == uid_count ? 0 : uid_list[i+1]; - } - uid_list_len = bitmask_len; - packed_flags |= UIDLIST_PACKED_FLAG_BITMASK; - } else { - bufp = uidbuf; - prev = 0; - for (i = 0; i < uid_count; i++) { - uid = uid_list[i]; - if (unlikely((uid & ~UID_LIST_MASK_RANGE) < prev)) { - if (!datastack) - i_free(uidbuf); - return -1; - } - if ((uid & UID_LIST_MASK_RANGE) == 0) { - squat_pack_num(&bufp, (uid - prev) << 1); - prev = uid + 1; - } else { - uid &= ~UID_LIST_MASK_RANGE; - squat_pack_num(&bufp, 1 | (uid - prev) << 1); - squat_pack_num(&bufp, uid_list[i+1] - uid - 1); - prev = uid_list[i+1] + 1; - i++; - } - } - uid_list_len = bufp - uidbuf; - if (uid_list_len > bitmask_len) { - bufp = uidbuf; - squat_pack_num(&bufp, base_uid); - goto bitmask_build; - } - } - - size_value = ((uid_list_len + - (listbufp - listbuf)) << 2) | packed_flags; - if (write_size) { - sizebufp = sizebuf; - squat_pack_num(&sizebufp, size_value); - o_stream_nsend(output, sizebuf, sizebufp - sizebuf); - } - o_stream_nsend(output, listbuf, listbufp - listbuf); - o_stream_nsend(output, uidbuf, uid_list_len); - if (!datastack) - i_free(uidbuf); - - *size_r = size_value; - return 0; -} - -static int -uidlist_write(struct ostream *output, const struct uidlist_list *list, - bool write_size, uint32_t *size_r) -{ - const uint32_t *uid_list = list->uid_list; - uint8_t buf[SQUAT_PACK_MAX_SIZE], *bufp; - uint32_t uid_count = list->uid_count; - uint32_t packed_flags = 0; - uint32_t offset = 0; - int ret; - - if (list->uid_begins_with_pointer) { - /* continued UID list */ - packed_flags |= UIDLIST_PACKED_FLAG_BEGINS_WITH_POINTER; - if ((uid_list[0] & UID_LIST_POINTER_MASK_LIST_IDX) != 0) { - offset = ((uid_list[0] & ~UID_LIST_POINTER_MASK_LIST_IDX) << 1) | 1; - if (list->uid_count == 1) { - bufp = buf; - squat_pack_num(&bufp, offset); - o_stream_nsend(output, buf, bufp - buf); - *size_r = (bufp - buf) << 2 | packed_flags; - return 0; - } - } else if (unlikely(output->offset <= uid_list[0])) { - i_assert(output->closed); - return -1; - } else { - i_assert(list->uid_count > 1); - offset = (output->offset - uid_list[0]) << 1; - } - uid_list++; - uid_count--; - } - - T_BEGIN { - ret = uidlist_write_array(output, uid_list, uid_count, - packed_flags, offset, - write_size, size_r); - } T_END; - return ret; -} - -static void squat_uidlist_map_blocks_set_pointers(struct squat_uidlist *uidlist) -{ - const void *base; - size_t end_index_size, end_size; - - base = CONST_PTR_OFFSET(uidlist->data, uidlist->hdr.block_list_offset + - sizeof(uint32_t)); - - end_index_size = uidlist->cur_block_count * sizeof(uint32_t); - end_size = end_index_size + uidlist->cur_block_count * sizeof(uint32_t); - if (end_size <= uidlist->data_size) { - uidlist->cur_block_end_indexes = base; - uidlist->cur_block_offsets = - CONST_PTR_OFFSET(base, end_index_size); - } else { - uidlist->cur_block_end_indexes = NULL; - uidlist->cur_block_offsets = NULL; - } -} - -static int uidlist_file_cache_read(struct squat_uidlist *uidlist, - size_t offset, size_t size) -{ - if (uidlist->file_cache == NULL) - return 0; - - if (file_cache_read(uidlist->file_cache, offset, size) < 0) { - i_error("read(%s) failed: %m", uidlist->path); - return -1; - } - uidlist->data = file_cache_get_map(uidlist->file_cache, - &uidlist->data_size); - squat_uidlist_map_blocks_set_pointers(uidlist); - return 0; -} - -static int squat_uidlist_map_blocks(struct squat_uidlist *uidlist) -{ - const struct squat_uidlist_file_header *hdr = &uidlist->hdr; - const void *base; - uint32_t block_count, blocks_offset, blocks_size, i, verify_count; - - if (hdr->block_list_offset == 0) { - /* empty file */ - uidlist->cur_block_count = 0; - return 1; - } - - /* get number of blocks */ - if (uidlist_file_cache_read(uidlist, hdr->block_list_offset, - sizeof(block_count)) < 0) - return -1; - blocks_offset = hdr->block_list_offset + sizeof(block_count); - if (blocks_offset > uidlist->data_size) { - squat_uidlist_set_corrupted(uidlist, "block list outside file"); - return 0; - } - - i_assert(uidlist->data != NULL); - base = CONST_PTR_OFFSET(uidlist->data, hdr->block_list_offset); - memcpy(&block_count, base, sizeof(block_count)); - - /* map the blocks */ - blocks_size = block_count * sizeof(uint32_t)*2; - if (uidlist_file_cache_read(uidlist, blocks_offset, blocks_size) < 0) - return -1; - if (blocks_offset + blocks_size > uidlist->data_size) { - squat_uidlist_set_corrupted(uidlist, "block list outside file"); - return 0; - } - - uidlist->cur_block_count = block_count; - squat_uidlist_map_blocks_set_pointers(uidlist); - - i_assert(uidlist->cur_block_end_indexes != NULL); - - /* verify just a couple of the end indexes to make sure they - look correct */ - verify_count = I_MIN(block_count, 8); - for (i = 1; i < verify_count; i++) { - if (unlikely(uidlist->cur_block_end_indexes[i-1] >= - uidlist->cur_block_end_indexes[i])) { - squat_uidlist_set_corrupted(uidlist, - "block list corrupted"); - return 0; - } - } - return 1; -} - -static int squat_uidlist_map_header(struct squat_uidlist *uidlist) -{ - if (uidlist->hdr.indexid == 0) { - /* still being built */ - return 1; - } - if (uidlist->hdr.indexid != uidlist->trie->hdr.indexid) { - /* see if trie was recreated */ - (void)squat_trie_open(uidlist->trie); - } - if (uidlist->hdr.indexid != uidlist->trie->hdr.indexid) { - squat_uidlist_set_corrupted(uidlist, "wrong indexid"); - return 0; - } - if (uidlist->hdr.used_file_size < sizeof(uidlist->hdr) || - (uidlist->hdr.used_file_size > uidlist->mmap_size && - uidlist->mmap_base != NULL)) { - squat_uidlist_set_corrupted(uidlist, "broken used_file_size"); - return 0; - } - return squat_uidlist_map_blocks(uidlist); -} - -static void squat_uidlist_unmap(struct squat_uidlist *uidlist) -{ - if (uidlist->mmap_size != 0) { - if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0) - i_error("munmap(%s) failed: %m", uidlist->path); - uidlist->mmap_base = NULL; - uidlist->mmap_size = 0; - } - uidlist->cur_block_count = 0; - uidlist->cur_block_end_indexes = NULL; - uidlist->cur_block_offsets = NULL; -} - -static int squat_uidlist_mmap(struct squat_uidlist *uidlist) -{ - struct stat st; - - if (fstat(uidlist->fd, &st) < 0) { - i_error("fstat(%s) failed: %m", uidlist->path); - return -1; - } - if (st.st_size < (off_t)sizeof(uidlist->hdr)) { - squat_uidlist_set_corrupted(uidlist, "File too small"); - return -1; - } - - squat_uidlist_unmap(uidlist); - uidlist->mmap_size = st.st_size; - uidlist->mmap_base = mmap(NULL, uidlist->mmap_size, - PROT_READ | PROT_WRITE, - MAP_SHARED, uidlist->fd, 0); - if (uidlist->mmap_base == MAP_FAILED) { - uidlist->data = uidlist->mmap_base = NULL; - uidlist->data_size = uidlist->mmap_size = 0; - i_error("mmap(%s) failed: %m", uidlist->path); - return -1; - } - uidlist->data = uidlist->mmap_base; - uidlist->data_size = uidlist->mmap_size; - return 0; -} - -static int squat_uidlist_map(struct squat_uidlist *uidlist) -{ - const struct squat_uidlist_file_header *mmap_hdr = uidlist->mmap_base; - int ret; - - if (mmap_hdr != NULL && !uidlist->building && - uidlist->hdr.block_list_offset == mmap_hdr->block_list_offset) { - /* file hasn't changed */ - return 1; - } - - if ((uidlist->trie->flags & SQUAT_INDEX_FLAG_MMAP_DISABLE) == 0) { - if (mmap_hdr == NULL || uidlist->building || - uidlist->mmap_size < mmap_hdr->used_file_size) { - if (squat_uidlist_mmap(uidlist) < 0) - return -1; - } - - if (!uidlist->building) { - memcpy(&uidlist->hdr, uidlist->mmap_base, - sizeof(uidlist->hdr)); - } - } else if (uidlist->building) { - /* we want to update blocks mapping, but using the header - in memory */ - } else { - ret = pread_full(uidlist->fd, &uidlist->hdr, - sizeof(uidlist->hdr), 0); - if (ret <= 0) { - if (ret < 0) { - i_error("pread(%s) failed: %m", uidlist->path); - return -1; - } - i_error("Corrupted %s: File too small", uidlist->path); - return 0; - } - uidlist->data = NULL; - uidlist->data_size = 0; - } - if (uidlist->file_cache == NULL && - (uidlist->trie->flags & SQUAT_INDEX_FLAG_MMAP_DISABLE) != 0) - uidlist->file_cache = file_cache_new_path(uidlist->fd, uidlist->path); - return squat_uidlist_map_header(uidlist); -} - -static int squat_uidlist_read_to_memory(struct squat_uidlist *uidlist) -{ - size_t i, page_size = mmap_get_page_size(); - - if (uidlist->file_cache != NULL) { - return uidlist_file_cache_read(uidlist, 0, - uidlist->hdr.used_file_size); - } - /* Tell the kernel we're going to use the uidlist data, so it loads - it into memory and keeps it there. */ - (void)madvise(uidlist->mmap_base, uidlist->mmap_size, MADV_WILLNEED); - /* It also speeds up a bit for us to sequentially load everything - into memory, although at least Linux catches up quite fast even - without this code. Compiler can quite easily optimize away this - entire for loop, but volatile seems to help with gcc 4.2. */ - for (i = 0; i < uidlist->mmap_size; i += page_size) - ((const volatile char *)uidlist->data)[i]; - return 0; -} - -static void squat_uidlist_free_from_memory(struct squat_uidlist *uidlist) -{ - size_t page_size = mmap_get_page_size(); - - if (uidlist->file_cache != NULL) { - file_cache_invalidate(uidlist->file_cache, - page_size, UOFF_T_MAX); - } else { - (void)madvise(uidlist->mmap_base, uidlist->mmap_size, - MADV_DONTNEED); - } -} - -struct squat_uidlist *squat_uidlist_init(struct squat_trie *trie) -{ - struct squat_uidlist *uidlist; - - uidlist = i_new(struct squat_uidlist, 1); - uidlist->trie = trie; - uidlist->path = i_strconcat(trie->path, ".uids", NULL); - uidlist->fd = -1; - - return uidlist; -} - -void squat_uidlist_deinit(struct squat_uidlist *uidlist) -{ - squat_uidlist_close(uidlist); - - i_free(uidlist->path); - i_free(uidlist); -} - -static int squat_uidlist_open(struct squat_uidlist *uidlist) -{ - squat_uidlist_close(uidlist); - - uidlist->fd = open(uidlist->path, O_RDWR); - if (uidlist->fd == -1) { - if (errno == ENOENT) { - i_zero(&uidlist->hdr); - return 0; - } - i_error("open(%s) failed: %m", uidlist->path); - return -1; - } - return squat_uidlist_map(uidlist) <= 0 ? -1 : 0; -} - -static void squat_uidlist_close(struct squat_uidlist *uidlist) -{ - i_assert(!uidlist->building); - - squat_uidlist_unmap(uidlist); - if (uidlist->file_cache != NULL) - file_cache_free(&uidlist->file_cache); - file_lock_free(&uidlist->file_lock); - if (uidlist->dotlock != NULL) - file_dotlock_delete(&uidlist->dotlock); - i_close_fd_path(&uidlist->fd, uidlist->path); - uidlist->corrupted = FALSE; -} - -int squat_uidlist_refresh(struct squat_uidlist *uidlist) -{ - /* we assume here that trie is locked, so that we don't need to worry - about it when reading the header */ - if (uidlist->fd == -1 || - uidlist->hdr.indexid != uidlist->trie->hdr.indexid) { - if (squat_uidlist_open(uidlist) < 0) - return -1; - } else { - if (squat_uidlist_map(uidlist) <= 0) - return -1; - } - return 0; -} - -static int squat_uidlist_is_file_stale(struct squat_uidlist *uidlist) -{ - struct stat st, st2; - - i_assert(uidlist->fd != -1); - - if (stat(uidlist->path, &st) < 0) { - if (errno == ENOENT) - return 1; - - i_error("stat(%s) failed: %m", uidlist->path); - return -1; - } - if (fstat(uidlist->fd, &st2) < 0) { - i_error("fstat(%s) failed: %m", uidlist->path); - return -1; - } - uidlist->locked_file_size = st2.st_size; - - return st.st_ino == st2.st_ino && - CMP_DEV_T(st.st_dev, st2.st_dev) ? 0 : 1; -} - -static int squat_uidlist_lock(struct squat_uidlist *uidlist) -{ - const char *error; - int ret; - - for (;;) { - i_assert(uidlist->fd != -1); - i_assert(uidlist->file_lock == NULL); - i_assert(uidlist->dotlock == NULL); - - if (uidlist->trie->lock_method != FILE_LOCK_METHOD_DOTLOCK) { - struct file_lock_settings lock_set = { - .lock_method = uidlist->trie->lock_method, - }; - ret = file_wait_lock(uidlist->fd, uidlist->path, - F_WRLCK, &lock_set, - SQUAT_TRIE_LOCK_TIMEOUT, - &uidlist->file_lock, &error); - if (ret < 0) { - i_error("squat uidlist %s: %s", - uidlist->path, error); - } - } else { - ret = file_dotlock_create(&uidlist->trie->dotlock_set, - uidlist->path, 0, - &uidlist->dotlock); - } - if (ret == 0) { - i_error("squat uidlist %s: Locking timed out", - uidlist->path); - return 0; - } - if (ret < 0) - return -1; - - ret = squat_uidlist_is_file_stale(uidlist); - if (ret == 0) - break; - - if (uidlist->file_lock != NULL) - file_unlock(&uidlist->file_lock); - else - file_dotlock_delete(&uidlist->dotlock); - if (ret < 0) - return -1; - - squat_uidlist_close(uidlist); - uidlist->fd = squat_trie_create_fd(uidlist->trie, - uidlist->path, 0); - if (uidlist->fd == -1) - return -1; - } - return 1; -} - -static int squat_uidlist_open_or_create(struct squat_uidlist *uidlist) -{ - int ret; - - if (uidlist->fd == -1) { - uidlist->fd = squat_trie_create_fd(uidlist->trie, - uidlist->path, 0); - if (uidlist->fd == -1) - return -1; - } - if (squat_uidlist_lock(uidlist) <= 0) - return -1; - - if (uidlist->locked_file_size != 0) { - if ((ret = squat_uidlist_map(uidlist)) < 0) - return -1; - if (ret == 0) { - /* broken file, truncate */ - if (ftruncate(uidlist->fd, 0) < 0) { - i_error("ftruncate(%s) failed: %m", - uidlist->path); - return -1; - } - uidlist->locked_file_size = 0; - } - } - if (uidlist->locked_file_size == 0) { - /* write using 0 until we're finished */ - i_zero(&uidlist->hdr); - if (write_full(uidlist->fd, &uidlist->hdr, - sizeof(uidlist->hdr)) < 0) { - i_error("write(%s) failed: %m", uidlist->path); - return -1; - } - } - return 0; -} - -int squat_uidlist_build_init(struct squat_uidlist *uidlist, - struct squat_uidlist_build_context **ctx_r) -{ - struct squat_uidlist_build_context *ctx; - int ret; - - i_assert(!uidlist->building); - - ret = squat_uidlist_open_or_create(uidlist); - if (ret == 0 && - lseek(uidlist->fd, uidlist->hdr.used_file_size, SEEK_SET) < 0) { - i_error("lseek(%s) failed: %m", uidlist->path); - ret = -1; - } - - if (ret < 0) { - if (uidlist->file_lock != NULL) - file_unlock(&uidlist->file_lock); - if (uidlist->dotlock != NULL) - file_dotlock_delete(&uidlist->dotlock); - return -1; - } - - ctx = i_new(struct squat_uidlist_build_context, 1); - ctx->uidlist = uidlist; - ctx->output = o_stream_create_fd(uidlist->fd, 0); - if (ctx->output->offset == 0) { - struct squat_uidlist_file_header hdr; - - i_zero(&hdr); - o_stream_nsend(ctx->output, &hdr, sizeof(hdr)); - } - o_stream_cork(ctx->output); - i_array_init(&ctx->lists, 10240); - i_array_init(&ctx->block_offsets, 128); - i_array_init(&ctx->block_end_indexes, 128); - ctx->list_start_idx = uidlist->hdr.count; - ctx->build_hdr = uidlist->hdr; - - uidlist->building = TRUE; - *ctx_r = ctx; - return 0; -} - -static void -uidlist_write_block_list_and_header(struct squat_uidlist_build_context *ctx, - struct ostream *output, - ARRAY_TYPE(uint32_t) *block_offsets, - ARRAY_TYPE(uint32_t) *block_end_indexes, - bool write_old_blocks) -{ - struct squat_uidlist *uidlist = ctx->uidlist; - unsigned int align, old_block_count, new_block_count; - uint32_t block_offset_count; - uoff_t block_list_offset; - - i_assert(uidlist->trie->hdr.indexid != 0); - ctx->build_hdr.indexid = uidlist->trie->hdr.indexid; - - if (array_count(block_end_indexes) == 0) { - ctx->build_hdr.used_file_size = output->offset; - ctx->build_hdr.block_list_offset = 0; - uidlist->hdr = ctx->build_hdr; - return; - } - - align = output->offset % sizeof(uint32_t); - if (align != 0) { - static char null[sizeof(uint32_t)-1] = { 0, }; - - o_stream_nsend(output, null, sizeof(uint32_t) - align); - } - block_list_offset = output->offset; - - new_block_count = array_count(block_offsets); - old_block_count = write_old_blocks ? uidlist->cur_block_count : 0; - - block_offset_count = new_block_count + old_block_count; - o_stream_nsend(output, &block_offset_count, sizeof(block_offset_count)); - /* write end indexes */ - o_stream_nsend(output, uidlist->cur_block_end_indexes, - old_block_count * sizeof(uint32_t)); - o_stream_nsend(output, array_front(block_end_indexes), - new_block_count * sizeof(uint32_t)); - /* write offsets */ - o_stream_nsend(output, uidlist->cur_block_offsets, - old_block_count * sizeof(uint32_t)); - o_stream_nsend(output, array_front(block_offsets), - new_block_count * sizeof(uint32_t)); - (void)o_stream_flush(output); - - /* update header - it's written later when trie is locked */ - ctx->build_hdr.block_list_offset = block_list_offset; - ctx->build_hdr.used_file_size = output->offset; - uidlist->hdr = ctx->build_hdr; -} - -void squat_uidlist_build_flush(struct squat_uidlist_build_context *ctx) -{ - struct uidlist_list *lists; - uint8_t buf[SQUAT_PACK_MAX_SIZE], *bufp; - unsigned int i, j, count, max; - uint32_t block_offset, block_end_idx, start_offset; - uint32_t list_sizes[UIDLIST_BLOCK_LIST_COUNT]; - size_t mem_size; - - if (ctx->uidlist->corrupted) - return; - - lists = array_get_modifiable(&ctx->lists, &count); - if (count == 0) - return; - - /* write the lists and save the written sizes to uid_list[0] */ - for (i = 0; i < count; i += UIDLIST_BLOCK_LIST_COUNT) { - start_offset = ctx->output->offset; - max = I_MIN(count - i, UIDLIST_BLOCK_LIST_COUNT); - for (j = 0; j < max; j++) { - if (uidlist_write(ctx->output, &lists[i+j], - FALSE, &list_sizes[j]) < 0) { - squat_uidlist_set_corrupted(ctx->uidlist, - "Broken uidlists"); - return; - } - } - - block_offset = ctx->output->offset; - block_end_idx = ctx->list_start_idx + i + max; - array_push_back(&ctx->block_offsets, &block_offset); - array_push_back(&ctx->block_end_indexes, &block_end_idx); - - /* write the full size of the uidlists */ - bufp = buf; - squat_pack_num(&bufp, block_offset - start_offset); - o_stream_nsend(ctx->output, buf, bufp - buf); - - /* write the sizes/flags */ - for (j = 0; j < max; j++) { - bufp = buf; - squat_pack_num(&bufp, list_sizes[j]); - o_stream_nsend(ctx->output, buf, bufp - buf); - } - } - - mem_size = ctx->lists.arr.buffer->used + - ctx->block_offsets.arr.buffer->used + - ctx->block_end_indexes.arr.buffer->used; - if (ctx->uidlist->max_size < mem_size) - ctx->uidlist->max_size = mem_size; - - ctx->list_start_idx += count; - array_clear(&ctx->lists); - - uidlist_write_block_list_and_header(ctx, ctx->output, - &ctx->block_offsets, - &ctx->block_end_indexes, TRUE); - - (void)squat_uidlist_map(ctx->uidlist); - - array_clear(&ctx->block_offsets); - array_clear(&ctx->block_end_indexes); -} - -int squat_uidlist_build_finish(struct squat_uidlist_build_context *ctx) -{ - if (ctx->uidlist->corrupted) - return -1; - - if (!ctx->output->closed) { - (void)o_stream_seek(ctx->output, 0); - o_stream_nsend(ctx->output, - &ctx->build_hdr, sizeof(ctx->build_hdr)); - (void)o_stream_seek(ctx->output, ctx->build_hdr.used_file_size); - } - - if (o_stream_finish(ctx->output) < 0) { - i_error("write() to %s failed: %s", ctx->uidlist->path, - o_stream_get_error(ctx->output)); - return -1; - } - return 0; -} - -void squat_uidlist_build_deinit(struct squat_uidlist_build_context **_ctx) -{ - struct squat_uidlist_build_context *ctx = *_ctx; - - *_ctx = NULL; - - i_assert(array_count(&ctx->lists) == 0 || ctx->uidlist->corrupted); - i_assert(ctx->uidlist->building); - ctx->uidlist->building = FALSE; - - if (ctx->uidlist->file_lock != NULL) - file_unlock(&ctx->uidlist->file_lock); - else - file_dotlock_delete(&ctx->uidlist->dotlock); - - if (ctx->need_reopen) - (void)squat_uidlist_open(ctx->uidlist); - - array_free(&ctx->block_offsets); - array_free(&ctx->block_end_indexes); - array_free(&ctx->lists); - o_stream_ignore_last_errors(ctx->output); - o_stream_unref(&ctx->output); - i_free(ctx); -} - -int squat_uidlist_rebuild_init(struct squat_uidlist_build_context *build_ctx, - bool compress, - struct squat_uidlist_rebuild_context **ctx_r) -{ - struct squat_uidlist_rebuild_context *ctx; - struct squat_uidlist_file_header hdr; - const char *temp_path; - int fd; - - if (build_ctx->build_hdr.link_count == 0) - return 0; - - if (!compress) { - if (build_ctx->build_hdr.link_count < - build_ctx->build_hdr.count*2/3) - return 0; - } - - /* make sure the entire uidlist is in memory before beginning, - otherwise the pages are faulted to memory in random order which - takes forever. */ - if (squat_uidlist_read_to_memory(build_ctx->uidlist) < 0) - return -1; - - temp_path = t_strconcat(build_ctx->uidlist->path, ".tmp", NULL); - fd = squat_trie_create_fd(build_ctx->uidlist->trie, temp_path, O_TRUNC); - if (fd == -1) - return -1; - - ctx = i_new(struct squat_uidlist_rebuild_context, 1); - ctx->uidlist = build_ctx->uidlist; - ctx->build_ctx = build_ctx; - ctx->fd = fd; - ctx->output = o_stream_create_fd(ctx->fd, 0); - ctx->next_uid_list_idx = 0x100; - o_stream_cork(ctx->output); - - i_zero(&hdr); - o_stream_nsend(ctx->output, &hdr, sizeof(hdr)); - - ctx->cur_block_start_offset = ctx->output->offset; - i_array_init(&ctx->new_block_offsets, - build_ctx->build_hdr.count / UIDLIST_BLOCK_LIST_COUNT); - i_array_init(&ctx->new_block_end_indexes, - build_ctx->build_hdr.count / UIDLIST_BLOCK_LIST_COUNT); - *ctx_r = ctx; - return 1; -} - -static void -uidlist_rebuild_flush_block(struct squat_uidlist_rebuild_context *ctx) -{ - uint8_t buf[SQUAT_PACK_MAX_SIZE], *bufp; - uint32_t block_offset, block_end_idx; - unsigned int i; - - ctx->new_count += ctx->list_idx; - - block_offset = ctx->output->offset; - block_end_idx = ctx->new_count; - array_push_back(&ctx->new_block_offsets, &block_offset); - array_push_back(&ctx->new_block_end_indexes, &block_end_idx); - - /* this block's contents started from cur_block_start_offset and - ended to current offset. write the size of this area. */ - bufp = buf; - squat_pack_num(&bufp, block_offset - ctx->cur_block_start_offset); - o_stream_nsend(ctx->output, buf, bufp - buf); - - /* write the sizes/flags */ - for (i = 0; i < ctx->list_idx; i++) { - bufp = buf; - squat_pack_num(&bufp, ctx->list_sizes[i]); - o_stream_nsend(ctx->output, buf, bufp - buf); - } - ctx->cur_block_start_offset = ctx->output->offset; -} - -uint32_t squat_uidlist_rebuild_next(struct squat_uidlist_rebuild_context *ctx, - const ARRAY_TYPE(uint32_t) *uids) -{ - int ret; - - T_BEGIN { - ret = uidlist_write_array(ctx->output, array_front(uids), - array_count(uids), 0, 0, FALSE, - &ctx->list_sizes[ctx->list_idx]); - } T_END; - if (ret < 0) - squat_uidlist_set_corrupted(ctx->uidlist, "Broken uidlists"); - - if (++ctx->list_idx == UIDLIST_BLOCK_LIST_COUNT) { - uidlist_rebuild_flush_block(ctx); - ctx->list_idx = 0; - } - return ctx->next_uid_list_idx++ << 1; -} - -uint32_t squat_uidlist_rebuild_nextu(struct squat_uidlist_rebuild_context *ctx, - const ARRAY_TYPE(seq_range) *uids) -{ - const struct seq_range *range; - ARRAY_TYPE(uint32_t) tmp_uids; - uint32_t seq, uid1, ret; - unsigned int i, count; - - range = array_get(uids, &count); - if (count == 0) - return 0; - - if (range[count-1].seq2 < 8) { - /* we can use a singleton bitmask */ - ret = 0; - for (i = 0; i < count; i++) { - for (seq = range[i].seq1; seq <= range[i].seq2; seq++) - ret |= 1 << (seq+1); - } - return ret; - } - if (count == 1 && range[0].seq1 == range[0].seq2) { - /* single UID */ - return (range[0].seq1 << 1) | 1; - } - - /* convert seq range to our internal representation and use the - normal _rebuild_next() to write it */ - i_array_init(&tmp_uids, 128); - for (i = 0; i < count; i++) { - if (range[i].seq1 == range[i].seq2) - array_push_back(&tmp_uids, &range[i].seq1); - else { - uid1 = range[i].seq1 | UID_LIST_MASK_RANGE; - array_push_back(&tmp_uids, &uid1); - array_push_back(&tmp_uids, &range[i].seq2); - } - } - ret = squat_uidlist_rebuild_next(ctx, &tmp_uids); - array_free(&tmp_uids); - return ret; -} - -int squat_uidlist_rebuild_finish(struct squat_uidlist_rebuild_context *ctx, - bool cancel) -{ - const char *temp_path; - int ret = 1; - - if (ctx->list_idx != 0) - uidlist_rebuild_flush_block(ctx); - if (cancel || ctx->uidlist->corrupted) - ret = 0; - - temp_path = t_strconcat(ctx->uidlist->path, ".tmp", NULL); - if (ret > 0) { - ctx->build_ctx->build_hdr.indexid = - ctx->uidlist->trie->hdr.indexid; - ctx->build_ctx->build_hdr.count = ctx->new_count; - ctx->build_ctx->build_hdr.link_count = 0; - uidlist_write_block_list_and_header(ctx->build_ctx, ctx->output, - &ctx->new_block_offsets, - &ctx->new_block_end_indexes, - FALSE); - (void)o_stream_seek(ctx->output, 0); - o_stream_nsend(ctx->output, &ctx->build_ctx->build_hdr, - sizeof(ctx->build_ctx->build_hdr)); - (void)o_stream_seek(ctx->output, - ctx->build_ctx->build_hdr.used_file_size); - - if (ctx->uidlist->corrupted) - ret = -1; - else if (o_stream_finish(ctx->output) < 0) { - i_error("write(%s) failed: %s", temp_path, - o_stream_get_error(ctx->output)); - ret = -1; - } else if (rename(temp_path, ctx->uidlist->path) < 0) { - i_error("rename(%s, %s) failed: %m", - temp_path, ctx->uidlist->path); - ret = -1; - } - ctx->build_ctx->need_reopen = TRUE; - } else { - o_stream_abort(ctx->output); - } - - /* we no longer require the entire uidlist to be in memory, - let it be used for something more useful. */ - squat_uidlist_free_from_memory(ctx->uidlist); - - o_stream_unref(&ctx->output); - if (close(ctx->fd) < 0) - i_error("close(%s) failed: %m", temp_path); - - if (ret <= 0) - i_unlink(temp_path); - array_free(&ctx->new_block_offsets); - array_free(&ctx->new_block_end_indexes); - i_free(ctx); - return ret < 0 ? -1 : 0; -} - -static void -uidlist_flush(struct squat_uidlist_build_context *ctx, - struct uidlist_list *list, uint32_t uid) -{ - uint32_t size, offset = ctx->output->offset; - - ctx->build_hdr.link_count++; - if (uidlist_write(ctx->output, list, TRUE, &size) < 0) - squat_uidlist_set_corrupted(ctx->uidlist, "Broken uidlists"); - - list->uid_count = 2; - list->uid_begins_with_pointer = TRUE; - - list->uid_list[0] = offset; - list->uid_list[1] = uid; -} - -static struct uidlist_list * -uidlist_add_new(struct squat_uidlist_build_context *ctx, unsigned int count, - uint32_t *uid_list_idx_r) -{ - struct uidlist_list *list; - - i_assert(array_count(&ctx->lists) + - ctx->list_start_idx == ctx->build_hdr.count); - *uid_list_idx_r = (ctx->build_hdr.count + 0x100) << 1; - list = array_append_space(&ctx->lists); - ctx->build_hdr.count++; - - list->uid_count = count; - return list; -} - -uint32_t squat_uidlist_build_add_uid(struct squat_uidlist_build_context *ctx, - uint32_t uid_list_idx, uint32_t uid) -{ - struct uidlist_list *list; - unsigned int idx, mask; - uint32_t *p; - - if ((uid_list_idx & 1) != 0) { - /* adding second UID */ - uint32_t prev_uid = uid_list_idx >> 1; - - i_assert(prev_uid != uid); - list = uidlist_add_new(ctx, 2, &uid_list_idx); - list->uid_list[0] = prev_uid; - if (prev_uid + 1 == uid) - list->uid_list[0] |= UID_LIST_MASK_RANGE; - list->uid_list[1] = uid; - return uid_list_idx; - } else if (uid_list_idx < (0x100 << 1)) { - uint32_t old_list_idx; - - if (uid < 8) { - /* UID lists containing only UIDs 0-7 are saved as - uidlist values 2..511. think of it as a bitmask. */ - uid_list_idx |= 1 << (uid + 1); - i_assert((uid_list_idx & 1) == 0); - return uid_list_idx; - } - - if (uid_list_idx == 0) { - /* first UID */ - return (uid << 1) | 1; - } - - /* create a new list */ - old_list_idx = uid_list_idx >> 1; - list = uidlist_add_new(ctx, 1, &uid_list_idx); - /* add the first UID ourself */ - idx = 0; - i_assert((old_list_idx & 0xff) != 0); - for (mask = 1; mask <= 128; mask <<= 1, idx++) { - if ((old_list_idx & mask) != 0) { - list->uid_list[0] = idx; - idx++; mask <<= 1; - break; - } - } - for (; mask <= 128; mask <<= 1, idx++) { - if ((old_list_idx & mask) != 0) { - (void)squat_uidlist_build_add_uid(ctx, - uid_list_idx, idx); - } - } - } - - /* add to existing list */ - idx = (uid_list_idx >> 1) - 0x100; - if (idx < ctx->list_start_idx) { - list = uidlist_add_new(ctx, 2, &uid_list_idx); - list->uid_list[0] = UID_LIST_POINTER_MASK_LIST_IDX | idx; - list->uid_list[1] = uid; - list->uid_begins_with_pointer = TRUE; - ctx->build_hdr.link_count++; - return uid_list_idx; - } - - idx -= ctx->list_start_idx; - if (idx >= array_count(&ctx->lists)) { - squat_uidlist_set_corrupted(ctx->uidlist, - "missing/broken uidlist"); - return 0; - } - list = array_idx_modifiable(&ctx->lists, idx); - i_assert(list->uid_count > 0); - - p = &list->uid_list[list->uid_count-1]; - i_assert(uid != *p || ctx->uidlist->corrupted || - (list->uid_count == 1 && list->uid_begins_with_pointer)); - if (uid == *p + 1 && - (list->uid_count > 1 || !list->uid_begins_with_pointer)) { - /* use a range */ - if (list->uid_count > 1 && (p[-1] & UID_LIST_MASK_RANGE) != 0 && - (list->uid_count > 2 || !list->uid_begins_with_pointer)) { - /* increase the existing range */ - *p += 1; - return uid_list_idx; - } - - if (list->uid_count == UIDLIST_LIST_SIZE) { - uidlist_flush(ctx, list, uid); - return uid_list_idx; - } - /* create a new range */ - *p |= UID_LIST_MASK_RANGE; - } else { - if (list->uid_count == UIDLIST_LIST_SIZE) { - uidlist_flush(ctx, list, uid); - return uid_list_idx; - } - } - - p++; - list->uid_count++; - - *p = uid; - return uid_list_idx; -} - -static void uidlist_array_append(ARRAY_TYPE(uint32_t) *uids, uint32_t uid) -{ - uint32_t *uidlist; - unsigned int count; - - uidlist = array_get_modifiable(uids, &count); - if (count == 0) { - array_push_back(uids, &uid); - return; - } - if (uidlist[count-1] + 1 == uid) { - if (count > 1 && (uidlist[count-2] & - UID_LIST_MASK_RANGE) != 0) { - uidlist[count-1]++; - return; - } - uidlist[count-1] |= UID_LIST_MASK_RANGE; - } - array_push_back(uids, &uid); -} - -static void uidlist_array_append_range(ARRAY_TYPE(uint32_t) *uids, - uint32_t uid1, uint32_t uid2) -{ - uint32_t *uidlist; - unsigned int count; - - i_assert(uid1 < uid2); - - uidlist = array_get_modifiable(uids, &count); - if (count == 0) { - uid1 |= UID_LIST_MASK_RANGE; - array_push_back(uids, &uid1); - array_push_back(uids, &uid2); - return; - } - if (uidlist[count-1] + 1 == uid1) { - if (count > 1 && (uidlist[count-2] & - UID_LIST_MASK_RANGE) != 0) { - uidlist[count-1] = uid2; - return; - } - uidlist[count-1] |= UID_LIST_MASK_RANGE; - } else { - uid1 |= UID_LIST_MASK_RANGE; - array_push_back(uids, &uid1); - } - array_push_back(uids, &uid2); -} - -static int -squat_uidlist_get_at_offset(struct squat_uidlist *uidlist, uoff_t offset, - uint32_t num, ARRAY_TYPE(uint32_t) *uids) -{ - const uint32_t *uid_list; - const uint8_t *p, *end; - uint32_t size, base_uid, next_uid, flags, prev; - uoff_t uidlist_data_offset; - unsigned int i, j, count; - - if (num != 0) - uidlist_data_offset = offset; - else { - /* not given, read it */ - if (uidlist_file_cache_read(uidlist, offset, - SQUAT_PACK_MAX_SIZE) < 0) - return -1; - - p = CONST_PTR_OFFSET(uidlist->data, offset); - end = CONST_PTR_OFFSET(uidlist->data, uidlist->data_size); - num = squat_unpack_num(&p, end); - uidlist_data_offset = p - (const uint8_t *)uidlist->data; - } - size = num >> 2; - - if (uidlist_file_cache_read(uidlist, uidlist_data_offset, size) < 0) - return -1; - if (uidlist_data_offset + size > uidlist->data_size) { - squat_uidlist_set_corrupted(uidlist, - "size points outside file"); - return -1; - } - - p = CONST_PTR_OFFSET(uidlist->data, uidlist_data_offset); - end = p + size; - - flags = num; - if ((flags & UIDLIST_PACKED_FLAG_BEGINS_WITH_POINTER) != 0) { - /* link to the file */ - prev = squat_unpack_num(&p, end); - - if ((prev & 1) != 0) { - /* pointer to uidlist */ - prev = ((prev >> 1) + 0x100) << 1; - if (squat_uidlist_get(uidlist, prev, uids) < 0) - return -1; - } else { - prev = offset - (prev >> 1); - if (squat_uidlist_get_at_offset(uidlist, prev, - 0, uids) < 0) - return -1; - } - uid_list = array_get(uids, &count); - next_uid = count == 0 ? 0 : uid_list[count-1] + 1; - } else { - next_uid = 0; - } - - num = base_uid = squat_unpack_num(&p, end); - if ((flags & UIDLIST_PACKED_FLAG_BITMASK) == 0) - base_uid >>= 1; - if (base_uid < next_uid) { - squat_uidlist_set_corrupted(uidlist, - "broken continued uidlist"); - return -1; - } - - if ((flags & UIDLIST_PACKED_FLAG_BITMASK) != 0) { - /* bitmask */ - size = end - p; - - uidlist_array_append(uids, base_uid++); - for (i = 0; i < size; i++) { - for (j = 0; j < 8; j++, base_uid++) { - if ((p[i] & (1 << j)) != 0) - uidlist_array_append(uids, base_uid); - } - } - } else { - /* range */ - for (;;) { - if ((num & 1) == 0) { - uidlist_array_append(uids, base_uid); - } else { - /* range */ - uint32_t seq1 = base_uid; - base_uid += squat_unpack_num(&p, end) + 1; - uidlist_array_append_range(uids, seq1, - base_uid); - } - if (p == end) - break; - - num = squat_unpack_num(&p, end); - base_uid += (num >> 1) + 1; - } - } - return 0; -} - -static int -squat_uidlist_get_offset(struct squat_uidlist *uidlist, uint32_t uid_list_idx, - uint32_t *offset_r, uint32_t *num_r) -{ - const uint8_t *p, *end; - unsigned int idx; - uint32_t num, skip_bytes, uidlists_offset; - size_t max_map_size; - - if (uidlist->fd == -1) { - squat_uidlist_set_corrupted(uidlist, "no uidlists"); - return -1; - } - - if (bsearch_insert_pos(&uid_list_idx, uidlist->cur_block_end_indexes, - uidlist->cur_block_count, - sizeof(uint32_t), uint32_cmp, &idx)) - idx++; - if (unlikely(idx == uidlist->cur_block_count)) { - squat_uidlist_set_corrupted(uidlist, "uidlist not found"); - return -1; - } - i_assert(uidlist->cur_block_end_indexes != NULL); - if (unlikely(idx > 0 && - uidlist->cur_block_end_indexes[idx-1] > uid_list_idx)) { - squat_uidlist_set_corrupted(uidlist, "broken block list"); - return -1; - } - - /* make sure everything is mapped */ - uid_list_idx -= idx == 0 ? 0 : uidlist->cur_block_end_indexes[idx-1]; - max_map_size = SQUAT_PACK_MAX_SIZE * (1+uid_list_idx); - if (uidlist_file_cache_read(uidlist, uidlist->cur_block_offsets[idx], - max_map_size) < 0) - return -1; - - /* find the uidlist inside the block */ - i_assert(uidlist->cur_block_offsets != NULL); - p = CONST_PTR_OFFSET(uidlist->data, uidlist->cur_block_offsets[idx]); - end = CONST_PTR_OFFSET(uidlist->data, uidlist->data_size); - - uidlists_offset = uidlist->cur_block_offsets[idx] - - squat_unpack_num(&p, end); - for (skip_bytes = 0; uid_list_idx > 0; uid_list_idx--) { - num = squat_unpack_num(&p, end); - skip_bytes += num >> 2; - } - *offset_r = uidlists_offset + skip_bytes; - *num_r = squat_unpack_num(&p, end); - - if (unlikely(p == end)) { - squat_uidlist_set_corrupted(uidlist, "broken file"); - return -1; - } - if (unlikely(*offset_r > uidlist->mmap_size && - uidlist->mmap_base != NULL)) { - squat_uidlist_set_corrupted(uidlist, "broken offset"); - return -1; - } - return 0; -} - -int squat_uidlist_get(struct squat_uidlist *uidlist, uint32_t uid_list_idx, - ARRAY_TYPE(uint32_t) *uids) -{ - unsigned int mask; - uint32_t uid, offset, num; - - if ((uid_list_idx & 1) != 0) { - /* single UID */ - uid = uid_list_idx >> 1; - uidlist_array_append(uids, uid); - return 0; - } else if (uid_list_idx < (0x100 << 1)) { - /* bitmask */ - for (uid = 0, mask = 2; mask <= 256; mask <<= 1, uid++) { - if ((uid_list_idx & mask) != 0) - uidlist_array_append(uids, uid); - } - return 0; - } - - uid_list_idx = (uid_list_idx >> 1) - 0x100; - if (squat_uidlist_get_offset(uidlist, uid_list_idx, &offset, &num) < 0) - return -1; - return squat_uidlist_get_at_offset(uidlist, offset, num, uids); -} - -uint32_t squat_uidlist_singleton_last_uid(uint32_t uid_list_idx) -{ - unsigned int idx, mask; - - if ((uid_list_idx & 1) != 0) { - /* single UID */ - return uid_list_idx >> 1; - } else if (uid_list_idx < (0x100 << 1)) { - /* bitmask */ - if (uid_list_idx == 2) { - /* just a quick optimization */ - return 0; - } - for (idx = 7, mask = 256; mask > 2; mask >>= 1, idx--) { - if ((uid_list_idx & mask) != 0) - return idx; - } - } - - i_unreached(); -} - -int squat_uidlist_get_seqrange(struct squat_uidlist *uidlist, - uint32_t uid_list_idx, - ARRAY_TYPE(seq_range) *seq_range_arr) -{ - ARRAY_TYPE(uint32_t) tmp_uid_arr; - struct seq_range range; - const uint32_t *tmp_uids; - unsigned int i, count; - int ret; - - i_array_init(&tmp_uid_arr, 128); - ret = squat_uidlist_get(uidlist, uid_list_idx, &tmp_uid_arr); - if (ret == 0) { - tmp_uids = array_get(&tmp_uid_arr, &count); - for (i = 0; i < count; i++) { - if ((tmp_uids[i] & UID_LIST_MASK_RANGE) == 0) - range.seq1 = range.seq2 = tmp_uids[i]; - else { - range.seq1 = tmp_uids[i] & ~UID_LIST_MASK_RANGE; - range.seq2 = tmp_uids[++i]; - } - array_push_back(seq_range_arr, &range); - } - } - array_free(&tmp_uid_arr); - return ret; -} - -int squat_uidlist_filter(struct squat_uidlist *uidlist, uint32_t uid_list_idx, - ARRAY_TYPE(seq_range) *uids) -{ - const struct seq_range *parent_range; - ARRAY_TYPE(seq_range) dest_uids; - ARRAY_TYPE(uint32_t) relative_uids; - const uint32_t *rel_range; - unsigned int i, rel_count, parent_idx, parent_count, diff, parent_uid; - uint32_t prev_seq, seq1, seq2; - int ret = 0; - - parent_range = array_get(uids, &parent_count); - if (parent_count == 0) - return 0; - - i_array_init(&relative_uids, 128); - i_array_init(&dest_uids, 128); - if (squat_uidlist_get(uidlist, uid_list_idx, &relative_uids) < 0) - ret = -1; - - parent_idx = 0; - rel_range = array_get(&relative_uids, &rel_count); - prev_seq = 0; parent_uid = parent_range[0].seq1; - for (i = 0; i < rel_count; i++) { - if (unlikely(parent_uid == (uint32_t)-1)) { - i_error("broken UID ranges"); - ret = -1; - break; - } - if ((rel_range[i] & UID_LIST_MASK_RANGE) == 0) - seq1 = seq2 = rel_range[i]; - else { - seq1 = (rel_range[i] & ~UID_LIST_MASK_RANGE); - seq2 = rel_range[++i]; - } - i_assert(seq1 >= prev_seq); - diff = seq1 - prev_seq; - while (diff > 0) { - if (unlikely(parent_uid == (uint32_t)-1)) { - i_error("broken UID ranges"); - ret = -1; - break; - } - - for (; parent_idx < parent_count; parent_idx++) { - if (parent_range[parent_idx].seq2 <= parent_uid) - continue; - if (parent_uid < parent_range[parent_idx].seq1) - parent_uid = parent_range[parent_idx].seq1; - else - parent_uid++; - break; - } - diff--; - } - diff = seq2 - seq1 + 1; - while (diff > 0) { - if (unlikely(parent_uid == (uint32_t)-1)) { - i_error("broken UID ranges"); - ret = -1; - break; - } - seq_range_array_add(&dest_uids, parent_uid); - for (; parent_idx < parent_count; parent_idx++) { - if (parent_range[parent_idx].seq2 <= parent_uid) - continue; - if (parent_uid < parent_range[parent_idx].seq1) - parent_uid = parent_range[parent_idx].seq1; - else - parent_uid++; - break; - } - diff--; - } - - prev_seq = seq2 + 1; - } - - buffer_set_used_size(uids->arr.buffer, 0); - array_append_array(uids, &dest_uids); - - array_free(&relative_uids); - array_free(&dest_uids); - return ret; -} - -size_t squat_uidlist_mem_used(struct squat_uidlist *uidlist, - unsigned int *count_r) -{ - *count_r = uidlist->hdr.count; - return uidlist->max_size; -} diff --git a/src/plugins/fts-squat/squat-uidlist.h b/src/plugins/fts-squat/squat-uidlist.h deleted file mode 100644 index 79ed791bf1..0000000000 --- a/src/plugins/fts-squat/squat-uidlist.h +++ /dev/null @@ -1,71 +0,0 @@ -#ifndef SQUAT_UIDLIST_H -#define SQUAT_UIDLIST_H - -struct squat_trie; -struct squat_uidlist_build_context; -struct squat_uidlist_rebuild_context; - -struct squat_uidlist_file_header { - uint32_t indexid; - uint32_t used_file_size; - uint32_t block_list_offset; - uint32_t count, link_count; -}; - -/* - uidlist file: - - struct uidlist_header; - - // size includes both prev_offset and uidlist - packed (size << 2) | packed_flags; // UIDLIST_PACKED_FLAG_* - [packed prev_offset;] // If UIDLIST_PACKED_FLAG_BEGINS_WITH_OFFSET is set - if (UIDLIST_PACKED_FLAG_BITMASK) { - packed base_uid; // first UID in uidlist - uint8_t bitmask[]; // first bit is base_uid+1 - } else { - // FIXME: packed range - } -*/ - -#define UIDLIST_IS_SINGLETON(idx) \ - (((idx) & 1) != 0 || (idx) < (0x100 << 1)) - -struct squat_uidlist *squat_uidlist_init(struct squat_trie *trie); -void squat_uidlist_deinit(struct squat_uidlist *uidlist); - -int squat_uidlist_refresh(struct squat_uidlist *uidlist); - -int squat_uidlist_build_init(struct squat_uidlist *uidlist, - struct squat_uidlist_build_context **ctx_r); -uint32_t squat_uidlist_build_add_uid(struct squat_uidlist_build_context *ctx, - uint32_t uid_list_idx, uint32_t uid); -void squat_uidlist_build_flush(struct squat_uidlist_build_context *ctx); -int squat_uidlist_build_finish(struct squat_uidlist_build_context *ctx); -void squat_uidlist_build_deinit(struct squat_uidlist_build_context **ctx); - -int squat_uidlist_rebuild_init(struct squat_uidlist_build_context *build_ctx, - bool compress, - struct squat_uidlist_rebuild_context **ctx_r); -uint32_t squat_uidlist_rebuild_next(struct squat_uidlist_rebuild_context *ctx, - const ARRAY_TYPE(uint32_t) *uids); -uint32_t squat_uidlist_rebuild_nextu(struct squat_uidlist_rebuild_context *ctx, - const ARRAY_TYPE(seq_range) *uids); -int squat_uidlist_rebuild_finish(struct squat_uidlist_rebuild_context *ctx, - bool cancel); - -int squat_uidlist_get(struct squat_uidlist *uidlist, uint32_t uid_list_idx, - ARRAY_TYPE(uint32_t) *uids); -uint32_t squat_uidlist_singleton_last_uid(uint32_t uid_list_idx); - -int squat_uidlist_get_seqrange(struct squat_uidlist *uidlist, - uint32_t uid_list_idx, - ARRAY_TYPE(seq_range) *seq_range_arr); -int squat_uidlist_filter(struct squat_uidlist *uidlist, uint32_t uid_list_idx, - ARRAY_TYPE(seq_range) *uids); - -void squat_uidlist_delete(struct squat_uidlist *uidlist); -size_t squat_uidlist_mem_used(struct squat_uidlist *uidlist, - unsigned int *count_r); - -#endif