From: Michael M Slusarz Date: Wed, 23 Feb 2022 16:33:44 +0000 (+0000) Subject: fts-flatcurve: Add doveadm plugin X-Git-Tag: 2.4.0~4107 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f40d40d4a8ffb26280875c0022185dfab6c6745c;p=thirdparty%2Fdovecot%2Fcore.git fts-flatcurve: Add doveadm plugin --- diff --git a/doc/man/doveadm-dump.1.in b/doc/man/doveadm-dump.1.in index d3ac7c2c41..e5ea106398 100644 --- a/doc/man/doveadm-dump.1.in +++ b/doc/man/doveadm-dump.1.in @@ -45,6 +45,11 @@ can be: Dump the list of expunged mails in .IR dovecot\-expunges.log . .TP +.B fts\-flatcurve +Dump the keywords indexed in +.I fts\-flatcurve +indexes directory and their frequencies. +.TP .B imapzlib Uncompress an IMAP traffic log, which contains data compressed using the IMAP COMPRESSION extension. diff --git a/src/plugins/fts-flatcurve/Makefile.am b/src/plugins/fts-flatcurve/Makefile.am index 3899cc9312..2f93774c2a 100644 --- a/src/plugins/fts-flatcurve/Makefile.am +++ b/src/plugins/fts-flatcurve/Makefile.am @@ -7,7 +7,10 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib-index \ -I$(top_srcdir)/src/lib-settings \ -I$(top_srcdir)/src/lib-storage \ - -I$(top_srcdir)/src/plugins/fts + -I$(top_srcdir)/src/lib-doveadm \ + -I$(top_srcdir)/src/doveadm \ + -I$(top_srcdir)/src/plugins/fts \ + $(XAPIAN_CXXFLAGS) AM_CXXFLAGS = \ $(XAPIAN_CXXFLAGS) @@ -31,6 +34,18 @@ lib21_fts_flatcurve_plugin_la_SOURCES = \ fts-backend-flatcurve-xapian.cc noinst_HEADERS = \ + doveadm-dump-flatcurve.h \ fts-flatcurve-plugin.h \ fts-backend-flatcurve.h \ fts-backend-flatcurve-xapian.h + +libdoveadm_fts_flatcurve_plugin_la_SOURCES = \ + doveadm-dump-flatcurve.c \ + doveadm-fts-flatcurve.c +libdoveadm_fts_flatcurve_plugin_la_LIBADD = $(LIBDOVECOT) +libdoveadm_fts_flatcurve_plugin_la_DEPENDENCIES = $(LIBDOVECOT_DEPS) +libdoveadm_fts_flatcurve_plugin_la_LDFLAGS = -module -avoid-version + +doveadm_moduledir = $(moduledir)/doveadm +doveadm_module_LTLIBRARIES = \ + libdoveadm_fts_flatcurve_plugin.la \ No newline at end of file diff --git a/src/plugins/fts-flatcurve/doveadm-dump-flatcurve.c b/src/plugins/fts-flatcurve/doveadm-dump-flatcurve.c new file mode 100644 index 0000000000..4835605cc8 --- /dev/null +++ b/src/plugins/fts-flatcurve/doveadm-dump-flatcurve.c @@ -0,0 +1,140 @@ +/* Copyright (c) the Dovecot authors, based on code by Michael Slusarz. + * See the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "doveadm-dump.h" +#include "doveadm-print.h" +#include "fts-backend-flatcurve.h" +#include "fts-backend-flatcurve-xapian.h" +#include "doveadm-dump-flatcurve.h" + +#define HEADER_TERMS TRUE +#define PAYLOAD_TERMS !HEADER_TERMS + +ARRAY_DEFINE_TYPE(fts_flatcurve_dump_term, struct fts_flatcurve_dump_term *); +struct fts_flatcurve_dump_term { + bool header; + const char *term; + unsigned int count; +}; + +static int +cmd_fts_flatcurve_dump_sort(struct fts_flatcurve_dump_term *const *p_lhs, + struct fts_flatcurve_dump_term *const *p_rhs) +{ + const struct fts_flatcurve_dump_term *lhs = *p_lhs; + const struct fts_flatcurve_dump_term *rhs = *p_rhs; + + int ret; + + ret = (int)rhs->count - (int)lhs->count; + if (ret != 0) return ret; + + ret = (int)rhs->header - (int)lhs->header; + if (ret != 0) return ret; + + ret = strcmp(lhs->term, rhs->term); + return ret; +} + +static void +cmd_fts_flatcurve_dump_array_push(bool header, + HASH_TABLE_TYPE(term_counter) *hterms, + ARRAY_TYPE(fts_flatcurve_dump_term) *aterms) +{ + char *key; + void *val; + struct fts_flatcurve_dump_term *term; + struct hash_iterate_context *iter = hash_table_iterate_init(*hterms); + while (hash_table_iterate(iter, *hterms, &key, &val)) { + term = t_new(struct fts_flatcurve_dump_term, 1); + term->header = header; + term->term = key; + term->count = POINTER_CAST_TO(val, unsigned int); + array_push_back(aterms, &term); + } + hash_table_iterate_deinit(&iter); +} + +static int +cmd_dump_fts_flatcurve_dump_terms(bool headers, const char *path, + ARRAY_TYPE(fts_flatcurve_dump_term) *aterms, + const char **error_r) +{ + HASH_TABLE_TYPE(term_counter) hterms; + hash_table_create(&hterms, pool_datastack_create(), 256, str_hash, strcmp); + int ret = fts_flatcurve_database_terms(headers, path, &hterms, error_r); + cmd_fts_flatcurve_dump_array_push(headers, &hterms, aterms); + hash_table_destroy(&hterms); + return ret; +} + +static void +cmd_dump_fts_flatcurve_print_terms(ARRAY_TYPE(fts_flatcurve_dump_term) *terms) +{ + doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW); + doveadm_print_header("terms", "terms", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + + struct fts_flatcurve_dump_term *term; + array_foreach_elem(terms, term) { + T_BEGIN { + doveadm_print(t_strdup_printf( + "%s(%u)\t%s", + term->header ? "H": "P", + term->count, term->term)); + } T_END; + } + doveadm_print_deinit(); +} + +static void +cmd_dump_fts_flatcurve_bundle(const char *arg_path, + const char *const *args ATTR_UNUSED) +{ + T_BEGIN { + const char *index_path, *error; + if (fts_flatcurve_database_locate_dir( + arg_path, &index_path, &error) < 0) + i_fatal("Can't use filename as FTS: %s - %s", + arg_path, error); + + ARRAY_TYPE(fts_flatcurve_dump_term) terms; + t_array_init(&terms, 256); + + if (cmd_dump_fts_flatcurve_dump_terms( + HEADER_TERMS, index_path, &terms, &error) < 0) + i_fatal("%s", error); + + if (cmd_dump_fts_flatcurve_dump_terms( + PAYLOAD_TERMS, index_path, &terms, &error) < 0) + i_fatal("%s", error); + + array_sort(&terms, cmd_fts_flatcurve_dump_sort); + cmd_dump_fts_flatcurve_print_terms(&terms); + } T_END; +} + +static bool +test_dump_fts_flatcurve_bundle(const char *arg_path) +{ + bool located; + T_BEGIN { + const char *index_path, *error; + located = fts_flatcurve_database_locate_dir( + arg_path, &index_path, &error) == 0; + } T_END; + return located; +} + +static const struct doveadm_cmd_dump doveadm_cmd_dump_fts_flatcurve_bundle = { + FTS_FLATCURVE_LABEL, + test_dump_fts_flatcurve_bundle, + cmd_dump_fts_flatcurve_bundle +}; + +void doveadm_dump_flatcurve_init(void) +{ + doveadm_dump_register(&doveadm_cmd_dump_fts_flatcurve_bundle); +} diff --git a/src/plugins/fts-flatcurve/doveadm-dump-flatcurve.h b/src/plugins/fts-flatcurve/doveadm-dump-flatcurve.h new file mode 100644 index 0000000000..062a9d1139 --- /dev/null +++ b/src/plugins/fts-flatcurve/doveadm-dump-flatcurve.h @@ -0,0 +1,9 @@ +/* Copyright (c) the Dovecot authors, based on code by Michael Slusarz. + * See the included COPYING file */ + +#ifndef DOVEADM_DUMP_FLATCURVE_H +#define DOVEADM_DUMP_FLATCURVE_H + +void doveadm_dump_flatcurve_init(void); + +#endif diff --git a/src/plugins/fts-flatcurve/doveadm-fts-flatcurve.c b/src/plugins/fts-flatcurve/doveadm-fts-flatcurve.c new file mode 100644 index 0000000000..3a075e4269 --- /dev/null +++ b/src/plugins/fts-flatcurve/doveadm-fts-flatcurve.c @@ -0,0 +1,310 @@ +/* Copyright (c) the Dovecot authors, based on code by Michael Slusarz. + * See the included COPYING file */ + +#include "lib.h" +#include "doveadm-mail.h" +#include "doveadm-mailbox-list-iter.h" +#include "doveadm-print.h" +#include "doveadm-dump-flatcurve.h" +#include "mail-search.h" +#include "str.h" +#include "fts-backend-flatcurve.h" +#include "fts-backend-flatcurve-xapian.h" + +#define DOVEADM_FLATCURVE_CMD_NAME_CHECK "fts flatcurve check" +#define DOVEADM_FLATCURVE_CMD_NAME_REMOVE "fts flatcurve remove" +#define DOVEADM_FLATCURVE_CMD_NAME_ROTATE "fts flatcurve rotate" +#define DOVEADM_FLATCURVE_CMD_NAME_STATS "fts flatcurve stats" + +const char *doveadm_fts_flatcurve_plugin_version = DOVECOT_ABI_VERSION; + +void doveadm_fts_flatcurve_plugin_init(struct module *module); +void doveadm_fts_flatcurve_plugin_deinit(void); + +enum fts_flatcurve_cmd_type { + FTS_FLATCURVE_CMD_CHECK, + FTS_FLATCURVE_CMD_REMOVE, + FTS_FLATCURVE_CMD_ROTATE, + FTS_FLATCURVE_CMD_STATS +}; + +struct fts_flatcurve_mailbox_cmd_context { + struct doveadm_mail_cmd_context ctx; + enum fts_flatcurve_cmd_type cmd_type; + struct mail_search_args *search_args; +}; + +static int +cmd_fts_flatcurve_mailbox_run_box(struct flatcurve_fts_backend *backend, + struct fts_flatcurve_mailbox_cmd_context *ctx, + struct mailbox *box, const char **error_r) +{ + struct fts_flatcurve_xapian_db_check check; + struct fts_flatcurve_xapian_db_stats stats; + uint32_t last_uid; + + int ret = 0; + bool result = FALSE; + switch (ctx->cmd_type) { + case FTS_FLATCURVE_CMD_CHECK: { + ret = fts_flatcurve_xapian_mailbox_check( + backend, &check, error_r); + result = check.shards > 0; + break; + } + case FTS_FLATCURVE_CMD_REMOVE: + ret = fts_backend_flatcurve_delete_dir(str_c( + backend->db_path), error_r); + result = ret > 0; + break; + case FTS_FLATCURVE_CMD_ROTATE: + ret = fts_flatcurve_xapian_mailbox_rotate( + backend, error_r); + result = ret == 0; + break; + case FTS_FLATCURVE_CMD_STATS: + ret = fts_flatcurve_xapian_mailbox_stats( + backend, &stats, error_r); + if (ret > 0 && stats.version > 0) { + ret = fts_flatcurve_xapian_get_last_uid( + backend, &last_uid, error_r); + result = ret >= 0; + } + break; + default: + i_unreached(); + } + + if (ret < 0 || !result) + return ret; + + struct mailbox_metadata metadata; + const char *guid = mailbox_get_metadata( + box, MAILBOX_METADATA_GUID, &metadata) < 0 ? + "" : guid_128_to_string(metadata.guid); + doveadm_print(str_c(backend->boxname)); + doveadm_print(guid); + + switch (ctx->cmd_type) { + case FTS_FLATCURVE_CMD_CHECK: + doveadm_print_num(check.errors); + doveadm_print_num(check.shards); + break; + case FTS_FLATCURVE_CMD_STATS: + doveadm_print_num(last_uid); + doveadm_print_num(stats.messages); + doveadm_print_num(stats.shards); + doveadm_print_num(stats.version); + break; + default: + break; + } + return 0; +} + +static int +cmd_fts_flatcurve_mailbox_run_do(struct flatcurve_fts_backend *backend, + struct mail_user *user, + struct fts_flatcurve_mailbox_cmd_context *ctx) +{ + enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_SKIP_ALIASES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + + struct doveadm_mailbox_list_iter *iter = + doveadm_mailbox_list_iter_init(&ctx->ctx, user, + ctx->search_args, iter_flags); + + const char *error; + const struct mailbox_info *info; + int ret = 0; + while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) { + struct mailbox *box = doveadm_mailbox_find( + ctx->ctx.cur_mail_user, info->vname); + + if (fts_backend_flatcurve_set_mailbox(backend, box, &error) < 0 || + cmd_fts_flatcurve_mailbox_run_box(backend, ctx, box, &error) < 0 || + fts_backend_flatcurve_close_mailbox(backend, &error) < 0) { + e_error(backend->event, "%s", error); + ret = -1; + } + + mailbox_free(&box); + } + + if (doveadm_mailbox_list_iter_deinit(&iter) < 0) { + e_error(backend->event, + "doveadm_mailbox_list_iter_deinit() failed"); + return -1; + } + return ret; +} + +static int +cmd_fts_flatcurve_mailbox_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct fts_flatcurve_mailbox_cmd_context *ctx = + (struct fts_flatcurve_mailbox_cmd_context *)_ctx; + struct fts_flatcurve_user *fuser = + FTS_FLATCURVE_USER_CONTEXT(user); + struct flatcurve_fts_backend *backend = fuser->backend; + + if (fuser == NULL) { + e_error(backend->event, FTS_FLATCURVE_LABEL " not enabled"); + doveadm_mail_failed_error(_ctx, MAIL_ERROR_NOTFOUND); + _ctx->exit_code = EX_UNAVAILABLE; + return -1; + } + + doveadm_print_header("mailbox", "mailbox", + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + doveadm_print_header_simple("guid"); + + switch (ctx->cmd_type) { + case FTS_FLATCURVE_CMD_CHECK: + doveadm_print_header_simple("errors"); + doveadm_print_header_simple("shards"); + break; + case FTS_FLATCURVE_CMD_STATS: + doveadm_print_header_simple("last_uid"); + doveadm_print_header_simple("messages"); + doveadm_print_header_simple("shards"); + doveadm_print_header_simple("version"); + break; + default: + break; + } + + int ret = cmd_fts_flatcurve_mailbox_run_do(backend, user, ctx); + if (ret < 0) + _ctx->exit_code = EX_TEMPFAIL; + return ret; +} + +static void +cmd_fts_flatcurve_mailbox_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct fts_flatcurve_mailbox_cmd_context *ctx = + (struct fts_flatcurve_mailbox_cmd_context *)_ctx; + + if (args[0] == NULL) { + switch (ctx->cmd_type) { + case FTS_FLATCURVE_CMD_CHECK: + doveadm_mail_help_name(DOVEADM_FLATCURVE_CMD_NAME_CHECK); + break; + case FTS_FLATCURVE_CMD_REMOVE: + doveadm_mail_help_name(DOVEADM_FLATCURVE_CMD_NAME_REMOVE); + break; + case FTS_FLATCURVE_CMD_ROTATE: + doveadm_mail_help_name(DOVEADM_FLATCURVE_CMD_NAME_ROTATE); + break; + case FTS_FLATCURVE_CMD_STATS: + doveadm_mail_help_name(DOVEADM_FLATCURVE_CMD_NAME_STATS); + break; + default: + i_unreached(); + } + } + + ctx->search_args = doveadm_mail_mailbox_search_args_build(args); +} + +static void +cmd_fts_flatcurve_mailbox_deinit(struct doveadm_mail_cmd_context *_ctx) +{ + struct fts_flatcurve_mailbox_cmd_context *ctx = + container_of(_ctx, struct fts_flatcurve_mailbox_cmd_context, ctx); + + if (ctx->search_args != NULL) + mail_search_args_unref(&ctx->search_args); +} + +static struct doveadm_mail_cmd_context * +cmd_fts_flatcurve_mailbox_alloc(enum fts_flatcurve_cmd_type type) +{ + struct fts_flatcurve_mailbox_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct fts_flatcurve_mailbox_cmd_context); + ctx->ctx.v.init = cmd_fts_flatcurve_mailbox_init; + ctx->ctx.v.deinit = cmd_fts_flatcurve_mailbox_deinit; + ctx->ctx.v.run = cmd_fts_flatcurve_mailbox_run; + ctx->cmd_type = type; + + doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW); + + return &ctx->ctx; +} + +static struct doveadm_mail_cmd_context *cmd_fts_flatcurve_check_alloc(void) +{ + return cmd_fts_flatcurve_mailbox_alloc(FTS_FLATCURVE_CMD_CHECK); +} + +static struct doveadm_mail_cmd_context *cmd_fts_flatcurve_remove_alloc(void) +{ + return cmd_fts_flatcurve_mailbox_alloc(FTS_FLATCURVE_CMD_REMOVE); +} + +static struct doveadm_mail_cmd_context *cmd_fts_flatcurve_rotate_alloc(void) +{ + return cmd_fts_flatcurve_mailbox_alloc(FTS_FLATCURVE_CMD_ROTATE); +} + +static struct doveadm_mail_cmd_context *cmd_fts_flatcurve_stats_alloc(void) +{ + return cmd_fts_flatcurve_mailbox_alloc(FTS_FLATCURVE_CMD_STATS); +} + +static struct doveadm_cmd_ver2 fts_flatcurve_commands[] = { + { + .name = DOVEADM_FLATCURVE_CMD_NAME_CHECK, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "", + .mail_cmd = cmd_fts_flatcurve_check_alloc, +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "mailbox-mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END + }, + { + .name = DOVEADM_FLATCURVE_CMD_NAME_REMOVE, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "", + .mail_cmd = cmd_fts_flatcurve_remove_alloc, +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "mailbox-mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END + }, + { + .name = DOVEADM_FLATCURVE_CMD_NAME_ROTATE, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "", + .mail_cmd = cmd_fts_flatcurve_rotate_alloc, +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "mailbox-mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END + }, + { + .name = DOVEADM_FLATCURVE_CMD_NAME_STATS, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "", + .mail_cmd = cmd_fts_flatcurve_stats_alloc, +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "mailbox-mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END + } +}; + +void doveadm_fts_flatcurve_plugin_init(struct module *module ATTR_UNUSED) +{ + doveadm_dump_flatcurve_init(); + for (unsigned int index = 0; + index < N_ELEMENTS(fts_flatcurve_commands); index++) + doveadm_cmd_register_ver2(fts_flatcurve_commands + index); +} + +void doveadm_fts_flatcurve_plugin_deinit(void) +{ +}