fts: Retry indexing for tika backend in case of internal server error

diff --git a/src/plugins/fts/fts-api-private.h b/src/plugins/fts/fts-api-private.h
index 2fdc40282b1b6e8148f634d6326eab924359ea86..a2eac7a099a1b8e5b10c95188f9ac7c4d9434313 100644 (file)
--- a/src/plugins/fts/fts-api-private.h
+++ b/src/plugins/fts/fts-api-private.h
@@ -84,7 +84,6 @@ struct fts_backend_update_context {
        normalizer_func_t *normalizer;
 
        struct mailbox *cur_box, *backend_box;
-       const char *error_msg;
 
        bool build_key_open:1;
        bool failed:1;

diff --git a/src/plugins/fts/fts-build-mail.c b/src/plugins/fts/fts-build-mail.c
index 611b03ee66f6b543bc2c89c7d3e627d10018340f..5c7eaae89443ee591da73ca88cfb7b890aca8b2a 100644 (file)
--- a/src/plugins/fts/fts-build-mail.c
+++ b/src/plugins/fts/fts-build-mail.c
@@ -435,10 +435,14 @@ static int fts_build_body_block(struct fts_mail_build_context *ctx,
        return fts_build_data(ctx, block->data, block->size, last);
 }
 
-static int fts_body_parser_finish(struct fts_mail_build_context *ctx)
+static int fts_body_parser_finish(struct fts_mail_build_context *ctx,
+                                 const char **error_msg_r,
+                                 bool *may_need_retry_r)
 {
        struct message_block block;
        int ret = 0;
+       int deinit_ret;
+       *may_need_retry_r = FALSE;
 
        do {
                i_zero(&block);
@@ -449,14 +453,19 @@ static int fts_body_parser_finish(struct fts_mail_build_context *ctx)
                }
        } while (block.size > 0);
 
-       if (fts_parser_deinit(&ctx->body_parser, &ctx->update_ctx->error_msg) < 0)
+       deinit_ret = fts_parser_deinit(&ctx->body_parser, error_msg_r);
+       if (deinit_ret == 0)
+               *may_need_retry_r = TRUE;
+       else if (deinit_ret < 0)
                ret = -1;
        return ret;
 }
 
 static int
 fts_build_mail_real(struct fts_backend_update_context *update_ctx,
-                   struct mail *mail)
+                   struct mail *mail,
+                   const char **error_msg,
+                   bool *may_need_retry)
 {
        struct fts_mail_build_context ctx;
        struct istream *input;
@@ -508,7 +517,7 @@ fts_build_mail_real(struct fts_backend_update_context *update_ctx,
                        /* body part changed. we're now parsing the end of
                           boundary, possibly followed by message epilogue */
                        if (ctx.body_parser != NULL) {
-                               if (fts_body_parser_finish(&ctx) < 0) {
+                               if (fts_body_parser_finish(&ctx, error_msg, may_need_retry) < 0) {
                                        ret = -1;
                                        break;
                                }
@@ -565,7 +574,7 @@ fts_build_mail_real(struct fts_backend_update_context *update_ctx,
        }
        if (ctx.body_parser != NULL) {
                if (ret == 0)
-                       ret = fts_body_parser_finish(&ctx);
+                       ret = fts_body_parser_finish(&ctx, error_msg, may_need_retry);
                else
                        (void)fts_parser_deinit(&ctx.body_parser, NULL);
        }
@@ -588,9 +597,20 @@ int fts_build_mail(struct fts_backend_update_context *update_ctx,
                   struct mail *mail)
 {
        int ret;
+       /* Number of attempts to be taken if retry is needed */
+       unsigned int attempts = 2;
+       const char *error_msg;
+       bool may_need_retry = FALSE;
 
        T_BEGIN {
-               ret = fts_build_mail_real(update_ctx, mail);
+               for (;;) {
+                       ret = fts_build_mail_real(update_ctx, mail, &error_msg, &may_need_retry);
+                       if (!may_need_retry || (--attempts == 0)) {
+                               if (may_need_retry)
+                                       i_info("%s - ignoring", error_msg);
+                               break;
+                       }
+               }
        } T_END;
        return ret;
 }

diff --git a/src/plugins/fts/fts-parser-tika.c b/src/plugins/fts/fts-parser-tika.c
index 10a6af0e6c78b84f43710749bad8bd90a4c9cb86..7759fdaf3dd7b7548567ece81d2553c0bb1fac2b 100644 (file)
--- a/src/plugins/fts/fts-parser-tika.c
+++ b/src/plugins/fts/fts-parser-tika.c
@@ -110,19 +110,12 @@ fts_tika_parser_response(const struct http_response *response,
                parser->payload = i_stream_create_from_data("", 0);
                break;
        case 500:
-               /* Server Error - the problem could be anything (in Tika or
-                  HTTP server or proxy) and might be retriable, but Tika has
-                  trouble processing some documents and throws up this error
-                  every time for those documents.
-
-                  Unfortunately we can't easily re-send the request here,
-                  because we would have to re-send the entire payload, which
-                  isn't available anymore here. So we'd need to indicate
-                  in fts_parser_deinit() that we want to retry.
-                  FIXME: do this in v2.3. For now we'll just ignore it. */
-               i_info("fts_tika: PUT %s failed: %s - ignoring",
-                      mail_user_plugin_getenv(parser->user, "fts_tika"),
-                      http_response_get_message(response));
+               parser->parser.may_need_retry = TRUE;
+               i_free(parser->parser.retriable_error_msg);
+               parser->parser.retriable_error_msg =
+                       i_strdup_printf("fts_tika: PUT %s failed: %s",
+                                       mail_user_plugin_getenv(parser->user, "fts_tika"),
+                                       http_response_get_message(response));
                parser->payload = i_stream_create_from_data("", 0);
                break;
 
@@ -240,9 +233,10 @@ static void fts_parser_tika_more(struct fts_parser *_parser,
 static int fts_parser_tika_deinit(struct fts_parser *_parser, const char **retriable_err_msg_r)
 {
        struct tika_fts_parser *parser = (struct tika_fts_parser *)_parser;
-       int ret = parser->failed ? -1 : 0;
+       int ret = _parser->may_need_retry ? 0: (parser->failed ? -1 : 1);
        if (retriable_err_msg_r != NULL)
-               *retriable_err_msg_r = NULL;
+               *retriable_err_msg_r = t_strdup(_parser->retriable_error_msg);
+       i_free(_parser->retriable_error_msg);
 
        /* remove io before unrefing payload - otherwise lib-http adds another
           timeout to ioloop unnecessarily */

diff --git a/src/plugins/fts/fts-parser.h b/src/plugins/fts/fts-parser.h
index 2a871347fe0bd8b864696ae760722686bde6bee6..3574ea41effb8e33e1e4f8e0cd4da91335b48443 100644 (file)
--- a/src/plugins/fts/fts-parser.h
+++ b/src/plugins/fts/fts-parser.h
@@ -22,6 +22,8 @@ struct fts_parser_vfuncs {
 struct fts_parser {
        struct fts_parser_vfuncs v;
        buffer_t *utf8_output;
+       bool may_need_retry;
+       char *retriable_error_msg;
 };
 
 extern struct fts_parser_vfuncs fts_parser_html;