imap: support external xap_helper process

Moving Xapian requests to an external process prevents expensive
searches from affecting non-search IMAP commands or any other
non-search requests within a -netd process if IMAP is sharing a
process with other public-facing protocols.

diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm
index 742497841b8473c13450b3992624f264d5773f9e..e72c646c4fb68309905035eb439c1a8afed11e87 100644 (file)
--- a/lib/PublicInbox/IMAP.pm
+++ b/lib/PublicInbox/IMAP.pm
@@ -1091,6 +1091,18 @@ sub parse_imap_query ($$) {
        $q;
 }
 
+sub mset_cb { # async_mset cb
+       my ($self, $tag, $want_msn, $srch, $opt, $mset, $err) = @_;
+       $self->write($err ? do {
+               warn "IMAP search error: $err\n";
+               \"$tag BAD program fault - command not performed\r\n"
+       } : do {
+               my $uids = $srch->mset_to_artnums($mset, $opt);
+               msn_convert($self, $uids) if scalar(@$uids) && $want_msn;
+               \"* SEARCH @$uids\r\n$tag OK Search done\r\n"
+       });
+}
+
 sub search_common {
        my ($self, $tag, $query, $want_msn) = @_;
        my $ibx = $self->{ibx} or return "$tag BAD No mailbox selected\r\n";
@@ -1109,10 +1121,9 @@ sub search_common {
                        limit => UID_SLICE,
                        uid_range => $range_info
                };
-               my $mset = $srch->mset($q, $opt);
-               my $uids = $srch->mset_to_artnums($mset, $opt);
-               msn_convert($self, $uids) if scalar(@$uids) && $want_msn;
-               "* SEARCH @$uids\r\n$tag OK Search done\r\n";
+               $srch->async_mset($q, $opt, \&mset_cb,
+                                       $self, $tag, $want_msn, $srch, $opt) ?
+                               undef : '';
        } else {
                "$tag BAD Error\r\n";
        }

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 98ad503438b15116a11afc33a8161f00fe9caa3d..52e1f33568ee21cf410ffd7933185e743c424b63 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -92,10 +92,12 @@ our @XH_SPEC = (
        'o=i', # offset
        'r', # 1=relevance then column
        't', # collapse threads
+       'u=i', # IMAP UID min
        'A=s@', # prefixes
        'K=i', # timeout kill after i seconds
        'O=s', # eidx_key
        'T=i', # threadid
+       'U=i', # IMAP UID max
        'Q=s@', # query prefixes "$user_prefix[:=]$XPREFIX"
 );
 
@@ -471,6 +473,9 @@ sub xh_opt ($$) {
        push @ret, '-t' if $opt->{threads};
        push @ret, '-T', $opt->{threadid} if defined $opt->{threadid};
        push @ret, '-O', $opt->{eidx_key} if defined $opt->{eidx_key};
+       if (my $uid = $opt->{uid_range}) {
+               push @ret, '-u', $uid->[0], '-U', $uid->[1];
+       }
        @ret;
 }
 

diff --git a/lib/PublicInbox/XapHelper.pm b/lib/PublicInbox/XapHelper.pm
index 15abed796c4822f8037daa1b41e4a07083826a24..d3c98f031d8a2d0c2b5d81c5d6f5a0814331ba82 100644 (file)
--- a/lib/PublicInbox/XapHelper.pm
+++ b/lib/PublicInbox/XapHelper.pm
@@ -162,6 +162,8 @@ sub cmd_mset { # to be used by WWW + IMAP
        $opt->{threads} = 1 if defined $req->{t};
        $opt->{git_dir} = $req->{g} if defined $req->{g};
        $opt->{eidx_key} = $req->{O} if defined $req->{O};
+       my @uid_range = @$req{qw(u U)};
+       $opt->{uid_range} = \@uid_range if grep(defined, @uid_range) == 2;
        $opt->{threadid} = $req->{T} if defined $req->{T};
        my $mset = $req->{srch}->mset($qry_str, $opt);
        say { $req->{0} } 'mset.size=', $mset->size,

diff --git a/lib/PublicInbox/XapHelperCxx.pm b/lib/PublicInbox/XapHelperCxx.pm
index 817abcfb417b7ba9f965c63f7150a6385d8a3f61..47807f9b6078c6ed50ab05413c812a282824efe2 100644 (file)
--- a/lib/PublicInbox/XapHelperCxx.pm
+++ b/lib/PublicInbox/XapHelperCxx.pm
@@ -35,6 +35,7 @@ my $ldflags = '-Wl,-O1';
 $ldflags .= ' -Wl,--compress-debug-sections=zlib' if $^O ne 'openbsd';
 my $xflags = ($ENV{CXXFLAGS} // '-Wall -ggdb3 -pipe') . ' ' .
        ' -DTHREADID=' . PublicInbox::Search::THREADID .
+       ' -DUID=' . PublicInbox::Search::UID .
        ' -DSHARD_COST=' . PublicInbox::Search::SHARD_COST .
        ' -DXH_SPEC="'.join('',
                map { s/=.*/:/; $_ } @PublicInbox::Search::XH_SPEC) . '" ' .

diff --git a/lib/PublicInbox/xap_helper.h b/lib/PublicInbox/xap_helper.h
index 9c8436dc7e089222aa32a7b95a57361b86dd0779..692765caa059ac7d5a3ce94fbd98d557820fe570 100644 (file)
--- a/lib/PublicInbox/xap_helper.h
+++ b/lib/PublicInbox/xap_helper.h
@@ -185,6 +185,7 @@ struct req { // argv and pfxv point into global rbuf
        unsigned long long max;
        unsigned long long off;
        unsigned long long threadid;
+       unsigned long long uid_min, uid_max;
        unsigned long timeout_sec;
        size_t nr_out;
        long sort_col; // value column, negative means BoolWeight
@@ -277,7 +278,14 @@ static Xapian::MSet mail_mset(struct req *req, const char *qry_str)
                qry = Xapian::Query(Xapian::Query::OP_FILTER, qry,
                                        Xapian::Query(req->Oeidx_key));
        }
-       // TODO: uid_range
+       // THREADID and UID are CPP macros defined by XapHelperCxx.pm
+       if (req->uid_min != ULLONG_MAX && req->uid_max != ULLONG_MAX) {
+               Xapian::Query range = Xapian::Query(
+                               Xapian::Query::OP_VALUE_RANGE, UID,
+                               Xapian::sortable_serialise(req->uid_min),
+                               Xapian::sortable_serialise(req->uid_max));
+               qry = Xapian::Query(Xapian::Query::OP_FILTER, qry, range);
+       }
        if (req->threadid != ULLONG_MAX) {
                std::string tid = Xapian::sortable_serialise(req->threadid);
                qry = Xapian::Query(Xapian::Query::OP_FILTER, qry,
@@ -286,7 +294,6 @@ static Xapian::MSet mail_mset(struct req *req, const char *qry_str)
        }
        Xapian::Enquire enq = prep_enquire(req);
        enq.set_query(qry);
-       // THREADID is a CPP macro defined on CLI (see) XapHelperCxx.pm
        if (req->collapse_threads && has_threadid(srch))
                enq.set_collapse_key(THREADID);
 
@@ -675,6 +682,11 @@ static void srch_init_extra(struct req *req)
        req->srch->qp_extra_done = true;
 }
 
+#define OPT_U(ch, var, fn, max) do { \
+       var = fn(optarg, &end, 10); \
+       if (*end || var == max) ABORT("-"#ch" %s", optarg); \
+} while (0)
+
 static void dispatch(struct req *req)
 {
        int c;
@@ -685,7 +697,7 @@ static void dispatch(struct req *req)
        } kbuf;
        char *end;
        FILE *kfp;
-       req->threadid = ULLONG_MAX;
+       req->threadid = req->uid_min = req->uid_max = ULLONG_MAX;
        for (c = 0; c < (int)MY_ARRAY_SIZE(cmds); c++) {
                if (cmds[c].fn_len == size &&
                        !memcmp(cmds[c].fn_name, req->argv[0], size)) {
@@ -723,34 +735,20 @@ static void dispatch(struct req *req)
                        case LONG_MAX: case LONG_MIN: ABORT("-k %s", optarg);
                        }
                        break;
-               case 'm':
-                       req->max = strtoull(optarg, &end, 10);
-                       if (*end || req->max == ULLONG_MAX)
-                               ABORT("-m %s", optarg);
-                       break;
-               case 'o':
-                       req->off = strtoull(optarg, &end, 10);
-                       if (*end || req->off == ULLONG_MAX)
-                               ABORT("-o %s", optarg);
-                       break;
+               case 'm': OPT_U(m, req->max, strtoull, ULLONG_MAX); break;
+               case 'o': OPT_U(o, req->off, strtoull, ULLONG_MAX); break;
                case 'r': req->relevance = true; break;
                case 't': req->collapse_threads = true; break;
+               case 'u': OPT_U(u, req->uid_min, strtoull, ULLONG_MAX); break;
                case 'A':
                        req->pfxv[req->pfxc++] = optarg;
                        if (MY_ARG_MAX == req->pfxc)
                                ABORT("too many -A");
                        break;
-               case 'K':
-                       req->timeout_sec = strtoul(optarg, &end, 10);
-                       if (*end || req->timeout_sec == ULONG_MAX)
-                               ABORT("-K %s", optarg);
-                       break;
+               case 'K': OPT_U(K, req->timeout_sec, strtoul, ULONG_MAX); break;
                case 'O': req->Oeidx_key = optarg - 1; break; // pad "O" prefix
-               case 'T':
-                       req->threadid = strtoull(optarg, &end, 10);
-                       if (*end || req->threadid == ULLONG_MAX)
-                               ABORT("-T %s", optarg);
-                       break;
+               case 'T': OPT_U(T, req->threadid, strtoull, ULLONG_MAX); break;
+               case 'U': OPT_U(U, req->uid_max, strtoull, ULLONG_MAX); break;
                case 'Q':
                        req->qpfxv[req->qpfxc++] = optarg;
                        if (MY_ARG_MAX == req->qpfxc) ABORT("too many -Q");