From: Eric Wong <e@80x24.org>
Date: Thu, 27 Feb 2025 00:08:18 +0000 (+0000)
Subject: imap: support external xap_helper process
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=09417bb8c68941bd3501016ac394c9557af00ed7;p=thirdparty%2Fpublic-inbox.git

imap: support external xap_helper process

Moving Xapian requests to an external process prevents expensive
searches from affecting non-search IMAP commands or any other
non-search requests within a -netd process if IMAP is sharing a
process with other public-facing protocols.
---

diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm
index 742497841..e72c646c4 100644
--- a/lib/PublicInbox/IMAP.pm
+++ b/lib/PublicInbox/IMAP.pm
@@ -1091,6 +1091,18 @@ sub parse_imap_query ($$) {
 	$q;
 }
 
+sub mset_cb { # async_mset cb
+	my ($self, $tag, $want_msn, $srch, $opt, $mset, $err) = @_;
+	$self->write($err ? do {
+		warn "IMAP search error: $err\n";
+		\"$tag BAD program fault - command not performed\r\n"
+	} : do {
+		my $uids = $srch->mset_to_artnums($mset, $opt);
+		msn_convert($self, $uids) if scalar(@$uids) && $want_msn;
+		\"* SEARCH @$uids\r\n$tag OK Search done\r\n"
+	});
+}
+
 sub search_common {
 	my ($self, $tag, $query, $want_msn) = @_;
 	my $ibx = $self->{ibx} or return "$tag BAD No mailbox selected\r\n";
@@ -1109,10 +1121,9 @@ sub search_common {
 			limit => UID_SLICE,
 			uid_range => $range_info
 		};
-		my $mset = $srch->mset($q, $opt);
-		my $uids = $srch->mset_to_artnums($mset, $opt);
-		msn_convert($self, $uids) if scalar(@$uids) && $want_msn;
-		"* SEARCH @$uids\r\n$tag OK Search done\r\n";
+		$srch->async_mset($q, $opt, \&mset_cb,
+					$self, $tag, $want_msn, $srch, $opt) ?
+				undef : '';
 	} else {
 		"$tag BAD Error\r\n";
 	}
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 98ad50343..52e1f3356 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -92,10 +92,12 @@ our @XH_SPEC = (
 	'o=i', # offset
 	'r', # 1=relevance then column
 	't', # collapse threads
+	'u=i', # IMAP UID min
 	'A=s@', # prefixes
 	'K=i', # timeout kill after i seconds
 	'O=s', # eidx_key
 	'T=i', # threadid
+	'U=i', # IMAP UID max
 	'Q=s@', # query prefixes "$user_prefix[:=]$XPREFIX"
 );
 
@@ -471,6 +473,9 @@ sub xh_opt ($$) {
 	push @ret, '-t' if $opt->{threads};
 	push @ret, '-T', $opt->{threadid} if defined $opt->{threadid};
 	push @ret, '-O', $opt->{eidx_key} if defined $opt->{eidx_key};
+	if (my $uid = $opt->{uid_range}) {
+		push @ret, '-u', $uid->[0], '-U', $uid->[1];
+	}
 	@ret;
 }
 
diff --git a/lib/PublicInbox/XapHelper.pm b/lib/PublicInbox/XapHelper.pm
index 15abed796..d3c98f031 100644
--- a/lib/PublicInbox/XapHelper.pm
+++ b/lib/PublicInbox/XapHelper.pm
@@ -162,6 +162,8 @@ sub cmd_mset { # to be used by WWW + IMAP
 	$opt->{threads} = 1 if defined $req->{t};
 	$opt->{git_dir} = $req->{g} if defined $req->{g};
 	$opt->{eidx_key} = $req->{O} if defined $req->{O};
+	my @uid_range = @$req{qw(u U)};
+	$opt->{uid_range} = \@uid_range if grep(defined, @uid_range) == 2;
 	$opt->{threadid} = $req->{T} if defined $req->{T};
 	my $mset = $req->{srch}->mset($qry_str, $opt);
 	say { $req->{0} } 'mset.size=', $mset->size,
diff --git a/lib/PublicInbox/XapHelperCxx.pm b/lib/PublicInbox/XapHelperCxx.pm
index 817abcfb4..47807f9b6 100644
--- a/lib/PublicInbox/XapHelperCxx.pm
+++ b/lib/PublicInbox/XapHelperCxx.pm
@@ -35,6 +35,7 @@ my $ldflags = '-Wl,-O1';
 $ldflags .= ' -Wl,--compress-debug-sections=zlib' if $^O ne 'openbsd';
 my $xflags = ($ENV{CXXFLAGS} // '-Wall -ggdb3 -pipe') . ' ' .
 	' -DTHREADID=' . PublicInbox::Search::THREADID .
+	' -DUID=' . PublicInbox::Search::UID .
 	' -DSHARD_COST=' . PublicInbox::Search::SHARD_COST .
 	' -DXH_SPEC="'.join('',
 		map { s/=.*/:/; $_ } @PublicInbox::Search::XH_SPEC) . '" ' .
diff --git a/lib/PublicInbox/xap_helper.h b/lib/PublicInbox/xap_helper.h
index 9c8436dc7..692765caa 100644
--- a/lib/PublicInbox/xap_helper.h
+++ b/lib/PublicInbox/xap_helper.h
@@ -185,6 +185,7 @@ struct req { // argv and pfxv point into global rbuf
 	unsigned long long max;
 	unsigned long long off;
 	unsigned long long threadid;
+	unsigned long long uid_min, uid_max;
 	unsigned long timeout_sec;
 	size_t nr_out;
 	long sort_col; // value column, negative means BoolWeight
@@ -277,7 +278,14 @@ static Xapian::MSet mail_mset(struct req *req, const char *qry_str)
 		qry = Xapian::Query(Xapian::Query::OP_FILTER, qry,
 					Xapian::Query(req->Oeidx_key));
 	}
-	// TODO: uid_range
+	// THREADID and UID are CPP macros defined by XapHelperCxx.pm
+	if (req->uid_min != ULLONG_MAX && req->uid_max != ULLONG_MAX) {
+		Xapian::Query range = Xapian::Query(
+				Xapian::Query::OP_VALUE_RANGE, UID,
+				Xapian::sortable_serialise(req->uid_min),
+				Xapian::sortable_serialise(req->uid_max));
+		qry = Xapian::Query(Xapian::Query::OP_FILTER, qry, range);
+	}
 	if (req->threadid != ULLONG_MAX) {
 		std::string tid = Xapian::sortable_serialise(req->threadid);
 		qry = Xapian::Query(Xapian::Query::OP_FILTER, qry,
@@ -286,7 +294,6 @@ static Xapian::MSet mail_mset(struct req *req, const char *qry_str)
 	}
 	Xapian::Enquire enq = prep_enquire(req);
 	enq.set_query(qry);
-	// THREADID is a CPP macro defined on CLI (see) XapHelperCxx.pm
 	if (req->collapse_threads && has_threadid(srch))
 		enq.set_collapse_key(THREADID);
 
@@ -675,6 +682,11 @@ static void srch_init_extra(struct req *req)
 	req->srch->qp_extra_done = true;
 }
 
+#define OPT_U(ch, var, fn, max) do { \
+	var = fn(optarg, &end, 10); \
+	if (*end || var == max) ABORT("-"#ch" %s", optarg); \
+} while (0)
+
 static void dispatch(struct req *req)
 {
 	int c;
@@ -685,7 +697,7 @@ static void dispatch(struct req *req)
 	} kbuf;
 	char *end;
 	FILE *kfp;
-	req->threadid = ULLONG_MAX;
+	req->threadid = req->uid_min = req->uid_max = ULLONG_MAX;
 	for (c = 0; c < (int)MY_ARRAY_SIZE(cmds); c++) {
 		if (cmds[c].fn_len == size &&
 			!memcmp(cmds[c].fn_name, req->argv[0], size)) {
@@ -723,34 +735,20 @@ static void dispatch(struct req *req)
 			case LONG_MAX: case LONG_MIN: ABORT("-k %s", optarg);
 			}
 			break;
-		case 'm':
-			req->max = strtoull(optarg, &end, 10);
-			if (*end || req->max == ULLONG_MAX)
-				ABORT("-m %s", optarg);
-			break;
-		case 'o':
-			req->off = strtoull(optarg, &end, 10);
-			if (*end || req->off == ULLONG_MAX)
-				ABORT("-o %s", optarg);
-			break;
+		case 'm': OPT_U(m, req->max, strtoull, ULLONG_MAX); break;
+		case 'o': OPT_U(o, req->off, strtoull, ULLONG_MAX); break;
 		case 'r': req->relevance = true; break;
 		case 't': req->collapse_threads = true; break;
+		case 'u': OPT_U(u, req->uid_min, strtoull, ULLONG_MAX); break;
 		case 'A':
 			req->pfxv[req->pfxc++] = optarg;
 			if (MY_ARG_MAX == req->pfxc)
 				ABORT("too many -A");
 			break;
-		case 'K':
-			req->timeout_sec = strtoul(optarg, &end, 10);
-			if (*end || req->timeout_sec == ULONG_MAX)
-				ABORT("-K %s", optarg);
-			break;
+		case 'K': OPT_U(K, req->timeout_sec, strtoul, ULONG_MAX); break;
 		case 'O': req->Oeidx_key = optarg - 1; break; // pad "O" prefix
-		case 'T':
-			req->threadid = strtoull(optarg, &end, 10);
-			if (*end || req->threadid == ULLONG_MAX)
-				ABORT("-T %s", optarg);
-			break;
+		case 'T': OPT_U(T, req->threadid, strtoull, ULLONG_MAX); break;
+		case 'U': OPT_U(U, req->uid_max, strtoull, ULLONG_MAX); break;
 		case 'Q':
 			req->qpfxv[req->qpfxc++] = optarg;
 			if (MY_ARG_MAX == req->qpfxc) ABORT("too many -Q");