lib/PublicInbox/gcf2_libgit2.h
lib/PublicInbox/xap_helper.h
lib/PublicInbox/xh_cidx.h
+lib/PublicInbox/xh_mset.h
sa_config/Makefile
sa_config/README
sa_config/root/etc/spamassassin/public-inbox.pre
lib/PublicInbox.pm : FORCE
VERSION=\$(VERSION) \$(PERL) -w ./version-gen.perl
+XH_TESTS = t/xap_helper.t t/cindex.t
+
test-asan : pure_all
- TEST_XH_CXX_ONLY=1 CXXFLAGS='-O0 -Wall -ggdb3 -fsanitize=address' \\
- prove -bvw t/xap_helper.t
+ TEST_XH_CXX_ONLY=1 CXXFLAGS='-Wall -ggdb3 -fsanitize=address' \\
+ prove -bvw \$(XH_TESTS)
VG_OPT = -v --trace-children=yes --track-fds=yes
VG_OPT += --leak-check=yes --track-origins=yes
test-valgrind : pure_all
TEST_XH_CXX_ONLY=1 VALGRIND="valgrind \$(VG_OPT)" \\
- prove -bvw t/xap_helper.t
+ prove -bvw \$(XH_TESTS)
EOF
}
);
our @MAIL_NRP;
+# Getopt::Long spec, only short options for portability in C++ implementation
+our @XH_SPEC = (
+ 'a', # ascending sort
+ 'c', # code search
+ 'd=s@', # shard dirs
+ 'g=s', # git dir (with -c)
+ 'k=i', # sort column (like sort(1))
+ 'm=i', # maximum number of results
+ 'o=i', # offset
+ 'p', # show percent
+ 'r', # 1=relevance then column
+ 't', # collapse threads
+ 'A=s@', # prefixes
+ 'D', # emit docdata
+ 'K=i', # timeout kill after i seconds
+ 'O=s', # eidx_key
+ 'T=i', # threadid
+);
+
sub load_xapian () {
return 1 if defined $Xap;
# n.b. PI_XAPIAN is intended for development use only
int(($docid - 1) / $nshard) + 1;
}
+sub docids_to_artnums {
+ my $nshard = shift->{nshard};
+ # XXX does array vs arrayref make a difference in modern Perls?
+ map { int(($_ - 1) / $nshard) + 1 } @_;
+}
+
sub mset_to_artnums {
my ($self, $mset) = @_;
my $nshard = $self->{nshard};
our (%SRCH, %WORKERS, $nworker, $workerset, $in);
our $stderr = \*STDERR;
-# only short options for portability in C++ implementation
-our @SPEC = (
- 'a', # ascending sort
- 'c', # code search
- 'd=s@', # shard dirs
- 'k=i', # sort column (like sort(1))
- 'm=i', # maximum number of results
- 'o=i', # offset
- 'r', # 1=relevance then column
- 't', # collapse threads
- 'A=s@', # prefixes
- 'O=s', # eidx_key
- 'T=i', # timeout in seconds
-);
-
sub cmd_test_inspect {
my ($req) = @_;
print { $req->{0} } "pid=$$ has_threadid=",
emit_mset_stats($req, $mset);
}
+sub mset_iter ($$) {
+ my ($req, $it) = @_;
+ eval {
+ my $buf = $it->get_docid;
+ $buf .= "\0".$it->get_percent if $req->{p};
+ my $doc = ($req->{A} || $req->{D}) ? $it->get_document : undef;
+ for my $p (@{$req->{A}}) {
+ $buf .= "\0".$p.$_ for xap_terms($p, $doc);
+ }
+ $buf .= "\0".$doc->get_data if $req->{D};
+ say { $req->{0} } $buf;
+ };
+ $@ ? iter_retry_check($req) : 0;
+}
+
+sub cmd_mset { # to be used by WWW + IMAP
+ my ($req, $qry_str) = @_;
+ $qry_str // die 'usage: mset [OPTIONS] QRY_STR';
+ my $opt = { limit => $req->{'m'}, offset => $req->{o} // 0 };
+ $opt->{relevance} = 1 if $req->{r};
+ $opt->{threads} = 1 if defined $req->{t};
+ $opt->{git_dir} = $req->{g} if defined $req->{g};
+ $opt->{eidx_key} = $req->{O} if defined $req->{O};
+ $opt->{threadid} = $req->{T} if defined $req->{T};
+ my $mset = $req->{srch}->mset($qry_str, $opt);
+ say { $req->{0} } 'mset.size=', $mset->size;
+ for my $it ($mset->items) {
+ for (my $t = 10; $t > 0; --$t) {
+ $t = mset_iter($req, $it) // $t;
+ }
+ }
+}
+
sub dispatch {
my ($req, $cmd, @argv) = @_;
my $fn = $req->can("cmd_$cmd") or return;
- $GLP->getoptionsfromarray(\@argv, $req, @SPEC) or return;
+ $GLP->getoptionsfromarray(\@argv, $req, @PublicInbox::Search::XH_SPEC)
+ or return;
my $dirs = delete $req->{d} or die 'no -d args';
my $key = join("\0", @$dirs);
$req->{srch} = $SRCH{$key} //= do {
substr($dir, 0, 0) = "$ENV{PERL_INLINE_DIRECTORY}/";
my $bin = "$dir/xap_helper";
my ($srcpfx) = (__FILE__ =~ m!\A(.+/)[^/]+\z!);
-my @srcs = map { $srcpfx.$_ } qw(xap_helper.h xh_cidx.h);
+my @srcs = map { $srcpfx.$_ } qw(xh_mset.h xh_cidx.h xap_helper.h);
my @pm_dep = map { $srcpfx.$_ } qw(Search.pm CodeSearch.pm);
my $ldflags = '-Wl,-O1';
$ldflags .= ' -Wl,--compress-debug-sections=zlib' if $^O ne 'openbsd';
my $xflags = ($ENV{CXXFLAGS} // '-Wall -ggdb3 -pipe') . ' ' .
' -DTHREADID=' . PublicInbox::Search::THREADID .
- ' ' . ($ENV{LDFLAGS} // $ldflags);
+ ' -DXH_SPEC="'.join('',
+ map { s/=.*/:/; $_ } @PublicInbox::Search::XH_SPEC) . '" ' .
+ ($ENV{LDFLAGS} // $ldflags);
my $xap_modversion;
sub xap_cfg (@) {
char *argv[MY_ARG_MAX];
char *pfxv[MY_ARG_MAX]; // -A <prefix>
struct srch *srch;
+ char *Pgit_dir;
char *Oeidx_key;
cmd fn;
unsigned long long max;
unsigned long long off;
+ unsigned long long threadid;
unsigned long timeout_sec;
size_t nr_out;
long sort_col; // value column, negative means BoolWeight
bool collapse_threads;
bool code_search;
bool relevance; // sort by relevance before column
+ bool emit_percent;
+ bool emit_docdata;
bool asc; // ascending sort
};
return enquire_mset(req, &enq);
}
+static bool starts_with(const std::string *s, const char *pfx, size_t pfx_len)
+{
+ return s->size() >= pfx_len && !memcmp(pfx, s->c_str(), pfx_len);
+}
+
+static void apply_roots_filter(struct req *req, Xapian::Query *qry)
+{
+ if (!req->Pgit_dir) return;
+ req->Pgit_dir[0] = 'P'; // modifies static rbuf
+ Xapian::Database *xdb = req->srch->db;
+ for (int i = 0; i < 9; i++) {
+ try {
+ std::string P = req->Pgit_dir;
+ Xapian::PostingIterator p = xdb->postlist_begin(P);
+ if (p == xdb->postlist_end(P)) {
+ warnx("W: %s not indexed?", req->Pgit_dir + 1);
+ return;
+ }
+ Xapian::TermIterator cur = xdb->termlist_begin(*p);
+ Xapian::TermIterator end = xdb->termlist_end(*p);
+ cur.skip_to("G");
+ if (cur == end) {
+ warnx("W: %s has no root commits?",
+ req->Pgit_dir + 1);
+ return;
+ }
+ Xapian::Query f = Xapian::Query(*cur);
+ for (++cur; cur != end; ++cur) {
+ std::string tn = *cur;
+ if (!starts_with(&tn, "G", 1))
+ continue;
+ f = Xapian::Query(Xapian::Query::OP_OR, f, tn);
+ }
+ *qry = Xapian::Query(Xapian::Query::OP_FILTER, *qry, f);
+ return;
+ } catch (const Xapian::DatabaseModifiedError & e) {
+ xdb->reopen();
+ }
+ }
+}
+
// for cindex
static Xapian::MSet commit_mset(struct req *req, const char *qry_str)
{
struct srch *srch = req->srch;
Xapian::Query qry = srch->qp->parse_query(qry_str, srch->qp_flags);
- // TODO: git_dir + roots_filter
+ apply_roots_filter(req, &qry);
// we only want commits:
qry = Xapian::Query(Xapian::Query::OP_FILTER, qry,
ABORT("BUG: %s caller only passed 1 FD", req->argv[0]);
}
-static bool starts_with(const std::string *s, const char *pfx, size_t pfx_len)
-{
- return s->size() >= pfx_len && !memcmp(pfx, s->c_str(), pfx_len);
-}
-
static int my_setlinebuf(FILE *fp) // glibc setlinebuf(3) can't report errors
{
return setvbuf(fp, NULL, _IOLBF, 0);
if (!fbuf->fp) err(EXIT_FAILURE, "open_memstream(fbuf)");
}
+static bool write_all(int fd, const struct fbuf *wbuf, size_t len)
+{
+ const char *p = wbuf->ptr;
+ assert(wbuf->len >= len);
+ do { // write to client FD
+ ssize_t n = write(fd, p, len);
+ if (n > 0) {
+ len -= n;
+ p += n;
+ } else {
+ perror(n ? "write" : "write (zero bytes)");
+ return false;
+ }
+ } while (len);
+ return true;
+}
+
+#define ERR_FLUSH(f) do { \
+ if (ferror(f) | fflush(f)) err(EXIT_FAILURE, "ferror|fflush "#f); \
+} while (0)
+
+#define ERR_CLOSE(f, e) do { \
+ if (ferror(f) | fclose(f)) \
+ e ? err(e, "ferror|fclose "#f) : perror("ferror|fclose "#f); \
+} while (0)
+
static void xclose(int fd)
{
if (close(fd) < 0 && errno != EINTR)
return false;
}
+#include "xh_mset.h" // read-only (WWW, IMAP, lei) stuff
#include "xh_cidx.h" // CodeSearchIdx.pm stuff
#define CMD(n) { .fn_len = sizeof(#n) - 1, .fn_name = #n, .fn = cmd_##n }
cmd fn;
} cmds[] = { // should be small enough to not need bsearch || gperf
// most common commands first
+ CMD(mset), // WWW and IMAP requests
CMD(dump_ibx), // many inboxes
CMD(dump_roots), // per-cidx shard
CMD(test_inspect), // least common commands last
char *end;
FILE *kfp;
struct srch **s;
- req->fn = NULL;
+ req->threadid = ULLONG_MAX;
for (c = 0; c < (int)MY_ARRAY_SIZE(cmds); c++) {
if (cmds[c].fn_len == size &&
!memcmp(cmds[c].fn_name, req->argv[0], size)) {
optarg = NULL;
MY_DO_OPTRESET();
- // keep sync with @PublicInbox::XapHelper::SPEC
- while ((c = getopt(req->argc, req->argv, "acd:k:m:o:rtA:O:T:")) != -1) {
+ // XH_SPEC is generated from @PublicInbox::Search::XH_SPEC
+ while ((c = getopt(req->argc, req->argv, XH_SPEC)) != -1) {
switch (c) {
case 'a': req->asc = true; break;
case 'c': req->code_search = true; break;
case 'd': fwrite(optarg, strlen(optarg) + 1, 1, kfp); break;
+ case 'g': req->Pgit_dir = optarg - 1; break; // pad "P" prefix
case 'k':
req->sort_col = strtol(optarg, &end, 10);
if (*end) ABORT("-k %s", optarg);
if (*end || req->off == ULLONG_MAX)
ABORT("-o %s", optarg);
break;
+ case 'p': req->emit_percent = true; break;
case 'r': req->relevance = true; break;
case 't': req->collapse_threads = true; break;
case 'A':
if (MY_ARG_MAX == req->pfxc)
ABORT("too many -A");
break;
- case 'O': req->Oeidx_key = optarg - 1; break; // pad "O" prefix
- case 'T':
+ case 'D': req->emit_docdata = true; break;
+ case 'K':
req->timeout_sec = strtoul(optarg, &end, 10);
if (*end || req->timeout_sec == ULONG_MAX)
+ ABORT("-K %s", optarg);
+ break;
+ case 'O': req->Oeidx_key = optarg - 1; break; // pad "O" prefix
+ case 'T':
+ req->threadid = strtoull(optarg, &end, 10);
+ if (*end || req->threadid == ULLONG_MAX)
ABORT("-T %s", optarg);
break;
default: ABORT("bad switch `-%c'", c);
}
}
- if (ferror(kfp) | fclose(kfp)) /* sets kbuf.srch */
- err(EXIT_FAILURE, "ferror|fclose"); // likely ENOMEM
+ ERR_CLOSE(kfp, EXIT_FAILURE); // may ENOMEM, sets kbuf.srch
kbuf.srch->db = NULL;
kbuf.srch->qp = NULL;
kbuf.srch->paths_len = size - offsetof(struct srch, paths);
stderr = orig_err;
return;
#endif
- if (ferror(stderr) | fflush(stderr))
- err(EXIT_FAILURE, "ferror|fflush stderr");
+ ERR_CLOSE(stderr, EXIT_FAILURE);
while (dup2(orig_err_fd, STDERR_FILENO) < 0) {
if (errno != EINTR)
err(EXIT_FAILURE, "dup2(%d => 2)", orig_err_fd);
stderr_set(req.fp[1]);
req.argc = (int)SPLIT2ARGV(req.argv, rbuf, len);
dispatch(&req);
- if (ferror(req.fp[0]) | fclose(req.fp[0]))
- perror("ferror|fclose fp[0]");
+ ERR_CLOSE(req.fp[0], 0);
if (req.fp[1]) {
stderr_restore(req.fp[1]);
- if (ferror(req.fp[1]) | fclose(req.fp[1]))
- perror("ferror|fclose fp[1]");
+ ERR_CLOSE(req.fp[1], 0);
}
}
}
fputs((const char *)ep->data, root_offs->fp);
}
fputc('\n', root_offs->fp);
- if (ferror(root_offs->fp) | fclose(root_offs->fp))
- err(EXIT_FAILURE, "ferror|fclose(root_offs)"); // ENOMEM
+ ERR_CLOSE(root_offs->fp, EXIT_FAILURE); // ENOMEM
root_offs->fp = NULL;
return true;
}
// buffering and rely on flock(2), here
static bool dump_roots_flush(struct req *req, struct dump_roots_tmp *drt)
{
- char *p;
- int fd = fileno(req->fp[0]);
bool ok = true;
+ off_t off = ftello(drt->wbuf.fp);
+ if (off < 0) EABORT("ftello");
+ if (!off) return ok;
+
+ ERR_FLUSH(drt->wbuf.fp); // ENOMEM
+ int fd = fileno(req->fp[0]);
- if (!drt->wbuf.fp) return true;
- if (fd < 0) EABORT("BUG: fileno");
- if (ferror(drt->wbuf.fp) | fclose(drt->wbuf.fp)) // ENOMEM?
- err(EXIT_FAILURE, "ferror|fclose(drt->wbuf.fp)");
- drt->wbuf.fp = NULL;
- if (!drt->wbuf.len) goto done_free;
while (flock(drt->root2off_fd, LOCK_EX)) {
if (errno == EINTR) continue;
err(EXIT_FAILURE, "LOCK_EX"); // ENOLCK?
}
- p = drt->wbuf.ptr;
- do { // write to client FD
- ssize_t n = write(fd, p, drt->wbuf.len);
- if (n > 0) {
- drt->wbuf.len -= n;
- p += n;
- } else {
- perror(n ? "write" : "write (zero bytes)");
- return false;
- }
- } while (drt->wbuf.len);
+ ok = write_all(fd, &drt->wbuf, (size_t)off);
while (flock(drt->root2off_fd, LOCK_UN)) {
if (errno == EINTR) continue;
err(EXIT_FAILURE, "LOCK_UN"); // ENOLCK?
}
-done_free: // OK to skip on errors, dump_roots_ensure calls fbuf_ensure
- free(drt->wbuf.ptr);
- drt->wbuf.ptr = NULL;
+ if (fseeko(drt->wbuf.fp, 0, SEEK_SET)) EABORT("fseeko");
return ok;
}
req->sort_col = -1;
Xapian::MSet mset = commit_mset(req, req->argv[optind + 1]);
+ fbuf_init(&drt.wbuf);
+
// @UNIQ_FOLD in CodeSearchIdx.pm can handle duplicate lines fine
// in case we need to retry on DB reopens
for (Xapian::MSetIterator i = mset.begin(); i != mset.end(); i++) {
- if (!drt.wbuf.fp)
- fbuf_init(&drt.wbuf);
for (int t = 10; t > 0; --t)
switch (dump_roots_iter(req, &drt, &i)) {
case ITER_OK: t = 0; break; // leave inner loop
--- /dev/null
+// Copyright (C) all contributors <meta@public-inbox.org>
+// License: GPL-2.0+ <https://www.gnu.org/licenses/gpl-2.0.txt>
+// This file is only intended to be included by xap_helper.h
+// it implements pieces used by WWW, IMAP and lei
+
+static void emit_doc_term(FILE *fp, const char *pfx, Xapian::Document *doc)
+{
+ Xapian::TermIterator cur = doc->termlist_begin();
+ Xapian::TermIterator end = doc->termlist_end();
+ size_t pfx_len = strlen(pfx);
+
+ for (cur.skip_to(pfx); cur != end; cur++) {
+ std::string tn = *cur;
+ if (!starts_with(&tn, pfx, pfx_len)) continue;
+ fputc(0, fp);
+ fwrite(tn.data(), tn.size(), 1, fp);
+ }
+}
+
+static enum exc_iter mset_iter(const struct req *req, FILE *fp, off_t off,
+ Xapian::MSetIterator *i)
+{
+ try {
+ fprintf(fp, "%llu", (unsigned long long)(*(*i))); // get_docid
+ if (req->emit_percent)
+ fprintf(fp, "%c%d", 0, i->get_percent());
+ if (req->pfxc || req->emit_docdata) {
+ Xapian::Document doc = i->get_document();
+ for (int p = 0; p < req->pfxc; p++)
+ emit_doc_term(fp, req->pfxv[p], &doc);
+ if (req->emit_docdata) {
+ std::string d = doc.get_data();
+ fputc(0, fp);
+ fwrite(d.data(), d.size(), 1, fp);
+ }
+ }
+ fputc('\n', fp);
+ } catch (const Xapian::DatabaseModifiedError & e) {
+ req->srch->db->reopen();
+ if (fseeko(fp, off, SEEK_SET) < 0) EABORT("fseeko");
+ return ITER_RETRY;
+ } catch (const Xapian::DocNotFoundError & e) { // oh well...
+ warnx("doc not found: %s", e.get_description().c_str());
+ if (fseeko(fp, off, SEEK_SET) < 0) EABORT("fseeko");
+ }
+ return ITER_OK;
+}
+
+#ifndef WBUF_FLUSH_THRESHOLD
+# define WBUF_FLUSH_THRESHOLD (BUFSIZ - 1000)
+#endif
+#if WBUF_FLUSH_THRESHOLD < 0
+# undef WBUF_FLUSH_THRESHOLD
+# define WBUF_FLUSH_THRESHOLD BUFSIZ
+#endif
+
+static bool cmd_mset(struct req *req)
+{
+ if (optind >= req->argc) ABORT("usage: mset [OPTIONS] WANT QRY_STR");
+ if (req->fp[1]) ABORT("mset only accepts 1 FD");
+ const char *qry_str = req->argv[optind];
+ CLEANUP_FBUF struct fbuf wbuf = {};
+ Xapian::MSet mset = req->code_search ? commit_mset(req, qry_str) :
+ mail_mset(req, qry_str);
+ fbuf_init(&wbuf);
+ fprintf(wbuf.fp, "mset.size=%llu\n", (unsigned long long)mset.size());
+ int fd = fileno(req->fp[0]);
+ for (Xapian::MSetIterator i = mset.begin(); i != mset.end(); i++) {
+ off_t off = ftello(wbuf.fp);
+ if (off < 0) EABORT("ftello");
+ /*
+ * TODO verify our fflush + fseeko use isn't affected by a
+ * glibc <2.25 bug:
+ * https://sourceware.org/bugzilla/show_bug.cgi?id=20181
+ * CentOS 7.x only has glibc 2.17. In any case, bug #20181
+ * shouldn't affect us since our use of fseeko is used to
+ * effectively discard data.
+ */
+ if (off > WBUF_FLUSH_THRESHOLD) {
+ ERR_FLUSH(wbuf.fp);
+ if (!write_all(fd, &wbuf, (size_t)off)) return false;
+ if (fseeko(wbuf.fp, 0, SEEK_SET)) EABORT("fseeko");
+ off = 0;
+ }
+ for (int t = 10; t > 0; --t)
+ switch (mset_iter(req, wbuf.fp, off, &i)) {
+ case ITER_OK: t = 0; break; // leave inner loop
+ case ITER_RETRY: break; // continue for-loop
+ case ITER_ABORT: return false; // error
+ }
+ }
+ off_t off = ftello(wbuf.fp);
+ if (off < 0) EABORT("ftello");
+ ERR_FLUSH(wbuf.fp);
+ return off > 0 ? write_all(fd, &wbuf, (size_t)off) : true;
+}
use_ok 'PublicInbox::CodeSearch';
+
+my @xh_args;
+my $exp = [ 'initial with NUL character', 'remove NUL character' ];
+my $zp_git = abs_path("$zp/.git");
if ('multi-repo search') {
my $csrch = PublicInbox::CodeSearch->new("$tmp/ext");
my $mset = $csrch->mset('NUL');
is(scalar($mset->items), 2, 'got results');
- my $exp = [ 'initial with NUL character', 'remove NUL character' ];
my @have = sort(map { $_->get_document->get_data } $mset->items);
is_xdeeply(\@have, $exp, 'got expected subjects');
$mset = $csrch->mset('NUL', { git_dir => "$tmp/wt0/.git" });
is(scalar($mset->items), 0, 'no results with other GIT_DIR');
- $mset = $csrch->mset('NUL', { git_dir => abs_path("$zp/.git") });
+ $mset = $csrch->mset('NUL', { git_dir => $zp_git });
@have = sort(map { $_->get_document->get_data } $mset->items);
is_xdeeply(\@have, $exp, 'got expected subjects w/ GIT_DIR filter');
my @xdb = $csrch->xdb_shards_flat;
$no_metadata_set->(0, ['indexlevel'], \@xdb);
+ @xh_args = $csrch->xh_args;
+}
+
+my $test_xhc = sub {
+ my ($xhc) = @_;
+ my $impl = $xhc->{impl};
+ my ($r, @l);
+ $r = $xhc->mkreq([], qw(mset -D -c -g), $zp_git, @xh_args, 'NUL');
+ chomp(@l = <$r>);
+ is(shift(@l), 'mset.size=2', "got expected header $impl");
+ my %docid2data;
+ my @got = sort map {
+ my @f = split /\0/;
+ is scalar(@f), 2, 'got 2 entries';
+ $docid2data{$f[0]} = $f[1];
+ $f[1];
+ } @l;
+ is_deeply(\@got, $exp, "expected doc_data $impl");
+
+ $r = $xhc->mkreq([], qw(mset -c -g), "$tmp/wt0/.git", @xh_args, 'NUL');
+ chomp(@l = <$r>);
+ is(shift(@l), 'mset.size=0', "got miss in wrong dir $impl");
+ is_deeply(\@l, [], "no extra lines $impl");
+
+ my $csrch = PublicInbox::CodeSearch->new("$tmp/ext");
+ while (my ($did, $expect) = each %docid2data) {
+ is_deeply($csrch->xdb->get_document($did)->get_data,
+ $expect, "docid=$did data matches");
+ }
+ ok(!$xhc->{io}->close, "$impl close");
+ is($?, 66 << 8, "got EX_NOINPUT from $impl exit");
+};
+
+SKIP: {
+ require_mods('+SCM_RIGHTS', 1);
+ require PublicInbox::XapClient;
+ my $xhc = PublicInbox::XapClient::start_helper('-j0');
+ $test_xhc->($xhc);
+ skip 'PI_NO_CXX set', 1 if $ENV{PI_NO_CXX};
+ $xhc->{impl} =~ /Cxx/ or
+ skip 'C++ compiler or xapian development libs missing', 1;
+ skip 'TEST_XH_CXX_ONLY set', 1 if $ENV{TEST_XH_CXX_ONLY};
+ local $ENV{PI_NO_CXX} = 1; # force XS or SWIG binding test
+ $xhc = PublicInbox::XapClient::start_helper('-j0');
+ $test_xhc->($xhc);
}
if ('--update') {
};
my @ibx_idx = glob("$v2->{inboxdir}/xap*/?");
+my @ibx_shard_args = map { ('-d', $_) } @ibx_idx;
my (@int) = glob("$crepo/public-inbox-cindex/cidx*/?");
my (@ext) = glob("$crepo/cidx-ext/cidx*/?");
is(scalar(@ext), 2, 'have 2 external shards') or diag explain(\@ext);
is($cinfo{has_threadid}, '0', 'has_threadid false for cindex');
is($cinfo{pid}, $info{pid}, 'PID unchanged for cindex');
- my @dump = (qw(dump_ibx -A XDFID), (map { ('-d', $_) } @ibx_idx),
- qw(13 rt:0..));
+ my @dump = (qw(dump_ibx -A XDFID), @ibx_shard_args, qw(13 rt:0..));
$r = $doreq->($s, @dump);
my @res;
while (sysread($r, my $buf, 512) != 0) { push @res, $buf }
my $res = do { local $/; <$r> };
is(join('', @res), $res, 'got identical response w/ error pipe');
my $stats = do { local $/; <$err_rd> };
- is($stats, "mset.size=6 nr_out=6\n", 'mset.size reported');
+ is($stats, "mset.size=6 nr_out=6\n", 'mset.size reported') or
+ diag "res=$res";
return wantarray ? ($ar, $s) : $ar if $cinfo{pid} == $pid;
is(scalar(@res), scalar(grep(/\A[0-9a-f]{40,} [0-9]+\n\z/, @res)),
'entries match format');
$err = do { local $/; <$err_r> };
- is($err, "mset.size=6 nr_out=5\n", "got expected status ($xhc->{impl})");
+ is $err, "mset.size=6 nr_out=5\n", "got expected status ($xhc->{impl})";
+
+ $r = $xhc->mkreq([], qw(mset -p -A XDFID -A Q), @ibx_shard_args,
+ 'dfn:lib/PublicInbox/Search.pm');
+ chomp((my $hdr, @res) = readline($r));
+ is $hdr, 'mset.size=1', "got expected header via mset ($xhc->{impl}";
+ is scalar(@res), 1, 'got one result';
+ @res = split /\0/, $res[0];
+ {
+ my $doc = $v2->search->xdb->get_document($res[0]);
+ my @q = PublicInbox::Search::xap_terms('Q', $doc);
+ is_deeply \@q, [ $mid ], 'docid usable';
+ }
+ ok $res[1] > 0 && $res[1] <= 100, 'pct > 0 && <= 100';
+ is $res[2], 'XDFID'.$dfid, 'XDFID result matches';
+ is $res[3], 'Q'.$mid, 'Q (msgid) mset result matches';
+ is scalar(@res), 4, 'only 4 columns in result';
+
+ $r = $xhc->mkreq([], qw(mset -p -A XDFID -A Q), @ibx_shard_args,
+ 'dt:19700101'.'000000..');
+ chomp(($hdr, @res) = readline($r));
+ is $hdr, 'mset.size=6',
+ "got expected header via multi-result mset ($xhc->{impl}";
+ is(scalar(@res), 6, 'got 6 rows');
+ for my $r (@res) {
+ my ($docid, $pct, @rest) = split /\0/, $r;
+ my $doc = $v2->search->xdb->get_document($docid);
+ ok $pct > 0 && $pct <= 100,
+ "pct > 0 && <= 100 #$docid ($xhc->{impl})";
+ my %terms;
+ for (@rest) {
+ s/\A([A-Z]+)// or xbail 'no prefix=', \@rest;
+ push @{$terms{$1}}, $_;
+ }
+ while (my ($pfx, $vals) = each %terms) {
+ @$vals = sort @$vals;
+ my @q = PublicInbox::Search::xap_terms($pfx, $doc);
+ is_deeply $vals, \@q,
+ "#$docid $pfx as expected ($xhc->{impl})";
+ }
+ }
}
done_testing;
projects / thirdparty / public-inbox.git / commitdiff
