extindex: support per-inbox indexheader+altid

This allows the venerable altid (e.g. gmane:1234) to finally
work for extindex users.  The newer indexheader directive works
here, too.  This allows a multi-inbox extindex to fully emulate
the capabilities of per-inbox Xapian indices.

For now, per-inbox indexheader and altid DO NOT work when
searching the extindex directly.  In other words, gmane:1234
might work on the /git/ inbox, but not the /all/ extindex
virtual inbox.  This may remain the case since altid is
typically per-inbox only, and stuff like X-Archives-Hash
can be global across inboxes.

diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index b40e96f1cbc34caf75396fcd55945637034ada92..cda3045e6348f464ed777e08d181c6f7b3dc0667 100644 (file)
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -571,7 +571,7 @@ sub _fill_ei ($$) {
        }
        return unless valid_foo_name($name, 'extindex');
        $es->{name} = $name;
-       $es->load_extra_indexers($es);
+       $es->load_extra_indexers($es); # extindex.*.{altid,indexheader}
        $es;
 }
 

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 094821a3b4516b37aa3a01488cd224031f05a935..cead0f8a1060326359e9345d372870fe7e2f1927 100644 (file)
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -21,6 +21,7 @@ use Carp qw(croak carp);
 use Scalar::Util qw(blessed);
 use Sys::Hostname qw(hostname);
 use File::Glob qw(bsd_glob GLOB_NOSORT);
+use PublicInbox::Isearch;
 use PublicInbox::MultiGit;
 use PublicInbox::Spawn ();
 use PublicInbox::Search;

diff --git a/lib/PublicInbox/Isearch.pm b/lib/PublicInbox/Isearch.pm
index 9566f71093bb6555c0dbb9ffec7d46ed7a5cb378..5f22c2f2fcdb5176f6d4abc041b012b02589761d 100644 (file)
--- a/lib/PublicInbox/Isearch.pm
+++ b/lib/PublicInbox/Isearch.pm
@@ -11,7 +11,11 @@ use PublicInbox::Search;
 
 sub new {
        my (undef, $ibx, $es) = @_;
-       bless { es => $es, eidx_key => $ibx->eidx_key }, __PACKAGE__;
+       my $self = bless { es => $es, eidx_key => $ibx->eidx_key }, __PACKAGE__;
+       # load publicinbox.*.{altid,indexheader}
+       PublicInbox::Search::load_extra_indexers($self, $ibx);
+       push @{$self->{-extra}}, @{$es->{-extra} // []} if $self->{-extra};
+       $self;
 }
 
 sub _ibx_id ($) {
@@ -55,14 +59,22 @@ SELECT MAX(docid) FROM xref3 WHERE ibx_id = ? AND xnum >= ? AND xnum <= ?
        \%opt;
 }
 
+sub _isrch_qparse ($) {
+       my ($self) = @_;
+       local $self->{es}->{-extra} = $self->{-extra};
+       $self->{es}->qparse_new; # XXX worth memoizing?
+}
+
 sub mset {
        my ($self, $str, $opt) = @_;
+       local $self->{es}->{qp} = _isrch_qparse($self) if $self->{-extra};
        $self->{es}->mset($str, eidx_mset_prep $self, $opt);
 }
 
 sub async_mset {
        my ($self, $str, $opt, $cb, @args) = @_;
        $opt = eidx_mset_prep $self, $opt;
+       local $self->{es}->{-extra} = $self->{-extra} if $self->{-extra};
        $self->{es}->async_mset($str, $opt, $cb, @args);
 }
 

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 53c16e555cb19a8df135d2d43997bf8b122956f3..7829c7d4721c629499b896ef4b12393c10acafa3 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -475,9 +475,8 @@ sub eml2doc ($$$;$) {
        term_generator($self)->set_document($doc);
        index_headers($self, $smsg);
 
-       if (defined(my $eidx_key = $smsg->{eidx_key})) {
-               $doc->add_boolean_term('O'.$eidx_key) if $eidx_key ne '.';
-       }
+       my $ekey = $smsg->{eidx_key};
+       $doc->add_boolean_term('O'.$ekey) if ($ekey // '.') ne '.';
        msg_iter($eml, \&index_xapian, [ $self, $doc ]);
        index_ids($self, $doc, $eml, $mids);
 
@@ -491,9 +490,10 @@ sub eml2doc ($$$;$) {
                my $data = $smsg->to_doc_data;
                $doc->set_data($data);
        }
-
-       for my $extra (@{$self->{-extra} // []}) {
-               $extra->index_extra($self, $eml, $mids);
+       my $xtra = defined $ekey ? $self->{"-extra\t$ekey"} : undef;
+       $xtra //= $self->{-extra};
+       for my $e (@$xtra) {
+               $e->index_extra($self, $eml, $mids);
        }
        $doc;
 }
@@ -1170,6 +1170,14 @@ sub eidx_shard_new {
        }, $class;
        $self->{-set_indexlevel_once} = 1 if $self->{indexlevel} eq 'medium';
        $self->load_extra_indexers($eidx);
+       require PublicInbox::Isearch;
+       my $all = $self->{-extra};
+       for my $ibx (@{$eidx->{ibx_active} // []}) {
+               my $isrch = PublicInbox::Isearch->new($ibx);
+               my $per_ibx = $isrch->{-extra} // next;
+               $self->{"-extra\t$isrch->{eidx_key}"} =
+                                       $all ? [ @$per_ibx, @$all ] : $per_ibx;
+       }
        $self;
 }
 

diff --git a/t/extsearch.t b/t/extsearch.t
index 0ea5bc5b811f71272207feaf8808833133224a95..28c43763ca313782e93028225ac9c12cda39b5e2 100644 (file)
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -593,6 +593,7 @@ test_lei(sub {
                'noted unindexed extindex is unsupported');
 });
 
+require PublicInbox::XhcMset;
 if ('indexheader support') {
        xsys_e [qw(git config extindex.all.indexheader
                boolean_term:xarchiveshash:X-Archives-Hash)],
@@ -608,20 +609,97 @@ if ('indexheader support') {
        $es = PublicInbox::Config->new($cfg_path)->ALL;
        my $mset = $es->mset('xarchiveshash:deadbeefcafe');
        is $mset->size, 1, 'extindex.*.indexheader works';
-       local $PublicInbox::Search::XHC = eval {
-               require PublicInbox::XhcMset;
-               PublicInbox::XapClient::start_helper('-j0');
-       } or xbail "no XHC: $@";
+       local $PublicInbox::Search::XHC =
+                       PublicInbox::XapClient::start_helper('-j0') or
+                       xbail "no XHC: $@";
        my @args;
        $es->async_mset('xarchiveshash:deadbeefcafe', {} , sub { @args = @_ });
-       is scalar(@args), 2, 'no extra args on hit';
-       is $args[0]->size, 1, 'async mset hit works';
-       ok !$args[1], 'no error on hit';
+       is scalar(@args), 2, 'no extra args on xarchiveshash hit';
+       is $args[0]->size, 1, 'async mset xarchiveshash hit works';
+       ok !$args[1], 'no error on xarchiveshash hit';
        @args = ();
        $es->async_mset('xarchiveshash:cafebeefdead', {} , sub { @args = @_ });
-       is scalar(@args), 2, 'no extra args on miss';
-       is $args[0]->size, 0, 'async mset miss works';
-       ok !$args[1], 'no error on miss';
+       is scalar(@args), 2, 'no extra args on xarchiveshash miss';
+       is $args[0]->size, 0, 'async mset xarchivehash miss works';
+       ok !$args[1], 'no error on xarchiveshash miss';
+}
+
+if ('per-inbox altid w/ extindex') {
+       my $another = 'another-nntp.sqlite3';
+       my $altid = [ "serial:gmane:file=$another" ];
+       my $aibx = create_inbox 'v2', version => 2, indexlevel => 'basic',
+                               altid => $altid, sub {
+               my ($im, $ibx) = @_;
+               my $mm = PublicInbox::Msgmap->new_file(
+                                       "$ibx->{inboxdir}/$another", 2);
+               $mm->mid_set(1234, 'a@example.com') == 1 or xbail 'mid_set';
+               $im->add(PublicInbox::Eml->new(<<'EOF')) or BAIL_OUT;
+From: a@example.com
+To: b@example.com
+Subject: boo!
+Message-ID: <a@example.com>
+X-Archives-Hash: dadfad
+Organization: felonious feline family
+
+hello world gmane:666
+EOF
+       };
+       PublicInbox::IO::write_file '>>', $cfg_path, <<EOF;
+[publicinbox "altid-test"]
+       inboxdir = $aibx->{inboxdir}
+       address = b\@example.com
+       altid = $altid->[0]
+       indexheader = phrase:organization:Organization
+EOF
+       ok run_script([qw(-extindex --all -vvv), $eidxdir]),
+               'extindex update w/ altid';
+       local $PublicInbox::Search::XHC =
+                       PublicInbox::XapClient::start_helper('-j0') or
+                       xbail "no XHC: $@";
+       my @args;
+       my $pi_cfg = PublicInbox::Config->new($cfg_path);
+       my $ibx = $pi_cfg->lookup('b@example.com');
+       my $mset = $ibx->isrch->mset('gmane:1234');
+
+       is $mset->size, 1, 'isrch->mset altid hit';
+       $ibx->isrch->async_mset('gmane:1234', {} , sub { @args = @_ });
+       is scalar(@args), 2, 'no extra args on altid hit';
+       is $args[0]->size, 1, 'isrch->async_mset altid hit';
+
+       $mset = $ibx->isrch->mset('organization:felonious');
+       is $mset->size, 1, 'isrch->mset indexheader hit';
+       @args = ();
+       $ibx->isrch->async_mset('organization:felonious', {} , sub { @args = @_ });
+       is scalar(@args), 2, 'no extra args on indexheader hit';
+       is $args[0]->size, 1, 'isrch->async_mset indexheader hit';
+
+       $mset = $ibx->isrch->mset('organization:world');
+       is $mset->size, 0, 'isrch->mset indexheader miss';
+       @args = ();
+       $ibx->isrch->async_mset('organization:world', {} , sub { @args = @_ });
+       is scalar(@args), 2, 'no extra args on indexheader miss';
+       is $args[0]->size, 0, 'isrch->async_mset indexheader miss';
+
+       $mset = $ibx->isrch->mset('xarchiveshash:deadbeefcafe');
+       is $mset->size, 0, 'isrch->mset does not cross inbox on indexheader';
+       $mset = $ibx->isrch->mset('xarchiveshash:dadfad');
+       is $mset->size, 1, 'isrch->mset hits global indexheader';
+
+       $es = $pi_cfg->ALL;
+       $mset = $es->mset('xarchiveshash:dadfad');
+       is $mset->size, 1, 'esrch->mset global indexheader hit';
+       $mset = $es->mset('gmane:1234');
+       is $mset->size, 1, '->mset altid hit works globally';
+
+       $mset = $es->mset('gmane:666');
+       is $mset->size, 0, 'global ->mset hits';
+       $mset = $ibx->isrch->mset('gmane:666');
+       is $mset->size, 0, 'isrch->mset altid miss works';
+
+       @args = ();
+       $ibx->isrch->async_mset('gmane:666', {} , sub { @args = @_ });
+       is scalar(@args), 2, 'no extra args on altid miss';
+       is $args[0]->size, 0, 'isrch->async_mset altid miss works';
 }
 
 done_testing;