extindex: support extindex.*.indexheader

Per-inbox (publicinbox.*.indexheader) is next...

diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 3af5f23c839821b1e4052092acb5fcaca59f5cb0..b40e96f1cbc34caf75396fcd55945637034ada92 100644 (file)
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -565,12 +565,13 @@ sub _fill_ei ($$) {
                my $v = get_1($self, "$pfx.$k") // next;
                $es->{$k} = $v;
        }
-       for my $k (qw(coderepo hide url infourl)) {
+       for my $k (qw(coderepo hide url infourl indexheader altid)) {
                my $v = $self->{"$pfx.$k"} // next;
                $es->{$k} = _array($v);
        }
        return unless valid_foo_name($name, 'extindex');
        $es->{name} = $name;
+       $es->load_extra_indexers($es);
        $es;
 }
 

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 68700c8b8e932f8d610c381c65103734e7b2fcb8..094821a3b4516b37aa3a01488cd224031f05a935 100644 (file)
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -35,6 +35,7 @@ use PublicInbox::Eml;
 use PublicInbox::DS qw(now add_timer);
 use DBI qw(:sql_types); # SQL_BLOB
 use PublicInbox::Admin qw(fmt_localtime);
+use PublicInbox::Config qw(rel2abs_collapsed);
 
 sub new {
        my (undef, $dir, $opt) = @_;
@@ -86,7 +87,21 @@ sub _ibx_attach { # each_inbox callback
 sub attach_config {
        my ($self, $cfg, $ibxs) = @_;
        $self->{cfg} = $cfg;
-       my $types;
+       my ($types, $ro);
+
+       # lookup extindex.$NAME.<indexheader|altid>
+       my $eidx_dir = rel2abs_collapsed($self->{topdir});
+       for my $k (grep(/\Aextindex\.(?:.+)\.topdir\z/, keys %$cfg)) {
+               next if rel2abs_collapsed($cfg->{$k}) ne $eidx_dir;
+               my $n = substr($k, length('extindex.'), -length('.topdir'));
+               $ro = $cfg->lookup_ei($n) and last;
+       }
+
+       # and copy from read-only to our read-write $self
+       for my $f (qw(altid indexheader)) {
+               $self->{$f} = $ro->{$f} if defined $ro->{$f};
+       }
+
        if ($ibxs) {
                for my $ibx (@$ibxs) {
                        $self->{ibx_map}->{$ibx->eidx_key} //= do {

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index b2576e52f545672ea44c036bf3f79efc57eb150f..53c16e555cb19a8df135d2d43997bf8b122956f3 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -1169,6 +1169,7 @@ sub eidx_shard_new {
                creat => 1,
        }, $class;
        $self->{-set_indexlevel_once} = 1 if $self->{indexlevel} eq 'medium';
+       $self->load_extra_indexers($eidx);
        $self;
 }
 

diff --git a/t/extsearch.t b/t/extsearch.t
index 16d75f63a017991d0d9220e70597018c5a6ab79f..0ea5bc5b811f71272207feaf8808833133224a95 100644 (file)
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -8,7 +8,7 @@ use PublicInbox::InboxWritable;
 require_git(2.6);
 require_mods(qw(json DBD::SQLite Xapian));
 use autodie qw(chmod open rename truncate unlink);
-require PublicInbox::Search;
+use PublicInbox::Search;
 use_ok 'PublicInbox::ExtSearch';
 use_ok 'PublicInbox::ExtSearchIdx';
 use_ok 'PublicInbox::OverIdx';
@@ -26,6 +26,7 @@ ok(run_script([qw(-init -Lbasic -V2 v2test --newsgroup v2.example),
        "$home/v2test", 'http://example.com/v2test', $v2addr ]), 'v2test init');
 my $env = { ORIGINAL_RECIPIENT => $v2addr };
 my $eml = eml_load('t/utf8.eml');
+my $eidxdir = "$home/extindex";
 
 $eml->header_set('List-Id', '<v2.example.com>');
 
@@ -592,4 +593,35 @@ test_lei(sub {
                'noted unindexed extindex is unsupported');
 });
 
+if ('indexheader support') {
+       xsys_e [qw(git config extindex.all.indexheader
+               boolean_term:xarchiveshash:X-Archives-Hash)],
+               { GIT_CONFIG => $cfg_path };
+       my $eml = eml_load('t/plack-qp.eml');
+       $eml->header_set('X-Archives-Hash', 'deadbeefcafe');
+       $in = \($eml->as_string);
+       $env->{ORIGINAL_RECIPIENT} = $v2addr;
+       run_script([qw(-mda --no-precheck)], $env, { 0 => $in }) or
+               xbail '-mda';
+       ok run_script([qw(-extindex --all -vvv), $eidxdir]),
+               'extindex update';
+       $es = PublicInbox::Config->new($cfg_path)->ALL;
+       my $mset = $es->mset('xarchiveshash:deadbeefcafe');
+       is $mset->size, 1, 'extindex.*.indexheader works';
+       local $PublicInbox::Search::XHC = eval {
+               require PublicInbox::XhcMset;
+               PublicInbox::XapClient::start_helper('-j0');
+       } or xbail "no XHC: $@";
+       my @args;
+       $es->async_mset('xarchiveshash:deadbeefcafe', {} , sub { @args = @_ });
+       is scalar(@args), 2, 'no extra args on hit';
+       is $args[0]->size, 1, 'async mset hit works';
+       ok !$args[1], 'no error on hit';
+       @args = ();
+       $es->async_mset('xarchiveshash:cafebeefdead', {} , sub { @args = @_ });
+       is scalar(@args), 2, 'no extra args on miss';
+       is $args[0]->size, 0, 'async mset miss works';
+       ok !$args[1], 'no error on miss';
+}
+
 done_testing;