cindex: preserve indexlevel across invocations

This matches the behavior of mail indexers and ensures `medium'
indices don't grow unexpectedly to be come `full' indices.

diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index 5f20325a68b8219100502f621e751c9a227d4231..3a3fc03eb20ead7bdba51702a05451b08942e93f 100644 (file)
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -85,7 +85,6 @@ sub new {
                xpfx => "$dir/cidx".  PublicInbox::CodeSearch::CIDX_SCHEMA_VER,
                cidx_dir => $dir,
                creat => 1, # TODO: get rid of this, should be implicit
-               indexlevel => $l,
                transact_bytes => 0, # for checkpoint
                total_bytes => 0, # for lock_release
                current_info => '',
@@ -617,16 +616,28 @@ sub cidx_init ($) {
        }
        $self->lock_acquire;
        my @shards;
+       my $l = $self->{indexlevel} //= $self->{-opt}->{indexlevel};
+
        for my $n (0..($self->{nshard} - 1)) {
                my $shard = bless { %$self, shard => $n }, ref($self);
                delete @$shard{qw(lockfh lock_path)};
-               $shard->idx_acquire;
+               my $xdb = $shard->idx_acquire;
+               if (!$n) {
+                       if (($l // '') eq 'medium') {
+                               $xdb->set_metadata('indexlevel', $l);
+                       } elsif (($l // '') eq 'full') {
+                               $xdb->set_metadata('indexlevel', ''); # unset
+                       }
+                       $l ||= $xdb->get_metadata('indexlevel') || 'full';
+               }
+               $shard->{indexlevel} = $l;
                $shard->idx_release;
                $shard->wq_workers_start("cidx shard[$n]", 1, $SIGSET, {
                        siblings => \@shards, # for ipc_atfork_child
                }, \&shard_done_wait, $self);
                push @shards, $shard;
        }
+       $self->{indexlevel} //= $l;
        # this warning needs to happen after idx_acquire
        state $once;
        warn <<EOM if $PublicInbox::Search::X{CLOEXEC_UNSET} && !$once++;

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index f36c8f97877d3e47d5b238208d806ffc9cf48666..699af432ff07f93063816396a2c551fc73cd3639 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -90,7 +90,7 @@ sub new {
        $self;
 }
 
-sub need_xapian ($) { $_[0]->{indexlevel} =~ $xapianlevels }
+sub need_xapian ($) { ($_[0]->{indexlevel} // 'full') =~ $xapianlevels }
 
 sub idx_release {
        my ($self, $wake) = @_;

diff --git a/t/cindex.t b/t/cindex.t
index 9da0ba692ff9fc57997ca7eb171976114ff3f9dd..d40f73ff6ac251c834dac7e6394686076cc8ed72 100644 (file)
--- a/t/cindex.t
+++ b/t/cindex.t
@@ -4,11 +4,13 @@
 use v5.12;
 use PublicInbox::TestCommon;
 use Cwd qw(getcwd abs_path);
+use List::Util qw(sum);
 require_mods(qw(json Search::Xapian));
 use_ok 'PublicInbox::CodeSearchIdx';
 require PublicInbox::Import;
 my ($tmp, $for_destroy) = tmpdir();
 my $pwd = getcwd();
+my @unused_keys = qw(last_commit has_threadid skip_docdata);
 
 # I reworked CodeSearchIdx->shard_worker to handle empty trees
 # in the initial commit generated by cvs2svn for xapian.git
@@ -71,7 +73,48 @@ ok(run_script([qw(-cindex --dangerous -q -d), "$tmp/ext", $zp, "$tmp/wt0"]),
 ok(-e "$tmp/ext/cidx.lock", 'external dir created');
 ok(!-d "$zp/.git/public-inbox-cindex", 'no cindex in original coderepo');
 
+ok(run_script([qw(-cindex -L medium --dangerous -q -d),
+       "$tmp/med", $zp, "$tmp/wt0"]), 'cindex external medium');
+
+my $no_metadata_set = sub {
+       my ($i, $extra, $xdb) = @_;
+       for my $xdb (@$xdb) {
+               for my $k (@unused_keys, @$extra) {
+                       is($xdb->get_metadata($k) // '', '',
+                               "metadata $k unset in shard #$i");
+               }
+               ++$i;
+       }
+};
+
+{
+       my $mid_size = sum(map { -s $_ } glob("$tmp/med/cidx*/*/*"));
+       my $full_size = sum(map { -s $_ } glob("$tmp/ext/cidx*/*/*"));
+       ok($full_size > $mid_size, 'full size > mid size') or
+               diag "full=$full_size mid=$mid_size";
+       for my $l (qw(med ext)) {
+               ok(run_script([qw(-cindex -q --reindex -u -d), "$tmp/$l"]),
+                       "reindex $l");
+       }
+       $mid_size = sum(map { -s $_ } glob("$tmp/med/cidx*/*/*"));
+       $full_size = sum(map { -s $_ } glob("$tmp/ext/cidx*/*/*"));
+       ok($full_size > $mid_size, 'full size > mid size after reindex') or
+               diag "full=$full_size mid=$mid_size";
+       my $csrch = PublicInbox::CodeSearch->new("$tmp/med");
+       my ($xdb0, @xdb) = $csrch->xdb_shards_flat;
+       $no_metadata_set->(0, [], [ $xdb0 ]);
+       is($xdb0->get_metadata('indexlevel'), 'medium',
+               'indexlevel set in shard #0');
+       $no_metadata_set->(1, ['indexlevel'], \@xdb);
+
+       ok(run_script([qw(-cindex -q -L full --reindex -u -d), "$tmp/med"]),
+               'reindex medium as full');
+       @xdb = $csrch->xdb_shards_flat;
+       $no_metadata_set->(0, ['indexlevel'], \@xdb);
+}
+
 use_ok 'PublicInbox::CodeSearch';
+
 if ('multi-repo search') {
        my $csrch = PublicInbox::CodeSearch->new("$tmp/ext");
        my $mset = $csrch->mset('NUL');
@@ -86,6 +129,8 @@ if ('multi-repo search') {
        $mset = $csrch->mset('NUL', { git_dir => abs_path("$zp/.git") });
        @have = sort(map { $_->get_document->get_data } $mset->items);
        is_xdeeply(\@have, $exp, 'got expected subjects w/ GIT_DIR filter');
+       my @xdb = $csrch->xdb_shards_flat;
+       $no_metadata_set->(0, ['indexlevel'], \@xdb);
 }
 
 if ('--update') {