www: support POST /$INBOX/$MSGID/?x=m&q=

This allows filtering the contents of any existing thread using
a search query.  It uses the existing THREADID column in Xapian
so we can internally add a Xapian OP_FILTER to the results.

This new functionality is orthogonal to the existing `t=1'
parameter which gives mairix-style thread expansion.  It doesn't
make sense to use `t=1' with this functionality, but it's not
disallowed, either.

The indentation change in Over->next_by_mid is to ensure
DBI->prepare_cached can share across both ->next_by_mid
and ->mid2tid.

I also noticed the existing regex for `POST /$INBOX/?x=m&q=' was
allowing extra characters.  With an added \z, it's now as strict
was originally intended and AFAIK nothing was generating invalid
URLs for it

Reported-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
Link: https://public-inbox.org/meta/aaniyhk7wfm4e6m5mbukcrhevzoc6ftctyrfwvmz4fkykwwtlj@mverfng6ytas/T/

diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index 18db9d38e12bdae939ce532c200eaffa5e73a4df..e1abf7ec068ae4c317e3ba6f4877a90de44baf44 100644 (file)
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -229,6 +229,11 @@ sub mbox_all {
                return PublicInbox::WWW::need($ctx, 'Overview');
 
        my $qopts = $ctx->{qopts} = { relevance => -2 }; # ORDER BY docid DESC
+
+       # {threadid} limits results to a given thread
+       # {threads} collapses results from messages in the same thread,
+       # allowing us to use ->expand_thread w/o duplicates in our own code
+       $qopts->{threadid} = $over->mid2tid($ctx->{mid}) if defined($ctx->{mid});
        $qopts->{threads} = 1 if $q->{t};
        $srch->query_approxidate($ctx->{ibx}->git, $q_string);
        my $mset = $srch->mset($q_string, $qopts);

diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm
index d6409b2a5fdb2017a69344aed92e60be77b2eb44..82034b30fdf399ccf9153f0e5b1c8d6e0f652ffa 100644 (file)
--- a/lib/PublicInbox/Over.pm
+++ b/lib/PublicInbox/Over.pm
@@ -282,13 +282,35 @@ SELECT eidx_key FROM inboxes WHERE ibx_id = ?
        $rows;
 }
 
+sub mid2tid {
+       my ($self, $mid) = @_;
+       my $dbh = dbh($self);
+
+       my $sth = $dbh->prepare_cached(<<'', undef, 1);
+SELECT id FROM msgid WHERE mid = ? LIMIT 1
+
+       $sth->execute($mid);
+       my $id = $sth->fetchrow_array or return;
+       $sth = $dbh->prepare_cached(<<'', undef, 1);
+SELECT num FROM id2num WHERE id = ? AND num > ?
+ORDER BY num ASC LIMIT 1
+
+       $sth->execute($id, 0);
+       my $num = $sth->fetchrow_array or return;
+       $sth = $dbh->prepare(<<'');
+SELECT tid FROM over WHERE num = ? LIMIT 1
+
+       $sth->execute($num);
+       $sth->fetchrow_array;
+}
+
 sub next_by_mid {
        my ($self, $mid, $id, $prev) = @_;
        my $dbh = dbh($self);
 
        unless (defined $$id) {
                my $sth = $dbh->prepare_cached(<<'', undef, 1);
-       SELECT id FROM msgid WHERE mid = ? LIMIT 1
+SELECT id FROM msgid WHERE mid = ? LIMIT 1
 
                $sth->execute($mid);
                $$id = $sth->fetchrow_array;

diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 5133a3b741bd2c284e8faf9281558eef7ecbfda2..6c3d9f93e1c8a1bd49b5ff3b11695eb3cc38fb7b 100644 (file)
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -386,6 +386,12 @@ sub mset {
                                        sortable_serialise($uid_range->[1]));
                $query = $X{Query}->new(OP_FILTER(), $query, $range);
        }
+       if (defined(my $tid = $opt->{threadid})) {
+               $tid = sortable_serialise($tid);
+               $query = $X{Query}->new(OP_FILTER(), $query,
+                               $X{Query}->new(OP_VALUE_RANGE(), THREADID, $tid, $tid));
+       }
+
        my $xdb = xdb($self);
        my $enq = $X{Enquire}->new($xdb);
        $enq->set_query($query);

diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index 9ffcb879d9ef5be1c17f199a351bb5906ab5bedc..a8f1ad1799ce573c2e0cb112d700224e90bc3851 100644 (file)
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -68,7 +68,9 @@ sub call {
                        my ($idx, $fn) = ($3, $4);
                        return invalid_inbox_mid($ctx, $1, $2) ||
                                get_attach($ctx, $idx, $fn);
-               } elsif ($path_info =~ m!$INBOX_RE/!o) {
+               } elsif ($path_info =~ m!$INBOX_RE/$MID_RE/\z!o) {
+                       return invalid_inbox_mid($ctx, $1, $2) || mbox_results($ctx);
+               } elsif ($path_info =~ m!$INBOX_RE/\z!o) {
                        return invalid_inbox($ctx, $1) || mbox_results($ctx);
                }
        }

diff --git a/t/psgi_v2.t b/t/psgi_v2.t
index 5b197a9f65b9d2d14d8e659bbb392f19dd7425c5..0a77adfb4bae30c773a90e66d6af1a2520063b8f 100644 (file)
--- a/t/psgi_v2.t
+++ b/t/psgi_v2.t
@@ -4,6 +4,7 @@
 use strict;
 use v5.10.1;
 use PublicInbox::TestCommon;
+use IO::Uncompress::Gunzip qw(gunzip);
 require_git(2.6);
 use PublicInbox::Eml;
 use PublicInbox::Config;
@@ -76,6 +77,30 @@ $new_mid //= do {
        local $/;
        <$fh>;
 };
+
+my $m2t = create_inbox 'mid2tid-1', version => 2, indexlevel => 'medium', sub {
+       my ($im, $ibx) = @_;
+       for my $n (1..3) {
+               $im->add(PublicInbox::Eml->new(<<EOM)) or xbail 'add';
+Date: Fri, 02 Oct 1993 00:0$n:00 +0000
+Message-ID: <t\@$n>
+Subject: tid $n
+From: x\@example.com
+References: <a-mid\@b>
+
+$n
+EOM
+               $im->add(PublicInbox::Eml->new(<<EOM)) or xbail 'add';
+Date: Fri, 02 Oct 1993 00:0$n:00 +0000
+Message-ID: <ut\@$n>
+Subject: unrelated tid $n
+From: x\@example.com
+References: <b-mid\@b>
+
+EOM
+       }
+};
+
 my $cfgpath = "$ibx->{inboxdir}/pi_config";
 {
        open my $fh, '>', $cfgpath or BAIL_OUT $!;
@@ -86,6 +111,9 @@ my $cfgpath = "$ibx->{inboxdir}/pi_config";
 [publicinbox "dup"]
        inboxdir = $dibx->{inboxdir}
        address = $dibx->{-primary_address}
+[publicinbox "m2t"]
+       inboxdir = $m2t->{inboxdir}
+       address = $m2t->{-primary_address}
 EOF
        close $fh or BAIL_OUT;
 }
@@ -178,20 +206,18 @@ my $client1 = sub {
        $cfg->each_inbox(sub { $_[0]->search->reopen });
 
        SKIP: {
-               eval { require IO::Uncompress::Gunzip };
-               skip 'IO::Uncompress::Gunzip missing', 6 if $@;
                my ($in, $out, $status);
                my $req = GET('/v2test/a-mid@b/raw');
                $req->header('Accept-Encoding' => 'gzip');
                $res = $cb->($req);
                is($res->header('Content-Encoding'), 'gzip', 'gzip encoding');
                $in = $res->content;
-               IO::Uncompress::Gunzip::gunzip(\$in => \$out);
+               gunzip(\$in => \$out);
                is($out, $raw, 'gzip response matches');
 
                $res = $cb->(GET('/v2test/a-mid@b/t.mbox.gz'));
                $in = $res->content;
-               $status = IO::Uncompress::Gunzip::gunzip(\$in => \$out);
+               $status = gunzip(\$in => \$out);
                unlike($out, qr/^From oldbug/sm, 'buggy "From_" line omitted');
                like($out, qr/^hello world$/m, 'got first in t.mbox.gz');
                like($out, qr/^hello world!$/m, 'got second in t.mbox.gz');
@@ -202,7 +228,7 @@ my $client1 = sub {
                # search interface
                $res = $cb->(POST('/v2test/?q=m:a-mid@b&x=m'));
                $in = $res->content;
-               $status = IO::Uncompress::Gunzip::gunzip(\$in => \$out);
+               $status = gunzip(\$in => \$out);
                unlike($out, qr/^From oldbug/sm, 'buggy "From_" line omitted');
                like($out, qr/^hello world$/m, 'got first in mbox POST');
                like($out, qr/^hello world!$/m, 'got second in mbox POST');
@@ -213,7 +239,7 @@ my $client1 = sub {
                # all.mbox.gz interface
                $res = $cb->(GET('/v2test/all.mbox.gz'));
                $in = $res->content;
-               $status = IO::Uncompress::Gunzip::gunzip(\$in => \$out);
+               $status = gunzip(\$in => \$out);
                unlike($out, qr/^From oldbug/sm, 'buggy "From_" line omitted');
                like($out, qr/^hello world$/m, 'got first in all.mbox');
                like($out, qr/^hello world!$/m, 'got second in all.mbox');
@@ -335,6 +361,18 @@ my $client3 = sub {
        local $SIG{__WARN__} = sub { push @warn, @_ };
        $res = $cb->(GET('/v2test/?t=1970'.'01'.'01'));
        is_deeply(\@warn, [], 'no warnings on YYYYMMDD only');
+
+       $res = $cb->(POST("/m2t/t\@1/?q=dt:19931002000300..&x=m"));
+       is($res->code, 200, 'got 200 on mid2tid query');
+       gunzip(\(my $in = $res->content) => \(my $out));
+       my @m = ($out =~ m!^Message-ID: <([^>]+)>\n!gms);
+       is_deeply(\@m, ['t@3'], 'only got latest result from query');
+
+       $res = $cb->(POST("/m2t/t\@1/?q=dt:19931002000400..&x=m"));
+       is($res->code, 404, '404 on out-of-range mid2tid query');
+
+       $res = $cb->(POST("/m2t/t\@1/?q=s:unrelated&x=m"));
+       is($res->code, 404, '404 on cross-thread search');
 };
 test_psgi(sub { $www->call(@_) }, $client3);
 test_httpd($env, $client3, 4);