favor poll(2) for most daemons

public-inbox-watch, lei-daemon, the master process of
public-inbox-(netd|httpd|imapd|nntpd|pop3d),
and the (mostly) Perl implementation of XapHelper do not
have many FDs to watch so epoll|kqueue end up being overkill.

Of course, *BSDs already have separate kqueue FDs emulating
signalfd and/or inotify, even.

In other words, only the worker processes of
public-inbox-(netd|httpd|imapd|nntpd|pop3d) are expected
to see C10K (or C100K) types of traffic where epoll|kqueue
shine.

Perhaps lei could benefit from epoll/kqueue on some virtual users
IMAP/JMAP system one day; as could -watch with many IMAP IDLE
folders; but we'll probably add a knob if/when it comes to that.

diff --git a/lib/PublicInbox/DS.pm b/lib/PublicInbox/DS.pm
index 9300ac7720cdcef8c11a42ec69a750e55a33e3cf..f2b147996f369ea2bb58ece7f36e2fc0d3e6ddcb 100644 (file)
--- a/lib/PublicInbox/DS.pm
+++ b/lib/PublicInbox/DS.pm
@@ -31,6 +31,7 @@ use Scalar::Util qw(blessed);
 use PublicInbox::Syscall qw(%SIGNUM
        EPOLLIN EPOLLOUT EPOLLONESHOT EPOLLEXCLUSIVE);
 use PublicInbox::Tmpfile;
+use PublicInbox::DSPoll;
 use Errno qw(EAGAIN EINVAL ECHILD EINTR);
 use Carp qw(carp croak);
 our @EXPORT_OK = qw(now msg_more awaitpid add_timer add_uniq_timer);
@@ -42,7 +43,7 @@ my $reap_armed;
 my $ToClose; # sockets to close when event loop is done
 our (
      %DescriptorMap,             # fd (num) -> PublicInbox::DS object
-     $Epoll,  # global Epoll, DSPoll, or DSKQXS ref
+     $Poller, # global Epoll, DSPoll, or DSKQXS ref
 
      @post_loop_do,              # subref + args to call at the end of each loop
 
@@ -75,13 +76,14 @@ sub Reset {
                my @q = delete @Stack{keys %Stack};
                for my $q (@q) { @$q = () }
                $AWAIT_PIDS = $nextq = $ToClose = undef;
-               $Epoll = undef; # may call DSKQXS::DESTROY
+               $Poller = undef; # may call DSKQXS::DESTROY
        } while (@Timers || keys(%Stack) || $nextq || $AWAIT_PIDS ||
                $ToClose || keys(%DescriptorMap) ||
                @post_loop_do || keys(%UniqTimer));
 
        $reap_armed = undef;
        $LoopTimeout = -1;  # no timeout by default
+       $Poller = PublicInbox::DSPoll->new;
 }
 
 =head2 C<< CLASS->SetLoopTimeout( $timeout ) >>
@@ -123,7 +125,7 @@ sub add_uniq_timer { # ($name, $secs, $coderef, @args) = @_;
        $UniqTimer{$_[0]} //= _add_named_timer(@_);
 }
 
-# caller sets return value to $Epoll
+# caller sets return value to $Poller
 sub _InitPoller () {
        my @try = ($^O eq 'linux' ? 'Epoll' : 'DSKQXS');
        my $cls;
@@ -269,7 +271,7 @@ sub unblockset ($) { sigset_prep $_[0], 'emptyset', 'addset' }
 # C<post_loop_do> for how to exit the loop.
 sub event_loop (;$$) {
        my ($sig, $oldset) = @_;
-       $Epoll //= _InitPoller();
+       $Poller //= _InitPoller();
        require PublicInbox::Sigfd if $sig;
        my $sigfd = $sig ? PublicInbox::Sigfd->new($sig) : undef;
        if ($sigfd && $sigfd->{is_kq}) {
@@ -298,7 +300,7 @@ sub event_loop (;$$) {
                my $timeout = RunTimers();
 
                # get up to 1000 events
-               $Epoll->ep_wait(1000, $timeout, \@events);
+               $Poller->ep_wait(1000, $timeout, \@events);
                for my $fd (@events) {
                        # it's possible epoll_wait returned many events,
                        # including some at the end that ones in the front
@@ -334,9 +336,9 @@ sub new {
     $self->{sock} = $sock;
     my $fd = fileno($sock);
 
-    $Epoll //= _InitPoller();
+    $Poller //= _InitPoller();
 retry:
-    if ($Epoll->ep_add($sock, $ev)) {
+    if ($Poller->ep_add($sock, $ev)) {
         if ($! == EINVAL && ($ev & EPOLLEXCLUSIVE)) {
             $ev &= ~EPOLLEXCLUSIVE;
             goto retry;
@@ -390,7 +392,7 @@ sub close {
 
     # if we're using epoll, we have to remove this from our epoll fd so we stop getting
     # notifications about it
-    $Epoll->ep_del($sock) and croak("EPOLL_CTL_DEL($self/$sock): $!");
+    $Poller->ep_del($sock) and croak("EPOLL_CTL_DEL($self/$sock): $!");
 
     # we explicitly don't delete from DescriptorMap here until we
     # actually close the socket, as we might be in the middle of
@@ -609,7 +611,7 @@ sub msg_more ($$) {
 
 sub epwait ($$) {
        my ($io, $ev) = @_;
-       $Epoll->ep_mod($io, $ev) and croak("EPOLL_CTL_MOD($io): $!");
+       $Poller->ep_mod($io, $ev) and croak("EPOLL_CTL_MOD($io): $!");
 }
 
 # return true if complete, false if incomplete (or failure)

diff --git a/lib/PublicInbox/Daemon.pm b/lib/PublicInbox/Daemon.pm
index 222093bc809a4d9a9baf75158c85d34bd42af803..078831530598f1060828e4d90971a98e837e7c11 100644 (file)
--- a/lib/PublicInbox/Daemon.pm
+++ b/lib/PublicInbox/Daemon.pm
@@ -556,6 +556,7 @@ sub start_worker ($) {
        } elsif ($pid == 0) {
                undef %WORKERS;
                PublicInbox::DS::Reset();
+               local $PublicInbox::DS::Poller; # allow epoll/kqueue
                srand($seed);
                eval { Net::SSLeay::randomize() };
                $set_user->() if $set_user;
@@ -677,6 +678,7 @@ sub daemon_loop () {
                $WORKER_SIG{USR2} = sub { worker_quit() if upgrade() };
                $refresh->();
        }
+       local $PublicInbox::DS::Poller; # allow epoll/kqueue
        worker_loop();
 }
 

diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index b7f1eb572fa0d25cc35b64904fd12ab40772f04b..17057e18db5bcb7c74abd8bf3af86f7d6311e259 100644 (file)
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -25,7 +25,7 @@ BEGIN {
                create_coderepo no_scm_rights
                tcp_host_port test_lei lei lei_ok $lei_out $lei_err $lei_opt
                test_httpd xbail require_cmd is_xdeeply tail_f
-               ignore_inline_c_missing);
+               ignore_inline_c_missing no_pollerfd);
        require Test::More;
        my @methods = grep(!/\W/, @Test::More::EXPORT);
        eval(join('', map { "*$_=\\&Test::More::$_;" } @methods));
@@ -843,7 +843,24 @@ sub test_httpd ($$;$$) {
        }
 };
 
-
+sub no_pollerfd ($) {
+       my ($pid) = @_;
+       my ($re, @cmd);
+       $^O eq 'linux' and
+               ($re, @cmd) = (qr/\Q[eventpoll]\E/, qw(lsof -p), $pid);
+       # n.b. *BSDs uses kqueue to emulate signalfd and/or inotify,
+       # and we can't distinguish which is which easily.
+       SKIP: {
+               (@cmd && $re) or
+                       skip 'open poller test is Linux-only', 1;
+               my $bin = require_cmd($cmd[0], 1) or skip "$cmd[0] missing", 1;
+               $cmd[0] = $bin;
+               my @of = xqx(\@cmd, {}, {2 => \(my $e)});
+               my $err = $?;
+               skip "$bin broken? (\$?=$err) ($e)", 1 if $err;
+               is(grep(/$re/, @of), 0, "no $re FDs") or diag explain(\@of);
+       }
+}
 package PublicInbox::TestCommon::InboxWakeup;
 use strict;
 sub on_inbox_unlock { ${$_[0]}->($_[1]) }

diff --git a/t/httpd-unix.t b/t/httpd-unix.t
index d90c6c3ed2a680bf48c28b18a818d7b5f0d8b2e4..95f589ad9a4f620f2bbe722f171523691ec8cbd6 100644 (file)
--- a/t/httpd-unix.t
+++ b/t/httpd-unix.t
@@ -135,6 +135,7 @@ SKIP: {
                check_sock($unix);
                ok(-s $pid_file, "$w pid file written");
                my $pid = $read_pid->($pid_file);
+               no_pollerfd($pid) if $w eq '-W1';
                is(kill('TERM', $pid), 1, "signaled daemonized $w process");
                vec(my $rvec = '', fileno($p0), 1) = 1;
                delete $td->{-extra}; # drop tail(1) process

diff --git a/t/lei-daemon.t b/t/lei-daemon.t
index 78ed265e24a9f7a2abe2030e744e1af7e20c2074..2be967be945de560737d34d4702430407fb677df 100644 (file)
--- a/t/lei-daemon.t
+++ b/t/lei-daemon.t
@@ -21,6 +21,7 @@ test_lei({ daemon_only => 1 }, sub {
        is($lei_err, '', 'no error from daemon-pid');
        like($lei_out, qr/\A[0-9]+\n\z/s, 'pid returned') or BAIL_OUT;
        chomp(my $pid = $lei_out);
+       no_pollerfd($pid);
        ok(kill(0, $pid), 'pid is valid');
        ok(-S $sock, 'sock created');
        is(-s $err_log, 0, 'nothing in errors.log');

diff --git a/t/watch_maildir.t b/t/watch_maildir.t
index 6836a3d9c11239c8ff452ba96ac92feb7020c934..d0df1c1e93c34ab689d55c9c36bffcb97e662e4a 100644 (file)
--- a/t/watch_maildir.t
+++ b/t/watch_maildir.t
@@ -151,6 +151,7 @@ More majordomo info at  http://vger.kernel.org/majordomo-info.html\n);
 
        # n.b. --no-scan is only intended for testing atm
        my $wm = start_script([qw(-watch --no-scan)], $env);
+       no_pollerfd($wm->{pid});
        my $eml = eml_load('t/data/0001.patch');
        $eml->header_set('Cc', $addr);
        my $em = PublicInbox::Emergency->new($maildir);

diff --git a/t/xap_helper.t b/t/xap_helper.t
index fe5d2d1429f2c39a572edc671a4833cf2348577a..54bef191d014f2f99142410403c89791c9c83584 100644 (file)
--- a/t/xap_helper.t
+++ b/t/xap_helper.t
@@ -93,7 +93,7 @@ my $test = sub {
        my $stats = do { local $/; <$err_rd> };
        is($stats, "mset.size=6 nr_out=6\n", 'mset.size reported');
 
-       return $ar if $cinfo{pid} == $pid;
+       return wantarray ? ($ar, $s) : $ar if $cinfo{pid} == $pid;
 
        # test worker management:
        kill('TERM', $cinfo{pid});
@@ -136,15 +136,16 @@ my $test = sub {
        is(scalar keys %pids, 1, 'have one pid') or diag explain(\%pids);
        is($info{pid}, (keys %pids)[0], 'kept oldest PID after TTOU');
 
-       $ar;
+       wantarray ? ($ar, $s) : $ar;
 };
 
 my @NO_CXX = (1);
 unless ($ENV{TEST_XH_CXX_ONLY}) {
        my $ar = $test->(qw[-MPublicInbox::XapHelper -e
                        PublicInbox::XapHelper::start('-j0')]);
-       $ar = $test->(qw[-MPublicInbox::XapHelper -e
+       ($ar, my $s) = $test->(qw[-MPublicInbox::XapHelper -e
                        PublicInbox::XapHelper::start('-j1')]);
+       no_pollerfd($ar->{pid});
 }
 SKIP: {
        eval {