Bug 832893: changes jobqueue.pl to spawn worker processes to deliver bugmail to avoid...

r=dkl, a=LpSolit

diff --git a/Bugzilla/JobQueue.pm b/Bugzilla/JobQueue.pm
index 5a33c12f237da0ebff1785bb234eb97c831b9701..2c3d8a773d860af464cd640a0aa7cc8ef5fa628c 100644 (file)
--- a/Bugzilla/JobQueue.pm
+++ b/Bugzilla/JobQueue.pm
@@ -12,7 +12,10 @@ use strict;
 use Bugzilla::Constants;
 use Bugzilla::Error;
 use Bugzilla::Install::Util qw(install_string);
+use File::Basename;
+use File::Slurp;
 use base qw(TheSchwartz);
+use fields qw(_worker_pidfile);
 
 # This maps job names for Bugzilla::JobQueue to the appropriate modules.
 # If you add new types of jobs, you should add a mapping here.
@@ -92,6 +95,57 @@ sub insert {
     return $retval;
 }
 
+# To avoid memory leaks/fragmentation which tends to happen for long running
+# perl processes; check for jobs, and spawn a new process to empty the queue.
+sub subprocess_worker {
+    my $self = shift;
+
+    my $command = "$0 -d -p '" . $self->{_worker_pidfile} . "' onepass";
+
+    while (1) {
+        my $time = (time);
+        my @jobs = $self->list_jobs({
+            funcname      => $self->{all_abilities},
+            run_after     => $time,
+            grabbed_until => $time,
+            limit         => 1,
+        });
+        if (@jobs) {
+            $self->debug("Spawning queue worker process");
+            # Run the worker as a daemon
+            system $command;
+            # And poll the PID to detect when the working has finished.
+            # We do this instead of system() to allow for the INT signal to
+            # interrup us and trigger kill_worker().
+            my $pid = read_file($self->{_worker_pidfile}, err_mode => 'quiet');
+            if ($pid) {
+                sleep(3) while(kill(0, $pid));
+            }
+            $self->debug("Queue worker process completed");
+        } else {
+            $self->debug("No jobs found");
+        }
+        sleep(5);
+    }
+}
+
+sub kill_worker {
+    my $self = Bugzilla->job_queue();
+    if ($self->{_worker_pidfile} && -e $self->{_worker_pidfile}) {
+        my $worker_pid = read_file($self->{_worker_pidfile});
+        if ($worker_pid && kill(0, $worker_pid)) {
+            $self->debug("Stopping worker process");
+            system "$0 -f -p '" . $self->{_worker_pidfile} . "' stop";
+        }
+    }
+}
+
+sub set_pidfile {
+    my ($self, $pidfile) = @_;
+    $self->{_worker_pidfile} = bz_locations->{'datadir'} .
+                               '/worker-' . basename($pidfile);
+}
+
 # Clear the request cache at the start of each run.
 sub work_once {
     my $self = shift;

diff --git a/Bugzilla/JobQueue/Runner.pm b/Bugzilla/JobQueue/Runner.pm
index 6c837369f978ce8f0089135a0db58dc1cac7ea74..8b5cf9b6ba6e21f7cf62ed9b868028262d33b666 100644 (file)
--- a/Bugzilla/JobQueue/Runner.pm
+++ b/Bugzilla/JobQueue/Runner.pm
@@ -36,6 +36,7 @@ our $initscript = "bugzilla-queue";
 sub gd_preconfig {
     my $self = shift;
 
+    $self->{_run_command} = 'subprocess_worker';
     my $pidfile = $self->{gd_args}{pidfile};
     if (!$pidfile) {
         $pidfile = bz_locations()->{datadir} . '/' . $self->{gd_progname} 
@@ -134,6 +135,7 @@ sub gd_can_install {
             print $config_fh <<END;
 #!/bin/sh
 BUGZILLA="$directory"
+# This user must have write access to Bugzilla's data/ directory.
 USER=$owner
 END
             close($config_fh);
@@ -181,21 +183,25 @@ sub gd_setup_signals {
     $SIG{TERM} = sub { $self->gd_quit_event(); }
 }
 
-sub gd_other_cmd {
-    my ($self) = shift;
-    if ($ARGV[0] eq "once") {
-        $self->_do_work("work_once");
+sub gd_quit_event {
+    Bugzilla->job_queue->kill_worker();
+    exit(1);
+}
 
-        exit(0);
+sub gd_other_cmd {
+    my ($self, $do, $locked) = @_;
+    if ($do eq "once") {
+        $self->{_run_command} = 'work_once';
+    } elsif ($do eq "onepass") {
+        $self->{_run_command} = 'work_until_done';
+    } else {
+        $self->SUPER::gd_other_cmd($do, $locked);
     }
-    
-    $self->SUPER::gd_other_cmd();
 }
 
 sub gd_run {
     my $self = shift;
-
-    $self->_do_work("work");
+    $self->_do_work($self->{_run_command});
 }
 
 sub _do_work {
@@ -203,11 +209,11 @@ sub _do_work {
 
     my $jq = Bugzilla->job_queue();
     $jq->set_verbose($self->{debug});
+    $jq->set_pidfile($self->{gd_pidfile});
     foreach my $module (values %{ Bugzilla::JobQueue->job_map() }) {
         eval "use $module";
         $jq->can_do($module);
     }
-
     $jq->$fn;
 }
 

diff --git a/jobqueue.pl b/jobqueue.pl
index 1b956917e088f8a53b4b34699959b793fe3c4a55..3b96b8dde865c026df79693f16d1638c9a219b73 100755 (executable)
--- a/jobqueue.pl
+++ b/jobqueue.pl
@@ -45,6 +45,7 @@ jobqueue.pl - Runs jobs in the background for Bugzilla.
              starts a new one.
    once      Checks the job queue once, executes the first item found (if
              any) and then exits
+   onepass   Checks the job queue, executes all items found, and then exits
    check     Report the current status of the daemon.
    install   On some *nix systems, this automatically installs and
              configures jobqueue.pl as a system service so that it will