Bug 1450990 - Refactor a bunch of the logging config files

diff --git a/Bugzilla.pm b/Bugzilla.pm
index a8b164b573421038f842d22148491803fdf0a114..d98298accbc2a63636712a0e34c3d404e983ddc4 100644 (file)
--- a/Bugzilla.pm
+++ b/Bugzilla.pm
@@ -56,7 +56,6 @@ use Encode;
 use File::Basename;
 use File::Spec::Functions;
 use Safe;
-use Sys::Syslog qw(:DEFAULT);
 use JSON::XS qw(decode_json);
 
 use parent qw(Bugzilla::CPAN);

diff --git a/Bugzilla/Constants.pm b/Bugzilla/Constants.pm
index 80d9c4e0c24acf171132474d3fa575159ec8d971..3aa7b0fa499c5a4f5b254bc1620c28f3bc63de34 100644 (file)
--- a/Bugzilla/Constants.pm
+++ b/Bugzilla/Constants.pm
@@ -693,6 +693,7 @@ sub _bz_locations {
         # The script should really generate these graphs directly...
         'webdotdir'      => "$datadir/webdot",
         'extensionsdir'  => "$libpath/extensions",
+        'logsdir'        => "$libpath/logs",
         'assetsdir'      => "$datadir/assets",
         'confdir'        => $confdir,
     };

diff --git a/Bugzilla/DaemonControl.pm b/Bugzilla/DaemonControl.pm
index 6586cc01b60fc0765c0b3d71a11c00ce015f66ae..2c6df1b87a5d961da1cf4ba42692e4bce31c3365 100644 (file)
--- a/Bugzilla/DaemonControl.pm
+++ b/Bugzilla/DaemonControl.pm
@@ -81,7 +81,13 @@ sub run_cereal {
         on_exception => on_exception( 'cereal', $exit_f ),
     );
     $exit_f->on_cancel( sub { $cereal->kill('TERM') } );
+    $exit_f->on_ready(
+        sub {
+            delete $ENV{LOG4PERL_STDERR_DISABLE};
+        }
+    );
     $loop->add($cereal);
+    $ENV{LOG4PERL_STDERR_DISABLE} = 1;
 
     return $exit_f;
 }

diff --git a/Bugzilla/Error.pm b/Bugzilla/Error.pm
index ef57303e98133172d065066d94fd5f4713ed204a..9fcd1638626a52051f40186bbaa0127f9c118675 100644 (file)
--- a/Bugzilla/Error.pm
+++ b/Bugzilla/Error.pm
@@ -169,7 +169,7 @@ sub ThrowUserError {
 
 sub ThrowCodeError {
     my ($error, $vars) = @_;
-    my $logfunc = _make_logfunc('User');
+    my $logfunc = _make_logfunc('Code');
     _add_vars_to_logging_fields($vars);
 
     _throw_error( 'global/code-error.html.tmpl', $error, $vars, $logfunc );

diff --git a/Bugzilla/Install/Filesystem.pm b/Bugzilla/Install/Filesystem.pm
index 70b195090e7956151dcc3113ab84bab2adb00a3f..094226891009cd46ecef343f63a94f8081aa4be0 100644 (file)
--- a/Bugzilla/Install/Filesystem.pm
+++ b/Bugzilla/Install/Filesystem.pm
@@ -36,6 +36,7 @@ use Cwd ();
 use File::Slurp;
 use IO::File;
 use POSIX ();
+use English qw(-no_match_vars $OSNAME);
 
 use base qw(Exporter);
 our @EXPORT = qw(
@@ -106,6 +107,7 @@ use constant HTTPD_ENV => qw(
     LOCALCONFIG_ENV
     BUGZILLA_UNSAFE_AUTH_DELEGATION
     LOG4PERL_CONFIG_FILE
+    LOG4PERL_STDERR_DISABLE
     USE_NYTPROF
     NYTPROF_DIR
 );
@@ -162,6 +164,8 @@ sub DIR_CGI_OVERWRITE { _group() ? 0770 : 0777 };
 # (or their subdirectories) to the user, via the webserver.
 sub DIR_ALSO_WS_SERVE { _suexec() ? 0001 : 0 };
 
+sub DIR_ALSO_WS_STICKY { $OSNAME eq 'linux' ? 02000 : 0 }
+
 # This looks like a constant because it effectively is, but
 # it has to call other subroutines and read the current filesystem,
 # so it's defined as a sub. This is not exported, so it doesn't have
@@ -186,6 +190,7 @@ sub FILESYSTEM {
     my $template_cache = bz_locations()->{'template_cache'};
     my $graphsdir      = bz_locations()->{'graphsdir'};
     my $assetsdir      = bz_locations()->{'assetsdir'};
+    my $logsdir        = bz_locations()->{'logsdir'};
 
     # We want to set the permissions the same for all localconfig files
     # across all PROJECTs, so we do something special with $localconfig,
@@ -277,6 +282,8 @@ sub FILESYSTEM {
                                   dirs => DIR_CGI_WRITE | DIR_ALSO_WS_SERVE },
          "$datadir/db"      => { files => CGI_WRITE,
                                   dirs => DIR_CGI_WRITE },
+         $logsdir           => { files => CGI_WRITE,
+                                 dirs  => DIR_CGI_WRITE | DIR_ALSO_WS_STICKY },
          $assetsdir         => { files => WS_SERVE,
                                   dirs => DIR_CGI_OVERWRITE | DIR_ALSO_WS_SERVE },
 
@@ -361,6 +368,7 @@ sub FILESYSTEM {
         $webdotdir              => DIR_CGI_WRITE | DIR_ALSO_WS_SERVE,
         $assetsdir              => DIR_CGI_WRITE | DIR_ALSO_WS_SERVE,
         $template_cache         => DIR_CGI_WRITE,
+        $logsdir                => DIR_CGI_WRITE | DIR_ALSO_WS_STICKY,
         # Directories that contain content served directly by the web server.
         "$skinsdir/custom"      => DIR_WS_SERVE,
         "$skinsdir/contrib"     => DIR_WS_SERVE,

diff --git a/Bugzilla/Logging.pm b/Bugzilla/Logging.pm
index 4a7abcb21e256819fe2c0be9d938aaf855d46081..f334435fc4d1f3004b93f19978f758dc29e8f029 100644 (file)
--- a/Bugzilla/Logging.pm
+++ b/Bugzilla/Logging.pm
@@ -12,12 +12,17 @@ use warnings;
 
 use Log::Log4perl qw(:easy);
 use Log::Log4perl::MDC;
-use File::Spec::Functions qw(rel2abs);
+use File::Spec::Functions qw(rel2abs catfile);
 use Bugzilla::Constants qw(bz_locations);
 use English qw(-no_match_vars $PROGRAM_NAME);
+use Taint::Util qw(untaint);
 
-sub is_interactive {
-    return not exists $ENV{SERVER_SOFTWARE}
+sub logfile {
+    my ($class, $name) = @_;
+
+    my $file = rel2abs(catfile(bz_locations->{logsdir}, $name));
+    untaint($file);
+    return $file;
 }
 
 sub fields {

diff --git a/README.rst b/README.rst
index adeb1a18efb15df8881346ade4644c0018371965..d8986fc7344a4f6e460062011c500df16ce651f8 100644 (file)
--- a/README.rst
+++ b/README.rst
@@ -377,6 +377,13 @@ LOG4PERL_CONFIG_FILE
 .. _`Devel::NYTProf`: https://metacpan.org/pod/Devel::NYTProf
 .. _`Log::Log4perl`: https://metacpan.org/pod/Log::Log4perl
 
+LOG4PERL_STDERR_DISABLE
+  Boolean. By default log messages are logged as plain text to `STDERR`.
+  Setting this to a true value disables this behavior.
+
+  Note: For programs that run using the `cereal` log aggregator, this environemnt
+  variable will be ignored.
+
 Persistent Data Volume
 ----------------------
 

diff --git a/Vagrantfile b/Vagrantfile
index 8776ac08f0a08bae91a45a9d3541608192597c6c..2a1c4e9c46d2747382a2428d1c96b74ef9dfbc5e 100644 (file)
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -38,7 +38,6 @@ Vagrant.configure('2') do |config|
 
   config.vm.provision 'main', type: 'ansible_local', run: 'always' do |ansible|
     ansible.playbook = 'vagrant_support/playbook.yml'
-    ansible.version = 'latest'
     ansible.extra_vars = {
       WEB_IP:            WEB_IP,
       DB_IP:             DB_IP,
@@ -51,7 +50,6 @@ Vagrant.configure('2') do |config|
   if ARGV.include? '--provision-with'
     config.vm.provision 'update', type: 'ansible_local', run: 'never' do |update|
       update.playbook = 'vagrant_support/update.yml'
-      update.version = 'latest'
     end
   end
 

diff --git a/conf/httpd.conf b/conf/httpd.conf
index 7fe859b59a593fdf440d762459bc2b36c5e77550..539ab4231501af22efd12e2ff401d033703c1eb3 100644 (file)
--- a/conf/httpd.conf
+++ b/conf/httpd.conf
@@ -86,6 +86,7 @@ Include /app/conf/env.conf
 
 PerlSwitches -wT
 PerlRequire /app/mod_perl.pl
+PerlSetEnv LOG4PERL_STDERR_DISABLE 1
 DirectoryIndex index.cgi
 DocumentRoot "/app"
 <IfDefine HTTPD_IN_SUBDIR>

diff --git a/conf/log4perl-docker.conf b/conf/log4perl-docker.conf
new file mode 100644 (file)
index 0000000..1f5a033
--- /dev/null
+++ b/conf/log4perl-docker.conf
@@ -0,0 +1,14 @@
+log4perl.rootLogger = DEBUG, Cereal, Screen
+log4perl.appender.Cereal = Log::Log4perl::Appender::Socket
+log4perl.appender.Cereal.PeerAddr=127.0.0.1
+log4perl.appender.Cereal.PeerPort=5880
+log4perl.appender.Cereal.defer_connection=1
+log4perl.appender.Cereal.layout = Log::Log4perl::Layout::PatternLayout
+log4perl.appender.Cereal.layout.ConversionPattern = %d %6p | %c | %m{chomp}%n
+
+log4perl.filter.LOG_TO_STDERR = sub { not $ENV{LOG4PERL_STDERR_DISABLE}  }
+log4perl.appender.Screen = Log::Log4perl::Appender::Screen
+log4perl.appender.Screen.Filter = LOG_TO_STDERR
+log4perl.appender.Screen.stderr = 1
+log4perl.appender.Screen.layout = Log::Log4perl::Layout::PatternLayout
+log4perl.appender.Screen.layout.ConversionPattern = %d %6p | %c | %m{chomp}%n

diff --git a/conf/log4perl-json.conf b/conf/log4perl-json.conf
index c5d7e2c1fd9d58e223aa678f39d6780705a6fec9..a9c4b6e1a387023cb7d05d18dc4384da3c2a1172 100644 (file)
--- a/conf/log4perl-json.conf
+++ b/conf/log4perl-json.conf
@@ -13,10 +13,12 @@ log4perl.appender.Cereal.layout.max_json_length = 16384
 # The default is Bugzilla. This is the "Logger" field
 # in https://wiki.mozilla.org/Firefox/Services/Logging#MozLog_JSON_schema
 #and it might be useful to pass in different values for different jobs.
-log4perl.appender.Cereal.layout.name = Bugzilla
+log4perl.appender.Cereal.layout.name = CEREAL
 
-log4perl.filter.IS_INTERACTIVE = sub { Bugzilla::Logging::is_interactive() }
+log4perl.filter.LOG_TO_STDERR = sub { not $ENV{LOG4PERL_STDERR_DISABLE}  }
 log4perl.appender.Screen = Log::Log4perl::Appender::Screen
-log4perl.appender.Screen.Filter = IS_INTERACTIVE
-log4perl.appender.Screen.layout = Log::Log4perl::Layout::PatternLayout
-log4perl.appender.Screen.layout.ConversionPattern = %-5.5p [%d] [%c] %m{chomp} at %F line %L (%M)%n
+log4perl.appender.Screen.Filter = LOG_TO_STDERR
+log4perl.appender.Screen.stderr = 1
+log4perl.appender.Screen.layout = Log::Log4perl::Layout::Mozilla
+log4perl.appender.Screen.layout.max_json_length = 16384
+log4perl.appender.Screen.layout.name = STDERR

diff --git a/conf/log4perl-test.conf b/conf/log4perl-test.conf
index 34b16307327d7322a8c0bd76f4317c12d4c542da..77fc00af845c8fad57df7d627cdb31c9226a2155 100644 (file)
--- a/conf/log4perl-test.conf
+++ b/conf/log4perl-test.conf
@@ -4,13 +4,14 @@ log4perl.appender.Cereal.PeerAddr=127.0.0.1
 log4perl.appender.Cereal.PeerPort=5880
 log4perl.appender.Cereal.defer_connection=1
 log4perl.appender.Cereal.layout = Log::Log4perl::Layout::PatternLayout
-log4perl.appender.Cereal.layout.ConversionPattern = %-5.5p [%d] [%c] %m{chomp} at %F line %L (%M)%n
+log4perl.appender.Cereal.layout.ConversionPattern = %d %6p | %c | %m{chomp}%n
 
-log4perl.filter.IS_INTERACTIVE = sub { Bugzilla::Logging::is_interactive() }
+log4perl.filter.LOG_TO_STDERR = sub { not $ENV{LOG4PERL_STDERR_DISABLE}  }
 log4perl.appender.Screen = Log::Log4perl::Appender::Screen
-log4perl.appender.Screen.Filter = IS_INTERACTIVE
+log4perl.appender.Screen.Filter = LOG_TO_STDERR
+log4perl.appender.Screen.stderr = 1
 log4perl.appender.Screen.layout = Log::Log4perl::Layout::PatternLayout
-log4perl.appender.Screen.layout.ConversionPattern = %-5.5p [%d] [%c] %m{chomp} at %F line %L (%M)%n
+log4perl.appender.Screen.layout.ConversionPattern = %d %6p | %c | %m{chomp}%n
 
 log4perl.appender.File = Log::Log4perl::Appender::File
 log4perl.appender.File.layout = Log::Log4perl::Layout::Mozilla

diff --git a/conf/log4perl-vagrant.conf b/conf/log4perl-vagrant.conf
new file mode 100644 (file)
index 0000000..d9c3e27
--- /dev/null
+++ b/conf/log4perl-vagrant.conf
@@ -0,0 +1,24 @@
+log4perl.rootLogger = DEBUG, TextFile, JSONFile, Screen
+
+log4perl.appender.TextFile = Log::Log4perl::Appender::File
+log4perl.appender.TextFile.layout = Log::Log4perl::Layout::PatternLayout
+log4perl.appender.TextFile.filename = sub { Bugzilla::Logging->logfile("bugzilla.log") }
+log4perl.appender.TextFile.layout.ConversionPattern = %-5.5p [%d] [%c] %m{chomp} at %F line %L (%M)%n
+log4perl.appender.TextFile.create_at_logtime = true
+log4perl.appender.TextFile.owner = vagrant
+log4perl.appender.TextFile.group = apache
+
+log4perl.appender.JSONFile = Log::Log4perl::Appender::File
+log4perl.appender.JSONFile.layout = Log::Log4perl::Layout::Mozilla
+log4perl.appender.JSONFile.filename = sub { Bugzilla::Logging->logfile("bugzilla-json.log") }
+log4perl.appender.JSONFile.mode = append
+log4perl.appender.JSONFile.create_at_logtime = true
+log4perl.appender.JSONFile.owner = vagrant
+log4perl.appender.JSONFile.group = apache
+
+log4perl.filter.LOG_TO_STDERR = sub { not $ENV{LOG4PERL_STDERR_DISABLE}  }
+log4perl.appender.Screen = Log::Log4perl::Appender::Screen
+log4perl.appender.Screen.Filter = LOG_TO_STDERR
+log4perl.appender.Screen.stderr = 1
+log4perl.appender.Screen.layout = Log::Log4perl::Layout::PatternLayout
+log4perl.appender.Screen.layout.ConversionPattern = %d %6p | %c | %m{chomp}%n

diff --git a/docker-compose.yml b/docker-compose.yml
index 5dfb6fad44a788227b72af620e24e63427c3b406..1ca6f5c90bac05ed1c4e82b6d897cddf99ff9290 100644 (file)
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -17,7 +17,7 @@ services:
       - /run
     environment: &bmo_env
       - LOCALCONFIG_ENV=1
-      - LOG4PERL_CONFIG_FILE=log4perl-test.conf
+      - LOG4PERL_CONFIG_FILE=log4perl-docker.conf
       - BUGZILLA_UNSAFE_AUTH_DELEGATION=1
       - PORT=80
       - BMO_db_host=bmo-db.vm

diff --git a/mod_perl.pl b/mod_perl.pl
index 73406be56cea624bbbe63a84bbe4705004bf8004..ff84e523e4086ebba2661057efddc70a72f10752 100644 (file)
--- a/mod_perl.pl
+++ b/mod_perl.pl
@@ -80,7 +80,7 @@ Bugzilla::CGI->compile(qw(:cgi :push));
 # sharing with the other httpd processes.
 my $limit = Bugzilla->localconfig->{apache_size_limit};
 if ($OSNAME eq 'linux' && ! eval { require Linux::Smaps }) {
-    warn "SizeLimit requires Linux::Smaps on linux. size limit set to 800MB";
+    WARN('SizeLimit requires Linux::Smaps on linux. size limit set to 800MB');
     $limit = 800_000;
 }
 Apache2::SizeLimit->set_max_unshared_size($limit);
@@ -167,13 +167,11 @@ sub handler : method {
         DB::enable_profile($file);
     }
     Bugzilla::init_page();
-    my $start = Time::HiRes::time();
     my $result = $class->SUPER::handler(@_);
     if (Bugzilla::ModPerl::USE_NYTPROF) {
         DB::disable_profile();
         DB::finish_profile();
     }
-    warn "[request_time] ", Bugzilla->cgi->request_uri, " took ", Time::HiRes::time() - $start, " seconds to execute";
 
     # When returning data from the REST api we must only return 200 or 304,
     # which tells Apache not to append its error html documents to the

diff --git a/vagrant_support/apache.j2 b/vagrant_support/apache.j2
index 773672fa14da86c83aeebb24b25a7bc3cafd2da9..2222de41b06db08c9fe53f24f91bf66dbaecc226 100644 (file)
--- a/vagrant_support/apache.j2
+++ b/vagrant_support/apache.j2
@@ -1,6 +1,8 @@
 PerlSwitches -wT
 PerlSetEnv USE_NYTPROF 0
 PerlSetEnv BUGZILLA_UNSAFE_AUTH_DELEGATION 1
+PerlPassEnv LOG4PERL_CONFIG_FILE
+PerlSetEnv LOG4PERL_STDERR_DISABLE=1
 PerlConfigRequire /vagrant/mod_perl.pl
 
 <IfModule mpm_prefork_module>

diff --git a/vagrant_support/playbook.yml b/vagrant_support/playbook.yml
index f38c1fc06226cb204eb80041d18d712d05e366dc..b394f5592fd3e2046109b67bfea888f18737049f 100644 (file)
--- a/vagrant_support/playbook.yml
+++ b/vagrant_support/playbook.yml
@@ -50,6 +50,15 @@
         group: root
         mode: 0644
 
+    - name: 'add LOG4PERL_CONFIG_FILE'
+      lineinfile:
+        dest: /etc/environment
+        regexp: 'LOG4PERL_CONFIG_FILE='
+        line: 'LOG4PERL_CONFIG_FILE=log4perl-vagrant.conf'
+        owner: root
+        group: root
+        mode: 0644
+
     - name: copy ntp.conf
       copy:
         src: ntp.conf