From 5fdfb4164ba31f29439dc324e6b4e0b6ac3c60c2 Mon Sep 17 00:00:00 2001 From: Dylan William Hardison Date: Tue, 16 Oct 2018 17:33:25 -0400 Subject: [PATCH] Bug 1495741 - memory issues: options to control memory usage --- Bugzilla/Install/Localconfig.pm | 9 ++- Bugzilla/Quantum.pm | 2 + Bugzilla/Quantum/Plugin/SizeLimit.pm | 77 ++++++++++++++++++++++++ Makefile.PL | 2 +- README.rst | 10 ++- conf/checksetup_answers.txt | 2 +- scripts/c9-install | 2 +- template/en/default/setup/strings.txt.pl | 9 ++- vagrant_support/checksetup_answers.j2 | 2 +- 9 files changed, 103 insertions(+), 12 deletions(-) create mode 100644 Bugzilla/Quantum/Plugin/SizeLimit.pm diff --git a/Bugzilla/Install/Localconfig.pm b/Bugzilla/Install/Localconfig.pm index 08cd9e80b..4b25b966b 100644 --- a/Bugzilla/Install/Localconfig.pm +++ b/Bugzilla/Install/Localconfig.pm @@ -23,6 +23,7 @@ use Bugzilla::Constants; use Bugzilla::Install::Util qw(bin_loc install_string); use Bugzilla::Util qw(generate_random_password wrap_hard); +use Mojo::JSON qw(encode_json); use Data::Dumper; use File::Basename qw(dirname); use English qw($EGID); @@ -148,8 +149,12 @@ use constant LOCALCONFIG_VARS => ( }, }, { - name => 'apache_size_limit', - default => 600000, + name => 'setrlimit', + default => encode_json({ RLIMIT_AS => 2e9 }), + }, + { + name => 'size_limit', + default => 750000, }, { name => 'memcached_servers', diff --git a/Bugzilla/Quantum.pm b/Bugzilla/Quantum.pm index 7f359830b..927729614 100644 --- a/Bugzilla/Quantum.pm +++ b/Bugzilla/Quantum.pm @@ -40,6 +40,8 @@ sub startup { $self->plugin('Bugzilla::Quantum::Plugin::Glue'); $self->plugin('Bugzilla::Quantum::Plugin::Hostage') unless $ENV{BUGZILLA_DISABLE_HOSTAGE}; + $self->plugin('Bugzilla::Quantum::Plugin::SizeLimit') + unless $ENV{BUGZILLA_DISABLE_SIZELIMIT}; $self->plugin('Bugzilla::Quantum::Plugin::BlockIP'); $self->plugin('Bugzilla::Quantum::Plugin::Helpers'); diff --git a/Bugzilla/Quantum/Plugin/SizeLimit.pm b/Bugzilla/Quantum/Plugin/SizeLimit.pm new file mode 100644 index 000000000..bd8d26443 --- /dev/null +++ b/Bugzilla/Quantum/Plugin/SizeLimit.pm @@ -0,0 +1,77 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Quantum::Plugin::SizeLimit; +use 5.10.1; +use Mojo::Base 'Mojolicious::Plugin'; +use Mojo::JSON qw(decode_json); +use Bugzilla::Logging; + +use constant MIN_SIZE_LIMIT => 750_000; +use constant HAVE_LINUX_SMAPS_TINY => eval { require Linux::Smaps::Tiny }; +use constant HAVE_BSD_RESOURCE => eval { require BSD::Resource }; + +BEGIN { + if (HAVE_LINUX_SMAPS_TINY) { + Linux::Smaps::Tiny->import('get_smaps_summary'); + } + if (HAVE_BSD_RESOURCE) { + BSD::Resource->import; + } +} + +my @RESOURCES = qw( + RLIMIT_CPU RLIMIT_FSIZE RLIMIT_DATA RLIMIT_STACK RLIMIT_CORE RLIMIT_RSS RLIMIT_MEMLOCK RLIMIT_NPROC RLIMIT_NOFILE + RLIMIT_OFILE RLIMIT_OPEN_MAX RLIMIT_LOCKS RLIMIT_AS RLIMIT_VMEM RLIMIT_PTHREAD RLIMIT_TCACHE RLIMIT_AIO_MEM + RLIMIT_AIO_OPS RLIMIT_FREEMEM RLIMIT_NTHR RLIMIT_NPTS RLIMIT_RSESTACK RLIMIT_SBSIZE RLIMIT_SWAP RLIMIT_MSGQUEUE + RLIMIT_RTPRIO RLIMIT_RTTIME RLIMIT_SIGPENDING +); + +my %RESOURCE; +if (HAVE_BSD_RESOURCE) { + $RESOURCE{$_} = eval $_ for @RESOURCES; +} + +sub register { + my ($self, $app, $conf) = @_; + + if (HAVE_BSD_RESOURCE) { + my $setrlimit = decode_json(Bugzilla->localconfig->{setrlimit}); + + # This trick means the master process will not a size limit. + Mojo::IOLoop->next_tick(sub { + foreach my $resource (keys %$setrlimit) { + setrlimit($RESOURCE{$resource}, $setrlimit->{$resource}, $setrlimit->{$resource}); + } + }); + } + + if (HAVE_LINUX_SMAPS_TINY) { + my $size_limit = Bugzilla->localconfig->{size_limit}; + return unless $size_limit; + + if ($size_limit < MIN_SIZE_LIMIT) { + WARN(sprintf "size_limit cannot be smaller than %d", MIN_SIZE_LIMIT); + $size_limit = MIN_SIZE_LIMIT; + } + + $app->hook( + after_dispatch => sub { + my $c = shift; + my $summary = get_smaps_summary(); + if ($summary->{Size} >= $size_limit) { + my $diff = $summary->{Size} - $size_limit; + INFO("memory size exceeded $size_limit by $diff ($summary->{Size})"); + $c->res->headers->connection('close'); + Mojo::IOLoop->singleton->stop_gracefully; + } + } + ); + } +} + +1; diff --git a/Makefile.PL b/Makefile.PL index 27a9e989e..b73080bea 100755 --- a/Makefile.PL +++ b/Makefile.PL @@ -300,7 +300,7 @@ my %optional_features = ( }, linux_smaps => { description => 'Linux::Smaps::Tiny for limiting memory usage', - prereqs => {runtime => {requires => {'Linux::Smaps::Tiny' => '0'}}}, + prereqs => {runtime => {requires => {'Linux::Smaps::Tiny' => '0', 'BSD::Resource' => 0}}}, }, linux_pdeath => { description => 'Linux::Pdeathsig for a good parent/child relationships', diff --git a/README.rst b/README.rst index a857a207d..1b556d915 100644 --- a/README.rst +++ b/README.rst @@ -362,9 +362,13 @@ BMO_shadowdbhost BMO_shadowdbport The port of the read-only database. -BMO_apache_size_limit - This is the max amount of unshared memory (in kb) that the apache process is - allowed to use before Apache::SizeLimit kills it. +BMO_setrlimit + This is a json object and can set any limit described in https://metacpan.org/pod/BSD::Resource. + Typically it used for setting RLIMIT_AS, and the default value is ``{ "RLIMIT_AS": 2000000000 }``. + +BMO_size_limit + This is the max amount of unshared memory the worker processes are allowed to + use before they will exit. Minimum 750000 (750MiB) BMO_mail_delivery_method Usually configured on the MTA section of admin interface, but may be set here for testing purposes. diff --git a/conf/checksetup_answers.txt b/conf/checksetup_answers.txt index f5c7559d6..402039ffc 100644 --- a/conf/checksetup_answers.txt +++ b/conf/checksetup_answers.txt @@ -4,7 +4,7 @@ $answer{'ADMIN_PASSWORD'} = 'vagrant01!'; $answer{'passwdqc_min'} = '8, 8, 8, 8, 8'; $answer{'ADMIN_REALNAME'} = 'Vagrant User'; $answer{'NO_PAUSE'} = 1; -$answer{'apache_size_limit'} = 700000; +$answer{'size_limit'} = 750000; $answer{'bugzilla_version'} = '1'; $answer{'create_htaccess'} = '1'; $answer{'db_check'} = 1; diff --git a/scripts/c9-install b/scripts/c9-install index d5bc537b4..a446e2b82 100755 --- a/scripts/c9-install +++ b/scripts/c9-install @@ -35,7 +35,7 @@ $answer{'ADMIN_OK'} = 'Y'; $answer{'ADMIN_PASSWORD'} = 'bmo4c9rocks'; $answer{'ADMIN_REALNAME'} = 'BMO Admin'; $answer{'NO_PAUSE'} = 1; -$answer{'apache_size_limit'} = 700000; +$answer{'size_limit'} = 750000; $answer{'bugzilla_version'} = '1'; $answer{'create_htaccess'} = ''; $answer{'diffpath'} = '/usr/bin'; diff --git a/template/en/default/setup/strings.txt.pl b/template/en/default/setup/strings.txt.pl index cb0ac4fe9..f5522aa00 100644 --- a/template/en/default/setup/strings.txt.pl +++ b/template/en/default/setup/strings.txt.pl @@ -272,9 +272,12 @@ and you cannot set this up any other way. YOU HAVE BEEN WARNED! If you set this to anything other than "", you will need to run checksetup.pl as ##root## or as a user who is a member of the specified group. END - localconfig_apache_size_limit => < < < <