Bug 1393643 - Add whitelist to rate limiting code (#220)

* Bug 1393643 - Add whitelist to rate limiting code

* use version that has new module

* add memcache to bloomfilter loading

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 199b066b0f6cc695e533f24f2861b28d8d62f3cf..a0b13539fb518f44c515a4abdee098eaba3b708f 100644 (file)
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -7,7 +7,7 @@ version: 2
 
 defaults:
   bmo_slim_image: &bmo_slim_image
-    image: mozillabteam/bmo-slim:20170818.1
+    image: mozillabteam/bmo-slim:20170824.1
     user: app
 
   mysql_image: &mysql_image

diff --git a/Bugzilla.pm b/Bugzilla.pm
index cf004d4fceb26778f7b66580556d8a41e4b7678c..bf8f99625699c69a5b84e46d2611e45f8b25153d 100644 (file)
--- a/Bugzilla.pm
+++ b/Bugzilla.pm
@@ -42,6 +42,7 @@ use Bugzilla::Token;
 use Bugzilla::User;
 use Bugzilla::Util;
 use Bugzilla::CPAN;
+use Bugzilla::Bloomfilter;
 
 use Bugzilla::Metrics::Collector;
 use Bugzilla::Metrics::Template;
@@ -765,7 +766,7 @@ sub elastic {
 }
 
 sub check_rate_limit {
-    my ($class, $name, $id) = @_;
+    my ($class, $name, $ip) = @_;
     my $params = Bugzilla->params;
     if ($params->{rate_limit_active}) {
         my $rules = decode_json($params->{rate_limit_rules});
@@ -774,9 +775,15 @@ sub check_rate_limit {
              warn "no rules for $name!";
              return 0;
         }
-        if (Bugzilla->memcached->should_rate_limit("$name:$id", @$limit)) {
-            Bugzilla->audit("[rate_limit] $id exceeds rate limit $name: " . join("/", @$limit));
-            ThrowUserError("rate_limit");
+        if (Bugzilla->memcached->should_rate_limit("$name:$ip", @$limit)) {
+            my $action = 'block';
+            my $filter = Bugzilla::Bloomfilter->lookup("rate_limit_whitelist");
+            if ($filter && $filter->test($ip)) {
+                $action = 'ignore';
+            }
+            my $limit = join("/", @$limit);
+            Bugzilla->audit("[rate_limit] action=$action, ip=$ip, limit=$limit");
+            ThrowUserError("rate_limit") if $action eq 'block';
         }
     }
 }

diff --git a/Bugzilla/Bloomfilter.pm b/Bugzilla/Bloomfilter.pm
new file mode 100644 (file)
index 0000000..0d329b2
--- /dev/null
+++ b/Bugzilla/Bloomfilter.pm
@@ -0,0 +1,67 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This Source Code Form is "Incompatible With Secondary Licenses", as
+# defined by the Mozilla Public License, v. 2.0.
+
+package Bugzilla::Bloomfilter;
+
+use 5.10.1;
+use strict;
+use warnings;
+
+use Bugzilla::Constants;
+use Algorithm::BloomFilter;
+use File::Temp qw(tempfile);
+
+sub _new_bloom_filter {
+    my ($n) = @_;
+    my $p = 0.01;
+    my $m = $n * abs(log $p) / log(2) ** 2;
+    my $k = $m / $n * log(2);
+    return Algorithm::BloomFilter->new($m, $k);
+}
+
+sub _filename {
+    my ($name) = @_;
+
+    my $datadir = bz_locations->{datadir};
+    return sprintf("%s/%s.bloom", $datadir, $name);
+}
+
+sub populate {
+    my ($class, $name, $items) = @_;
+    my $memcached = Bugzilla->memcached;
+
+    my $filter = _new_bloom_filter(@$items + 0);
+    foreach my $item (@$items) {
+        $filter->add($item);
+    }
+
+    my ($fh, $filename) = tempfile( "${name}XXXXXX", DIR => bz_locations->{datadir}, UNLINK => 0);
+    binmode $fh, ':bytes';
+    print $fh $filter->serialize;
+    close $fh;
+    rename($filename, _filename($name)) or die "failed to rename $filename: $!";
+    $memcached->clear_bloomfilter({name => $name});
+}
+
+sub lookup {
+    my ($class, $name) = @_;
+    my $memcached   = Bugzilla->memcached;
+    my $filename    = _filename($name);
+    my $filter_data = $memcached->get_bloomfilter( { name => $name } );
+
+    if (!$filter_data && -f $filename) {
+        open my $fh, '<:bytes', $filename;
+        local $/ = undef;
+        $filter_data = <$fh>;
+        close $fh;
+        $memcached->set_bloomfilter({ name => $name, filter => $filter_data });
+    }
+
+    return Algorithm::BloomFilter->deserialize($filter_data);
+}
+
+1;

diff --git a/Bugzilla/Memcached.pm b/Bugzilla/Memcached.pm
index 233db31f273107a284b516fc53cd350cf53a0795..1623296f8a2e35dbb073170705ea64c93e811002 100644 (file)
--- a/Bugzilla/Memcached.pm
+++ b/Bugzilla/Memcached.pm
@@ -127,6 +127,41 @@ sub get_config {
     }
 }
 
+sub set_bloomfilter {
+    my ($self, $args) = @_;
+    return unless $self->{memcached};
+    if (exists $args->{name}) {
+        return $self->_set($self->_bloomfilter_prefix . '.' . $args->{name}, $args->{filter});
+    }
+    else {
+        ThrowCodeError('params_required', { function => "Bugzilla::Memcached::set_bloomfilter",
+                                            params   => [ 'name' ] });
+    }
+}
+
+sub get_bloomfilter {
+    my ($self, $args) = @_;
+    return unless $self->{memcached};
+    if (exists $args->{name}) {
+        return $self->_get($self->_bloomfilter_prefix . '.' . $args->{name});
+    }
+    else {
+        ThrowCodeError('params_required', { function => "Bugzilla::Memcached::set_bloomfilter",
+                                            params   => [ 'name' ] });
+    }
+}
+
+sub clear_bloomfilter {
+    my ($self, $args) = @_;
+    return unless $self->{memcached};
+    if ($args && exists $args->{name}) {
+        $self->_delete($self->_config_prefix . '.' . $args->{name});
+    }
+    else {
+        $self->_inc_prefix("bloomfilter");
+    }
+}
+
 sub clear {
     my ($self, $args) = @_;
     return unless $self->{memcached};
@@ -244,6 +279,10 @@ sub _config_prefix {
     return $_[0]->_prefix("config");
 }
 
+sub _bloomfilter_prefix {
+    return $_[0]->_prefix("bloomfilter");
+}
+
 sub _encode_key {
     my ($self, $key) = @_;
     $key = $self->_global_prefix . '.' . uri_escape_utf8($key);

diff --git a/Dockerfile b/Dockerfile
index d6057775e6939d47b8bed12a1cfc2603f92a620f..01a846bc7783b8da24ef78f81b39ccf3ffc1f6e0 100644 (file)
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM mozillabteam/bmo-slim:latest
+FROM mozillabteam/bmo-slim:20170824.1
 MAINTAINER Dylan William Hardison <dylan@mozilla.com>
 
 ENV BUNDLE=https://s3.amazonaws.com/moz-devservices-bmocartons/bmo/vendor.tar.gz

diff --git a/Makefile.PL b/Makefile.PL
index 3217101b8a3289b924643cb9a60378faed1c3873..33319ce92c9cc1faead3f889c596b35acef60933 100755 (executable)
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -37,6 +37,7 @@ BEGIN {
 
 # PREREQ_PM
 my %requires = (
+    'Algorithm::BloomFilter'   => 0,
     'CGI'                      => '<= 3.63',
     'CPAN::Meta::Prereqs'      => '2.132830',
     'CPAN::Meta::Requirements' => '2.121',

diff --git a/scripts/bloomfilter-populate.pl b/scripts/bloomfilter-populate.pl
new file mode 100644 (file)
index 0000000..c591a61
--- /dev/null
+++ b/scripts/bloomfilter-populate.pl
@@ -0,0 +1,21 @@
+#!/usr/bin/perl -w
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+use strict;
+use warnings;
+use lib qw(. lib local/lib/perl5);
+
+use Bugzilla;
+use Bugzilla::Constants;
+use Bugzilla::Bloomfilter;
+
+# set Bugzilla usage mode to USAGE_MODE_CMDLINE
+Bugzilla->usage_mode(USAGE_MODE_CMDLINE);
+
+my $name = shift @ARGV or die "usage: $0 \$name < list\n";
+my @lines = <STDIN>;
+chomp @lines;
+Bugzilla::Bloomfilter->populate($name, \@lines);
+