From: Dylan William Hardison Date: Mon, 1 Apr 2019 16:24:06 +0000 (-0400) Subject: Bug 1506144 - Access bugzilla security bug metadata via STMO X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=938ee70271b73538fb49ac76c16a3743785b4cf9;p=thirdparty%2Fbugzilla.git Bug 1506144 - Access bugzilla security bug metadata via STMO Add system for sending JSON documents to an arbitrary location --- diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a55540cd..41b13594e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ version: 2 defaults: bmo_slim_image: &bmo_slim_image - image: mozillabteam/bmo-slim:20190208.2 + image: mozillabteam/bmo-slim:20190322.1 user: app mysql_image: &mysql_image diff --git a/Bugzilla/App/Command/report_ping.pm b/Bugzilla/App/Command/report_ping.pm new file mode 100644 index 000000000..a08d5f3d6 --- /dev/null +++ b/Bugzilla/App/Command/report_ping.pm @@ -0,0 +1,136 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::App::Command::report_ping; ## no critic (Capitalization) +use Mojo::Base 'Mojolicious::Command'; + +use Bugzilla::Constants; +use JSON::MaybeXS; +use Mojo::File 'path'; +use Mojo::Util 'getopt'; +use PerlX::Maybe 'maybe'; +use Module::Runtime 'require_module'; + +has description => 'send a report ping to a url'; +has usage => sub { shift->extract_usage }; + +sub run { + my ($self, @args) = @_; + my $json + = JSON::MaybeXS->new(convert_blessed => 1, canonical => 1, pretty => 1); + my $report_type = 'Simple'; + my $page = 1; + my ($rows, $base_url, $test, $dump_schema); + + + Bugzilla->usage_mode(USAGE_MODE_CMDLINE); + getopt \@args, + 'base-url|u=s' => \$base_url, + 'page|p=i' => \$page, + 'rows|r=i' => \$rows, + 'dump-schema' => \$dump_schema, + 'report-type=s' => \$report_type, + 'test' => \$test; + + $base_url = 'http://localhost' if $dump_schema || $test; + die $self->usage unless $base_url; + + my $report_class = "Bugzilla::Report::Ping::$report_type"; + require_module($report_class); + my $report = $report_class->new( + model => Bugzilla->dbh->model, + base_url => $base_url, + maybe rows => $rows, + maybe page => $page, + ); + + if ($dump_schema) { + print $json->encode( $report->validator->schema->data ); + exit; + } + + my $rs = $report->resultset; + if ($test) { + foreach my $p ($report->page .. $report->pager->last_page) { + # get the next page, except for page 1. + $rs = $rs->page($p) if $p > $report->page; + say "Testing page $p of ", $report->pager->last_page; + foreach my $result ($rs->all) { + my @error = $report->test($result); + if (@error) { + my (undef, $doc) = $report->prepare($result); + die $json->encode({errors => \@error, result => $doc}); + } + } + } + } + else { + foreach my $p ($report->page .. $report->pager->last_page) { + # get the next page, except for page 1. + $rs = $rs->page($p) if $p > $page; + say "Sending page $p of ", $report->pager->last_page; + Mojo::Promise->all(map { $report->send($_) } $rs->all)->wait; + } + } +} + +1; + +__END__ + +=head1 NAME + +Bugzilla::App::Command::report_ping - descriptionsend a report ping to a url'; + +=head1 SYNOPSIS + + Usage: APPLICATION report_ping + + ./bugzilla.pl report_ping --base-url=http://example.com/path + + Options: + -h, --help Print a brief help message and exits. + -u, --base-url URL to send the json documents to. + -r, --rows num (Optional) Number of requests to send at once. Default: 10. + -p, --page num (Optional) Page to start on. Default: 1 + --report-type word (Optional) Report class to use. Default: Simple + --test Validate the json documents against the json schema. + --dump-schema Print the json schema. + +=head1 DESCRIPTION + +send a report ping to a url. + +=head1 ATTRIBUTES + +L inherits all attributes from +L and implements the following new ones. + +=head2 description + + my $description = $report_ping->description; + $rereport r = $re$port_ping->description('Foo'); + +Short description of this command, used for the command list. + +=head2 usage + + my $usage = $report_ping->usage; + $report_ping = $report_ping->usage('Foo'); + +Usage information for this command, used for the help screen. + +=head1 METHODS + +L inherits all methods from +L and implements the following new ones. + +=head2 run + + $report_ping->run(@ARGV); + +Run this command. diff --git a/Bugzilla/Model/Result/Bug.pm b/Bugzilla/Model/Result/Bug.pm index e4e052e4b..844701584 100644 --- a/Bugzilla/Model/Result/Bug.pm +++ b/Bugzilla/Model/Result/Bug.pm @@ -8,8 +8,16 @@ package Bugzilla::Model::Result::Bug; use Mojo::Base 'DBIx::Class::Core'; +__PACKAGE__->load_components('Helper::Row::NumifyGet'); + __PACKAGE__->table(Bugzilla::Bug->DB_TABLE); __PACKAGE__->add_columns(Bugzilla::Bug->DB_COLUMN_NAMES); +__PACKAGE__->add_columns( + '+bug_id' => {is_numeric => 1}, + '+reporter' => {is_numeric => 1} + '+qa_contact' => {is_numeric => 1} + '+assigned_to' => {is_numeric => 1} +); __PACKAGE__->set_primary_key(Bugzilla::Bug->ID_FIELD); __PACKAGE__->has_one( @@ -27,19 +35,31 @@ __PACKAGE__->might_have( ); __PACKAGE__->has_many( - bug_keywords => 'Bugzilla::Model::Result::BugKeyword', + map_keywords => 'Bugzilla::Model::Result::BugKeyword', 'bug_id' ); -__PACKAGE__->many_to_many(keywords => 'bug_keywords', 'keyword'); +__PACKAGE__->many_to_many(keywords => 'map_keywords', 'keyword'); __PACKAGE__->has_many(flags => 'Bugzilla::Model::Result::Flag', 'bug_id'); __PACKAGE__->has_many( - bug_groups => 'Bugzilla::Model::Result::BugGroup', + map_groups => 'Bugzilla::Model::Result::BugGroup', 'bug_id' ); -__PACKAGE__->many_to_many(groups => 'bug_groups', 'group'); +__PACKAGE__->many_to_many(groups => 'map_groups', 'group'); + +__PACKAGE__->has_many( + map_depends_on => 'Bugzilla::Model::Result::Dependency', + 'dependson' +); +__PACKAGE__->many_to_many(depends_on => 'map_depends_on', 'depends_on'); + +__PACKAGE__->has_many( + map_blocked_by => 'Bugzilla::Model::Result::Dependency', + 'blocked' +); +__PACKAGE__->many_to_many(blocked_by => 'map_depends_on', 'blocked_by'); __PACKAGE__->has_one( product => 'Bugzilla::Model::Result::Product', @@ -52,5 +72,22 @@ __PACKAGE__->has_one( ); +__PACKAGE__->has_many( + map_duplicates => 'Bugzilla::Model::Result::Duplicate', + 'dupe_of' +); + +__PACKAGE__->many_to_many('duplicates', 'map_duplicates', 'duplicate'); + +__PACKAGE__->might_have( map_duplicate_of => 'Bugzilla::Model::Result::Duplicate', 'dupe'); + +sub duplicate_of { + my ($self) = @_; + + my $duplicate = $self->map_duplicate_of; + return $duplicate->duplicate_of if $duplicate; + return undef; +} + 1; diff --git a/Bugzilla/Model/Result/Dependency.pm b/Bugzilla/Model/Result/Dependency.pm new file mode 100644 index 000000000..c9037a97e --- /dev/null +++ b/Bugzilla/Model/Result/Dependency.pm @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Model::Result::Dependency; +use Mojo::Base 'DBIx::Class::Core'; + +__PACKAGE__->load_components('Helper::Row::NumifyGet'); + +__PACKAGE__->table('dependencies'); +__PACKAGE__->add_columns(qw[ blocked dependson ]); +__PACKAGE__->set_primary_key(qw[ blocked dependson ]); + +__PACKAGE__->add_columns( + '+blocked' => {is_numeric => 1}, + '+dependson' => {is_numeric => 1}, +); + +__PACKAGE__->belongs_to( + blocked_by => 'Bugzilla::Model::Result::Bug', + {'foreign.bug_id' => 'self.blocked'} +); + +__PACKAGE__->belongs_to( + depends_on => 'Bugzilla::Model::Result::Bug', + {'foreign.bug_id' => 'self.dependson'} +); + + +1; diff --git a/Bugzilla/Model/Result/Duplicate.pm b/Bugzilla/Model/Result/Duplicate.pm new file mode 100644 index 000000000..04ed3ad73 --- /dev/null +++ b/Bugzilla/Model/Result/Duplicate.pm @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Model::Result::Duplicate; +use Mojo::Base 'DBIx::Class::Core'; + +__PACKAGE__->table('duplicates'); +__PACKAGE__->add_columns(qw[ dupe_of dupe ]); +__PACKAGE__->set_primary_key(qw[ dupe ]); + +__PACKAGE__->belongs_to( + duplicate => 'Bugzilla::Model::Result::Bug', + {'foreign.bug_id' => 'self.dupe'} +); + +__PACKAGE__->belongs_to( + duplicate_of => 'Bugzilla::Model::Result::Bug', + {'foreign.bug_id' => 'self.dupe_of'} +); + +1; diff --git a/Bugzilla/Model/Result/Flag.pm b/Bugzilla/Model/Result/Flag.pm index 5bb36f08e..6a2a1a393 100644 --- a/Bugzilla/Model/Result/Flag.pm +++ b/Bugzilla/Model/Result/Flag.pm @@ -8,8 +8,20 @@ package Bugzilla::Model::Result::Flag; use Mojo::Base 'DBIx::Class::Core'; +__PACKAGE__->load_components('Helper::Row::NumifyGet'); + __PACKAGE__->table(Bugzilla::Flag->DB_TABLE); __PACKAGE__->add_columns(Bugzilla::Flag->DB_COLUMN_NAMES); + +__PACKAGE__->add_columns( + '+id' => {is_numeric => 1}, + '+type_id' => {is_numeric => 1}, + '+bug_id' => {is_numeric => 1}, + '+attach_id' => {is_numeric => 1}, + '+setter_id' => {is_numeric => 1}, + '+requestee_id' => {is_numeric => 1}, +); + __PACKAGE__->set_primary_key(Bugzilla::Flag->ID_FIELD); __PACKAGE__->belongs_to(bug => 'Bugzilla::Model::Result::Bug', 'bug_id'); diff --git a/Bugzilla/Model/Result/User.pm b/Bugzilla/Model/Result/User.pm index d127c6bea..921fc37d4 100644 --- a/Bugzilla/Model/Result/User.pm +++ b/Bugzilla/Model/Result/User.pm @@ -8,8 +8,11 @@ package Bugzilla::Model::Result::User; use Mojo::Base 'DBIx::Class::Core'; +__PACKAGE__->load_components('Helper::Row::NumifyGet'); + __PACKAGE__->table(Bugzilla::User->DB_TABLE); __PACKAGE__->add_columns(Bugzilla::User->DB_COLUMN_NAMES); +__PACKAGE__->add_columns('+userid' => {is_numeric => 1}); __PACKAGE__->set_primary_key(Bugzilla::User->ID_FIELD); sub name { diff --git a/Bugzilla/Report/Ping.pm b/Bugzilla/Report/Ping.pm new file mode 100644 index 000000000..c2a32479b --- /dev/null +++ b/Bugzilla/Report/Ping.pm @@ -0,0 +1,109 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Report::Ping; +use 5.10.1; +use Moo::Role; + +use Type::Utils qw(class_type); +use Bugzilla::Types qw(URL); +use Types::Standard qw(Str Num Int); +use Scalar::Util qw(blessed); +use JSON::Validator; +use Mojo::Promise; + +has 'model' => + (is => 'ro', required => 1, isa => class_type({class => 'Bugzilla::Model'})); + +has '_base_url' => ( + is => 'ro', + init_arg => 'base_url', + required => 1, + isa => URL, + coerce => 1, + handles => {base_url => 'clone'} +); + +has 'page' => (is => 'ro', isa => Int, default => 1); + +has 'rows' => (is => 'ro', default => 10); + +has 'user_agent' => ( + is => 'lazy', + init_arg => undef, + isa => class_type({class => 'Mojo::UserAgent'}) +); + +sub _build_user_agent { + return Mojo::UserAgent->new; +} + +has 'validator' => ( + is => 'lazy', + init_arg => undef, + isa => class_type({class => 'JSON::Validator'}), + handles => ['validate'], +); + +requires '_build_validator'; + +has 'resultset' => ( + is => 'lazy', + init_arg => undef, + isa => class_type({class => 'DBIx::Class::ResultSet'}), + handles => ['pager'], +); + +requires '_build_resultset'; + +around '_build_resultset' => sub { + my ($method, $self, @args) = @_; + my $rs = $self->$method(@args); + $rs = $rs->rows($self->rows)->page($self->page) if defined $rs; + + return $rs; +}; + +has 'namespace' => (is => 'lazy', init_arg => undef, isa => Str); + +sub _build_namespace { + return 'bugzilla'; +} + +has 'doctype' => (is => 'lazy', init_arg => undef, isa => Str); + +sub _build_doctype { + my ($self) = @_; + my @class_parts = split(/::/, blessed $self); + return lc $class_parts[-1]; +} + +has 'docversion' => (is => 'lazy', init_arg => undef, isa => Num); + +sub _build_docversion { + my ($self) = @_; + return $self->VERSION; +} + +requires 'prepare'; + +sub send { + my ($self, $row) = @_; + my ($id, $doc) = $self->prepare($row); + my $url = $self->base_url; + push @{$url->path}, $self->namespace, $self->doctype, $self->docversion, $id; + return $self->user_agent->put_p($url, json => $doc); +} + +sub test { + my ($self, $row) = @_; + my ($id, $doc) = $self->prepare($row); + + return $self->validate($doc); +} + +1; diff --git a/Bugzilla/Report/Ping/Simple.pm b/Bugzilla/Report/Ping/Simple.pm new file mode 100644 index 000000000..89aa5bd0d --- /dev/null +++ b/Bugzilla/Report/Ping/Simple.pm @@ -0,0 +1,109 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::Report::Ping::Simple; +use 5.10.1; +use Moo; + +use JSON::Validator qw(joi); + +our $VERSION = '1'; + +with 'Bugzilla::Report::Ping'; + +sub _build_validator { + my ($self) = @_; + + # For prototyping we use joi, but after protyping + # $schema should be set to the file path or url of a json schema file. + my $schema = joi->object->strict->props({ + reporter => joi->integer->required, + assigned_to => joi->integer->required, + qa_contact => joi->type([qw[null integer]])->required, + bug_id => joi->integer->required->min(1), + product => joi->string->required, + component => joi->string->required, + bug_status => joi->string->required, + keywords => joi->array->required->items(joi->string)->required, + groups => joi->array->required->items(joi->string)->required, + flags => joi->array->required->items(joi->object->strict->props({ + name => joi->string->required, + status => joi->string->enum([qw[? + -]])->required, + setter_id => joi->integer->required, + requestee_id => joi->type([qw[null integer]])->required, + })), + priority => joi->string->required, + bug_severity => joi->string->required, + resolution => joi->string, + blocked_by => joi->array->required->items(joi->integer), + depends_on => joi->array->required->items(joi->integer), + duplicate_of => joi->type([qw[null integer]])->required, + duplicates => joi->array->required->items(joi->integer), + target_milestone => joi->string->required, + version => joi->string->required, + }); + + return JSON::Validator->new( + schema => Mojo::JSON::Pointer->new($schema->compile)); +} + + +sub _build_resultset { + my ($self) = @_; + my $bugs = $self->model->resultset('Bug'); + my $query = {}; + my $options = { + order_by => 'me.bug_id', + }; + return $bugs->search($query, $options); +} + +sub prepare { + my ($self, $bug) = @_; + my $doc = { + reporter => $bug->reporter->id, + assigned_to => $bug->assigned_to->id, + qa_contact => $bug->qa_contact ? $bug->qa_contact->id : undef, + bug_id => 0 + $bug->id, + product => $bug->product->name, + component => $bug->component->name, + bug_status => $bug->bug_status, + priority => $bug->priority, + resolution => $bug->resolution, + bug_severity => $bug->bug_severity, + keywords => [map { $_->name } $bug->keywords->all], + groups => [map { $_->name } $bug->groups->all], + duplicate_of => $bug->duplicate_of ? $bug->duplicate_of->id : undef, + duplicates => [map { $_->id } $bug->duplicates->all ], + version => $bug->version, + target_milestone => $bug->target_milestone, + blocked_by => [ + map { $_->dependson } $bug->map_blocked_by->all + ], + depends_on => [ + map { $_->blocked } $bug->map_depends_on->all + ], + flags => [ + map { $self->_prepare_flag($_) } $bug->flags->all + ], + }; + + return ($bug->id, $doc); +} + +sub _prepare_flag { + my ($self, $flag) = @_; + + return { + name => $flag->type->name, + status => $flag->status, + requestee_id => $flag->requestee_id, + setter_id => $flag->setter_id, + }; +} + +1; diff --git a/Bugzilla/Types.pm b/Bugzilla/Types.pm index 63422c8b7..d464bb890 100644 --- a/Bugzilla/Types.pm +++ b/Bugzilla/Types.pm @@ -12,7 +12,7 @@ use strict; use warnings; use Type::Library -base, - -declare => qw( Bug User Group Attachment Comment JSONBool Task ); + -declare => qw( Bug User Group Attachment Comment JSONBool URI URL Task ); use Type::Utils -all; use Types::Standard -types; @@ -22,6 +22,11 @@ class_type Group, {class => 'Bugzilla::Group'}; class_type Attachment, {class => 'Bugzilla::Attachment'}; class_type Comment, {class => 'Bugzilla::Comment'}; class_type JSONBool, {class => 'JSON::PP::Boolean'}; +class_type URI {class => 'URI'}; +class_type URL {class => 'Mojo::URL'}; role_type Task, {role => 'Bugzilla::Task'}; +coerce URL, from Str() => q{ Mojo::URL->new($_) }, + from URI() => q{ Mojo::URL->new("$_") }; + 1; diff --git a/Dockerfile b/Dockerfile index ffaaede99..dd37244cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM mozillabteam/bmo-slim:20190208.2 +FROM mozillabteam/bmo-slim:20190322.1 ARG CI ARG CIRCLE_SHA1 diff --git a/Makefile.PL b/Makefile.PL index ad1e692c0..fc586951e 100755 --- a/Makefile.PL +++ b/Makefile.PL @@ -65,6 +65,7 @@ my %requires = ( 'IO::Async' => '0.71', 'IPC::System::Simple' => 0, 'JSON::MaybeXS' => '1.003008', + 'JSON::Validator' => '3.05', 'JSON::XS' => '2.01', 'LWP::Protocol::https' => '6.07', 'LWP::UserAgent' => '6.26', diff --git a/t/report-ping-simple.t b/t/report-ping-simple.t new file mode 100644 index 000000000..f1f532b4a --- /dev/null +++ b/t/report-ping-simple.t @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. +use 5.10.1; +use strict; +use warnings; +use lib qw( . lib local/lib/perl5 ); + +BEGIN { + unlink('data/db/report_ping_simple') if -f 'data/db/report_ping_simple'; + $ENV{test_db_name} = 'report_ping_simple'; +} + +use Bugzilla::Test::MockDB; +use Bugzilla::Test::MockParams (password_complexity => 'no_constraints'); +use Bugzilla::Test::Util qw(create_bug create_user); +use Bugzilla; +use Bugzilla::Constants; +use Bugzilla::Hook; +BEGIN { Bugzilla->extensions } +use Test2::V0; +use Test2::Tools::Mock qw(mock mock_accessor); +use Test2::Tools::Exception qw(dies lives); +use PerlX::Maybe qw(provided); +use ok 'Bugzilla::Report::Ping::Simple'; + +Bugzilla->dbh->model->resultset('Keyword') + ->create({name => 'regression', description => 'the regression keyword'}); + +my $user = create_user('reportuser@invalid.tld', '*'); +Bugzilla->set_user($user); +create_bug( + short_desc => "test bug $_", + comment => "Hello, world: $_", + provided $_ % 3 == 0, keywords => ['regression'], + assigned_to => 'reportuser@invalid.tld' +) for (1..250); + +my $report = Bugzilla::Report::Ping::Simple->new( + base_url => 'http://localhost', + model => Bugzilla->dbh->model, +); + +my $rs = $report->resultset->page(1); +is($rs->count, 10, "got 10 items"); +my $pager = $rs->pager; +is($pager->last_page, 25, "got 25 pages"); + +is($rs->first->id, 1, "first bug of page 1 is 1"); + +my ($first, $second, $third, @rest) = $rs->all; +{ + my ($id, $doc) = $report->prepare( $first ); + is($id, 1, "doc id is 1"); + is($doc->{product}, 'Firefox'); + is($doc->{keywords}, []); + is([map { "$_" } $report->validate($doc)], [], "No errors for first doc"); +} + +{ + my ($id, $doc) = $report->prepare( $third ); + is($id, 3, "doc id is 3"); + is($doc->{product}, 'Firefox'); + is($doc->{keywords}, ['regression']); +} + +{ + my $rs2 = $rs->page($pager->next_page); + my $pager2 = $rs2->pager; + + is($rs2->first->id, 11, "first bug of page 2 is 11"); + isnt($pager, $pager2, "pagers are different"); +} + +done_testing; +