sub ES_TYPE {'bug'}
+sub ES_INDEX { Bugzilla->params->{elasticsearch_index} }
+
+sub ES_SETTINGS {
+ return {
+ number_of_shards => 2,
+ analysis => {
+ filter => {
+ asciifolding_original => {
+ type => "asciifolding",
+ preserve_original => \1,
+ },
+ },
+ analyzer => {
+ autocomplete => {
+ type => 'custom',
+ tokenizer => 'keyword',
+ filter => [ 'lowercase', 'asciifolding_original' ],
+ },
+ folding => {
+ tokenizer => 'standard',
+ filter => [ 'standard', 'lowercase', 'asciifolding_original' ],
+ },
+ bz_text_analyzer => {
+ type => 'standard',
+ filter => [ 'lowercase', 'stop' ],
+ max_token_length => '20'
+ },
+ bz_equals_analyzer => {
+ type => 'custom',
+ filter => ['lowercase'],
+ tokenizer => 'keyword',
+ },
+ whiteboard_words => {
+ type => 'custom',
+ tokenizer => 'whiteboard_words_pattern',
+ filter => ['stop']
+ },
+ whiteboard_shingle_words => {
+ type => 'custom',
+ tokenizer => 'whiteboard_words_pattern',
+ filter => [ 'stop', 'shingle', 'lowercase' ]
+ },
+ whiteboard_tokens => {
+ type => 'custom',
+ tokenizer => 'whiteboard_tokens_pattern',
+ filter => [ 'stop', 'lowercase' ]
+ },
+ whiteboard_shingle_tokens => {
+ type => 'custom',
+ tokenizer => 'whiteboard_tokens_pattern',
+ filter => [ 'stop', 'shingle', 'lowercase' ]
+ }
+ },
+ tokenizer => {
+ whiteboard_tokens_pattern => {
+ type => 'pattern',
+ pattern => '\\s*([,;]*\\[|\\][\\s\\[]*|[;,])\\s*'
+ },
+ whiteboard_words_pattern => {
+ type => 'pattern',
+ pattern => '[\\[\\];,\\s]+'
+ },
+ },
+ },
+ };
+}
+
sub _bz_field {
my ($field, @fields) = @_;
with 'Bugzilla::Elastic::Role::ChildObject';
-use constant ES_TYPE => 'comment';
-use constant ES_PARENT_TYPE => 'bug';
+use constant ES_TYPE => 'comment';
+use constant ES_PARENT_CLASS => 'Bugzilla::Bug';
sub ES_OBJECTS_AT_ONCE { 50 }
use Bugzilla::Util qw(trick_taint);
with 'Bugzilla::Elastic::Role::HasClient';
-with 'Bugzilla::Elastic::Role::HasIndexName';
sub suggest_users {
my ($self, $text) = @_;
my $result = eval {
$self->client->suggest(
- index => $self->index_name,
+ index => Bugzilla::User->ES_INDEX,
body => {
$field => {
text => $text,
use List::MoreUtils qw(natatime);
use Storable qw(dclone);
use Scalar::Util qw(looks_like_number);
+use Time::HiRes;
use namespace::clean;
with 'Bugzilla::Elastic::Role::HasClient';
-with 'Bugzilla::Elastic::Role::HasIndexName';
has 'shadow_dbh' => ( is => 'lazy' );
predicate => 'has_progress_bar',
);
-sub create_index {
- my ($self) = @_;
- my $indices = $self->client->indices;
-
- $indices->create(
- index => $self->index_name,
- body => {
- settings => {
- number_of_shards => 2,
- analysis => {
- filter => {
- asciifolding_original => {
- type => "asciifolding",
- preserve_original => \1,
- },
- },
- analyzer => {
- autocomplete => {
- type => 'custom',
- tokenizer => 'keyword',
- filter => ['lowercase', 'asciifolding_original'],
- },
- folding => {
- tokenizer => 'standard',
- filter => ['standard', 'lowercase', 'asciifolding_original'],
- },
- bz_text_analyzer => {
- type => 'standard',
- filter => ['lowercase', 'stop'],
- max_token_length => '20'
- },
- bz_equals_analyzer => {
- type => 'custom',
- filter => ['lowercase'],
- tokenizer => 'keyword',
- },
- whiteboard_words => {
- type => 'custom',
- tokenizer => 'whiteboard_words_pattern',
- filter => ['stop']
- },
- whiteboard_shingle_words => {
- type => 'custom',
- tokenizer => 'whiteboard_words_pattern',
- filter => ['stop', 'shingle', 'lowercase']
- },
- whiteboard_tokens => {
- type => 'custom',
- tokenizer => 'whiteboard_tokens_pattern',
- filter => ['stop', 'lowercase']
- },
- whiteboard_shingle_tokens => {
- type => 'custom',
- tokenizer => 'whiteboard_tokens_pattern',
- filter => ['stop', 'shingle', 'lowercase']
- }
- },
- tokenizer => {
- whiteboard_tokens_pattern => {
- type => 'pattern',
- pattern => '\\s*([,;]*\\[|\\][\\s\\[]*|[;,])\\s*'
- },
- whiteboard_words_pattern => {
- type => 'pattern',
- pattern => '[\\[\\];,\\s]+'
- },
- },
- },
- },
- }
- ) unless $indices->exists(index => $self->index_name);
+
+sub _create_index {
+ my ($self, $class) = @_;
+ my $indices = $self->client->indices;
+ my $index_name = $class->ES_INDEX;
+
+ unless ($indices->exists(index => $index_name)) {
+ $indices->create(
+ index => $index_name,
+ body => { settings => $class->ES_SETTINGS },
+ );
+ }
}
sub _bulk_helper {
my ($self, $class) = @_;
return $self->client->bulk_helper(
- index => $self->index_name,
+ index => $class->ES_INDEX,
type => $class->ES_TYPE,
);
}
-
sub _find_largest {
my ($self, $class, $field) = @_;
my $result = $self->client->search(
- index => $self->index_name,
+ index => $class->ES_INDEX,
type => $class->ES_TYPE,
body => {
aggs => { $field => { extended_stats => { field => $field } } },
return $self->_find_largest($class, $class->ID_FIELD);
}
-sub put_mapping {
+sub _put_mapping {
my ($self, $class) = @_;
my %body = ( properties => scalar $class->ES_PROPERTIES );
}
$self->client->indices->put_mapping(
- index => $self->index_name,
+ index => $class->ES_INDEX,
type => $class->ES_TYPE,
body => \%body,
);
sub bulk_load {
my ( $self, $class ) = @_;
+ $self->_create_index($class);
+
my $bulk = $self->_bulk_helper($class);
my $last_mtime = $self->_find_largest_mtime($class);
my $last_id = $self->_find_largest_id($class);
my $new_ids = $self->_select_all_ids($class, $last_id);
my $updated_ids = $self->_select_updated_ids($class, $last_mtime);
- $self->put_mapping($class);
+ $self->_put_mapping($class);
$self->_bulk_load_ids($bulk, $class, $new_ids) if @$new_ids;
$self->_bulk_load_ids($bulk, $class, $updated_ids) if @$updated_ids;
sub bulk_load_ids {
my ($self, $class, $ids) = @_;
- $self->put_mapping($class);
+ $self->_create_index($class);
+ $self->_put_mapping($class);
$self->_bulk_load_ids($self->_bulk_helper($class), $class, $ids);
}
}
my $total = 0;
- use Time::HiRes;
my $start = time;
while (my @ids = $iter->()) {
if ($progress_bar) {
with 'Bugzilla::Elastic::Role::Object';
-requires qw(ES_PARENT_TYPE es_parent_id);
+requires qw(ES_PARENT_CLASS es_parent_id);
+
+sub ES_PARENT_TYPE { $_[0]->ES_PARENT_CLASS->ES_TYPE }
+sub ES_INDEX { $_[0]->ES_PARENT_CLASS->ES_INDEX }
+sub ES_SETTINGS { $_[0]->ES_PARENT_CLASS->ES_SETTINGS }
1;
+++ /dev/null
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-#
-# This Source Code Form is "Incompatible With Secondary Licenses", as
-# defined by the Mozilla Public License, v. 2.0.
-package Bugzilla::Elastic::Role::HasIndexName;
-
-use 5.10.1;
-use Moo::Role;
-use Search::Elasticsearch;
-
-has 'index_name' => ( is => 'ro', default => sub { Bugzilla->params->{elasticsearch_index} } );
-
-
-1;
use 5.10.1;
use Role::Tiny;
-requires qw(ES_TYPE ES_PROPERTIES es_document);
+requires qw(ES_TYPE ES_INDEX ES_SETTINGS ES_PROPERTIES es_document);
requires qw(ID_FIELD DB_TABLE);
sub ES_OBJECTS_AT_ONCE { 100 }
return $doc;
};
+
+
1;
has 'invalid_order_columns' => ( is => 'lazy' );
with 'Bugzilla::Elastic::Role::HasClient';
-with 'Bugzilla::Elastic::Role::HasIndexName';
with 'Bugzilla::Elastic::Role::Search';
my @SUPPORTED_FIELDS = qw(
my $body = $self->es_query;
my $result = eval {
$self->client->search(
- index => $self->index_name,
- type => 'bug',
- body => $body,
+ index => Bugzilla::Bug->ES_INDEX,
+ type => Bugzilla::Bug->ES_TYPE,
+ body => $body,
);
};
die $@ unless $result;
with 'Bugzilla::Elastic::Role::Object';
+sub ES_INDEX {
+ my ($class) = @_;
+ sprintf("%s_%s", Bugzilla->params->{elasticsearch_index}, $class->ES_TYPE);
+}
+
sub ES_TYPE { 'user' }
sub ES_OBJECTS_AT_ONCE { 5000 }
return ("SELECT $id FROM $table WHERE $id > ? AND is_enabled AND NOT disabledtext ORDER BY $id", [$last_id // 0]);
}
+sub ES_SETTINGS {
+ return {
+ number_of_shards => 2,
+ analysis => {
+ filter => {
+ asciifolding_original => {
+ type => "asciifolding",
+ preserve_original => \1,
+ },
+ },
+ analyzer => {
+ autocomplete => {
+ type => 'custom',
+ tokenizer => 'keyword',
+ filter => [ 'lowercase', 'asciifolding_original' ],
+ },
+ folding => {
+ tokenizer => 'standard',
+ filter => [ 'standard', 'lowercase', 'asciifolding_original' ],
+ },
+ }
+ }
+ };
+}
+
sub ES_PROPERTIES {
return {
suggest_user => {
}
my $indexer = Bugzilla::Elastic::Indexer->new(
- $debug_sql ? ( debug_sql => 1 ) : (),
+ $debug_sql ? ( debug_sql => 1 ) : (),
$progress_bar ? ( progress_bar => 'Term::ProgressBar' ) : (),
);
-$indexer->create_index;
-
my $run_time = time;
my $loop = IO::Async::Loop->new;
my $timer = IO::Async::Timer::Periodic->new(
projects / thirdparty / bugzilla.git / commitdiff
