Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ESConfig: configurable Elasticsearch document types to allow splitting index #1296

Merged
merged 31 commits into from
Oct 28, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
1489794
add MetaCPAN::ESConfig module to centralize ES config
haarg Oct 22, 2024
edb8e78
adapt Mapping script to use ESConfig module
haarg Oct 22, 2024
4e2c206
remove analysis configuration from MetaCPAN::Model
haarg Oct 22, 2024
1ce41ec
explicitly load modules for ES sub types
haarg Oct 23, 2024
94c84ea
configure MetaCPAN::Model via ESConfig
haarg Oct 22, 2024
491e640
check for compilation errors in document set modules
haarg Oct 26, 2024
aff19bf
disable critic rule prohibiting prototypes
haarg Oct 23, 2024
967b34e
add doc method to MetaCPAN::Model
haarg Oct 23, 2024
f9f7dea
mapping script: require specifying a source index if copying
haarg Oct 23, 2024
a2e9202
Scripts: refresh all indices
haarg Oct 23, 2024
2f5b7d1
find ESXM types via model rather than index
haarg Oct 23, 2024
0e55e2e
find es index/type via ESConfig rather than passing index around
haarg Oct 23, 2024
d709a5e
fix model type alias for ESBool
haarg Oct 23, 2024
373da0d
query modules don't need to be given an index name anymore
haarg Oct 23, 2024
74fcce0
remove out of date comment
haarg Oct 23, 2024
75b7021
no need to prebuild an index object
haarg Oct 23, 2024
7097883
add ES and ESModel Catalyst model classes
haarg Oct 23, 2024
f72bdf1
testserver: get mapping via esconfig
haarg Oct 23, 2024
c44ac51
mapping: list document types, not types from index
haarg Oct 23, 2024
c61c516
scripts don't need index method
haarg Oct 23, 2024
e5c5376
ESConfig: overrides with undef removes element
haarg Oct 26, 2024
ca7b31f
create distributions with upsert
haarg Oct 26, 2024
762f54d
default backup all indexes
haarg Oct 26, 2024
407b552
validate index is actually deleted
haarg Oct 26, 2024
b9c246d
check indexes and aliases we're actually using
haarg Oct 26, 2024
374abed
remove old ES config from test config
haarg Oct 26, 2024
a29eff9
always use suggester for autocomplete
haarg Oct 27, 2024
c0b70ae
download_url: account for elasticsearch 6
haarg Oct 27, 2024
38877ad
fix script query syntax when using newer Elasticsearch
haarg Oct 27, 2024
8d40746
remove use_dis_max from query_string query
haarg Oct 27, 2024
d6571ec
ignore system indexes when deleting for testing
haarg Oct 27, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .perlcriticrc
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ theme = core
[-ValuesAndExpressions::ProhibitNoisyQuotes]
[-Variables::ProhibitPunctuationVars]

# doesn't understand signatures
[-Subroutines::ProhibitSubroutinePrototypes]

[CodeLayout::RequireTrailingCommas]
severity = 4

Expand Down
25 changes: 4 additions & 21 deletions lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use Moose;
extends 'Catalyst::Plugin::Session::Store';
use MooseX::Types::ElasticSearch qw( ES );

use MetaCPAN::ESConfig qw( es_doc_path );
use MetaCPAN::Server::Config ();
use MetaCPAN::Util qw( true false );

Expand All @@ -17,26 +18,12 @@ has _session_es => (
default =>
sub { MetaCPAN::Server::Config::config()->{elasticsearch_servers} },
);
has _session_es_index => (
required => 1,
is => 'ro',
default => sub { shift->_session_plugin_config->{index} || 'user' }
);
has _session_es_type => (
required => 1,
is => 'ro',
default => sub { shift->_session_plugin_config->{type} || 'session' }
);

sub get_session_data {
my ( $self, $key ) = @_;
if ( my ($sid) = $key =~ /^\w+:(.*)/ ) {
my $data = eval {
$self->_session_es->get(
index => $self->_session_es_index,
type => $self->_session_es_type,
id => $sid,
);
$self->_session_es->get( es_doc_path('session'), id => $sid, );
} || return undef;
if ( $key =~ /^expires:/ ) {
return $data->{_source}->{_expires};
Expand All @@ -52,8 +39,7 @@ sub store_session_data {
if ( my ($sid) = $key =~ /^session:(.*)/ ) {
$session->{_expires} = $self->session_expires;
$self->_session_es->index(
index => $self->_session_es_index,
type => $self->_session_es_type,
es_doc_path('session'),
id => $sid,
body => $session,
refresh => true,
Expand All @@ -66,8 +52,7 @@ sub delete_session_data {
if ( my ($sid) = $key =~ /^session:(.*)/ ) {
eval {
$self->_session_es->delete(
index => $self->_session_es_index,
type => $self->_session_es_type,
es_doc_path('session'),
id => $sid,
refresh => true,
);
Expand All @@ -93,8 +78,6 @@ sub delete_expired_sessions { }
MyApp->config(
'Plugin::Session' => {
servers => ':9200',
index => 'user',
type => 'session',
} );

=head1 DESCRIPTION
Expand Down
6 changes: 3 additions & 3 deletions lib/MetaCPAN/API/Model/Cover.pm
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package MetaCPAN::API::Model::Cover;

use MetaCPAN::ESConfig qw( es_doc_path );
use MetaCPAN::Moose;

use MetaCPAN::Util qw(hit_total);
Expand All @@ -12,9 +13,8 @@ sub find_release_coverage {
my $query = +{ term => { release => $release } };

my $res = $self->_run_query(
index => 'cover',
type => 'cover',
body => {
es_doc_path('cover'),
body => {
query => $query,
size => 999,
}
Expand Down
4 changes: 2 additions & 2 deletions lib/MetaCPAN/API/Model/User.pm
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package MetaCPAN::API::Model::User;

use MetaCPAN::ESConfig qw( es_doc_path );
use MetaCPAN::Moose;

with 'MetaCPAN::API::Model::Role::ES';
Expand All @@ -17,8 +18,7 @@ sub lookup {
};

my $res = $self->_run_query(
index => 'user',
type => 'account',
es_doc_path('account'),
body => { query => $query },
search_type => 'dfs_query_then_fetch',
);
Expand Down
5 changes: 1 addition & 4 deletions lib/MetaCPAN/API/Plugin/Model.pm
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,7 @@ has download => sub {

has search => sub {
my $self = shift;
return MetaCPAN::Query::Search->new(
es => $self->app->es,
index_name => 'cpan',
);
return MetaCPAN::Query::Search->new( es => $self->app->es, );
};

has user => sub {
Expand Down
7 changes: 4 additions & 3 deletions lib/MetaCPAN/Document/Author.pm
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@ use ElasticSearchX::Model::Document::Types qw( Location );
use ElasticSearchX::Model::Document;

# load order not important
use Gravatar::URL ();
use MetaCPAN::Types qw( ESBool Profile );
use MetaCPAN::Types::TypeTiny qw(
use Gravatar::URL ();
use MetaCPAN::Document::Author::Profile ();
use MetaCPAN::Types qw( ESBool Profile );
use MetaCPAN::Types::TypeTiny qw(
ArrayRef
ArrayRefPromote
Blog
Expand Down
5 changes: 1 addition & 4 deletions lib/MetaCPAN/Document/Author/Set.pm
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,7 @@ has query_author => (

sub _build_query_author {
my $self = shift;
return MetaCPAN::Query::Author->new(
es => $self->es,
index_name => $self->index->name,
);
return MetaCPAN::Query::Author->new( es => $self->es );
}

__PACKAGE__->meta->make_immutable;
Expand Down
5 changes: 1 addition & 4 deletions lib/MetaCPAN/Document/CVE/Set.pm
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,7 @@ has query_cve => (

sub _build_query_cve {
my $self = shift;
return MetaCPAN::Query::CVE->new(
es => $self->es,
index_name => 'cve',
);
return MetaCPAN::Query::CVE->new( es => $self->es );
}

__PACKAGE__->meta->make_immutable;
Expand Down
5 changes: 1 addition & 4 deletions lib/MetaCPAN/Document/Contributor/Set.pm
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,7 @@ has query_contributor => (

sub _build_query_contributor {
my $self = shift;
return MetaCPAN::Query::Contributor->new(
es => $self->es,
index_name => 'contributor',
);
return MetaCPAN::Query::Contributor->new( es => $self->es );
}

__PACKAGE__->meta->make_immutable;
Expand Down
5 changes: 1 addition & 4 deletions lib/MetaCPAN/Document/Cover/Set.pm
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,7 @@ has query_cover => (

sub _build_query_cover {
my $self = shift;
return MetaCPAN::Query::Cover->new(
es => $self->es,
index_name => 'cover',
);
return MetaCPAN::Query::Cover->new( es => $self->es );
}

__PACKAGE__->meta->make_immutable;
Expand Down
2 changes: 1 addition & 1 deletion lib/MetaCPAN/Document/Distribution.pm
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ has river => (

sub releases {
my $self = shift;
return $self->index->type("release")
return $self->index->model->doc("release")
->query( { term => { "distribution" => $self->name } } );
}

Expand Down
5 changes: 1 addition & 4 deletions lib/MetaCPAN/Document/Distribution/Set.pm
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,7 @@ has query_distribution => (

sub _build_query_distribution {
my $self = shift;
return MetaCPAN::Query::Distribution->new(
es => $self->es,
index_name => 'cpan',
);
return MetaCPAN::Query::Distribution->new( es => $self->es );
}

__PACKAGE__->meta->make_immutable;
Expand Down
5 changes: 1 addition & 4 deletions lib/MetaCPAN/Document/Favorite/Set.pm
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,7 @@ has query_favorite => (

sub _build_query_favorite {
my $self = shift;
return MetaCPAN::Query::Favorite->new(
es => $self->es,
index_name => $self->index->name,
);
return MetaCPAN::Query::Favorite->new( es => $self->es );
}

__PACKAGE__->meta->make_immutable;
Expand Down
Loading