diff --git a/lib/MetaCPAN/Document/Author/Set.pm b/lib/MetaCPAN/Document/Author/Set.pm deleted file mode 100644 index 2b041ed9d..000000000 --- a/lib/MetaCPAN/Document/Author/Set.pm +++ /dev/null @@ -1,23 +0,0 @@ -package MetaCPAN::Document::Author::Set; - -use Moose; - -use MetaCPAN::Query::Author (); - -extends 'ElasticSearchX::Model::Document::Set'; - -has query_author => ( - is => 'ro', - isa => 'MetaCPAN::Query::Author', - lazy => 1, - builder => '_build_query_author', - handles => [qw< by_ids by_user search prefix_search >], -); - -sub _build_query_author { - my $self = shift; - return MetaCPAN::Query::Author->new( es => $self->es ); -} - -__PACKAGE__->meta->make_immutable; -1; diff --git a/lib/MetaCPAN/Document/CVE/Set.pm b/lib/MetaCPAN/Document/CVE/Set.pm deleted file mode 100644 index 4602a7c5e..000000000 --- a/lib/MetaCPAN/Document/CVE/Set.pm +++ /dev/null @@ -1,27 +0,0 @@ -package MetaCPAN::Document::CVE::Set; - -use Moose; - -use MetaCPAN::Query::CVE (); - -extends 'ElasticSearchX::Model::Document::Set'; - -has query_cve => ( - is => 'ro', - isa => 'MetaCPAN::Query::CVE', - lazy => 1, - builder => '_build_query_cve', - handles => [ qw< - find_cves_by_cpansa - find_cves_by_release - find_cves_by_dist - > ], -); - -sub _build_query_cve { - my $self = shift; - return MetaCPAN::Query::CVE->new( es => $self->es ); -} - -__PACKAGE__->meta->make_immutable; -1; diff --git a/lib/MetaCPAN/Document/Contributor/Set.pm b/lib/MetaCPAN/Document/Contributor/Set.pm deleted file mode 100644 index 9e9b91f60..000000000 --- a/lib/MetaCPAN/Document/Contributor/Set.pm +++ /dev/null @@ -1,23 +0,0 @@ -package MetaCPAN::Document::Contributor::Set; - -use Moose; - -use MetaCPAN::Query::Contributor (); - -extends 'ElasticSearchX::Model::Document::Set'; - -has query_contributor => ( - is => 'ro', - isa => 'MetaCPAN::Query::Contributor', - lazy => 1, - builder => '_build_query_contributor', - handles => [qw< find_author_contributions find_release_contributors >], -); - -sub _build_query_contributor { - my $self = shift; - return MetaCPAN::Query::Contributor->new( es => $self->es ); -} - -__PACKAGE__->meta->make_immutable; -1; diff --git a/lib/MetaCPAN/Document/Cover/Set.pm b/lib/MetaCPAN/Document/Cover/Set.pm deleted file mode 100644 index 52b833bdc..000000000 --- a/lib/MetaCPAN/Document/Cover/Set.pm +++ /dev/null @@ -1,23 +0,0 @@ -package MetaCPAN::Document::Cover::Set; - -use Moose; - -use MetaCPAN::Query::Cover (); - -extends 'ElasticSearchX::Model::Document::Set'; - -has query_cover => ( - is => 'ro', - isa => 'MetaCPAN::Query::Cover', - lazy => 1, - builder => '_build_query_cover', - handles => [qw< find_release_coverage >], -); - -sub _build_query_cover { - my $self = shift; - return MetaCPAN::Query::Cover->new( es => $self->es ); -} - -__PACKAGE__->meta->make_immutable; -1; diff --git a/lib/MetaCPAN/Document/Distribution/Set.pm b/lib/MetaCPAN/Document/Distribution/Set.pm deleted file mode 100644 index b7d4ece89..000000000 --- a/lib/MetaCPAN/Document/Distribution/Set.pm +++ /dev/null @@ -1,23 +0,0 @@ -package MetaCPAN::Document::Distribution::Set; - -use Moose; - -use MetaCPAN::Query::Distribution (); - -extends 'ElasticSearchX::Model::Document::Set'; - -has query_distribution => ( - is => 'ro', - isa => 'MetaCPAN::Query::Distribution', - lazy => 1, - builder => '_build_query_distribution', - handles => [qw< get_river_data_by_dist get_river_data_by_dists >], -); - -sub _build_query_distribution { - my $self = shift; - return MetaCPAN::Query::Distribution->new( es => $self->es ); -} - -__PACKAGE__->meta->make_immutable; -1; diff --git a/lib/MetaCPAN/Document/Favorite/Set.pm b/lib/MetaCPAN/Document/Favorite/Set.pm deleted file mode 100644 index 60b0649a6..000000000 --- a/lib/MetaCPAN/Document/Favorite/Set.pm +++ /dev/null @@ -1,27 +0,0 @@ -package MetaCPAN::Document::Favorite::Set; - -use Moose; - -use MetaCPAN::Query::Favorite (); - -extends 'ElasticSearchX::Model::Document::Set'; - -has query_favorite => ( - is => 'ro', - isa => 'MetaCPAN::Query::Favorite', - lazy => 1, - builder => '_build_query_favorite', - handles => [ qw< agg_by_distributions - by_user - leaderboard - recent - users_by_distribution > ], -); - -sub _build_query_favorite { - my $self = shift; - return MetaCPAN::Query::Favorite->new( es => $self->es ); -} - -__PACKAGE__->meta->make_immutable; -1; diff --git a/lib/MetaCPAN/Document/File/Set.pm b/lib/MetaCPAN/Document/File/Set.pm index 569bda6df..3e3cd66bb 100644 --- a/lib/MetaCPAN/Document/File/Set.pm +++ b/lib/MetaCPAN/Document/File/Set.pm @@ -2,214 +2,11 @@ package MetaCPAN::Document::File::Set; use Moose; -use List::Util qw( max ); -use MetaCPAN::ESConfig qw( es_doc_path ); -use MetaCPAN::Query::Favorite (); -use MetaCPAN::Query::File (); -use MetaCPAN::Query::Release (); -use MetaCPAN::Util qw( true false ); +use MetaCPAN::ESConfig qw( es_doc_path ); +use MetaCPAN::Util qw( true false ); extends 'ElasticSearchX::Model::Document::Set'; -has query_file => ( - is => 'ro', - isa => 'MetaCPAN::Query::File', - lazy => 1, - builder => '_build_query_file', - handles => [ qw( - dir - interesting_files - files_by_category - ) ], -); - -sub _build_query_file { - my $self = shift; - return MetaCPAN::Query::File->new( es => $self->es ); -} - -has query_favorite => ( - is => 'ro', - isa => 'MetaCPAN::Query::Favorite', - lazy => 1, - builder => '_build_query_favorite', - handles => [qw< agg_by_distributions >], -); - -sub _build_query_favorite { - my $self = shift; - return MetaCPAN::Query::Favorite->new( es => $self->es ); -} - -has query_release => ( - is => 'ro', - isa => 'MetaCPAN::Query::Release', - lazy => 1, - builder => '_build_query_release', - handles => [qw< find_download_url >], -); - -sub _build_query_release { - my $self = shift; - return MetaCPAN::Query::Release->new( es => $self->es ); -} - -my @ROGUE_DISTRIBUTIONS = qw( - Acme-DependOnEverything - Bundle-Everything - kurila - perl-5.005_02+apache1.3.3+modperl - perlbench - perl_debug - pod2texi - spodcxx -); - -sub find { - my ( $self, $module ) = @_; - - my $query = { - bool => { - must => [ - { term => { indexed => true } }, - { term => { authorized => true } }, - { term => { status => 'latest' } }, - { - bool => { - should => [ - { term => { documentation => $module } }, - { - nested => { - path => "module", - query => { - bool => { - must => [ - { - term => { "module.name" => - $module } - }, - { - bool => { should => - [ - { term => - { "module.authorized" - => true - } }, - { exists => - { field => - 'module.associated_pod' - } }, - ], - } - }, - ], - }, - }, - } - }, - ] - } - }, - ], - }, - }; - - my $res = $self->es->search( - es_doc_path('file'), - search_type => 'dfs_query_then_fetch', - body => { - query => $query, - sort => [ - '_score', - { 'version_numified' => { order => 'desc' } }, - { 'date' => { order => 'desc' } }, - { 'mime' => { order => 'asc' } }, - { 'stat.mtime' => { order => 'desc' } } - ], - _source => [ - qw( documentation module.indexed module.authoried module.name ) - ], - size => 100, - }, - ); - - my @candidates = @{ $res->{hits}{hits} }; - - my ($file) = grep { - grep { $_->{indexed} && $_->{authorized} && $_->{name} eq $module } - @{ $_->{module} || [] } - } grep { !$_->{documentation} || $_->{documentation} eq $module } - @candidates; - - $file ||= shift @candidates; - return $file ? $self->get( $file->{_id} ) : undef; -} - -sub find_pod { - my ( $self, $name ) = @_; - my $file = $self->find($name); - return $file unless ($file); - my ($module) - = grep { $_->indexed && $_->authorized && $_->name eq $name } - @{ $file->module || [] }; - if ( $module && ( my $pod = $module->associated_pod ) ) { - my ( $author, $release, @path ) = split( /\//, $pod ); - return $self->get( { - author => $author, - release => $release, - path => join( '/', @path ), - } ); - } - else { - return $file; - } -} - -sub documented_modules { - my ( $self, $release ) = @_; - return $self->query( { - bool => { - must => [ - { term => { release => $release->{name} } }, - { term => { author => $release->{author} } }, - { exists => { field => "documentation" } }, - { - bool => { - should => [ - { - bool => { - must => [ - { - exists => - { field => 'module.name' } - }, - { - term => - { 'module.indexed' => true } - }, - ], - } - }, - { - bool => { - must => [ - { - exists => - { field => 'pod.analyzed' } - }, - { term => { indexed => true } }, - ], - } - }, - ], - } - }, - ], - }, - } )->size(999) - ->source( [qw(name module path documentation distribution)] )->all; -} - =head2 history Find the history of a given module/documentation. @@ -271,154 +68,5 @@ sub history { return $search->sort( [ { date => 'desc' } ] ); } -sub _autocomplete { - my ( $self, $query ) = @_; - - my $search_size = 100; - - my $sugg_res = $self->es->search( - es_doc_path('file'), - body => { - suggest => { - documentation => { - text => $query, - completion => { - field => "suggest", - size => $search_size, - }, - }, - } - }, - ); - - my %docs; - for my $suggest ( @{ $sugg_res->{suggest}{documentation}[0]{options} } ) { - $docs{ $suggest->{text} } = max grep {defined} - ( $docs{ $suggest->{text} }, $suggest->{score} ); - } - - my $res = $self->es->search( - es_doc_path('file'), - body => { - query => { - bool => { - must => [ - { term => { indexed => true } }, - { term => { authorized => true } }, - { term => { status => 'latest' } }, - { terms => { documentation => [ keys %docs ] } }, - ], - must_not => [ - { - terms => { distribution => \@ROGUE_DISTRIBUTIONS } - }, - ], - } - }, - _source => [ qw( - author - date - deprecated - distribution - documentation - release - ) ], - size => $search_size, - }, - ); - - my $hits = $res->{hits}{hits}; - - my $fav_res - = $self->agg_by_distributions( - [ map $_->{_source}{distribution}, @$hits ] ); - - my $favs = $fav_res->{favorites}; - - my %valid = map { - my $source = $_->{_source}; - ( - $source->{documentation} => { - %$source, favorites => $favs->{ $source->{distribution} }, - } - ); - } @{ $res->{hits}{hits} }; - - # remove any exact match, it will be added later - my $exact = delete $valid{$query}; - - no warnings 'uninitialized'; - my @sorted = map { $valid{$_} } - sort { - my $a_data = $valid{$a}; - my $b_data = $valid{$b}; - $a_data->{deprecated} <=> $b_data->{deprecated} - || $b_data->{favorites} <=> $a_data->{favorites} - || $docs{$b} <=> $docs{$a} - || length($a) <=> length($b) - || $a cmp $b - } - keys %valid; - - return { - took => $sugg_res->{took} + $res->{took} + $fav_res->{took}, - suggestions => \@sorted, - }; -} - -sub autocomplete { - my ( $self, @terms ) = @_; - my $data = $self->_autocomplete( join ' ', @terms ); - - return { - took => $data->{took}, - hits => { - hits => [ - map { - my $source = $_; - +{ - fields => { - map +( $_ => $source->{$_} ), qw( - documentation - release - author - distribution - ), - }, - }; - } @{ $data->{suggestions} } - ], - }, - }; -} - -sub autocomplete_suggester { - my ( $self, @terms ) = @_; - my $data = $self->_autocomplete( join ' ', @terms ); - - return { - took => $data->{took}, - suggestions => [ - map +{ - author => $_->{author}, - date => $_->{date}, - deprecated => $_->{deprecated}, - distribution => $_->{distribution}, - name => $_->{documentation}, - release => $_->{release}, - }, - @{ $data->{suggestions} } - ], - }; -} - -sub find_changes_files { - my ( $self, $author, $release ) = @_; - my $result = $self->files_by_category( $author, $release, ['changelog'], - { _source => true } ); - my ($file) = @{ $result->{categories}{changelog} || [] }; - return $file; -} - __PACKAGE__->meta->make_immutable; 1; diff --git a/lib/MetaCPAN/Document/Mirror/Set.pm b/lib/MetaCPAN/Document/Mirror/Set.pm deleted file mode 100644 index 7fb91a04a..000000000 --- a/lib/MetaCPAN/Document/Mirror/Set.pm +++ /dev/null @@ -1,23 +0,0 @@ -package MetaCPAN::Document::Mirror::Set; - -use Moose; - -use MetaCPAN::Query::Mirror (); - -extends 'ElasticSearchX::Model::Document::Set'; - -has query_mirror => ( - is => 'ro', - isa => 'MetaCPAN::Query::Mirror', - lazy => 1, - builder => '_build_query_mirror', - handles => [qw< search >], -); - -sub _build_query_mirror { - my $self = shift; - return MetaCPAN::Query::Mirror->new( es => $self->es ); -} - -__PACKAGE__->meta->make_immutable; -1; diff --git a/lib/MetaCPAN/Document/Package/Set.pm b/lib/MetaCPAN/Document/Package/Set.pm deleted file mode 100644 index 981c61c53..000000000 --- a/lib/MetaCPAN/Document/Package/Set.pm +++ /dev/null @@ -1,23 +0,0 @@ -package MetaCPAN::Document::Package::Set; - -use Moose; - -use MetaCPAN::Query::Package (); - -extends 'ElasticSearchX::Model::Document::Set'; - -has query_package => ( - is => 'ro', - isa => 'MetaCPAN::Query::Package', - lazy => 1, - builder => '_build_query_package', - handles => [qw< get_modules >], -); - -sub _build_query_package { - my $self = shift; - return MetaCPAN::Query::Package->new( es => $self->es ); -} - -__PACKAGE__->meta->make_immutable; -1; diff --git a/lib/MetaCPAN/Document/Permission/Set.pm b/lib/MetaCPAN/Document/Permission/Set.pm deleted file mode 100644 index d4192b248..000000000 --- a/lib/MetaCPAN/Document/Permission/Set.pm +++ /dev/null @@ -1,23 +0,0 @@ -package MetaCPAN::Document::Permission::Set; - -use Moose; - -use MetaCPAN::Query::Permission (); - -extends 'ElasticSearchX::Model::Document::Set'; - -has query_permission => ( - is => 'ro', - isa => 'MetaCPAN::Query::Permission', - lazy => 1, - builder => '_build_query_permission', - handles => [qw< by_author by_modules >], -); - -sub _build_query_permission { - my $self = shift; - return MetaCPAN::Query::Permission->new( es => $self->es ); -} - -__PACKAGE__->meta->make_immutable; -1; diff --git a/lib/MetaCPAN/Document/Release/Set.pm b/lib/MetaCPAN/Document/Release/Set.pm deleted file mode 100644 index 7cd365d3c..000000000 --- a/lib/MetaCPAN/Document/Release/Set.pm +++ /dev/null @@ -1,70 +0,0 @@ -package MetaCPAN::Document::Release::Set; - -use Moose; - -use MetaCPAN::Query::Release (); - -extends 'ElasticSearchX::Model::Document::Set'; - -has query_release => ( - is => 'ro', - isa => 'MetaCPAN::Query::Release', - lazy => 1, - builder => '_build_query_release', - handles => [ qw< - activity - all_by_author - author_status - by_author - by_author_and_name - by_author_and_names - find - get_contributors - get_files - latest_by_author - latest_by_distribution - modules - predecessor - recent - requires - reverse_dependencies - top_uploaders - versions - > ], -); - -sub _build_query_release { - my $self = shift; - return MetaCPAN::Query::Release->new( es => $self->es ); -} - -sub find_github_based { - shift->query( { - bool => { - must => [ - { term => { status => 'latest' } }, - { - bool => { - should => [ - { - prefix => { - "resources.bugtracker.web" => - 'http://github.com/' - } - }, - { - prefix => { - "resources.bugtracker.web" => - 'https://github.com/' - } - }, - ], - } - }, - ], - }, - } ); -} - -__PACKAGE__->meta->make_immutable; -1; diff --git a/lib/MetaCPAN/Query.pm b/lib/MetaCPAN/Query.pm new file mode 100644 index 000000000..dda368b21 --- /dev/null +++ b/lib/MetaCPAN/Query.pm @@ -0,0 +1,52 @@ +package MetaCPAN::Query; +use Moose; + +use Module::Runtime qw( require_module ); +use Module::Pluggable::Object (); +use MooseX::Types::ElasticSearch qw( ES ); + +has es => ( + is => 'ro', + required => 1, + isa => ES, + coerce => 1, +); + +my @plugins = Module::Pluggable::Object->new( + search_path => [__PACKAGE__], + max_depth => 3, + require => 0, +)->plugins; + +for my $class (@plugins) { + require_module($class); + my $name = $class->can('name') && $class->name + or next; + + my $in = "_in_$name"; + my $gen = "_gen_$name"; + + has $in => ( + is => 'ro', + init_arg => $name, + weak_ref => 1, + ); + + has $gen => ( + is => 'ro', + init_arg => undef, + lazy => 1, + default => sub { + my $self = shift; + $class->new( + es => $self->es, + query => $self, + ); + }, + ); + + no strict 'refs'; + *$name = sub { $_[0]->$in // $_[0]->$gen }; +} + +1; diff --git a/lib/MetaCPAN/Query/Distribution.pm b/lib/MetaCPAN/Query/Distribution.pm index 7ffd8a506..71140d573 100644 --- a/lib/MetaCPAN/Query/Distribution.pm +++ b/lib/MetaCPAN/Query/Distribution.pm @@ -7,6 +7,20 @@ use MetaCPAN::Util qw(hit_total); with 'MetaCPAN::Query::Role::Common'; +sub rogue_list { + return qw( + Acme-DependOnEverything + Bundle-Everything + kurila + perl-5.005_02+apache1.3.3+modperl + perlbench + perl_debug + perl_mlb + pod2texi + spodcxx + ); +} + sub get_river_data_by_dist { my ( $self, $dist ) = @_; diff --git a/lib/MetaCPAN/Query/File.pm b/lib/MetaCPAN/Query/File.pm index da760f45f..8e7a421ac 100644 --- a/lib/MetaCPAN/Query/File.pm +++ b/lib/MetaCPAN/Query/File.pm @@ -2,6 +2,7 @@ package MetaCPAN::Query::File; use MetaCPAN::Moose; +use List::Util qw( max ); use MetaCPAN::ESConfig qw( es_doc_path ); use MetaCPAN::Util qw( hit_total true false ); @@ -348,5 +349,333 @@ sub files_by_category { return $return; } +sub find_changes_files { + my ( $self, $author, $release ) = @_; + my $result = $self->files_by_category( $author, $release, ['changelog'], + { _source => true } ); + my ($file) = @{ $result->{categories}{changelog} || [] }; + return $file; +} + +sub _autocomplete { + my ( $self, $query ) = @_; + + my $search_size = 100; + + my $sugg_res = $self->es->search( + es_doc_path('file'), + body => { + suggest => { + documentation => { + text => $query, + completion => { + field => "suggest", + size => $search_size, + }, + }, + } + }, + ); + + my %docs; + for my $suggest ( @{ $sugg_res->{suggest}{documentation}[0]{options} } ) { + $docs{ $suggest->{text} } = max grep {defined} + ( $docs{ $suggest->{text} }, $suggest->{score} ); + } + + my $res = $self->es->search( + es_doc_path('file'), + body => { + query => { + bool => { + must => [ + { term => { indexed => true } }, + { term => { authorized => true } }, + { term => { status => 'latest' } }, + { terms => { documentation => [ keys %docs ] } }, + ], + must_not => [ + { + terms => { + distribution => [ + $self->query->distribution->rogue_list + ] + }, + }, + ], + } + }, + _source => [ qw( + author + date + deprecated + distribution + documentation + release + ) ], + size => $search_size, + }, + ); + + my $hits = $res->{hits}{hits}; + + my $fav_res + = $self->query->favorite->agg_by_distributions( + [ map $_->{_source}{distribution}, @$hits ] ); + + my $favs = $fav_res->{favorites}; + + my %valid = map { + my $source = $_->{_source}; + ( + $source->{documentation} => { + %$source, favorites => $favs->{ $source->{distribution} }, + } + ); + } @{ $res->{hits}{hits} }; + + # remove any exact match, it will be added later + my $exact = delete $valid{$query}; + + no warnings 'uninitialized'; + my @sorted = map { $valid{$_} } + sort { + my $a_data = $valid{$a}; + my $b_data = $valid{$b}; + $a_data->{deprecated} <=> $b_data->{deprecated} + || $b_data->{favorites} <=> $a_data->{favorites} + || $docs{$b} <=> $docs{$a} + || length($a) <=> length($b) + || $a cmp $b + } + keys %valid; + + return { + took => $sugg_res->{took} + $res->{took} + $fav_res->{took}, + suggestions => \@sorted, + }; +} + +sub autocomplete { + my ( $self, @terms ) = @_; + my $data = $self->_autocomplete( join ' ', @terms ); + + return { + took => $data->{took}, + hits => { + hits => [ + map { + my $source = $_; + +{ + fields => { + map +( $_ => $source->{$_} ), qw( + documentation + release + author + distribution + ), + }, + }; + } @{ $data->{suggestions} } + ], + }, + }; +} + +sub autocomplete_suggester { + my ( $self, @terms ) = @_; + my $data = $self->_autocomplete( join ' ', @terms ); + + return { + took => $data->{took}, + suggestions => [ + map +{ + author => $_->{author}, + date => $_->{date}, + deprecated => $_->{deprecated}, + distribution => $_->{distribution}, + name => $_->{documentation}, + release => $_->{release}, + }, + @{ $data->{suggestions} } + ], + }; +} + +sub documented_modules { + my ( $self, $author, $release ) = @_; + my $query = { + bool => { + must => [ + { term => { author => $author } }, + { term => { release => $release } }, + { exists => { field => "documentation" } }, + { + bool => { + should => [ + { + bool => { + must => [ + { + exists => + { field => 'module.name' } + }, + { + term => + { 'module.indexed' => true } + }, + ], + } + }, + { + bool => { + must => [ + { + exists => + { field => 'pod.analyzed' } + }, + { term => { indexed => true } }, + ], + } + }, + ], + } + }, + ], + }, + }; + my $res = $self->es->search( + es_doc_path('file'), + body => { + query => $query, + size => 999, + _source => [qw(name module path documentation distribution)], + }, + ); + + return { + took => $res->{took}, + files => [ map $_->{_source}, @{ $res->{hits}{hits} } ], + }; +} + +sub find_module { + my ( $self, $module, $fields ) = @_; + + my $query = { + bool => { + must => [ + { term => { indexed => true } }, + { term => { authorized => true } }, + { term => { status => 'latest' } }, + { + bool => { + should => [ + { term => { documentation => $module } }, + { + nested => { + path => "module", + query => { + bool => { + must => [ + { + term => { "module.name" => + $module } + }, + { + bool => { should => + [ + { term => + { "module.authorized" + => true + } }, + { exists => + { field => + 'module.associated_pod' + } }, + ], + } + }, + ], + }, + }, + } + }, + ] + } + }, + ], + }, + }; + + my $res = $self->es->search( + es_doc_path('file'), + search_type => 'dfs_query_then_fetch', + body => { + query => $query, + sort => [ + '_score', + { 'version_numified' => { order => 'desc' } }, + { 'date' => { order => 'desc' } }, + { 'mime' => { order => 'asc' } }, + { 'stat.mtime' => { order => 'desc' } } + ], + _source => [ + qw( documentation module.indexed module.authoried module.name ) + ], + size => 100, + }, + ); + + my @candidates = @{ $res->{hits}{hits} }; + + my ($file) = grep { + grep { $_->{indexed} && $_->{authorized} && $_->{name} eq $module } + @{ $_->{module} || [] } + } grep { !$_->{documentation} || $_->{documentation} eq $module } + @candidates; + + $file ||= shift @candidates; + return undef + if !$file; + return $self->es->get_source( + es_doc_path('file'), + id => $file->{_id}, + ( $fields ? ( _source => $fields ) : () ), + ); +} + +sub find_pod { + my ( $self, $name ) = @_; + my $file = $self->find_module($name); + return $file + unless $file; + my ($module) + = grep { $_->{indexed} && $_->{authorized} && $_->{name} eq $name } + @{ $file->{module} || [] }; + if ( $module && ( my $pod = $module->{associated_pod} ) ) { + my ( $author, $release, @path ) = split( /\//, $pod ); + my $query = { + bool => { + must => [ + { term => { author => $author } }, + { term => { release => $release } }, + { term => { path => join( '/', @path ) } }, + ], + }, + }; + my $pod_file = $self->es->search( + es_doc_path('file'), + body => { + query => $query, + }, + ); + return $pod_file->{hits}{hits}[0]; + } + else { + return $file; + } +} + __PACKAGE__->meta->make_immutable; 1; diff --git a/lib/MetaCPAN/Query/Role/Common.pm b/lib/MetaCPAN/Query/Role/Common.pm index dd492357a..751d8fa2e 100644 --- a/lib/MetaCPAN/Query/Role/Common.pm +++ b/lib/MetaCPAN/Query/Role/Common.pm @@ -1,7 +1,46 @@ package MetaCPAN::Query::Role::Common; - use Moose::Role; -has es => ( is => 'ro', ); +use MooseX::Types::ElasticSearch qw( ES ); + +has es => ( + is => 'ro', + required => 1, + isa => ES, + coerce => 1, +); + +sub name { + my $self = shift; + my $class = ref $self || $self; + + $class =~ /^MetaCPAN::Query::([^:]+)$/ + or return undef; + return lc $1; +} + +has _in_query => ( + is => 'ro', + init_arg => 'query', + weak_ref => 1, +); + +has _gen_query => ( + is => 'ro', + lazy => 1, + init_arg => undef, + default => sub { + my $self = shift; + my $name = $self->name; + + require MetaCPAN::Query; + MetaCPAN::Query->new( + es => $self->es, + ( $name ? ( $name => $self ) : () ), + ); + }, +); + +sub query { $_[0]->_in_query // $_[0]->_gen_query } 1; diff --git a/lib/MetaCPAN/Query/Search.pm b/lib/MetaCPAN/Query/Search.pm index 9a32e87b2..9628e55db 100644 --- a/lib/MetaCPAN/Query/Search.pm +++ b/lib/MetaCPAN/Query/Search.pm @@ -240,8 +240,11 @@ sub build_query { } }, ], - must_not => - [ { terms => { distribution => \@ROGUE_DISTRIBUTIONS } }, ], + must_not => [ { + terms => { + distribution => [ $self->query->distribution->rogue_list ] + } + } ], must => [ { bool => { diff --git a/lib/MetaCPAN/Script/Tickets.pm b/lib/MetaCPAN/Script/Tickets.pm index 25dd88709..05ce5dac5 100644 --- a/lib/MetaCPAN/Script/Tickets.pm +++ b/lib/MetaCPAN/Script/Tickets.pm @@ -104,14 +104,45 @@ sub index_github_bugs { log_debug {'Fetching GitHub issues'}; - my $scroll - = $self->model->doc('release')->find_github_based->scroll('5m'); + my $scroll = $self->es->scroll_helper( + scroll => '5m', + es_doc_path('release'), + body => { + query => { + bool => { + must => [ + { term => { status => 'latest' } }, + { + bool => { + should => [ + { + prefix => { + "resources.bugtracker.web" => + 'http://github.com/' + } + }, + { + prefix => { + "resources.bugtracker.web" => + 'https://github.com/' + } + }, + ], + }, + }, + ], + }, + }, + sort => ['_doc'], + }, + ); + log_debug { sprintf( "Found %s repos", $scroll->total ) }; my %summary; RELEASE: while ( my $release = $scroll->next ) { - my $resources = $release->resources; + my $resources = $release->{resources}; my ( $user, $repo, $source ) = $self->github_user_repo_from_resources($resources); next unless $user; diff --git a/lib/MetaCPAN/Server/Controller/Activity.pm b/lib/MetaCPAN/Server/Controller/Activity.pm index 811b2f261..34965f8d6 100644 --- a/lib/MetaCPAN/Server/Controller/Activity.pm +++ b/lib/MetaCPAN/Server/Controller/Activity.pm @@ -13,7 +13,7 @@ sub get : Path('') : Args(0) { my ( $self, $c ) = @_; $c->stash_or_detach( - $c->model('ESModel')->doc('release')->activity( $c->req->params ) ); + $c->model('ESQuery')->release->activity( $c->req->params ) ); } __PACKAGE__->meta->make_immutable; diff --git a/lib/MetaCPAN/Server/Controller/Author.pm b/lib/MetaCPAN/Server/Controller/Author.pm index 2d9ff8d4a..43cd9c7a9 100644 --- a/lib/MetaCPAN/Server/Controller/Author.pm +++ b/lib/MetaCPAN/Server/Controller/Author.pm @@ -16,33 +16,34 @@ sub get : Path('') : Args(1) { $c->cdn_max_age('1y'); my $file = $self->model($c)->raw->get($id); $c->stash_or_detach( - $c->model('ESModel')->doc('release')->author_status( $id, $file ) ); + $c->model('ESQuery')->release->author_status( $id, $file ) ); } # /author/search?q=QUERY sub qsearch : Path('search') : Args(0) { my ( $self, $c ) = @_; - $c->stash_or_detach( - $self->model($c)->search( @{ $c->req->params }{qw( q from )} ) ); + $c->stash_or_detach( $c->model('ESQuery') + ->author->search( @{ $c->req->params }{qw( q from )} ) ); } # /author/by_ids?id=PAUSE_ID1&id=PAUSE_ID2... sub by_ids : Path('by_ids') : Args(0) { my ( $self, $c ) = @_; - $c->stash_or_detach( $self->model($c)->by_ids( $c->read_param('id') ) ); + $c->stash_or_detach( + $c->model('ESQuery')->author->by_ids( $c->read_param('id') ) ); } # /author/by_user/USER_ID sub by_user : Path('by_user') : Args(1) { my ( $self, $c, $user ) = @_; - $c->stash_or_detach( $self->model($c)->by_user($user) ); + $c->stash_or_detach( $c->model('ESQuery')->author->by_user($user) ); } # /author/by_user?user=USER_ID1&user=USER_ID2... sub by_users : Path('by_user') : Args(0) { my ( $self, $c ) = @_; $c->stash_or_detach( - $self->model($c)->by_user( $c->read_param('user') ) ); + $c->model('ESQuery')->author->by_user( $c->read_param('user') ) ); } # /author/by_prefix/PAUSE_ID_PREFIX @@ -51,8 +52,11 @@ sub by_prefix : Path('by_prefix') : Args(1) { my $size = $c->req->param('size') // 500; my $from = $c->req->param('from') // 0; - $c->stash_or_detach( $self->model($c) - ->prefix_search( $prefix, { size => $size, from => $from } ) ); + $c->stash_or_detach( + $c->model('ESQuery')->author->prefix_search( + $prefix, { size => $size, from => $from } + ) + ); } 1; diff --git a/lib/MetaCPAN/Server/Controller/CVE.pm b/lib/MetaCPAN/Server/Controller/CVE.pm index 0fd0db983..a6da3de90 100644 --- a/lib/MetaCPAN/Server/Controller/CVE.pm +++ b/lib/MetaCPAN/Server/Controller/CVE.pm @@ -11,20 +11,22 @@ with 'MetaCPAN::Server::Role::JSONP'; sub get : Path('') : Args(1) { my ( $self, $c, $cpansa_id ) = @_; - $c->stash_or_detach( $self->model($c)->find_cves_by_cpansa($cpansa_id) ); + $c->stash_or_detach( + $c->model('ESQuery')->cve->find_cves_by_cpansa($cpansa_id) ); } sub release : Path('release') : Args(2) { my ( $self, $c, $author, $release ) = @_; $c->stash_or_detach( - $self->model($c)->find_cves_by_release( $author, $release ) ); + $c->model('ESQuery')->cve->find_cves_by_release( $author, $release ) + ); } sub dist : Path('dist') : Args(1) { my ( $self, $c, $dist ) = @_; my $version = $c->req->query_params->{version}; $c->stash_or_detach( - $self->model($c)->find_cves_by_dist( $dist, $version ) ); + $c->model('ESQuery')->cve->find_cves_by_dist( $dist, $version ) ); } 1; diff --git a/lib/MetaCPAN/Server/Controller/Changes.pm b/lib/MetaCPAN/Server/Controller/Changes.pm index c775d07bd..f66c06d7a 100644 --- a/lib/MetaCPAN/Server/Controller/Changes.pm +++ b/lib/MetaCPAN/Server/Controller/Changes.pm @@ -25,8 +25,7 @@ sub get : Chained('index') : PathPart('') : Args(2) { $c->cdn_max_age('1y'); my $file - = $c->model('ESModel')->doc('file') - ->find_changes_files( $author, $release ); + = $c->model('ESQuery')->file->find_changes_files( $author, $release ); $file or $c->detach( '/not_found', [] ); my $source = $c->model('Source')->path( @$file{qw(author release path)} ) @@ -53,7 +52,7 @@ sub get : Chained('index') : PathPart('') : Args(2) { sub find : Chained('index') : PathPart('') : Args(1) { my ( $self, $c, $name ) = @_; - my $release = eval { $c->model('ESModel')->doc('release')->find($name); } + my $release = eval { $c->model('ESQuery')->release->find($name); } or $c->detach( '/not_found', [] ); $c->forward( 'get', [ @$release{qw( author name )} ] ); @@ -77,8 +76,8 @@ sub by_releases : Path('by_releases') : Args(0) { return; } - my $ret = $c->model('ESModel')->doc('release') - ->by_author_and_names( \@releases ); + my $ret + = $c->model('ESQuery')->release->by_author_and_names( \@releases ); my @changes; for my $release ( @{ $ret->{releases} } ) { diff --git a/lib/MetaCPAN/Server/Controller/Contributor.pm b/lib/MetaCPAN/Server/Controller/Contributor.pm index f930800f5..11353d1b5 100644 --- a/lib/MetaCPAN/Server/Controller/Contributor.pm +++ b/lib/MetaCPAN/Server/Controller/Contributor.pm @@ -11,14 +11,15 @@ with 'MetaCPAN::Server::Role::JSONP'; sub get : Path('') : Args(2) { my ( $self, $c, $author, $name ) = @_; - $c->stash_or_detach( - $self->model($c)->find_release_contributors( $author, $name ) ); + $c->stash_or_detach( $c->model('ESQuery') + ->contributor->find_release_contributors( $author, $name ) ); } sub by_pauseid : Path('by_pauseid') : Args(1) { my ( $self, $c, $pauseid ) = @_; $c->stash_or_detach( - $self->model($c)->find_author_contributions($pauseid) ); + $c->model('ESQuery')->contributor->find_author_contributions($pauseid) + ); } 1; diff --git a/lib/MetaCPAN/Server/Controller/Cover.pm b/lib/MetaCPAN/Server/Controller/Cover.pm index 1b0eefafb..6eb9011b2 100644 --- a/lib/MetaCPAN/Server/Controller/Cover.pm +++ b/lib/MetaCPAN/Server/Controller/Cover.pm @@ -11,7 +11,8 @@ with 'MetaCPAN::Server::Role::JSONP'; sub get : Path('') : Args(1) { my ( $self, $c, $release ) = @_; - $c->stash_or_detach( $self->model($c)->find_release_coverage($release) ); + $c->stash_or_detach( + $c->model('ESQuery')->cover->find_release_coverage($release) ); } 1; diff --git a/lib/MetaCPAN/Server/Controller/Diff.pm b/lib/MetaCPAN/Server/Controller/Diff.pm index 3789ac9be..f6b6d2bd2 100644 --- a/lib/MetaCPAN/Server/Controller/Diff.pm +++ b/lib/MetaCPAN/Server/Controller/Diff.pm @@ -33,8 +33,8 @@ sub release : Chained('index') : PathPart('release') : Args(1) { my ( $latest, $previous ); try { - $latest = $c->model('ESModel')->doc('release')->find($name); - $previous = $c->model('ESModel')->doc('release')->predecessor($name); + $latest = $c->model('ESQuery')->release->find($name); + $previous = $c->model('ESQuery')->release->predecessor($name); } catch { $c->detach('/not_found'); diff --git a/lib/MetaCPAN/Server/Controller/Distribution.pm b/lib/MetaCPAN/Server/Controller/Distribution.pm index a47a66ec1..767966800 100644 --- a/lib/MetaCPAN/Server/Controller/Distribution.pm +++ b/lib/MetaCPAN/Server/Controller/Distribution.pm @@ -12,13 +12,17 @@ with 'MetaCPAN::Server::Role::JSONP'; sub river_data_by_dist : Path('river') : Args(1) { my ( $self, $c, $dist ) = @_; - $c->stash_or_detach( $self->model($c)->get_river_data_by_dist($dist) ); + $c->stash_or_detach( + $c->model('ESQuery')->distribution->get_river_data_by_dist($dist) ); } sub river_data_by_dists : Path('river') : Args(0) { my ( $self, $c ) = @_; - $c->stash_or_detach( $self->model($c) - ->get_river_data_by_dists( $c->read_param('distribution') ) ); + $c->stash_or_detach( + $c->model('ESQuery')->distribution->get_river_data_by_dists( + $c->read_param('distribution') + ) + ); } __PACKAGE__->meta->make_immutable; diff --git a/lib/MetaCPAN/Server/Controller/Favorite.pm b/lib/MetaCPAN/Server/Controller/Favorite.pm index 43fa2a7e6..f3691fc49 100644 --- a/lib/MetaCPAN/Server/Controller/Favorite.pm +++ b/lib/MetaCPAN/Server/Controller/Favorite.pm @@ -28,20 +28,21 @@ sub find : Path('') : Args(2) { sub by_user : Path('by_user') : Args(1) { my ( $self, $c, $user ) = @_; - $c->stash_or_detach( - $self->model($c)->by_user( $user, $c->req->param('size') || 250 ) ); + $c->stash_or_detach( $c->model('ESQuery') + ->favorite->by_user( $user, $c->req->param('size') || 250 ) ); } sub users_by_distribution : Path('users_by_distribution') : Args(1) { my ( $self, $c, $distribution ) = @_; $c->stash_or_detach( - $self->model($c)->users_by_distribution($distribution) ); + $c->model('ESQuery')->favorite->users_by_distribution($distribution) + ); } sub recent : Path('recent') : Args(0) { my ( $self, $c ) = @_; $c->stash_or_detach( - $self->model($c)->recent( + $c->model('ESQuery')->favorite->recent( $c->req->param('page') || 1, $c->req->param('page_size') || $c->req->param('size') || 100, ) @@ -50,13 +51,13 @@ sub recent : Path('recent') : Args(0) { sub leaderboard : Path('leaderboard') : Args(0) { my ( $self, $c ) = @_; - $c->stash_or_detach( $self->model($c)->leaderboard() ); + $c->stash_or_detach( $c->model('ESQuery')->favorite->leaderboard() ); } sub agg_by_distributions : Path('agg_by_distributions') : Args(0) { my ( $self, $c ) = @_; $c->stash_or_detach( - $self->model($c)->agg_by_distributions( + $c->model('ESQuery')->favorite->agg_by_distributions( $c->read_param('distribution'), $c->req->param('user') # optional ) diff --git a/lib/MetaCPAN/Server/Controller/File.pm b/lib/MetaCPAN/Server/Controller/File.pm index c37c11e23..5b0df974c 100644 --- a/lib/MetaCPAN/Server/Controller/File.pm +++ b/lib/MetaCPAN/Server/Controller/File.pm @@ -31,7 +31,7 @@ sub find : Path('') { sub dir : Path('dir') { my ( $self, $c, @path ) = @_; - $c->stash_or_detach( $self->model($c)->dir(@path) ); + $c->stash_or_detach( $c->model('ESQuery')->file->dir(@path) ); } 1; diff --git a/lib/MetaCPAN/Server/Controller/Mirror.pm b/lib/MetaCPAN/Server/Controller/Mirror.pm index 78cdb3e5e..38a2eefff 100644 --- a/lib/MetaCPAN/Server/Controller/Mirror.pm +++ b/lib/MetaCPAN/Server/Controller/Mirror.pm @@ -11,7 +11,8 @@ with 'MetaCPAN::Server::Role::JSONP'; sub search : Path('search') : Args(0) { my ( $self, $c ) = @_; - $c->stash_or_detach( $self->model($c)->search( $c->req->param('q') ) ); + $c->stash_or_detach( + $c->model('ESQuery')->mirror->search( $c->req->param('q') ) ); } 1; diff --git a/lib/MetaCPAN/Server/Controller/Module.pm b/lib/MetaCPAN/Server/Controller/Module.pm index b2a04cf54..db1b60dfd 100644 --- a/lib/MetaCPAN/Server/Controller/Module.pm +++ b/lib/MetaCPAN/Server/Controller/Module.pm @@ -12,14 +12,12 @@ has '+type' => ( default => 'file' ); sub get : Path('') : Args(1) { my ( $self, $c, $name ) = @_; - my $file = $self->model($c)->raw->find($name); + my $file + = $c->model('ESQuery')->file->find_module( $name, $c->req->fields ); if ( !defined $file ) { $c->detach( '/not_found', [] ); } - $c->stash( $file->{_source} - || single_valued_arrayref_to_scalar( $file->{fields} ) ) - || $c->detach( '/not_found', - ['The requested field(s) could not be found'] ); + $c->stash($file); } __PACKAGE__->meta->make_immutable(); diff --git a/lib/MetaCPAN/Server/Controller/Package.pm b/lib/MetaCPAN/Server/Controller/Package.pm index c2117e64f..c58a49c1a 100644 --- a/lib/MetaCPAN/Server/Controller/Package.pm +++ b/lib/MetaCPAN/Server/Controller/Package.pm @@ -11,11 +11,12 @@ with 'MetaCPAN::Server::Role::JSONP'; sub modules : Path('modules') : Args(1) { my ( $self, $c, $dist ) = @_; - my $last = $c->model('ESModel')->doc('release')->find($dist); + my $last = $c->model('ESQuery')->release->find($dist); $c->detach( '/not_found', ["Cannot find last release for $dist"] ) unless $last; $c->stash_or_detach( - $self->model($c)->get_modules( $dist, $last->{version} ) ); + $c->model('ESQuery')->package->get_modules( $dist, $last->{version} ) + ); } __PACKAGE__->meta->make_immutable; diff --git a/lib/MetaCPAN/Server/Controller/Permission.pm b/lib/MetaCPAN/Server/Controller/Permission.pm index b37704b5a..3eb3ff37d 100644 --- a/lib/MetaCPAN/Server/Controller/Permission.pm +++ b/lib/MetaCPAN/Server/Controller/Permission.pm @@ -9,18 +9,20 @@ with 'MetaCPAN::Server::Role::JSONP'; sub by_author : Path('by_author') : Args(1) { my ( $self, $c, $pauseid ) = @_; - $c->stash_or_detach( $self->model($c)->by_author($pauseid) ); + $c->stash_or_detach( + $c->model('ESQuery')->permission->by_author($pauseid) ); } sub by_module : Path('by_module') : Args(1) { my ( $self, $c, $module ) = @_; - $c->stash_or_detach( $self->model($c)->by_modules($module) ); + $c->stash_or_detach( + $c->model('ESQuery')->permission->by_modules($module) ); } sub by_modules : Path('by_module') : Args(0) { my ( $self, $c ) = @_; - $c->stash_or_detach( - $self->model($c)->by_modules( $c->read_param('module') ) ); + $c->stash_or_detach( $c->model('ESQuery') + ->permission->by_modules( $c->read_param('module') ) ); } __PACKAGE__->meta->make_immutable; diff --git a/lib/MetaCPAN/Server/Controller/Pod.pm b/lib/MetaCPAN/Server/Controller/Pod.pm index b903a211f..9f2edc84b 100644 --- a/lib/MetaCPAN/Server/Controller/Pod.pm +++ b/lib/MetaCPAN/Server/Controller/Pod.pm @@ -36,28 +36,29 @@ sub find : Path('') { sub get : Path('') : Args(1) { my ( $self, $c, $module ) = @_; - $module = $c->model('ESModel')->doc('file')->find_pod($module) + $module = $c->model('ESQuery')->file->find_pod($module) or $c->detach( '/not_found', [] ); - $c->forward( 'find', [ map { $module->$_ } qw(author release path) ] ); + $c->forward( 'find', + [ map { $module->{_source}{$_} } qw(author release path) ] ); } sub find_dist_links { my ( $self, $c, $author, $release, $permalinks ) = @_; - my @modules = $c->model('ESModel')->doc('file') - ->documented_modules( { name => $release, author => $author } ); + my $modules + = $c->model('ESQuery')->file->documented_modules( $author, $release ); + my $files = $modules->{files}; my $links = {}; - for my $file (@modules) { - next - unless $file->has_documentation; - my $name = $file->documentation; + for my $file (@$files) { + my $name = $file->{documentation} + or next; my ($module) - = grep { $_->name eq $name } @{ $file->module }; - if ( $module && $module->authorized && $module->indexed ) { + = grep { $_->{name} eq $name } @{ $file->{module} }; + if ( $module && $module->{authorized} && $module->{indexed} ) { if ($permalinks) { $links->{$name} = join '/', - 'release', $author, $release, $file->path; + 'release', $author, $release, $file->{path}; } else { $links->{$name} = $name; @@ -67,11 +68,11 @@ sub find_dist_links { if exists $links->{$name}; if ($permalinks) { $links->{$name} = join '/', - 'release', $author, $release, $file->path; + 'release', $author, $release, $file->{path}; } else { $links->{$name} = join '/', - 'distribution', $file->distribution, $file->path; + 'distribution', $file->{distribution}, $file->{path}; } } return $links; diff --git a/lib/MetaCPAN/Server/Controller/Release.pm b/lib/MetaCPAN/Server/Controller/Release.pm index 753d88fbf..b71b9bade 100644 --- a/lib/MetaCPAN/Server/Controller/Release.pm +++ b/lib/MetaCPAN/Server/Controller/Release.pm @@ -11,7 +11,7 @@ with 'MetaCPAN::Server::Role::JSONP'; sub find : Path('') : Args(1) { my ( $self, $c, $name ) = @_; - my $file = $self->model($c)->find($name); + my $file = $c->model('ESQuery')->release->find($name); $c->detach( '/not_found', [] ) unless $file; $c->stash($file); } @@ -21,26 +21,28 @@ sub get : Path('') : Args(2) { $c->add_author_key($author); $c->cdn_max_age('1y'); $c->stash_or_detach( - $self->model($c)->raw->by_author_and_name( $author, $name ) ); + $c->model('ESQuery')->release->by_author_and_name( $author, $name ) ); } sub contributors : Path('contributors') : Args(2) { my ( $self, $c, $author, $release ) = @_; $c->stash_or_detach( - $self->model($c)->get_contributors( $author, $release ) ); + $c->model('ESQuery')->release->get_contributors( $author, $release ) + ); } sub files : Path('files') : Args(1) { my ( $self, $c, $name ) = @_; my $files = $c->req->params->{files}; $c->detach( '/bad_request', ['No files requested'] ) unless $files; - $c->stash_or_detach( - $self->model($c)->get_files( $name, [ split /,/, $files ] ) ); + $c->stash_or_detach( $c->model('ESQuery') + ->release->get_files( $name, [ split /,/, $files ] ) ); } sub modules : Path('modules') : Args(2) { my ( $self, $c, $author, $name ) = @_; - $c->stash_or_detach( $self->model($c)->modules( $author, $name ) ); + $c->stash_or_detach( + $c->model('ESQuery')->release->modules( $author, $name ) ); } sub recent : Path('recent') : Args(0) { @@ -49,7 +51,8 @@ sub recent : Path('recent') : Args(0) { my $type = $params->{type}; my $page = $params->{page}; my $size = $params->{page_size}; - $c->stash_or_detach( $self->model($c)->recent( $type, $page, $size ) ); + $c->stash_or_detach( + $c->model('ESQuery')->release->recent( $type, $page, $size ) ); } sub by_author : Path('by_author') : Args(1) { @@ -58,19 +61,21 @@ sub by_author : Path('by_author') : Args(1) { my $page = $params->{page}; my $size = $params->{page_size} // $params->{size}; $c->stash_or_detach( - $self->model($c)->by_author( $pauseid, $page, $size ) ); + $c->model('ESQuery')->release->by_author( $pauseid, $page, $size ) ); } sub latest_by_distribution : Path('latest_by_distribution') : Args(1) { my ( $self, $c, $dist ) = @_; $c->add_dist_key($dist); $c->cdn_max_age('1y'); - $c->stash_or_detach( $self->model($c)->latest_by_distribution($dist) ); + $c->stash_or_detach( + $c->model('ESQuery')->release->latest_by_distribution($dist) ); } sub latest_by_author : Path('latest_by_author') : Args(1) { my ( $self, $c, $pauseid ) = @_; - $c->stash_or_detach( $self->model($c)->latest_by_author($pauseid) ); + $c->stash_or_detach( + $c->model('ESQuery')->release->latest_by_author($pauseid) ); } sub all_by_author : Path('all_by_author') : Args(1) { @@ -79,7 +84,8 @@ sub all_by_author : Path('all_by_author') : Args(1) { my $page = $params->{page}; my $size = $params->{page_size}; $c->stash_or_detach( - $self->model($c)->all_by_author( $pauseid, $page, $size ) ); + $c->model('ESQuery')->release->all_by_author( $pauseid, $page, $size ) + ); } sub versions : Path('versions') : Args(1) { @@ -87,8 +93,8 @@ sub versions : Path('versions') : Args(1) { my %params = %{ $c->req->params }{qw( plain versions )}; $c->add_dist_key($dist); $c->cdn_max_age('1y'); - my $data = $self->model($c) - ->versions( $dist, [ split /,/, $params{versions} || '' ] ); + my $data = $c->model('ESQuery') + ->release->versions( $dist, [ split /,/, $params{versions} || '' ] ); if ( $params{plain} ) { my $data = join "\n", @@ -105,21 +111,22 @@ sub versions : Path('versions') : Args(1) { sub top_uploaders : Path('top_uploaders') : Args() { my ( $self, $c ) = @_; my $range = $c->req->param('range') || 'weekly'; - $c->stash_or_detach( $self->model($c)->top_uploaders($range) ); + $c->stash_or_detach( + $c->model('ESQuery')->release->top_uploaders($range) ); } sub interesting_files : Path('interesting_files') : Args(2) { my ( $self, $c, $author, $release ) = @_; my $categories = $c->read_param( 'category', 1 ); - $c->stash_or_detach( $c->model('ESModel')->doc('file') - ->interesting_files( $author, $release, $categories ) ); + $c->stash_or_detach( $c->model('ESQuery') + ->file->interesting_files( $author, $release, $categories ) ); } sub files_by_category : Path('files_by_category') : Args(2) { my ( $self, $c, $author, $release ) = @_; my $categories = $c->read_param( 'category', 1 ); - $c->stash_or_detach( $c->model('ESModel')->doc('file') - ->files_by_category( $author, $release, $categories ) ); + $c->stash_or_detach( $c->model('ESQuery') + ->file->files_by_category( $author, $release, $categories ) ); } __PACKAGE__->meta->make_immutable; diff --git a/lib/MetaCPAN/Server/Controller/ReverseDependencies.pm b/lib/MetaCPAN/Server/Controller/ReverseDependencies.pm index 186d6c4dc..e2749b499 100644 --- a/lib/MetaCPAN/Server/Controller/ReverseDependencies.pm +++ b/lib/MetaCPAN/Server/Controller/ReverseDependencies.pm @@ -14,7 +14,7 @@ with 'MetaCPAN::Server::Role::JSONP'; sub dist : Path('dist') : Args(1) { my ( $self, $c, $dist ) = @_; $c->stash_or_detach( - $c->model('ESModel')->doc('release')->reverse_dependencies( + $c->model('ESQuery')->release->reverse_dependencies( $dist, @{ $c->req->params }{qw< page page_size sort >} ) ); @@ -23,7 +23,7 @@ sub dist : Path('dist') : Args(1) { sub module : Path('module') : Args(1) { my ( $self, $c, $module ) = @_; $c->stash_or_detach( - $c->model('ESModel')->doc('release')->requires( + $c->model('ESQuery')->release->requires( $module, @{ $c->req->params }{qw< page page_size sort >} ) ); diff --git a/lib/MetaCPAN/Server/Controller/Scroll.pm b/lib/MetaCPAN/Server/Controller/Scroll.pm index f2876c2bc..1c3bcbcef 100644 --- a/lib/MetaCPAN/Server/Controller/Scroll.pm +++ b/lib/MetaCPAN/Server/Controller/Scroll.pm @@ -55,7 +55,7 @@ sub index : Path('/_search/scroll') : Args { } my $res = eval { - $c->model('ESModel')->es->scroll( { + $c->model('ES')->scroll( { scroll_id => $scroll_id, scroll => $c->req->params->{scroll}, } ); diff --git a/lib/MetaCPAN/Server/Controller/Search/Autocomplete.pm b/lib/MetaCPAN/Server/Controller/Search/Autocomplete.pm index 7005c7dfa..88e8e87b2 100644 --- a/lib/MetaCPAN/Server/Controller/Search/Autocomplete.pm +++ b/lib/MetaCPAN/Server/Controller/Search/Autocomplete.pm @@ -14,13 +14,13 @@ has '+type' => ( default => 'file' ); sub get : Local : Path('') : Args(0) { my ( $self, $c ) = @_; $c->stash_or_detach( - $self->model($c)->autocomplete( $c->req->param("q") ) ); + $c->model('ESQuery')->file->autocomplete( $c->req->param("q") ) ); } sub suggest : Local : Path('/suggest') : Args(0) { my ( $self, $c ) = @_; - $c->stash_or_detach( - $self->model($c)->autocomplete_suggester( $c->req->param("q") ) ); + $c->stash_or_detach( $c->model('ESQuery') + ->file->autocomplete_suggester( $c->req->param("q") ) ); } 1; diff --git a/lib/MetaCPAN/Server/Controller/Search/DownloadURL.pm b/lib/MetaCPAN/Server/Controller/Search/DownloadURL.pm index 1b16ab609..e04c1d4e7 100644 --- a/lib/MetaCPAN/Server/Controller/Search/DownloadURL.pm +++ b/lib/MetaCPAN/Server/Controller/Search/DownloadURL.pm @@ -15,8 +15,8 @@ sub get : Local : Path('/download_url') : Args(1) { my ( $self, $c, $module ) = @_; my $type = $module eq 'perl' ? 'dist' : 'module'; my $data - = $self->model($c) - ->find_download_url( $type, $module, $c->req->params ); + = $c->model('ESQuery') + ->release->find_download_url( $type, $module, $c->req->params ); return $c->detach( '/not_found', [] ) unless $data; $c->stash($data); } diff --git a/lib/MetaCPAN/Server/Controller/Source.pm b/lib/MetaCPAN/Server/Controller/Source.pm index d667b3d5d..868e177cd 100644 --- a/lib/MetaCPAN/Server/Controller/Source.pm +++ b/lib/MetaCPAN/Server/Controller/Source.pm @@ -63,9 +63,12 @@ sub module : Chained('index') : PathPart('') : Args(1) { $c->cdn_never_cache(1); - $module = $c->model('ESModel')->doc('file')->find($module) + my $file + = $c->model('ESQuery') + ->file->find_module( $module, [qw(author release path)] ) or $c->detach( '/not_found', [] ); - $c->forward( 'get', [ map { $module->$_ } qw(author release path) ] ); + + $c->forward( 'get', [ map { $file->{$_} } qw(author release path) ] ); } 1; diff --git a/lib/MetaCPAN/Server/Model/ESQuery.pm b/lib/MetaCPAN/Server/Model/ESQuery.pm new file mode 100644 index 000000000..e31615cc4 --- /dev/null +++ b/lib/MetaCPAN/Server/Model/ESQuery.pm @@ -0,0 +1,31 @@ +package MetaCPAN::Server::Model::ESQuery; + +use Moose; + +use MetaCPAN::Query (); + +extends 'Catalyst::Model'; + +has es => ( + is => 'ro', + writer => '_set_es', +); + +has _esx_query => ( + is => 'ro', + lazy => 1, + default => sub { + my $self = shift; + MetaCPAN::Query->new( es => $self->es ); + }, +); + +sub ACCEPT_CONTEXT { + my ( $self, $c ) = @_; + if ( !$self->es ) { + $self->_set_es( $c->model('ES') ); + } + return $self->_esx_query; +} + +1; diff --git a/lib/MetaCPAN/Server/Role/Request.pm b/lib/MetaCPAN/Server/Role/Request.pm index 39623e135..943291265 100644 --- a/lib/MetaCPAN/Server/Role/Request.pm +++ b/lib/MetaCPAN/Server/Role/Request.pm @@ -14,4 +14,10 @@ around [qw(content_type header)] => sub { : $header; }; +sub fields { + my $self = shift; + my @fields = map { split /,/ } $self->param('fields'); + return @fields ? \@fields : undef; +} + 1; diff --git a/t/lib/MetaCPAN/Server/Test.pm b/t/lib/MetaCPAN/Server/Test.pm index 98ef26885..601e0ce5a 100644 --- a/t/lib/MetaCPAN/Server/Test.pm +++ b/t/lib/MetaCPAN/Server/Test.pm @@ -2,18 +2,22 @@ package MetaCPAN::Server::Test; use strict; use warnings; +use feature qw(state); -use HTTP::Request::Common qw( DELETE GET POST ); ## no perlimports -use MetaCPAN::Model (); -use MetaCPAN::Server (); -use MetaCPAN::Server::Config (); -use Plack::Test; ## no perlimports +use HTTP::Request::Common qw( DELETE GET POST ); ## no perlimports +use MetaCPAN::Model (); +use MetaCPAN::Server (); +use MetaCPAN::Server::Config (); +use MooseX::Types::ElasticSearch qw( ES ); +use Plack::Test; ## no perlimports use base 'Exporter'; our @EXPORT_OK = qw( POST GET DELETE + es model test_psgi app + query ); # Begin the load-order dance. @@ -38,9 +42,19 @@ sub app { return $app; } +sub es { + state $es = do { + my $c = MetaCPAN::Server::Config::config(); + ES->assert_coerce( $c->{elasticsearch_servers} ); + }; +} + sub model { - my $c = MetaCPAN::Server::Config::config(); - MetaCPAN::Model->new( es => $c->{elasticsearch_servers} ); + state $model = MetaCPAN::Model->new( es => es() ); +} + +sub query { + state $query = MetaCPAN::Query->new( es => es() ); } 1; diff --git a/t/model/release/reverse_dependencies.t b/t/model/release/reverse_dependencies.t index 81eddf44e..606418d8f 100644 --- a/t/model/release/reverse_dependencies.t +++ b/t/model/release/reverse_dependencies.t @@ -13,8 +13,8 @@ subtest 'distribution reverse_dependencies' => sub { sort { $a->[1] cmp $b->[1] } map +[ @{$_}{qw(author name)} ], @{ - $c->model('ESModel')->doc('release') - ->raw->reverse_dependencies('Multiple-Modules')->{data} + $c->model('ESQuery') + ->release->reverse_dependencies('Multiple-Modules')->{data} } ]; @@ -32,8 +32,8 @@ subtest 'module reverse_dependencies' => sub { my $data = [ map +[ @{$_}{qw(author name)} ], @{ - $c->model('ESModel')->doc('release') - ->raw->requires('Multiple::Modules')->{data} + $c->model('ESQuery')->release->requires('Multiple::Modules') + ->{data} } ]; @@ -46,8 +46,7 @@ subtest 'module reverse_dependencies' => sub { subtest 'no reverse_dependencies' => sub { my $data - = $c->model('ESModel')->doc('release')->raw->requires('DoesNotExist') - ->{data}; + = $c->model('ESQuery')->release->requires('DoesNotExist')->{data}; is_deeply( $data, [], 'Found no reverse dependencies for module.' ); }; diff --git a/t/query.t b/t/query.t new file mode 100644 index 000000000..b56c7d6b1 --- /dev/null +++ b/t/query.t @@ -0,0 +1,44 @@ +use strict; +use warnings; + +use lib 't/lib'; + +use MetaCPAN::Query; +use MetaCPAN::Server::Test (); +use Test::More; +use Scalar::Util qw(weaken refaddr); + +my $es = MetaCPAN::Server::Test::model->es; + +{ + my $query = MetaCPAN::Query->new( es => $es ); + my $release = $query->release; + + ok $release->isa('MetaCPAN::Query::Release'), + 'release object is correct class'; + is refaddr $release->query, refaddr $query, 'got same parent object'; + + weaken $release; + weaken $query; + ok !defined $query, 'parent object properly released' + or diag explain $query; + ok !defined $release, 'release object properly released' + or diag explain $release; +} + +{ + my $release = MetaCPAN::Query::Release->new( es => $es ); + my $query = $release->query; + + ok $query->isa('MetaCPAN::Query'), 'query object is correct class'; + is refaddr $query->release, refaddr $release, 'got same child object'; + + weaken $release; + weaken $query; + ok !defined $query, 'parent object properly released' + or diag explain $query; + ok !defined $release, 'release object properly released' + or diag explain $release; +} + +done_testing; diff --git a/t/release/moose.t b/t/release/moose.t index f3b28a03d..5582bfb8f 100644 --- a/t/release/moose.t +++ b/t/release/moose.t @@ -51,12 +51,6 @@ ok( is( $ppport->name, 'ppphdoc', 'name doesn\'t contain a dot' ); -ok( my $moose = $model->doc('file')->find('Moose'), 'find Moose module' ); - -is( $moose->name, 'Moose.pm', 'defined in Moose.pm' ); - -is( $moose->module->[0]->associated_pod, 'DOY/Moose-0.02/lib/Moose.pm' ); - my $signature; $signature = $model->doc('file')->query( { bool => { diff --git a/t/release/pm-PL.t b/t/release/pm-PL.t index 9abd3cd4a..3facfb9c5 100644 --- a/t/release/pm-PL.t +++ b/t/release/pm-PL.t @@ -2,40 +2,46 @@ use strict; use warnings; use lib 't/lib'; -use MetaCPAN::Server::Test qw( app GET model test_psgi ); +use MetaCPAN::ESConfig qw( es_doc_path ); +use MetaCPAN::Server::Test qw( app GET query es test_psgi ); use Test::More; -my $model = model(); +my $query = query(); # Module::Faker will generate a regular pm for the main module. -is( $model->doc('file')->find('uncommon::sense')->path, +is( $query->file->find_module('uncommon::sense')->{path}, 'lib/uncommon/sense.pm', 'find main module' ); # This should be the .pm.PL file we specified. -ok( my $pm = $model->doc('file')->find('less::sense'), +ok( my $pm = $query->file->find_module('less::sense'), 'find sense.pm.PL module' ); -is( $pm->name, 'sense.pm.PL', 'name is correct' ); +is( $pm->{name}, 'sense.pm.PL', 'name is correct' ); is( - $pm->module->[0]->associated_pod, + $pm->{module}->[0]->{associated_pod}, 'MO/uncommon-sense-0.01/sense.pod', 'has associated pod file' ); # Ensure that $VERSION really came from file and not dist. -is( $pm->module->[0]->version, +is( $pm->{module}->[0]->{version}, '4.56', 'pm.PL module version is (correctly) different than main dist' ) # TRAVIS 5.16 - or diag( Test::More::explain( $pm->meta->get_data($pm) ) ); + or diag($pm); { # Verify all the files we expect to be contained in the release. - my $files - = $model->doc('file') - ->query( { term => { release => 'uncommon-sense-0.01' } } ) - ->raw->size(20)->all->{hits}->{hits}; + my $files = es->search( + es_doc_path('file'), + body => { + query => { + term => { release => 'uncommon-sense-0.01' }, + }, + size => 20, + }, + )->{hits}->{hits}; $files = [ map { $_->{_source} } @$files ]; is_deeply( diff --git a/t/release/pod-pm.t b/t/release/pod-pm.t index b24a217cd..0730bafe3 100644 --- a/t/release/pod-pm.t +++ b/t/release/pod-pm.t @@ -2,18 +2,18 @@ use strict; use warnings; use lib 't/lib'; -use MetaCPAN::Server::Test qw( model ); +use MetaCPAN::Server::Test qw( query ); use Test::More; -my $model = model(); +my $query = query(); -ok( my $pod_pm = $model->doc('file')->find('Pod::Pm'), +ok( my $pod_pm = $query->file->find_module('Pod::Pm'), 'find Pod::Pm module' ); -is( $pod_pm->name, 'Pm.pm', 'defined in Pm.pm' ); +is( $pod_pm->{name}, 'Pm.pm', 'defined in Pm.pm' ); is( - $pod_pm->module->[0]->associated_pod, + $pod_pm->{module}->[0]->{associated_pod}, 'MO/Pod-Pm-0.01/lib/Pod/Pm.pod', 'has associated pod file' ); diff --git a/t/release/versions.t b/t/release/versions.t index 90f25ba9a..a08bb379b 100644 --- a/t/release/versions.t +++ b/t/release/versions.t @@ -2,10 +2,10 @@ use strict; use warnings; use lib 't/lib'; -use MetaCPAN::Server::Test qw( model ); +use MetaCPAN::Server::Test qw( query ); use Test::More; -my $model = model(); +my $query = query(); my %modules = ( 'Versions::Our' => '1.45', @@ -16,14 +16,15 @@ my %modules = ( while ( my ( $module, $version ) = each %modules ) { - ok( my $file = $model->doc('file')->find($module), "find $module" ) + ok( my $file = $query->file->find_module($module), "find $module" ) or next; ( my $path = "lib/$module.pm" ) =~ s/::/\//; - is( $file->path, $path, 'expected path' ); + is( $file->{path}, $path, 'expected path' ); # Check module version (different than dist version). - is( $file->module->[0]->version, $version, 'version parsed from file' ); + is( $file->{module}->[0]->{version}, + $version, 'version parsed from file' ); }