From 3001c75aac512e2e0eb68dbfa91109036dc4334f Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 26 Oct 2024 17:47:42 +0200 Subject: [PATCH 1/5] remove old convert_author.pl script --- bin/convert_authors.pl | 105 ----------------------------------------- 1 file changed, 105 deletions(-) delete mode 100755 bin/convert_authors.pl diff --git a/bin/convert_authors.pl b/bin/convert_authors.pl deleted file mode 100755 index 573e3f75b..000000000 --- a/bin/convert_authors.pl +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env perl -# PODNAME: foo - -use strict; -use warnings; -use Cpanel::JSON::XS qw( decode_json ); -use File::Find qw( find ); - -my @files; -find( - sub { - push( @files, $File::Find::name ); - }, - 'conf/authors' -); - -foreach my $file (@files) { - next unless ( -f $file ); - next if ( $file =~ /1/ ); - next unless ( $file =~ /\.json$/ ); - my $json; - { - local $/ = undef; - local *FILE; - open FILE, "<$file"; - $json = ; - close FILE - } - warn $file; - my $data = decode_json($json); - my ($author) = keys %$data; - ($data) = values %$data; - my $raw = { - donation => [], - profile => [], - }; - my %profiles = ( - "delicious_username" => 'delicious', - "facebook_public_profile" => 'facebook', - "github_username" => 'github', - "linkedin_public_profile" => "linkedin", - "stackoverflow_public_profile" => 'stackoverflow', - "perlmonks_username" => 'perlmonks', - "twitter_username" => 'twitter', - "slideshare_url" => 'slideshare', - "youtube_channel_url" => 'youtube', - slashdot_username => 'slashdot', - "amazon_author_profile" => 'amazon', - aim => 'aim', - icq => 'icq', - jabber => 'jabber', - msn_messenger => 'msn_messenger', - "oreilly_author_profile" => 'oreilly', - slideshare_username => 'slideshare', - stumbleupon_profile => 'stumbleupon', - xing_public_profile => 'xing', - ACT_id => 'act', - irc_nick => 'irc', - irc_nickname => 'irc' - ); - - while ( my ( $k, $v ) = each %profiles ) { - next unless ( my $value = delete $data->{$k} ); - $value =~ s/^.*\///; - push( - @{ $raw->{profile} }, - { - name => $v, - id => $value - } - ); - } - - if ( my $pp = delete $data->{paypal_address} ) { - delete $data->{accepts_donations}; - push( - @{ $raw->{donation} }, - { - id => $pp, - name => 'paypal' - } - ); - } - - if ( $data->{blog_url} ) { - $raw->{blog} = [ { - url => delete $data->{blog_url}, - feed => delete $data->{blog_feed} - } ]; - } - delete $data->{perlmongers} if ( ref $data->{perlmongers} ); - if ( $data->{perlmongers} ) { - $raw->{perlmongers} = { - name => delete $data->{perlmongers}, - url => delete $data->{perlmongers_url}, - }; - } - $raw->{$_} = delete $data->{$_} - for (qw(city country email region website openid)); - unlink $file; - ( my $base = $file ) =~ s/^(.*)\/.*?$/$1/; - open FILE, '>', "$base/author-1.0.json"; - print FILE Cpanel::JSON::XS->new->pretty->encode($raw); - close FILE; -} From 0f205adcaffed5e7cb1853fbf8f8c9e266b9429c Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 26 Oct 2024 17:49:27 +0200 Subject: [PATCH 2/5] remove check_json.pl script Checking JSON can easily be doing with json_pp or jq. --- bin/check_json.pl | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100755 bin/check_json.pl diff --git a/bin/check_json.pl b/bin/check_json.pl deleted file mode 100755 index f798368d2..000000000 --- a/bin/check_json.pl +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env perl -# PODNAME: check_json.pl -use 5.010; - -use Data::Dumper qw( Dumper ); -use Cpanel::JSON::XS qw( decode_json ); - -foreach my $file (@ARGV) { - say "Processing $file"; - eval { - my $hash = decode_json( - do { local ( @ARGV, $/ ) = $file; <> } - ); - print Dumper($hash); - }; - - if ($@) { say "\terror in $file: $@" } -} From a384ecbd7ad8cefb342963da8ffb238f1e0d2970 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 26 Oct 2024 17:51:39 +0200 Subject: [PATCH 3/5] remove docker-elasticsearch script Our current setup for elasticsearch in docker for testing is in the metacpan-docker repository. If we wanted something to put directly in this repo, it would probably be a docker-compose setup. --- bin/docker-elasticsearch | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100755 bin/docker-elasticsearch diff --git a/bin/docker-elasticsearch b/bin/docker-elasticsearch deleted file mode 100755 index d0cc95629..000000000 --- a/bin/docker-elasticsearch +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -# bin/docker-elasticsearch /tmp/metacpan.yml /tmp/es 127.0.0.1:9200 - -ES_CONFIG_FILE="$1" -ES_CONFIG_DIR="$2" -ES_PORT="$3" -PUPPET_SCRIPTS="https://github.com/metacpan/metacpan-puppet/raw/master/modules/metacpan_elasticsearch/files/etc/scripts" - -sudo mkdir "$ES_CONFIG_DIR" && sudo chmod 777 "$ES_CONFIG_DIR" - -curl -L "$PUPPET_SCRIPTS"/prefer_shorter_module_names_100.groovy > "$ES_CONFIG_DIR"/prefer_shorter_module_names_100.groovy - -curl -L "$PUPPET_SCRIPTS"/prefer_shorter_module_names_400.groovy > "$ES_CONFIG_DIR"/prefer_shorter_module_names_400.groovy - -curl -L "$PUPPET_SCRIPTS"/status_is_latest.groovy > "$ES_CONFIG_DIR"/status_is_latest.groovy - -curl -L "$PUPPET_SCRIPTS"/score_version_numified.groovy > "$ES_CONFIG_DIR"/score_version_numified.groovy - -curl -L https://raw.githubusercontent.com/metacpan/metacpan-docker/master/elasticsearch/metacpan.yml > "$ES_CONFIG_FILE" - -docker pull elasticsearch:2.4-alpine -docker run -d -p "$ES_PORT":9200 -v "$ES_CONFIG_FILE":/usr/share/elasticsearch/config/metacpan.yml -v "$ES_CONFIG_DIR":/usr/share/elasticsearch/config/scripts elasticsearch:2.4-alpine From 8d89c2410381cb10bcdbfaf5446f812c2d766064 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 26 Oct 2024 17:55:03 +0200 Subject: [PATCH 4/5] remove unused and broken unlisted_prereqs.pl script --- bin/unlisted_prereqs.pl | 98 ----------------------------------------- 1 file changed, 98 deletions(-) delete mode 100755 bin/unlisted_prereqs.pl diff --git a/bin/unlisted_prereqs.pl b/bin/unlisted_prereqs.pl deleted file mode 100755 index bc94996da..000000000 --- a/bin/unlisted_prereqs.pl +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env perl -# PODNAME: check_prereqs.pl - -# TODO: this stuff should be in other modules somewhere - -use strict; -use warnings; -use Perl::PrereqScanner 1.014 (); -use CPAN::Meta::Requirements (); -use File::Find::Rule::Perl (); -use List::Util qw( sum ); -use version 0.77; - -# TODO: use CPAN::Meta::Prereqs - -my @found = File::Find::Rule->not( - File::Find::Rule->name( qw( - .git - t/var/tmp - var - ) )->prune->discard, -)->perl_file->in('.'); - -my $local = {}; -my $files = {}; - -foreach (@found) { - - # FIXME: unix slashes - my $phase = - - # beneath t/ or xt/ - m{^(\./)?x?t/} - ? 'build' - : 'runtime'; - - push @{ $files->{$phase} }, $_; - - if (m{^(?:\./)?(?:t/)?lib/(.+?)\.pm$}) { - ( my $pm = $1 ) =~ s!/!::!g; - $local->{$pm} = $_; - } -} - -my $scanner = Perl::PrereqScanner->new( - - # TODO: extra_scanners => [qw( PlackMiddleware Catalyst )], -); - -my $reqs = {}; - -foreach my $phase ( keys %$files ) { - my $pr = CPAN::Meta::Requirements->new; - foreach my $file ( @{ $files->{$phase} } ) { - $pr->add_requirements( $scanner->scan_file($file) ); - } - - # ignore packages we provide locally - $pr->clear_requirement($_) - for grep { exists $local->{$_} } $pr->required_modules; - - $reqs->{$phase} = $pr->as_string_hash; -} - -# don't duplicate runtime deps into build deps -foreach my $dep ( keys %{ $reqs->{runtime} } ) { - - # TODO: check version - delete $reqs->{build}{$dep}; -} - -sub check_prereqs { - my ( $scanned, $mm ) = @_; - foreach my $dep ( keys %$scanned ) { - if ( exists( $mm->{$dep} ) ) { - delete $scanned->{$dep} - if version->parse( $scanned->{$dep} ) - <= version->parse( $mm->{$dep} ); - } - } -} - -my ( $PREREQ_PM, $BUILD_REQUIRES, $MIN_PERL_VERSION ); - -my $mm_prereqs = qx{$^X Makefile.PL PREREQ_PRINT=1}; -eval $mm_prereqs; - -check_prereqs( $reqs->{runtime}, $PREREQ_PM ); -check_prereqs( $reqs->{build}, $BUILD_REQUIRES ); -delete $reqs->{runtime}{perl} - if version->parse( $reqs->{runtime}{perl} ) - <= version->parse($MIN_PERL_VERSION); - -use Data::Dumper (); -$Data::Dumper::Sortkeys = 1; -print Data::Dumper->Dump( [$reqs], ['requires'] ); - -exit sum map { scalar keys %{ $reqs->{$_} } } keys %$reqs; From 5fdc3fbd39e8ca8ced7d2636f04c5424b483b3f2 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 26 Oct 2024 17:56:41 +0200 Subject: [PATCH 5/5] tidy and strict for all Remove exemptions from perlcritic, add xt to tidy, and fix critic and tidy issues. --- bin/mirror_cpan_for_developers.pl | 3 ++ tidyall.ini | 53 ++----------------------------- xt/search_web.t | 2 +- 3 files changed, 6 insertions(+), 52 deletions(-) diff --git a/bin/mirror_cpan_for_developers.pl b/bin/mirror_cpan_for_developers.pl index 865b77cb5..7a8d4de10 100755 --- a/bin/mirror_cpan_for_developers.pl +++ b/bin/mirror_cpan_for_developers.pl @@ -1,4 +1,7 @@ #!/usr/bin/env perl +use strict; +use warnings; + # This script is only needed if you are developing metacpan, # on the live servers we use File::Rsync::Mirror::Recent # https://github.com/metacpan/metacpan-puppet/tree/master/modules/rrrclient diff --git a/tidyall.ini b/tidyall.ini index b27c1fd9d..b29f471e9 100644 --- a/tidyall.ini +++ b/tidyall.ini @@ -1,64 +1,15 @@ jobs = 5 [PerlTidy] -select = {bin,lib,t}/**/*.{pl,pm,t,psgi} +select = {bin,lib,t,xt}/**/*.{pl,pm,t,psgi} select = bin/{metacpan,wait-for-open} select = app.psgi ignore = t/var/**/* argv = --profile=$ROOT/.perltidyrc [PerlCritic] -select = {bin,lib,t}/**/*.{pl,pm,t,psgi} +select = {bin,lib,t,xt}/**/*.{pl,pm,t,psgi} select = bin/{metacpan,wait-for-open} -ignore = t/var/**/* -ignore = bin/build_test_CPAN_dir.pl -ignore = bin/check_json.pl -ignore = bin/convert_authors.pl -ignore = bin/get_fields.pl -ignore = bin/mirror_cpan_for_developers.pl -ignore = bin/unlisted_prereqs.pl -ignore = bin/write_config_json -ignore = lib/Catalyst/Action/Deserialize/MetaCPANSanitizedJSON.pm -ignore = lib/Catalyst/Authentication/Store/Proxy.pm -ignore = lib/Catalyst/Plugin/OAuth2/Provider.pm -ignore = lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm -ignore = lib/MetaCPAN/Document/Distribution.pm -ignore = lib/MetaCPAN/Document/File.pm -ignore = lib/MetaCPAN/Document/Release.pm -ignore = lib/MetaCPAN/Model.pm -ignore = lib/MetaCPAN/Pod/XHTML.pm -ignore = lib/MetaCPAN/Role/Common.pm -ignore = lib/MetaCPAN/Script/Author.pm -ignore = lib/MetaCPAN/Script/Backup.pm -ignore = lib/MetaCPAN/Script/CPANTesters.pm -ignore = lib/MetaCPAN/Script/Check.pm -ignore = lib/MetaCPAN/Script/First.pm -ignore = lib/MetaCPAN/Script/Latest.pm -ignore = lib/MetaCPAN/Script/Mapping.pm -ignore = lib/MetaCPAN/Script/Pagerank.pm -ignore = lib/MetaCPAN/Script/PerlMongers.pm -ignore = lib/MetaCPAN/Script/Query.pm -ignore = lib/MetaCPAN/Script/ReindexDist.pm -ignore = lib/MetaCPAN/Script/Release.pm -ignore = lib/MetaCPAN/Script/Runner.pm -ignore = lib/MetaCPAN/Script/Tickets.pm -ignore = lib/MetaCPAN/Script/Watcher.pm -ignore = lib/MetaCPAN/Server/Controller.pm -ignore = lib/MetaCPAN/Server/Controller/Changes.pm -ignore = lib/MetaCPAN/Server/Controller/File.pm -ignore = lib/MetaCPAN/Server/Controller/Login.pm -ignore = lib/MetaCPAN/Server/Controller/Login/PAUSE.pm -ignore = lib/MetaCPAN/Server/Controller/Login/Twitter.pm -ignore = lib/MetaCPAN/Server/Controller/Root.pm -ignore = lib/MetaCPAN/Server/Controller/Search/Autocomplete.pm -ignore = lib/MetaCPAN/Server/Controller/Source.pm -ignore = lib/MetaCPAN/Server/Controller/User/Favorite.pm -ignore = lib/MetaCPAN/Server/Model/CPAN.pm -ignore = lib/MetaCPAN/Server/Model/Source.pm -ignore = lib/MetaCPAN/Server/View/JSON.pm -ignore = lib/MetaCPAN/Server/View/Pod.pm -ignore = lib/MetaCPAN/Util.pm -ignore = lib/Plack/Session/Store/ElasticSearch.pm [UniqueLines] select = .gitignore diff --git a/xt/search_web.t b/xt/search_web.t index d2bf281f8..14293bfd4 100644 --- a/xt/search_web.t +++ b/xt/search_web.t @@ -12,7 +12,7 @@ use Test::More; # Just use this to get an es object. my $server = MetaCPAN::TestServer->new; my $search = MetaCPAN::Query::Search->new( - es => $server->es_client, + es => $server->es_client, index_name => 'cpan', );