diff --git a/bin/mirrors.pl b/bin/mirrors.pl new file mode 100644 index 0000000..ac60ca0 --- /dev/null +++ b/bin/mirrors.pl @@ -0,0 +1,78 @@ +use strict; +use warnings; +use v5.36; + +use Getopt::Long; +use MetaCPAN::Logger qw< :log :dlog >; +use Cpanel::JSON::XS (); + +use MetaCPAN::ES; +use MetaCPAN::Ingest qw< cpan_dir >; + +# args +my ( $distribution, $files_only, $undo ); +GetOptions( + "distribution=s" => \$distribution, + "files_only" => \$files_only, + "undo" => \$undo, +); + +# setup +my $cpan = cpan_dir(); +my $es = MetaCPAN::ES->new( type => "mirror" ); + + +index_mirrors(); + +$es->index_refresh; + +# TODO: +# cdn_purge_now( { keys => ['MIRRORS'], } ); + +log_info {"done"}; + +### + +sub index_mirrors () { + log_info { 'Getting mirrors.json file from ' . $cpan }; + + my $json = $cpan->child( 'indices', 'mirrors.json' )->slurp; + + # Clear out everything in the index + # so don't end up with old mirrors + $es->clear_type; + + my $mirrors = Cpanel::JSON::XS::decode_json($json); + foreach my $mirror (@$mirrors) { + $mirror->{location} = { + lon => delete $mirror->{longitude}, + lat => delete $mirror->{latitude} + }; + + #Dlog_trace {"Indexing $_"} $mirror; + log_debug {sprintf("Indexing %s", $mirror->{name})}; + + my @doc = + map { $_ => $mirror->{$_} } + grep { defined $mirror->{$_} } + keys %$mirror; + + $es->index( body => { @doc } ); + } +} + +1; + +__END__ + +=pod + +=head1 SYNOPSIS + + $ bin/mirrors.pl + +=head1 SOURCE + +L + +=cut diff --git a/lib/MetaCPAN/ES.pm b/lib/MetaCPAN/ES.pm index d726cec..49d2963 100644 --- a/lib/MetaCPAN/ES.pm +++ b/lib/MetaCPAN/ES.pm @@ -4,6 +4,7 @@ use strict; use warnings; use v5.36; +use MetaCPAN::Logger qw< :log :dlog >; use Search::Elasticsearch; use MetaCPAN::Ingest qw< config >; @@ -108,4 +109,25 @@ sub count ( $self, %args ) { ); } +sub clear_type ( $self ) { + my $bulk = $self->bulk; + my $scroll = $self->scroll( + query => { match_all => {} }, + sort => '_doc', + ); + + my @ids; + while ( my $search = $scroll->next ) { + push @ids => $search->{_id}; + log_debug { "deleting id=" . $search->{_id} }; + if ( @ids == 500 ) { + $bulk->delete_ids(@ids); + @ids = (); + } + } + $bulk->delete_ids(@ids); + + $bulk->flush; +} + 1;