Multiple collectors

Changes

@@ -1,5 +1,10 @@
 {{$NEXT}}
 
+    - Make collector options configurable
+    - Add collector option for max poll events
+    - Add collector option for max open jobs
+    - Add option to allow multiple collectors
+
 1.000038  2020-11-02 20:49:12-08:00 America/Los_Angeles
 
     - Add shellcall and aux output capture for plugins

lib/App/Yath/Command/collector.pm

@@ -34,22 +34,27 @@ sub run {
 
     my $run = Test2::Harness::Run->new(%{decode_json(<STDIN>)});
 
-    my $collector = $collector_class->new(
+    my $collector;
+    $collector = $collector_class->new(
         %args,
         settings   => $settings,
         workdir    => $dir,
         run_id     => $run_id,
         runner_pid => $runner_pid,
         run        => $run,
+        output_fh  => $fh,
         # as_json may already have the json form of the event cached, if so
         # we can avoid doing an extra call to encode_json
-        action => sub { print $fh defined($_[0]) ? $_[0]->as_json . "\n" : "null\n"; },
+        action => sub { print {$collector->output_fh} defined($_[0]) ? ref($_[0]) ? $_[0]->as_json . "\n" : $_[0] : "null\n"; },
     );
 
     local $SIG{PIPE} = 'IGNORE';
     my $ok = eval { $collector->process(); 1 };
     my $err = $@;
 
+    # Avoid refcycle
+    delete $collector->{action};
+
     eval { print $fh "null\n"; 1 } or warn $@;
 
     die $err unless $ok;

lib/App/Yath/Command/start.pm

@@ -35,6 +35,7 @@ include_options(
     'App::Yath::Options::Runner',
     'App::Yath::Options::Workspace',
     'App::Yath::Options::Persist',
+    'App::Yath::Options::Collector',
 );
 
 option_group {prefix => 'runner', category => "Persistent Runner Options"} => sub {

lib/App/Yath/Command/test.pm

@@ -61,6 +61,7 @@ include_options(
     'App::Yath::Options::Run',
     'App::Yath::Options::Runner',
     'App::Yath::Options::Workspace',
+    'App::Yath::Options::Collector',
 );
 
 sub MAX_ATTACH() { 1_048_576 }

lib/App/Yath/Options/Collector.pm

@@ -0,0 +1,82 @@
+package App::Yath::Options::Collector;
+use strict;
+use warnings;
+
+our $VERSION = '1.000039';
+
+use App::Yath::Options;
+
+option_group {prefix => 'collector', category => "Collector Options"} => sub {
+    option max_open_jobs => (
+        type => 's',
+        description => 'Maximum number of jobs a collector can process at a time, if more jobs are pending their output will be delayed until the earlier jobs have been processed. (Default: 300)',
+        default => 300,
+        long_examples  => [' 300'],
+        short_examples => [' 300'],
+    );
+
+    option spawn_worker_at_max => (
+        type => 'b',
+        description => 'Spawn an extra collector whenever we hit max_open_jobs. (Default: off)',
+        default => 0,
+    );
+
+    option max_poll_events => (
+        type => 's',
+        description => 'Maximum number of events to poll from a job before jumping to the next job. (Default: 1000)',
+        default => 1000,
+        long_examples  => [' 1000'],
+        short_examples => [' 1000'],
+    );
+};
+
+1;
+
+__END__
+
+
+=pod
+
+=encoding UTF-8
+
+=head1 NAME
+
+App::Yath::Options::Collector - collector options for Yath.
+
+=head1 DESCRIPTION
+
+This is where the command line options for the collector are defined.
+
+=head1 PROVIDED OPTIONS POD IS AUTO-GENERATED
+
+=head1 SOURCE
+
+The source code repository for Test2-Harness can be found at
+F<http://github.com/Test-More/Test2-Harness/>.
+
+=head1 MAINTAINERS
+
+=over 4
+
+=item Chad Granum E<lt>[email protected]<gt>
+
+=back
+
+=head1 AUTHORS
+
+=over 4
+
+=item Chad Granum E<lt>[email protected]<gt>
+
+=back
+
+=head1 COPYRIGHT
+
+Copyright 2020 Chad Granum E<lt>[email protected]<gt>.
+
+This program is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+See F<http://dev.perl.org/licenses/>
+
+=cut