Send flaky test output to sentry to ease debuging

lib/rspecq/formatters/failure_recorder.rb

@@ -31,18 +31,20 @@ def message(n)
       def example_failed(notification)
         example = notification.example
 
+        presenter = RSpec::Core::Formatters::ExceptionPresenter.new(
+          example.exception, example
+        )
+
+        msg = presenter.fully_formatted(nil, @colorizer)
+
         if @queue.requeue_job(example, @max_requeues, @worker_id)
           # HACK: try to avoid picking the job we just requeued; we want it
           # to be picked up by a different worker
+          @queue.record_flaky_failure(notification.example.id, msg)
           sleep 0.5
           return
         end
 
-        presenter = RSpec::Core::Formatters::ExceptionPresenter.new(
-          example.exception, example
-        )
-
-        msg = presenter.fully_formatted(nil, @colorizer)
         msg << "\n"
         msg << @colorizer.wrap(
           "bin/rspec --seed #{RSpec.configuration.seed} #{example.location_rerun_argument}",

lib/rspecq/queue.rb

@@ -171,6 +171,10 @@ def record_example_failure(example_id, message)
       @redis.hset(key_failures, example_id, message)
     end
 
+    def record_flaky_failure(example_id, message)
+      @redis.hset(key_flaky_failures, example_id, message)
+    end
+
     # For errors occured outside of examples (e.g. while loading a spec file)
     def record_non_example_error(job, message)
       @redis.hset(key_errors, job, message)
@@ -224,6 +228,10 @@ def example_failures
       @redis.hgetall(key_failures)
     end
 
+    def flaky_failures
+      @redis.hgetall(key_flaky_failures)
+    end
+
     def non_example_errors
       @redis.hgetall(key_errors)
     end
@@ -331,6 +339,13 @@ def key_failures
       key("example_failures")
     end
 
+    # Contains flaky RSpec example failures.
+    #
+    # redis: HASH<example_id => error message>
+    def key_flaky_failures
+      key("flaky_failures")
+    end
+
     # Contains errors raised outside of RSpec examples
     # (e.g. a syntax error in spec_helper.rb).
     #

lib/rspecq/reporter.rb

@@ -61,7 +61,7 @@ def report
       puts summary(@queue.example_failures, @queue.non_example_errors,
         flaky_jobs, humanize_duration(tests_duration))
 
-      flaky_jobs_to_sentry(flaky_jobs, tests_duration)
+      flaky_jobs_to_sentry(flaky_jobs, tests_duration, @queue.flaky_failures)
 
       exit 1 if [email protected]_successful?
     end
@@ -130,7 +130,7 @@ def humanize_duration(seconds)
       Time.at(seconds).utc.strftime("%H:%M:%S")
     end
 
-    def flaky_jobs_to_sentry(jobs, build_duration)
+    def flaky_jobs_to_sentry(jobs, build_duration, failures)
       return if jobs.empty?
 
       jobs.each do |job|
@@ -142,7 +142,8 @@ def flaky_jobs_to_sentry(jobs, build_duration)
           build_duration: build_duration,
           location: @queue.job_location(job),
           rerun_command: @queue.job_rerun_command(job),
-          worker: @queue.failed_job_worker(job)
+          worker: @queue.failed_job_worker(job),
+          output: failures[job]
         }
 
         tags = {