repo init

CHANGELOG.md

@@ -0,0 +1,22 @@
+# Changelog
+
+## Unreleased
+
+### Added
+
+- None
+
+### Fixed
+
+- None
+
+## 0.1.0 - 2020-08-24
+
+### Added
+
+- A `JsonFormatter` to produce a json for each of the parallel_test_suite, in a file given by user in `--output` option.
+- A Report generator which parses the above file to generate a report having a list of top 20 slowest examples, failures, errors and runtime checks.
+
+## Previous versions
+
+No previous versions.

Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem 'rake', '~> 12.3.3'
+
+gemspec

Gemfile.lock

@@ -0,0 +1,24 @@
+PATH
+  remote: .
+  specs:
+    parallel_tests_report (0.1.0)
+      nokogiri
+      rake (~> 12.3.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    mini_portile2 (2.4.0)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    rake (12.3.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  parallel_tests_report!
+  rake (~> 12.3.3)
+
+BUNDLED WITH
+   2.1.4

README.md

@@ -0,0 +1,34 @@
+# parallel-tests-report
+
+Works with [parallel_tests](https://github.com/grosser/parallel_tests) gem to generate a consolidated report for the spec groups executed by parallel_tests.
+
+The report generated will include:
+- List of top 20 slowest examples.
+- Rspec command to reproduce the failed example with the bisect option and seed value used.
+
+This gem will also verify the time taken for a test against configured threshold value and report if the time has exceeded.
+
+## How it works
+- parallel_tests gem is configured to use a custom formatter provided by this gem using `--format` and `--out` options.
+- Once tests are executed a rake task provided by this gem can be executed to parse the json and generate the report.
+
+## Installation
+Include the gem in your Gemfile
+
+`gem 'parallel_tests_report'`
+
+`$ bundle install`
+
+Add the following to the Rakefile before load_task(In Rails application):
+
+`require 'parallel_tests_report'`
+
+## Usage
+- add `--format` and `--out` option to `.rspec` or `.rspec_parallel`
+  - `--format ParallelTestsReport::JsonFormatter --out tmp/test-results/rspec.json`
+- execute the rake task after specs are executed 
+  - `bundle exec parallel_tests_report rake generate:report <TIME_LIMIT_IN_SECONDS> tmp/test-results/rspec.json`
+  - <TIME_LIMIT_IN_SECONDS> is the maximum time an example can take. Default is 10 seconds.
+  - <OUTPUT_FILE> is the file specified in the --out option. Default is 'tmp/test-results/rspec.json'
+
+#### This rake task can be configured to run after specs are executed in a continuous integration setup, it also produces a junit xml file for time limit exceeding check.

Rakefile

@@ -0,0 +1,3 @@
+require 'parallel_tests_report'
+
+Dir.glob("lib/parallel_tests_report/tasks/*.rake").each { |f| import f }

bin/parallel_tests_report

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+gem_dir = File.expand_path("..",File.dirname(__FILE__))
+$LOAD_PATH.unshift gem_dir
+exec_type = ARGV[0]
+if exec_type == 'rake' then
+    require 'rake'
+    require 'pp'
+    pwd=Dir.pwd
+    Dir.chdir(gem_dir)
+    Rake.application.init
+    Rake.application.load_rakefile
+    Dir.chdir(pwd)
+    Rake::Task[ARGV[1]].invoke(ARGV[2], ARGV[3])
+end

lib/parallel_tests_report.rb

@@ -0,0 +1,6 @@
+require "rspec/core"
+require "rspec/core/formatters/base_formatter"
+
+module ParallelTestsReport
+  require 'parallel_tests_report/railtie' if defined?(Rails)
+end

lib/parallel_tests_report/generate_report.rb

@@ -0,0 +1,114 @@
+require 'parallel_tests_report'
+require 'json'
+require 'nokogiri'
+
+class ParallelTestsReport::GenerateReport
+  def start(time_limit, output)
+    all_examples = []
+    slowest_examples = []
+    failed_examples = []
+    time_exceeding_examples = []
+    rerun_failed = []
+    errors = []
+
+    return if File.zero?(output)
+
+    File.foreach(output) do |line|
+      parallel_suite = JSON.parse(line)
+      all_examples += parallel_suite["examples"]
+      slowest_examples += parallel_suite["profile"]["examples"]
+      failed_examples += parallel_suite["examples"].select {|ex| ex["status"] == "failed" }
+      time_exceeding_examples += parallel_suite["examples"].select {|ex| ex["run_time"] >= time_limit}
+      errors << parallel_suite["messages"][0] if parallel_suite["examples"].size == 0
+    end
+
+    if slowest_examples.size > 0
+      slowest_examples = slowest_examples.sort_by do |ex|
+        -ex["run_time"]
+      end.first(20)
+      puts "Top #{slowest_examples.size} slowest examples\n"
+      slowest_examples.each do |ex|
+        puts <<-TEXT
+  #{ex["full_description"]}
+      #{ex["run_time"]} #{"seconds"} #{ex["file_path"]} #{ex["line_number"]}
+        TEXT
+      end
+    end
+
+    if failed_examples.size > 0
+      puts "\nFailed Examples:\n"
+      failed_examples.each do |ex|
+        puts <<-TEXT
+  => #{ex["full_description"]}
+      #{ex["run_time"]} #{"seconds"} #{ex["file_path"]} #{ex["line_number"]}
+        #{ex["exception"]["message"]}
+        TEXT
+        all_examples.each do |e|
+          rerun_failed << e["file_path"].to_s if e["parallel_test_proessor"] == ex["parallel_test_proessor"] && !rerun_failed.include?(e["file_path"])
+        end
+        str = ""
+        rerun_failed.each do |e|
+          str += e + " "
+        end
+        puts <<-TEXT
+  \n\s\sIn case the failure: "#{ex["full_description"]}" is due to random ordering, run the following command to isolate the minimal set of examples that reproduce the same failures:
+    `bundle exec rspec #{str} --seed #{ex['seed']} --bisect`\n
+        TEXT
+        rerun_failed.clear
+      end
+    end
+
+    if errors.size > 0
+      puts "\Errors:\n"
+      errors.each do |err|
+        puts <<-TEXT
+        #{err}
+        TEXT
+      end
+    end
+
+    if time_exceeding_examples.size > 0 || errors.size > 0
+      generate_xml(errors, time_exceeding_examples, time_limit)
+    end
+
+    if time_exceeding_examples.size > 0
+      puts "\nExecution time is exceeding the threshold of #{@time_limit} seconds for following tests:"
+      time_exceeding_examples.each do |ex|
+        puts <<-TEXT
+  => #{ex["full_description"]}: #{ex["run_time"]} #{"Seconds"}
+        TEXT
+      end
+    else
+      puts "Runtime check Passed."
+    end
+
+    if failed_examples.size > 0 || errors.size > 0 || time_exceeding_examples.size > 0
+      fail_message = "Tests Failed"
+      puts "\e[31m#{fail_message}\e[0m"
+      exit 1
+    end
+  end
+
+  def generate_xml(errors, time_exceeding_examples, time_limit)
+    builder = Nokogiri::XML::Builder.new(:encoding => 'UTF-8') do |xml|
+      xml.testsuite {
+        time_exceeding_examples.each do |arr|
+          classname = "#{arr["file_path"]}".sub(%r{\.[^/]*\Z}, "").gsub("/", ".").gsub(%r{\A\.+|\.+\Z}, "")
+          xml.testcase("classname" => "#{classname}", "name" => "#{arr["full_description"]}", "file" => "#{arr["file_path"]}", "time" => "#{arr["run_time"]}") {
+            xml.failure "Execution time is exceeding the threshold of #{time_limit} seconds"
+          }
+        end
+        errors.each do |arr|
+          file_path = arr[/(?<=An error occurred while loading ).*/]
+          classname = "#{file_path}".sub(%r{\.[^/]*\Z}, "").gsub("/", ".").gsub(%r{\A\.+|\.+\Z}, "")
+          xml.testcase("classname" => "#{classname}", "name" => "An error occurred while loading", "file" => "#{file_path}", "time" => "0.0") {
+            xml.failure arr.gsub(/\e\[([;\d]+)?m/, "").gsub(/An error occurred while loading #{file_path}\n/, "")
+          }
+        end
+      }
+    end
+    File.open('tmp/test-results/time_limit_exceeded.xml', 'w') do |file|
+      file << builder.to_xml
+    end
+  end
+end

lib/parallel_tests_report/json_formatter.rb

@@ -0,0 +1,113 @@
+# With this JsonFormatter we are generating a file which contains a json for each and every parallel_test_suite. It contains all the passed, failed, pending and profiled examples with their full_description, file_path, run_time, seed value and parallel_test_proessor number.
+# We parse each line of this file, to generate a report to show slowest examples, failed examples, errors and runtime checks.
+
+# And example to show the structure of the file:
+# This is a single line containing details for one parallel_test_suite
+=begin
+{"messages":["Run options: exclude {:asrun=\u003etrue, :to_be_implemented=\u003etrue, :integration=\u003etrue}"],"seed":2121,"examples":[{"full_description":"An example", updated_at, id, media_id","status":"passed","file_path":"/path/to/the/example","line_number":01,"run_time":2.269736609,"parallel_test_proessor":1,"seed":2121},{"full_description":"Another Example","status":"passed","file_path":"path/to/another/exmple","line_number":20,"run_time":4.139183023,"parallel_test_proessor":1,"seed":2121}],"profile":{"examples":[{"full_description":"An example", updated_at, id, media_id","status":"passed","file_path":"/path/to/the/example","line_number":01,"run_time":2.269736609,"parallel_test_proessor":1,"seed":2121},{"full_description":"Another Example","status":"passed","file_path":"path/to/another/exmple","line_number":20,"run_time":4.139183023,"parallel_test_proessor":1,"seed":2121}]}}
+=end
+
+require 'parallel_tests_report'
+
+class ParallelTestsReport::JsonFormatter < RSpec::Core::Formatters::BaseFormatter
+  RSpec::Core::Formatters.register self, :message, :dump_profile, :seed, :stop, :close
+  attr_reader :output_hash, :output
+  def initialize(output)
+    super
+    @output ||= output
+    if String === @output
+      #open the file given as argument in --out
+      FileUtils.mkdir_p(File.dirname(@output))
+      # overwrite previous results
+      File.open(@output, 'w'){}
+      @output = File.open(@output, 'a')
+      # close and restart in append mode
+    elsif File === @output
+      @output.close
+      @output = File.open(@output.path, 'a')
+    end
+    @output_hash = {}
+
+    if ENV['TEST_ENV_NUMBER'].to_i != 0
+      @n = ENV['TEST_ENV_NUMBER'].to_i
+    else
+      @n = 1
+    end
+  end
+
+  def message(notification)
+    (@output_hash[:messages] ||= []) << notification.message
+  end
+
+  def seed(notification)
+    return unless notification.seed_used?
+    @output_hash[:seed] = notification.seed
+  end
+
+  def close(_notification)
+    #close the file after all the processes are finished
+    @output.close if (IO === @output) & (@output != $stdout)
+  end
+
+  def stop(notification)
+    #adds to @output_hash, an array of examples which run in a particular processor
+    @output_hash[:examples] = notification.examples.map do |example|
+      format_example(example).tap do |hash|
+        e = example.exception
+        if e
+          hash[:exception] =  {
+            :class => e.class.name,
+            :message => e.message,
+            :backtrace => e.backtrace,
+          }
+        end
+      end
+    end
+  end
+
+  def dump_profile(profile)
+    dump_profile_slowest_examples(profile)
+  end
+
+  def dump_profile_slowest_examples(profile)
+    #adds to @output_hash, an array of 20 slowest examples
+    lock_output do
+      @output_hash[:profile] = {}
+      @output_hash[:profile][:examples] = profile.slowest_examples.map do |example|
+        format_example(example)
+      end
+    end
+    #write the @output_hash to the file
+    output.puts @output_hash.to_json
+    output.flush
+  end
+
+protected
+  #to make a single file for all the parallel processes
+  def lock_output
+    if File === @output
+      begin
+        @output.flock File::LOCK_EX
+        yield
+      ensure
+        @output.flock File::LOCK_UN
+      end
+    else
+      yield
+    end
+  end
+
+private
+
+  def format_example(example)
+    {
+      :full_description => example.full_description,
+      :status => example.execution_result.status.to_s,
+      :file_path => example.metadata[:file_path],
+      :line_number  => example.metadata[:line_number],
+      :run_time => example.execution_result.run_time,
+      :parallel_test_proessor => @n,
+      :seed => @output_hash[:seed]
+    }
+  end
+end

lib/parallel_tests_report/railtie.rb

@@ -0,0 +1,13 @@
+require 'parallel_tests_report'
+require 'rails'
+
+module ParallelTestsReport
+  class Railtie < Rails::Railtie
+    railtie_name :parallel_tests_report
+
+    rake_tasks do
+      path = File.expand_path(__dir__)
+      Dir.glob("#{path}/tasks/**/*.rake").each { |f| load f }
+    end
+  end
+end

lib/parallel_tests_report/tasks/generate_report.rake

@@ -0,0 +1,24 @@
+require_relative '../../parallel_tests_report/generate_report.rb'
+
+namespace :generate do
+  task :report, [:time_limit, :output] do |t,args|
+    output = args[:output].to_s
+    if output == ""
+      if(args[:time_limit] != nil && args[:time_limit].to_f == 0.0 && args[:time_limit] != '0.0') # If only one argument is given while calling the rake_task and that is :output.
+        #Since, first argument is :time_limit, assigning that to output.
+        output = args[:time_limit].to_s
+      else
+        output = 'tmp/test-results/rspec.json' # default :output file
+      end
+    end
+    time_limit = args[:time_limit].to_f
+    if time_limit == 0.0
+      if args[:time_limit] == "0.0" #if :time_limit itself is 0.0
+        time_limit = 0.0
+      else
+        time_limit = 10.0 # default :time_limit
+      end
+    end
+    ParallelTestsReport::GenerateReport.new.start time_limit,output
+  end
+end

parallel_tests_report.gemspec

@@ -0,0 +1,14 @@
+Gem::Specification.new do |s|
+  s.name        = 'parallel_tests_report'
+  s.version     = '0.1.0'
+  s.date        = '2020-08-17'
+  s.summary     = "Generate report for parallel_tests"
+  s.description = "Works with parallel_tests ruby gem to generate a report having a list of slowest and failed examples."
+  s.authors     = ["Akshat Birani"]
+  s.email       = '[email protected]'
+  s.homepage    = 'https://github.com/amagimedia/parallel-tests-report'
+  s.files       = Dir["{lib,bin}/**/*", "README.md", "Rakefile"]
+  s.executables << 'parallel_tests_report'
+  s.add_dependency 'rake', '~> 12.3.3'
+  s.add_dependency 'nokogiri'
+end