A metadata-driven flaky test reporting tool for RSpec, Shortcut, and CircleCI

flaky.rb

require 'digest/sha1'
require 'net/http'

RSpec.configure do |config|
  # Use this way:
  #
  #     let(:user) { create(:user) }
  #     let(:contact) { create(:contact, contact_of: user) }
  #
  #     # The `:flaky` metadata signals that this should be treated as a flaky
  #     # spec, and the current values of `:user` and `:contact` should be recorded
  #     # to help understand why this spec only occasionally fails. (In this simple
  #     # example, `:user` and `:contact` aren't actually used in the body of the
  #     # spec, but typically you would only want to record available spec variables
  #     # that the spec actually uses. If there are none, you can simply mark the
  #     # spec with `:flaky` and no variables.)
  #     it "should always pass but sometimes fails", flaky: [:user, :contact] do
  #       phone_number = Faker::PhoneNumber.phone_number
  #
  #       # The metadata approach above can't give FlakySpecs access to the spec's
  #       # local variables, so if the spec stores any test data in locals, you
  #       # should call `flaky_local_variables` to record those and their values.
  #       # Its parameter is an implicit Hash, so it can record multiple local
  #       # variables and their values in a single call.
  #       flaky_local_variables(phone_number: phone_number)
  #
  #       # This will fail when phone_number has an extension
  #       expect(phone_number).not_to match(/x/)
  #     end
  #
  # When the expectation fails, the spec will *not* fail (because it's been
  # identified as a flaky spec) but it will be reported to Shortcut as an
  # intermittent failure, along with the current values of the random seed and
  # `user`, `contact`, and `phone_number`, which can be used to help track
  # down the circumstances under which the spec fails. Only one Shortcut story
  # will be created for each [spec_name, exception_type] combination (yes, I've
  # seen a flaky spec that was flaky in multiple different ways that produced
  # different exceptions) and diagnostic data about each failure is recorded as
  # a comment on that story. In other words, each flaky spec becomes one item
  # on the team backlog, and that item gradually accumulates more data to help
  # the team understand the flakiness and how to fix it.
  #
  # As mentioned above, this tool sits at the intersection of RSpec, Shortcut,
  # and CircleCI. The basic idea, though, should be adaptable to any combination
  # of test framework, backlog management tool, and CI system.
  
  config.around(:example, :flaky) do |example|
    example.example_group.let(:__flaky_locals) { {} }

    example.run
    unless example.exception.nil?
      begin
        FlakySpecs::ShortcutReporter.report(config, example, self)
        example.example.display_exception = nil # suppress the failure
        # TODO: sneak the diagnostic info into the XML report somehow
      rescue => err
        STDERR.puts "Error while processing flaky spec failure: #{err.class} #{err.message}"
        STDERR.puts err.backtrace.join("\n")
      end
    end
  end
end

module FlakySpecs
  module FlakyHelpers

    def flaky_local_variables(names_and_values)
      names_and_values.each do |name, val|
        __flaky_locals[name.to_sym] = val
      end
    end
    alias :flaky_local_variable :flaky_local_variables
  end

  class ShortcutReporter
    def self.report(config, example, hook)
      new(config, example, hook).report
    end

    attr_reader :config, :example, :hook

    def initialize(config, example, hook)
      @config = config
      @example = example
      @hook = hook
    end

    def report
      if ENV["CI"] == "true"
        create_or_update_tracking_issue
      else
        print_local_message
      end
    end

    def print_local_message
      STDERR.puts "===> Suppressing flaky test failure:"
      STDERR.puts example.exception
      STDERR.puts "===> If this test fails in CI, its history will be tracked in Shortcut"
    end

    def create_or_update_tracking_issue
      WebMock.disable!
      VCR.eject_cassette
      VCR.turned_off do
        issue = find_issue
        if issue.present?
          add_comment issue
        else
          create_issue
        end
      end
    ensure
      WebMock.enable!
    end

    def spec_ident
      @spec_ident ||=
        begin
          pieces = [ example.file_path,
                     example.full_description,
                     example.exception.class.name ]
          Digest::SHA1.base64digest pieces.join("|")
        end
    end

    def issue_search_params
      { archived: false,
        external_id: spec_ident,
        story_type: "bug",
      }.to_json
    end

    def find_issue
      story_search_uri = URI("https://api.app.shortcut.com/api/v3/stories/search?token=#{shortcut_token}")
      res = Net::HTTP.post(story_search_uri, issue_search_params, "Content-Type" => "application/json")
      if res.code !~ /^2/
        STDERR.puts "===> Got response code #{res.code} while searching for tracking issue"
        return nil
      end
      payload = JSON.parse(res.body)
      return nil if payload.size == 0
      STDERR.puts "===> Got multiple tracking issues for external_id #{spec_ident}" if payload.size > 1
      payload.first
    end

    def issue_create_params
      { comments: [issue_comment],
        description: issue_description,
        external_id: spec_ident,
        name: issue_name,
        project_id: shortcut_project_id,
        story_type: "bug",
      }.to_json
    end

    def create_issue
      story_create_uri = URI("https://api.app.shortcut.com/api/v3/stories?token=#{shortcut_token}")
      res = Net::HTTP.post(story_create_uri, issue_create_params, "Content-Type" => "application/json")
      if res.code !~ /^2/
        STDERR.puts "===> Got response code #{res.code} while creating tracking issue"
        return nil
      end
      JSON.parse(res.body)
    end

    def add_comment(issue)
      comment_create_uri = URI("https://api.app.shortcut.com/api/v3/stories/#{issue["id"]}/comments?token=#{shortcut_token}")
      res = Net::HTTP.post(comment_create_uri, issue_comment.to_json, "Content-Type" => "application/json")
      if res.code !~ /^2/
        STDERR.puts "===> Got response code #{res.code} while creating issue comment"
        STDERR.puts "===> URI: 'https://api.app.shortcut.com/api/v3/stories/#{issue["id"]}/comments?token=REDACTED'"
        return nil
      end
      JSON.parse(res.body)
    end

    def issue_description
      # For some reason, Shortcut doesn't give the full Markdown treatment to
      # descriptions that come in through the API. So we transform this so
      # that each paragraph is one line.
      <<~"MARKDOWN".split(/\n\n+/).map(&:squish).join("\n\n")
        The spec "#{example.full_description}" has been identified as a flaky
        spec (one that fails intermittently). It can be helpful to just flag
        flaky specs rather than fix them right away, because sometimes it
        requires examining details from a series of failures before the
        patterns of success and failure become clear and the cause of the
        flakiness can be determined. The cause can be time-related issues,
        geography, or simply the particular random test values that are used
        in some situations.

        This story has been created by FlakySpecs to track information about
        cases where the spec fails in CI. Each failure will be attached here
        as a comment.
      MARKDOWN
    end

    def issue_name
      "Flaky Spec: #{example.full_description}"
    end

    def issue_comment
      { text: <<~"MARKDOWN"
                An intermittent failure occurred in example `#{example.full_description}`.

                Failure timestamp: #{Time.now.utc}.
                The CI build details can be found at: #{build_link}.
                Part of pull request(s): #{pull_requests}
                RSpec config.seed: #{config.seed}

                Failure information:
                ```
                #{example.exception.class}: #{example.exception.message}
                ```

                Details provided:
                ```
                #{diagnostic_details}
                ```

                Backtrace:
                ```
                #{example.exception.backtrace.join("\n")}
                ```
              MARKDOWN
      }
    end

    def diagnostic_details
      return "None." if detail_pairs.empty?

      detail_pairs.map{ |name, value|
        "  #{name}: '#{value.inspect}'"
      }.join("\n")
    end

    def detail_pairs
      result = []

      Array(example.metadata[:flaky]).each do |name|
        next unless name.kind_of?(Symbol) || name.kind_of?(String)
        sym = name.to_sym
        begin
          result << [sym, hook.send(sym)]
        rescue NoMethodError
          STDERR.puts "could not find value for metadata[:flaky][#{sym.inspect}]"
        end
      end

      hook.__flaky_locals.each do |name, value|
        result << [name.to_sym, value]
      end

      result
    end

    def shortcut_token
      ENV.fetch("FLAKY_SPECS_SHORTCUT_TOKEN") { raise(ArgumentError, "You must configure a Shortcut token at ENV['FLAKY_SPECS_SHORTCUT_TOKEN']") }
    end

    def shortcut_project_id
      ENV.fetch("SHORTCUT_PROJECT_ID")
    end

    # documentation about CircleCI-defined environment variables is at
    # https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables

    def build_link
      ENV.fetch("CIRCLE_BUILD_URL")
    end

    def pull_requests
      ENV["CIRCLE_PULL_REQUESTS"]
    end
  end
end

class RSpec::Core::ExampleGroup
  include FlakySpecs::FlakyHelpers
end