Last active
June 8, 2025 16:03
-
-
Save glv/787bfcf04ef5cc0e5785edb8f496d710 to your computer and use it in GitHub Desktop.
A metadata-driven flaky test reporting tool for RSpec, Shortcut, and CircleCI
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require 'digest/sha1' | |
require 'net/http' | |
RSpec.configure do |config| | |
# Use this way: | |
# | |
# let(:user) { create(:user) } | |
# let(:contact) { create(:contact, contact_of: user) } | |
# | |
# # The `:flaky` metadata signals that this should be treated as a flaky | |
# # spec, and the current values of `:user` and `:contact` should be recorded | |
# # to help understand why this spec only occasionally fails. (In this simple | |
# # example, `:user` and `:contact` aren't actually used in the body of the | |
# # spec, but typically you would only want to record available spec variables | |
# # that the spec actually uses. If there are none, you can simply mark the | |
# # spec with `:flaky` and no variables.) | |
# it "should always pass but sometimes fails", flaky: [:user, :contact] do | |
# phone_number = Faker::PhoneNumber.phone_number | |
# | |
# # The metadata approach above can't give FlakySpecs access to the spec's | |
# # local variables, so if the spec stores any test data in locals, you | |
# # should call `flaky_local_variables` to record those and their values. | |
# # Its parameter is an implicit Hash, so it can record multiple local | |
# # variables and their values in a single call. | |
# flaky_local_variables(phone_number: phone_number) | |
# | |
# # This will fail when phone_number has an extension | |
# expect(phone_number).not_to match(/x/) | |
# end | |
# | |
# When the expectation fails, the spec will *not* fail (because it's been | |
# identified as a flaky spec) but it will be reported to Shortcut as an | |
# intermittent failure, along with the current values of the random seed and | |
# `user`, `contact`, and `phone_number`, which can be used to help track | |
# down the circumstances under which the spec fails. Only one Shortcut story | |
# will be created for each [spec_name, exception_type] combination (yes, I've | |
# seen a flaky spec that was flaky in multiple different ways that produced | |
# different exceptions) and diagnostic data about each failure is recorded as | |
# a comment on that story. In other words, each flaky spec becomes one item | |
# on the team backlog, and that item gradually accumulates more data to help | |
# the team understand the flakiness and how to fix it. | |
# | |
# As mentioned above, this tool sits at the intersection of RSpec, Shortcut, | |
# and CircleCI. The basic idea, though, should be adaptable to any combination | |
# of test framework, backlog management tool, and CI system. | |
config.around(:example, :flaky) do |example| | |
example.example_group.let(:__flaky_locals) { {} } | |
example.run | |
unless example.exception.nil? | |
begin | |
FlakySpecs::ShortcutReporter.report(config, example, self) | |
example.example.display_exception = nil # suppress the failure | |
# TODO: sneak the diagnostic info into the XML report somehow | |
rescue => err | |
STDERR.puts "Error while processing flaky spec failure: #{err.class} #{err.message}" | |
STDERR.puts err.backtrace.join("\n") | |
end | |
end | |
end | |
end | |
module FlakySpecs | |
module FlakyHelpers | |
def flaky_local_variables(names_and_values) | |
names_and_values.each do |name, val| | |
__flaky_locals[name.to_sym] = val | |
end | |
end | |
alias :flaky_local_variable :flaky_local_variables | |
end | |
class ShortcutReporter | |
def self.report(config, example, hook) | |
new(config, example, hook).report | |
end | |
attr_reader :config, :example, :hook | |
def initialize(config, example, hook) | |
@config = config | |
@example = example | |
@hook = hook | |
end | |
def report | |
if ENV["CI"] == "true" | |
create_or_update_tracking_issue | |
else | |
print_local_message | |
end | |
end | |
def print_local_message | |
STDERR.puts "===> Suppressing flaky test failure:" | |
STDERR.puts example.exception | |
STDERR.puts "===> If this test fails in CI, its history will be tracked in Shortcut" | |
end | |
def create_or_update_tracking_issue | |
WebMock.disable! | |
VCR.eject_cassette | |
VCR.turned_off do | |
issue = find_issue | |
if issue.present? | |
add_comment issue | |
else | |
create_issue | |
end | |
end | |
ensure | |
WebMock.enable! | |
end | |
def spec_ident | |
@spec_ident ||= | |
begin | |
pieces = [ example.file_path, | |
example.full_description, | |
example.exception.class.name ] | |
Digest::SHA1.base64digest pieces.join("|") | |
end | |
end | |
def issue_search_params | |
{ archived: false, | |
external_id: spec_ident, | |
story_type: "bug", | |
}.to_json | |
end | |
def find_issue | |
story_search_uri = URI("https://api.app.shortcut.com/api/v3/stories/search?token=#{shortcut_token}") | |
res = Net::HTTP.post(story_search_uri, issue_search_params, "Content-Type" => "application/json") | |
if res.code !~ /^2/ | |
STDERR.puts "===> Got response code #{res.code} while searching for tracking issue" | |
return nil | |
end | |
payload = JSON.parse(res.body) | |
return nil if payload.size == 0 | |
STDERR.puts "===> Got multiple tracking issues for external_id #{spec_ident}" if payload.size > 1 | |
payload.first | |
end | |
def issue_create_params | |
{ comments: [issue_comment], | |
description: issue_description, | |
external_id: spec_ident, | |
name: issue_name, | |
project_id: shortcut_project_id, | |
story_type: "bug", | |
}.to_json | |
end | |
def create_issue | |
story_create_uri = URI("https://api.app.shortcut.com/api/v3/stories?token=#{shortcut_token}") | |
res = Net::HTTP.post(story_create_uri, issue_create_params, "Content-Type" => "application/json") | |
if res.code !~ /^2/ | |
STDERR.puts "===> Got response code #{res.code} while creating tracking issue" | |
return nil | |
end | |
JSON.parse(res.body) | |
end | |
def add_comment(issue) | |
comment_create_uri = URI("https://api.app.shortcut.com/api/v3/stories/#{issue["id"]}/comments?token=#{shortcut_token}") | |
res = Net::HTTP.post(comment_create_uri, issue_comment.to_json, "Content-Type" => "application/json") | |
if res.code !~ /^2/ | |
STDERR.puts "===> Got response code #{res.code} while creating issue comment" | |
STDERR.puts "===> URI: 'https://api.app.shortcut.com/api/v3/stories/#{issue["id"]}/comments?token=REDACTED'" | |
return nil | |
end | |
JSON.parse(res.body) | |
end | |
def issue_description | |
# For some reason, Shortcut doesn't give the full Markdown treatment to | |
# descriptions that come in through the API. So we transform this so | |
# that each paragraph is one line. | |
<<~"MARKDOWN".split(/\n\n+/).map(&:squish).join("\n\n") | |
The spec "#{example.full_description}" has been identified as a flaky | |
spec (one that fails intermittently). It can be helpful to just flag | |
flaky specs rather than fix them right away, because sometimes it | |
requires examining details from a series of failures before the | |
patterns of success and failure become clear and the cause of the | |
flakiness can be determined. The cause can be time-related issues, | |
geography, or simply the particular random test values that are used | |
in some situations. | |
This story has been created by FlakySpecs to track information about | |
cases where the spec fails in CI. Each failure will be attached here | |
as a comment. | |
MARKDOWN | |
end | |
def issue_name | |
"Flaky Spec: #{example.full_description}" | |
end | |
def issue_comment | |
{ text: <<~"MARKDOWN" | |
An intermittent failure occurred in example `#{example.full_description}`. | |
Failure timestamp: #{Time.now.utc}. | |
The CI build details can be found at: #{build_link}. | |
Part of pull request(s): #{pull_requests} | |
RSpec config.seed: #{config.seed} | |
Failure information: | |
``` | |
#{example.exception.class}: #{example.exception.message} | |
``` | |
Details provided: | |
``` | |
#{diagnostic_details} | |
``` | |
Backtrace: | |
``` | |
#{example.exception.backtrace.join("\n")} | |
``` | |
MARKDOWN | |
} | |
end | |
def diagnostic_details | |
return "None." if detail_pairs.empty? | |
detail_pairs.map{ |name, value| | |
" #{name}: '#{value.inspect}'" | |
}.join("\n") | |
end | |
def detail_pairs | |
result = [] | |
Array(example.metadata[:flaky]).each do |name| | |
next unless name.kind_of?(Symbol) || name.kind_of?(String) | |
sym = name.to_sym | |
begin | |
result << [sym, hook.send(sym)] | |
rescue NoMethodError | |
STDERR.puts "could not find value for metadata[:flaky][#{sym.inspect}]" | |
end | |
end | |
hook.__flaky_locals.each do |name, value| | |
result << [name.to_sym, value] | |
end | |
result | |
end | |
def shortcut_token | |
ENV.fetch("FLAKY_SPECS_SHORTCUT_TOKEN") { raise(ArgumentError, "You must configure a Shortcut token at ENV['FLAKY_SPECS_SHORTCUT_TOKEN']") } | |
end | |
def shortcut_project_id | |
ENV.fetch("SHORTCUT_PROJECT_ID") | |
end | |
# documentation about CircleCI-defined environment variables is at | |
# https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables | |
def build_link | |
ENV.fetch("CIRCLE_BUILD_URL") | |
end | |
def pull_requests | |
ENV["CIRCLE_PULL_REQUESTS"] | |
end | |
end | |
end | |
class RSpec::Core::ExampleGroup | |
include FlakySpecs::FlakyHelpers | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment