Add SpecMetrics to your Rails project

README.md

SpecMetrics

SpecMetrics is a solution to get insights from your test suite. It's currently a work in progress (not even a gem yet) and it comes with the SpecMetrics Dashboard project to help get data-driven insights from the reports generated by this configuration.

SpecMetrics is deeply inspired from https://www.foraker.com/blog/profiling-your-rspec-suite and https://github.com/foraker/rspec_profiling.

AWS Setup

• Create dedicated IAM user.
• Create S3 bucket.
• Create S3 write-only policy for the bucket:

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "",
            "Effect": "Allow",
            "Action": [
                "s3:PutObject"
            ],
            "Resource": [
                "arn:aws:s3:::spec-metrics/*"
            ]
        }
    ]
}

Troubleshooting

VCR

If using VCR, be sure to add the following to your VCR configuration:

VCR.configure do |config|
  config.ignore_hosts '<YOUR-S3-BUCKET>.s3-eu-west-1.amazonaws.com'
end

counter.rb

# spec/support/spec_metrics/counter.rb
module SpecMetrics
  class Counter

    def initialize
      @queries_count = 0
      @queries_duration = 0
      @requests_count = 0
      @requests_duration = 0
    end

    def new_query(start, finish)
      @queries_count += 1
      @queries_duration += (finish - start)
    end

    def new_request(request)
      @requests_count += 1
      @requests_duration += request[:view_runtime].to_f / 1000 # in ms
    end

    # @return [Hash]
    def output
      {
        queries_count: @queries_count,
        queries_duration: @queries_duration,
        requests_count: @requests_count,
        requests_duration: @requests_duration
      }
    end
  end
end

example_output.rb

# spec/support/spec_metrics/example_output.rb
module SpecMetrics
  class ExampleOutput

    # @param metadata: `notification.example.metadata`
    # @param counter [SpecMetrics::ExampleCounter]
    def self.build(metadata, counter)
      new(metadata, counter).build
    end

    def initialize(metadata, counter)
      @metadata = metadata
      @counter = counter
    end

    def build
      filter(@metadata).merge(@counter.output)
    end

    private

    def filter(metadata)
      filter_base(metadata).merge(
        example_group: filter_group(metadata[:example_group]),
        execution_result: metadata[:execution_result].to_hash
      )
    end

    def filter_base(metadata)
      {
        description: metadata[:full_description],
        file_path: metadata[:file_path],
        line_number: metadata[:line_number]
      }
    end

    def filter_group(metadata)
      result = filter_base(metadata)
      return result if metadata[:parent_example_group].nil?
      result.merge(
        example_group: filter_group(metadata[:parent_example_group])
      )
    end

  end
end

Gemfile

# Add this gem to your Gemfile
gem 'aws-sdk', '~> 2.5.1'

git_output.rb

# spec/support/spec_metrics/git_output.rb
module SpecMetrics
  class GitOutput
    class << self

      def build
        sha = `git rev-parse HEAD`.strip
        {
          branch: `git rev-parse --abbrev-ref HEAD`.strip,
          date: Time.parse(`git show -s --format=%ci #{sha}`),
          remote_origin: `git config --get remote.origin.url`.strip,
          sha: sha,
          status: {
            deleted: status_deleted,
            modified: status_modified,
            new: status_new,
            untracked: status_untracked
          }
        }
      end

      def status_deleted
        status_lines.map { |l| l[/deleted:\s+(.+)\Z/, 1] }.compact
      end

      def status_modified
        status_lines.map { |l| l[/modified:\s+(.+)\Z/, 1] }.compact
      end

      def status_new
        status_lines.map { |l| l[/new file:\s+(.+)\Z/, 1] }.compact
      end

      def status_untracked
        lines = status_lines
        untracked_index = lines.index('Untracked files:')
        return [] if untracked_index.nil?
        after_untracked_empty_index = lines[untracked_index + 1..-1].index('')
        if after_untracked_empty_index.nil?
          lines[untracked_index + 1..-1]
        else
          lines[untracked_index + untracked_index + 1..after_untracked_empty_index]
        end
      end

      def status_lines
        `git status`.split("\n").map(&:strip)
      end
    end
    
  end
end

listener.rb

# spec/support/spec_metrics/listener.rb
require_relative './counter'
require_relative './git_output'

module SpecMetrics
  # Listener plugged into RSpec. Events to be triggered are setup in the
  # RSpec config object. See usage below.
  #
  # Taken from https://www.foraker.com/blog/profiling-your-rspec-suite
  # See https://github.com/foraker/rspec_profiling too
  #
  # Usage:
  #
  #     listener = SpecMetrics::Listener.new(aws_credentials)
  #     config.reporter.register_listener listener,
  #       :start, :example_started, :example_passed,
  #       :example_failed, :dump_summary, :seed
  #
  # Available events are defined in RSpec::Core::Reporter::RSPEC_NOTIFICATIONS
  # See https://github.com/rspec/rspec-core/blob/master/lib/rspec/core/reporter.rb
  class Listener

    LISTENED_EVENTS = %i(
      start dump_summary seed
      example_started example_passed example_failed
    ).freeze

    EXAMPLE_FIELDS = %w(
      file line_number description result time query_count
      query_time request_count request_time
    ).freeze

    IGNORED_QUERIES_PATTERN = %r{(
      pg_table|
      pg_attribute|
      pg_namespace|
      show\stables|
      pragma|
      sqlite_master/rollback|
      ^TRUNCATE TABLE|
      ^ALTER TABLE|
      ^BEGIN|
      ^COMMIT|
      ^ROLLBACK|
      ^RELEASE|
      ^SAVEPOINT
    )}xi

    # @param rspec_config [RSpec::Config]: RSpec configuration used to setup
    #   listener methods
    # @param collector_config [Hash]: an hash providing configuration for the output
    #   store. Currently, only available store is AWS S3, so the expected config
    #   contains :access_key_id, :secret_access_key and :region.
    def self.setup(rspec_config, collector_config)
      listener = new(collector_config)
      rspec_config.reporter.register_listener listener, *LISTENED_EVENTS
    end

    # @param collector_config [Hash]
    def initialize(collector_config)
      @aws_credentials = Aws::Credentials.new(
        collector_config[:access_key_id],
        collector_config[:secret_access_key]
      )
      @aws_region = collector_config[:region]
      @around_counter = Counter.new
    end

    def finalize
      output[:git] = GitOutput.build
      output[:system] = system_info
      output[:around] = @around_counter.output
      send_to_s3(output)
    end

    def start(_notification)
      start_counting_queries
      start_counting_requests
    end

    def seed(notification)
      output[:seed] = notification.seed
      output[:seed_used] = notification.seed_used?
    end

    def dump_summary(notification)
      %i(duration example_count failure_count pending_count).each do |key|
        output[key] = notification.send(key)
      end
      finalize
    end

    def example_started(_notification)
      @example_counter = Counter.new
    end

    def example_finished(notification)
      output[:examples].push(ExampleOutput.build(notification.example.metadata, @example_counter))
      @example_counter = nil
    end

    alias example_passed example_finished
    alias example_failed example_finished

    def start_counting_queries
      ActiveSupport::Notifications.subscribe('sql.active_record') do |_, start, finish, _, query|
        unless query[:sql] =~ IGNORED_QUERIES_PATTERN
          counter.new_query(start, finish)
        end
      end
    end

    def start_counting_requests
      ActiveSupport::Notifications.subscribe('process_action.action_controller') do |_, _, _, _, request|
        counter.new_request(request)
      end
    end

    def counter
      @example_counter ? @example_counter : @around_counter
    end

    private

    def send_to_s3(output)
      client = Aws::S3::Client.new(
        credentials: @aws_credentials,
        region: @aws_region
      )
      bucket = Aws::S3::Bucket.new('jt-spec-metrics', client: client)
      bucket.put_object(key: key(output), body: output.to_json)

    rescue Aws::S3::Errors::ServiceError => e
      # rescues all errors returned by Amazon Simple Storage Service
      puts("Failed to send spec-metrics data to S3 (#{e})")
      # TODO: dump locally and display a message on how to retry
      #       the S3 dump.
    end

    def key(output)
      remote = output[:git][:remote_origin].gsub(/[^\w]/, '-')
      branch = output[:git][:branch]
      sha = output[:git][:sha][0...8]
      time = Time.now.strftime('%Y%m%d%H%M%S%L')

      key = [
        'raw',
        remote,
        "#{branch}-#{time}-#{sha}.json"
      ].join('/')
      key
    end

    def output
      @output ||= {
        examples: []
      }
    end

    def system_info
      {
        current_dir: File.expand_path('.').split('/').last,
        hostname: `hostname`.strip
      }
    end

  end
end

spec_helper.rb

# spec/spec_helper.rb
# Your code ignored...

RSpec.configure do |config|
  # More of your code ignored...
  
  # SpecMetrics profiling (send reports to S3)
  require Rails.root.join('spec', 'support', 'spec_metrics', 'listener').to_s
  SpecMetrics::Listener.setup(config,
    access_key_id: "REPLACE-ME",
    secret_access_key: "REPLACE-ME",
    region: "REPLACE-ME"
  )
end

# More of your code ignored...