open_telemetry_reporter.rb

# need to plug into library-level lifecycle hooks to get best timing info from
# minitest
module TimingPlugin
  def after_setup
    metadata[:start_time] = Minitest.clock_time
    super
  end

  def before_teardown
    super
    metadata[:end_time] = Minitest.clock_time
  end
end

class Minitest::Test
  include TimingPlugin
end

module Minitest
  module Reporters
    class OpenTelemetryReporter < BaseReporter
      def initialize(...)
        if ENV["OTEL_EXPORTER_OTLP_ENDPOINT"].blank? || ENV["OTEL_EXPORTER_OTLP_HEADERS"].blank?
          raise "ENV values must be set for OTEL_EXPORTER_OTLP_ENDPOINT and OTEL_EXPORTER_OTLP_HEADERS"
        end

        super
      end

      # beginning of entire test run (before anything is run)
      def start
        @start_clock = Minitest.clock_time
        @start_time = Time.current
      end

      # for each test, record will be called with a test Result
      def record(r)
        results << r
      end

      # runs at the end, when all tests have completed
      def report
        end_clock = Minitest.clock_time

        OpenTelemetry.tracer_provider.shutdown
        # also needs those ENV variables set to work properly
        # OpenTelemetry::SDK.configure do |c|
        #   c.service_name = "monorail-test-suite"
        #   c.error_handler = ->(exception:, message:) { raise(exception || message) }
        #   c.logger = Logger.new($stderr, level: ENV.fetch("OTEL_LOG_LEVEL", "fatal").to_sym)
        # end

        # OpenTelemetry.tracer_provider.force_flush(timeout: 120)
        tracer_provider = OpenTelemetry::SDK::Trace::TracerProvider.new(
          resource: OpenTelemetry::SDK::Resources::Resource.create({OpenTelemetry::SemanticConventions::Resource::SERVICE_NAME => "monorail-test-suite"})
        )
        otlp_exporter = OpenTelemetry::Exporter::OTLP::Exporter.new
        processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(otlp_exporter)
        # tracer_provider.add_span_processor(OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(OpenTelemetry::SDK::Trace::Export::ConsoleSpanExporter.new))
        tracer_provider.add_span_processor(processor)

        tracer = tracer_provider.tracer("monorail-test-suite")
        root_span = tracer.start_root_span("minitest suite", start_timestamp: @start_time)
        root_span.add_attributes({
          "minitest.test_count" => results.count,
          "minitest.suite_result" => passed? ? "passed" : "failed"
        })

        root_token = OpenTelemetry::Context.attach(OpenTelemetry::Trace.context_with_span(root_span, parent_context: OpenTelemetry::Context.empty))

        # other interesting ideas for result instrumentations
        # machine vs CI
        # github branch / pr / local
        results.each do |r|
          span = tracer.start_span(r.name, start_timestamp: time(r.metadata[:start_time]))

          result = if r.passed?
            "passed"
          elsif r.skipped?
            "skipped"
          elsif r.failure
            "failed"
          end

          span.add_attributes({
            "minitest.class" => r.klass,
            "minitest.assertions" => r.assertions,
            "minitest.source_location" => r.source_location.join(":"),
            "minitest.result" => result,
            "minitest.test_type" => test_type(r)
          })

          if !r.skipped? && r.failure.present? && (error = r.failure&.error)
            span.record_exception(error)
            span.status = OpenTelemetry::Trace::Status.error(error.message)
          end

          span.finish(end_timestamp: time(r.metadata[:end_time]))
        end

        root_span.finish(end_timestamp: time(end_clock))
        OpenTelemetry::Context.detach(root_token)

        tracer_provider.force_flush(timeout: 120)
        tracer_provider.shutdown(timeout: 120)

        nil
      end

      def passed?
        results.all? { |r| r.skipped? || r.passed? }
      end

      private

      def test_type(result)
        test_class = result.klass.constantize
        ancestors = test_class.ancestors.to_set
        if ancestors.include? ActionDispatch::SystemTestCase
          "system_test"
        elsif ancestors.include? ActionDispatch::IntegrationTest
          "controller_test"
        elsif ancestors.include? ActiveJob::TestCase
          "job_test"
        elsif ancestors.include? ActionMailer::TestCase
          "mailer_test"
        elsif ancestors.include? ViewComponent::TestCase
          "view_component_test"
        else
          "unit_test"
        end
      end

      # helper to translate monotonic time to human-readable timestamp
      def time(clock)
        offset = clock - @start_clock
        @start_time + offset
      end
    end
  end
end