export_bitrise_test_report.rb

require_relative './helpers/config_item_validation_helpers'
require 'trainer'

module Fastlane
  module Actions
    class ExportBitriseTestReportAction < Action
      extend ConfigItemValidationHelpers

      class Runner
        attr_reader :name, :xcresult_path, :test_result_dir, :deploy_dir

        def initialize(params)
          @name = params[:name]
          @xcresult_path = params[:xcresult_path] || Actions.lane_context[Actions::SharedValues::SCAN_GENERATED_XCRESULT_PATH]
          @test_result_dir = params[:test_result_dir]
          @deploy_dir = params[:deploy_dir]
        end

        def run
          # When BITRISE_TEST_RESULT_DIR isn't set, we're likely not in a step so there is nothing to do.
          if test_result_dir.nil?
            UI.important("Skipping becuase BITRISE_TEST_RESULT_DIR was not set. Are you running in a Bitrise step?")
            return
          end

          # Make sure that the xcresult bundle exists
          unless xcresult_path && File.exist?(xcresult_path)
            UI.user_error!("An xcresult bundle could not be found. Run scan with result_bundle set to true, or provide the xcresult_path option")
          end

          # Copy the xcresult bundle into the test result directory
          UI.message("Using '#{xcresult_path}' for the Test Report")
          report_dir = File.join(test_result_dir, name)
          FileUtils.mkdir_p(report_dir)
          FileUtils.cp_r(xcresult_path, File.join(report_dir, File.basename(xcresult_path)))

          # Write the test-info.json
          payload = {'test-name' => name}
          File.open(File.join(report_dir, 'test-info.json'), "w") do |f|
            f.write(payload.to_json)
          end

          # Deploy the .xcresult bundle as a zip archive if the deployment directory was also set
          if deploy_dir && File.exist?(deploy_dir)
            UI.message("Creating a zip archive of .xcresult bundle for deployment as well")
            Actions::ZipAction.run({
              :path => xcresult_path,
              :output_path => File.join(deploy_dir, "#{name}.xcresult.zip")
            })
          end

          # Export the test result into environment variables for use by other steps
          if envman_installed?
            UI.message("Exporting test results to environment for other steps")
            parser = Trainer::TestParser.new(xcresult_path)

            # Calculate test results
            total_tests = parser.number_of_tests_excluding_retries
            failed_tests = parser.number_of_failures_excluding_retries
            passed_tests = total_tests - failed_tests
            retried_tests = retried_test_count(parser)
            result = total_tests > 0 && failed_tests == 0 ? 'succeeded' : 'failed'

            # Find the failed test names if there were any
            failures = failed_tests_excluding_retries(parser)

            # Set them with envman
            Action.sh(envman_tool_name, 'add', '--key', 'BITRISE_XCRESULT_PATH', '--value', xcresult_path)
            Action.sh(envman_tool_name, 'add', '--key', 'BITRISE_XCODE_TEST_RESULT', '--value', result)
            Action.sh(envman_tool_name, 'add', '--key', 'BITRISE_XCODE_TEST_TOTAL_COUNT', '--value', total_tests.to_s)
            Action.sh(envman_tool_name, 'add', '--key', 'BITRISE_XCODE_TEST_PASSED_COUNT', '--value', passed_tests.to_s)
            Action.sh(envman_tool_name, 'add', '--key', 'BITRISE_XCODE_TEST_FAILED_COUNT', '--value', failed_tests.to_s)
            Action.sh(envman_tool_name, 'add', '--key', 'BITRISE_XCODE_TEST_RETRIED_COUNT', '--value', retried_tests.to_s)
            Action.sh(envman_tool_name, 'add', '--key', 'BITRISE_XCODE_TEST_FAILURES', '--value', failures.join(',')) unless failures.empty?
            # Note: This value can only be populated via the `SCAN_DEVICES` env var meaning that passing the `device`/`devices` option manually will not work
            Action.sh(envman_tool_name, 'add', '--key', 'BITRISE_XCODE_TEST_DEVICES', '--value', ENV['SCAN_DEVICES']) if ENV['SCAN_DEVICES']

            # Return nothing
            return
          end
        end

        def envman_installed?
          `which #{envman_tool_name}`.to_s.length != 0
        end

        def envman_tool_name
          'envman'
        end

        def retried_test_count(parser)
          # Each item in the array contains the identifier and the status.
          # To know the number of retries, we filter the results and tally the number of tests for a given identifier
          # Any identifier that was seen more than once represents a retry that might have been either successful for a failure
          flattened_test_results(parser)
            .map { _1[:identifier] }
            .tally
            .select { |_, value| value > 1 }
            .length
        end

        def failed_tests_excluding_retries(parser)
          return [] if parser.number_of_failures_excluding_retries == 0
          failures = []

          # Trainer doesn't specifically tell us which tests failed even after all retries...
          # But we can figure it out!
          #
          # It's not obvious though, because we technically don't know how many times a test should retry.
          # However given that we do know that at least one test have exhausted all retry attempts then
          # the number of times that test failed == the retry count.
          # This means that we can ignore any tests that failed less than that value.

          # Flatten all data into a single array, select the failed tests and tally the failures against their identifier
          # See the export_bitrise_test_report_spec.rb for an example of the `parser.data` structure
          failures = flattened_test_results(parser)
            .select { _1[:status] == "Failure" }
            .map { _1[:identifier] }
            .tally

          # Calculate the maximum number of retries based on the test that failed the most
          max_retries = failures.values.max

          # Return the identifiers (key) that exhausted all retries
          failures
            .select { |_, failure_count| failure_count == max_retries }
            .keys
        end

        def flattened_test_results(parser)
          parser.data
            .map { _1[:tests] }
            .flatten
        end
      end

      def self.run(params)
        Runner.new(params).run
      end

      def self.description
        "Prepares an xcresult bundle for display in the Bitrise Test Report"
      end

      def self.available_options
        [
          FastlaneCore::ConfigItem.new(key: :name,
                                        env_name: "BITRISE_TEST_RESULT_NAME",
                                        description: "Test name displayed on the tab of the Test Reports page",
                                        optional: false,
                                        type: String,
                                        verify_block: blank_value_not_allowed),
          FastlaneCore::ConfigItem.new(key: :xcresult_path,
                                        env_name: "BITRISE_XCRESULT_PATH",
                                        description: "Path to xcresult bundle",
                                        optional: true, # Uses SharedValues::SCAN_GENERATED_XCRESULT_PATH if not provided
                                        type: String),
          FastlaneCore::ConfigItem.new(key: :test_result_dir,
                                        env_name: "BITRISE_TEST_RESULT_DIR",
                                        description: "Root directory for all test results created by the Bitrise CLI",
                                        optional: true, # Action will skip if not set
                                        type: String),
          FastlaneCore::ConfigItem.new(key: :deploy_dir,
                                        env_name: "BITRISE_DEPLOY_DIR",
                                        description: "Root directory for artefacts being deployed",
                                        optional: true, # When set, a .zip of the .xcresult bundle will be made here for deployment
                                        type: String),
        ]
      end
    end
  end
end

export_bitrise_test_report_spec.rb

describe Fastlane::Actions::ExportBitriseTestReportAction::Runner do
  describe '#run' do
    it 'should skip when test_result_dir is not defined' do
      expect(Fastlane::UI).to receive(:important).with('Skipping becuase BITRISE_TEST_RESULT_DIR was not set. Are you running in a Bitrise step?').exactly(1).times

      subject = described_class.new({ :name => 'Test', :xcresult_path => nil, :test_result_dir => nil })
      subject.run
    end

    describe 'copying the xcresult bundle and writing test-info.json in test_result_dir' do
      include_context :uses_temp_dir

      let(:xcresult_path) {
        # Create a result bundle structure (a dir with a file)
        path = File.join(temp_dir, 'build/output/Test.xcresult')
        FileUtils.mkdir_p(path)
        FileUtils.touch(File.join(path, 'Data'))

        path
      }

      let(:test_result_dir) {
        path = File.join(temp_dir, 'tmp/test_results')
        FileUtils.mkdir_p(path)

        path
      }

      let(:deploy_dir) {
        path = File.join(temp_dir, 'tmp/deployments')
        FileUtils.mkdir_p(path)

        path
      }

      it 'should error if the xcresult_path could not be found' do
        subject = described_class.new({ :name => 'Test', :xcresult_path => nil, :test_result_dir => test_result_dir })

        expect { subject.run }.to(
          raise_error(%r|An xcresult bundle could not be found. Run scan with result_bundle set to true, or provide the xcresult_path option|)
        )
      end

      it 'should work with explicit xcresult_path' do
        expect_any_instance_of(described_class).to receive(:envman_installed?).and_return(false)

        subject = described_class.new({ :name => 'Test', :xcresult_path => xcresult_path, :test_result_dir => test_result_dir })
        subject.run

        expect(File).to exist(expected_xcresult_path)
        expect(File).to exist(expected_test_info_path)
        expect(File.read(expected_test_info_path)).to eq('{"test-name":"Test"}')
        expect(Dir.empty?(deploy_dir)).to eq(true)
      end

      it 'should use scans shared context when xcresult_path is nil' do
        expect_any_instance_of(described_class).to receive(:envman_installed?).and_return(false)

        Fastlane::Actions.lane_context[Fastlane::Actions::SharedValues::SCAN_GENERATED_XCRESULT_PATH] = xcresult_path

        subject = described_class.new({ :name => 'Test', :xcresult_path => nil, :test_result_dir => test_result_dir })
        subject.run

        expect(File).to exist(expected_xcresult_path)
        expect(File).to exist(expected_test_info_path)
        expect(File.read(expected_test_info_path)).to eq('{"test-name":"Test"}')
        expect(Dir.empty?(deploy_dir)).to eq(true)
      end

      it 'should also deploy a .zip archive when deploy_dir is set' do
        expect_any_instance_of(described_class).to receive(:envman_installed?).and_return(false)

        allow(FastlaneCore::Helper).to receive(:sh_enabled?).and_return(true)

        subject = described_class.new({ :name => 'Test', :xcresult_path => xcresult_path, :test_result_dir => test_result_dir, :deploy_dir => deploy_dir })
        subject.run

        expect(File).to exist(expected_xcresult_zip_path)
      end

      def expected_xcresult_path
        File.join(test_result_dir, 'Test/Test.xcresult')
      end

      def expected_test_info_path
        File.join(test_result_dir, 'Test/test-info.json')
      end

      def expected_xcresult_zip_path
        File.join(deploy_dir, 'Test.xcresult.zip')
      end

      it 'should set environment variables using envman upon success' do
        expect_any_instance_of(described_class).to receive(:envman_installed?).and_return(true)

        # trainer will report 3 tests with 1 failure
        parser = double(Trainer::TestParser, number_of_tests_excluding_retries: 3, number_of_failures_excluding_retries: 1, data: [
          {
            tests: [
              { identifier: "testA()", status: "Success" },
              { identifier: "testB()", status: "Failure" },
              { identifier: "testB()", status: "Success" },
              { identifier: "testC()", status: "Failure" },
              { identifier: "testC()", status: "Failure" },
              { identifier: "testC()", status: "Failure" }
            ]
          }
        ])
        expect(Trainer::TestParser).to receive(:new).and_return(parser)

        device_name = 'iPhone 14'
        ENV['SCAN_DEVICES'] = device_name

        # envman should be called with all expected variables
        expect(Fastlane::Action).to receive(:sh).with('envman', 'add', '--key', 'BITRISE_XCRESULT_PATH', '--value', xcresult_path)
        expect(Fastlane::Action).to receive(:sh).with('envman', 'add', '--key', 'BITRISE_XCODE_TEST_RESULT', '--value', 'failed')
        expect(Fastlane::Action).to receive(:sh).with('envman', 'add', '--key', 'BITRISE_XCODE_TEST_TOTAL_COUNT', '--value', '3')
        expect(Fastlane::Action).to receive(:sh).with('envman', 'add', '--key', 'BITRISE_XCODE_TEST_PASSED_COUNT', '--value', '2')
        expect(Fastlane::Action).to receive(:sh).with('envman', 'add', '--key', 'BITRISE_XCODE_TEST_FAILED_COUNT', '--value', '1')
        expect(Fastlane::Action).to receive(:sh).with('envman', 'add', '--key', 'BITRISE_XCODE_TEST_RETRIED_COUNT', '--value', '2')
        expect(Fastlane::Action).to receive(:sh).with('envman', 'add', '--key', 'BITRISE_XCODE_TEST_FAILURES', '--value', 'testC()')
        expect(Fastlane::Action).to receive(:sh).with('envman', 'add', '--key', 'BITRISE_XCODE_TEST_DEVICES', '--value', device_name)

        subject = described_class.new({ :name => 'Test', :xcresult_path => xcresult_path, :test_result_dir => test_result_dir })
        subject.run
      end
    end
  end
end