CircleCI Insights tool for comparing credits and runtimes.

circleci_insights

#!/usr/bin/env ruby

# Script to get information from the CircleCI Insights API for the number of credits used and time
# taken for running workflows across one or more branches.

require "net/http"
require "json"
require 'optparse'
require 'optparse/time'
require 'set'

limit = 10
org = ""
project = ""
branches = []
workflow_filter = nil
include_jobs = false
start_date = nil
end_date = nil

OptionParser.new do |parser|
  parser.on("--include-jobs", "Get metrics on jobs as well as workflows") do |arg|
    include_jobs = arg
  end
  parser.on("--limit LIMIT", Integer, "Max number of successful runs to analyze (default 10)") do |arg|
    limit = arg
  end
  parser.on("--org ORG", String, "Organization path") do |arg|
    org = arg
  end
  parser.on("-p PROJECT", "--project PROJECT", String, "Project name") do |arg|
    project = arg
  end
  parser.on("-b A,B,C", "--branch A,B,C", Array, "Branch names") do |arg|
    branches.concat(arg)
  end
  parser.on("-w A,B,C", "--workflows A,B,C", Array, "List of workflows to use") do |arg|
    workflow_filter = arg
  end
  parser.on("--start-date DATETIME", Time, "Start datetime for getting run data") do |arg|
    start_date = arg.utc
  end
  parser.on("--end-date DATETIME", Time, "End datetime for getting run data") do |arg|
    end_date = arg.utc
  end
  parser.on("-h", "--help", "Prints this help") do
    puts parser
    puts "CircleCI API key is passed using environment variable CIRCLECI_API_KEY."
    exit
  end
end.parse!

if org.empty?
  $stderr.puts "Missing --org option"
  exit 1
end

if project.empty?
  $stderr.puts "Missing --project option"
  exit 1
end

branches.uniq!
if branches.empty?
  $stderr.puts "Missing --branch option"
  exit 1
end

date_limit = Time.now - (3600 * 24 * 90) + 120
if start_date && (start_date < date_limit || start_date > Time.now)
  $stderr.puts "Start date must be within 90 days of now"
  exit 1
end

if end_date && (end_date < date_limit || end_date > Time.now)
  $stderr.puts "End date must be within 90 days of now"
  exit 1
end

if start_date && end_date && start_date >= end_date
  $stderr.puts "Start date must be before the end date"
  exit 1
end

if end_date && !start_date
  $stderr.puts "Start date must be provided along with the end date"
  exit 1
end

CIRCLECI_API_KEY = ENV.fetch("CIRCLECI_API_KEY", "")
if CIRCLECI_API_KEY.empty?
  $stderr.puts "Missing value for CIRCLECI_API_KEY environment variable"
  exit 1
end

def call_cicleci_api(path)
  $stderr.write(".") if $stderr.isatty
  uri = URI("https://circleci.com/api/v2#{path}")
  Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
    request = Net::HTTP::Get.new(uri)
    request["Circle-Token"] = CIRCLECI_API_KEY
    response = http.request(request)
    response.value
    return JSON.parse(response.body)
  end
end

def call_insights_api(org, project, path)
  call_cicleci_api("/insights/#{org}/#{project}#{path}")
end

def call_project_api(org, project, path)
  call_cicleci_api("/project/#{org}/#{project}#{path}")
end

def workflows_for_branch(org, project, branch, include_jobs:)
  workflows = {}
  call_project_api(org, project, "/pipeline?branch=#{URI.escape(branch)}")["items"].each do |pipeline|
    if pipeline["errors"].empty?
      call_cicleci_api("/pipeline/#{pipeline['id']}/workflow")["items"].each do |workflow|
        if include_jobs
          jobs = call_cicleci_api("/workflow/#{workflow['id']}/job")["items"].map { |job| job["name"] }
          workflows[workflow["name"]] = jobs
        else
          workflows[workflow["name"]] = []
        end
      end
      break
    end
  end
  workflows
end

def median(values)
  return 0 if values.empty?
  ((values[(values.size - 1) / 2] + values[values.size / 2]) / 2.0).round
end

def mean(values)
  return 0 if values.empty?
  (values.sum.to_f / values.size).round
end

def path_with_params(path, params)
  first = !path.include?("?")
  params.each do |name, value|
    next if value.nil?
    path = "#{path}#{first ? '?' : '&'}#{URI.escape(name.to_s)}=#{URI.escape(value.to_s)}"
    first = false
  end
  path
end

def get_workflow_summary_data(org:, project:, branch:, workflow:, limit:, start_date: nil, end_date: nil)
  path = path_with_params("/workflows/#{workflow}", "branch" => branch, "start-date" => start_date&.iso8601, "end-date" => end_date&.iso8601)
  workflow_summary_data = call_insights_api(org, project, path)
  workflow_summary_data["items"].select { |info| info["status"] == "success" }.take(limit)
end

def get_workflow_job_data(org:, project:, branch:, workflow:, jobs:, limit:, start_date: nil, end_date: nil)
  workflow_data = {}
  jobs.each do |job_name|
    job_data = []
    workflow_jobs_path = path_with_params("/workflows/#{workflow}/jobs/#{job_name}", "branch" => branch, "start-date" => start_date&.iso8601, "end-date" => end_date&.iso8601)
    next_page_token = nil

    loop do
      job_info = call_insights_api(org, project, path_with_params(workflow_jobs_path, next_page_token: next_page_token))
      items = Array(job_info["items"])
      break if items.empty?

      job_data.concat(items.select { |job| job["status"] == "success" })
      break if job_data.size >= limit

      next_page_token = job_info["next_page_token"]
      break if next_page_token.to_s.empty?
    end

    workflow_data[job_name] = job_data.take(limit)
  end
  workflow_data
end

workflow_data = {}
job_data = {}

branches.each do |branch_name|
  job_data[branch_name] = {}
  workflow_data[branch_name] = {}

  workflows_for_branch(org, project, branch_name, include_jobs: include_jobs).each do |workflow_name, job_names|
    next if workflow_filter && !workflow_filter.include?(workflow_name)
    workflow_data[branch_name][workflow_name] = get_workflow_summary_data(
      org: org,
      project: project,
      branch: branch_name,
      workflow: workflow_name,
      limit: limit,
      start_date: start_date,
      end_date: end_date
    )
    if include_jobs
      job_data[branch_name][workflow_name] = get_workflow_job_data(
      org: org,
      project: project,
      branch: branch_name,
      workflow: workflow_name,
      jobs: job_names,
      limit: limit,
      start_date: start_date,
      end_date: end_date
    )
    end
  end
end

# Clear progress dots
$stderr.write("\r\033[K") if $stderr.isatty

output = []

branches.each_with_index do |branch_name, index|
  output << :newline if index > 0
  branch_workflow_data = workflow_data[branch_name]
  branch_job_data = job_data[branch_name]

  if branch_workflow_data.empty?
    puts "No workflow runs found for #{branch}\n"
    next
  end

  output << [branch_name, "Success", "Mean Credits", "Mean Time", "Median Time"]
  output << :separator

  total_runs = 0
  total_credits = 0
  max_mean_time = 0
  max_median_time = 0

  branch_workflow_data.keys.sort.each do |workflow_name|
    workflow_summary_data = branch_workflow_data[workflow_name]
    workflow_credits = workflow_summary_data.map { |workflow| workflow["credits_used"] }
    workflow_durations = workflow_summary_data.map { |workflow| workflow["duration"] }
    workflow_mean_time = mean(workflow_durations)
    workflow_median_time = median(workflow_durations)

    total_runs = workflow_summary_data.size if workflow_summary_data.size > total_runs
    total_credits += workflow_credits.sum
    max_mean_time = workflow_mean_time if workflow_mean_time > max_mean_time
    max_median_time = workflow_median_time if workflow_median_time > max_median_time

    output << [workflow_name, workflow_summary_data.size, mean(workflow_credits), workflow_mean_time, workflow_median_time]

    if include_jobs
      workflow_job_data = branch_job_data[workflow_name]
      workflow_job_data.keys.sort.each do |job_name|
        jobs = workflow_job_data[job_name]
        job_credits = jobs.map { |job| job["credits_used"] }
        job_durations = jobs.map { |job| job["duration"] }
        output << ["  #{job_name}", job_credits.size, mean(job_credits), mean(job_durations), median(job_durations)]
      end
      output << :separator
    end
  end

  output << :separator unless include_jobs
  mean_total_credits = (total_runs > 0 ? (total_credits.to_f / total_runs.to_f).round : 0)
  output << ["Total", total_runs, mean_total_credits, max_mean_time, max_median_time]
end

label_padding = output.reject { |v| v.is_a?(Symbol) }.map(&:first).map(&:size).max
output.each do |name, runs, mean_credits, mean_time, median_time|
  if name == :separator
    puts "-" * (label_padding + 51)
  elsif name == :newline
    puts ""
  else
    puts "#{name.ljust(label_padding)} | #{runs.to_s.rjust(7)} | #{mean_credits.to_s.rjust(12)} | #{mean_time.to_s.rjust(9)} | #{median_time.to_s.rjust(11)}"
  end
end