Ruby KPI gatherer script

main.rb

require 'octokit'
require 'rest-client'
require 'json'
require './ignored_repos.rb'

def get_cc_issue_details(num_issues, repo_id, snapshot_id, debug)
  url = "https://api.codeclimate.com/v1/repos/#{repo_id}/snapshots/#{snapshot_id}/issues?page[size]=100"
  page = 1

  categories = Hash.new()
  severities = Hash.new()
  engines = Hash.new()

  while num_issues > 0 do

    puts "CC Issue URL: #{url}&page[number]=#{page}" if debug
    issue_response = RestClient.get("#{url}&page[number]=#{page}", {Authorization: "Token token=#{ENV['CC_TOKEN']}"})
    issue_results = JSON.parse(issue_response.to_str)
    puts "Issue Response Body: #{issue_results}" if debug

    issue_results["data"].each do |issue|
      # CC seems to only ever return 1 category, but returns it in an array
      puts "Issue attributes: #{issue["attributes"]}" if debug
      category = "#{issue["attributes"]["categories"][0]}"
      puts "Category: #{category}" if debug
      severity = "#{issue["attributes"]["severity"]}"
      puts "Severity: #{severity}" if debug
      engine = "#{issue["attributes"]["engine_name"]}"
      puts "Engine: #{engine}" if debug

      if categories.has_key? category then
        categories[category] += 1
      else
        categories[category] = 1
      end

      puts "Category #{category}: #{categories[category]}" if debug

      if severities.has_key? severity then
        severities[severity] += 1
      else
        severities[severity] = 1
      end

      puts "Severity #{severity}: #{severities[severity]}" if debug

      if engines.has_key? engine then
        engines[engine] += 1
      else
        engines[engine] = 1
      end

      puts "Engine #{engine}: #{engines[engine]}" if debug
    end

    num_issues -= 100
    page += 1
  end

  return categories, severities, engines
end

total_repos = 0
unarchived_repos = 0
skipped_repos = 0
github_orgs = %w(conjurdemos conjurinc cyberark)

# Skip repos owned by groups other than Conjur R&D or focused on recruiting candidates
repo_blocklist = get_repos_to_ignore()

client = Octokit::Client.new(:access_token => ENV['GH_TOKEN'])
client.auto_paginate = true

debug_repos = []
puts "Repo,Language,GHIssueCount,CCLinesOfCode,CCLetterGrade,CCTotalIssues,CCRemediationTime,CoveragePercent,CoverageGrade,IssueBuckets"

total_language_bytes = Hash.new

# Get the repositories and issue counts from GitHub
github_orgs.each do |org|
  repos = client.org_repos(org)

  total_repos += repos.count

  repos.each do |repo|
    # Skip archived repos
    next if repo["archived"]

    # Skip other repos if we're debugging certain ones
    debug = debug_repos.include? repo["full_name"]
    if debug_repos.count > 0 && !debug then
      skipped_repos += 1
      next
    end

    if repo_blocklist.include? repo["full_name"] then
      skipped_repos += 1
      next
    end

    unarchived_repos += 1

    name = repo["full_name"]
    puts "Repo name: #{name}" if debug

    primary_language = repo["language"]

    languages = Hash.new
    client.languages(repo["full_name"]).each do |language, size|
      languages[language] = size

      if total_language_bytes.has_key? language then
        total_language_bytes[language] += size
      else
        total_language_bytes[language] = size
      end
    end
    puts languages if debug

    issue_count = repo["open_issues"]

    url = "https://api.codeclimate.com/v1/repos?github_slug=#{name}&include[]=latest_default_branch_snapshot,latest_default_branch_test_report"
    puts "CC URL: #{url}" if debug

    codeclimate_response = RestClient.get(url, {Authorization: "Token token=#{ENV['CC_TOKEN']}"})
    puts "Request headers: #{codeclimate_response.request.headers}" if debug
    puts "Request url: #{codeclimate_response.request.url}" if debug
    puts "Response status code: #{codeclimate_response.code}" if debug
    puts "Response headers: #{codeclimate_response.headers}" if debug

    cc_results = JSON.parse(codeclimate_response.to_str)
    puts "Parsed response body: #{cc_results}" if debug

    lines_of_code = nil
    letter_grade = nil
    cc_issue_count = nil
    remediation_time = nil
    coverage_percent = nil
    coverage_grade = nil
    output = nil

    categories = Hash.new()
    severities = Hash.new()
    engines = Hash.new()

    if (cc_results["data"] != []) then
      puts "data exists" if debug
      if (!cc_results["included"].nil? && cc_results["included"] != []) then
        puts "#{cc_results["included"].count} reports exist" if debug
      #   cc_results["included"].each do |report|
      #     if report["type"] == "snapshots" then
      #       puts "in snapshot report" if debug
      #       puts "Report body: #{report}" if debug
      #       lines_of_code = report["attributes"]["lines_of_code"]
      #       letter_grade = report["attributes"]["ratings"][0]["letter"] if report["attributes"]["ratings"] != []
      #       cc_issue_count = report["meta"]["issues_count"]
      #       if cc_issue_count > 0 then
      #         categories, severities, engines = get_cc_issue_details(cc_issue_count,
      #           cc_results["data"][0]["id"], report["id"], debug)
      #       end
      #       remediation_time = report["meta"]["measures"]["remediation"]["value"]
      #     elsif report["type"] == "test_reports" then
      #       puts "in test report" if debug
      #       puts "Report body: #{report}" if debug
      #       coverage_percent = report["attributes"]["covered_percent"]
      #       coverage_grade = report["attributes"]["rating"]["letter"]
      #     else
      #       puts "in unknown report" if debug
      #       puts "Report body: #{report}" if debug
      #     end
      #   end
      end
    end

    puts "#{name},#{primary_language},#{issue_count},#{lines_of_code},#{letter_grade},#{cc_issue_count},#{remediation_time},#{coverage_percent},#{coverage_grade},\"#{languages}\",\"#{categories}\",\"#{severities}\",\"#{engines}\""
  end
end
$stderr.puts "#{total_repos} total repos, #{unarchived_repos} unarchived repos, #{skipped_repos} skipped repos"
$stderr.puts "Total bytes across all repos by language: #{total_language_bytes}"

total_bytes = 0
total_language_bytes.each do |language, size|
  total_bytes += size
end

total_language_bytes.each do |language, size|
  $stderr.puts "#{language} => #{(size.to_f/total_bytes) * 100}%"
end