crater hunter, some parts taken from github.com/rust-lang-nursery/crater

main.rs

extern crate serde;
#[macro_use]
extern crate serde_derive;
extern crate futures;
extern crate percent_encoding;
extern crate reqwest;
extern crate serde_json;
extern crate tokio_core;
extern crate url;

use std::cmp::min;
use std::collections::HashMap;
use std::fmt::Write;
use std::fs;
use std::path::PathBuf;

macro_rules! string_enum {
    (pub enum $name:ident { $($item:ident => $str:expr,)* }) => {
        #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Copy, Clone)]
        pub enum $name {
            $($item,)*
        }

        impl ::std::str::FromStr for $name {
            type Err = ();

            fn from_str(s: &str) -> Result<$name, ()> {
                Ok(match s {
                    $($str => $name::$item,)*
                    s => panic!("invalid {}: {}", stringify!($name), s),
                })
            }
        }

        impl ::std::fmt::Display for $name {
            fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
                write!(f, "{}", self.to_str())
            }
        }

        impl $name {
            pub fn to_str(&self) -> &'static str {
                match *self {
                    $($name::$item => $str,)*
                }
            }

            pub fn possible_values() -> &'static [&'static str] {
                &[$($str,)*]
            }
        }
    }
}

use reqwest::Client;

fn main() -> Result<(), Box<std::error::Error>> {
    let mut nll: TestResults = serde_json::from_slice(&fs::read("results-nll-1.json")?)?;
    let master: TestResults = serde_json::from_slice(&fs::read("results-pr-51762.json")?)?;

    let mut master_map = HashMap::new();
    for (idx, res) in master.crates.iter().enumerate() {
        assert!(master_map.insert(res.name.clone(), idx).is_none());
    }

    for res in &mut nll.crates {
        if !master_map.contains_key(&res.name) {
            // skip
            continue;
        }
        let master_res = &master.crates[master_map[&res.name]];
        if let Some(result) = &master_res.runs[0] {
            res.runs[0] = Some(result.clone());
            let cmp = match (result.res, res.runs[1].clone().unwrap().res) {
                (TestResult::BuildFail, TestResult::BuildFail) => Comparison::SameBuildFail,
                (TestResult::TestFail, TestResult::TestFail) => Comparison::SameTestFail,
                (TestResult::TestSkipped, TestResult::TestSkipped) => Comparison::SameTestSkipped,
                (TestResult::TestPass, TestResult::TestPass) => Comparison::SameTestPass,
                (TestResult::BuildFail, TestResult::TestFail)
                | (TestResult::BuildFail, TestResult::TestSkipped)
                | (TestResult::BuildFail, TestResult::TestPass)
                | (TestResult::TestFail, TestResult::TestPass) => Comparison::Fixed,
                (TestResult::TestPass, TestResult::TestFail)
                | (TestResult::TestPass, TestResult::BuildFail)
                | (TestResult::TestSkipped, TestResult::BuildFail)
                | (TestResult::TestFail, TestResult::BuildFail) => Comparison::Regressed,
                (TestResult::TestFail, TestResult::TestSkipped)
                | (TestResult::TestPass, TestResult::TestSkipped)
                | (TestResult::TestSkipped, TestResult::TestFail)
                | (TestResult::TestSkipped, TestResult::TestPass) => {
                    panic!("can't compare");
                }
            };
            res.res = cmp;
        }

        if let Some(run) = &mut res.runs[0] {
            run.log = format!(
                "https://cargobomb-reports.s3.amazonaws.com/pr-51762/{}",
                run.log
            );
        }
        if let Some(run) = &mut res.runs[1] {
            run.log = format!(
                "https://cargobomb-reports.s3.amazonaws.com/nll-1/{}",
                run.log
            );
        }
    }

    let mut count = 0;
    for res in &nll.crates {
        if res.res == Comparison::Regressed {
            if let Some(_) = &res.runs[1] {
                let path = PathBuf::from(format!("logs/nll-1/{}", res.name));
                if !path.exists() {
                    count += 1;
                }
            }
        }
    }

    let mut logs = Vec::new();

    let client = Client::new();
    for res in &nll.crates {
        let path = PathBuf::from(format!("logs/nll-1/{}", res.name));
        if path.exists() {
            continue;
        }
        if res.res == Comparison::Regressed {
            if let Some(nll_run) = &res.runs[1] {
                fs::create_dir_all(&path)?;
                let url = format!("{}/log.txt", nll_run.log).replace("+", "%2B");
                let mut response = client.get(&url).send()?;
                if response.status().is_success() {
                    let content = response.text()?;
                    fs::write(path.join("nll.log"), content)?;
                    count -= 1;
                    if count % 100 == 0 {
                        println!("crates left: {}", count);
                    }
                } else {
                    panic!("could not request {}: {:?}", nll_run.log, response);
                }
            }
        }
    }
    for res in &nll.crates {
        let path = PathBuf::from(format!("logs/nll-1/{}", res.name));
        let log = match fs::read_to_string(path.join("nll.log")) {
            Ok(s) => s,
            Err(_) => continue,
        };
        if !log.contains("internal compiler error") {
            continue;
        }
        logs.push((res, log));
    }
    let mut errors = Vec::new();
    for (res, log) in &logs {
        const ICE: &str = "internal compiler error";
        let error = log.lines()
            .find(|l| l.contains(&format!("{}: ", ICE)))
            .unwrap_or_else(|| {
                panic!("could not find ICE in log for {}", res.name);
            });
        let source = if error.contains("universal_regions.rs:825") {
            let start = error.find(&format!("{}: ", ICE)).unwrap();
            let start = start + format!("{}: ", ICE).len();
            let end = start + error[start..].find(": ").unwrap();
            &error[start..end]
        } else {
            let start = error.find(&format!("{}: ", ICE)).unwrap();
            let start = start + format!("{}: ", ICE).len();
            &error[start..]
        };
        errors.push((source, res.name.clone(), res));
    }

    errors.sort_by_key(|c| c.0.clone());

    let mut current_category = String::new();
    let mut report_md = String::new();
    for (source, _, res) in errors {
        let log_url = format!("{}/log.txt", res.runs[1].as_ref().unwrap().log).replace("+", "%2B");
        let category = source[..min(60, source.len())].to_owned();
        if category != current_category {
            writeln!(report_md, "#### {}", category)?;
            current_category = category;
        }
        writeln!(
            report_md,
            " - [{}]({}): [log]({})",
            res.name, res.url, log_url,
        )?;
    }
    fs::write("report.md", report_md)?;

    println!("nll results: {}", nll.crates.len());
    println!("master results: {}", master.crates.len());

    let out = serde_json::to_string(&nll)?;
    fs::write("results-merged.json", &out)?;

    Ok(())
}

#[derive(Serialize, Deserialize)]
pub struct TestResults {
    crates: Vec<CrateResult>,
}

#[derive(Serialize, Deserialize)]
struct CrateResult {
    name: String,
    url: String,
    res: Comparison,
    runs: [Option<BuildTestResult>; 2],
}

#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
enum Comparison {
    Regressed,
    Fixed,
    Skipped,
    Unknown,
    SameBuildFail,
    SameTestFail,
    SameTestSkipped,
    SameTestPass,
}

#[derive(Clone, Serialize, Deserialize)]
struct BuildTestResult {
    res: TestResult,
    log: String,
}

string_enum!(pub enum TestResult {
    BuildFail => "build-fail",
    TestFail => "test-fail",
    TestSkipped => "test-skipped",
    TestPass => "test-pass",
});