benchie.cpp

#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <string.h>
#include <vector>
#include <string>
#include <chrono>
using namespace std;

#ifndef _WIN32
    #define redir_str " >/dev/null 2>/dev/null"
    #include <sys/times.h>
    #include <unistd.h>
    bool supports_color()
    {
        if (getenv("FORCECOLOR"))
            return true;
        if (getenv("NOCOLOR"))
            return false;
        if (!isatty(fileno(stdout)))
            return false;
        char * term = getenv("TERM");
        if (term && (strstr(term, "color") || strstr(term, "xterm")))
            return true;
        if (getenv("COLORTERM"))
            return true;
        return false;
    }
    
    double get_cpu_time()
    {
        struct tms my_time;
        clock_t my_clock = times(&my_time);
        (void)my_clock;
        double utime = (double)my_time.tms_cutime / sysconf(_SC_CLK_TCK);
        double stime = (double)my_time.tms_cstime / sysconf(_SC_CLK_TCK);
        auto ret = utime;
        ret += stime;
        return ret;
    }
    void system_prepare();
#else
    #define redir_str ""
    #define WIN32_LEAN_AND_MEAN
    #define VC_EXTRALEAN
    #include <windows.h>
    bool supports_color()
    {
        if (getenv("FORCECOLOR"))
            return true;
        if (getenv("NOCOLOR"))
            return false;
        CONSOLE_SCREEN_BUFFER_INFO info;
        GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info);
        if (info.dwSize.X && info.dwSize.Y)
            return true;
        return false;
    }
    double _time = 0.0;
    double get_cpu_time() { return _time; }
    HANDLE subproc_id = 0;
    int system(char * cmd)
    {
        STARTUPINFO sinfo = {};
        sinfo.cb = sizeof(STARTUPINFO);
        sinfo.dwFlags = STARTF_USESTDHANDLES;
        
        HANDLE hNul = CreateFile("NUL", GENERIC_WRITE, FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
        sinfo.hStdOutput = hNul;
        sinfo.hStdError = hNul;
        
        PROCESS_INFORMATION pinfo = {};
        if (!CreateProcess(0, cmd, 0, 0, 0, 0, 0, 0, &sinfo, &pinfo))
            return -1;
        WaitForSingleObject(pinfo.hProcess, INFINITE);
        
        FILETIME ctime, xtime, stime, utime;
        GetProcessTimes(pinfo.hProcess, &ctime, &xtime, &stime, &utime);

        ULARGE_INTEGER stime_u, utime_u;
        stime_u.LowPart = stime.dwLowDateTime;
        stime_u.HighPart = stime.dwHighDateTime;
        utime_u.LowPart = utime.dwLowDateTime;
        utime_u.HighPart = utime.dwHighDateTime;

        // Convert to milliseconds
        double stime_s = stime_u.QuadPart / 10000000.0;
        double utime_s = utime_u.QuadPart / 10000000.0;
        _time += utime_s;
        _time += stime_s;

        DWORD ret;
        GetExitCodeProcess(pinfo.hProcess, &ret);
        
        CloseHandle(hNul);
        CloseHandle(pinfo.hProcess);
        CloseHandle(pinfo.hThread);
        return ret;
    }
#endif
    
static inline double get_time()
{
    auto now = std::chrono::steady_clock::now();
    auto duration = now.time_since_epoch();
    double ret = std::chrono::duration<double>(duration).count();
    return ret;
}

const char * c_reset = "";
const char * c_green = "";
const char * c_blue = "";
const char * c_pink = "";
const char * c_yellow = "";

static inline char * f2s(double f, int sizelimit, const char * color)
{
    static char x[256][256] = {{}};
    static size_t i = 0;
    memset(x[i], 0, 256);
    auto m = pow(10.0, (double)sizelimit-2);
    auto m2 = pow(10.0, (double)sizelimit-1);
    if (f > m2)
        snprintf(x[i], 255, "%s%.*e%s", color, sizelimit-6, f, c_reset);
    else
    {
        int lten = max(0, (int)ceil(log10(fabs(f)))-1);
        f *= m;
        f = round(f);
        f /= m;
        snprintf(x[i], 255, "%s%*.*f%s", color, sizelimit, sizelimit-2-lten, f, c_reset);
    }
    
    return x[i++];
}

double get_conf_r(int degrees)
{
    if (degrees < 0)
        return 1.0/0.0;
    // critical values of student's t distribution for 99% confidence interval
    if (degrees <= 16)
    {
        double asdf[17] = {
            1000000000000.0,
            63.6551,
            9.9247,
            5.8408,
            4.6041,
            4.0322,
            3.7074,
            3.4995,
            3.3554,
            3.2498,
            3.1693,
            3.1058,
            3.0545,
            3.0123,
            2.9768,
            2.9467,
            2.9208,
        };
        return asdf[degrees];
    }
    // cruddy approximation (varies from +0.0 to +0.01ish the real value, conservatively)
    return 61.4 / exp(pow((log(log(degrees) + 1.0)) * 18.1, 0.9)) + 2.576;
}

int main(int argc, char ** argv)
{
    if (argc == 1)
        return puts("usage: benchify \"cmd1\" \"cmd2\" ...");
    vector<string> cmds;
    vector<vector<double>> times;
    vector<vector<double>> times2;
    for (size_t i = 1; i < (unsigned)argc; i++)
    {
        cmds.push_back(argv[i]);
        times.push_back({});
    }
    
    if (supports_color())
    {
        c_reset = "\033[0m";
        c_green = "\033[92m";
        c_blue = "\033[36m";
        c_pink = "\033[35m";
        c_yellow = "\033[93m";
    }
    
    size_t warmups = 1;
    size_t runs = 10;
    
    for (size_t i = 0; i < cmds.size(); i++)
    {
        size_t max_runs = warmups + runs;
        printf("command %zd/%zd: %s%s%s...\n", i+1, cmds.size(), c_yellow, cmds[i].data(), c_reset);
        double t2 = 0.0;
        double t_min = 1.0/0.0;
        double t_max = -1.0;
        
        int num_outliers = 0;
        
        double outlying_lo = 0.0;
        double outlying_hi = 1.0/0.0;
        
        for (size_t j = 0; j < max_runs; j++)
        {
            if (j < warmups)
                printf("\r%s...Warmup %s%zd%s/%s%zd%s...%s\r", c_green, c_reset, j+1, c_green, c_reset, warmups, c_green, c_reset);
            else if (j == warmups)
                printf("\r%s...Initial run...%s                     \r", c_green, c_reset);
            fflush(stdout);
            double start = get_time();
            double ustart = get_cpu_time();
            
            auto ret = system((cmds[i] + redir_str).data());
            
            /*
            // for debugging extreme outlier rejection
            if (j == 5)
            #ifndef _WIN32
                sleep(3);
            #else
                Sleep(3000);
            #endif
            */
            
            double uend = get_cpu_time();
            double end = get_time();
            if (ret)
            {
                printf("subcommand failed: %s\n", cmds[i].data());
                exit(ret);
            }
            if (j < warmups) continue;
            times[i].push_back(end - start);
            t2 += uend-ustart;
            
            double mean = 0.0;
            size_t n = 0;
            for (size_t k = 0; k < times[i].size(); k++)
            {
                if (times[i][k] < 0.0)
                    continue;
                mean += times[i][k];
                n += 1;
            }
            mean /= n;
            
            double stddev = 0.0;
            for (size_t k = 0; k < times[i].size(); k++)
            {
                if (times[i][k] < 0.0)
                    continue;
                double x = times[i][k]-mean;
                stddev += x * x;
            }
            if (stddev == 0.0) stddev += 0.000001;
            stddev /= n-1;
            stddev = sqrt(stddev);
            if (mean < t_min) t_min = mean;
            if (mean > t_max) t_max = mean;
            //printf("\r...%f   ", times[i].back());
            double r = get_conf_r(n-1) / sqrt(double(n));
            printf("\r%4zd/%-4zd... %ss  (%s cpu)  range:%s~%s  99%%:%s~%s  \r",
                j+1-warmups, runs,
                f2s(mean, 8, c_green),
                f2s(t2 / (j + 1- warmups), 8, c_blue),
                f2s(t_min, 6, c_pink),
                f2s(t_max, 6, c_pink),
                f2s(mean-stddev*r, 6, c_pink),
                f2s(mean+stddev*r, 6, c_pink));
            fflush(stdout);
            
            outlying_lo = mean-stddev*max(0.0, r)*2.0 - 0.1;
            outlying_hi = mean+stddev*max(0.0, r)*2.0 + 0.1;
            
            // extreme outlier discarding
            if (j - warmups < runs*2)
            {
                int new_num_outliers = 0;
                for (size_t k = 0; k < times[i].size(); k++)
                {
                    if (times[i][k] < 0.0 || times[i][k] < outlying_lo || times[i][k] > outlying_hi)
                    {
                        new_num_outliers += 1;
                        times[i][k] = -1.0;
                    }
                }
                max_runs += new_num_outliers - num_outliers;
                num_outliers = new_num_outliers;
            }
        }
        double mean = 0.0;
        size_t n = 0;
        for (size_t k = 0; k < times[i].size(); k++)
        {
            if (times[i][k] < 0.0)
                continue;
            mean += times[i][k];
            n += 1;
        }
        mean /= n;
        double stddev = 0.0;
        for (size_t k = 0; k < times[i].size(); k++)
        {
            if (times[i][k] < 0.0)
                continue;
            double x = times[i][k]-mean;
            stddev += x * x;
        }
        stddev /= n-1;
        stddev = sqrt(stddev);
        double r = get_conf_r(n-1) / (sqrt((double)n));
        printf("\r%4zd/%-4zd    %ss  (%s cpu)  range:%s~%s  99%%:%s~%s  \n",
            max_runs-warmups, runs,
            f2s(mean, 8, c_green),
            f2s(t2 / runs, 8, c_blue),
            f2s(t_min, 6, c_pink),
            f2s(t_max, 6, c_pink),
            f2s(mean-stddev*r, 6, c_pink),
            f2s(mean+stddev*r, 6, c_pink));
    }

    return 0;
}