read_bench.m

%% Benchmark Analysis Script
% A Brock, 11.16.2016
%
% This quick script runs through and determines the fastest method from a
% given set of benchmarks.
%
% Note that this script is really only set up to work well with a single
% benchmark file, as the indexing isn't quite perfect for the multipl
% inputs case. Extending it should be easy enough if desired.
%% Clear the playing field
clc
clear all
close all
fclose all;
% Get all files
f = {'benchmark_TITAN_stack2.txt'};


%% Pull results
for i = 1:length(f)
    fid = fopen(f{i},'r');
    j = 0;
    k = 0;
    while ~feof(fid);
        s = fgets(fid);
        deets = strfind(s,'CONFIG');
        sub = strfind(s,'Subpixel');
        normal = strfind(s,'Normal');
        im2col = strfind(s,'IM2COL');
        batchroll = strfind(s,'Batchroll');
        batchstack = strfind(s,'BatchStack');
        
        % This could be done much more elegantly with a switch-case and a
        % struct or cell array, but I wrote it as fast as possible (AFAP)
        % so it's just all copy-pastes, all the way down.
        if deets
            e = sscanf(s,'CONFIG: input = %f x %f x %f * ker = %f x %f x 3 x 3 ( bs = 128 , stride = 1 scale = %f )');
            j = j+1;
            input_size(j,1:3)=e(1:3);
            kern_size(j,1:2)=e(4:5);
            scale(j)=e(end);
        elseif sub
            if strfind(s,'fprop')
                sub_fprop(i,j) = sscanf(s,'Subpixel DCD                                      ==> fprop         ==>      %f');
            elseif strfind(s,'input')
                sub_bprop_input(i,j) = sscanf(s,'Subpixel DCD                                      ==> bprop inputs  ==>      %f');
            elseif strfind(s,'weights')
                sub_bprop_weights(i,j) = sscanf(s,'Subpixel DCD                                      ==> bprop weights ==>      %f');
            end
        elseif normal
            if strfind(s,'fprop')
                normal_fprop(i,j) = sscanf(s,'Normal DCD                                        ==> fprop         ==>       %f');
            elseif strfind(s,'input')
                normal_bprop_input(i,j) = sscanf(s,'Normal DCD                                        ==> bprop inputs  ==>       %f');
            elseif strfind(s,'weights')
                normal_bprop_weights(i,j) = sscanf(s,'Normal DCD                                        ==> bprop weights ==>       %f');
            end
            
        elseif im2col
            if strfind(s,'fprop')
                im2col_fprop(i,j) = sscanf(s,'IM2COL DCD                                        ==> fprop         ==>       %f');
            elseif strfind(s,'input')
                im2col_bprop_input(i,j) = sscanf(s,'IM2COL DCD                                        ==> bprop inputs  ==>       %f');
            elseif strfind(s,'weights')
                im2col_bprop_weights(i,j) = sscanf(s,'IM2COL DCD                                        ==> bprop weights ==>       %f');
            end
            
        elseif batchroll
            if strfind(s,'fprop')
                batchroll_fprop(i,j) = sscanf(s,'Batchroll DCD                                        ==> fprop         ==>       %f');
            elseif strfind(s,'input')
                batchroll_bprop_input(i,j) = sscanf(s,'Batchroll DCD                                        ==> bprop inputs  ==>       %f');
            elseif strfind(s,'weights')
                batchroll_bprop_weights(i,j) = sscanf(s,'Batchroll DCD                                        ==> bprop weights ==>       %f');
            end
        elseif batchstack
            if strfind(s,'fprop')
                batchstack_fprop(i,j) = sscanf(s,'BatchStack DCD                                        ==> fprop         ==>       %f');
            elseif strfind(s,'input')
                batchstack_bprop_input(i,j) = sscanf(s,'BatchStack DCD                                        ==> bprop inputs  ==>       %f');
            elseif strfind(s,'weights')
                batchstack_bprop_weights(i,j) = sscanf(s,'BatchStack DCD                                        ==> bprop weights ==>       %f');
            end
        end
    end
end
fclose(fid);
%% Organize reslts and find fastest 
sub_speed = sub_fprop+sub_bprop_input+sub_bprop_weights;
no_speed = normal_fprop+normal_bprop_input+normal_bprop_weights;
im2col_speed = im2col_fprop+im2col_bprop_input+im2col_bprop_weights;
batchroll_speed = batchroll_fprop+batchroll_bprop_input+batchroll_bprop_weights;
batchstack_speed = batchstack_fprop+batchstack_bprop_input+batchstack_bprop_weights;

names = {'subpixel','normal','im2col','batchroll','batchstack'};
overall_speed = [sub_speed' no_speed' im2col_speed' batchroll_speed' batchstack_speed'];
fprop =[sub_fprop' normal_fprop' im2col_fprop' batchroll_fprop' batchstack_fprop'];
bprop_input =[sub_bprop_input' normal_bprop_input' im2col_bprop_input' batchroll_bprop_input' batchstack_bprop_input'];
bprop_weights =[sub_bprop_weights' normal_bprop_weights' im2col_bprop_weights' batchroll_bprop_weights' batchstack_bprop_weights'];
[~,win_all] = min(overall_speed');
[~,win_fprop] = min(fprop');
[~,win_bprop_i] = min(bprop_input');
[~,win_bprop_w] = min(bprop_weights');

for i=1:length(sub_speed)
    fprintf('For input size %i,%i,%i, kernel size %i,%i,scale %i: ' ,input_size(i,:),kern_size(i,:),scale(i));
    fprintf('%s wins with %i ms overall, ',names{win_all(i)},overall_speed(i,win_all(i)));
    fprintf('%s wins fprop with %i ms, ',names{win_fprop(i)},fprop(i,win_fprop(i)));
    fprintf('%s wins bprop inputs with %i ms, ',names{win_bprop_i(i)},bprop_input(i,win_bprop_i(i)));
    fprintf('%s wins bprop weights with %i.\n ',names{win_bprop_w(i)},bprop_weights(i,win_bprop_w(i)));
end


%% Scatter3 some results
% We'll go ahead and scatter some results to get a more qualitative feel
% for performance comparison. The X and Y axes will be the number of input
% and output feature maps, respectively, and the Z axis will represent
% the inverse of total time taken. Each scatter plot is for a constant 
% set of spatial dimensions (i.e. 16x16,32x32,64x64) and dilation factor (2 or 4)
dims=[64,32,16,8,4];
marker  = {'bo','g*','rx','mh','ks'};
for i = 1:length(dims)
    indices = find( (input_size(:,2)==dims(i)).*(scale(:)==2)  );
    figure;hold on;
    for j = 1:size(overall_speed,2)
        scatter3(kern_size(indices,1),kern_size(indices,2),overall_speed(indices,j),marker{j});
    end
    title(sprintf('Speed Comparison at %ix%i, dilation factor 2',dims(i),dims(i)))
    xlabel('Input Size')
    ylabel('Output Size')
    zlabel('Time (ms)')
    legend(names);
    view([30,30])
    grid on;
    
end