Skip to content

Instantly share code, notes, and snippets.

@ajbrock
Created November 16, 2016 19:32
Show Gist options
  • Save ajbrock/61f1ea092f4178abbc53b6eb993f1970 to your computer and use it in GitHub Desktop.
Save ajbrock/61f1ea092f4178abbc53b6eb993f1970 to your computer and use it in GitHub Desktop.
%% Benchmark Analysis Script
% A Brock, 11.16.2016
%
% This quick script runs through and determines the fastest method from a
% given set of benchmarks.
%
% Note that this script is really only set up to work well with a single
% benchmark file, as the indexing isn't quite perfect for the multipl
% inputs case. Extending it should be easy enough if desired.
%% Clear the playing field
clc
clear all
close all
fclose all;
% Get all files
f = {'benchmark_TITAN_stack2.txt'};
%% Pull results
for i = 1:length(f)
fid = fopen(f{i},'r');
j = 0;
k = 0;
while ~feof(fid);
s = fgets(fid);
deets = strfind(s,'CONFIG');
sub = strfind(s,'Subpixel');
normal = strfind(s,'Normal');
im2col = strfind(s,'IM2COL');
batchroll = strfind(s,'Batchroll');
batchstack = strfind(s,'BatchStack');
% This could be done much more elegantly with a switch-case and a
% struct or cell array, but I wrote it as fast as possible (AFAP)
% so it's just all copy-pastes, all the way down.
if deets
e = sscanf(s,'CONFIG: input = %f x %f x %f * ker = %f x %f x 3 x 3 ( bs = 128 , stride = 1 scale = %f )');
j = j+1;
input_size(j,1:3)=e(1:3);
kern_size(j,1:2)=e(4:5);
scale(j)=e(end);
elseif sub
if strfind(s,'fprop')
sub_fprop(i,j) = sscanf(s,'Subpixel DCD ==> fprop ==> %f');
elseif strfind(s,'input')
sub_bprop_input(i,j) = sscanf(s,'Subpixel DCD ==> bprop inputs ==> %f');
elseif strfind(s,'weights')
sub_bprop_weights(i,j) = sscanf(s,'Subpixel DCD ==> bprop weights ==> %f');
end
elseif normal
if strfind(s,'fprop')
normal_fprop(i,j) = sscanf(s,'Normal DCD ==> fprop ==> %f');
elseif strfind(s,'input')
normal_bprop_input(i,j) = sscanf(s,'Normal DCD ==> bprop inputs ==> %f');
elseif strfind(s,'weights')
normal_bprop_weights(i,j) = sscanf(s,'Normal DCD ==> bprop weights ==> %f');
end
elseif im2col
if strfind(s,'fprop')
im2col_fprop(i,j) = sscanf(s,'IM2COL DCD ==> fprop ==> %f');
elseif strfind(s,'input')
im2col_bprop_input(i,j) = sscanf(s,'IM2COL DCD ==> bprop inputs ==> %f');
elseif strfind(s,'weights')
im2col_bprop_weights(i,j) = sscanf(s,'IM2COL DCD ==> bprop weights ==> %f');
end
elseif batchroll
if strfind(s,'fprop')
batchroll_fprop(i,j) = sscanf(s,'Batchroll DCD ==> fprop ==> %f');
elseif strfind(s,'input')
batchroll_bprop_input(i,j) = sscanf(s,'Batchroll DCD ==> bprop inputs ==> %f');
elseif strfind(s,'weights')
batchroll_bprop_weights(i,j) = sscanf(s,'Batchroll DCD ==> bprop weights ==> %f');
end
elseif batchstack
if strfind(s,'fprop')
batchstack_fprop(i,j) = sscanf(s,'BatchStack DCD ==> fprop ==> %f');
elseif strfind(s,'input')
batchstack_bprop_input(i,j) = sscanf(s,'BatchStack DCD ==> bprop inputs ==> %f');
elseif strfind(s,'weights')
batchstack_bprop_weights(i,j) = sscanf(s,'BatchStack DCD ==> bprop weights ==> %f');
end
end
end
end
fclose(fid);
%% Organize reslts and find fastest
sub_speed = sub_fprop+sub_bprop_input+sub_bprop_weights;
no_speed = normal_fprop+normal_bprop_input+normal_bprop_weights;
im2col_speed = im2col_fprop+im2col_bprop_input+im2col_bprop_weights;
batchroll_speed = batchroll_fprop+batchroll_bprop_input+batchroll_bprop_weights;
batchstack_speed = batchstack_fprop+batchstack_bprop_input+batchstack_bprop_weights;
names = {'subpixel','normal','im2col','batchroll','batchstack'};
overall_speed = [sub_speed' no_speed' im2col_speed' batchroll_speed' batchstack_speed'];
fprop =[sub_fprop' normal_fprop' im2col_fprop' batchroll_fprop' batchstack_fprop'];
bprop_input =[sub_bprop_input' normal_bprop_input' im2col_bprop_input' batchroll_bprop_input' batchstack_bprop_input'];
bprop_weights =[sub_bprop_weights' normal_bprop_weights' im2col_bprop_weights' batchroll_bprop_weights' batchstack_bprop_weights'];
[~,win_all] = min(overall_speed');
[~,win_fprop] = min(fprop');
[~,win_bprop_i] = min(bprop_input');
[~,win_bprop_w] = min(bprop_weights');
for i=1:length(sub_speed)
fprintf('For input size %i,%i,%i, kernel size %i,%i,scale %i: ' ,input_size(i,:),kern_size(i,:),scale(i));
fprintf('%s wins with %i ms overall, ',names{win_all(i)},overall_speed(i,win_all(i)));
fprintf('%s wins fprop with %i ms, ',names{win_fprop(i)},fprop(i,win_fprop(i)));
fprintf('%s wins bprop inputs with %i ms, ',names{win_bprop_i(i)},bprop_input(i,win_bprop_i(i)));
fprintf('%s wins bprop weights with %i.\n ',names{win_bprop_w(i)},bprop_weights(i,win_bprop_w(i)));
end
%% Scatter3 some results
% We'll go ahead and scatter some results to get a more qualitative feel
% for performance comparison. The X and Y axes will be the number of input
% and output feature maps, respectively, and the Z axis will represent
% the inverse of total time taken. Each scatter plot is for a constant
% set of spatial dimensions (i.e. 16x16,32x32,64x64) and dilation factor (2 or 4)
dims=[64,32,16,8,4];
marker = {'bo','g*','rx','mh','ks'};
for i = 1:length(dims)
indices = find( (input_size(:,2)==dims(i)).*(scale(:)==2) );
figure;hold on;
for j = 1:size(overall_speed,2)
scatter3(kern_size(indices,1),kern_size(indices,2),overall_speed(indices,j),marker{j});
end
title(sprintf('Speed Comparison at %ix%i, dilation factor 2',dims(i),dims(i)))
xlabel('Input Size')
ylabel('Output Size')
zlabel('Time (ms)')
legend(names);
view([30,30])
grid on;
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment