Created
November 16, 2016 19:32
-
-
Save ajbrock/61f1ea092f4178abbc53b6eb993f1970 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
%% Benchmark Analysis Script | |
% A Brock, 11.16.2016 | |
% | |
% This quick script runs through and determines the fastest method from a | |
% given set of benchmarks. | |
% | |
% Note that this script is really only set up to work well with a single | |
% benchmark file, as the indexing isn't quite perfect for the multipl | |
% inputs case. Extending it should be easy enough if desired. | |
%% Clear the playing field | |
clc | |
clear all | |
close all | |
fclose all; | |
% Get all files | |
f = {'benchmark_TITAN_stack2.txt'}; | |
%% Pull results | |
for i = 1:length(f) | |
fid = fopen(f{i},'r'); | |
j = 0; | |
k = 0; | |
while ~feof(fid); | |
s = fgets(fid); | |
deets = strfind(s,'CONFIG'); | |
sub = strfind(s,'Subpixel'); | |
normal = strfind(s,'Normal'); | |
im2col = strfind(s,'IM2COL'); | |
batchroll = strfind(s,'Batchroll'); | |
batchstack = strfind(s,'BatchStack'); | |
% This could be done much more elegantly with a switch-case and a | |
% struct or cell array, but I wrote it as fast as possible (AFAP) | |
% so it's just all copy-pastes, all the way down. | |
if deets | |
e = sscanf(s,'CONFIG: input = %f x %f x %f * ker = %f x %f x 3 x 3 ( bs = 128 , stride = 1 scale = %f )'); | |
j = j+1; | |
input_size(j,1:3)=e(1:3); | |
kern_size(j,1:2)=e(4:5); | |
scale(j)=e(end); | |
elseif sub | |
if strfind(s,'fprop') | |
sub_fprop(i,j) = sscanf(s,'Subpixel DCD ==> fprop ==> %f'); | |
elseif strfind(s,'input') | |
sub_bprop_input(i,j) = sscanf(s,'Subpixel DCD ==> bprop inputs ==> %f'); | |
elseif strfind(s,'weights') | |
sub_bprop_weights(i,j) = sscanf(s,'Subpixel DCD ==> bprop weights ==> %f'); | |
end | |
elseif normal | |
if strfind(s,'fprop') | |
normal_fprop(i,j) = sscanf(s,'Normal DCD ==> fprop ==> %f'); | |
elseif strfind(s,'input') | |
normal_bprop_input(i,j) = sscanf(s,'Normal DCD ==> bprop inputs ==> %f'); | |
elseif strfind(s,'weights') | |
normal_bprop_weights(i,j) = sscanf(s,'Normal DCD ==> bprop weights ==> %f'); | |
end | |
elseif im2col | |
if strfind(s,'fprop') | |
im2col_fprop(i,j) = sscanf(s,'IM2COL DCD ==> fprop ==> %f'); | |
elseif strfind(s,'input') | |
im2col_bprop_input(i,j) = sscanf(s,'IM2COL DCD ==> bprop inputs ==> %f'); | |
elseif strfind(s,'weights') | |
im2col_bprop_weights(i,j) = sscanf(s,'IM2COL DCD ==> bprop weights ==> %f'); | |
end | |
elseif batchroll | |
if strfind(s,'fprop') | |
batchroll_fprop(i,j) = sscanf(s,'Batchroll DCD ==> fprop ==> %f'); | |
elseif strfind(s,'input') | |
batchroll_bprop_input(i,j) = sscanf(s,'Batchroll DCD ==> bprop inputs ==> %f'); | |
elseif strfind(s,'weights') | |
batchroll_bprop_weights(i,j) = sscanf(s,'Batchroll DCD ==> bprop weights ==> %f'); | |
end | |
elseif batchstack | |
if strfind(s,'fprop') | |
batchstack_fprop(i,j) = sscanf(s,'BatchStack DCD ==> fprop ==> %f'); | |
elseif strfind(s,'input') | |
batchstack_bprop_input(i,j) = sscanf(s,'BatchStack DCD ==> bprop inputs ==> %f'); | |
elseif strfind(s,'weights') | |
batchstack_bprop_weights(i,j) = sscanf(s,'BatchStack DCD ==> bprop weights ==> %f'); | |
end | |
end | |
end | |
end | |
fclose(fid); | |
%% Organize reslts and find fastest | |
sub_speed = sub_fprop+sub_bprop_input+sub_bprop_weights; | |
no_speed = normal_fprop+normal_bprop_input+normal_bprop_weights; | |
im2col_speed = im2col_fprop+im2col_bprop_input+im2col_bprop_weights; | |
batchroll_speed = batchroll_fprop+batchroll_bprop_input+batchroll_bprop_weights; | |
batchstack_speed = batchstack_fprop+batchstack_bprop_input+batchstack_bprop_weights; | |
names = {'subpixel','normal','im2col','batchroll','batchstack'}; | |
overall_speed = [sub_speed' no_speed' im2col_speed' batchroll_speed' batchstack_speed']; | |
fprop =[sub_fprop' normal_fprop' im2col_fprop' batchroll_fprop' batchstack_fprop']; | |
bprop_input =[sub_bprop_input' normal_bprop_input' im2col_bprop_input' batchroll_bprop_input' batchstack_bprop_input']; | |
bprop_weights =[sub_bprop_weights' normal_bprop_weights' im2col_bprop_weights' batchroll_bprop_weights' batchstack_bprop_weights']; | |
[~,win_all] = min(overall_speed'); | |
[~,win_fprop] = min(fprop'); | |
[~,win_bprop_i] = min(bprop_input'); | |
[~,win_bprop_w] = min(bprop_weights'); | |
for i=1:length(sub_speed) | |
fprintf('For input size %i,%i,%i, kernel size %i,%i,scale %i: ' ,input_size(i,:),kern_size(i,:),scale(i)); | |
fprintf('%s wins with %i ms overall, ',names{win_all(i)},overall_speed(i,win_all(i))); | |
fprintf('%s wins fprop with %i ms, ',names{win_fprop(i)},fprop(i,win_fprop(i))); | |
fprintf('%s wins bprop inputs with %i ms, ',names{win_bprop_i(i)},bprop_input(i,win_bprop_i(i))); | |
fprintf('%s wins bprop weights with %i.\n ',names{win_bprop_w(i)},bprop_weights(i,win_bprop_w(i))); | |
end | |
%% Scatter3 some results | |
% We'll go ahead and scatter some results to get a more qualitative feel | |
% for performance comparison. The X and Y axes will be the number of input | |
% and output feature maps, respectively, and the Z axis will represent | |
% the inverse of total time taken. Each scatter plot is for a constant | |
% set of spatial dimensions (i.e. 16x16,32x32,64x64) and dilation factor (2 or 4) | |
dims=[64,32,16,8,4]; | |
marker = {'bo','g*','rx','mh','ks'}; | |
for i = 1:length(dims) | |
indices = find( (input_size(:,2)==dims(i)).*(scale(:)==2) ); | |
figure;hold on; | |
for j = 1:size(overall_speed,2) | |
scatter3(kern_size(indices,1),kern_size(indices,2),overall_speed(indices,j),marker{j}); | |
end | |
title(sprintf('Speed Comparison at %ix%i, dilation factor 2',dims(i),dims(i))) | |
xlabel('Input Size') | |
ylabel('Output Size') | |
zlabel('Time (ms)') | |
legend(names); | |
view([30,30]) | |
grid on; | |
end | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment