Beginner example of using BenchmarkTools.jl

benchmarktools.jl

#From the post: https://randyzwitch.com/benchmarktools-julia-benchmarking/
using Random, OmniSci, BenchmarkTools, Base.Threads

#change defaults, since examples long-running
BenchmarkTools.DEFAULT_PARAMETERS.seconds = 200
BenchmarkTools.DEFAULT_PARAMETERS.samples = 5

#generate test data
gendata(x, T) = [rand(typemin(T):typemax(T)) for y in 1:x]
int64_10x6 = gendata(10^6, Int64)
int64_10x7 = gendata(10^7, Int64)

#Test whether broadcasting more/less efficient than pre-allocating results array
function preallocate(x)

    v = Vector{OmniSci.TStringValue}(undef, length(x))

    for idx in 1:length(x)
        v[idx] = OmniSci.TStringValue(x[idx])
    end

    return v
end

function preallocate_inbounds(x)

    v = Vector{OmniSci.TStringValue}(undef, length(x))

    for idx in 1:length(x)
        @inbounds v[idx] = OmniSci.TStringValue(x[idx])
    end

    return v
end

function preallocate_threads(x)

    v = Vector{OmniSci.TStringValue}(undef, length(x))

    Threads.@threads for idx in 1:length(x)
        v[idx] = OmniSci.TStringValue(x[idx])
    end

    return v
end

function preallocate_threads_inbounds(x)

    v = Vector{OmniSci.TStringValue}(undef, length(x))

    Threads.@threads for idx in 1:length(x)
        @inbounds v[idx] = OmniSci.TStringValue(x[idx])
    end

    return v
end

##RESULTS

julia> @benchmark v61 = OmniSci.TStringValue.($int64_10x6)
BenchmarkTools.Trial: 
  memory estimate:  297.55 MiB
  allocs estimate:  6000002
  --------------
  minimum time:     752.568 ms (0.00% GC)
  median time:      990.071 ms (25.20% GC)
  mean time:        1.204 s (30.86% GC)
  maximum time:     1.976 s (45.76% GC)
  --------------
  samples:          5
  evals/sample:     1

julia> @benchmark v62 = preallocate($int64_10x6)
BenchmarkTools.Trial: 
  memory estimate:  297.55 MiB
  allocs estimate:  6000002
  --------------
  minimum time:     748.719 ms (0.00% GC)
  median time:      988.012 ms (25.31% GC)
  mean time:        1.194 s (31.32% GC)
  maximum time:     1.930 s (47.09% GC)
  --------------
  samples:          5
  evals/sample:     1

julia> @benchmark v63 = preallocate_inbounds($int64_10x6)
BenchmarkTools.Trial: 
  memory estimate:  297.55 MiB
  allocs estimate:  6000002
  --------------
  minimum time:     738.013 ms (0.00% GC)
  median time:      967.184 ms (28.03% GC)
  mean time:        1.215 s (31.80% GC)
  maximum time:     2.088 s (45.90% GC)
  --------------
  samples:          5
  evals/sample:     1

julia> @benchmark v64 = preallocate_threads($int64_10x6)
BenchmarkTools.Trial: 
  memory estimate:  297.32 MiB
  allocs estimate:  5995489
  --------------
  minimum time:     249.117 ms (0.00% GC)
  median time:      253.161 ms (0.00% GC)
  mean time:        252.061 ms (0.00% GC)
  maximum time:     254.137 ms (0.00% GC)
  --------------
  samples:          5
  evals/sample:     1

julia> @benchmark v65 = preallocate_threads_inbounds($int64_10x6)
BenchmarkTools.Trial: 
  memory estimate:  297.34 MiB
  allocs estimate:  5995887
  --------------
  minimum time:     241.585 ms (0.00% GC)
  median time:      246.792 ms (0.00% GC)
  mean time:        246.043 ms (0.00% GC)
  maximum time:     249.109 ms (0.00% GC)
  --------------
  samples:          5
  evals/sample:     1

julia> @benchmark v71 = OmniSci.TStringValue.($int64_10x7)
BenchmarkTools.Trial: 
  memory estimate:  2.77 GiB
  allocs estimate:  57361655
  --------------
  minimum time:     26.316 s (70.63% GC)
  median time:      39.332 s (76.28% GC)
  mean time:        39.332 s (77.44% GC)
  maximum time:     48.395 s (79.44% GC)
  --------------
  samples:          5
  evals/sample:     1

julia> @benchmark v72 = preallocate($int64_10x7)
BenchmarkTools.Trial: 
  memory estimate:  2.77 GiB
  allocs estimate:  57294866
  --------------
  minimum time:     27.064 s (71.96% GC)
  median time:      38.925 s (76.22% GC)
  mean time:        38.962 s (77.70% GC)
  maximum time:     55.167 s (82.53% GC)
  --------------
  samples:          5
  evals/sample:     1

julia> @benchmark v73 = preallocate_inbounds($int64_10x7)
BenchmarkTools.Trial: 
  memory estimate:  2.77 GiB
  allocs estimate:  57304370
  --------------
  minimum time:     26.219 s (70.93% GC)
  median time:      39.387 s (76.64% GC)
  mean time:        38.960 s (77.58% GC)
  maximum time:     54.665 s (82.19% GC)
  --------------
  samples:          5
  evals/sample:     1

julia> @benchmark v74 = preallocate_threads($int64_10x7)
BenchmarkTools.Trial: 
  memory estimate:  2.66 GiB
  allocs estimate:  54989344
  --------------
  minimum time:     2.717 s (0.00% GC)
  median time:      17.659 s (78.92% GC)
  mean time:        15.455 s (75.91% GC)
  maximum time:     26.784 s (82.93% GC)
  --------------
  samples:          5
  evals/sample:     1

julia> @benchmark v75 = preallocate_threads_inbounds($int64_10x7)
BenchmarkTools.Trial: 
  memory estimate:  2.72 GiB
  allocs estimate:  56262010
  --------------
  minimum time:     2.641 s (0.00% GC)
  median time:      16.659 s (76.25% GC)
  mean time:        15.024 s (74.27% GC)
  maximum time:     24.701 s (79.15% GC)
  --------------
  samples:          5
  evals/sample:     1


#make embedded plots
using ECharts

x = ["broadcast", "pre-allocate", "pre-allocate/inbounds", "threads", "threads/inbounds"]
ymin = [752.568, 748.719, 738.013, 249.117, 241.585]
ymed = [990.071, 988.012, 967.184, 253.161, 246.792]

plot_10_6 = bar(x, hcat(ymin, ymed), ec_width = 1000, ec_height = 500)
seriesnames!(plot_10_6, ["Min", "Median"])

ymin2 = [26.316,27.064,26.219,2.717,2.641]
ymed2 = [39.332,38.925,39.387,17.659,16.659]

plot_10_7 = bar(x, hcat(ymin2, ymed2), ec_width = 1000, ec_height = 500)
seriesnames!(plot_10_7, ["Min", "Median"])
colorscheme!(plot_10_7, ("acw", "ZenAndTea"))