Skip to content

Instantly share code, notes, and snippets.

@keynslug

keynslug/bench-report

Last active August 25, 2023 11:58
Show Gist options
  • Save keynslug/f80cf9c3be5e9299d0edca20b60b0b6f to your computer and use it in GitHub Desktop.
Save keynslug/f80cf9c3be5e9299d0edca20b60b0b6f to your computer and use it in GitHub Desktop.
Download ZIP
Perf comparison / PR#11481 @ 2332eb2 (zm) vs PR#11517 @ cf45e80 (head)
Raw
bench-report
> emqx_topicidx_bench:run(zm, head, fixed_15k).
x Base Match / Fixed Topics (15000)
+ Head Match / Fixed Topics (15000)
+--------------------------------------------------------------------------+
|+++++++++ xxxxxxxxxxxx x x x|
|++++++++ xxxxxxxxxxx |
| ++++++ xxxxxxxx |
| ++++++ xxxxxxxx |
| ++++++ xxxxxxxx |
| ++++++ xxxxxxx |
| ++++++ xxxxxx |
| ++++++ xxxxxx |
| ++++++ xxxxxx |
| +++++ xxxxx |
| +++++ xxxx |
| +++++ xxx |
| +++++ xxx |
| +++++ xx |
| ++ + xx |
| ++ xx |
| + xx |
| + xx |
| + |
| + |
| + |
| + |
| + |
| + |
| + |
| + |
| + |
| |____MA______| |
| |_A_| |
+--------------------------------------------------------------------------+
Dataset: x N=100 CI=95.0000
Statistic Value [ Bias] (Bootstrapped LB‥UB)
Min: 878.000
1st Qu. 887.000
Median: 891.000
3rd Qu. 896.000
Max: 1076.00
Average: 894.100 [ 3.21700e-2] ( 891.620 ‥ 901.510)
Std. Dev: 20.3015 [ -1.78707] ( 8.01193 ‥ 44.6308)
Outliers: 0/5 = 5 (μ=894.132, σ=18.5144)
Outlier variance: 0.161241 (moderate)
------
Dataset: + N=100 CI=95.0000
Statistic Value [ Bias] (Bootstrapped LB‥UB)
Min: 846.000
1st Qu. 854.000
Median: 859.000
3rd Qu. 863.000
Max: 872.000
Average: 858.880 [ 2.53900e-3] ( 857.780 ‥ 860.000)
Std. Dev: 5.52913 [ -3.48216e-2] ( 4.99837 ‥ 6.21500)
Outliers: 0/0 = 0 (μ=858.883, σ=5.49431)
Outlier variance: 9.90000e-3 (not affected by outliers)
Difference at 95.0% confidence
-35.2200 ± 4.12403
-3.93916% ± 0.461250%
(Student's t, pooled s = 14.8782)
------
> emqx_topicidx_bench:run(head, zm, fixed_15k).
x Base Match / Fixed Topics (15000)
+ Head Match / Fixed Topics (15000)
+--------------------------------------------------------------------------+
| xxxxxxxxx+++***+++++ + + x|
| xxxxxxxxx++++++++++ + |
| xxxxxxxx ++++++++ + |
| xxxxxx x ++++++ |
| xxxxxx x ++++++ |
| xxxxxx ++++++ |
| xxxxxx ++++++ |
| xxxxx ++++++ |
| xxxxx +++++ |
| xxxxx +++++ |
| xxx x +++++ |
| xxx x +++++ |
| xx x + ++ |
| xx x + ++ |
| xx x + + |
| xx + + |
| xx + + |
| xx + + |
| xx + |
| xx + |
| x + |
||_____MA______| |
| |___A__| |
+--------------------------------------------------------------------------+
Dataset: x N=100 CI=95.0000
Statistic Value [ Bias] (Bootstrapped LB‥UB)
Min: 851.000
1st Qu. 858.000
Median: 862.000
3rd Qu. 867.000
Max: 1073.00
Average: 865.140 [ -7.41500e-3] ( 862.450 ‥ 873.610)
Std. Dev: 22.4918 [ -2.59199] ( 7.47254 ‥ 50.5489)
Outliers: 0/4 = 4 (μ=865.133, σ=19.8998)
Outlier variance: 0.199805 (moderate)
------
Dataset: + N=100 CI=95.0000
Statistic Value [ Bias] (Bootstrapped LB‥UB)
Min: 880.000
1st Qu. 890.000
Median: 895.000
3rd Qu. 900.000
Max: 977.000
Average: 896.060 [ -5.29500e-3] ( 894.430 ‥ 899.110)
Std. Dev: 11.1354 [ -0.440139] ( 7.17597 ‥ 20.8100)
Outliers: 0/3 = 3 (μ=896.055, σ=10.6952)
Outlier variance: 9.90000e-3 (not affected by outliers)
Difference at 95.0% confidence
30.9200 ± 4.91908
3.57399% ± 0.568587%
(Student's t, pooled s = 17.7465)
------
> emqx_topicidx_bench:run(zm, head, fixed).
x Base Match / Fixed Topics
+ Head Match / Fixed Topics
+--------------------------------------------------------------------------+
|++++++*****xxxxxx xxx x|
|+++++++****xxxxxx |
|+++++++**xxxxxxx |
| ++++++**xxxxxx |
| +++++++*xxxxxx |
| +++++++xxxxxxx |
| ++++++xxxxxxx |
| +++++ xxxxxxx |
| +++++ xxxxxx |
| +++++ xxxxx |
| +++++ xxxxx |
| +++++ xxxxx |
| +++++ xxxxx |
| +++++ xxxxx |
| +++++ xxxx |
| +++++ xxxx |
| +++++ xxxx |
| +++++ xxx |
| +++++ xxx |
| +++++ xxx |
| +++++ xxx |
| +++++ xxx |
| +++++ xx |
| +++++ xx |
| +++++ xx |
| +++++ xx |
| ++++ xx |
| ++++ xx |
| ++++ xx |
| ++++ xx |
| + ++ xx |
| + ++ xx |
| + ++ xx |
| + ++ xx |
| + ++ xx |
| + + xx |
| + + xx |
| + + xx |
| + xx |
| + x |
| + x |
| + x |
| + x |
| + x |
| + x |
| + x |
| |___MA___| |
| |_A_| |
+--------------------------------------------------------------------------+
Dataset: x N=200 CI=95.0000
Statistic Value [ Bias] (Bootstrapped LB‥UB)
Min: 382.000
1st Qu. 393.000
Median: 396.000
3rd Qu. 399.000
Max: 602.000
Average: 398.510 [ -8.71250e-3] ( 397.140 ‥ 402.855)
Std. Dev: 15.9799 [ -1.54188] ( 6.45759 ‥ 35.6499)
Outliers: 1/17 = 18 (μ=398.501, σ=14.4380)
Outlier variance: 0.544763 (severe, the data set is probably unusable)
------
Dataset: + N=200 CI=95.0000
Statistic Value [ Bias] (Bootstrapped LB‥UB)
Min: 364.000
1st Qu. 372.000
Median: 377.000
3rd Qu. 381.000
Max: 397.000
Average: 376.985 [ 1.88900e-3] ( 376.165 ‥ 377.850)
Std. Dev: 6.07944 [ -2.48665e-2] ( 5.50395 ‥ 6.83108)
Outliers: 0/2 = 2 (μ=376.987, σ=6.05457)
Outlier variance: 0.161318 (moderate)
Difference at 95.0% confidence
-21.5250 ± 2.36956
-5.40137% ± 0.594605%
(Student's t, pooled s = 12.0896)
------
> emqx_topicidx_bench:run(head, zm, fixed).
x Base Match / Fixed Topics
+ Head Match / Fixed Topics
+--------------------------------------------------------------------------+
| xxx********++***++ xxx + x x|
| xxxxx*****+++++* + |
| xxxxx*****+++ +* + |
| xxxxx*****+++ ++ |
| xxxx*****+++ + |
| xxxx****++++ |
| xxxxx*+*++++ |
| xxxxx*++++++ |
| xxxxx*++++++ |
| xxxxx*++++++ |
| xxxxx*++++++ |
| xxxxx*++++++ |
| xxxxx*++++++ |
| xxxxx+++++++ |
| xxxxx+++++ + |
| xxxxx+++++ |
| xxx +++++ |
| xxx +++++ |
| xxx +++++ |
| xx +++++ |
| xx +++++ |
| xx +++++ |
| xx +++++ |
| xx ++ ++ |
| xx + ++ |
| xx + ++ |
| xx + + |
| xx + + |
| xx + + |
| x + + |
| x + + |
| x + + |
| x + + |
| x + |
| x + |
| x + |
| x + |
| x + |
| x + |
| x + |
| x + |
||____M_A______| |
| |__A__| |
+--------------------------------------------------------------------------+
Dataset: x N=200 CI=95.0000
Statistic Value [ Bias] (Bootstrapped LB‥UB)
Min: 374.000
1st Qu. 381.000
Median: 383.000
3rd Qu. 386.000
Max: 541.000
Average: 386.325 [ -4.47050e-3] ( 384.735 ‥ 389.770)
Std. Dev: 16.0774 [ -0.739208] ( 7.75892 ‥ 28.1949)
Outliers: 0/21 = 21 (μ=386.321, σ=15.3382)
Outlier variance: 0.560086 (severe, the data set is probably unusable)
------
Dataset: + N=200 CI=95.0000
Statistic Value [ Bias] (Bootstrapped LB‥UB)
Min: 383.000
1st Qu. 391.000
Median: 395.000
3rd Qu. 399.000
Max: 434.000
Average: 395.925 [ -1.61450e-3] ( 395.090 ‥ 396.900)
Std. Dev: 6.54013 [ -4.51889e-2] ( 5.69919 ‥ 8.55076)
Outliers: 0/5 = 5 (μ=395.923, σ=6.49494)
Outlier variance: 0.166232 (moderate)
Difference at 95.0% confidence
9.60000 ± 2.40552
2.48495% ± 0.622666%
(Student's t, pooled s = 12.2730)
------
> emqx_topicidx_bench:run(zm, head, fixed_5k_rules).
x Base Match / Fixed Rules (5000)
+ Head Match / Fixed Rules (5000)
+--------------------------------------------------------------------------+
|+ ++++++*x*xxx*xx x xx x x|
| ++++++*xxxxxxxx x |
| ++++++xxxxxxxx |
| ++++++xxxxxxxx |
| ++++++ xxxxxxx |
| ++++++ xxxxxxx |
| ++++++ xxxxxx |
| +++++ xxxxxx |
| +++++ xxxx x |
| ++++ xxx |
| ++++ xxx |
| ++++ xxx |
| ++++ xxx |
| ++++ x |
| +++ x |
| +++ x |
| ++ x |
| ++ x |
| ++ x |
| ++ x |
| ++ x |
| ++ x |
| ++ x |
| + x |
| + x |
| + x |
| |____M_A_____| |
| |_A_| |
+--------------------------------------------------------------------------+
Dataset: x N=100 CI=95.0000
Statistic Value [ Bias] (Bootstrapped LB‥UB)
Min: 677.000
1st Qu. 683.000
Median: 686.000
3rd Qu. 693.000
Max: 876.000
Average: 690.450 [ -1.65590e-2] ( 687.960 ‥ 697.960)
Std. Dev: 20.6331 [ -1.99801] ( 8.02926 ‥ 45.3997)
Outliers: 0/6 = 6 (μ=690.433, σ=18.6351)
Outlier variance: 0.248172 (moderate)
------
Dataset: + N=100 CI=95.0000
Statistic Value [ Bias] (Bootstrapped LB‥UB)
Min: 652.000
1st Qu. 661.000
Median: 664.000
3rd Qu. 668.000
Max: 695.000
Average: 665.310 [ -2.94300e-3] ( 664.290 ‥ 666.620)
Std. Dev: 5.94197 [ -8.04227e-2] ( 4.83063 ‥ 8.44277)
Outliers: 0/2 = 2 (μ=665.307, σ=5.86154)
Outlier variance: 9.90000e-3 (not affected by outliers)
Difference at 95.0% confidence
-25.1400 ± 4.20844
-3.64110% ± 0.609522%
(Student's t, pooled s = 15.1827)
------
> emqx_topicidx_bench:run(head, zm, fixed_5k_rules).
x Base Match / Fixed Rules (5000)
+ Head Match / Fixed Rules (5000)
+--------------------------------------------------------------------------+
| xxxxxx****++**+ +x x|
| xxxxx****+++*+ |
| xxxxx****+++++ |
| xxxxx*+**++++ |
| xxxxxx *+++++ |
| xxxxx *+++++ |
| xxxxx *+++++ |
| xxxx *+++++ |
| xxxx *++++ |
| xxxx +++++ |
| xxxx +++++ |
| xxxx ++++ |
| xxxx ++++ |
| xxx ++++ |
| xxx ++ + |
| xx ++ + |
| x + |
| x + |
| x + |
| x + |
| x + |
| x + |
| + |
||_____MA_______| |
| |_A_| |
+--------------------------------------------------------------------------+
Dataset: x N=100 CI=95.0000
Statistic Value [ Bias] (Bootstrapped LB‥UB)
Min: 658.000
1st Qu. 665.000
Median: 669.000
3rd Qu. 675.000
Max: 862.000
Average: 672.530 [ 1.10950e-2] ( 669.960 ‥ 680.730)
Std. Dev: 20.9672 [ -2.02549] ( 7.86997 ‥ 46.5125)
Outliers: 0/5 = 5 (μ=672.541, σ=18.9418)
Outlier variance: 0.267397 (moderate)
------
Dataset: + N=100 CI=95.0000
Statistic Value [ Bias] (Bootstrapped LB‥UB)
Min: 674.000
1st Qu. 682.000
Median: 685.000
3rd Qu. 690.000
Max: 710.000
Average: 686.020 [ 4.35100e-3] ( 684.850 ‥ 687.240)
Std. Dev: 6.12658 [ -4.59945e-2] ( 5.37554 ‥ 7.63143)
Outliers: 0/1 = 1 (μ=686.024, σ=6.08058)
Outlier variance: 9.90000e-3 (not affected by outliers)
Difference at 95.0% confidence
13.4900 ± 4.28142
2.00586% ± 0.636615%
(Student's t, pooled s = 15.4460)
------
Raw
emqx_topicidx_bench.erl
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_topicidx_bench).
-compile([export_all, nowarn_export_all]).
-include_lib("stdlib/include/assert.hrl").
% -include_lib("proper/include/proper.hrl").
-define(FILTERS, [
<<"t/#">>,
<<"t/+/#">>,
<<"+/+/+/sub">>,
<<"dev/global/sensor">>,
<<"dev/+/sensor/#">>
]).
-define(FILTERGEN, [
{1, <<"t/+/+">>},
{1, <<"t/+/+/+">>},
{1000, [<<"dev">>, '+', fun() -> io_lib:format("~B", [next(dev)]) end, '#']},
{500, [<<"t">>, fun() -> io_lib:format("~3.10.0B", [next(fwd1)]) end, '#']},
{500, [<<"t">>, fun() -> io_lib:format("~3.10.0B", [next(fwd2)]) end, '+']},
{500, [<<"t">>, <<"fwd">>, fun() -> io_lib:format("~3.10.0B", [next(fwd1)]) end]},
{500, [<<"t">>, <<"fwd">>, fun() -> io_lib:format("~3.10.0B", [next(fwd2)]) end, '#']},
{100, [<<"$sys">>, '+', fun() -> io_lib:format("node~3.10.0B", [next(sys)]) end, '+']},
{500, ['+', <<"bridged">>, fun() -> io_lib:format("remote~3.10.0B", [next(plus1)]) end, '+']},
{500, [
'+',
<<"bridged">>,
fun() -> io_lib:format("remote~3.10.0B", [next(plus2)]) end,
<<"sub">>,
'+'
]},
{1000, [
<<"msg">>, <<"bridged">>, fun() -> io_lib:format("remote~3.10.0B", [next(msg)]) end, '#'
]}
]).
-define(NODES, [
node1,
node2,
node3
]).
-define(TOPICS, [
<<"dev/global/sensor/042">>,
<<"dev/global/sensor">>,
<<"dev/global">>,
<<"dev/local/temp">>,
<<"dev/local/humidity">>,
<<"dev/local">>,
<<"dev/local/0">>,
<<"dev/local/1">>,
<<"dev/local/2">>,
<<"dev/local/3">>,
<<"dev/local/4">>,
<<"dev/local/5">>,
<<"dev/local/6">>,
<<"dev/local/7">>,
<<"dev/local/8">>,
<<"dev/local/9">>,
<<"dev/local/sensor/temp/*">>,
<<"dev/local/sensor/temp/CPU/0">>,
<<"dev/local/sensor/temp/CPU/1">>,
<<"dev/local/sensor/temp/CPU/2">>,
<<"dev/local/sensor/temp/CPU/3">>,
<<"dev/local/sensor/temp/CPU/4">>,
<<"dev/local/sensor/temp/CPU/5">>,
<<"dev/local/sensor/temp/CPU/6">>,
<<"dev/local/sensor/temp/CPU/7">>,
<<"dev/local/sensor/temp/CPU/8">>,
<<"dev/local/sensor/temp/CPU/9">>,
<<"t/fwd/001">>,
<<"t/fwd/002">>,
<<"t/fwd/003">>,
<<"t/fwd/004">>,
<<"t/fwd/005">>,
<<"t/fwd/006">>,
<<"t/fwd/007">>,
<<"t/fwd/008">>,
<<"t/fwd/009">>,
<<"t/fwd/010">>,
<<"t/fwd/011">>,
<<"t/fwd/012">>,
<<"t/fwd/013">>,
<<"t/fwd/014">>,
<<"t/fwd/015">>,
<<"t/fwd/016">>,
<<"t/fwd/017">>,
<<"t/fwd/018">>,
<<"t/fwd/019">>,
<<"t/fwd/020">>,
<<"t/001">>,
<<"t/002">>,
<<"t/003">>,
<<"t/004">>,
<<"t/005">>,
<<"t/006">>,
<<"t/007">>,
<<"t/008">>,
<<"t/009">>,
<<"t/010">>,
<<"t/011">>,
<<"t/012">>,
<<"t/013">>,
<<"t/014">>,
<<"t/015">>,
<<"t/016">>,
<<"t/017">>,
<<"t/018">>,
<<"t/019">>,
<<"t/020">>,
<<"t/bridged/remote001/sub">>,
<<"t/bridged/remote002/sub">>,
<<"t/bridged/remote003/sub">>,
<<"t/bridged/remote004/sub">>,
<<"t/bridged/remote005/sub">>,
<<"t/bridged/remote006/sub">>,
<<"t/bridged/remote007/sub">>,
<<"t/bridged/remote008/sub">>,
<<"t/bridged/remote009/sub">>,
<<"t/bridged/remote010/sub">>,
<<"t/bridged/remote011/sub">>,
<<"t/bridged/remote012/sub">>,
<<"t/bridged/remote013/sub">>,
<<"t/bridged/remote014/sub">>,
<<"t/bridged/remote015/sub">>,
<<"t/bridged/remote016/sub">>,
<<"t/bridged/remote017/sub">>,
<<"t/bridged/remote018/sub">>,
<<"t/bridged/remote019/sub">>,
<<"t/bridged/remote020/sub">>,
<<"msg/bridged/remote001/sub">>,
<<"msg/bridged/remote002/sub">>,
<<"msg/bridged/remote003/sub">>,
<<"msg/bridged/remote004/sub">>,
<<"msg/bridged/remote005/sub">>,
<<"msg/bridged/remote006/sub">>,
<<"msg/bridged/remote007/sub">>,
<<"msg/bridged/remote008/sub">>,
<<"msg/bridged/remote009/sub">>,
<<"msg/bridged/remote010/sub">>,
<<"msg/bridged/remote011/sub">>,
<<"msg/bridged/remote012/sub">>,
<<"msg/bridged/remote013/sub">>,
<<"msg/bridged/remote014/sub">>,
<<"msg/bridged/remote015/sub">>,
<<"msg/bridged/remote016/sub">>,
<<"msg/bridged/remote017/sub">>,
<<"msg/bridged/remote018/sub">>,
<<"msg/bridged/remote019/sub">>,
<<"msg/bridged/remote020/sub">>,
<<"msg/bridged/remote001/sub/01">>,
<<"msg/bridged/remote002/sub/02">>,
<<"msg/bridged/remote003/sub/03">>,
<<"msg/bridged/remote004/sub/04">>,
<<"msg/bridged/remote005/sub/05">>,
<<"msg/bridged/remote006/sub/06">>,
<<"msg/bridged/remote007/sub/07">>,
<<"msg/bridged/remote008/sub/08">>,
<<"msg/bridged/remote009/sub/09">>,
<<"msg/bridged/remote010/sub/10">>,
<<"msg/bridged/remote011/sub/10">>,
<<"msg/bridged/remote012/sub/10">>,
<<"msg/bridged/remote013/sub/10">>,
<<"msg/bridged/remote014/sub/10">>,
<<"msg/bridged/remote015/sub/10">>,
<<"msg/bridged/remote016/sub/10">>,
<<"msg/bridged/remote017/sub/10">>,
<<"msg/bridged/remote018/sub/10">>,
<<"msg/bridged/remote019/sub/10">>,
<<"msg/bridged/remote020/sub/10">>,
<<"$sys/broker/node001/sub">>,
<<"$sys/broker/node002/sub">>,
<<"$sys/broker/node003/sub">>,
<<"$sys/broker/node004/sub">>,
<<"$sys/broker/node005/sub">>,
<<"$sys/broker/node006/sub">>,
<<"$sys/broker/node007/sub">>,
<<"$sys/broker/node008/sub">>,
<<"$sys/broker/node009/sub">>,
<<"$sys/broker/node010/sub">>,
<<"$sys/broker/node011/sub">>,
<<"$sys/broker/node012/sub">>,
<<"$sys/broker/node013/sub">>,
<<"$sys/broker/node014/sub">>,
<<"$sys/broker/node015/sub">>,
<<"$sys/broker/node016/sub">>,
<<"$sys/broker/node017/sub">>,
<<"$sys/broker/node018/sub">>,
<<"$sys/broker/node019/sub">>,
<<"$sys/broker/node020/sub">>
]).
prepare(ImplBase, ImplHead, Filters, Topics) ->
TabOrig = init(ImplBase, Filters),
TabHead = init(ImplHead, Filters),
_ = ?assertEqual(
[[topic(ImplBase, M) || M <- lists:sort(match(ImplBase, T, TabOrig))] || T <- Topics],
[[topic(ImplHead, M) || M <- lists:sort(match(ImplHead, T, TabHead))] || T <- Topics]
),
{TabOrig, TabHead}.
run(X) ->
run(orig, head, X).
run(ImplBase, ImplHead, Alias) when is_atom(Alias) ->
run(ImplBase, ImplHead, workload(Alias));
run(ImplBase, ImplHead, Workload = #{name := Name, entries := Entries, matches := Matches}) ->
Samples = maps:get(samples, Workload, 200),
run(Name, ImplBase, ImplHead, Entries, Matches, Samples).
workload(fixed) ->
#{
name => "Fixed Topics",
entries => cartesian(?FILTERS, ?NODES),
matches => ?TOPICS
};
workload(fixed_rules) ->
#{
name => "Fixed Rules",
entries => numbered(?FILTERS),
matches => ?TOPICS
};
workload(fixed_15k) ->
#{
name => "Fixed Topics (15000)",
entries => cartesian(gen(?FILTERGEN), ?NODES),
matches => ?TOPICS,
samples => 100
};
workload(fixed_5k_rules) ->
#{
name => "Fixed Rules (5000)",
entries => numbered(gen(?FILTERGEN)),
matches => ?TOPICS,
samples => 100
};
workload(topics_only) ->
#{
name => "Topics = Filters",
entries => cartesian(?TOPICS, ?NODES),
matches => ?TOPICS
};
workload(topics_rules) ->
#{
name => "Topics = Rules",
entries => numbered(?TOPICS),
matches => ?TOPICS
};
workload(topics_with_filters) ->
#{
name => "Topics = Filters",
entries => cartesian(?FILTERS ++ ?TOPICS, ?NODES),
matches => ?TOPICS
}.
run(Name, ImplBase, ImplHead, Entries, Topics, NSamples) ->
{TabOrig, TabHead} = prepare(ImplBase, ImplHead, Entries, Topics),
io:format("~p: ~p~n", [ImplBase, ets:info(TabOrig)]),
io:format("~p: ~p~n", [ImplHead, ets:info(TabHead)]),
ROrig = eministat:s(
"Base Match / " ++ Name,
fun() -> sample(ImplBase, Topics, TabOrig) end,
NSamples
),
RHead = eministat:s(
"Head Match / " ++ Name,
fun() -> sample(ImplHead, Topics, TabHead) end,
NSamples
),
eministat:x(95.0, ROrig, RHead).
init(head, Entries) ->
Tab = emqx_topic_index:new(),
_ = [emqx_topic_index:insert(F, ID, [], Tab) || {F, ID} <- Entries],
Tab;
init(zm, Entries) ->
Tab = emqx_topic_index:new(),
_ = [emqx_topic_index:insert(F, ID, [], Tab) || {F, ID} <- Entries],
Tab;
init(orig, Entries) ->
Tab = emqx_topic_index_r52:new(),
_ = [emqx_topic_index_r52:insert(F, ID, [], Tab) || {F, ID} <- Entries],
Tab.
match(orig, Topic, Tab) ->
match_orig(Topic, Tab);
match(head, Topic, Tab) ->
match_head(Topic, Tab);
match(zm, Topic, Tab) ->
match_zm(Topic, Tab).
-compile({inline, match_orig/2}).
match_orig(Topic, Tab) ->
emqx_topic_index_r52:matches(Topic, Tab, []).
-compile({inline, match_head/2}).
match_head(Topic, Tab) ->
emqx_topic_index:matches(Topic, Tab, []).
-compile({inline, match_zm/2}).
match_zm(Topic, Tab) ->
emqx_topic_index:matches_zm(Topic, Tab, []).
sample(orig, Topics, Tab) ->
lists:foreach(fun(T) -> match_orig(T, Tab) end, Topics);
sample(head, Topics, Tab) ->
lists:foreach(fun(T) -> match_head(T, Tab) end, Topics);
sample(zm, Topics, Tab) ->
lists:foreach(fun(T) -> match_zm(T, Tab) end, Topics).
topic(orig, Match) ->
emqx_topic_index_r52:get_topic(Match);
topic(head, Match) ->
emqx_topic_index:get_topic(Match);
topic(zm, Match) ->
emqx_topic_index:get_topic(Match).
filters() ->
?FILTERS.
topics() ->
?TOPICS.
cartesian(A, B) ->
[{X, Y} || X <- A, Y <- B].
numbered(A) ->
lists:zip(A, lists:seq(1, length(A))).
%%
gen(Gens) when is_list(Gens) ->
Tab = ets:new(?MODULE, [set]),
_ = put({?MODULE, gentable}, Tab),
Result = lists:flatmap(fun({N, Gen}) -> gen(N, Gen) end, Gens),
_ = ets:delete(Tab),
Result.
gen(N, Generator) when is_integer(N) ->
[emit(Generator) || _ <- lists:seq(1, N)].
emit(Topic) when is_binary(Topic) ->
Topic;
emit(Topic) when is_list(Topic) ->
emqx_topic:join(
lists:map(
fun
(F) when is_function(F) -> F();
(W) when is_binary(W) -> W;
(W) when is_atom(W) -> W
end,
Topic
)
).
next(CounterName) ->
ets:update_counter(get({?MODULE, gentable}), CounterName, {2, 1}, {CounterName, 0}).
Raw
meter_memory_pressure.erl
%%--------------------------------------------------------------------
%% Copyright (c) 2019-2021 RBK.money, Ltd.
%% Copyright (c) 2022 Vality.dev.
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(meter_memory_pressure).
-compile(export_all).
-compile(nowarn_export_all).
-type microseconds() :: non_neg_integer().
-type words() :: non_neg_integer().
-type metrics() :: #{
minor_gcs := non_neg_integer(),
minor_gcs_duration := microseconds(),
major_gcs := non_neg_integer(),
minor_gcs_duration := microseconds(),
heap_reclaimed := words(),
offheap_bin_reclaimed := words(),
stack_min := words(),
stack_max := words()
}.
-export([measure/2]).
-export([export/3]).
-export_type([metrics/0]).
%%
-type runner() :: fun(() -> _) | {module(), atom(), [term()]}.
-type opts() :: #{
iterations => pos_integer(),
spawn_opts => [{atom(), _}],
dump_traces => file:filename()
}.
-export_type([runner/0]).
-export_type([opts/0]).
-spec measure(runner(), opts()) -> metrics().
measure(Runner, Opts0) ->
Opts = maps:merge(get_default_opts(), Opts0),
Token = make_ref(),
Tracer = start_tracer(Token, Opts),
ok = run(Runner, Tracer, Opts),
Metrics = collect_metrics(Tracer, Token),
Metrics.
get_default_opts() ->
#{
iterations => 100,
spawn_opts => [{fullsweep_after, 0}]
}.
run(Runner, Tracer, Opts) ->
SpawnOpts = [monitor, {priority, high}] ++ maps:get(spawn_opts, Opts),
{Staging, MRef} = erlang:spawn_opt(
fun() -> run_staging(Runner, Tracer, Opts) end,
SpawnOpts
),
receive
{'DOWN', MRef, process, Staging, normal} ->
ok
end.
run_staging(Runner, Tracer, Opts) ->
N = maps:get(iterations, Opts),
TraceOpts = [garbage_collection, timestamp, {tracer, Tracer}],
_ = erlang:trace(self(), true, TraceOpts),
iterate(Runner, N).
iterate(Runner, N) when N > 0 ->
_ = execute(Runner),
iterate(Runner, N - 1);
iterate(_Runner, 0) ->
ok.
execute({Module, Function, Args}) ->
apply(Module, Function, Args);
execute(Runner) when is_function(Runner, 0) ->
Runner().
%%
start_tracer(Token, Opts) ->
Self = self(),
erlang:spawn_link(fun() -> run_tracer(Self, Token, Opts) end).
collect_metrics(Tracer, Token) ->
_ = Tracer ! Token,
receive
{?MODULE, {metrics, Metrics}} ->
Metrics
end.
run_tracer(MeterPid, Token, Opts) ->
_ =
receive
Token -> ok
end,
Traces = collect_traces(),
Metrics = analyze_traces(Traces),
ok = maybe_dump_traces(Traces, Opts),
MeterPid ! {?MODULE, {metrics, Metrics}}.
collect_traces() ->
collect_traces([]).
collect_traces(Acc) ->
receive
{trace_ts, _Pid, Trace, Info, Clock} ->
collect_traces([{Trace, Info, Clock} | Acc]);
Unexpected ->
error({unexpected, Unexpected})
after 0 -> lists:reverse(Acc)
end.
maybe_dump_traces(Traces, #{dump_traces := Filename}) ->
file:write_file(Filename, erlang:term_to_binary(Traces));
maybe_dump_traces(_, #{}) ->
ok.
analyze_traces(Traces) ->
analyze_traces(Traces, #{
minor_gcs => 0,
minor_gcs_duration => 0,
major_gcs => 0,
major_gcs_duration => 0,
heap_reclaimed => 0,
offheap_bin_reclaimed => 0
}).
analyze_traces([{gc_minor_start, InfoStart, C1}, {gc_minor_end, InfoEnd, C2} | Rest], M0) ->
M1 = increment(minor_gcs, M0),
M2 = increment(minor_gcs_duration, timer:now_diff(C2, C1), M1),
analyze_traces(Rest, analyze_gc(InfoStart, InfoEnd, M2));
analyze_traces([{gc_major_start, InfoStart, C1}, {gc_major_end, InfoEnd, C2} | Rest], M0) ->
M1 = increment(major_gcs, M0),
M2 = increment(major_gcs_duration, timer:now_diff(C2, C1), M1),
analyze_traces(Rest, analyze_gc(InfoStart, InfoEnd, M2));
analyze_traces([], M) ->
M.
analyze_gc(InfoStart, InfoEnd, M0) ->
M1 = increment(heap_reclaimed, difference(heap_size, InfoEnd, InfoStart), M0),
M2 = increment(offheap_bin_reclaimed, difference(bin_vheap_size, InfoEnd, InfoStart), M1),
M3 = update(stack_min, fun erlang:min/2, min(stack_size, InfoStart, InfoEnd), M2),
M4 = update(stack_max, fun erlang:max/2, max(stack_size, InfoStart, InfoEnd), M3),
M4.
difference(Name, Info1, Info2) ->
combine(Name, fun(V1, V2) -> erlang:max(0, V2 - V1) end, Info1, Info2).
min(Name, Info1, Info2) ->
combine(Name, fun erlang:min/2, Info1, Info2).
max(Name, Info1, Info2) ->
combine(Name, fun erlang:max/2, Info1, Info2).
combine(Name, Fun, Info1, Info2) ->
{_, V1} = lists:keyfind(Name, 1, Info1),
{_, V2} = lists:keyfind(Name, 1, Info2),
Fun(V1, V2).
increment(Name, Metrics) ->
increment(Name, 1, Metrics).
increment(Name, Delta, Metrics) ->
maps:update_with(Name, fun(V) -> V + Delta end, Metrics).
update(Name, Fun, I, Metrics) ->
maps:update_with(Name, fun(V) -> Fun(V, I) end, I, Metrics).
%%
-spec export(file:filename(), file:filename(), csv) -> ok.
export(FilenameIn, FilenameOut, Format) ->
{ok, Content} = file:read_file(FilenameIn),
Traces = erlang:binary_to_term(Content),
{ok, FileOut} = file:open(FilenameOut, [write, binary]),
ok = format_traces(Traces, Format, FileOut),
ok = file:close(FileOut).
format_traces(Traces, csv, FileOut) ->
_ = format_csv_header(FileOut),
_ = lists:foreach(fun(T) -> format_csv_trace(T, FileOut) end, Traces),
ok.
format_csv_header(Out) ->
Line = " ~s , ~s , ~s , ~s , ~s , ~s , ~s , ~s , ~s , ~s , ~s , ~s ~n",
io:fwrite(Out, Line, [
"Time",
"End?",
"Major?",
"Stack",
"Heap",
"HeapBlock",
"BinHeap",
"BinHeapBlock",
"OldHeap",
"OldHeapBlock",
"OldBinHeap",
"OldBinHeapBlock"
]).
format_csv_trace({Event, Info, Clock}, Out) ->
Line = " ~B , ~B , ~B , ~B , ~B , ~B , ~B , ~B , ~B , ~B , ~B , ~B ~n",
io:fwrite(Out, Line, [
clock_to_mcs(Clock),
bool_to_integer(lists:member(Event, [gc_minor_end, gc_major_end])),
bool_to_integer(lists:member(Event, [gc_major_start, gc_major_end])),
get_info(stack_size, Info),
get_info(heap_size, Info),
get_info(heap_block_size, Info),
get_info(bin_vheap_size, Info),
get_info(bin_vheap_block_size, Info),
get_info(old_heap_size, Info),
get_info(old_heap_block_size, Info),
get_info(bin_old_vheap_size, Info),
get_info(bin_old_vheap_block_size, Info)
]).
get_info(Name, Info) ->
{_Name, V} = lists:keyfind(Name, 1, Info),
V.
clock_to_mcs({MSec, Sec, USec}) ->
(MSec * 1000000 + Sec) * 1000000 + USec.
bool_to_integer(false) ->
0;
bool_to_integer(true) ->
1.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment