Skip to content

Instantly share code, notes, and snippets.

@cyb70289

cyb70289/gist:98fab345783f23dde9ddce000aaa0a12

Created June 17, 2024 07:04
Show Gist options
  • Save cyb70289/98fab345783f23dde9ddce000aaa0a12 to your computer and use it in GitHub Desktop.
Save cyb70289/98fab345783f23dde9ddce000aaa0a12 to your computer and use it in GitHub Desktop.
Download ZIP
bluewhale latency vs. bandwidth
Raw
gistfile1.txt
Bluewhale memory latency vs. bandwidth
======================================
max bw = 574*8*32 MB/s = 147GB/s
bw = 0%
-------
$ numactl -m0 -N0 /usr/lib/lmbench/bin/lat_mem_rd -P 1 512 4096
"stride=4096
0.00391 1.429
0.00586 1.429
0.00781 1.429
0.01172 1.429
0.01562 1.429
0.02344 1.429
0.03125 1.429
0.04688 1.429
0.06250 1.429
0.09375 5.122
0.12500 5.070
0.18750 5.032
0.25000 5.014
0.37500 5.010
0.50000 5.003
0.75000 5.003
1.00000 5.009
1.50000 5.005
2.00000 5.005
3.00000 5.003
4.00000 5.003
6.00000 5.003
8.00000 5.003
12.00000 5.003
16.00000 5.326
24.00000 7.118
32.00000 12.170
48.00000 20.202
64.00000 22.360
96.00000 49.745
128.00000 82.307
192.00000 106.349
256.00000 117.425
384.00000 118.940
512.00000 119.097
bw = 17%
--------
$ numactl -m0 -N0 /usr/lib/lmbench/bin/bw_mem -P 1 -N2000 1G rd
$ numactl -m0 -N0 /usr/lib/lmbench/bin/lat_mem_rd -P 1 512 4096
bw = 96M*8*32/s
"stride=4096
0.00391 1.430
0.00586 1.430
0.00781 1.430
0.01172 1.430
0.01562 1.430
0.02344 1.430
0.03125 1.430
0.04688 1.430
0.06250 1.430
0.09375 5.123
0.12500 5.048
0.18750 5.003
0.25000 5.003
0.37500 5.003
0.50000 5.003
0.75000 5.007
1.00000 5.003
1.50000 5.004
2.00000 5.004
3.00000 5.004
4.00000 5.003
6.00000 5.004
8.00000 5.004
12.00000 5.004
16.00000 5.070
24.00000 6.046
32.00000 13.386
48.00000 19.115
64.00000 23.554
96.00000 91.274
128.00000 108.651
192.00000 119.975
256.00000 115.218
384.00000 121.088
512.00000 120.498
bw = 33%
--------
numactl -m0 -N0 /usr/lib/lmbench/bin/bw_mem -P 2 -N 200 1G rd
numactl -m0 -N0 /usr/lib/lmbench/bin/lat_mem_rd -P 1 512 4096
bw = 191M*8*32 ~ 196k
"stride=4096
0.00391 1.430
0.00586 1.430
0.00781 1.430
0.01172 1.430
0.01562 1.430
0.02344 1.430
0.03125 1.430
0.04688 1.430
0.06250 1.430
0.09375 5.004
0.12500 5.048
0.18750 5.815
0.25000 5.763
0.37500 5.637
0.50000 5.464
0.75000 5.289
1.00000 5.235
1.50000 5.160
2.00000 5.119
3.00000 5.083
4.00000 5.061
6.00000 5.044
8.00000 5.031
12.00000 5.038
16.00000 5.053
24.00000 6.083
32.00000 12.542
48.00000 19.201
64.00000 21.667
96.00000 93.573
128.00000 111.295
192.00000 119.411
256.00000 124.701
384.00000 123.319
512.00000 123.028
bw = 48%
--------
numactl -m0 -N0 /usr/lib/lmbench/bin/bw_mem -P 3 -N 100 1G rd
numactl -m0 -N0 /usr/lib/lmbench/bin/lat_mem_rd -P 1 512 4096
bw = 279M*8*32
"stride=4096
0.00391 1.430
0.00586 1.430
0.00781 1.430
0.01172 1.430
0.01562 1.430
0.02344 1.430
0.03125 1.430
0.04688 1.430
0.06250 1.430
0.09375 5.122
0.12500 5.070
0.18750 5.887
0.25000 6.544
0.37500 6.090
0.50000 5.869
0.75000 6.182
1.00000 5.936
1.50000 5.996
2.00000 5.833
3.00000 5.480
4.00000 5.359
6.00000 5.225
8.00000 5.207
12.00000 5.120
16.00000 5.203
24.00000 6.215
32.00000 11.580
48.00000 19.028
64.00000 21.616
96.00000 98.941
128.00000 102.285
192.00000 119.673
256.00000 122.957
384.00000 126.793
512.00000 132.268
bw = 62%
--------
numactl -m0 -N0 /usr/lib/lmbench/bin/bw_mem -P 4 -N 100 1G rd
numactl -m0 -N0 /usr/lib/lmbench/bin/lat_mem_rd -P 1 512 4096
bw = 354M*8*32 ~ 358
"stride=4096
0.00391 1.430
0.00586 1.430
0.00781 1.430
0.01172 1.430
0.01562 1.430
0.02344 1.430
0.03125 1.430
0.04688 1.430
0.06250 1.430
0.09375 5.122
0.12500 5.026
0.18750 5.942
0.25000 5.919
0.37500 6.236
0.50000 6.472
0.75000 6.345
1.00000 6.313
1.50000 5.515
2.00000 5.830
3.00000 5.487
4.00000 5.738
6.00000 5.250
8.00000 5.368
12.00000 5.145
16.00000 5.214
24.00000 7.032
32.00000 11.207
48.00000 20.131
64.00000 23.182
96.00000 60.542
128.00000 108.371
192.00000 137.993
256.00000 138.172
384.00000 134.242
512.00000 135.022
bw = 75%
--------
numactl -m0 -N0 /usr/lib/lmbench/bin/bw_mem -P 5 -N 100 1G rd
numactl -m0 -N0 /usr/lib/lmbench/bin/lat_mem_rd -P 1 512 4096
bw = 342M*8*32 ~ 358
"stride=4096
0.00391 1.430
0.00586 1.430
0.00781 1.430
0.01172 1.430
0.01562 1.430
0.02344 1.430
0.03125 1.430
0.04688 1.430
0.06250 1.430
0.09375 5.003
0.12500 5.026
0.18750 5.893
0.25000 6.394
0.37500 6.157
0.50000 5.894
0.75000 6.120
1.00000 6.472
1.50000 5.929
2.00000 5.344
3.00000 5.351
4.00000 5.184
6.00000 5.159
8.00000 5.092
12.00000 5.089
16.00000 5.088
24.00000 6.718
32.00000 11.889
48.00000 19.106
64.00000 21.939
96.00000 101.262
128.00000 94.486
192.00000 144.925
256.00000 138.883
384.00000 138.295
512.00000 149.376
bw = 85%
--------
numactl -m0 -N0 /usr/lib/lmbench/bin/bw_mem -P 6 -N 100 1G rd
numactl -m0 -N0 /usr/lib/lmbench/bin/lat_mem_rd -P 1 512 4096
bw = 489M*8*32 ~ 493
"stride=4096
0.00391 1.430
0.00586 1.430
0.00781 1.430
0.01172 1.430
0.01562 1.430
0.02344 1.430
0.03125 1.430
0.04688 1.430
0.06250 1.430
0.09375 5.003
0.12500 5.048
0.18750 6.432
0.25000 6.108
0.37500 6.328
0.50000 6.109
0.75000 6.136
1.00000 6.067
1.50000 5.645
2.00000 5.557
3.00000 5.325
4.00000 5.314
6.00000 5.165
8.00000 5.167
12.00000 5.094
16.00000 5.204
24.00000 6.622
32.00000 11.815
48.00000 20.377
64.00000 23.645
96.00000 122.715
128.00000 103.656
192.00000 147.377
256.00000 151.897
384.00000 154.773
512.00000 157.705
bw = 92%
--------
numactl -m0 -N0 /usr/lib/lmbench/bin/bw_mem -P 7 -N 100 1G rd
numactl -m0 -N0 /usr/lib/lmbench/bin/lat_mem_rd -P 1 512 4096
bw = 532M*8*32
"stride=4096
0.00391 1.430
0.00586 1.430
0.00781 1.430
0.01172 1.430
0.01562 1.430
0.02344 1.430
0.03125 1.430
0.04688 1.430
0.06250 1.430
0.09375 5.033
0.12500 5.003
0.18750 5.870
0.25000 6.187
0.37500 6.268
0.50000 6.279
0.75000 5.405
1.00000 6.198
1.50000 5.843
2.00000 6.477
3.00000 5.517
4.00000 5.714
6.00000 5.249
8.00000 5.214
12.00000 5.085
16.00000 5.295
24.00000 6.827
32.00000 11.598
48.00000 19.626
64.00000 24.877
96.00000 65.536
128.00000 138.918
192.00000 160.913
256.00000 178.941
384.00000 173.373
512.00000 180.008
bw = 97%
--------
numactl -m0 -N0 /usr/lib/lmbench/bin/bw_mem -P 8 -N 100 1G rd
numactl -m0 -N0 /usr/lib/lmbench/bin/lat_mem_rd -P 1 512 4096
bw = 556M*8*32
"stride=4096
0.00391 1.430
0.00586 1.430
0.00781 1.430
0.01172 1.430
0.01562 1.430
0.02344 1.430
0.03125 1.430
0.04688 1.430
0.06250 1.430
0.09375 5.063
0.12500 5.070
0.18750 5.092
0.25000 5.963
0.37500 6.377
0.50000 6.525
0.75000 6.105
1.00000 5.951
1.50000 6.379
2.00000 5.577
3.00000 5.532
4.00000 5.372
6.00000 5.328
8.00000 5.291
12.00000 5.260
16.00000 5.253
24.00000 6.407
32.00000 13.153
48.00000 21.064
64.00000 24.562
96.00000 143.447
128.00000 182.688
192.00000 195.648
256.00000 198.184
384.00000 199.390
512.00000 195.149
bw = 98%
--------
numactl -m0 -N0 /usr/lib/lmbench/bin/bw_mem -P 9 -N 100 1G rd
numactl -m0 -N0 /usr/lib/lmbench/bin/lat_mem_rd -P 1 512 4096
bw = 565M*8*32
"stride=4096
0.00391 1.430
0.00586 1.430
0.00781 1.430
0.01172 1.430
0.01562 1.430
0.02344 1.430
0.03125 1.430
0.04688 1.430
0.06250 1.430
0.09375 5.003
0.12500 5.048
0.18750 5.778
0.25000 5.003
0.37500 5.811
0.50000 6.520
0.75000 5.841
1.00000 5.539
1.50000 5.273
2.00000 5.418
3.00000 5.138
4.00000 5.235
6.00000 5.072
8.00000 5.177
12.00000 5.038
16.00000 5.126
24.00000 5.879
32.00000 11.304
48.00000 19.292
64.00000 24.802
96.00000 154.718
128.00000 197.321
192.00000 195.842
256.00000 218.901
384.00000 210.298
512.00000 215.099
bw = 99%
--------
numactl -m0 -N0 /usr/lib/lmbench/bin/bw_mem -P 10 -N 100 1G rd
numactl -m0 -N0 /usr/lib/lmbench/bin/lat_mem_rd -P 1 512 4096
bw = 571M*8*32
"stride=4096
0.00391 1.430
0.00586 1.430
0.00781 1.430
0.01172 1.430
0.01562 1.430
0.02344 1.430
0.03125 1.430
0.04688 1.430
0.06250 1.430
0.09375 5.092
0.12500 5.070
0.18750 6.179
0.25000 6.243
0.37500 6.619
0.50000 6.349
0.75000 6.331
1.00000 5.750
1.50000 5.609
2.00000 5.395
3.00000 5.163
4.00000 5.170
6.00000 5.150
8.00000 5.122
12.00000 5.056
16.00000 5.158
24.00000 6.305
32.00000 14.388
48.00000 19.875
64.00000 26.120
96.00000 91.108
128.00000 217.904
192.00000 231.902
256.00000 224.484
384.00000 241.242
512.00000 234.990
bw = 100%
---------
numactl -m0 -N0 /usr/lib/lmbench/bin/bw_mem -P 11 -N 100 1G rd
numactl -m0 -N0 /usr/lib/lmbench/bin/lat_mem_rd -P 1 512 4096
bw = 574M*8*32
"stride=4096
0.00391 1.430
0.00586 1.429
0.00781 1.430
0.01172 1.430
0.01562 1.430
0.02344 1.430
0.03125 1.430
0.04688 1.430
0.06250 1.430
0.09375 5.003
0.12500 5.003
0.18750 5.799
0.25000 5.908
0.37500 5.666
0.50000 6.539
0.75000 5.973
1.00000 5.364
1.50000 5.444
2.00000 5.656
3.00000 5.224
4.00000 5.324
6.00000 5.117
8.00000 5.163
12.00000 5.062
16.00000 5.112
24.00000 6.064
32.00000 12.013
48.00000 19.543
64.00000 24.830
96.00000 188.989
128.00000 198.572
192.00000 258.530
256.00000 263.336
384.00000 254.254
512.00000 258.133
bw = 100%
---------
numactl -m0 -N0 /usr/lib/lmbench/bin/bw_mem -P 12 -N 100 1G rd
numactl -m0 -N0 /usr/lib/lmbench/bin/lat_mem_rd -P 1 512 4096
bw = 574M*8*32
"stride=4096
0.00391 1.429
0.00586 1.429
0.00781 1.429
0.01172 1.429
0.01562 1.429
0.02344 1.429
0.03125 1.429
0.04688 1.429
0.06250 1.429
0.09375 5.003
0.12500 5.025
0.18750 5.032
0.25000 5.957
0.37500 5.713
0.50000 6.312
0.75000 6.397
1.00000 6.279
1.50000 5.740
2.00000 5.807
3.00000 5.738
4.00000 5.274
6.00000 5.421
8.00000 5.364
12.00000 5.145
16.00000 5.150
24.00000 7.199
32.00000 12.285
48.00000 19.557
64.00000 28.358
96.00000 195.286
128.00000 200.648
192.00000 265.381
256.00000 259.416
384.00000 266.772
512.00000 271.280
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment