l16s xfs

[root@node-21 k8s.westeurope.azure resource]$ fio fio-rand-RW.fio
file1: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=16
...
fio-3.1
Starting 4 processes
file1: Laying out IO file (1 file / 10240MiB)
Jobs: 4 (f=4): [m(4)][100.0%][r=1536MiB/s,w=1023MiB/s][r=393k,w=262k IOPS][eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2506: Mon Nov 19 21:44:14 2018
   read: IOPS=82.7k, BW=323MiB/s (339MB/s)(284GiB/900001msec)
    slat (nsec): min=1400, max=70170k, avg=3709.03, stdev=63623.30
    clat (usec): min=3, max=70557, avg=109.85, stdev=243.87
     lat (usec): min=6, max=70560, avg=114.01, stdev=260.23
    clat percentiles (usec):
     |  1.00th=[   74],  5.00th=[   78], 10.00th=[   80], 20.00th=[   83],
     | 30.00th=[   86], 40.00th=[   89], 50.00th=[   93], 60.00th=[   97],
     | 70.00th=[  102], 80.00th=[  111], 90.00th=[  123], 95.00th=[  137],
     | 99.00th=[  478], 99.50th=[  963], 99.90th=[ 1975], 99.95th=[ 2311],
     | 99.99th=[ 5080]
   bw (  KiB/s): min= 2416, max=408928, per=24.97%, avg=330833.08, stdev=109946.87, samples=1799
   iops        : min=  604, max=102232, avg=82708.28, stdev=27486.73, samples=1799
  write: IOPS=55.1k, BW=215MiB/s (226MB/s)(189GiB/900001msec)
    slat (nsec): min=1900, max=62408k, avg=4599.58, stdev=25201.22
    clat (usec): min=8, max=70556, avg=109.73, stdev=242.01
     lat (usec): min=12, max=70560, avg=114.80, stdev=243.51
    clat percentiles (usec):
     |  1.00th=[   74],  5.00th=[   78], 10.00th=[   80], 20.00th=[   83],
     | 30.00th=[   86], 40.00th=[   89], 50.00th=[   93], 60.00th=[   97],
     | 70.00th=[  102], 80.00th=[  111], 90.00th=[  123], 95.00th=[  137],
     | 99.00th=[  474], 99.50th=[  963], 99.90th=[ 1958], 99.95th=[ 2311],
     | 99.99th=[ 5080]
   bw (  KiB/s): min= 1544, max=270496, per=24.97%, avg=220563.86, stdev=73308.36, samples=1799
   iops        : min=  386, max=67624, avg=55140.94, stdev=18327.09, samples=1799
  lat (usec)   : 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=67.18%
  lat (usec)   : 250=31.26%, 500=0.62%, 750=0.29%, 1000=0.18%
  lat (msec)   : 2=0.39%, 4=0.08%, 10=0.01%, 20=0.01%, 50=0.01%
  lat (msec)   : 100=0.01%
  cpu          : usr=24.23%, sys=65.76%, ctx=394412, majf=0, minf=11
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwt: total=74436748,49626362,0, short=0,0,0, dropped=0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=16
file1: (groupid=0, jobs=1): err= 0: pid=2507: Mon Nov 19 21:44:14 2018
   read: IOPS=82.0k, BW=324MiB/s (340MB/s)(285GiB/900001msec)
    slat (nsec): min=1400, max=70251k, avg=3686.42, stdev=63277.27
    clat (usec): min=3, max=70566, avg=109.46, stdev=243.06
     lat (usec): min=5, max=70571, avg=113.59, stdev=259.27
    clat percentiles (usec):
     |  1.00th=[   74],  5.00th=[   77], 10.00th=[   80], 20.00th=[   83],
     | 30.00th=[   86], 40.00th=[   89], 50.00th=[   92], 60.00th=[   96],
     | 70.00th=[  102], 80.00th=[  110], 90.00th=[  123], 95.00th=[  135],
     | 99.00th=[  469], 99.50th=[  955], 99.90th=[ 1958], 99.95th=[ 2311],
     | 99.99th=[ 5014]
   bw (  KiB/s): min= 2424, max=408824, per=25.06%, avg=332001.39, stdev=110360.34, samples=1799
   iops        : min=  606, max=102206, avg=83000.32, stdev=27590.09, samples=1799
  write: IOPS=55.3k, BW=216MiB/s (227MB/s)(190GiB/900001msec)
    slat (nsec): min=1800, max=62410k, avg=4592.90, stdev=25213.10
    clat (usec): min=30, max=70361, avg=109.34, stdev=240.18
     lat (usec): min=37, max=70365, avg=114.40, stdev=241.67
    clat percentiles (usec):
     |  1.00th=[   74],  5.00th=[   77], 10.00th=[   80], 20.00th=[   83],
     | 30.00th=[   86], 40.00th=[   89], 50.00th=[   92], 60.00th=[   96],
     | 70.00th=[  101], 80.00th=[  110], 90.00th=[  123], 95.00th=[  135],
     | 99.00th=[  469], 99.50th=[  955], 99.90th=[ 1958], 99.95th=[ 2278],
     | 99.99th=[ 4948]
   bw (  KiB/s): min= 1624, max=272032, per=25.06%, avg=221404.40, stdev=73595.64, samples=1799
   iops        : min=  406, max=68008, avg=55351.07, stdev=18398.90, samples=1799
  lat (usec)   : 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=67.74%
  lat (usec)   : 250=30.70%, 500=0.62%, 750=0.29%, 1000=0.17%
  lat (msec)   : 2=0.38%, 4=0.08%, 10=0.01%, 20=0.01%, 50=0.01%
  lat (msec)   : 100=0.01%
  cpu          : usr=24.03%, sys=65.94%, ctx=395303, majf=0, minf=10
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwt: total=74698057,49814332,0, short=0,0,0, dropped=0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=16
file1: (groupid=0, jobs=1): err= 0: pid=2508: Mon Nov 19 21:44:14 2018
   read: IOPS=82.8k, BW=323MiB/s (339MB/s)(284GiB/900001msec)
    slat (nsec): min=1400, max=69339k, avg=3702.79, stdev=63341.31
    clat (usec): min=3, max=69450, avg=109.71, stdev=242.28
     lat (usec): min=6, max=69453, avg=113.86, stdev=258.46
    clat percentiles (usec):
     |  1.00th=[   74],  5.00th=[   78], 10.00th=[   80], 20.00th=[   83],
     | 30.00th=[   86], 40.00th=[   89], 50.00th=[   93], 60.00th=[   96],
     | 70.00th=[  102], 80.00th=[  110], 90.00th=[  123], 95.00th=[  137],
     | 99.00th=[  474], 99.50th=[  963], 99.90th=[ 1975], 99.95th=[ 2311],
     | 99.99th=[ 5080]
   bw (  KiB/s): min= 2416, max=406504, per=24.99%, avg=331186.85, stdev=110111.44, samples=1799
   iops        : min=  604, max=101628, avg=82796.69, stdev=27527.86, samples=1799
  write: IOPS=55.2k, BW=216MiB/s (226MB/s)(190GiB/900001msec)
    slat (nsec): min=1800, max=62410k, avg=4601.98, stdev=25547.03
    clat (usec): min=24, max=69450, avg=109.63, stdev=243.01
     lat (usec): min=32, max=69453, avg=114.71, stdev=244.52
    clat percentiles (usec):
     |  1.00th=[   74],  5.00th=[   78], 10.00th=[   80], 20.00th=[   83],
     | 30.00th=[   86], 40.00th=[   89], 50.00th=[   93], 60.00th=[   96],
     | 70.00th=[  102], 80.00th=[  110], 90.00th=[  123], 95.00th=[  135],
     | 99.00th=[  474], 99.50th=[  963], 99.90th=[ 1975], 99.95th=[ 2311],
     | 99.99th=[ 5080]
   bw (  KiB/s): min= 1808, max=271072, per=25.00%, avg=220834.09, stdev=73418.52, samples=1799
   iops        : min=  452, max=67768, avg=55208.50, stdev=18354.62, samples=1799
  lat (usec)   : 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=67.42%
  lat (usec)   : 250=31.01%, 500=0.62%, 750=0.29%, 1000=0.17%
  lat (msec)   : 2=0.38%, 4=0.08%, 10=0.01%, 20=0.01%, 50=0.01%
  lat (msec)   : 100=0.01%
  cpu          : usr=24.09%, sys=65.88%, ctx=394356, majf=0, minf=10
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwt: total=74516486,49687576,0, short=0,0,0, dropped=0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=16
file1: (groupid=0, jobs=1): err= 0: pid=2509: Mon Nov 19 21:44:14 2018
   read: IOPS=82.8k, BW=323MiB/s (339MB/s)(284GiB/900001msec)
    slat (nsec): min=1300, max=69693k, avg=3701.90, stdev=63753.41
    clat (usec): min=3, max=69811, avg=109.75, stdev=240.78
     lat (usec): min=6, max=69814, avg=113.90, stdev=257.11
    clat percentiles (usec):
     |  1.00th=[   74],  5.00th=[   78], 10.00th=[   80], 20.00th=[   83],
     | 30.00th=[   86], 40.00th=[   89], 50.00th=[   93], 60.00th=[   97],
     | 70.00th=[  102], 80.00th=[  111], 90.00th=[  123], 95.00th=[  137],
     | 99.00th=[  474], 99.50th=[  963], 99.90th=[ 1975], 99.95th=[ 2311],
     | 99.99th=[ 5080]
   bw (  KiB/s): min= 2488, max=408640, per=24.99%, avg=331098.26, stdev=110035.13, samples=1799
   iops        : min=  622, max=102160, avg=82774.53, stdev=27508.77, samples=1799
  write: IOPS=55.2k, BW=216MiB/s (226MB/s)(189GiB/900001msec)
    slat (nsec): min=1900, max=62414k, avg=4590.48, stdev=24047.84
    clat (usec): min=9, max=69816, avg=109.69, stdev=243.56
     lat (usec): min=12, max=69821, avg=114.76, stdev=244.90
    clat percentiles (usec):
     |  1.00th=[   74],  5.00th=[   78], 10.00th=[   80], 20.00th=[   83],
     | 30.00th=[   86], 40.00th=[   89], 50.00th=[   93], 60.00th=[   97],
     | 70.00th=[  102], 80.00th=[  110], 90.00th=[  123], 95.00th=[  135],
     | 99.00th=[  474], 99.50th=[  963], 99.90th=[ 1975], 99.95th=[ 2311],
     | 99.99th=[ 5145]
   bw (  KiB/s): min= 1456, max=273896, per=24.98%, avg=220696.73, stdev=73355.32, samples=1799
   iops        : min=  364, max=68474, avg=55174.15, stdev=18338.83, samples=1799
  lat (usec)   : 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=67.30%
  lat (usec)   : 250=31.13%, 500=0.62%, 750=0.29%, 1000=0.17%
  lat (msec)   : 2=0.39%, 4=0.08%, 10=0.01%, 20=0.01%, 50=0.01%
  lat (msec)   : 100=0.01%
  cpu          : usr=24.13%, sys=65.82%, ctx=395577, majf=0, minf=12
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwt: total=74493820,49654736,0, short=0,0,0, dropped=0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=16

Run status group 0 (all jobs):
   READ: bw=1294MiB/s (1357MB/s), 323MiB/s-324MiB/s (339MB/s-340MB/s), io=1137GiB (1221GB), run=900001-900001msec
  WRITE: bw=863MiB/s (905MB/s), 215MiB/s-216MiB/s (226MB/s-227MB/s), io=758GiB (814GB), run=900001-900001msec

Disk stats (read/write):
  sdb: ios=1571433/14889416, merge=0/1635, ticks=357712/68708039, in_queue=69186662, util=67.97%