Reproducing CrystalDiskMark tests with fio - fixes for https://unix.stackexchange.com/revisions/480191/9

cdm_fio.sh

#!/bin/bash

# This script is based on https://unix.stackexchange.com/revisions/480191/9 .
# The following changes proved to be necessary to make it work on CentOS 7:
#  * removed disk info (model, size) - not very useful, might not work in many cases.
#  * using "bw" instead of "bw_bytes" to support fio version 3.1 (those availible through yum @base)
#  * escaping exclamation mark in sed command
#  * the ".fiomark.txt" is not auto-removed 

LOOPS=5 #How many times to run each test
SIZE=1024 #Size of each test, multiples of 32 recommended for Q32 tests to give the most accurate results.
WRITEZERO=0 #Set whether to write zeroes or randoms to testfile (random is the default for both fio and crystaldiskmark); dd benchmarks typically only write zeroes which is why there can be a speed difference.

QSIZE=$(($SIZE / 32)) #Size of Q32Seq tests
SIZE+=m
QSIZE+=m

if [ -z $1 ]; then
    TARGET=$HOME
    echo "Defaulting to $TARGET for testing"
else
    TARGET="$1"
    echo "Testing in $TARGET"
fi



echo "Configuration: Size:$SIZE Loops:$LOOPS Write Only Zeroes:$WRITEZERO
Running Benchmark,  please wait...
"

fio --loops=$LOOPS --size=$SIZE --filename="$TARGET/.fiomark.tmp" --stonewall --ioengine=libaio --direct=1 --zero_buffers=$WRITEZERO --output-format=json \
  --name=Bufread --loops=1 --bs=$SIZE --iodepth=1 --numjobs=1 --rw=readwrite \
  --name=Seqread --bs=$SIZE --iodepth=1 --numjobs=1 --rw=read \
  --name=Seqwrite --bs=$SIZE --iodepth=1 --numjobs=1 --rw=write \
  --name=512kread --bs=512k --iodepth=1 --numjobs=1 --rw=read \
  --name=512kwrite --bs=512k --iodepth=1 --numjobs=1 --rw=write \
  --name=SeqQ32T1read --bs=$QSIZE --iodepth=32 --numjobs=1 --rw=read \
  --name=SeqQ32T1write --bs=$QSIZE --iodepth=32 --numjobs=1 --rw=write \
  --name=4kread --bs=4k --iodepth=1 --numjobs=1 --rw=randread \
  --name=4kwrite --bs=4k --iodepth=1 --numjobs=1 --rw=randwrite \
  --name=4kQ32T1read --bs=4k --iodepth=32 --numjobs=1 --rw=randread \
  --name=4kQ32T1write --bs=4k --iodepth=32 --numjobs=1 --rw=randwrite \
  --name=4kQ8T8read --bs=4k --iodepth=8 --numjobs=8 --rw=randread \
  --name=4kQ8T8write --bs=4k --iodepth=8 --numjobs=8 --rw=randwrite > "$TARGET/.fiomark.txt"

SEQR="$(($(cat "$TARGET/.fiomark.txt" | grep -A15 '"name" : "Seqread"' | grep bw | cut -d: -f2 | sed s:,::g)/1024))MB/s IOPS=$(cat "$TARGET/.fiomark.txt" | grep -A15 '"name" : "Seqread"' | grep -m1 iops | cut -d: -f2 | cut -d. -f1 | sed 's: ::g')"
SEQW="$(($(cat "$TARGET/.fiomark.txt" | grep -A80 '"name" : "Seqwrite"' | grep bw | grep -v '_' | sed 2\!d | cut -d: -f2 | sed s:,::g)/1024))MB/s IOPS=$(cat "$TARGET/.fiomark.txt" | grep -A80 '"name" : "Seqwrite"' | grep iops | sed '7!d' | cut -d: -f2 | cut -d. -f1 | sed 's: ::g')"
F12KR="$(($(cat "$TARGET/.fiomark.txt" | grep -A15 '"name" : "512kread"' | grep bw | cut -d: -f2 | sed s:,::g)/1024))MB/s IOPS=$(cat "$TARGET/.fiomark.txt" | grep -A15 '"name" : "512kread"' | grep -m1 iops | cut -d: -f2 | cut -d. -f1 | sed 's: ::g')"
F12KW="$(($(cat "$TARGET/.fiomark.txt" | grep -A80 '"name" : "512kwrite"' | grep bw | grep -v '_' | sed 2\!d | cut -d: -f2 | sed s:,::g)/1024))MB/s IOPS=$(cat "$TARGET/.fiomark.txt" | grep -A80 '"name" : "512kwrite"' | grep iops | sed '7!d' | cut -d: -f2 | cut -d. -f1 | sed 's: ::g')"
SEQ32R="$(($(cat "$TARGET/.fiomark.txt" | grep -A15 '"name" : "SeqQ32T1read"' | grep bw | cut -d: -f2 | sed s:,::g)/1024))MB/s IOPS=$(cat "$TARGET/.fiomark.txt" | grep -A15 '"name" : "SeqQ32T1read"' | grep -m1 iops | cut -d: -f2 | cut -d. -f1 | sed 's: ::g')"
SEQ32W="$(($(cat "$TARGET/.fiomark.txt" | grep -A80 '"name" : "SeqQ32T1write"' | grep bw | grep -v '_' | sed 2\!d | cut -d: -f2 | sed s:,::g)/1024))MB/s IOPS=$(cat "$TARGET/.fiomark.txt" | grep -A80 '"name" : "SeqQ32T1write"' | grep iops | sed '7!d' | cut -d: -f2 | cut -d. -f1 | sed 's: ::g')"
FKR="$(($(cat "$TARGET/.fiomark.txt" | grep -A15 '"name" : "4kread"' | grep bw | cut -d: -f2 | sed s:,::g)/1024))MB/s IOPS=$(cat "$TARGET/.fiomark.txt" | grep -A15 '"name" : "4kread"' | grep -m1 iops | cut -d: -f2 | cut -d. -f1 | sed 's: ::g')"
FKW="$(($(cat "$TARGET/.fiomark.txt" | grep -A80 '"name" : "4kwrite"' | grep bw | grep -v '_' | sed 2\!d | cut -d: -f2 | sed s:,::g)/1024))MB/s IOPS=$(cat "$TARGET/.fiomark.txt" | grep -A80 '"name" : "4kwrite"' | grep iops | sed '7!d' | cut -d: -f2 | cut -d. -f1 | sed 's: ::g')"
FK32R="$(($(cat "$TARGET/.fiomark.txt" | grep -A15 '"name" : "4kQ32T1read"' | grep bw | cut -d: -f2 | sed s:,::g)/1024))MB/s IOPS=$(cat "$TARGET/.fiomark.txt" | grep -A15 '"name" : "4kQ32T1read"' | grep -m1 iops | cut -d: -f2 | cut -d. -f1 | sed 's: ::g')"
FK32W="$(($(cat "$TARGET/.fiomark.txt" | grep -A80 '"name" : "4kQ32T1write"' | grep bw | grep -v '_' | sed 2\!d | cut -d: -f2 | sed s:,::g)/1024))MB/s IOPS=$(cat "$TARGET/.fiomark.txt" | grep -A80 '"name" : "4kQ32T1write"' | grep iops | sed '7!d' | cut -d: -f2 | cut -d. -f1 | sed 's: ::g')"
FK8R="$(($(cat "$TARGET/.fiomark.txt" | grep -A15 '"name" : "4kQ8T8read"' | grep bw | sed 's/        "bw" : //g' | sed 's:,::g' | awk '{ SUM += $1} END { print SUM }')/1024))MB/s IOPS=$(cat "$TARGET/.fiomark.txt" | grep -A15 '"name" : "4kQ8T8read"' | grep iops | sed 's/        "iops" : //g' | sed 's:,::g' | awk '{ SUM += $1} END { print SUM }' | cut -d. -f1)"
FK8W="$(($(cat "$TARGET/.fiomark.txt" | grep -A80 '"name" : "4kQ8T8write"' | grep bw | sed 's/        "bw" : //g' | sed 's:,::g' | awk '{ SUM += $1} END { print SUM }')/1024))MB/s IOPS=$(cat "$TARGET/.fiomark.txt" | grep -A80 '"name" : "4kQ8T8write"' | grep '"iops" '| sed 's/        "iops" : //g' | sed 's:,::g' | awk '{ SUM += $1} END { print SUM }' | cut -d. -f1)"

echo -e "
Results:  
\033[0;33m
Sequential Read: $SEQR
Sequential Write: $SEQW
\033[0;32m
512KB Read: $F12KR
512KB Write: $F12KW
\033[1;36m
Sequential Q32T1 Read: $SEQ32R
Sequential Q32T1 Write: $SEQ32W
\033[0;36m
4KB Read: $FKR
4KB Write: $FKW
\033[1;33m
4KB Q32T1 Read: $FK32R
4KB Q32T1 Write: $FK32W
\033[1;35m
4KB Q8T8 Read: $FK8R
4KB Q8T8 Write: $FK8W
"

# rm "$TARGET/.fiomark.txt"
rm "$TARGET/.fiomark.tmp"

Feature request, since I don't have the points on Stack Exchange to post a comment there:

Support paths that are mounted from /dev/mapper or /dev/ devices, as is the case in my setup, where the volumes are mounted from a LVM2 group inside a LUKS container. Thankfully, the $TARGET variable is correct, so it writes the test files to the correct location always, but it doesn't support resolving the disk model info from mapper or volume group paths.

However, the benchmark shows that performance is really great even when there's encryption on top of it.

@kode54 I can look into supporting /dev/ devices, I'm not very familiar with encryption and logical partitions so I'm not sure what supporting paths from /dev/mapper involves (that is what /dev/mapper is for right?)

Well, really, all it needs to do is resolve the actual host device from them for the drive info, and properly display a full name for the logs and such.

For example, it needs to resolve /dev// into as a volume group. That can be used with the volume group tools to identify which device the volume group is hosted on. In my case, this would resolve to a /dev/mapper/cryptlvm path, which means it's hosted on the LUKS device named cryptlvm.

Then /dev/mapper/ paths need to be resolved like that to use the cryptsetup tool or perhaps some /sys paths where applicable to resolve that to the partition/device that the LUKS is hosted on.

Conversely, some people have also been known to run LUKS on top of LVM, so it would be a /dev/mapper device that resolves to a /dev// path, that then resolves to a host device or partition.

It doesn't need any of this to still benchmark, the filesystem path is still valid for hosting the test file(s), and produces somewhat accurate results despite the overhead.

The only artifact as of now is that it ends up logging that the drive is called mappe. The stats are otherwise fine.

I've tried to reproduce CrystalDiskMark's interface and its behavior with Qt and fio. If you're interested, please take a look at my repo.

In my gist, I've updated this script to use jq tool for json parsing:

QUERY='def read_bw(name): [.jobs[] | select(.jobname==name+"read").read.bw] | add / 1024 | floor;
       def read_iops(name): [.jobs[] | select(.jobname==name+"read").read.iops] | add | floor;
       def write_bw(name): [.jobs[] | select(.jobname==name+"write").write.bw] | add / 1024 | floor;
       def write_iops(name): [.jobs[] | select(.jobname==name+"write").write.iops] | add | floor;
       def job_summary(name): read_bw(name), read_iops(name), write_bw(name), write_iops(name);
       job_summary("Seq"), job_summary("512k"), job_summary("SeqQ32T1"),
       job_summary("4k"), job_summary("4kQ32T1"), job_summary("4kQ8T8")'
read -d '\n' -ra V <<< "$(jq "$QUERY" "$TARGET/.fiomark.txt")"

I think this makes it much more maintainable and should solve problems like:
553046/1024: syntax error in expression (error token is "553046/1024")

@BobbyWibowo
It is failing because fio is returning two rows that match the bw pattern (bw and bw_bytes) for the --rw=read command. We need to exclude those results, using something like grep -v '_'.

@qqgg231
Your issue must be being caused by a newer version of fio than this script originally worked with, as I'm experiencing the same issue. I had to break the executions of fio into multiple commands.

Here is an updated version of this script that accounts for both of my above statements (tested in CentOS 7 minimal with vim, fio and df installed): https://gist.github.com/BAGELreflex/c04e7a25d64e989cbd9376a9134b8f6d

just using "grep -w bw" fixes the problem.

I got these errors on almost everytime try to assign READ values:
cdm_fio.sh: line 47: 3770160898
3681797/1024: syntax error in expression (error token is "3681797/1024")
cdm_fio.sh: line 49: 2653835452
2591636/1024: syntax error in expression (error token is "2591636/1024")
cdm_fio.sh: line 51: 4006499343
3912597/1024: syntax error in expression (error token is "3912597/1024")
cdm_fio.sh: line 53: 98398290
96092/1024: syntax error in expression (error token is "96092/1024")
cdm_fio.sh: line 55: 279897248
273337/1024: syntax error in expression (error token is "273337/1024")
Results:
Sequential Read:
Sequential Write: 184MB/s IOPS=0
512KB Read:
512KB Write: 244MB/s IOPS=488
Sequential Q32T1 Read:
Sequential Q32T1 Write: 137MB/s IOPS=4
4KB Read:
4KB Write: 75MB/s IOPS=19244
4KB Q32T1 Read:
4KB Q32T1 Write: 116MB/s IOPS=29714
4KB Q8T8 Read: 599MB/s IOPS=153598
4KB Q8T8 Write: 138MB/s IOPS=35407

I forgot to mention I'm on Ubuntu server 22.04