Skip to content

Instantly share code, notes, and snippets.

@smj10j

smj10j/test-bs.command

Last active February 14, 2017 16:44
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save smj10j/ed9b001b10aa7009c8a6 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save smj10j/ed9b001b10aa7009c8a6 to your computer and use it in GitHub Desktop.
Download ZIP
Simple bash script to test optimal dd block size on OSX and Linux
Raw
test-bs.command
#!/bin/bash
###
### Example usage: "sudo ./test-bs.sh -p -s 134217728 -w -f /Volumes/SDCARD/dd_test"
###
### This would test the write speed (-w) of the (presumably) attached SD Card by writing
### 134217728 (-s) bytes to /Volumes/SDCARD/dd_test_output (-f) using each block size.
### Before each test is run, the disk cache will be purged (-p)
###
### Generating results
### OF=/Volumes/SDCARD/dd_test_output && \
### (sudo hdiutil imageinfo $(df "$(dirname "$OF")" | awk 'NR == 2 {print $1}') && \
### time sudo ./test-bs.sh -p -s 1073741824 -w -o "$OF") | tee ~/Downloads/Results-SDCARD-1024MB-12-18-2015.txt
###
### Or, when in the directory where files are to be read and written
### (sudo hdiutil imageinfo $(df "$(pwd)" | awk 'NR == 2 {print $1}') && \
### time sudo /Code/smj10j/gists/test-bs.sh -p -s 1073741824 -w) | tee ~/Downloads/Results-WDMyBook1230Media-1024MB-12-18-2015.txt
###
# Constants
BLOCK_SIZES=( 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288 1048576 2097152 4194304 8388608 16777216 33554432 67108864 )
# Settable by flags
VERBOSE=0
PURGE_CACHES=0
RUN_READ_TESTS=1
RUN_WRITE_TESTS=1
NUM_ITERATIONS=1
TEST_FILE_SIZE=134217728
TEST_FILE_PREFIX="dd_bs_test_file"
function usage {
echo "Usage: $0 [-hpvfio]"
echo "Options"
echo " -f Path and file prefix to use for input and output streams"
echo " -i Path of file to use for input stream"
echo " -o Path of file to use for output stream"
echo " -n Number of iterations"
echo " -s Test file size in bytes. Default 134217728 (128 MB)"
echo " -p Purge disk caches before each test. Requires root privileges (run with sudo)"
echo " -w Only test writes"
echo " -v Be verbose"
echo " -h Show this menu"
}
while getopts ":hpwvf:i:o:n:s:" opt; do
case $opt in
f)
TEST_FILE_PREFIX=$OPTARG
;;
i)
TEST_INPUT_FILE=$OPTARG
;;
o)
TEST_OUTPUT_FILE=$OPTARG
;;
n)
NUM_ITERATIONS=$OPTARG
;;
s)
TEST_FILE_SIZE=$OPTARG
;;
p)
PURGE_CACHES=1
;;
w)
RUN_READ_TESTS=0
RUN_WRITE_TESTS=1
;;
v)
VERBOSE=1
;;
h)
usage >&2
exit 1
;;
\?)
echo "Invalid option: -$OPTARG" >&2
usage >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
usage >&2
exit 1
;;
esac
done
TEST_INPUT_FILE=${TEST_INPUT_FILE-"${TEST_FILE_PREFIX}_input"}
TEST_OUTPUT_FILE=${TEST_OUTPUT_FILE-"${TEST_FILE_PREFIX}_output"}
TEST_FILE_SIZE_KB=$((TEST_FILE_SIZE / 1024))
TEST_FILE_SIZE_MB=$((TEST_FILE_SIZE_KB / 1024))
REMOVE_TEST_INPUT_FILE_ON_EXIT=0
# Exit if either file exists already
if [[ $RUN_READ_TESTS -eq 1 && -e $TEST_INPUT_FILE ]]; then
echo "Test file $TEST_INPUT_FILE exists and we will use it for reading" >&2
fi
if [[ -e $TEST_OUTPUT_FILE ]]; then
echo "Test file $TEST_OUTPUT_FILE exists, aborting so we don't overwrite it" >&2
exit 1
fi
### Done with option parsing ###
function run_tests {
# Abort if any errors occur
set -e
if [[ $RUN_READ_TESTS -eq 1 ]]; then
# echo ""
# local DEVICE=$(df "$TEST_INPUT_FILE" | awk 'NR == 2 {print $1}')
# echo "-------- $TEST_INPUT_FILE is stored on device $DEVICE ---------"
# sudo hdiutil imageinfo $DEVICE
# echo "---------------------------------------------------------------"
echo ""
echo "Testing transfer rates to read $TEST_FILE_SIZE_MB MB ($TEST_FILE_SIZE_KB KB) from $TEST_INPUT_FILE"
for BLOCK_SIZE in "${BLOCK_SIZES[@]}"; do
local DD_ARGS=(if="$TEST_INPUT_FILE" of=/dev/null bs=$BLOCK_SIZE)
[[ $VERBOSE -eq 1 ]] && echo "Testing read block size with DD_ARGS: ${DD_ARGS[*]}"
purge_caches
local DD_RESULT
DD_RESULT=$(dd "${DD_ARGS[@]}" 2>&1)
RESULTS_READ[$BLOCK_SIZE]=${RESULTS_READ[$BLOCK_SIZE]}' "'$DD_RESULT'"'
print_rates "$BLOCK_SIZE" "$(get_transfer_rate_str "$DD_RESULT")"
done
fi
if [[ $RUN_WRITE_TESTS -eq 1 ]]; then
echo ""
echo "Testing transfer rates to write $TEST_FILE_SIZE_MB MB ($TEST_FILE_SIZE_KB KB) to $TEST_OUTPUT_FILE"
for BLOCK_SIZE in "${BLOCK_SIZES[@]}"; do
local COUNT=$((TEST_FILE_SIZE / BLOCK_SIZE))
local DD_ARGS=(if=/dev/zero of="$TEST_OUTPUT_FILE" bs=$BLOCK_SIZE count=$COUNT)
[[ $VERBOSE -eq 1 ]] && echo "Testing write block size with DD_ARGS: ${DD_ARGS[*]}"
purge_caches
local DD_RESULT
DD_RESULT=$(dd "${DD_ARGS[@]}" 2>&1)
RESULTS_WRITE[$BLOCK_SIZE]=${RESULTS_WRITE[$BLOCK_SIZE]}' "'$DD_RESULT'"'
print_rates "$BLOCK_SIZE" "$(get_transfer_rate_str "$DD_RESULT")"
done
fi
# Disable exit-on-error
set +e
}
function cleanup {
if [[ $RUN_READ_TESTS -eq 1 && $REMOVE_TEST_INPUT_FILE_ON_EXIT -eq 1 ]]; then
# Clean up the test files if we created any
[[ -f $TEST_INPUT_FILE ]] && rm "$TEST_INPUT_FILE"
fi
[[ $RUN_WRITE_TESTS -eq 1 && -f $TEST_OUTPUT_FILE ]] && rm "$TEST_OUTPUT_FILE"
}
trap cleanup SIGHUP SIGINT SIGTERM EXIT ERR
function purge_caches {
if [[ $PURGE_CACHES -eq 1 ]]; then
set +e
if [[ "$(uname)" == "Darwin" ]]; then
/usr/sbin/purge || echo "Run this command with sudo to be able to purge disk caches" >&2
else
echo 3 > /proc/sys/vm/drop-caches || echo "Run this command with sudo to be able to purge disk caches" >&2
fi
set -e
fi
}
function create_test_input_file {
if [[ $RUN_READ_TESTS -eq 1 && ! -e $TEST_INPUT_FILE ]]; then
# Create test file
REMOVE_TEST_INPUT_FILE_ON_EXIT=1
echo "Generating $TEST_FILE_SIZE_MB MB ($TEST_FILE_SIZE_KB KB) test file at '$TEST_INPUT_FILE'"
local BLOCK_SIZE=$((64 * 1024))
local COUNT=$((TEST_FILE_SIZE / BLOCK_SIZE))
local DD_ARGS=(if=/dev/urandom of="$TEST_INPUT_FILE" bs=$BLOCK_SIZE count=$COUNT)
local OUTPUT
OUTPUT=$(dd "${DD_ARGS[@]}" 2>&1)
[[ $? -eq 1 ]] && echo "$OUTPUT" && exit 1
fi
}
function get_transfer_rate {
local TRANSFER_RATE_STR, TRANSFER_RATE_VALUE, TRANSFER_RATE_UNITS
local DD_RESULT=$1
TRANSFER_RATE_STR=$(get_transfer_rate_str "$DD_RESULT")
TRANSFER_RATE_VALUE="$(echo "$TRANSFER_RATE_STR" | egrep --only-matching '[0-9.]+' | tr -d '\n')"
TRANSFER_RATE_UNITS=${TRANSFER_RATE_STR//$TRANSFER_RATE_VALUE /}
echo '("'$TRANSFER_RATE_VALUE'" "'$TRANSFER_RATE_UNITS'")'
}
function get_transfer_rate_str {
local TRANSFER_RATE_STR
local DD_RESULT=$1
TRANSFER_RATE_STR="$(echo "$DD_RESULT" | egrep --only-matching '[0-9.]+ ([MGk]?B|bytes)/s(ec)?')"
echo "$TRANSFER_RATE_STR"
}
function print_rates {
local BLOCK_SIZE=$1
local TRANSFER_RATE=$2
local TRANSFER_RATE_UNITS=$3
local TRANSFER_RATE_SUM=$4
local TRANSFER_RATE_STDDEV=$5
local TRANSFER_RATE_VAR=$6
local SAMPLE_COUNT=$7
local SCORE=$8
local BLOCK_SIZE_PRINT=$BLOCK_SIZE
if [[ $BLOCK_SIZE -ge 1048576 ]]; then
BLOCK_SIZE_PRINT=$(( BLOCK_SIZE / $((1024*1024)) ))M
elif [[ $BLOCK_SIZE -ge 1024 ]]; then
BLOCK_SIZE_PRINT=$(( BLOCK_SIZE / 1024 ))K
fi
echo -en "${BLOCK_SIZE_PRINT}\t"
if [[ -n "${TRANSFER_RATE_SUM}" ]]; then
echo -en "avg: ${TRANSFER_RATE}\t"
echo -en "sum: ${TRANSFER_RATE_SUM}\t"
echo -en "std dev: ${TRANSFER_RATE_STDDEV}\t"
echo -en "variance: ${TRANSFER_RATE_VAR}\t"
echo -en "count: ${SAMPLE_COUNT}\t"
echo -en "units: ${TRANSFER_RATE_UNITS}\t"
echo -e "score: ${SCORE}"
else
echo -e "${TRANSFER_RATE} ${TRANSFER_RATE_UNITS}"
fi
}
function print_statistics {
local TITLE=${1}
local RESULTS=("${!2}")
echo ""
echo "$TITLE - $START_TIME_STR"
echo "Script Arguments: '$ARGS'"
echo ""
for i in $(seq ${#BLOCK_SIZES[@]}); do
local BLOCK_SIZE=${BLOCK_SIZES[i-1]}
[[ $VERBOSE -eq 1 ]] && echo "Calculating results for block size $BLOCK_SIZE"
local DD_RESULTS_STR=${RESULTS[i-1]}
local DD_RESULTS
eval "DD_RESULTS=($DD_RESULTS_STR)"
local TRANSFER_RATE=0
local TRANSFER_RATE_VALUE=0
local TRANSFER_RATE_UNITS='unknown'
local TRANSFER_RATE_SUM=0
local TRANSFER_RATE_AVG=0
local TRANSFER_RATE_STDDEV=0
local TRANSFER_RATE_VAR=0
local SCORE=0
for DD_RESULT in "${DD_RESULTS[@]}"; do
[[ $VERBOSE -eq 1 ]] && echo "-- $DD_RESULT --"
eval "TRANSFER_RATE=$(get_transfer_rate "$DD_RESULT")"
TRANSFER_RATE_VALUE=${TRANSFER_RATE[0]}
TRANSFER_RATE_UNITS=${TRANSFER_RATE[1]}
TRANSFER_RATE_SUM=$(echo "$TRANSFER_RATE_SUM + $TRANSFER_RATE_VALUE" | bc)
done
TRANSFER_RATE_AVG=$(echo "$TRANSFER_RATE_SUM / ${#DD_RESULTS[@]}" | bc)
for DD_RESULT in "${DD_RESULTS[@]}"; do
eval "TRANSFER_RATE=$(get_transfer_rate "$DD_RESULT")"
TRANSFER_RATE_VALUE=${TRANSFER_RATE[0]}
TRANSFER_RATE_VAR=$(echo "$TRANSFER_RATE_VAR + ($TRANSFER_RATE_VALUE - $TRANSFER_RATE_AVG)^2" | bc)
done
TRANSFER_RATE_STDDEV=$(echo "sqrt($TRANSFER_RATE_VAR)" | bc)
SCORE=$(( TRANSFER_RATE_AVG - (TRANSFER_RATE_STDDEV*2) ))
print_rates "$BLOCK_SIZE" "$TRANSFER_RATE_AVG" "$TRANSFER_RATE_UNITS" "$TRANSFER_RATE_SUM" "$TRANSFER_RATE_STDDEV" "$TRANSFER_RATE_VAR" "${#DD_RESULTS[@]}" "$SCORE"
[[ $VERBOSE -eq 1 ]] && echo ""
done
}
########################
###### Go go go! #######
########################
ARGS="${*}"
START_TIME_STR=$(date --rfc-3339=seconds)
[[ $RUN_READ_TESTS -eq 1 ]] && create_test_input_file
for i in $(seq "$NUM_ITERATIONS"); do
if [[ $NUM_ITERATIONS -gt 1 ]]; then
echo ""
echo "Iteration $i of $NUM_ITERATIONS"
fi
run_tests
done
if [[ $NUM_ITERATIONS -gt 1 ]]; then
[[ $RUN_READ_TESTS -eq 1 ]] && print_statistics "Read Statistics" RESULTS_READ[@]
[[ $RUN_WRITE_TESTS -eq 1 ]] && print_statistics "Write Statistics" RESULTS_WRITE[@]
fi
Raw
zResults-InternalSSD-1024MB-12-18-2015.txt
Backing Store Information:
URL: file:///dev/rdisk1
Name: rdisk1
Class Name: CDevBackingStore
Class Name: CRawDiskImage
Checksum Type: none
Size Information:
Total Bytes: 483127984128
Compressed Ratio: 1
Sector Count: 943609344
Total Non-Empty Bytes: 483127984128
Compressed Bytes: 483127984128
Total Empty Bytes: 0
Format: RAW*
Format Description: raw read/write
Checksum Value:
Properties:
Encrypted: false
Kernel Compatible: false
Checksummed: false
Software License Agreement: false
Partitioned: false
Compressed: no
Segments:
0: /dev/rdisk1
partitions:
partition-scheme: none
block-size: 512
appendable: true
partitions:
0:
partition-name: whole disk
partition-start: 0
partition-synthesized: true
partition-length: 943609344
partition-hint: Apple_HFS
partition-filesystems:
HFS+:
burnable: true
Resize limits (per hdiutil resize -limits):
943608944 943609344 943609344
Testing transfer rates to write 1024 MB (1048576 KB) to /Users/steve/Downloads/dd_test_output
512 247 MB/s
1K 424 MB/s
2K 606 MB/s
4K 634 MB/s
8K 645 MB/s
16K 710 MB/s
32K 616 MB/s
64K 706 MB/s
128K 623 MB/s
256K 730 MB/s
512K 666 MB/s
1M 681 MB/s
2M 673 MB/s
4M 694 MB/s
8M 713 MB/s
16M 674 MB/s
32M 660 MB/s
64M 631 MB/s
Raw
zResults-SDCARD-1024MB-12-18-2015.txt
Backing Store Information:
URL: file:///dev/rdisk4s1
Name: rdisk4s1
Class Name: CDevBackingStore
Class Name: CRawDiskImage
Checksum Type: none
Size Information:
Total Bytes: 32123125760
Compressed Ratio: 1
Sector Count: 62740480
Total Non-Empty Bytes: 32123125760
Compressed Bytes: 32123125760
Total Empty Bytes: 0
Format: RAW*
Format Description: raw read/write
Checksum Value:
Properties:
Encrypted: false
Kernel Compatible: false
Checksummed: false
Software License Agreement: false
Partitioned: false
Compressed: no
Segments:
0: /dev/rdisk4s1
partitions:
partition-scheme: none
block-size: 512
appendable: false
partitions:
0:
partition-name: whole disk
partition-start: 0
partition-synthesized: true
partition-length: 62740480
partition-hint: DOS_FAT_32
partition-filesystems:
FAT32: SDCARD
burnable: false
Resize limits (per hdiutil resize -limits):
62740480 62740480 62740480
Testing transfer rates to write 1024 MB (1048576 KB) to /Volumes/SDCARD/dd_test_output
512 37.8 MB/s
1K 40.1 MB/s
2K 40.7 MB/s
4K 38.9 MB/s
8K 40.6 MB/s
16K 36.6 MB/s
32K 37.7 MB/s
64K 41.0 MB/s
128K 40.1 MB/s
256K 39.9 MB/s
512K 40.7 MB/s
1M 40.0 MB/s
2M 30.9 MB/s
4M 34.3 MB/s
8M 34.2 MB/s
16M 32.8 MB/s
32M 37.0 MB/s
64M 36.8 MB/s
Raw
zResults-WDMyBook1230Media-1024MB-12-18-2015.txt
Backing Store Information:
URL: file:///dev/rdisk7
Name: rdisk7
Class Name: CDevBackingStore
Class Name: CRawDiskImage
Checksum Type: none
Size Information:
Total Bytes: 999250984960
Compressed Ratio: 1
Sector Count: 1951662080
Total Non-Empty Bytes: 999250984960
Compressed Bytes: 999250984960
Total Empty Bytes: 0
Format: RAW*
Format Description: raw read/write
Checksum Value:
Properties:
Encrypted: false
Kernel Compatible: false
Checksummed: false
Software License Agreement: false
Partitioned: false
Compressed: no
Segments:
0: /dev/rdisk7
partitions:
partition-scheme: none
block-size: 512
appendable: true
partitions:
0:
partition-name: whole disk
partition-start: 0
partition-synthesized: true
partition-length: 1951662080
partition-hint: Apple_HFS
partition-filesystems:
HFS+:
burnable: true
Resize limits (per hdiutil resize -limits):
245891336 1951662080 1951662080
Testing transfer rates to write 1024 MB (1048576 KB) to dd_bs_test_file_output
512 73.4 MB/s
1K 76.4 MB/s
2K 62.8 MB/s
4K 62.7 MB/s
8K 67.2 MB/s
16K 63.9 MB/s
32K 66.5 MB/s
64K 68.9 MB/s
128K 63.3 MB/s
256K 63.6 MB/s
512K 68.2 MB/s
1M 58.7 MB/s
2M 61.2 MB/s
4M 62.3 MB/s
8M 63.2 MB/s
16M 58.4 MB/s
32M 62.8 MB/s
64M 65.4 MB/s
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment