parallel decompression speed test

test_decomp.sh

#!/bin/bash
#
# This is a rather minimal example Argbash potential
# Example taken from http://argbash.readthedocs.io/en/stable/example.html
#
# ARG_OPTIONAL_BOOLEAN([delete-tmp],[],[delete output directory])
# ARG_POSITIONAL_SINGLE([num-threads],[number of threads to run with],[])
# ARG_POSITIONAL_SINGLE([input],[input file list],[])
# ARG_HELP([Test how long it takes to decompress many files in parallel])
# ARGBASH_GO()
# needed because of Argbash --> m4_ignore([
### START OF CODE GENERATED BY Argbash v2.9.0 one line above ###
# Argbash is a bash code generator used to get arguments parsing right.
# Argbash is FREE SOFTWARE, see https://argbash.io for more info
# Generated online by https://argbash.io/generate


die()
{
	local _ret="${2:-1}"
	test "${_PRINT_HELP:-no}" = yes && print_help >&2
	echo "$1" >&2
	exit "${_ret}"
}


begins_with_short_option()
{
	local first_option all_short_options='h'
	first_option="${1:0:1}"
	test "$all_short_options" = "${all_short_options/$first_option/}" && return 1 || return 0
}

# THE DEFAULTS INITIALIZATION - POSITIONALS
_positionals=()
# THE DEFAULTS INITIALIZATION - OPTIONALS
_arg_delete_tmp="off"


print_help()
{
	printf '%s\n' "Test how long it takes to decompress many files in parallel"
	printf 'Usage: %s [--(no-)delete-tmp] [-h|--help] <num-threads> <input>\n' "$0"
	printf '\t%s\n' "<num-threads>: number of threads to run with"
	printf '\t%s\n' "<input>: input file list"
	printf '\t%s\n' "--delete-tmp, --no-delete-tmp: delete output directory (off by default)"
	printf '\t%s\n' "-h, --help: Prints help"
}


parse_commandline()
{
	_positionals_count=0
	while test $# -gt 0
	do
		_key="$1"
		case "$_key" in
			--no-delete-tmp|--delete-tmp)
				_arg_delete_tmp="on"
				test "${1:0:5}" = "--no-" && _arg_delete_tmp="off"
				;;
			-h|--help)
				print_help
				exit 0
				;;
			-h*)
				print_help
				exit 0
				;;
			*)
				_last_positional="$1"
				_positionals+=("$_last_positional")
				_positionals_count=$((_positionals_count + 1))
				;;
		esac
		shift
	done
}


handle_passed_args_count()
{
	local _required_args_string="'num-threads' and 'input'"
	test "${_positionals_count}" -ge 2 || _PRINT_HELP=yes die "FATAL ERROR: Not enough positional arguments - we require exactly 2 (namely: $_required_args_string), but got only ${_positionals_count}." 1
	test "${_positionals_count}" -le 2 || _PRINT_HELP=yes die "FATAL ERROR: There were spurious positional arguments --- we expect exactly 2 (namely: $_required_args_string), but got ${_positionals_count} (the last one was: '${_last_positional}')." 1
}


assign_positional_args()
{
	local _positional_name _shift_for=$1
	_positional_names="_arg_num_threads _arg_input "

	shift "$_shift_for"
	for _positional_name in ${_positional_names}
	do
		test $# -gt 0 || break
		eval "$_positional_name=\${1}" || die "Error during argument parsing, possibly an Argbash bug." 1
		shift
	done
}

parse_commandline "$@"
handle_passed_args_count
assign_positional_args 1 "${_positionals[@]}"

# OTHER STUFF GENERATED BY Argbash

### END OF CODE GENERATED BY Argbash (sortof) ### ])
# [ <-- needed because of Argbash


echo "Value of --num-threads: $_arg_num_threads"
echo "Value of --input: $_arg_input"
echo "delete-tmp is $_arg_delete_tmp"

# ] <-- needed because of Argbash

inputlst=$_arg_input
threads=$_arg_num_threads
listname=${inputlst##*/}

echo "Running parallel decompression on ${inputlst}"
tfname="par_decomp_t=${threads}_${listname}.time"
mkdir -p DECOMP_OUT
/usr/bin/time -v -o ${tfname} ./rush -j ${threads} -i ${inputlst} -v nt=${threads} 'gunzip -c {} >> DECOMP_OUT/thread_$(({#} % {nt}))' 

if [ "$_arg_delete_tmp" = "on" ]; then
	echo "deleting output directory"
	rm -fr DECOMP_OUT
fi