Skip to content

Instantly share code, notes, and snippets.

@MewX
Created June 4, 2021 09:28
Show Gist options
  • Save MewX/9f382e4e798136aa643f267d7b1ca218 to your computer and use it in GitHub Desktop.
Save MewX/9f382e4e798136aa643f267d7b1ca218 to your computer and use it in GitHub Desktop.
downsampler-threaded.sh v4
#!/bin/bash
#########################
#
# NAME:
# downsampler-threaded.sh - A Bash script to automate resampling of 24 bit FLAC files using multiple threads.
#
# SYNOPSIS:
# downsampler-threaded.sh [OPTION [ARGUMENT]...] [--] FILE_OR_FOLDER [FILE_OR_FOLDER...]
#
# DESCRIPTION:
# Automatically resamples 24 bit FLAC files to 16 bit and a common multiple of their sample rate.
#
# Uses SoX with multiple threads if either GNU Parallel or moreutils Parallel is detected, or with a single
# thread when neither 'parallel' variety is found.
#
# Optionally supports 24 bit outputs for 176.4 and 192 KHz sources, and/or dithering 24/44.1 and 24/48 sources
# to 16 bit without resampling.
#
# metaflac is optionally used on successfully converted files to add padding and/or to add either of 2 tags
# which detail 1) the sox command used, and 2) the source file's bit depth and sample rate.
#
# DEPENDS:
# basename, dirname, file, metaflac, sox.
#
# RECOMMENDS:
# parallel (GNU or moreutils).
#
# nproc from GNU coreutils is only used for moreutils parallel with the "threads_unused / -T / --unthreads" option.
#
# realpath was used in previous versions, and can still be used by un-commenting lines 320, 321, and 323, but this
# version now defaults to using the function (now named "absolute_path") formerly used only as a no-realpath fallback.
#
# For both realpath (if re-enabled) and nproc, Mac/BSD coreutils binaries with a 'g' prepended to their name will
# be detected and used if they are in the $PATH.
#
# VERSION: 04
#
#########################
#### ####
#### Default User Settings ####
#### ####
# Threads - requires a parallel, default (when a parallel is present) is to use all available threads, this script runs in single-threaded mode when both options are enabled
threads_used="0" # "0" to disable, or maximum number of threads to use
threads_unused="0" # "0" to disable, or number of threads to keep available - moreutils requires nproc to use this option
# SoX Verbosity - "0" absolutely nothing ever, "1" errors, "2" errors+warnings, "3" errors+warnings+sox_processing_info, "4"+ SoX_debugging
sox_verbosity_level="1" # "1" is recommended, must be set, with a number, anything between 0-4 is acceptable
# SoX dither # "dither" is recommended, but noise-shaped dither ("dither -s") may help with sources for which -G/--guard is
sox_dither="dither" # not sufficient to prevent clipping, and any other valid dither effect command may be set here.
# SoX rate
sox_rate="rate -v -L" # Set any valid rate effect command for SoX here, "rate -v -L" is recommended.
# FLAC Padding - set to "0" to disable adding padding to output files
flac_padding="4096" # length in bytes of the padding block added by metaflac to converted files (+4 more bytes for padding block header)
# Homebrew binaries folder on MacOS - in most cases only needed for Automator users, unless the correct path is not in your shell's $PATH by default.
homebrew_bin="/usr/local/bin" # set the location of your Homebrew binaries
# Script Features - setting the value for options below to "1" enables them, any other text (or lack thereof) between the double-quotes disables.
use_24_44_and_24_48_input="1" # output 16/44 and 16/48 from 24/44 and 24/48 sources
use_24_88_and_24_96_output="0" # output 24/88.2 and 24/96 from 24/176.4 and 24/192 sources
use_SOXCOMMAND_tag="1" # create a tag detailing the SoX command used to convert the file
use_SOXSOURCE_tag="1" # create a tag detailing the source file's bit depth and sample rate
use_progress_bar="0" # use GNU parallel's progress bar for SoX jobs - no effect when using moreutils parallel or single-threaded mode
#sox_stderr_logging="0" # not yet implemented, redirection creates the file before there's any output...
# should we just accept that, and check for/delete empty log files after? hmmm
#### ####
#### End of Default Settings ####
#### ####
print_usage() { printf 'Usage:
%s [OPTION [ARGUMENT]...] [--] FILE_OR_FOLDER [FILE_OR_FOLDER...]
Options:
Script:
-h, -H, --help Print this help text and exit.
-- End of options. Following arguments taken as files/folders for conversion.
-f, --4448 Use 24/44 and 24/48 input files.
-F, --no-4448 Don'\''t use 24/44 and 24/48 input files.
-e, --8896 Output 24/88.2 and 24/96 from 24/176.4 and 24/192 sources.
-E, --no-8896 Don'\''t output 24/88.2 & 24/96 from 24/176.4 & 24/192 (but do output 16/44 and 16/48)
SoX:
-v ARG, --sox-verbosity ARG SoX'\''s -V option, must be set with a number between 0 and 4.
-d '\''ARG'\'', --dither '\''ARG'\'' Dither effect command for SoX, arguments with spaces should be quoted.
-D, --default-dither Sets dither command to '\''dither'\''.
-r '\''ARG'\'', --rate '\''ARG'\'' Rate effect command for SoX, arguments with spaces should be quoted
-R, --default-rate Sets rate command to '\''rate -v -L'\'' (very high quality, linear phase response)
Threads:
-t ARG, --threads ARG Number of threads to use. Set to 0 for (parallel'\''s) defaults. Disables -T/--unthreads
-T ARG, --unthreads ARG Number of threads to leave unused. Set to 0 for (parallel'\''s) defaults. Disables -t/--threads
GNU Parallel:
-b, --progress-bar Use GNU Parallel'\''s progress bar. No effect with moreutils parallel
-B, --no-progress-bar Don'\''t use GNU Parallel'\''s progress bar. No effect with moreutils parallel
FLAC:
-p ARG, --padding ARG Number of bytes to use for FLAC padding. Set to 0 to disable.
Tags:
-c, --command-tag Tag output with the SoX command used to convert the file.
-C, --no-command-tag Do not tag output with the SoX command used to convert the file.
-s, --source-tag Tag output with the source file'\''s bit depth and sample rate.
-S, --no-source-tag Do not tag output with the source file'\''s bit depth and sample rate.
Homebrew:
-w, --homebrew-bin '\''ARG'\'' On MacOS, set the location of your Homebrew binaries (in most cases only needed with Automator).
-W, --default-homebrew On MacOS, use the default Homebrew binary location, /usr/local/bin (in most cases only needed with Automator).
' "$0" ; }
#-P ARG, --parallel-binary ARG let user specify their parallel binary - should be do-able with minimal changes...? worth doing?
#-(?), --summary print summary of actions taken on each file / ie: script verbosity / eg: print table containing data from our arrays, easy
absolute_path() { ( cd -P "$( dirname "$1" )" 2>/dev/null && printf '%s/%s\n' "$PWD" "$( basename "$1" )" ) ; }
printf '\n'
# colourful printf # tput works everywhere?
red="$( tput setaf 1 )" # RED='\033[0;31m'
green="$( tput setaf 2 )" # GREEN='\033[0;32m'
orange="$( tput setaf 3 )" # ORANGE='\033[0;33m'
default="$( tput sgr0 )" # NC='\033[0m'
# runtime options
while : ;do
case "$1" in
-b|--progress-bar)
use_progress_bar="1"
shift
;;
-B|--no-progress-bar)
use_progress_bar="0"
shift
;;
-c|--command-tag)
use_SOXCOMMAND_tag="1"
shift
;;
-C|--no-command-tag)
use_SOXCOMMAND_tag="0"
shift
;;
-d|--dither)
printf '%sWARNING%s: Validity of the dither effect command is your responsibility. Arguments with spaces should be quoted.\n\n' "$orange" "$default"
sox_dither="$2"
shift 2
;;
-D|--default-dither)
sox_dither="dither"
shift
;;
-e|--8896)
use_24_88_and_24_96_output="1"
shift
;;
-E|--no-8896)
use_24_88_and_24_96_output="0"
shift
;;
-f|--4448)
use_24_44_and_24_48_input="1"
shift
;;
-F|--no-4448)
use_24_44_and_24_48_input="0"
shift
;;
-h|-H|--help)
print_usage ;exit
;;
-p|--padding)
flac_padding="$2"
shift 2
;;
-r|--rate)
printf '%sWARNING%s: Validity of the rate effect command is your responsibility. Arguments with spaces should be quoted.\n\n' "$orange" "$default"
sox_rate="$2"
shift 2
;;
-R|--default-rate)
sox_rate="rate -v -L"
shift
;;
-s|--source-tag)
use_SOXSOURCE_tag="1"
shift
;;
-S|--no-source-tag)
use_SOXSOURCE_tag="0"
shift
;;
-t|--threads)
threads_used="$2" ;threads_unused="0"
shift 2
;;
-T|--unthreads)
threads_unused="$2" ;threads_used="0"
shift 2
;;
-v|--sox-verbosity)
sox_verbosity_level="$2"
shift 2
;;
-w|--homebrew-bin)
homebrew_bin="$2"
shift 2
;;
-W|--default-homebrew)
homebrew_bin="/usr/local/bin"
shift
;;
--)
break
;;
-?*)
printf '%sERROR%s: Unknown option '\''%s'\''\n\n' "$red" "$default" "$1" >&2 ;print_usage ;exit 1
;;
*)
break
;;
esac
done
# argument(s) required
[[ "$#" -ge "1" ]] || { printf '%sERROR%s: Nothing to do, please specify at least one file or folder.\n\n' "$red" "$default" >&2 ;print_usage ;exit 1 ; }
# validate threads
[[ "$threads_used" != *[!0123456789]* ]] || { printf '%sERROR%s: Argument for '\''-t / --threads / threads_used'\'' must be zero or a positive integer.\n\n' "$red" "$default" >&2 ;exit 1 ; }
[[ "$threads_unused" != *[!0123456789]* ]] || { printf '%sERROR%s: Argument for '\''-T / --unthreads / threads_unused'\'' must be zero or a positive integer.\n\n' "$red" "$default" >&2 ;exit 1 ; }
# validate flac padding
if [[ "$flac_padding" != *[!0123456789]* ]] ;then
if [[ "$flac_padding" -ge "1" ]] ;then
[[ "$flac_padding" -le "512" ]] && printf '%sWARNING%s: Using only %s bytes for FLAC padding. Maybe this is enough for your needs, but it'\''s not very much.\n\n' "$orange" "$default" "$flac_padding" >&2
[[ "$flac_padding" -gt "8192" ]] && printf '%sWARNING%s: Using more than 8KB for FLAC padding. Maybe your needs require that much, but it'\''s quite a lot.\n\n' "$orange" "$default" >&2
fi
else
printf '%sERROR%s: Argument for '\''-p / --padding / flac_padding'\'' must be zero or a positive integer.\n\n' "$red" "$default" >&2
exit 1
fi
# validate sox verbosity
[[ "$sox_verbosity_level" == [01234] ]] || { printf '%sERROR%s: Argument for '\''-v / --sox-verbosity / sox_verbosity_level'\'' must be an integer between 0 and 4.\n\n' "$red" "$default" >&2 ;exit 1 ; }
# ctrl+c exits script, not just sox/whatever
trap "exit 1" INT
# dependencies
#[[ "$PATH" != *"/usr/local/bin"* ]] && PATH=$PATH:/usr/local/bin # Automator defaults to ignoring Homebrew
[[ "$PATH" != *"$homebrew_bin"* ]] && PATH=$PATH:"$homebrew_bin" # Automator defaults to ignoring Homebrew
command -v basename >/dev/null 2>&1 || { printf '%sERROR%s: '\''basename'\'' not found, aborting.\n\n' "$red" "$default" >&2 ; exit 1 ; }
command -v dirname >/dev/null 2>&1 || { printf '%sERROR%s: '\''dirname'\'' not found, aborting.\n\n' "$red" "$default" >&2 ; exit 1 ; }
command -v file >/dev/null 2>&1 || { printf '%sERROR%s: '\''file'\'' (the program) not found, aborting.\n\n' "$red" "$default" >&2 ; exit 1 ; }
if [[ "$flac_padding" != "0" || "$use_SOXSOURCE_tag" == "1" || "$use_SOXCOMMAND_tag" == "1" ]] ;then
command -v metaflac >/dev/null 2>&1 || { printf '%sERROR%s: '\''metaflac'\'' not found, aborting.\n\n' "$red" "$default" >&2 ; exit 1 ; }
fi
command -v sox >/dev/null 2>&1 || { printf '%sERROR%s: '\''sox'\'' not found, aborting.\n\n' "$red" "$default" >&2 ; exit 1 ; }
# recommends
if command -v nproc >/dev/null 2>&1 ;then nproc="nproc" ;elif command -v gnproc >/dev/null 2>&1 ;then nproc="gnproc" ;else nproc="no_nproc" ;fi
if command -v parallel >/dev/null 2>&1 ;then
parallel_help="$( parallel -h )" # or maybe we should $( file --mime-type ) = perl then_gnu else_moreutils ?
if [[ "$parallel_help" == *"GNU"* ]] ;then
our_parallel="GNU"
parallel_divider=":::"
parallel_citation="--will-cite"
[[ "$threads_unused" -gt "0" ]] && paralleljobs[0]="-j -$threads_unused"
[[ "$use_progress_bar" == "1" ]] && parallel_progress_bar[0]="--bar"
elif [[ "$parallel_help" == *"parallel [OPTIONS] command"* ]] ;then
our_parallel="moreutils"
parallel_divider="--"
if [[ "$threads_unused" -gt "0" ]] ;then
if [[ "$nproc" == "no_nproc" ]] ;then
printf '%sERROR%s: '\''nproc'\'' not found, required to use moreutils parallel with threads_unused option -- try threads_used instead.\n\n' "$orange" "$default" >&2
paralleljobs[0]=""
else
paralleljobs[0]="-j $( "$nproc" --ignore="$threads_unused" )"
fi
fi
else
printf '%sERROR%s: '\''parallel'\'' exists but is not recognized, %srunning in single-threaded mode%s.\n\n' "$orange" "$default" "$orange" "$default" >&2
our_parallel="no_parallel"
fi
else
printf '%sWARNING%s: No '\''parallel'\'' found, %srunning in single-threaded mode%s.\n\n' "$orange" "$default" "$orange" "$default"
our_parallel="no_parallel"
fi
if [[ "$our_parallel" != "no_parallel" ]] ;then
[[ "$threads_unused" -gt "0" && "$threads_used" -gt "0" ]] && {
printf '%sERROR%s: the options '\''threads_used'\'' and '\''threads_unused'\'' are mutually exclusive, %srunning in single-threaded mode%s.\n\n' "$orange" "$default" "$orange" "$default" >&2
our_parallel="no_parallel" ; }
[[ "$threads_used" -gt "0" ]] && paralleljobs[0]="-j $threads_used"
fi
#if command -v realpath >/dev/null 2>&1 ;then realpath="realpath" ;elif command -v grealpath >/dev/null 2>&1 ;then realpath="grealpath"
#else
realpath="absolute_path"
#fi
# get all the items, including folder and subfolder contents, from command line arguments
user_args=( "$@" )
for arg in "${user_args[@]}" ; do
if [[ -d "$arg" ]] ; then
user_args+=( "$arg"/* )
for subdir in "$arg"/* ; do
if [[ -d "$subdir" ]] ; then
user_args+=( "$subdir"/* )
fi
done
fi
done
printf 'Reading input file(s). '
# just the flacs, just the 24 bit flacs
for flacfile in "${user_args[@]}" ;do
[[ "$( file -b --mime-type "$flacfile" )" == "audio/"*"flac" ]] && [[ "$( sox --i -b "$flacfile" )" -eq "24" ]] && absolute_flac_names+=( "$( "$realpath" "$flacfile" )" )
done
# candidate flacs must exist
[[ "${#absolute_flac_names[@]}" -ge "1" ]] || { printf '%sERROR%s: No candidate FLAC files found, aborting.\n\n' "$red" "$default" >&2 ; exit 1 ; }
# source data
for index in "${!absolute_flac_names[@]}" ;do
flac_filenames[$index]="$( basename "${absolute_flac_names[$index]}" )"
absolute_flac_dirs[$index]="$( dirname "${absolute_flac_names[$index]}" )"
flac_sample_rates[$index]="$( sox --i -r "${absolute_flac_names[$index]}" )"
done
printf 'Found %s candidate FLAC file(s). Configuring output. ' "${#absolute_flac_names[@]}"
# target data
for index in "${!absolute_flac_names[@]}" ;do
# 24/44 and 24/48 --> 16/44 and 16/48
if [[ "$use_24_44_and_24_48_input" == "1" ]] ;then
[[ "${flac_sample_rates[$index]}" -eq "44100" || "${flac_sample_rates[$index]}" -eq "48000" ]] && {
target_bit_depths[$index]="16"
target_sample_rates[$index]=""
target_folders[$index]="${absolute_flac_dirs[$index]}/unresampled-16bit"
target_flacs[$index]="${target_folders[$index]}/${flac_filenames[$index]}" ; }
fi
# 24/176 and 24/192 --> 24/88 and 24/96
if [[ "$use_24_88_and_24_96_output" == "1" ]] ;then
if [[ "${flac_sample_rates[$index]}" -eq "176400" || "${flac_sample_rates[$index]}" -eq "192000" ]] ;then
[[ "${flac_sample_rates[$index]}" -eq "176400" ]] && {
target_sample_rates[$index]="${sox_rate[0]} 88200"
target_folders[$index]="${absolute_flac_dirs[$index]}/resampled-24-88" ; }
[[ "${flac_sample_rates[$index]}" -eq "192000" ]] && {
target_sample_rates[$index]="${sox_rate[0]} 96000"
target_folders[$index]="${absolute_flac_dirs[$index]}/resampled-24-96" ; }
target_bit_depths[$index]="24"
target_flacs[$index]="${target_folders[$index]}/${flac_filenames[$index]}"
# 24/88 and 24/96 --> 16/44 and 16/48
elif [[ "${flac_sample_rates[$index]}" -eq "88200" || "${flac_sample_rates[$index]}" -eq "96000" ]] ;then
[[ "${flac_sample_rates[$index]}" -eq "88200" ]] && {
target_sample_rates[$index]="${sox_rate[0]} 44100"
target_folders[$index]="${absolute_flac_dirs[$index]}/resampled-16-44" ; }
[[ "${flac_sample_rates[$index]}" -eq "96000" ]] && {
target_sample_rates[$index]="${sox_rate[0]} 48000"
target_folders[$index]="${absolute_flac_dirs[$index]}/resampled-16-48" ; }
target_bit_depths[$index]="16"
target_flacs[$index]="${target_folders[$index]}/${flac_filenames[$index]}"
fi
else
# 24/{88,96,176,192} --> 16/$common_multiple
if [[ "${flac_sample_rates[$index]}" -eq "88200" || "${flac_sample_rates[$index]}" -eq "96000" ]] ||
[[ "${flac_sample_rates[$index]}" -eq "176400" || "${flac_sample_rates[$index]}" -eq "192000" ]] ;then
[[ "${flac_sample_rates[$index]}" -eq "88200" || "${flac_sample_rates[$index]}" -eq "176400" ]] && {
target_sample_rates[$index]="${sox_rate[0]} 44100"
target_folders[$index]="${absolute_flac_dirs[$index]}/resampled-16-44" ; }
[[ "${flac_sample_rates[$index]}" -eq "96000" || "${flac_sample_rates[$index]}" -eq "192000" ]] && {
target_sample_rates[$index]="${sox_rate[0]} 48000"
target_folders[$index]="${absolute_flac_dirs[$index]}/resampled-16-48" ; }
target_bit_depths[$index]="16"
target_flacs[$index]="${target_folders[$index]}/${flac_filenames[$index]}"
fi
fi
done
# target flacs must exist
[[ "${#target_flacs[@]}" -ge "1" ]] || { printf '%sERROR%s: No targets for conversion, aborting.\n\n' "$red" "$default" >&2 ; exit 1 ; }
# construct lists of sox and metaflac commands
for index in "${!target_flacs[@]}" ;do
[[ ! -d "${target_folders[$index]}" ]] && mkdir "${target_folders[$index]}"
cmdlistsox[$index]="sox -V$sox_verbosity_level \"${absolute_flac_names[$index]}\" -G -b ${target_bit_depths[$index]} \"${target_flacs[$index]}\" ${target_sample_rates[$index]} ${sox_dither[0]}"
#cmdlistsox[$index]="sox -V$sox_verbosity_level \"${absolute_flac_names[$index]}\" -G -b ${target_bit_depths[$index]} \"${target_flacs[$index]}\" ${target_sample_rates[$index]} ${sox_dither[0]} 2>\"$PWD\"/sox-stderr-\"$index\".txt"
cmdlistmfsc[$index]="metaflac --set-tag=SOXCOMMAND=\"sox input.flac -G -b ${target_bit_depths[$index]} output.flac ${target_sample_rates[$index]} ${sox_dither[0]}\" \"${target_flacs[$index]}\""
cmdlistmfss[$index]="metaflac --set-tag=SOXSOURCE=\"24 bit, \"${flac_sample_rates[$index]}\" Hz\" \"${target_flacs[$index]}\""
done
# run command lists, if possible using multiple threads
if [[ "$our_parallel" == "no_parallel" ]] ;then
printf 'Converting %s target(s) with SoX.\n\n' "${#target_flacs[@]}"
for index in "${!cmdlistsox[@]}" ;do
printf 'Converting %s.\n' "${target_flacs[$index]}"
if eval "${cmdlistsox[$index]}" ;then
printf ' %sSuccess%s! ' "$green" "$default"
[[ "$flac_padding" != "0" ]] && { printf 'Adding padding... ' ;metaflac --add-padding="$flac_padding" "${target_flacs[$index]}" || printf '%sERROR%s: Failure adding padding to %s.\n ' "$red" "$default" "${target_flacs[$index]}" >&2 ; }
[[ "$use_SOXCOMMAND_tag" == "1" ]] && { printf 'Adding SoX command tag... ' ;eval "${cmdlistmfsc[$index]}" || printf '%sERROR%s: Failure adding SOXCOMMAND tag to %s.\n ' "$red" "$default" "${target_flacs[$index]}" >&2 ; }
[[ "$use_SOXSOURCE_tag" == "1" ]] && { printf 'Adding SoX source tag... ' ;eval "${cmdlistmfss[$index]}" || printf '%sERROR%s: Failure adding SOXSOURCE tag to %s.\n ' "$red" "$default" "${target_flacs[$index]}" >&2 ; }
printf '%sDone%s!\n\n' "$green" "$default"
else
printf '%sERROR%s: SoX had non-zero exit status converting %s, aborting follow-up tasks for this file.\n\n' "$red" "$default" "${target_flacs[$index]}" >&2
fi
done
else
printf 'Converting %s target(s) with SoX and %s%s%s Parallel.\n' "${#target_flacs[@]}" "$orange" "$our_parallel" "$default"
if parallel ${parallel_citation[0]} ${parallel_progress_bar[0]} ${paralleljobs[0]} "${parallel_divider[0]}" "${cmdlistsox[@]}" ;then
printf '%sSuccess%s! ' "$green" "$default"
[[ "$flac_padding" != "0" ]] && { printf 'Adding padding... ' ;parallel ${parallel_citation[0]} ${paralleljobs[0]} metaflac --add-padding="$flac_padding" "${parallel_divider[0]}" "${target_flacs[@]}" || printf '%sERROR%s: Failure adding padding to file(s).\n' "$red" "$default" >&2 ; }
[[ "$use_SOXCOMMAND_tag" == "1" ]] && { printf 'Adding SoX command tag... ' ;parallel ${parallel_citation[0]} ${paralleljobs[0]} "${parallel_divider[0]}" "${cmdlistmfsc[@]}" || printf '%sERROR%s: Failure adding SOXCOMMAND tag(s).\n' "$red" "$default" >&2 ; }
[[ "$use_SOXSOURCE_tag" == "1" ]] && { printf 'Adding SoX source tag... ' ;parallel ${parallel_citation[0]} ${paralleljobs[0]} "${parallel_divider[0]}" "${cmdlistmfss[@]}" || printf '%sERROR%s: Failure adding SOXSOURCE tag(s).\n' "$red" "$default" >&2 ; }
printf '%sDone%s!\n\n' "$green" "$default"
else
printf '%sERROR%s: Parallel/SoX had non-zero exit status, aborting follow-up tasks.\n\n' "$red" "$default" >&2
exit 1
fi
fi
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment