Last active
December 21, 2018 19:16
-
-
Save zish/9fa639c134e9e29474f9cc2b3a532319 to your computer and use it in GitHub Desktop.
Bash multi forking (not multithreading). Crude way of creating a limited number of child processes, while allowing to perform a set number of tasks.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
func () { | |
#-- These will make sense shortly. | |
local TMP_NAME="${1}-${2}" | |
local TMP_FILENAME="/tmp/Xout-${TMP_NAME}.tmp" | |
#-- We just assume that if it exists, it's there from a previous run. | |
# Se we just removw it. | |
if [ -e ${TMP_FILENAME} ]; then /bin/rm -f ${TMP_FILENAME}; fi | |
#-- Xin is some array generated elsewhere in the code | |
for nn in $(seq 0 $((( ${#Xin[@]} - 1 )))); do | |
#-- Output the values to a unique TMP file, where we can retrieve it later. | |
echo -e "Xout1[${TMP_NAME}-$nn]=\"$(foo1 "${Xin[$nn]}")\"" >> ${TMP_FILENAME} | |
echo -e "Xout2[${TMP_NAME}-$nn]=\"$(foo2 "${Xin[$nn]}")\"" >> ${TMP_FILENAME} | |
done | |
} | |
#-- This is the number of total tasks to run. | |
MAX_TASKS=80 | |
#-- This is the maximum number of child processes to execute per-iteration. | |
# It's to make sure we don't overload the system. | |
MAX_CHILDREN=4 | |
#-- Loop through all iterations. | |
# Each one generates a unique temp file. We retrieve the contents later. | |
for N1 in $(seq 1 $(( ${MAX_TASKS}/ ${MAX_CHILDREN} ))); do | |
for N2 in $(seq 1 ${MAX_CHILDREN}); do | |
#-- Start up to MAX_CHILDREN child 'func' calls. | |
# Don't fork the child, if we are running the max amount for this iteration. | |
# This ensures that we don't overload our system. | |
if [ ${N2} -lt ${MAX_CHILDREN} ]; then | |
func ${N1} ${N2} & | |
else | |
echo "[Run time ${SECONDS} seconds] Max children per-iteration reached. Not forking." 1>&2 | |
func ${N1} ${N2} | |
fi | |
done | |
done | |
#-- Retrieve the saved data from the temp files. | |
declare -A Xout1 Xout2 | |
for TMP_FILENAME in /tmp/Xout-*.tmp; do | |
eval "$(cat ${TMP_FILENAME})" | |
done | |
#-- This should show you that everything worked :) | |
declare -p Xout1 | |
declare -p Xout2 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment