parallel_gzip.bash

#!/bin/bash

GZIP="gzip --best --rsyncable --quiet"

for DEST in $* ; do
  [ ! -d "${DEST}" ] && exit -1
  echo Running \"${GZIP}\" on everything in \"${DEST}\"

  TRACKING="/tmp/${RANDOM}_parallel_gzip"
  mkdir -p "${TRACKING}"
  FIND="${TRACKING}/find"
  find "$DEST" -type f -not -name '*gz' > "$FIND"
  
  CPU_COUNT=$(lscpu -p | tail -1 | cut -f 1 -d ,)
  
  declare -a todo
  declare -a stat
  
  for (( c=0; c<=$CPU_COUNT; c++ )) ; do
    todo[$c]="${TRACKING}/todo_${c}"
    stat[$c]="${TRACKING}/stat_${c}"
    echo "CPU $c still running" > ${stat[$c]}
  done
  
  mapfile -t < "$FIND" found
  file_count=${#found[@]}
  
  echo "Planning to gzip $file_count files"
  
  for (( i=1; i<=$file_count; i++ )) ; do
    for (( cpu=0; cpu<=$CPU_COUNT; cpu++ )) ; do
      file_name=${found[$i]}
  
      if [ -s "$file_name" ] ; then
        /bin/echo -ne "$file_name\00" >> "${todo[$cpu]}"
        let i++
      fi
    done
  done
  
  for (( cpu=0; cpu<=$CPU_COUNT; cpu++ )) ; do
    echo "Spawning gzip for CPU $cpu"
    taskset -c $cpu xargs -r -0 --arg-file="${todo[$cpu]}" $GZIP && echo "$cpu done" > "${stat[$cpu]}" &
  done
  
  while [[ $(cat ${stat[@]} | grep -shc running) -gt 0 ]]; do
    cat ${stat[@]}
    sleep 10
  done
  
  echo Finished \"${DEST}\"
done