Read chunks from log file via bash

producer.sh

#!/bin/bash

set -e

cd "$(dirname "${BASH_SOURCE[0]}")"

do_exit() {
  local -r the_result=$1

  echo
  echo "log_file=${log_file}"
  echo "log_file_subscriber=${log_file_subscriber}"
  echo
  echo "Exit code is ${the_result}."
  exit "${the_result}"
}

exit_result=0
trap 'do_exit "$exit_result"' SIGINT SIGTERM EXIT

log_file="$(mktemp)"
exec > >(tee -i "${log_file}")
exec 2>&1

log_file_subscriber="$(mktemp)"
./subscriber.sh "${log_file}" >"${log_file_subscriber}" 2>&1 &

# Some pause (just testing) ...
sleep 3

# Generate a lot of output.
counter=0
while true; do
  echo "counter=${counter} // line 01 of 16"
  echo "counter=${counter} // line 02 of 16"
  echo "counter=${counter} // line 03 of 16"
  echo "counter=${counter} // line 04 of 16"
  echo "counter=${counter} // line 05 of 16"
  echo "counter=${counter} // line 06 of 16"
  echo "counter=${counter} // line 07 of 16"
  echo "counter=${counter} // line 08 of 16"
  echo "counter=${counter} // line 09 of 16"
  echo "counter=${counter} // line 10 of 16"
  echo "counter=${counter} // line 11 of 16"
  echo "counter=${counter} // line 12 of 16"
  echo "counter=${counter} // line 13 of 16"
  echo "counter=${counter} // line 14 of 16"
  echo "counter=${counter} // line 15 of 16"
  echo "counter=${counter} // line 16 of 16"

  counter=$((counter + 1))

  sleep 1
done

sample_output.txt

$ cat /tmp/tmp.JPirEkkcmg
PID=30218

Processing log chunk (16 lines):
****************************************
counter=0 // line 01 of 16
counter=0 // line 02 of 16
counter=0 // line 03 of 16
counter=0 // line 04 of 16
counter=0 // line 05 of 16
counter=0 // line 06 of 16
counter=0 // line 07 of 16
counter=0 // line 08 of 16
counter=0 // line 09 of 16
counter=0 // line 10 of 16
counter=0 // line 11 of 16
counter=0 // line 12 of 16
counter=0 // line 13 of 16
counter=0 // line 14 of 16
counter=0 // line 15 of 16
counter=0 // line 16 of 16
****************************************

Processing log chunk (16 lines):
****************************************
counter=1 // line 01 of 16
counter=1 // line 02 of 16
counter=1 // line 03 of 16
counter=1 // line 04 of 16
counter=1 // line 05 of 16
counter=1 // line 06 of 16
counter=1 // line 07 of 16
counter=1 // line 08 of 16
counter=1 // line 09 of 16
counter=1 // line 10 of 16
counter=1 // line 11 of 16
counter=1 // line 12 of 16
counter=1 // line 13 of 16
counter=1 // line 14 of 16
counter=1 // line 15 of 16
counter=1 // line 16 of 16
****************************************

Processing log chunk (16 lines):
****************************************
counter=2 // line 01 of 16
counter=2 // line 02 of 16
counter=2 // line 03 of 16
counter=2 // line 04 of 16
counter=2 // line 05 of 16
counter=2 // line 06 of 16
counter=2 // line 07 of 16
counter=2 // line 08 of 16
counter=2 // line 09 of 16
counter=2 // line 10 of 16
counter=2 // line 11 of 16
counter=2 // line 12 of 16
counter=2 // line 13 of 16
counter=2 // line 14 of 16
counter=2 // line 15 of 16
counter=2 // line 16 of 16
****************************************

Processing log chunk (16 lines):
****************************************
counter=3 // line 01 of 16
counter=3 // line 02 of 16
counter=3 // line 03 of 16
counter=3 // line 04 of 16
counter=3 // line 05 of 16
counter=3 // line 06 of 16
counter=3 // line 07 of 16
counter=3 // line 08 of 16
counter=3 // line 09 of 16
counter=3 // line 10 of 16
counter=3 // line 11 of 16
counter=3 // line 12 of 16
counter=3 // line 13 of 16
counter=3 // line 14 of 16
counter=3 // line 15 of 16
counter=3 // line 16 of 16
****************************************

Processing log chunk (10 lines):
****************************************

log_file=/tmp/tmp.nrbm7Hde9G
log_file_subscriber=/tmp/tmp.JPirEkkcmg

Exit code is 0.

log_file=/tmp/tmp.nrbm7Hde9G
log_file_subscriber=/tmp/tmp.JPirEkkcmg

Exit code is 0.
****************************************

subscriber.sh

#!/bin/bash

set -e

log_file=$1

echo "PID=$$"

sleep_duration=0.5

chunk_start=1
chunk_length=0
chunk_content=''

while true; do
  chunk_content=$(tail --lines="+${chunk_start}" "${log_file}")

  if [ "${chunk_content}" == '' ]; then
    # No new log messages were produced.
    sleep "${sleep_duration}"
    continue
  fi

  chunk_length=$(echo "${chunk_content}" | wc -l)
  chunk_start=$((chunk_start + chunk_length))

  # Do something more meaningful here.
  echo
  echo "Processing log chunk (${chunk_length} lines):"
  echo '****************************************'
  echo "${chunk_content}"
  echo '****************************************'

  sleep "${sleep_duration}"
done