Run a zrepl job and block/wait until it's finished.

zrepl-run-and-wait-for-job-with-progress.sh

#!/bin/bash
#
# Author: Riyad Preukschas <[email protected]>
# License: Mozilla Public License 2.0
#
# Run a zrepl job and block/wait until it's finished (with progress indication).
#
# Note: this is basically a hack until there's a properly integrated solution for this.
# see https://github.com/zrepl/zrepl/issues/427

set -o nounset  # complain when reading unset vars

run-and-wait-for-zrepl-job-with-progress() {
  zrepl_job_result=''  # result
  zrepl_error=''  # result
  local zrepl_job="$1"

  echo "Running job ${zrepl_job} ..."

  zrepl signal wakeup "${zrepl_job}"
  local bytes_replicated_last=0
  local bps_avg_last=0
  local delta_t=1  # seconds between loop iterations
  echo "NOTE: not all steps can be size-estimated, progress estimates may be imprecise."
  while :
  do
    sleep "${delta_t}"

    # The magic: zrepl status -> jq *something* -> bash eval()
    # Everything comes together in the last line of the jq script. We use jq's text interpolation
    # to define shell variables whose values are calculated by the functions defined before.
    # The interpolated shell string is then eval()ed, defining/overwriting variables for the next
    # loop iteration and printing the current progress.
    #
    # Output variables:
    # zrepl_job_result: the state of all the stages unified into a single array
    # bytes_replicated_last: bytes replicated until now
    # bps_avg_last: bytes per second (weighted average)
    #
    # Notes:
    # bytes_to_human_* adapted from https://stackoverflow.com/questions/10420352/converting-file-size-in-bytes-to-human-readable-string/20732091#20732091
    # LC_NUMERIC=C prevents errors like "printf: 174.39417266845703: invalid number" with locales where decimal separator is ","
    eval "$( \
      zrepl status --mode raw \
        | ZREPL_JOB="${zrepl_job}" DELTA_T="${delta_t}" BYTES_REPLICATED_LAST="${bytes_replicated_last}" BPS_AVG_LAST="${bps_avg_last}" jq -c -r '
          def job_results: .Jobs[$ENV.ZREPL_JOB].push | [.PruningReceiver.State?,.PruningSender.State?,.Replication.Attempts[-1].State] | map(tostring | ascii_downcase);
          def steps_total: .Jobs[$ENV.ZREPL_JOB].push | [.Replication.Attempts[].Filesystems[].Steps[]] | length;
          def steps_replicated: .Jobs[$ENV.ZREPL_JOB].push | [.Replication.Attempts[].Filesystems[].CurrentStep] | reduce .[] as $item (0; . + $item);
          def bytes_expected: .Jobs[$ENV.ZREPL_JOB].push | [.Replication.Attempts[].Filesystems[].Steps[].Info.BytesExpected] | reduce .[] as $item (0; . + $item);
          def bytes_replicated: .Jobs[$ENV.ZREPL_JOB].push | [.Replication.Attempts[].Filesystems[].Steps[].Info.BytesReplicated] | reduce .[] as $item (0; . + $item);
          def bps_avg($bytes_replicated): (($bytes_replicated - ($ENV.BYTES_REPLICATED_LAST | tonumber)) / ($ENV.DELTA_T | tonumber)) as $rate | 0.5 as $weight | (1-$weight)*($ENV.BPS_AVG_LAST | tonumber) + $weight*$rate;
          def bytes_to_human_number: if . == 0 then 0 else ((. | log2) / (1024 | log2) | floor) as $i | . / pow(1024; $i) end;
          def bytes_to_human_unit: if . == 0 then "B" else ((. | log2) / (1024 | log2) | floor) as $i | ["B", "KiB", "MiB", "GiB", "TiB", "PiB"][$i] end;
          @sh "zrepl_job_result=\(job_results | tojson) bytes_replicated_last=\(bytes_replicated) bps_avg_last=\(bps_avg(bytes_replicated)); LC_NUMERIC=C printf \"  Progress: %i / %i snapshots (estimated: %.1f %s / %.1f %s @ %.1f %s/s) ...         \\r\" \(steps_replicated) \(steps_total) \"\(bytes_replicated | bytes_to_human_number)\" \(bytes_replicated | bytes_to_human_unit) \"\(bytes_expected | bytes_to_human_number)\" \(bytes_expected | bytes_to_human_unit) \"\(bps_avg(bytes_replicated) | bytes_to_human_number)\" \(bps_avg(bytes_replicated) | bytes_to_human_unit)"
        ' \
    )"

    # error? .State in ["execerr", "filesystem-error", "planning-error", "planerr"]
    if [[ "${zrepl_job_result}" = *err* ]]; then
    #if [[ $(jq 'any(. == "error")' <<< "${zrepl_job_result}") = "true" ]]; then
      zrepl_error=1
      break
    fi

    # all done
    [[ $(jq 'all(. == "done")' <<< "${zrepl_job_result}") = "true" ]] && break
  done
  echo # keep last progress string

  if [[ -z "${zrepl_error}" ]]; then
    echo "Running job ${zrepl_job} ... done"
  else
    echo "Error encountered while processing ${zrepl_job} job."
    echo "Run the following command for details: zrepl status --job ${zrepl_job}"
    echo "Running job ${zrepl_job} ... failed"
  fi
}

run-and-wait-for-zrepl-job-with-progress "$@"

zrepl-run-and-wait-for-job.sh

#!/bin/bash
#
# Author: Riyad Preukschas <[email protected]>
# License: Mozilla Public License 2.0
#
# Run a zrepl job and block/wait until it's finished (without progress indication).
#
# Note: this is basically a hack until there's a properly integrated solution for this.
# see https://github.com/zrepl/zrepl/issues/427

set -o nounset  # complain when reading unset vars

run-and-wait-for-zrepl-job() {
  zrepl_job_result=''  # result
  zrepl_error=''  # result
  local zrepl_job="$1"

  echo "Running job ${zrepl_job} ..."

  zrepl signal wakeup "${zrepl_job}"
  local delta_t=5  # seconds between loop iterations
  echo "NOTE: not all steps can be size-estimated, progress estimates may be imprecise."
  while :
  do
    sleep "${delta_t}"

    # pluck the state of all the stages and unify them into a single array
    zrepl_job_result="$( \
      zrepl status --mode raw \
        | ZREPL_JOB="${zrepl_job}" jq -c -r '
          .Jobs[$ENV.ZREPL_JOB].push | [.PruningReceiver.State?,.PruningSender.State?,.Replication.Attempts[-1].State] | map(tostring | ascii_downcase)
        ' \
    )"

    # error? .State in ["execerr", "filesystem-error", "planning-error", "planerr"]
    if [[ "${zrepl_job_result}" = *err* ]]; then
    #if [[ $(jq 'any(. == "error")' <<< "${zrepl_job_result}") = "true" ]]; then
      zrepl_error=1
      break
    fi

    # all done
    [[ $(jq 'all(. == "done")' <<< "${zrepl_job_result}") = "true" ]] && break
  done

  if [[ -z "${zrepl_error}" ]]; then
    echo "Running job ${zrepl_job} ... done"
  else
    echo "Error encountered while processing ${zrepl_job} job."
    echo "Run the following command for details: zrepl status --job ${zrepl_job}"
    echo "Running job ${zrepl_job} ... failed"
  fi
}

run-and-wait-for-zrepl-job "$@"