Sometimes it takes more than one run. This script runs puppet until it is idempotent or returns an error.

puppet_until_idempotent.sh

#!/bin/bash

[ "$PT_noop" = "true" ] && NOOP_FLAG="--noop" || unset NOOP_FLAG

puppet_command="/opt/puppetlabs/bin/puppet agent --onetime --verbose --no-daemonize --no-splay --no-usecacheonfailure --no-use_cached_catalog $NOOP_FLAG"

# Sometimes it takes more than one run ...
# Retries until idempotent or error.
#
# Waits for up to five minutes for an in-progress puppet run to complete.
# Regarding stale locks, see also: https://github.com/tkishel/unlock_puppet/blob/master/tasks/init.rb

execute_puppet_until_idempotent() {
  puppet_command=$1
  maximum_number_of_puppet_runs=10
  sleep_seconds_when_puppet_run_in_progress=30
  retry_when_success_with_failures=false

  for run in $(seq 1 $maximum_number_of_puppet_runs); do
    echo "Executing: '$puppet_command'"

    # To both capture and output command output ...
    exec 5>&1
    puppet_command_output=$($puppet_command | tee /dev/fd/5; exit "${PIPESTATUS[0]}")
    puppet_exit_code=$?

    if echo "$puppet_command_output" | grep -q "Run of Puppet configuration client already in progress"; then
      puppet_run_in_progress=true
    else
      puppet_run_in_progress=false
    fi

    # 0: The run succeeded with no changes or failures; the system was already in the desired state
    if [ $puppet_exit_code -eq 0 ]; then
      break
    fi

    # 1: The run failed, or wasn't attempted due to another run already in progress.
    if [ $puppet_exit_code -eq 1 ]; then
      if [ "$puppet_run_in_progress" = true ]; then
        echo "Sleeping $sleep_seconds_when_puppet_run_in_progress seconds while waiting for another puppet run already in progress"
        sleep $sleep_seconds_when_puppet_run_in_progress
        continue
      else
        echo "ERROR: '$puppet_command' failed with exit code: $puppet_exit_code"
        exit 1
      fi
    fi

    # 2: The run succeeded, and some resources were changed.
    if [ $puppet_exit_code -eq 2 ]; then
      continue
    fi

    # 4: The run succeeded, and some resources failed.
    if [ $puppet_exit_code -eq 4 ]; then
      if [ "$retry_when_success_with_failures" = true ]; then
        continue
      else
        echo "ERROR: '$puppet_command' failed with exit code: $puppet_exit_code"
        exit 1
      fi
    fi

    # 6: The run succeeded, and included both changes and failures.
    if [ $puppet_exit_code -eq 6 ]; then
      if [ "$retry_when_success_with_failures" = true ] ; then
        continue
      else
        echo "ERROR: '$puppet_command' failed with exit code: $puppet_exit_code"
        exit 1
      fi
    fi
  done
}

execute_puppet_until_idempotent "$puppet_command"
exit 0

This may be useful as an alternative to https://github.com/puppetlabs/puppetlabs-peadm/blob/master/tasks/puppet_runonce.sh