Git Hooks and associated scripts for checking puppet commits

check_erb.sh

#!/bin/bash
#Script to test puppet files have valid syntax.
#Intended for use with hudson/jenkins.

set -e
set -u

fail=0
#TODO: Run these in parallel - we have 4 cores.
#TODO: Control the environment (through the config dir?).
#      We want to parse for all environments.
#      Is this being done, contrary to puppet report?
#TODO: Even with --ignoreimport, some may be pulling in others,
#      meaning we're checking multiple times.
all_files=`find -name "*.pp" -o -name "*.erb"`
num_files=`echo $all_files | wc -w`
if [[ $num_files -eq "0" ]]; then
  echo "ERROR: no .pp or .erb files found"
  exit 1
fi
echo "Checking $num_files *.pp and *.erb files for syntax errors."
echo "Puppet version is: `puppet --version`"

for x in $all_files; do
  set +e
  case $x in
  *.pp )
    puppet --parseonly --ignoreimport --color=false $x ;;
  *.erb )
    cat $x | erb -x -T - | ruby -c > /dev/null ;;
  esac
  rc=$?
  set -e
  if [[ $rc -ne 0 ]] ; then
    fail=1
    echo "ERROR in $x (see above)"
  fi
done

if [[ $fail -ne 0 ]] ; then
  echo "FAIL: at least one file failed syntax check."
else
  echo "SUCCESS: all .pp and *.erb files pass syntax check."
fi
exit $fail

fix_whitespace.sh

#!/bin/sh
# Pre-commit hook for git which removes trailing whitespace, converts tabs to spaces, and enforces a max line length.

if git-rev-parse --verify HEAD >/dev/null 2>&1 ; then
   against=HEAD
else
   # Initial commit: diff against an empty tree object
   against=4b825dc642cb6eb9a060e54bf8d69288fbee4904
fi

files_with_whitespace=`git diff-index --check --cached $against | # Find all changed files
                       sed '/^[+-]/d' |                           # Remove lines which start with + or -
                       sed -E 's/:[0-9]+:.*//' |                  # Remove end of lines which contains numbers, etc.
                       sed '/Generated/d' |                       # Ignore generated files
                       sed '/Libraries/d' |                       # Ignore libraries
                       sed '/\.[mh]\$/!d' |                       # Only process .m and .h files
                       uniq`                                      # Remove duplicate files

# Change field separator to newline so that for correctly iterates over lines
IFS=$'\n'

# Find files with trailing whitespace
for FILE in $files_with_whitespace ; do
    echo "Fixing whitespace in $FILE" >&2
    # Replace tabs with four spaces
    sed -i "" $'s/\t/    /g' "$FILE"
    # Strip trailing whitespace
    sed -i '' -E 's/[[:space:]]*$//' "$FILE"
    git add "$FILE"
done

# Detect too long lines in .m and .h files:
changed_source_files=`git diff-index --cached $against --numstat |
                      cut -f3 |
                      egrep '\.[hm]$'`

if [[ -n "$changed_source_files" ]]; then
    found_offenses=''

    for file in $changed_source_files ; do
        too_long=`git diff-index --cached -p $against -- "$file" |
                  egrep '^\+.{117,}' |
                  sed -E 's/^\+//'`

        if [[ -n "$too_long" ]]; then
            found_offenses=YES
            printf "\n$file:\n%s\n" "$too_long" >&2
        fi
    done

    if [[ -n $found_offenses ]]; then
        echo "\nAborting commit because you added lines longer than 116 chars." >&2
        exit 1
    fi
fi

pre-commit.sh

#!/bin/bash

# External requirements:
#
#    * Git
#    * sed
#    * Ruby with ERB and YAML support
#    * Puppet >= 2.7
#    * puppet-lint
#    * r10k
#
#    Adjust LINTFLAGS as appropriate

# Redirect output to stderr.
exec 1>&2

PUPPETLINT_FLAGS=${PUPPETLINT_FLAGS:-'--no-autoloader_layout-check --no-80chars-check'}
TMPDIR=${TMPDIR:-'/tmp'}
TMPFILE=$(mktemp "${TMPDIR}"/tmp.XXXXXXXXXX)
TMPR10K=$(mktemp -d "${TMPDIR}"/tmp.XXXXXXXXXX)
STATUS=0

# Register exit trap for removing temporary files
trap 'rm -rf $TMPFILE $TMPR10K' EXIT INT HUP

# Check for ruby binary
which ruby >/dev/null 2>&1 || exit 1

# Check for Puppet binary
which puppet >/dev/null 2>&1 || exit 1

# Check for puppet-lint
which puppet-lint >/dev/null 2>&1 || exit 1

# Check for erb
which erb >/dev/null 2>&1 || exit 1

# Get correct git revision
if git rev-parse --quiet --verify HEAD > /dev/null
then
    revision=HEAD
else
    # Initial commit: diff against an empty tree object
    revision=4b825dc642cb6eb9a060e54bf8d69288fbee4904
fi

IFS="
 "

# Get a list of files changed in this transaction
declare -a FILES
FILES=$(git diff --cached --name-only --diff-filter=ACM "${revision}")

for file in ${FILES[@]}
do
    # Don't check empty files
    if [[ $(git cat-file -s ":0:${file}") -eq 0 ]]; then
        continue
    fi

    extension="${file##*.}"
    git cat-file blob ":0:${file}" > $TMPFILE

    if [[ $? -ne 0 ]]; then
        echo "Unable to checkout ${file}"
        STATUS=2
    else
        case $extension in
            pp)
                # Remove import lines while parsing
                # http://projects.puppetlabs.com/issues/9670#note-14
                sed -i -e '/^import / d' $TMPFILE >/dev/null 2>&1
                # Puppet syntax check
                puppet parser validate $TMPFILE >/dev/null 2>&1
                if [[ $? -ne 0 ]]; then
                    echo "Puppet syntax error in ${file}. Run 'puppet parser validate ${file}'" >&2
                    STATUS=2
                fi

                # puppet-lint check
                puppet-lint $PUPPETLINT_FLAGS --log-format "${file}:%{linenumber} %{KIND} - %{message}" $TMPFILE 2> /dev/null
                if [[ $? -ne 0 ]] ; then
                    STATUS=2
                fi
            ;;

            erb)
                # syntax check templates - this doesn't catch a lot of mistakes,
                # but it should catch gross mistakes
                erb -x -T - "${TMPFILE}" | ruby -c >/dev/null 2>&1
                if [[ $? -ne 0 ]]; then
                    echo "ERB syntax error in ${file}" >&2
                    STATUS=2
                fi
            ;;
            yml|yaml)
                # syntax YAML files, https://ttboj.wordpress.com/2013/08/25/finding-yaml-errors-in-puppet/
                ruby -ryaml -e "YAML.parse(File.open('${TMPFILE}'))" >/dev/null 2>&1
                if [[ $? -ne 0 ]]; then
                    echo "YAML syntax error in ${file}" >&2
                    STATUS=2
                fi
            ;;
            json)
                # syntax YAML files, https://ttboj.wordpress.com/2013/08/25/finding-yaml-errors-in-puppet/
                ruby -rjson -e "JSON.parse(File.open('${TMPFILE}').read)" >/dev/null 2>&1
                if [[ $? -ne 0 ]]; then
                    echo "JSON syntax error in ${file}" >&2
                    STATUS=2
                fi
            ;;
        esac

        # r10k Puppetfile syntax
        which r10k >/dev/null 2>&1
        if [[ $? -eq 0 ]]; then
            if [[ "$(basename $file)" == 'Puppetfile' ]]; then
                cp "${TMPFILE}" "${TMPR10K}/Puppetfile"
                ( cd "${TMPR10K}" && r10k puppetfile check ) 2> "${TMPFILE}"
                if [[ $? -ne 0 ]]; then
                    echo "r10k syntax error in ${file}" >&2
                    sed "s,${TMPR10K}/Puppetfile,${file}," >&2 < "${TMPFILE}"
                    STATUS=2
                fi
            fi
        fi
    fi
done

exit $STATUS