Find CDN failures

fix_bad_cdn_utf8.sh

##
# Find CDN output that has invalid UTF8 byte sequences.
# Fix them.
#
# NB: This requires an iconv that supports --byte-subst
#

FAILURES=utf8_failures
FAILURE_STRING=__WUBWUB__

echo "Finding failures"
trap "rm $FAILURES; exit" SIGHUP SIGINT SIGTERM

rm $FAILURES

# Find duff files by attempting to convert.
# Take stderr failures and pipe them to ./$FAILURES
for f in cdn*.tsv
do
  (iconv -t UTF8 $f 2>&1 >/dev/null | grep -Po "cdn.*tsv") >> $FAILURES
done

echo "Fixing failures"
# Look at the failures and substitute a failure string
# Discard lines with failures
for f in `cat $FAILURES`
do
  iconv --byte-subst=$FAILURE_STRING -f UTF8 -t UTF8 $f > "$f.new"
  cat "$f.new" | (mawk "\$0 !~ /$FAILURE_STRING/" > $f) && rm "$f.new"
done

rm $FAILURES