Watch for termination of spot instance, SIGHUP passed process if so.

monitor.sh

#!/bin/bash
pid=$1

echo "Watching for spot instance termination; will SIGHUP $pid if so."

while kill -0 $pid >/dev/null 2>/dev/null; do
  if curl -s http://169.254.169.254/latest/meta-data/spot/termination-time | grep -q .*T.*Z; then
    echo 'oh noes! telling caffe to snapshot.'
    kill -SIGHUP $pid
  fi
  sleep 5;
done

echo "$pid ended, and we're still alive. Huzzah!"