yar_log_dump.sh

#!/bin/bash
set -x

LOCAL_DIR=/home/hadoop/yarn_logs
mkdir $LOCAL_DIR

for row in $( yarn application -appStates ALL -list | awk '/application_/{print $1 "," $2}' )
do
    fields=($(echo $row | tr "," "\n"))
    yarn logs -applicationId ${fields[0]} > $LOCAL_DIR/yarn_${fields[0]}_${fields[1]}.log
done
if [ -z "$(ls -A $LOCAL_DIR)" ]; then
    echo "($LOCAL_DIR) is empty"
else
    aws s3 cp $LOCAL_DIR/*.log s3://out/
fi