containers.awk

FNR > 6 {
   print $1
}

yarn-get-container-logs.sh

#!/bin/bash
YARN_APP_ID=$1
if [ -z "${YARN_APP_ID}"]; then
  echo getlogs.sh [yarn application id]
  exit 1
fi

echo YARN Application ID is $1
if [ ! -d "${YARN_APP_ID}" ]; then
  echo "mkdir ${YARN_APP_ID}"
  mkdir ${YARN_APP_ID}
else
  echo Dir already exists: ${YARN_APP_ID}
fi

YARN_ATTEMPTS=${YARN_APP_ID}/attempt.txt
if [ ! -e "${YARN_ATTEMPTS}" ]; then
  #echo Listing attempts to ${YARN_ATTEMPTS}
  echo "yarn applicationattempt -list ${YARN_APP_ID} > ${YARN_ATTEMPTS}"
  yarn applicationattempt -list ${YARN_APP_ID} > ${YARN_ATTEMPTS}
else
  echo File already exists: ${YARN_ATTEMPTS}
fi

YARN_ATTEMPT_ID=$(cat ${YARN_ATTEMPTS} | tail -1 | awk '{ print $1}')
if [ -z "${YARN_ATTEMPT_ID}" ]; then
  echo "Unable to get YARN Attempt ID"
  exit 1
else
  echo "YARN Attempt ID is ${YARN_ATTEMPT_ID}"
fi

YARN_CONTAINERS=${YARN_APP_ID}/containers.txt
if [ ! -e "${YARN_CONTAINERS}" ]; then
  echo Listing containers to ${YARN_CONTAINERS}
  yarn container -list ${YARN_ATTEMPT_ID} > ${YARN_CONTAINERS}
else
  echo File already exists: ${YARN_CONTAINERS}
fi

YARN_CONTAINER_IDS=${YARN_APP_ID}/container_ids.txt
awk -f containers.awk ${YARN_CONTAINERS} | sort > ${YARN_CONTAINER_IDS}
while read p; do
  echo container is $p
  yarn logs -applicationId ${YARN_APP_ID} -containerId $p > $p.log 2>&1
done < ${YARN_CONTAINER_IDS}