Export Elasticsearch spark driver log messages

.1_readme.txt

# usage

nvm use .
npm install
chmod +x export-spark-driver-logs.sh

./export-spark-driver-logs.sh $ES_IP:$ES_PORT driver-##############-####

.gitignore

node_modules/
driver-*.json

.nvmrc

v8.11.4

export-filebeat-errors.sh

#!/usr/bin/env bash

ES_HOST=$1
SPARK_DRIVER=$2

if [ -z "$ES_HOST" ]; then echo "ERROR: ES_HOST required."; exit 1; fi

DATE_STRING=$(date '+%Y%m%d%H%M')

read -r -d '' JSON_DATA <<EOF
{
  "query": {
    "bool": {
      "must": [
        {
          "match": {
            "message": "error"
          }
        }
      ]
    }
  },
  "sort": [
    {
      "@timestamp": {
        "order": "asc"
      }
    }
  ]
}
EOF

OUTPUT_FILE="${DATE_STRING}.json"

if [ -f $OUTPUT_FILE ]; then
  echo "Output file already exists, would you like to replace it? (y|n)"
  read RESPONSE
  if [ $RESPONSE = "y" ]; then
    rm $OUTPUT_FILE
  fi
fi

./node_modules/elasticdump/bin/elasticdump \
  --input="http://${ES_HOST}/filebeat-*" \
  --output="${OUTPUT_FILE}" \
  --searchBody "$(echo $JSON_DATA)" \
  --limit 10000

cat "${OUTPUT_FILE}" | jq '._source.message'

export-spark-driver-logs.sh

#!/usr/bin/env bash

ES_HOST=$1
SPARK_DRIVER=$2

if [ -z "$ES_HOST" ]; then echo "ERROR: ES_HOST required."; exit 1; fi
if [ -z "$SPARK_DRIVER" ]; then echo "ERROR: SPARK_DRIVER required."; exit 1; fi

read -r -d '' JSON_DATA <<EOF
{
  "query": {
    "bool": {
      "must": [
        {
          "term": {
            "executor": "$SPARK_DRIVER"
          }
        }
      ]
    }
  },
  "sort": [
    {
      "@timestamp": {
        "order": "asc"
      }
    }
  ]
}
EOF

OUTPUT_FILE="${SPARK_DRIVER}.json"

if [ -f $OUTPUT_FILE ]; then
  echo "Output file already exists, would you like to replace it? (y|n)"
  read RESPONSE
  if [ $RESPONSE = "y" ]; then
    rm $OUTPUT_FILE
  fi
fi

./node_modules/elasticdump/bin/elasticdump \
  --input="http://${ES_HOST}/filebeat-*" \
  --output="${OUTPUT_FILE}" \
  --searchBody "$(echo $JSON_DATA)"

cat "${OUTPUT_FILE}" | jq '._source.message'

package.json

{
  "name": "export-spark-driver-logs",
  "version": "1.0.0",
  "description": "",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "Eric London",
  "license": "ISC",
  "dependencies": {
    "elasticdump": "^3.3.19"
  }
}