esx.sh: glue together a bunch of shell programs to run queries on Elasticsearch

esx.sh

#!/bin/sh

set -e

NAME=$(basename "$0")

RC_DIR="$HOME/.esx"
HISTORY_DIR="$RC_DIR/history"
LAST_RESULTS_FILE="$RC_DIR/last-result.json"
QUERY_TEMPLATE_URL="https://gist.githubusercontent.com/marzocchi/71cc6c82eda0e0c36fb8dd36f7da92cf/raw/template.json"
QUERY_TEMPLATE="$RC_DIR/template.json" 
QUERY_FILE=$(pwd)/"$(basename "$NAME" .sh)"-query.json

gethostname() {
  echo "$1" | awk -F/ '{print $3}' | awk -F: '{print $1}'
}

log() {
	echo "$@" > /dev/stderr
}

usage() {
  log Usage: "$NAME" query
}

create_query_file_from_template() {
  if [ -z "$1" ]; then
    log No file given
    log
    return 1
  fi

  if [ ! -f "$QUERY_TEMPLATE" ]; then 
    log Query template "$QUERY_TEMPLATE" not found!
    usage
    log
    return 1
  fi

  cp "$QUERY_TEMPLATE" "$1"
}

prettify() {
  cat - | jq --compact-output --unbuffered '.hits.hits[]'
}

run_query() {
  curl -# -XPOST "$ES_URL" -d @-
}

check_required_program() {
  for i in "$@"; do
    which "$i" >/dev/null 2>&1 || (log Required program \'"$i"\' not installed.; return 1)
  done
}

ensure_environment() {
  check_required_program curl jq || return 1

  [ -z "$ES_URL" ] || (log "The ES_URL environment variable is not set."; return 1)

  [ -d "$RC_DIR" ] || (log "Creating missing directory $RC_DIR"; mkdir "$RC_DIR" || return 1)
  [ -d "$HISTORY_DIR" ] || (log "Creating missing directory $HISTORY_DIR"; mkdir "$HISTORY_DIR" || return 1)

  if [ ! -s "$QUERY_TEMPLATE" ]; then
    log "Template $QUERY_TEMPLATE not found, grabbing default template from $QUERY_TEMPLATE_URL"
    curl -#L "$QUERY_TEMPLATE_URL" > "$QUERY_TEMPLATE" || return 1
  fi
}

command_history() {
  tmp=$(mktemp)
  ranger --choosefile="$tmp" "$HISTORY_DIR"
  choosenfile=$(cat "$tmp")
  rm "$tmp"

  if [ ! -s "$choosenfile" ]; then
    log No file selected!
    log
    return 1
  fi

  cp "$choosenfile" "$QUERY_FILE"
  command_query "$QUERY_FILE"
}

command_query() {
  if [ -n "$1" ]; then
    QUERY_FILE="$1"
  else 
    log Using existing query file: "$QUERY_FILE"
  fi

  if [ ! -s "$QUERY_FILE" ]; then
    create_query_file_from_template "$QUERY_FILE" || exit 1
  fi

  "$EDITOR" "$QUERY_FILE" > /dev/tty

  HISTORY_FILE="$HISTORY_DIR/"$(date +"%Y%m%d%H%M%S")-$(gethostname "$ES_URL").json
  cp "$QUERY_FILE" "$HISTORY_FILE"

  run_query < "$HISTORY_FILE" | tee "$LAST_RESULTS_FILE" | prettify

  log 
  log "Query saved in $HISTORY_FILE"
  log
}

command_last() {
  if [ ! -f "$LAST_RESULTS_FILE" ]; then
    log "No previous query result"
    usage
    log
    return 1
  fi

  prettify < "$LAST_RESULTS_FILE" 
}

if [ "$#" -lt 1 ]; then
  if [ -f "$HISTORY_FILE" ]; then
    COMMAND="last"
  else
    COMMAND="query"
  fi
else
  COMMAND="$1"
  shift
fi

ensure_environment || (log "Could not prepare environment."; exit 1)
command_"$COMMAND" "$@" || exit 1

template.json

{
  "size": 100,
  "query": {
    "filtered": {
      "query": {
        "query_string": {
          "query": "message:Exception",
          "analyze_wildcard": true
        }
      },
      "filter": {
        "bool": {
          "must": [
            {
              "range": {
                "@timestamp": {
                  "lte": "2017-01-10T12:00:00",
                  "gte": "2017-01-10T00:00:00"
                }
              }
            }
          ]
        }
      }
    }
  },
  "sort": [
    {
      "@timestamp": {
        "order": "desc",
        "unmapped_type": "boolean"
      }
    }
  ],
  "fields": [
    "*",
    "_source"
  ],
  "script_fields": {},
  "fielddata_fields": [
    "@timestamp",
    "time"
  ]
}