Export Elastic search as JSON through Kibana console proxy

elastic-scroll.sh

#!/bin/bash

kb_host="https://XXXXXXXX"
kb_url="$kb_host/api/console/proxy"

index=$1
passw_file=$2
query_file=$3
output_file=$4

if [ -z "$index" -o  -z "$passw_file" -o -z "$query_file" -o -z "$output_file" ]; then
  echo "usage: elastic-scroll [index] [netrc_file] [query_file] [output_file]" 1>&2
  exit 1
fi

function urlencode()
{
  echo "$1" | jq -sRr @uri
}

function call_elastic()
{
  local path=$(urlencode $2)
  local response=$(curl -s "$kb_url?path=$path&method=$1" -d "$3" --netrc-file "$passw_file" -H 'kbn-xsrf: reporting' -H 'Content-Type: application/json')

  local error=$(echo "$response" | jq -r '.error | length')
  if [ "$error" != "0" ]; then
    echo "$response" | jq '.' 1>&2
    exit 1
  fi
  
  echo "$response"
}

function extract_scroll_id()
{
  local scroll_id=$(echo "$1" | jq -r '._scroll_id')
  [ "$scroll_id" = "null" ] && echo "" || echo "$scroll_id"
}

function extract_hits_count()
{
  local hits_count=$(echo "$1" | jq -r '.hits.hits | length')
  [ -z "$hits_count" ] && echo "0" || echo "$hits_count"
}

function extract_total_hits_count()
{
  local hits_count=$(echo "$1" | jq -r '.hits.total.value')
  [ -z "$hits_count" ] && echo "0" || echo "$hits_count"
}

function progression()
{
  LC_ALL=C printf "%3.0f%% | %*s/%s" "$(bc <<< "scale=2;$1/$2*100")" "$(echo -n "$2" | wc -c)" "$1" "$2"
}

function extract_and_format_output()
{
  echo -n "$1" | jq '.hits.hits[]._source' | sed ':a;N;$!ba;s/}\n{/},\n{/g'
}

response=$(call_elastic 'POST' "$index/_search?scroll=30s" "@$query_file" )

scroll_id=$(extract_scroll_id "$response")
hits_count=$(extract_hits_count "$response")
hits_so_far="$hits_count"
hits_total=$(extract_total_hits_count "$response")

echo "Got initial response with $hits_count hits of a total of $hits_total hits"

echo '[' > "$output_file"

if [ "$hits_count" != "0" ]; then
  echo $(extract_and_format_output "$response") >> "$output_file"
fi

while [ "$hits_count" != "0" -a ! -z "$scroll_id" ]; do
  response=$(call_elastic 'POST' '_search/scroll' '{"scroll": "30s", "scroll_id": "'$scroll_id'"}')

  scroll_id=$(extract_scroll_id "$response")
  hits_count=$(extract_hits_count "$response")
  hits_so_far=$((hits_so_far + hits_count))

  echo -e "\e[1A\e[0K\rKibana scroll progression: " "$(progression $hits_so_far $hits_total)"
 
  if [ "$hits_count" = "0" ]; then
    break
  fi

  echo ',' >> "$output_file"
  echo $(extract_and_format_output "$response") >> "$output_file"
done

echo >> "$output_file"
echo ']' >> "$output_file"

echo Done!

password.netrc

machine XXXXXXXX
login XXXXXX
password XXXXX

query.json

{
  "size": 500,
  "_source": ["field1", "field2", ...],
  "query": { .... }
}