scroll Elasticsearch with bash

es2csv.sh

#!/bin/bash
# from https://gist.github.com/cb372/4567f624894706c70e65
es_url=$1
index=$2

response=$(curl -s -H 'content-type: application/json' $es_url/$index/_search?scroll=1m -d @query.json)
scroll_id=$(echo $response | jq -r ._scroll_id)
hits_count=$(echo $response | jq -r '.hits.hits | length')
hits_so_far=hits_count
echo Got initial response with $hits_count hits and scroll ID $scroll_id

# TODO process first page of results here
echo $response | jq -r '.hits.hits[]._id' > es.csv

while [ "$hits_count" != "0" ]; do
  response=$(curl -s -H 'content-type: application/json' $es_url/_search/scroll -d "{ \"scroll\": \"1m\", \"scroll_id\": \"$scroll_id\" }")
  scroll_id=$(echo $response | jq -r ._scroll_id)
  hits_count=$(echo $response | jq -r '.hits.hits | length')
  hits_so_far=$((hits_so_far + hits_count))
  echo "Got response with $hits_count hits (hits so far: $hits_so_far), new scroll ID $scroll_id"

  # TODO process page of results
  echo $response | jq -r '.hits.hits[]._id' >> es.csv
done

echo Done!

{
"_source": {
"includes": [ "_id" ]
},
"query": { "match_all": {} },
"size": 1000
}