Unload logs from kibana

kibana_hits.py

#!/usr/bin/evn python

# Dependencies:
#    pip3 install ujson ipdb tqdm requests

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import sys
import re
import tempfile

try:
    import requests    
    import ujson
    import ipdb
    from tqdm import tqdm
except ImportError:
    print("At first, you should install dependencies: pip3 install ujson ipdb tqdm requests")
    sys.exit(0)

#-----------------------------------------------------------------------------
# Constants
#-----------------------------------------------------------------------------

kibana_proxy_url = "https://...host-here.../api/console/proxy"
common_headers={
    "Accept": "application/json",
    "Referer": "https://...host-here.../app/kibana",
    "kbn-version": "5.4.1"
}

#-----------------------------------------------------------------------------
# Functions
#-----------------------------------------------------------------------------

def iter_hits(cookie):
    headers = {
        **common_headers,
        "Cookie": cookie
    }
    resp = requests.post(
        url=kibana_proxy_url, 
        params={
            "method": "POST",
            "path": "logstash-prod-lopush*/_search?scroll=30m",
        },
        headers=headers, 
        json={
            "query": {
                "match_phrase": {
                    "message": "processed"
                }
            },
            "size": 10_000
        })

    if resp.status_code != 200:
        ipdb.set_trace()
        return

    total = int(resp.json()['hits']['total'])
    with tqdm(total=total, desc="Loading data from Kibana") as pbar:
        while resp.status_code == 200 and len(resp.json()['hits']['hits']) > 0:
            json = resp.json()
            scroll_id = json['_scroll_id']

            for hit in json['hits']['hits']:
                pbar.update()
                yield hit

            resp = requests.post(
                url=kibana_proxy_url, 
                params={
                    "method": "POST",
                    "path": "_search/scroll"
                },
                headers=headers, 
                json={
                    "scroll": "30m",
                    "scroll_id": scroll_id
                })
            json = resp.json()