get errors from elastic search failed logs

aws-elastisearch-failed.py

import json
import base64

def each_data(stream):
    for line in stream.readlines():
        data = json.loads(line)
        data['errorMessage'] = json.loads(data['errorMessage'])
        yield data, json.loads(base64.b64decode(data['rawData']).decode())

import boto3

BUCKET=''
PREFIX=''
    
def s3_streams(bucket, prefix):
    s3 = boto3.client('s3')
    response = s3.list_objects(Bucket=bucket, Prefix=prefix)    
    for key in map(lambda x: x['Key'], response['Contents']):
        print("reading %s"%key)
        yield io.BytesIO(get_log_from_s3(s3, key, bucket))

import gzip, io

def get_log_from_s3(s3, key, bucket):   
    response = s3.get_object(Bucket=BUCKET,
                         Key=key)
    gzipped_body = response['Body'].read()
    return gzip.decompress(gzipped_body)

errors = {}
# FOCUS ON THIS ONLY
def per_error(data, event):
    print(event)
    print(data['errorMessage']['reason'])

for stream in s3_streams(BUCKET, PREFIX):
    for data, event in each_data(stream):
        per_error(data, event)