s3accesslogs.py

import boto3
import re
import requests
from requests_aws4auth import AWS4Auth
region = 'us-east-1'
service = 'es'
credentials = boto3.Session().get_credentials()
awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service, session_token=credentials.token)
host='XXXXXX'
RoleArn = 'arn:aws:iam::XXXX:role/cross-account-role'
external_id = 'XXXX'
index = 's3-access-logs'
type = 's3-logs'
url = host + '/' + index + '/' + type
headers = { "Content-Type": "application/json" }
ip_pattern = re.compile('(\d+\.\d+\.\d+\.\d+)')
time_pattern = re.compile('\[(\d+\/\w\w\w\/\d\d\d\d:\d\d:\d\d:\d\d\s-\d\d\d\d)\]')
message_pattern = re.compile('\"(.+)\"')
print "function starts from here"
def lambda_handler(event, context):
    #print event
    key = event['Records'][0]['s3']['object']['key']
    bucket = event['Records'][0]['s3']['bucket']['name']
    sts = boto3.client('sts')
    response = sts.assume_role(RoleArn=RoleArn, RoleSessionName='s3file', DurationSeconds=900, ExternalId=external_id)
    secret_Key = response['Credentials']['SecretAccessKey']
    session_Token = response['Credentials']['SessionToken']
    access_Key = response['Credentials']['AccessKeyId']
    s3 = boto3.client('s3', aws_access_key_id=access_Key, aws_secret_access_key= secret_Key, aws_session_token= session_Token,region_name= region)
    obj = s3.get_object(Bucket=bucket, Key=key)
    body = obj['Body'].read()
    lines = body.splitlines()
    for line in lines:
        remote_ip = ip_pattern.search(line).group(1)
        bucket_owner = line.split(' ')[0]
        bucket_name = line.split(' ')[1]
        time = line.split(' ')[2] + line.split(' ')[3]
        requester = line.split(' ')[5]
        request_id = line.split(' ')[6]
        operation = line.split(' ')[7]
        key = line.split(' ')[8]
        http_status = line.split(' ')[12]
        bytes_sent = line.split(' ')[13]
        object_size = line.split(' ')[14]
        total_time = line.split(' ')[15]
        turn_around_time = line.split(' ')[16]
        referrer = line.split(' ')[20]
        document = { "remote_ip": remote_ip, "bucket_owner": bucket_owner, "bucket_name": bucket_name, "time": time, "requester": requester, "request_id": request_id, "operation" : operation,  "key" : key,"http_status": http_status ,"bytes_sent": bytes_sent, "object_size": object_size, "total_time":total_time,"turn_around_time": turn_around_time, "referrer": referrer }
        r = requests.post(url, auth=awsauth, json=document, headers=headers)
    print "function ended"