Lambda Example

lambda_example.py

# preliminaries
import re
import json
import logging
import boto3
import urllib
import base64

# fire up logger
logger = logging.getLogger()

# fire up s3 client
s3 = boto3.client('s3')

# constituent regexp patterns
remote_host    = '([\w\.]+)'
remote_logname = '(\-)'
remote_user    = '(\-)'
time           = '\[([^\]]+)\]'
method         = '\"([A-Z]{3,4})'
url            = '(\S+)'
junk           = '(.*)'

# concat constituent patterns
log_pattern    = '\\s+'.join([remote_host, remote_logname, remote_user, time, method, url])

# compile pattern as regexp
log_line = re.compile(log_pattern)

def to_json(match_list):
  return json.dumps(
    {
      "remote_host" : match_list.group(1),
      "remote_logname" : match_list.group(2),
      "remote_user" : match_list.group(3),
      "time" : match_list.group(4),
      "method" : match_list.group(5),
      "url" : match_list.group(6)
    }, sort_keys = True
  )


def lambda_handler(event, context):
  logger.info('Log stream name: ', context.log_stream_name)
  
  for record in event['Records']:
    payload = base64.b64decode(record['kinesis']['data'])
    matches = log_line.search(payload)
    return to_json(matches)

  # for record in event['Records']:
  #   try:
  #     payload      = base64.b64decode(record['kinesis']['data'])
  #     payload_json = json.loads(payload)
  #     return str(payload_json)
  #     # event_type   = str(payload_json['event'])
  #     # matches      = log_line.search(payload)
  #     # return to_json(matches)
  #   except Exception:
  #     pass