A inefficient but effective way to scan logs on s3

s3logs.py

from __future__ import print_function

import StringIO
import gzip
from boto.s3.connection import S3Connection

conn = S3Connection("<key_id>", "<secret>")
bucket = conn.get_bucket('<bucket>')


def get_matching_lines(prefix, match):
    """
    prefix - the prefix of files to match, useful for limiting date ranges
    match - the string present in relevant log entries
    """
    with open('logs.out', 'w') as f:
        count = 0
        keys = bucket.get_all_keys(prefix=prefix)
        print("Searching {} log files".format(len(keys)))
        for key in keys:
            fileobj = StringIO.StringIO(key.get_contents_as_string())
            gzip_file_handle = gzip.GzipFile(fileobj=fileobj)
            log = gzip_file_handle.read()
            logs = log.split('\n')
            logs = [l for l in logs if match in l]
            for l in logs:
                print(l, file=f)
                count += 1
    print("Found {} matching lines".format(count))