Last active
November 8, 2019 14:26
-
-
Save gpetrousov/0b1b831c516db43574f2e11b970d4f72 to your computer and use it in GitHub Desktop.
Lazy loading in Python Lambda when reading from S3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" The name of the S3 bucket and object is contained inside the event that triggered the Lambda """ | |
def lambda_handler(event, context): | |
# Define client | |
s3 = boto3.client('s3') | |
# Get the bucket name | |
source_bucket_name = event['Records'][0]['s3']['bucket']['name'] | |
# Get the object name | |
key_name = event['Records'][0]['s3']['object']['key'] | |
# Get object from s3 [dict type] | |
response = s3.get_object(Bucket=source_bucket_name, Key=key_name) | |
# Open the stream in binary mode [objects in s3 are binary] | |
# TextIOWrapper decodes in chunks. | |
stream_content = io.TextIOWrapper(gzip.GzipFile(None, 'rb', fileobj=response['Body'])) | |
# Count HTTP 460s per targetgroup in AWS ELB [dict] | |
http_460s_per_targetgroup = {} | |
for each_line in stream_content: | |
if (" 460 " in each_line) and ("targetgroup" in each_line): | |
# Do some regex magic to find "targetgroup" | |
targetgroup_name = return_target_group_from_string(each_line) | |
increment_targetgroup_460_counter(http_460s_per_targetgroup, targetgroup_name) | |
else: | |
logging.info("False HTTP 460 log does not contain a target group") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment