iam_policy.json

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "logs:CreateLogGroup",
                "logs:CreateLogStream",
                "logs:PutLogEvents"
            ],
            "Resource": "arn:aws:logs:*:*:*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "s3:GetObject",
                "s3:PutObject",
                "s3:DeleteObject",
                "s3:CopyObject",
                "s3:PutObjectAcl"
            ],
            "Resource": [
                "arn:aws:s3:::pb-govuk-travel-advice-email/*"
            ]
        }
    ]
}

s3_handler.py

import datetime
import email
from email.header import decode_header
import json
import re

import boto3
from botocore.exceptions import ClientError

SES_KEY_REGEX = re.compile(r'^[0-9a-z]{40}$')
SUMMARY_REGEX = re.compile(r'Latest update: (.*)$', re.MULTILINE | re.IGNORECASE)
TRAVEL_ADVICE_REGEX = re.compile(r' travel advice$')

S3_CLIENT = boto3.client('s3')

def handle_s3_event(event, context):
    for record in event['Records']:
        if record['s3'] and SES_KEY_REGEX.match(record['s3']['object']['key']):
            handle_email_key(record['s3']['bucket']['name'], record['s3']['object']['key'])

def handle_email_key(bucket_name, key):
    message = email.message_from_file(fetch_key(bucket_name, key))
    status = country_status(message)
    if status:
        merge_status(bucket_name, status)
        country = status.keys()[0]
        dest = '%s/%s.msg' % (
            country.lower().replace(' ', '-'),
            status[country]['received']
        )

        source = '%s/%s' % (bucket_name, key)
        S3_CLIENT.copy_object(
            Bucket=bucket_name,
            Key=dest,
            CopySource=source
        )

        S3_CLIENT.delete_object(Bucket=bucket_name, Key=key)

def fetch_key(bucket_name, key):
    try:
        response = S3_CLIENT.get_object(Bucket=bucket_name, Key=key)
        return response["Body"]
    except ClientError as e:
        if e.response['Error']['Code'] == 'NoSuchKey':
            return None
        else:
            raise e

def country_status(message):
    subject = decode_header(message["subject"])[0][0]
    if TRAVEL_ADVICE_REGEX.search(subject):
        country = TRAVEL_ADVICE_REGEX.sub('', subject)
    else:
        return None

    received = None
    if message['Date']:
        date_tuple=email.utils.parsedate_tz(message['Date'])
        if date_tuple:
            received=datetime.datetime.fromtimestamp(email.utils.mktime_tz(date_tuple))

    text_part = [p for p in message.get_payload() if p.get_content_type() == 'text/plain'][0]
    body = text_part.get_payload(None, True)
    summary = SUMMARY_REGEX.search(body).groups()[0].rstrip()

    return {
        country: {
            'received': received.isoformat(),
            'summary': summary
        }
    }

def merge_status(bucket_name, status):
    contents = fetch_key(bucket_name, 'index.json')
    if contents:
        index = json.load(contents)
    else:
        index = dict()
    index.update(status)
    S3_CLIENT.put_object(
        Bucket=bucket_name,
        Key='index.json',
        Body=json.dumps(index),
        ContentType='application/json'
    )
    S3_CLIENT.put_object_acl(
        ACL='public-read',
        Bucket=bucket_name,
        Key='index.json'
    )