Example for setting S3 bucket "ExpiredObjectDeleteMarker" automatically from CloudFormation

lambda.py

# Extracted from the CloudFormation template.json below with:
# jq -r .Resources.RemoveExpiredObjectDeleteMarkerFunction.Properties.Code.ZipFile < template.json
import uuid
import httplib
import urlparse
import json
import boto3
import botocore
import traceback

def send_response(request, response, status=None, reason=None):
    if status is not None:
        response['Status'] = status

    if reason is not None:
        response['Reason'] = reason

    body = json.dumps(response)

    print('send_response: response body: "%(body)s"' % locals())

    if 'ResponseURL' in request and request['ResponseURL']:
        print('send_response: ResponseURL: "%s"' % request['ResponseURL'])
        url = urlparse.urlparse(request['ResponseURL'])
        https = httplib.HTTPSConnection(url.hostname)
        https.request('PUT', url.path + '?' +url.query, body)
    else:
        print('send_response: no ResponseURL in request: body: "%(body)s"' % locals())

    return response

def update(buckets, remove_expired_object_delete_marker):
    client = boto3.client('s3')
    for bucket in buckets:
        try:
            need_update = False
            lifecycle = client.get_bucket_lifecycle_configuration(Bucket = bucket)
            for rule in lifecycle['Rules']:
                if remove_expired_object_delete_marker:
                    if ('Expiration' not in rule
                      or 'ExpiredObjectDeleteMarker' not in rule['Expiration']):
                        rule['Expiration'] = { 'ExpiredObjectDeleteMarker': True }
                        need_update = True
                    else:
                        print('update: bucket: "%s" not adding rule: "%s"' %
                            (bucket, json.dumps(rule)))
                elif 'Expiration' in rule:
                    del rule['Expiration']
                    need_update = True

            print('update: bucket: "%s"; updated: %s; policy: "%s"; ' %
                (bucket, need_update, json.dumps(lifecycle)))

            if need_update:
                client.put_bucket_lifecycle_configuration(
                    Bucket = bucket,
                    LifecycleConfiguration = {
                        'Rules': lifecycle['Rules']
                    },
                )

        except botocore.exceptions.ClientError as e:
            print('update: exception: remove: %s; %s' %
              (remove_expired_object_delete_marker, json.dumps(e.response)))
            if (e.response['Error']['Code'] == 'NoSuchLifecycleConfiguration'):
                print('update: "%(bucket)s": no lifecycle' % locals())
            else:
                raise e


    return 'OK'

def handler(event, context):
    response = {
        'StackId': event['StackId'],
        'RequestId': event['RequestId'],
        'LogicalResourceId': event['LogicalResourceId'],
        'Status': 'SUCCESS'
    }

    if 'PhysicalResourceId' in event:
        response['PhysicalResourceId'] = event['PhysicalResourceId']
    else:
        response['PhysicalResourceId'] = str(uuid.uuid4())

    try:
        for key in ['Buckets', 'RemoveExpiredObjectDeleteMarker']:
            if (key not in event['ResourceProperties'] or
              not event['ResourceProperties'][key]):
                return send_response(
                    event, response, status='FAILED',
                    reason='The properties "%(key)s" must not be empty' % locals()
                )
        buckets = event['ResourceProperties']['Buckets']
        remove_expired_object_delete_marker = (
            event['ResourceProperties']['RemoveExpiredObjectDeleteMarker']
            if event['RequestType'] != 'Delete'
            else 'false'
        )
        expire_object_delete_marker = (
            False if remove_expired_object_delete_marker == 'false'
            else True
        )
        action_response = update(buckets, remove_expired_object_delete_marker)
        response['Reason'] = 'Response: "%(action_response)s"' % locals()

    except:
        response['Status'] = 'FAILED'
        response['Reason'] = 'handler Failed: "%s"' % traceback.format_exc()

    return send_response(event, response)

template.json

{
    "AWSTemplateFormatVersion": "2010-09-09",
    "Description": "Example S3 bucket with clean deleted markers",
    "Parameters": {
      "BucketName": {
        "Description": "Name of bucket to create",
        "Type": "String",
        "Default": "test-s3-with-clean-delete-marker"
      },
      "ExpiryInDays": {
        "Description": "Number of days to set expiry",
        "Type": "Number",
        "Default": "1"
      }
    },
    "Resources": {
        "bucket": {
            "Type": "AWS::S3::Bucket",
            "Properties": {
                "BucketName": { "Ref": "BucketName" },
                "VersioningConfiguration": {
                    "Status": "Enabled"
                },
                "LifecycleConfiguration": {
                    "Rules": [
                        {
                            "Status": "Enabled",
                            "Prefix": "/",
                            "Id": "ExpireAfterDays",
                            "NoncurrentVersionExpirationInDays": {
                              "Ref": "ExpiryInDays"
                            }
                        }
                    ]
                }
            }
        },
        "setRemoveExpiredObjectDeleteMarker": {
            "Type": "Custom::SetRemoveExpiredObjectDeleteMarker",
            "Properties": {
                "Buckets": [
                    { "Ref": "BucketName" }
                ],
                "ServiceToken": {
                    "Fn::GetAtt": [
                        "RemoveExpiredObjectDeleteMarkerFunction",
                        "Arn"
                    ]
                },
                "DepoendsOn": [
                    "bucket",
                    "RemoveExpiredObjectDeleteMarkerFunction"
                ],
                "RemoveExpiredObjectDeleteMarker": true
            }
        },
        "LambdaExecutionRole": {
            "Type": "AWS::IAM::Role",
            "Properties": {
                "Path": "/",
                "Policies": [
                    {
                        "PolicyName": "root",
                        "PolicyDocument": {
                            "Version": "2012-10-17",
                            "Statement": [
                                {
                                    "Action": [
                                        "logs:*"
                                    ],
                                    "Resource": "arn:aws:logs:*:*:*",
                                    "Effect": "Allow"
                                },
                                {
                                    "Action": [
                                        "s3:*"
                                    ],
                                    "Resource": "*",
                                    "Effect": "Allow"
                                }
                            ]
                        }
                    }
                ],
                "AssumeRolePolicyDocument": {
                    "Version": "2012-10-17",
                    "Statement": [
                        {
                            "Action": [
                                "sts:AssumeRole"
                            ],
                            "Effect": "Allow",
                            "Principal": {
                                "Service": [
                                    "lambda.amazonaws.com"
                                ]
                            }
                        }
                    ]
                }
            }
        },
        "RemoveExpiredObjectDeleteMarkerFunction": {
            "Type": "AWS::Lambda::Function",
            "Properties": {
                "Code": {
                    "ZipFile": "import uuid\nimport httplib\nimport urlparse\nimport json\nimport boto3\nimport botocore\nimport traceback\n\ndef send_response(request, response, status=None, reason=None):\n    if status is not None:\n        response['Status'] = status\n\n    if reason is not None:\n        response['Reason'] = reason\n\n    body = json.dumps(response)\n\n    print('send_response: response body: \"%(body)s\"' % locals())\n\n    if 'ResponseURL' in request and request['ResponseURL']:\n        print('send_response: ResponseURL: \"%s\"' % request['ResponseURL'])\n        url = urlparse.urlparse(request['ResponseURL'])\n        https = httplib.HTTPSConnection(url.hostname)\n        https.request('PUT', url.path + '?' +url.query, body)\n    else:\n        print('send_response: no ResponseURL in request: body: \"%(body)s\"' % locals())\n\n    return response\n\ndef update(buckets, remove_expired_object_delete_marker):\n    client = boto3.client('s3')\n    for bucket in buckets:\n        try:\n            need_update = False\n            lifecycle = client.get_bucket_lifecycle_configuration(Bucket = bucket)\n            for rule in lifecycle['Rules']:\n                if remove_expired_object_delete_marker:\n                    if ('Expiration' not in rule\n                      or 'ExpiredObjectDeleteMarker' not in rule['Expiration']):\n                        rule['Expiration'] = { 'ExpiredObjectDeleteMarker': True }\n                        need_update = True\n                    else:\n                        print('update: bucket: \"%s\" not adding rule: \"%s\"' %\n                            (bucket, json.dumps(rule)))\n                elif 'Expiration' in rule:\n                    del rule['Expiration']\n                    need_update = True\n\n            print('update: bucket: \"%s\"; updated: %s; policy: \"%s\"; ' %\n                (bucket, need_update, json.dumps(lifecycle)))\n\n            if need_update:\n                client.put_bucket_lifecycle_configuration(\n                    Bucket = bucket,\n                    LifecycleConfiguration = {\n                        'Rules': lifecycle['Rules']\n                    },\n                )\n\n        except botocore.exceptions.ClientError as e:\n            print('update: exception: remove: %s; %s' %\n              (remove_expired_object_delete_marker, json.dumps(e.response)))\n            if (e.response['Error']['Code'] == 'NoSuchLifecycleConfiguration'):\n                print('update: \"%(bucket)s\": no lifecycle' % locals())\n            else:\n                raise e\n\n\n    return 'OK'\n\ndef handler(event, context):\n    response = {\n        'StackId': event['StackId'],\n        'RequestId': event['RequestId'],\n        'LogicalResourceId': event['LogicalResourceId'],\n        'Status': 'SUCCESS'\n    }\n\n    if 'PhysicalResourceId' in event:\n        response['PhysicalResourceId'] = event['PhysicalResourceId']\n    else:\n        response['PhysicalResourceId'] = str(uuid.uuid4())\n\n    try:\n        for key in ['Buckets', 'RemoveExpiredObjectDeleteMarker']:\n            if (key not in event['ResourceProperties'] or\n              not event['ResourceProperties'][key]):\n                return send_response(\n                    event, response, status='FAILED',\n                    reason='The properties \"%(key)s\" must not be empty' % locals()\n                )\n        buckets = event['ResourceProperties']['Buckets']\n        remove_expired_object_delete_marker = (\n            event['ResourceProperties']['RemoveExpiredObjectDeleteMarker']\n            if event['RequestType'] != 'Delete'\n            else 'false'\n        )\n        expire_object_delete_marker = (\n            False if remove_expired_object_delete_marker == 'false'\n            else True\n        )\n        action_response = update(buckets, remove_expired_object_delete_marker)\n        response['Reason'] = 'Response: \"%(action_response)s\"' % locals()\n\n    except:\n        response['Status'] = 'FAILED'\n        response['Reason'] = 'handler Failed: \"%s\"' % traceback.format_exc()\n\n    return send_response(event, response)\n"
                },
                "MemorySize": 128,
                "Handler": "index.handler",
                "Role": {
                    "Fn::GetAtt": [
                        "LambdaExecutionRole",
                        "Arn"
                    ]
                },
                "Timeout": "10",
                "Runtime": "python2.7"
            }
        },
    }
}