Batch restore s3 deleted objects with bucket versioning -- Delete Delete Markers

delete_delete_markers.py

import boto3

client = boto3.client('s3')


def get_object_versions(bucket, prefix, max_key, key_marker):
    kwargs = dict(
        Bucket=bucket,
        EncodingType='url',
        MaxKeys=max_key,
        Prefix=prefix
    )

    if key_marker:
        kwargs['KeyMarker'] = key_marker

    response = client.list_object_versions(**kwargs)

    return response


def get_delete_markers_info(bucket, prefix, key_marker):
    markers = []
    max_markers = 500
    version_batch_size = 500

    while True:
        response = get_object_versions(bucket, prefix, version_batch_size, key_marker)
        key_marker = response.get('NextKeyMarker')
        delete_markers = response.get('DeleteMarkers', [])

        markers = markers + [dict(Key=x.get('Key'), VersionId=x.get('VersionId')) for x in delete_markers if
                             x.get('IsLatest')]

        print '{0} -- {1} delete markers ...'.format(key_marker, len(markers))

        if len(markers) >= max_markers or key_marker is None:
            break

    return {"delete_markers": markers, "key_marker": key_marker}


def delete_delete_markers(bucket, prefix):
    key_marker = None

    while True:
        info = get_delete_markers_info(bucket, prefix, key_marker)
        key_marker = info.get('key_marker')
        delete_markers = info.get('delete_markers', [])

        if len(delete_markers) > 0:
            response = client.delete_objects(
                Bucket=bucket,
                Delete={
                    'Objects': delete_markers,
                    'Quiet': True
                }
            )

            print 'Deleting {0} delete markers ... '.format(len(delete_markers))
            print 'Done with status {0}'.format(response.get('ResponseMetadata', {}).get('HTTPStatusCode'))
        else:
            print 'No more delete markers found\n'
            break


delete_delete_markers(bucket='data-global', prefix='2017/02/18')