Multipart API of AWS S3

delete_unfinished_multipart_upload.py

#!/bin/python

import os
import json

bucket_name="your_bucket_name"

output = os.popen('aws s3api list-multipart-uploads --bucket %s'%(bucket_name))


x=json.load(output)
total=len(x["Uploads"])
n=0
for upload in x["Uploads"]:
    n+=1
    print("deleting:%s progress:%d/%d"%(upload["Key"],n,total))
    cmd="aws s3api abort-multipart-upload --bucket %s --key '%s' --upload-id '%s'"%(bucket_name,upload["Key"],upload["UploadId"])
    os.system(cmd)

etag.py

import hashlib

def hexstring(bytes):
    r=""
    for b in bytes:
       r+=hex(b)[2:]
    return r

file=open("/Users/unity/Downloads/test.html", 'rb')
content=file.read()
d1=hashlib.md5(content)
d2=hashlib.md5(d1.digest())
print(hexstring(d2.digest()))

multipart.py

#!/bin/python

import os
import json

bucket_name = "your bucket name"


def abort_unfinished(bucket):
    output = os.popen('aws s3api list-multipart-uploads --bucket %s' % bucket)

    x = json.load(output)
    total = len(x["Uploads"])
    n = 0
    for upload in x["Uploads"]:
        n += 1
        if upload["Initiated"] < '2018-07-13T09:25:55.000Z':
            print("deleting:%s %s %s progress:%d/%d" % (upload["Key"], upload["UploadId"],upload["Initiated"], n, total))
            cmd = "aws s3api abort-multipart-upload --bucket %s --key '%s' --upload-id '%s'" % (
            bucket, upload["Key"], upload["UploadId"])
            # cmd="aws s3api complete-multipart-upload --bucket %s--key '%s' --upload-id '%s'"%(
            # bucket_name, upload["Key"], upload["UploadId"])
            os.system(cmd)


def list_unfinished(bucket):
    output = os.popen('aws s3api list-multipart-uploads --bucket %s' % (bucket))
    x = json.load(output)
    total = len(x["Uploads"])
    n = 0
    for upload in x["Uploads"]:
        n += 1
        print("%d/%d: %s %s %s" % (n, total, upload["Key"], upload["UploadId"],upload["Initiated"]))
        #print(upload)


def list_parts(bucket):
    output = os.popen('aws s3api list-multipart-uploads --bucket %s' % bucket)
    x = json.load(output)
    total = len(x["Uploads"])
    n=0
    for upload in x["Uploads"]:
        n += 1
        cmd = "aws s3api list-parts --bucket %s --key %s --upload-id %s" % (
            bucket, upload["Key"], upload["UploadId"])
        os.system(cmd)


#abort_unfinished(bucket_name)
list_unfinished(bucket_name)
#list_parts(bucket_name)