etag_s3_swift.py

import hashlib
import sys


def etag_checksum_s3(filename, chunk_size=6 * 1024 * 1024):
    md5s = []
    with open(filename, 'rb') as f:
        for data in iter(lambda: f.read(chunk_size), b''):
            md5s.append(hashlib.md5(data).digest())
            print("s3api segment(hex): %s"% hashlib.md5(data).hexdigest())

    m = hashlib.md5("".join(md5s))
    return '{}-{}'.format(m.hexdigest(), len(md5s))

def etag_checksum_swift(filename, chunk_size=6 * 1024 * 1024):
    md5s = []
    with open(filename, 'rb') as f:
        for data in iter(lambda: f.read(chunk_size), b''):
            md5s.append(hashlib.md5(data).hexdigest())
            print("s3api segment(hex): %s"% hashlib.md5(data).hexdigest())

    m = hashlib.md5("".join(md5s))
    return '{}-N'.format(m.hexdigest())


print("      file: %s\n" % sys.argv[1])
print("   S3 ETag: %s\n" % etag_checksum_s3(sys.argv[1]))
print("Swift ETag: %s\n" % etag_checksum_swift(sys.argv[1]))