s3 reposync

gistfile1.txt

#!/bin/env python

import boto
import hashlib
import os

BUCKET_NAME = 'reponame'
AWS_ACCESS_KEY_ID = ''
AWS_SECRET_ACCESS_KEY = ''

REPO_PATH = '/home/repo'

def build_connection():
   return boto.connect_s3(AWS_ACCESS_KEY_ID,
                          AWS_SECRET_ACCESS_KEY)


def md5sum(filename, blocksize=1024*1024):
   md5 = hashlib.md5()
   file = open(filename)
   while True:
       data = file.read(blocksize)
       if not data:
           break
       md5.update(data)
   return md5.hexdigest()


def upload(k, bucket, filename):
    k.set_contents_from_filename(filename)
    bucket.set_acl('public-read', k.key)


def upload_to_s3(srcpath, bucket_name=BUCKET_NAME):
   from boto.s3.key import Key
   conn = build_connection()
   bucket = conn.get_bucket(bucket_name)
   k = Key(bucket)
   for path,dir,files in os.walk(srcpath):
      for file in files:
          filename =  os.path.join(path,file)
          k.key = os.path.relpath(filename, srcpath)
          s3key = bucket.get_key(k.key)
          if s3key:
              etag = s3key.etag.strip('"') or None
              if md5sum(filename) != etag:
                  print filename
                  upload(k, bucket, filename)
          else:
              print filename
              upload(k, bucket, filename)

if __name__ == '__main__':
   upload_to_s3(REPO_PATH)