-
-
Save tsungtwu/f83f3c06b40b35c8227c63e0674ffc04 to your computer and use it in GitHub Desktop.
Watch a directory for changes with Python Watchdog then multipart upload to S3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import sys | |
import os | |
import time | |
from watchdog.observers import Observer | |
from watchdog.events import FileModifiedEvent, FileCreatedEvent | |
import boto3 | |
import mimetypes | |
from botocore.exceptions import ClientError | |
# Create an S3 client | |
s3 = boto3.client('s3') | |
bucket_names = ['bucket1', 'bucket2'] | |
class go2s3: | |
def dispatch(self, event): | |
filename = os.path.basename(event.src_path) | |
ct = mimetypes.guess_type(filename)[0] or 'binary/octet-stream' | |
if type(event) in [FileModifiedEvent, FileCreatedEvent]: | |
print "Event: %s" % event | |
# Uploads the given file using a managed uploader, which will split up large | |
# files automatically and upload parts in parallel. | |
for bucket in bucket_names: | |
try: | |
s3.upload_file(Bucket=bucket, Filename=event.src_path, Key=filename, ExtraArgs={'ContentType': ct, 'ServerSideEncryption': 'AES256'}) | |
print "Uploaded %s to %s successfully" % (filename, bucket) | |
except ClientError as e: | |
print "Upload of %s to %s failed with: %s" % (filename, bucket, e) | |
# print dir(event) | |
if __name__ == "__main__": | |
path = sys.argv[1] if len(sys.argv) > 1 else '.' | |
if not os.path.exists(path): | |
os.makedirs(path) | |
event_handler = go2s3() | |
observer = Observer() | |
observer.schedule(event_handler, path, recursive=False) | |
observer.start() | |
try: | |
while True: | |
time.sleep(1) | |
except KeyboardInterrupt: | |
observer.stop() | |
observer.join() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
[Unit] | |
Description=Watch /_dir_to_watch/ for new/modified files then upload to s3 | |
[Service] | |
Type=simple | |
User=root | |
Group=root | |
Restart=on-failure | |
ExecStart=/usr/bin/python /root/bin/go2s3.py /_dir_to_watch/ | |
[Install] | |
WantedBy=multi-user.target |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment