Skip to content

Instantly share code, notes, and snippets.

@zduey
Created September 18, 2016 02:26
Example of using Luigi for backup to S3
import os
import boto3
import luigi
import shutil
import datetime
from luigi.s3 import S3Target
# Globals
today = datetime.datetime.today().strftime(format='%Y_%m_%d')
folder_to_archive = ''
archive_name = '' + today
archive_type = ''
s3_bucket_name = ''
class CompressDocumentsFolder(luigi.ExternalTask):
"""
Archives the user's specified directory
"""
def run(self):
shutil.make_archive(archive_name, archive_type, folder_to_archive)
def output(self):
return luigi.LocalTarget(archive_name + "." + archive_type)
class UploadArchive(luigi.Task):
"""
Uploads the archive to S3
"""
def requires(self):
return CompressDocumentsFolder()
def run(self):
s3 = boto3.resource('s3')
bucket = s3.Bucket(s3_bucket_name)
bucket.upload_file(archive_name + "." + archive_type, archive_name)
os.remove(archive_name + "." + archive_type)
return
def output(self):
full_s3_path = 's3://' + s3_bucket_name + '/' + archive_name
return S3Target(full_s3_path)
if __name__ == "__main__":
luigi.run(main_task_cls=UploadArchive)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment