Last active
February 26, 2019 15:56
-
-
Save hoangdh/9827e9cd0c64861a448aaeb260c1ac31 to your computer and use it in GitHub Desktop.
Upload, delete files on AWS S3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import boto3 | |
from datetime import datetime, timedelta | |
# Requeriment | |
## python2.7, boto3 (pip install boto3) | |
## ~/.aws/credentials - Contain your credentail (API, Secret) | |
## ~/.aws/config - Contain your config (Region,...) | |
## More details: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html | |
# Upload a file into Bucket | |
# Usage: upload_file("s3bucket.tenten.vn", "tenten/database", "abc.h2") | |
def upload_file(bucket, folder, file): | |
dest = "%s/%s" % (folder, file) | |
s3 = boto3.client('s3') | |
s3.upload_file(file, bucket, dest) | |
# Delete a file on bucket | |
# Usage: delete_file("s3bucket.tenten.vn", "tenten/database/abc.h2") | |
def delete_file(bucket, file): | |
s3 = boto3.resource("s3") | |
obj = s3.Object(bucket, file) | |
obj.delete() | |
# Listing all matched objects (files) on bucket with pattern | |
# Usage: list_file("s3bucket.tenten.vn","h2") | |
def list_file(bucket, pattern): | |
s3 = boto3.client('s3') | |
resp = s3.list_objects_v2(Bucket=bucket) | |
keys = [] | |
for obj in resp['Contents']: | |
keys.append(obj['Key']) | |
match = [s for s in keys if pattern in s] | |
return match | |
# Get X date X days ago and remove all file contain pattern YYYYMMDD | |
# Usage: remove_old_files("s3bucket.tenten.vn", 4) | |
def remove_old_files(bucket, day): | |
days_ago = datetime.now() - timedelta(days=day) | |
old = list_file(bucket,str(days_ago.strftime('%Y%m%d'))) | |
for f in old: | |
delete_file(bucket, f) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment