aws_utils.py

from django.conf import settings

import boto3
from utils.files import get_related_filename
from utils.files import get_fragments_list
from utils.files import get_fragments_list_no_intro
from utils.files import get_m3u8_files

from utils.aws_templates.templates import distribution_config
import time
import logging


def _get_aws_credentials(aws_config, use_system_defaults=True):
    """
    Returns the aws s3 credentials from the given user config
    """

    credentials = {}

    # System default AWS credentials
    if use_system_defaults:
        credentials = {
            "aws_access_key_id": settings.AWS_ACCESS_KEY_ID,
            "aws_secret_access_key": settings.AWS_SECRET_ACCESS_KEY,
        }

    # Update AWS credentials from config
    for key in ("aws_access_key_id", "aws_secret_access_key"):
        if key in aws_config:
            credentials[key] = aws_config[key]

    return credentials


def file_upload(src, key, acl="public-read", content_type="application/octet-stream", aws_config=None, s3=None):
    """
    Single files Amazon S3 upload
    """

    if aws_config is None:
        aws_config = {}

    # Get bucket name and file url from User config or system
    bucket = None
    file_url = ""

    if "bucket_name" in aws_config:
        # use user config
        bucket = aws_config.get("bucket_name")
        cloudfront_url = aws_config.get("cloudfront_url")
        # use bucket cloud front if it has one. If it doesn't use the buckek url.
        if cloudfront_url:
            file_url = settings.AWS_S3_CLOUDFRONT_URL % {"cloudfront_url": cloudfront_url, "key": key}
        else:
            file_url = settings.AWS_S3_KEY_URL % {"bucket": bucket, "key": key}
    else:
        # use system config
        bucket = settings.AWS_S3_VIDEO_BUCKET
        file_url = settings.AWS_S3_CLOUDFRONT_URL % {"cloudfront_url": settings.AWS_S3_MEDIA_CLOUDFRONT_URL, "key": key}

    if s3 is None:
        # AWS credentials
        credentials = _get_aws_credentials(aws_config)

        # AWS S3 client
        s3 = boto3.resource("s3", **credentials)
    try:
        s3.Object(bucket, key).upload_file(src, ExtraArgs={
            "ACL": acl, "ContentType": content_type
        })
    except:
        raise ValueError("Upload fails.")

    return file_url


def validate_config(aws_config):
    """
    Validate user AWS config
    """
    s3 = boto3.client("s3",
                      aws_access_key_id=aws_config.get("aws_access_key_id"),
                      aws_secret_access_key=aws_config.get("aws_secret_access_key"))

    try:
        s3.head_bucket(Bucket=aws_config.get("bucket_name"))
    except:
        valid = False
    else:
        valid = True

    return valid


def check_uploaded_files(video, aws_config=None):
    """Checks if all fragments and playlist files in the local machine are in S3.
    """
    if aws_config is None:
        aws_config = {}

    # Get bucket name
    bucket = aws_config.get("bucket_name", settings.AWS_S3_VIDEO_BUCKET)

    # AWS credentials
    credentials = _get_aws_credentials(aws_config)

    # Creates a AWS S3 client.
    client = boto3.client("s3", **credentials)

    # Creates a reusable Paginator.
    paginator = client.get_paginator('list_objects')

    # Creates a PageIterator from the Paginator.
    page_iterator = paginator.paginate(Bucket=bucket, Prefix=video.ref_str)

    # Gets all files in the given bucket.
    uploaded_files = []
    for page in page_iterator:
        if 'Contents' in page:
            for elem in page['Contents']:
                uploaded_files.append(elem['Key'])
        else:
            return False

    # Gets all generated files in disk.
    local_files = []
    local_files.extend(get_fragments_list(video))
    local_files.extend(get_m3u8_files(video))

    # Validates that the amount of files uploaded is equal to the amount of
    # files in disk.
    if len(uploaded_files) == len(local_files):
        return False

    # Validates that each local file has been uploaded.
    for local_file in local_files:
        if get_related_filename(video, local_file) not in uploaded_files:
            logging.info("File not found {0}".format(get_related_filename(video, local_file)))
            return False
    return True


def check_uploaded_fragments(video, aws_config=None):
    """Checks if all fragments and playlist files in the local machine are in S3.
    """
    if aws_config is None:
        aws_config = {}

    # Get bucket name
    bucket = aws_config.get("bucket_name", settings.AWS_S3_VIDEO_BUCKET)

    # AWS credentials
    credentials = _get_aws_credentials(aws_config)

    # Creates a AWS S3 client.
    client = boto3.client("s3", **credentials)

    # Creates a reusable Paginator.
    paginator = client.get_paginator('list_objects')

    # Creates a PageIterator from the Paginator.
    page_iterator = paginator.paginate(Bucket=bucket, Prefix=video.ref_str)

    # Gets all files in the given bucket.
    uploaded_files = []
    for page in page_iterator:
        if 'Contents' in page:
            for elem in page['Contents']:
                uploaded_files.append(elem['Key'])
        else:
            return False
    # Gets all generated files in disk.
    local_files = []
    local_files.extend(get_fragments_list_no_intro(video))
    # local_files.extend(get_m3u8_files(video))

    # Validates that the amount of files uploaded is equal to the amount of files in disk.
    if len(uploaded_files) == (len(local_files) - len(get_m3u8_files(video))):
        return False

    # Validates that each local file has been uploaded.
    for local_file in local_files:
        if get_related_filename(video, local_file) not in uploaded_files:
            logging.info("Fragment not found %s", get_related_filename(video, local_file))
            return False
    return True


def create_cloudfront_distribution(access_key_id, secret_access_key, account_ref, origin, comment):

    client = boto3.client('cloudfront', aws_access_key_id=access_key_id, aws_secret_access_key=secret_access_key)

    # Generates unique caller reference.
    caller_reference = '{0}_{1}'.format(account_ref, time.time())
    # Origin ID
    origin_ID = 'MyOrigin'

    distribution_config['CallerReference']  = caller_reference
    distribution_config['Origins']['Items'][0]['DomainName'] = origin
    distribution_config['Origins']['Items'][0]['Id'] = origin_ID
    distribution_config['DefaultCacheBehavior']['TargetOriginId'] = origin_ID
    distribution_config['DefaultCacheBehavior']['MinTTL'] = 1814400  # 21 days in seconds
    distribution_config['DefaultCacheBehavior']['DefaultTTL'] = 1814400  # 21 days in seconds
    distribution_config['DefaultCacheBehavior']['MaxTTL'] = 1814400  # 21 days in seconds
    distribution_config['Enabled'] = True
    distribution_config['Comment'] = comment

    response = client.create_distribution(DistributionConfig=distribution_config)

    cf_id = response['Distribution']['Id']
    cf_domain_name = response['Distribution']['DomainName']

    return cf_id, cf_domain_name