shentonfreude

#!/usr/bin/env python3
# Setup logs so our app shows INFO but boto shows at WARNING

import logging
import boto3

# Don't show boto INFO; if we don't call this our own INFO logs aren't seen!
logging.basicConfig(level=logging.WARNING)

# Define our logger to show INFO logs

shentonfreude

# ...
[loggers]
keys = root, rival, sqlalchemy, boto3

# ...

[logger_boto3]
# boto3.resources.action spews binary contents of file put_object at INFO, suppress
level = WARN
handlers =

shentonfreude

#!/usr/bin/env python3
"""
We want to query of a page 'size' of N results and return them, and also a
'next' and 'previous' link. Those CloudSearch 'start' offsets are based on the
page (for our app, starting at 1) and the size. So with size=100: page=1 has
start=0, page=2 has start=100, page=10 has start=100. We don't want to return
'previous' if we're at page=1, and don't want to return 'next' if it's beyond
the maximum results 'found'.  So if size=100 and found=543, the last page=5,
there is no page=6. More generally: lastpage = int(found / size).
"""

shentonfreude

import boto3

# found is always the same
# start is the index into the total results, not a page of size results

cs = boto3.client('cloudsearchdomain',
                  endpoint_url='https://search-name-xxx.us-east-1.cloudsearch.amazonaws.com',
                  region_name='us-east-1')

size = 4

shentonfreude

import boto3

# Returned 'cursor' can be of seemingly any length, and changes on each return

cs = boto3.client('cloudsearchdomain',
                  endpoint_url='https://search-thing-xxx.us-east-1.cloudsearch.amazonaws.com',
                  region_name='us-east-1')
size = 4
cursor = 'initial'
while True:

shentonfreude

#!/usr/bin/env python

# An exponential backoff around Boto3 DynamoDB, whose own backoff eventually
# fails on long multipage scans. We'd like to use this as a wrapper somehow,
# see: https://gist.github.com/numberoverzero/cec21b8ca715401c5662

from time import sleep

import boto3
from boto3.dynamodb.conditions import Attr

shentonfreude

    # EC2 only
    # Can't use (EC2) SecurityGroups with Subnet
    # Can't use (VPC) SecurityGroupIds with NetworkInterfaces
    # Can't use SubnetId with NetworkInterfaces

    def add_ec2(self):
        name, tags = self._name_tags('ec2')
        self.ec2 = self.t.add_resource(
            ec2.Instance(
                name,

shentonfreude

    aws_region = config['AWS']['region']
    search_domain_name = config['AWS']['cloudsearch_domain_name']  # e.g., avail-search-dev
    cs = boto3.client("cloudsearch")
    cs_domain = cs.describe_domains(DomainNames=[search_domain_name])['DomainStatusList'][0]
    cs_doc_ep = cs_domain['DocService']['Endpoint']
    cs_doc = boto3.client('cloudsearchdomain',
                          endpoint_url='https://' + cs_doc_ep,
                          region_name=aws_region)
    cs_search_ep = cs_domain['SearchService']['Endpoint']
    cs_search = boto3.client('cloudsearchdomain',

shentonfreude

def asset_update(asset_id, state, extra={}):
    asset = extra.copy()
    asset.update({'dt': datetime.now().isoformat(),
                  'state': state})

    # We need update_item placeholder and values like:
    #   UpdateExpression='SET dt = :dt, #state = :state'   # 'state' is a reserved word
    #   ExpressionAttributeNames={'#state': 'state'}       # proxy for reserved word 'state'
    #   ExpressionAttributeValues={':dt': datetime.now().isoformat(), ':state': 'uploaded'}
    # We can't use "reserved words" like 'state' use EAN for *all* names,

shentonfreude

# Create AWS ElasticTranscoder preset based on AWS 1080p with 'auto' framerate.
# This uses same output FPS as incoming, so we don't up/down sample.

import boto3

et = boto3.client('elastictranscoder')
res = et.read_preset(Id='1351620000001-000001')  # Generic 1080p
preset = res['Preset']

del preset['Type']