Getting

boto3_funcs.py

import boto3

def get_paginated_results(func, key, **kwargs):
    """
    Many boto3 methods return only a limited number of results at once,
    with pagination information. This function handles the pagination to
    retrieve the entire result set.

    @param func
        The function to call to get the data.
    @param key
        The key in the result set returned from the function containing the data
        that we want
    @param kwargs
        The arguments to pass to each call to func.
    @returns
        An iterable containing the results of the consecutive calls to func.
    """
    next_token = ''
    more = True
    while more:
        result = func(**kwargs, nextToken=next_token)
        things = result[key]
        next_token = result.get('nextToken', None)
        more = next_token != None
        for thing in things:
            yield thing


def bundle(ids, block_size):
    """
    Given an iterable, bundles it into lists, each block_size long.

    @param ids
        The things to bundle
    @param block_size
        The maximum size of each block
    @returns
        An iterable of blocks, each containing up to block_size things.
    """

    block = []

    for id in ids:
        block.append(id)
        if len(block) == block_size:
            yield block
            block = []
    if len(block):
        yield block


def bundled_map(ids, block_size, func):
    """
    A map function where the mapping function is applied to blocks of items
    rather than to individual things.

    @param ids
        The things to be mapped.
    @param block_size
        The number of things to be mapped at a time
    @param func
        A mapping function that maps a block of block_size things at a time.
    """
    blocks = bundle(ids, block_size)
    for block in blocks:
        things = func(block)
        for thing in things:
            yield thing


# ====== Example usage ====== #

def get_services(cluster):
    ecs = boto3.client('ecs')
    service_ids = get_paginated_results(ecs.list_services, 'serviceArns', cluster=cluster)
    services = bundled_map(
        service_ids, 10,
        lambda x: ecs.describe_services(cluster=cluster, services=x)['services']
    )
    return services