Iterate over a Kinesis stream

kinesisIterate.sh

#!/bin/bash -eux

# This script iterates over a Kinesis stream and dumps out the records.
# I used it to test whether items were getting pushed to my stream.
# Requires jq.

AWS_PROFILE=dev
AWS_REGION=us-east-1
SHARD_ID=myShard
STREAM_NAME=myStream

SHARD_ITERATOR=$(AWS_PROFILE=$AWS_PROFILE aws kinesis get-shard-iterator --shard-id $SHARD_ID --shard-iterator-type TRIM_HORIZON --stream-name $STREAM_NAME --region $AWS_REGION --query 'ShardIterator')

while [[ 1 -eq 1 ]]
do
    OUTPUT=$(AWS_PROFILE=$AWS_PROFILE stringer aws kinesis get-records --region $AWS_REGION --shard-iterator $SHARD_ITERATOR)
    PAYLOADS=$(echo $OUTPUT | jq '.Records[] | select(.).Data')
    for PAYLOAD in $PAYLOADS; do
        # Trim extraneous quotes
        echo $PAYLOAD | sed -e 's/^"//' -e 's/"$//' | base64 -d
    done
    SHARD_ITERATOR=$(echo $OUTPUT | jq -r 'select(.) | .NextShardIterator')
done