Shell script to export Apteligent / Crittercism crash data using their REST API

crittercism-export.sh

#!/bin/sh
# set -x

# Requires:
# JSON.sh (https://github.com/dominictarr/JSON.sh)

TOKEN=''
AUTH="Authorization: Bearer $TOKEN"

BASE='https://developers.crittercism.com/v2'
STARTDATE='2011-01-01'
ENDDATE=$(date +'%Y-%m-%d')

if [ ! -d 'cache' ]; then
    mkdir 'cache'
fi
CACHE='cache/apps.json'
if [ ! -e "$CACHE" ]; then
    curl -s -o "$CACHE" -H "$AUTH" "$BASE/apps"
fi
APPS=$(cat 'cache/apps.json' | sh JSON.sh | egrep -o '\["data","[a-z0-9]+"\]' | cut -f2 -d',' | tr -d '"' | tr -d ']')
# APPS=$(cat 'cache/apps.json')
# APPS=$(python <<EOF
# import json, sys
# apps = json.loads('$APPS')
# map(sys.stdout.write, (k + ' ' for k in apps['data'].keys()))
# EOF
# )

function get_crash_details {
    app="$1"
    crash="$2"
    cache="cache/crash-detail-${app}-${crash}.json"
    if [ ! -e "$cache" ]; then
        curl -s -o "$cache" -H "$AUTH" "$BASE/crash/${app}/${crash}?diagnostics=true&get_other_crashes=true&dailyOccurrences=true"
    fi
    cache="cache/crash-userdata-${app}-${crash}.json"
    if [ ! -e "$cache" ]; then
        curl -s -o "$cache" -H "$AUTH" "$BASE/crash/userData/${app}/${crash}"
    fi
}

function get_crashes {
    app="$1"
    count=1
    total=1
    while [ $total -gt 0 ] && [ $count -le $total ]; do
        cache="cache/crash-${app}-${count}.json"
        if [ ! -e "$cache" ]; then
            curl -s -o "$cache" -H "$AUTH" "$BASE/crash/paginatedtable/${app}?appVersion=all&startDate=${STARTDATE}&endDate=${ENDDATE}&sortBy=lastOccurred&sortOrder=descending&pageNum=${count}"
        fi
        total=$(cat "$cache" | sh JSON.sh | egrep '\["data","pagination","totalPages"\]' | cut -f2)
        count=$(cat "$cache" | sh JSON.sh | egrep '\["data","pagination","pageNum"\]' | cut -f2)
        if [ -z "$count" ]; then
            count=$total
        fi
        let count++
        hashes=$(cat "$cache" | sh JSON.sh | egrep '\["data","errors",[0-9]+,"hash"\]')
        if [ -n $"$hashes" ]; then
            echo "$hashes" | while IFS= read -r line; do
                hash=$(echo "$line" | cut -f2 | tr -d '"')
                get_crash_details "$app" "$hash"
            done
        fi
    done
}

function get_exception_details {
    app="$1"
    exception="$2"
    cache="cache/exception-detail-${app}-${exception}.json"
    if [ ! -e "$cache" ]; then
        curl -s -o "$cache" -H "$AUTH" "$BASE/exception/${app}/${exception}?diagnostics=true"
    fi
    cache="cache/exception-userdata-${app}-${exception}.json"
    if [ ! -e "$cache" ]; then
        curl -s -o "$cache" -H "$AUTH" "$BASE/exception/userData/${app}/${exception}"
    fi
}

function get_exceptions {
    app="$1"
    count=1
    total=1
    while [ $total -gt 0 ] && [ $count -le $total ]; do
        cache="cache/exception-${app}-${count}.json"
        if [ ! -e "$cache" ]; then
            curl -s -o "$cache" -H "$AUTH" "$BASE/exception/paginatedtable/${app}?appVersion=all&startDate=${STARTDATE}&endDate=${ENDDATE}&pageNum=${count}&dailyOccurrences=true"
        fi
        total=$(cat "$cache" | sh JSON.sh | egrep '\["data","pagination","totalPages"\]' | cut -f2)
        count=$(cat "$cache" | sh JSON.sh | egrep '\["data","pagination","pageNum"\]' | cut -f2)
        if [ -z "$count" ]; then
            count=$total
        fi
        let count++
        hashes=$(cat "$cache" | sh JSON.sh | egrep '\["data","errors",[0-9]+,"hash"\]')
        if [ -n $"$hashes" ]; then
            echo "$hashes" | while IFS= read -r line; do
                hash=$(echo "$line" | cut -f2 | tr -d '"')
                get_exception_details "$app" "$hash"
            done
        fi
    done
}

for APP in $APPS; do
    get_crashes "$APP"
    get_exceptions "$APP"
done