download_all_pages.sh

#!/bin/bash

first_url="$1"
dir="$2"

if [[ -z "$first_url" || -z "$dir" ]]; then
    echo "usage: $0 <request url> <output directory>" >&2
    exit 1
fi

access_token=""
if [[ "$first_url" =~ access_token= ]]; then
    access_token=$(echo "$first_url" | sed -E 's/.*(&access_token=[^&]*).*/\1/')
fi

page_idx=-1

mkdir -p "$dir"

loop() {
    url="$1$access_token"
    echo "$url"
    page_idx=$((page_idx + 1))
    content=$(curl -s "$url") || exit 3
    echo "$content" > "$dir/page-${page_idx}.json"
    next_page=$(echo "$content" | sed -E 's/.*"next_page":("([^"]*)"|null).*/\2/')
    [[ -n "$next_page" ]] && loop "$next_page"
    sleep 0.2
}
loop "$first_url"