export-cognito-to-json-and-csv.sh

#!/bin/bash
set -euo pipefail

# initial export modeled on https://workos.com/docs/migrate/aws-cognito/1-exporting-cognito-user-data

if [ $# -eq 0 ]; then
  echo "Usage: $0 <user-pool-id> <region>"
  exit 1
fi
if ! [ -x "$(command -v aws)" ]; then
  echo "Error: aws cli is not installed." >&2
  exit 1
fi
if ! [ -x "$(command -v jq)" ]; then
  echo "Error: jq is not installed." >&2
  exit 1
fi
if [ -z "$1" ] || [ -z "$2" ]; then
  echo "Error: user-pool-id and region are required." >&2
  exit 1
fi

user_pool_id="$1"
region="$2"
output_dir="cognito_exports"
file_index=1

mkdir -p "$output_dir"

export_users() {
  # list users 
  aws cognito-idp list-users --user-pool-id "$user_pool_id" --region "$region" $1 | \
  jq '.' > "$output_dir/users_$2.json"
}

next_token=""
while true; do
  next_token=$(export_users "${next_token:+--pagination-token $next_token}" "$file_index" | jq -r '.PaginationToken // empty')
  [ -z "$next_token" ] && break
  ((file_index++))
done
echo "Export complete, combining user files..."

# merge all .Users arrays in input files into one file
jq -s '                                                        
    reduce .[] as $item (null; .Users += $item.Users)' $output_dir/users_*.json > "$output_dir/users.json"

echo "Combined exported user files to $output_dir/users.json"

# use jq to create csv with top level keys, plus values of all attributes (prefixed with "attr:")
jq -r '
.Users as $users |
($users | map(keys) | add | unique | del(.[] | select(. == "Attributes"))) as $topLevelKeys |
($users | map(.Attributes.[].Name) | unique) as $attributeKeys |
($topLevelKeys + ($attributeKeys | map("attr:" + .))) as $headerRow |
($users | map(. as $user | ($topLevelKeys | map($user[.] // "" | tostring) + ($user.Attributes | from_entries as $in | $attributeKeys | map($in[.] // "" | tostring))))) as $rows |
($headerRow, $rows[]) | @csv' $output_dir/users.json > "$output_dir/users.csv"

echo "Exported csv to $output_dir/users.csv"