Paginated API Requests on bash with curl & jq (GitLab API Example)

gitlab-api-paginated.sh

#!/usr/bin/env bash

# GitLab API Pagination
# Fetches paginated data from GitLab API using curl and jq
#
# Usage:
#   - export SOURCE_GITLAB_URL, SOURCE_GITLAB_TOKEN for first GitLab instance
#   - export TARGET_GITLAB_URL, TARGET_GITLAB_TOKEN for second GitLab instance
#   - use api::src <path> [args] or api::dst <path> [args] to work with GitLab API
#
# Requires: curl, jq

set -euo pipefail

: "${SOURCE_GITLAB_URL:?}"
: "${SOURCE_GITLAB_TOKEN:?}"
: "${TARGET_GITLAB_URL:=$SOURCE_GITLAB_URL}"
: "${TARGET_GITLAB_TOKEN:=$SOURCE_GITLAB_TOKEN}"

# Makes a paginated GitLab API request
api() {
  local url="${1:?}" token="${2:?}" path="${3:?}" args=("${@:4}")
  local page=1 per_page=100 response on_page _url

  while :; do
    if [[ $path == *'?'* ]]; then
      _url="$url/api/v4/$path&page=$page&per_page=$per_page"
    else
      _url="$url/api/v4/$path?page=$page&per_page=$per_page"
    fi

    response="$(
      curl -sSfLH "PRIVATE-TOKEN: $token" "${args[@]}" --url "$_url" | jq -cer
    )"
    on_page="$(jq -er 'length' <<<"$response")"

    [ "$on_page" -eq 0 ] && break
    echo "$response"
    [ "$on_page" -lt "$per_page" ] && break
    ((page++)) || :
  done
}

# Make curl request to source GitLab API (api path, args...)
api::src() {
  api "$SOURCE_GITLAB_URL" "$SOURCE_GITLAB_TOKEN" "$1" "${@:2}" \
  | jq -cers '[inputs] | if length > 1 then add else .[0] end'
}

# Make curl request to target GitLab API (api path, args...)
api::dst() {
  api "$TARGET_GITLAB_URL" "$TARGET_GITLAB_TOKEN" "$1" "${@:2}" \
  | jq -cers '[inputs] | if length > 1 then add else .[0] end'
}

# Example usage
# while read -r group; do
#   echo "Create group: $group"
#   api::dst groups -X POST -d "name=$group&path=$group" > /dev/null
# done < <(api::src groups | jq -er '.[] | .name')