export collections from firestore and import to bq

export_firestore_import_to_bq.sh

#!/bin/bash

function fail {
    printf '%s\n' "$1" >&2 ## Send message to stderr.
    exit "${2-1}" ## Return a code specified by $2, or 1 by default.
}

projectId=""
bucket=""
dataset=""

ts=$(date +%Y%m%d-%H_%M_%S)
location="gs://$bucket/$ts"

# give subcollection name directly. No paths with /. So subcollections with same names won't work.
colls=( "abcd" "efgh" )

collsString=$(IFS=, ; echo "${colls[*]}")

gcloud config set project "$projectId" || fail "gcloud config set project $projectId failed"

echo "Collections String: $collsString"
echo "Exporting to: $location"
gcloud beta firestore export --collection-ids="$collsString" "$location" || fail "gcloud firestore export failed"

for coll in "${colls[@]}"
do
  echo "Loading Collection to bq: $coll"

  bq load --replace --source_format=DATASTORE_BACKUP "$dataset"."$coll" "$location"/all_namespaces/kind_"$coll"/all_namespaces_kind_"$coll".export_metadata
done