get-google-buckets

#!/bin/bash -e
# Copyright 2020 Paul Brewer, Economic and Financial Technology Consulting LLC
# Open Source License: The MIT License, see https://opensource.org/licenses/MIT
# rsyncs (read-only) all files in all accessible Google Cloud Storage Buckets to ~/gb/<bucketname>/
# Important: This script can use many gigabytes of Google Cloud outgoing bandwidth even in a short period of time.
# All fees for Google bandwidth are the responsibility of the user running this software.
mkdir -p ~/gb
cd ~/gb
BUCKETLIST=$(gsutil ls gs://)
for b in $BUCKETLIST
do (
   DEST=./$(echo $b | sed -e 's/gs://' | sed -e 's/\///g' )/
   mkdir -p $DEST
   echo $b $DEST
   gsutil -m rsync -r $b $DEST
)
done