Skip to content

Instantly share code, notes, and snippets.

@chazlarson
Last active November 9, 2021 23:33
Show Gist options
  • Select an option

  • Save chazlarson/63e2dfb274a3e3178fb88485fe62943f to your computer and use it in GitHub Desktop.

Select an option

Save chazlarson/63e2dfb274a3e3178fb88485fe62943f to your computer and use it in GitHub Desktop.
script to do some Google Drive stuff. This script will install safire, create 3 projects, create 3 shared drives, create 100 Service Accounts in each project, download all the SA JSON files, add all those SAs to a Google Group, add that group to all three Shared Drives, then create four rclone remotes [one for each SD, one union]
#!/bin/bash
# Assumptions:
# 1. You have created a google project as described here: https://docs.saltbox.dev/reference/google-project-setup/
# 2. You have the credential JSON to hand
# 3. You have created a google group as described here: https://docs.saltbox.dev/reference/google-group-setup/
# 4. You have that group address to hand
# 5. You have rclone installed
# 6. You are running python 3.8 and have run sudo apt install python3.8-venv -y
# Probably other python3 works, the assumption is that I can create a venv
# SETTINGS YOU MAY WANT TO ADJUST
# How many projects to create
# 100 SAs will be created in each project
project_count=3
# If you have existing service accounts and want them merged in here
# Enter a prefix or prefixes that can be used to identify them
# alternative_prefixes="bing bang boing"
alternative_prefixes=""
# enter your google group address; if this is left empty you will be prompted for it
google_group=""
# If you have existing groups you want added to the shared drives
# enter them here.
# alternative_prefixes="google-accounts@bing.bang all-sas@bang.boing"
alternative_groups=""
# Shoudl we download the JSON files?
download_json=1
# END USER SERVICEABLE PARTS
# None of these should need to change
# Seriously, leave them alone
saf_dir=safire
config_file=~/$saf_dir/config.py
prefix_file=/opt/$saf_dir/prefix_file
creds_file=~/$saf_dir/creds/creds.json
token_file=~/$saf_dir/creds/token.pickle
grptoken_file=~/$saf_dir/creds/grptoken.pickle
# temp other control files
shared_drive_names=/opt/$saf_dir/drive_file
drive_create_log=/opt/$saf_dir/drive_create_log
drive_ids=/opt/$saf_dir/drive_ids
user_drive_list=/opt/$saf_dir/list_drives
drives_to_create=/opt/$saf_dir/drive_create
function first_half {
# i'm assuming this has been done
# sudo apt install python3.8-venv -y
echo "---------- Setting up safire"
cd /opt
if [ ! -d $saf_dir ]; then
git clone https://github.com/chazlarson/safire $saf_dir
fi
cd $saf_dir/safire
if [ ! -d safire-venv ]; then
python3 -m venv safire-venv
find_and_activate_safire
pip install wheel
pip install -r /opt/$saf_dir/requirements.txt
fi
mkdir -p /opt/sa/all
mkdir -p ~/$saf_dir/creds
echo "---------- Checking $prefix_file"
if [ -f $prefix_file ]; then
echo "---------- FOUND $prefix_file"
eval $(cat "$prefix_file")
else
echo "---------- File $prefix_file does not exist."
prefix=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c6 ;)
echo "export prefix=$prefix" > "$prefix_file"
eval $(cat "$prefix_file")
fi
echo "---------- Prefix set to $prefix"
sed "s/project_prefix = \"\"/project_prefix = \"$prefix\"/" default_config.py | sed "s/email_prefix = \".*\"/email_prefix = \"$prefix\"/" > ~/$saf_dir/config.py
echo "---------- IMPORTANT NOTE"
echo "| If you DO NOT WANT to use the standard three Shared Drives:"
echo "| Movies Music TV"
echo "| [Note that a prefix will be prepended to those names: $prefix]_"
echo "| OR if you DO NOT WANT to use the standard file system:"
echo "| Media"
echo "| ├── Movies"
echo "| ├── Music"
echo "| └── TV"
echo "| Create a file here: $user_drive_list"
echo "| the file should contain one line for each Shared Drive and path:"
echo "| Movies|/Media/Movies"
echo "| Music|/Media/Music"
echo "| TV|/Media/TV"
echo "| If you do nothing those defaults wil be used."
echo "| If this is confusing to you, choose to do nothing."
echo "---------- IMPORTANT NOTE"
}
function find_and_activate_safire() {
echo "---------- Changing dir and activating virtualenv "
cd /opt/$saf_dir/safire
source safire-venv/bin/activate
}
function do_safire_auth() {
find_and_activate_safire
echo "---------- Running authentication; follow prompts "
./safire.py auth all
echo "---------- Authenticated to Google "
}
function check_auth {
echo "---------- Checking authentication "
if [ -f $creds_file ]; then
find_and_activate_safire
if [ -f $token_file ]; then
if [ -f $grptoken_file ]; then
echo "---------- Both authentication files exist"
else
do_safire_auth
fi
else
do_safire_auth
fi
else
echo "---------- You need to copy your Google credentials JSON file to $creds_file"
exit 1
fi
}
function second_half {
check_auth
eval $(cat "/opt/$saf_dir/prefix_file")
echo "---------- Prefix set to $prefix"
if [ -z "$google_group" ]
then
echo "---------- Enter Your Google Groups email: "
read google_group
else
echo "---------- Using Google Group: $google_group"
fi
find_and_activate_safire
if [ ! -f $user_drive_list ]; then
echo 'Movies|/Media/Movies
Music|/Media/Music
TV|/Media/TV' > $user_drive_list
fi
if [ -f $user_drive_list ]; then
rm -f $shared_drive_names
while IFS="|" read -r drive dir; do
echo "${prefix}_${drive}" >> $shared_drive_names
done < $user_drive_list
else
echo "---------- no drive list file"
exit 1
fi
# get list of existing shared drives
./safire.py list drives $prefix
drive_list_file=~/$saf_dir/data/drives_list_${prefix}_.csv
# /home/chaz/safire/data/drives_list_$prefix_.csv
# now contains a CSV with drives in it there were any.
echo "---------- checking $drive_list_file"
if [ -f "$drive_list_file" ]; then
rm -f $drives_to_create
drivenames=()
while IFS= read -r line; do
# drivenames+=("$line")
OLDIFS=$IFS
IFS=','
foundOne=0
while read id driveid name
do
if [ "$name" = "$line" ]; then
foundOne=1
fi
done < $drive_list_file
if [ ! "$foundOne" ]; then
echo "---------- Need to create $line"
echo $line >> $drives_to_create
fi
IFS=$OLDIFS
done < $shared_drive_names
else
echo "---------- NO $drive_list_file"
cp $shared_drive_names $drives_to_create
fi
echo "---------- Creating Drives "
if [ -f "$drives_to_create" ]; then
rm -f $drive_create_log
./safire.py add drives $drives_to_create >> $drive_create_log
else
echo "---------- No drives to create"
fi
#
# #Creating fJUV8T-Music
#
# # Drive ID for fJUV8T-Music is 0AIUbfs4dVRkdUk9PVA
if [ ! -f $drive_ids ]; then
awk '/Drive ID/{print}' $drive_create_log > interim
awk -F' ' '{print $4"|"$6}' interim > $drive_ids
rm -f interim
fi
# fJUV8T_Movies|0ANLi8MUHFDloUk9PVA
# fJUV8T_Music|0APXyWYQ5hmmnUk9PVA
# fJUV8T_TV|0ADMGTvX4xif6Uk9PVA
echo "---------- Creating Projects "
./safire.py add projects $project_count
echo "---------- Creating Service Accounts "
./safire.py add sas $prefix
echo "---------- Adding Service Accounts to $google_group "
./safire.py add members $prefix $google_group
echo "---------- Adding Alternative Service Accounts to $google_group "
for alt_prefix in $alternative_prefixes; do
./safire.py add members $alt_prefix $google_group
done
echo "---------- Adding $google_group to Shared Drives "
./safire.py add user $google_group $prefix
echo "---------- Adding alternative groups to Shared Drives "
for alt_group in $alternative_groups; do
./safire.py add user $alt_group $prefix
done
echo "---------- Downloading Service Account JSON files "
if [ ! "$download_json" ]; then
./safire.py add jsons
else
echo "-------------- Skipping as requested "
fi
echo "---------- Syncing Service Account JSON files to /opt/sa/all "
rsync -av ~/safire/svcaccts/ /opt/sa/all
echo "---------- Creating rclone remotes "
sd_names=""
while IFS="|" read -r sd_name sd_id; do
rclone config create $sd_name drive scope=drive service_account_file=/opt/sa/all/000150.json team_drive=$sd_id
rclone touch $sd_name:/saltbox-created-$sd_name
rclone touch $sd_name:/$sd_name-mount.bin
sd_names+="$sd_name: "
done < $drive_ids
echo "---------- Creating standard file systems "
while IFS="|" read -r drive dir; do
rclone mkdir ${prefix}_${drive}:$dir
done < $user_drive_list
echo "---------- Creating rclone union remote "
rclone config create google union upstreams="$sd_names"
echo "---------- all done, deactivating virtual env "
deactivate
}
if [ -f $config_file ]; then
echo "---------- Second half!"
second_half
else
echo "---------- First half!"
first_half
fi
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment