Last active
November 9, 2021 23:33
-
-
Save chazlarson/63e2dfb274a3e3178fb88485fe62943f to your computer and use it in GitHub Desktop.
script to do some Google Drive stuff. This script will install safire, create 3 projects, create 3 shared drives, create 100 Service Accounts in each project, download all the SA JSON files, add all those SAs to a Google Group, add that group to all three Shared Drives, then create four rclone remotes [one for each SD, one union]
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| # Assumptions: | |
| # 1. You have created a google project as described here: https://docs.saltbox.dev/reference/google-project-setup/ | |
| # 2. You have the credential JSON to hand | |
| # 3. You have created a google group as described here: https://docs.saltbox.dev/reference/google-group-setup/ | |
| # 4. You have that group address to hand | |
| # 5. You have rclone installed | |
| # 6. You are running python 3.8 and have run sudo apt install python3.8-venv -y | |
| # Probably other python3 works, the assumption is that I can create a venv | |
| # SETTINGS YOU MAY WANT TO ADJUST | |
| # How many projects to create | |
| # 100 SAs will be created in each project | |
| project_count=3 | |
| # If you have existing service accounts and want them merged in here | |
| # Enter a prefix or prefixes that can be used to identify them | |
| # alternative_prefixes="bing bang boing" | |
| alternative_prefixes="" | |
| # enter your google group address; if this is left empty you will be prompted for it | |
| google_group="" | |
| # If you have existing groups you want added to the shared drives | |
| # enter them here. | |
| # alternative_prefixes="google-accounts@bing.bang all-sas@bang.boing" | |
| alternative_groups="" | |
| # Shoudl we download the JSON files? | |
| download_json=1 | |
| # END USER SERVICEABLE PARTS | |
| # None of these should need to change | |
| # Seriously, leave them alone | |
| saf_dir=safire | |
| config_file=~/$saf_dir/config.py | |
| prefix_file=/opt/$saf_dir/prefix_file | |
| creds_file=~/$saf_dir/creds/creds.json | |
| token_file=~/$saf_dir/creds/token.pickle | |
| grptoken_file=~/$saf_dir/creds/grptoken.pickle | |
| # temp other control files | |
| shared_drive_names=/opt/$saf_dir/drive_file | |
| drive_create_log=/opt/$saf_dir/drive_create_log | |
| drive_ids=/opt/$saf_dir/drive_ids | |
| user_drive_list=/opt/$saf_dir/list_drives | |
| drives_to_create=/opt/$saf_dir/drive_create | |
| function first_half { | |
| # i'm assuming this has been done | |
| # sudo apt install python3.8-venv -y | |
| echo "---------- Setting up safire" | |
| cd /opt | |
| if [ ! -d $saf_dir ]; then | |
| git clone https://github.com/chazlarson/safire $saf_dir | |
| fi | |
| cd $saf_dir/safire | |
| if [ ! -d safire-venv ]; then | |
| python3 -m venv safire-venv | |
| find_and_activate_safire | |
| pip install wheel | |
| pip install -r /opt/$saf_dir/requirements.txt | |
| fi | |
| mkdir -p /opt/sa/all | |
| mkdir -p ~/$saf_dir/creds | |
| echo "---------- Checking $prefix_file" | |
| if [ -f $prefix_file ]; then | |
| echo "---------- FOUND $prefix_file" | |
| eval $(cat "$prefix_file") | |
| else | |
| echo "---------- File $prefix_file does not exist." | |
| prefix=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c6 ;) | |
| echo "export prefix=$prefix" > "$prefix_file" | |
| eval $(cat "$prefix_file") | |
| fi | |
| echo "---------- Prefix set to $prefix" | |
| sed "s/project_prefix = \"\"/project_prefix = \"$prefix\"/" default_config.py | sed "s/email_prefix = \".*\"/email_prefix = \"$prefix\"/" > ~/$saf_dir/config.py | |
| echo "---------- IMPORTANT NOTE" | |
| echo "| If you DO NOT WANT to use the standard three Shared Drives:" | |
| echo "| Movies Music TV" | |
| echo "| [Note that a prefix will be prepended to those names: $prefix]_" | |
| echo "| OR if you DO NOT WANT to use the standard file system:" | |
| echo "| Media" | |
| echo "| ├── Movies" | |
| echo "| ├── Music" | |
| echo "| └── TV" | |
| echo "| Create a file here: $user_drive_list" | |
| echo "| the file should contain one line for each Shared Drive and path:" | |
| echo "| Movies|/Media/Movies" | |
| echo "| Music|/Media/Music" | |
| echo "| TV|/Media/TV" | |
| echo "| If you do nothing those defaults wil be used." | |
| echo "| If this is confusing to you, choose to do nothing." | |
| echo "---------- IMPORTANT NOTE" | |
| } | |
| function find_and_activate_safire() { | |
| echo "---------- Changing dir and activating virtualenv " | |
| cd /opt/$saf_dir/safire | |
| source safire-venv/bin/activate | |
| } | |
| function do_safire_auth() { | |
| find_and_activate_safire | |
| echo "---------- Running authentication; follow prompts " | |
| ./safire.py auth all | |
| echo "---------- Authenticated to Google " | |
| } | |
| function check_auth { | |
| echo "---------- Checking authentication " | |
| if [ -f $creds_file ]; then | |
| find_and_activate_safire | |
| if [ -f $token_file ]; then | |
| if [ -f $grptoken_file ]; then | |
| echo "---------- Both authentication files exist" | |
| else | |
| do_safire_auth | |
| fi | |
| else | |
| do_safire_auth | |
| fi | |
| else | |
| echo "---------- You need to copy your Google credentials JSON file to $creds_file" | |
| exit 1 | |
| fi | |
| } | |
| function second_half { | |
| check_auth | |
| eval $(cat "/opt/$saf_dir/prefix_file") | |
| echo "---------- Prefix set to $prefix" | |
| if [ -z "$google_group" ] | |
| then | |
| echo "---------- Enter Your Google Groups email: " | |
| read google_group | |
| else | |
| echo "---------- Using Google Group: $google_group" | |
| fi | |
| find_and_activate_safire | |
| if [ ! -f $user_drive_list ]; then | |
| echo 'Movies|/Media/Movies | |
| Music|/Media/Music | |
| TV|/Media/TV' > $user_drive_list | |
| fi | |
| if [ -f $user_drive_list ]; then | |
| rm -f $shared_drive_names | |
| while IFS="|" read -r drive dir; do | |
| echo "${prefix}_${drive}" >> $shared_drive_names | |
| done < $user_drive_list | |
| else | |
| echo "---------- no drive list file" | |
| exit 1 | |
| fi | |
| # get list of existing shared drives | |
| ./safire.py list drives $prefix | |
| drive_list_file=~/$saf_dir/data/drives_list_${prefix}_.csv | |
| # /home/chaz/safire/data/drives_list_$prefix_.csv | |
| # now contains a CSV with drives in it there were any. | |
| echo "---------- checking $drive_list_file" | |
| if [ -f "$drive_list_file" ]; then | |
| rm -f $drives_to_create | |
| drivenames=() | |
| while IFS= read -r line; do | |
| # drivenames+=("$line") | |
| OLDIFS=$IFS | |
| IFS=',' | |
| foundOne=0 | |
| while read id driveid name | |
| do | |
| if [ "$name" = "$line" ]; then | |
| foundOne=1 | |
| fi | |
| done < $drive_list_file | |
| if [ ! "$foundOne" ]; then | |
| echo "---------- Need to create $line" | |
| echo $line >> $drives_to_create | |
| fi | |
| IFS=$OLDIFS | |
| done < $shared_drive_names | |
| else | |
| echo "---------- NO $drive_list_file" | |
| cp $shared_drive_names $drives_to_create | |
| fi | |
| echo "---------- Creating Drives " | |
| if [ -f "$drives_to_create" ]; then | |
| rm -f $drive_create_log | |
| ./safire.py add drives $drives_to_create >> $drive_create_log | |
| else | |
| echo "---------- No drives to create" | |
| fi | |
| # | |
| # #Creating fJUV8T-Music | |
| # | |
| # # Drive ID for fJUV8T-Music is 0AIUbfs4dVRkdUk9PVA | |
| if [ ! -f $drive_ids ]; then | |
| awk '/Drive ID/{print}' $drive_create_log > interim | |
| awk -F' ' '{print $4"|"$6}' interim > $drive_ids | |
| rm -f interim | |
| fi | |
| # fJUV8T_Movies|0ANLi8MUHFDloUk9PVA | |
| # fJUV8T_Music|0APXyWYQ5hmmnUk9PVA | |
| # fJUV8T_TV|0ADMGTvX4xif6Uk9PVA | |
| echo "---------- Creating Projects " | |
| ./safire.py add projects $project_count | |
| echo "---------- Creating Service Accounts " | |
| ./safire.py add sas $prefix | |
| echo "---------- Adding Service Accounts to $google_group " | |
| ./safire.py add members $prefix $google_group | |
| echo "---------- Adding Alternative Service Accounts to $google_group " | |
| for alt_prefix in $alternative_prefixes; do | |
| ./safire.py add members $alt_prefix $google_group | |
| done | |
| echo "---------- Adding $google_group to Shared Drives " | |
| ./safire.py add user $google_group $prefix | |
| echo "---------- Adding alternative groups to Shared Drives " | |
| for alt_group in $alternative_groups; do | |
| ./safire.py add user $alt_group $prefix | |
| done | |
| echo "---------- Downloading Service Account JSON files " | |
| if [ ! "$download_json" ]; then | |
| ./safire.py add jsons | |
| else | |
| echo "-------------- Skipping as requested " | |
| fi | |
| echo "---------- Syncing Service Account JSON files to /opt/sa/all " | |
| rsync -av ~/safire/svcaccts/ /opt/sa/all | |
| echo "---------- Creating rclone remotes " | |
| sd_names="" | |
| while IFS="|" read -r sd_name sd_id; do | |
| rclone config create $sd_name drive scope=drive service_account_file=/opt/sa/all/000150.json team_drive=$sd_id | |
| rclone touch $sd_name:/saltbox-created-$sd_name | |
| rclone touch $sd_name:/$sd_name-mount.bin | |
| sd_names+="$sd_name: " | |
| done < $drive_ids | |
| echo "---------- Creating standard file systems " | |
| while IFS="|" read -r drive dir; do | |
| rclone mkdir ${prefix}_${drive}:$dir | |
| done < $user_drive_list | |
| echo "---------- Creating rclone union remote " | |
| rclone config create google union upstreams="$sd_names" | |
| echo "---------- all done, deactivating virtual env " | |
| deactivate | |
| } | |
| if [ -f $config_file ]; then | |
| echo "---------- Second half!" | |
| second_half | |
| else | |
| echo "---------- First half!" | |
| first_half | |
| fi |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment