deploy-lookml-headless-production

#!/bin/bash
# Author: [email protected]
#
# The purpose of this script is to copy a LookML project and some of its dependencies (connections, user attributes, models) from
# a source environment to a destination environment. This script will not copy users, groups, roles, or permission sets.

# Please update the following environment variables:
export SRC_REPO_URL="https://github.com/drewgillson/looker-data-applications-exercise.git"
export SRC_CLIENT_ID="..."
export SRC_CLIENT_SECRET="..."
export SRC_URL="http://cluster-a0c230e8.eastus.cloudapp.azure.com"

export DEST_CLIENT_ID="..."
export DEST_CLIENT_SECRET="..."
export DEST_HOST="ec2-44-225-46-169.us-west-2.compute.amazonaws.com"
export DEST_MODELS_DIR="/mnt/lookerfiles"
export DEST_PROJECT="looker-data-applications-exercise"
export DEST_SSH_USER="ubuntu"
export DEST_SSH_PORT="22"
export DEST_URL="https://looker-elb-1979379080.us-west-2.elb.amazonaws.com"
# No updates needed past this point

export SRC_API_URL="$SRC_URL:19999/api/3.1"
export SRC_AUTH_TOKEN=`curl -k -s -X POST -d "client_id=$SRC_CLIENT_ID&client_secret=$SRC_CLIENT_SECRET" "$SRC_API_URL/login" | jq -r ".access_token"`
export DEST_API_URL="$DEST_URL:19999/api/3.1"
export DEST_AUTH_TOKEN=`curl -k -s -X POST -d "client_id=$DEST_CLIENT_ID&client_secret=$DEST_CLIENT_SECRET" "$DEST_API_URL/login" | jq -r ".access_token"`

type jq >/dev/null 2>&1 || { echo >&2 "Please install the jq utility. Aborting."; exit 1; }

# The script accepts an optional parameter -t UNIX_TIMESTAMP that will deploy content found in an existing timestamp directory
# that was created during a previous execution of the script. If -t is not passed, the script downloads the current content
# from the source environment and saves it into a new timestamp directory.
export TS="`date +%s`"
export FORCE_TS="false"
while getopts ":t:" flag
do
    case $flag in
        t) TS=$OPTARG
           FORCE_TS="true"
           ;;
        \?) echo "ERROR: Invalid option, restore from a timestamp like this: bash $0 -t 1566512069"
            exit 1
            ;;
    esac
done
 
# Switch Looker user on destination to development mode
function enter_dev_mode () {
    curl -k -s -d '{"workspace_id":"dev"}' -X PATCH "$DEST_API_URL/session?access_token=$DEST_AUTH_TOKEN" | jq
}
 
# Download the master branch from the source repository and copy it to the destination
function pull_source_repo_and_deploy () {
    # Remove an existing bare repository with the same name if it exists (this needs to be cleaned up because Looker doesn't delete the directory when it deletes a project)
    ssh -t -l $DEST_SSH_USER -p $DEST_SSH_PORT $DEST_HOST "sudo -S rm -rf $DEST_MODELS_DIR/bare_models/$DEST_PROJECT.git"
 
    # Create a new project (this will fail if one already exists)
    curl -k -s -d "{\"id\":\"$DEST_PROJECT\",\"name\":\"$DEST_PROJECT\",\"pull_request_mode\":\"off\",\"validation_required\":false,\"folders_enabled\":true,\"allow_warnings\":false,\"uses_git\":true,\"is_example\":false}" -X POST "$DEST_API_URL/projects?access_token=$DEST_AUTH_TOKEN" | jq
 
    # Create a new bare repository
    curl -k -s -d "{\"id\":\"$DEST_PROJECT\",\"name\":\"$DEST_PROJECT\",\"git_remote_url\":null,\"git_service_name\":\"bare\"}" -X PATCH "$DEST_API_URL/projects/$DEST_PROJECT?access_token=$DEST_AUTH_TOKEN" | jq
 
    if [ "$FORCE_TS" = "false" ]; then
        # Git clone from source repository to the local file system
        rm -rf $DEST_PROJECT
        mkdir $DEST_PROJECT
        git clone $SRC_REPO_URL ./$DEST_PROJECT
        rm -rf ./$DEST_PROJECT/.git
        find $DEST_PROJECT -name "*.gitkeep" -type f -delete
        # Copy the contents of the git repository that we just cloned to the local file system to the models directory on the destination
        # First, the files on the share need to be writable
        ssh -t -l $DEST_SSH_USER -p $DEST_SSH_PORT $DEST_HOST "sudo -S chmod -R 777 $DEST_MODELS_DIR/models/$DEST_PROJECT"
        scp -P $DEST_SSH_PORT -rp $DEST_PROJECT $DEST_SSH_USER@$DEST_HOST:$DEST_MODELS_DIR/models
 
        # Make a copy of the files in a time-stamped release directory
        mv $DEST_PROJECT $DEST_PROJECT$TS
        mkdir $DEST_PROJECT$TS/json
    else
        # Or, copy the contents of an existing time-stamped release directory to the destination
        mv $DEST_PROJECT$TS/json json
        mv $DEST_PROJECT$TS $DEST_PROJECT
        ssh -t -l $DEST_SSH_USER -p $DEST_SSH_PORT $DEST_HOST "sudo -S chmod -R 777 $DEST_MODELS_DIR/models/$DEST_PROJECT"
        scp -P $DEST_SSH_PORT -rp $DEST_PROJECT $DEST_SSH_USER@$DEST_HOST:$DEST_MODELS_DIR/models
        mv $DEST_PROJECT $DEST_PROJECT$TS
        mv json $DEST_PROJECT$TS/json
    fi
 
    # Commit the uploaded repository contents to a new repository in bare_models
    ssh -t -l $DEST_SSH_USER -p $DEST_SSH_PORT $DEST_HOST "cd $DEST_MODELS_DIR/models/$DEST_PROJECT && git add * && git commit -am 'Initial commit' && sudo -S git push -u origin master"
    curl -k -s "$DEST_URL/webhooks/projects/$DEST_PROJECT/deploy"
}
 
# Copy top-level spaces and their child looks from the source to the destination. If -t parameter is passed,
# use JSON files from timestamp directory rather than downloading most recent ones
function create_spaces_and_looks () {
    if [ "$FORCE_TS" = "false" ]; then
        curl -k -s "$SRC_API_URL/folders/search?fields=name%2Cparent_id%2Cid&parent_id=1&access_token=$SRC_AUTH_TOKEN" | jq -c '.[]' > ./$DEST_PROJECT$TS/json/folders.json
    fi
    while read -u 3 row
    do
        echo $row | jq
        export SPACE_ID="`echo $row | jq '.id'`"
        # Make sure spaces in folder names are properly URI encoded
        export SPACE_NAME="`echo $row | jq -r '.name' | jq -sRr @uri | sed 's/%0A//'`"
        # Create new space
        curl -k -s -d "$row" -H "Content-Type:application/json" -X POST "$DEST_API_URL/folders?access_token=$DEST_AUTH_TOKEN" | jq
        curl -k -s "$DEST_API_URL/folders/search?fields=id&name=$SPACE_NAME&access_token=$DEST_AUTH_TOKEN" | jq
        export NEW_SPACE_ID="`curl -k -s "$DEST_API_URL/folders/search?fields=id&name=$SPACE_NAME&access_token=$DEST_AUTH_TOKEN" | jq '.[0].id'`"
        if [ "$FORCE_TS" = "false" ]; then
            curl -k -s "$SRC_API_URL/looks/search?fields=id&space_id=$SPACE_ID&access_token=$SRC_AUTH_TOKEN" | jq -c '.[]' > ./$DEST_PROJECT$TS/json/looks_in_space_$SPACE_ID.json
        fi
        while read -u 4 look
        do
            # In order to create looks on the destination we have to create new queries first
            export LOOK_ID="`echo $look | jq '.id'`"
            if [ "$FORCE_TS" = "false" ]; then
                curl -k -s "$SRC_API_URL/looks/$LOOK_ID?access_token=$SRC_AUTH_TOKEN" > ./$DEST_PROJECT$TS/json/look_$LOOK_ID.json
            fi
            export QUERY_ID="`cat ./$DEST_PROJECT$TS/json/look_$LOOK_ID.json | jq '.query.id'`"
            if [ "$FORCE_TS" = "false" ]; then
                curl -k -s "$SRC_API_URL/queries/$QUERY_ID?access_token=$SRC_AUTH_TOKEN" > ./$DEST_PROJECT$TS/json/query_$QUERY_ID.json
            fi
            export QUERY="`cat ./$DEST_PROJECT$TS/json/query_$QUERY_ID.json`"
 
            # Create a new query on the destination to associate our new look with
            export NEW_QUERY_ID="`curl -k -s -d "$QUERY" -H "Content-Type:application/json" -X POST "$DEST_API_URL/queries?access_token=$DEST_AUTH_TOKEN" | jq '.id'`"

            # Substitute the query ID on the source look with the new query ID from the destination
            sed -i '' -e "s/\"query_id\":$QUERY_ID/\"query_id\":$NEW_QUERY_ID/" ./$DEST_PROJECT$TS/json/look_$LOOK_ID.json
            sed -i '' -e "s/\"space_id\":$SPACE_ID/\"space_id\":$NEW_SPACE_ID/" ./$DEST_PROJECT$TS/json/look_$LOOK_ID.json
            export NEW_LOOK="`cat ./$DEST_PROJECT$TS/json/look_$LOOK_ID.json`"
 
            # We can copy the source look to the destination now that it has been associated with a new query
            curl -k -s -d "$NEW_LOOK" -H "Content-Type:application/json" -X POST "$DEST_API_URL/looks?access_token=$DEST_AUTH_TOKEN" | jq
 
        done 4< ./$DEST_PROJECT$TS/json/looks_in_space_$SPACE_ID.json
    done 3< ./$DEST_PROJECT$TS/json/folders.json
}
 
# Copy connections from the source to the destination. Note that database connection passwords need to be set manually
function create_connections () {
    if [ "$FORCE_TS" = "false" ]; then
        curl -k -s "$SRC_API_URL/connections?access_token=$SRC_AUTH_TOKEN" | jq -c '.[]' > ./$DEST_PROJECT$TS/json/connections.json
    fi
    cat ./$DEST_PROJECT$TS/json/connections.json | while read row
    do
        echo $row | jq
        # TODO: passwords aren't specified automatically and you need to log in to the UI to finish setting up the connection, or you can insert a password node into the JSON body in $row
        curl -k -s -d "$row" -H "Content-Type:application/json" -X POST "$DEST_API_URL/connections?access_token=$DEST_AUTH_TOKEN" | jq
    done
}
 
# Copy models from the source to the destination
function create_models () {
    if [ "$FORCE_TS" = "false" ]; then
        curl -k -s "$SRC_API_URL/lookml_models?access_token=$SRC_AUTH_TOKEN" | jq -c '.[]' > ./$DEST_PROJECT$TS/json/models.json
    fi
    cat ./$DEST_PROJECT$TS/json/models.json | while read row
    do
        echo $row | jq
        curl -k -s -d "$row" -H "Content-Type:application/json" -X POST "$DEST_API_URL/lookml_models?access_token=$DEST_AUTH_TOKEN" | jq
    done
}
 
# Copy user attributes from the source to the destination
function create_user_attributes () {
    if [ "$FORCE_TS" = "false" ]; then
        # The hidden_value_domain_whitelist attribute needs to be removed in order to form a valid request
        curl -k -s "$SRC_API_URL/user_attributes?access_token=$SRC_AUTH_TOKEN" | jq -c '.[]' | sed 's/,"hidden_value_domain_whitelist":null,/,/' > ./$DEST_PROJECT$TS/json/user_attributes.json
    fi
    cat ./$DEST_PROJECT$TS/json/user_attributes.json | while read row
    do
        echo $row | jq
        curl -k -s -d "$row" -H "Content-Type:application/json" -X POST "$DEST_API_URL/user_attributes?access_token=$DEST_AUTH_TOKEN" | jq
    done
}
 
# Script entry point:
enter_dev_mode
pull_source_repo_and_deploy
create_connections
create_user_attributes
create_models
create_spaces_and_looks