Set of bash scripts for configuring applications via EC2 tags

ec2-config-refresh.sh

#!/bin/bash

# Refreshes configuration of this instance
#
# When ran the script will read tags currently set on this instance and save them
# as files in $tag_location. Files that do not match the tags will be removed or updated.
#
# Only the modified files will be changed.
#
# !!IMPORTANT!! Tag names cannot contain "==" (double equal sign). If they do, things will break.
#
# Copyright (c) 2019 Uber Technologies, Inc.
# SPDX-License-Identifier: Apache-2.0

set -euo pipefail

declare tag_location="/var/lib/ec2-config-reloader/tags"

function get_instance_id() {
    /usr/bin/ec2metadata --instance-id
}

function get_region() {
    local az
    local region

    az=$(/usr/bin/ec2metadata --availability-zone)
    region=${az%?}
    echo "$region"
}

#
# Get AWS tags currently set on this instance
#
function get_raw_ec2_tags() {
    local instance_id=$1
    local region=$2

    aws ec2 describe-tags --filters Name=resource-id,Values="$instance_id" --region="$region" --output=text | cut -f2,5 --output-delimiter="=="
}

#
# Return a list of tags in the format key==val as currently set in files on this machine
#
function get_raw_file_tags() {
    for file in "$tag_location"/*; do
        [[ -e $file ]] || break
        echo "$(basename "$file")"=="$(cat "$file")"
    done;
}

#
# Remove files representing the tags, given a list of tags
#
function remove_file_tags() {
    # input is in form "tag_name==tag_value"
    local tag_name

    for tag in "$@"; do
        tag_name=${tag%%==*}
        rm "$tag_location"/"$tag_name"
    done;
}

#
# Given two arguments, each a new-line separate string, returns elements only present in the second.
# argument. Example below.
# Both inputs should be sorted
# Example:
# Left =
#    One
#    Two
# Right =
#    Three
#    Two
# Will return:
#    One
#
function diff_added() {
    local left=$1
    local right=$2
    comm -1 -3 <(echo "$left") <(echo "$right")
}

#
# Write tags to files, given a list of "tag_name==tag_value" arguments
# Using tmp files to make the operation atomic
#
function write_tags() {
    local tmp_file
    local tag_name
    local tag_value

    for tag in "$@"; do
        tag_name=${tag%%==*}
        tag_value=${tag#*==}
        tmp_file=$(mktemp)
        echo "$tag_value" > "$tmp_file"
        mv "$tmp_file" "$tag_location"/"$tag_name"
        chmod 644 "$tag_location"/"$tag_name"
    done;
}

function refresh_tags_from_ec2() {
    local raw_ec2_tags
    local raw_file_tags
    local tmp_tags

    local instance_id
    local region

    local -a tags_to_remove
    local -a tags_to_write

    mkdir -p $tag_location

    instance_id=$(get_instance_id)
    region=$(get_region)

    raw_ec2_tags=$(get_raw_ec2_tags "$instance_id" "$region" | sort)
    raw_file_tags=$(get_raw_file_tags | sort)

    # Here be dragons
    # order of the below matters for "value of the tag was updated" case
    # ==> we want to provide atomic "replace" rather than "delete" and "write new" sequentially

    # first, write new + update existing tags
    tmp_tags=$(diff_added "$raw_file_tags" "$raw_ec2_tags")
    readarray -t tags_to_write < <(echo "$tmp_tags")

    if [ ! -z "${tags_to_write[0]:-}" ]; then
        write_tags "${tags_to_write[@]}"
    fi

    # re-read the tags from the disk, some might have been overriden (update case)
    raw_file_tags=$(get_raw_file_tags | sort)
    # check what really needs to be removed
    tmp_tags=$(diff_added "$raw_ec2_tags" "$raw_file_tags")
    readarray -t tags_to_remove < <(echo "$tmp_tags")

    if [ ! -z "${tags_to_remove[0]:-}" ]; then
        remove_file_tags "${tags_to_remove[@]}"
    fi

}

usage() {
    {
        echo "Reload configuration of this EC2 instance."
        echo
        echo "The script reads tags set on this EC2 instance and saves them in"
        echo "${tag_location} to be used by other resources."
        echo
        echo "Usage: ${0##*/}"
        echo
        echo "The script currently takes no arguments."
    } >&2
}

function main() {
    if [ $# -ne 0 ]; then
        usage
        exit 1
    fi
    refresh_tags_from_ec2 "$@"
}


if [ "$0" == "${BASH_SOURCE[0]}" ]; then
    main "$@"
fi

export-vars.sh

#!/bin/bash

# Copyright (c) 2019 Uber Technologies, Inc.
# SPDX-License-Identifier: Apache-2.0

set -euo pipefail

declare allowed_pattern="^[A-Za-z0-9_-]+=[A-Za-z0-9_-]+$"

export_vars() {
    IFS=';' read -ra vars

    [ -z "${vars:-}" ] && return 0

    for var in "${vars[@]}"; do
        if [[ ! "$var" =~ $allowed_pattern ]]; then
            echo "ERROR: Line contains disallowed chars: \"$var\", does not match: \"$allowed_pattern\"" >&2
            return 1
        fi
        export "${var?}"
    done
}

test-ec2-config-refresh.sh

#!/bin/bash

# Copyright (c) 2019 Uber Technologies, Inc.
# SPDX-License-Identifier: Apache-2.0

# shellcheck disable=SC1091
. "ec2-config-refresh.sh"

# unused since each test mocks out get_raw_ec2_tags too, but has to be mocked
function get_instance_id() {
    echo "id-pawel"
}

# unused since each test mocks out get_raw_ec2_tags too, but has to be mocked
function get_region() {
    echo "eu-pawel-1"
}

# test util function
function number_of_tags_on_disk() {
    find "$tag_location" -type f| wc -l
}

function test_adding_a_tag() {

    function get_raw_ec2_tags() {
        echo "tag_name==tag_value"
    }

    refresh_tags_from_ec2

    [ "$(number_of_tags_on_disk)" == 1 ] || return 1
    [ "$(cat "$tag_location"/tag_name)" == "tag_value" ] || return 1

}

function test_removing_a_tag() {

    function get_raw_ec2_tags() {
        echo ""
    }

    echo "tag_value" > "$tag_location"/tag_name
    [ "$(number_of_tags_on_disk)" == 1 ] || return 1

    refresh_tags_from_ec2

    [ "$(number_of_tags_on_disk)" == 0 ] || return 1
}

function test_nothing_changes() {

    function get_raw_ec2_tags() {
        echo "tag_name==tag_value"
    }

    echo "tag_value" > "$tag_location"/tag_name
    [ "$(number_of_tags_on_disk)" == 1 ] || return 1

    refresh_tags_from_ec2

    [ "$(number_of_tags_on_disk)" == 1 ] || return 1
    [ "$(cat "$tag_location"/tag_name)" == "tag_value" ] || return 1
}

function test_overwrite() {

    function get_raw_ec2_tags() {
        echo "tag_name==new_tag_value"
    }

    echo "old_tag_value" > "$tag_location"/tag_name
    [ "$(number_of_tags_on_disk)" == 1 ] || return 1
    [ "$(cat "$tag_location"/tag_name)" == "old_tag_value" ] || return 1

    refresh_tags_from_ec2

    [ "$(number_of_tags_on_disk)" == 1 ] || return 1
    [ "$(cat "$tag_location"/tag_name)" == "new_tag_value" ] || return 1
}

function test_no_tags() {

    function get_raw_ec2_tags() {
        echo ""
    }

    [ "$(number_of_tags_on_disk)" == 0 ] || return 1

    refresh_tags_from_ec2

    [ "$(number_of_tags_on_disk)" == 0 ] || return 1
}

function test_second_tag() {
    function get_raw_ec2_tags() {
        echo "Name1==tag_value1"
        echo "Name2==tag_value2"
    }

    echo "tag_value1" > "$tag_location"/Name1
    [ "$(number_of_tags_on_disk)" == 1 ] || return 1

    refresh_tags_from_ec2

    [ "$(number_of_tags_on_disk)" == 2 ] || return 1
    [ "$(cat "$tag_location"/Name1)" == "tag_value1" ] || return 1
    [ "$(cat "$tag_location"/Name2)" == "tag_value2" ] || return 1
}

function test_add_and_remove() {
    function get_raw_ec2_tags() {
        echo "Name1==tag_value1"
        echo "Name3==tag_value3"
    }

    echo "tag_value1" > "$tag_location"/Name1
    echo "tag_value2" > "$tag_location"/Name2
    [ "$(number_of_tags_on_disk)" == 2 ] || return 1

    refresh_tags_from_ec2

    [ "$(number_of_tags_on_disk)" == 2 ] || return 1
    [ "$(cat "$tag_location"/Name1)" == "tag_value1" ] || return 1
    [ "$(cat "$tag_location"/Name3)" == "tag_value3" ] || return 1
}

function test_with_equals_in_tag_value() {
    function get_raw_ec2_tags() {
        echo "Name1==tag=value1"
    }
    
    [ "$(number_of_tags_on_disk)" == 0 ] || return 1

    refresh_tags_from_ec2

    [ "$(number_of_tags_on_disk)" == 1 ] || return 1
    [ "$(cat "$tag_location"/Name1)" == "tag=value1" ] || return 1
}

#
# This behaviour (== in tag name) is currently unsupported, but deterministic.
# Testing just in case.
#
function test_with_equals_in_tag_name() {
    function get_raw_ec2_tags() {
        echo "tag==name==tag_value1"
    }
    
    [ "$(number_of_tags_on_disk)" == 0 ] || return 1

    refresh_tags_from_ec2

    [ "$(number_of_tags_on_disk)" == 1 ] || return 1
    [ "$(cat "$tag_location"/tag)" == "name==tag_value1" ] || return 1
}

run_tests() {
    local tests=
    local ret=0

    if [ $# -ne 0 ]; then
        tests="$*"
    else
        tests=$(compgen -A function | grep "test_")
    fi

    for test in $tests; do
        declare tag_location
        tag_location=$(mktemp -d)

        echo -n "$test: "
        if $test 2>"/tmp/test.log"; then
            echo "OK"
        else
            echo "FAILED"
            ret=1
        fi
    done

    exit $ret
}

if [ "$0" == "${BASH_SOURCE[0]}" ]; then
    run_tests "$@"
fi

test-export-vars.sh

#!/bin/bash

# Copyright (c) 2019 Uber Technologies, Inc.
# SPDX-License-Identifier: Apache-2.0

set -eou pipefail

. "${0%/*}/export-vars.sh"

declare runtime_vars_path="/tmp/test-run.service-env-tags"

test_empty() {
    export BLA=00
    echo "" > $runtime_vars_path

    export_vars < $runtime_vars_path

    [ $BLA == "00" ]
}

test_single() {
    export BLA=00
    local value=$RANDOM
    echo "BLA=$value;" > $runtime_vars_path

    export_vars < $runtime_vars_path

    [ $BLA == "$value" ]
}

test_double() {
    local value1=$RANDOM
    local value2=$RANDOM
    export BLA=00
    export BLU=00
    echo "BLA=$value1;BLU=$value2" > $runtime_vars_path

    export_vars < $runtime_vars_path

    [ $BLA == "$value1" ] && [ $BLU == "$value2" ]
}

test_all_chars() {
    local value=0123456789azAZ-_
    export BLA=00
    echo "BLA=$value;" > $runtime_vars_path

    export_vars < $runtime_vars_path

    [ $BLA == "$value" ]
}

test_invalid_chars() {
    local value1="$RANDOM"
    export BLA=00
    echo "BLA=$value1#$%#" > $runtime_vars_path

    export_vars < $runtime_vars_path

    [ "$?" -eq 1 ] && [ $BLA == 00 ]
}

test_malicious_suffix_not_executed() {
    local tmp_target_path="/tmp/test-export-vars.$RANDOM"
    echo value2="BLA=$RANDOM; touch $tmp_target_path" > $runtime_vars_path

    export_vars < $runtime_vars_path

    [ "$?" -eq 1 ] && [ ! -f $tmp_target_path ]

}

test_vars_exported() {
    echo "foo=bar" > $runtime_vars_path

    export_vars < $runtime_vars_path

    bash -c '([ "$foo" == "bar" ])'
}

run_tests() {
    local tests=
    local ret=0

    if [ $# -ne 0 ]; then
        tests="$@"
    else
        tests=$(compgen -A function | grep "test_")
    fi

    for test in $tests; do
        :>/tmp/test.log

        echo -n "$test: "
        if (set -x; $test) 2>/tmp/test.log; then
            echo -e "\033[32mOK\033[0m"
        else
            echo -e "\033[31mFAILED\033[0m"
            ret=1

            {
                echo
                echo "Trace:"

                echo -e "\033[33m"
                cat /tmp/test.log
                echo -e "\033[0m"
            } >&2
        fi
    done

    exit $ret
}

if [ "$0" == "$BASH_SOURCE" ]; then
    run_tests "$@"
fi