Setup my local environment for development/testing at StreamSets

README.md

I have a set of files in my ~/bin folder named env.*.zsh that setup the env vars and aliases that I need for development or testing on different environments. One of the env vars defined is $HELP, that I can echo to be remined of what alias I may want to use in that environment.

All the env.*.zsh files rely on some stuff specific to my laptop:

• credentials stored in IBM's 1password
  • local.platform: username and password
  • local.3x: username and password
  • platform.dev.my-org: username, password, website, CRED_ID and CRED_TOKEN
• python virtual envs:
  • $HOME/py-envs/3.x-stf/bin/activate
  • $HOME/py-envs/4.x-stf/bin/activate

Some external dependencies:

• a script in another gist that updates the prompt set-prompt-for-env.zsh
• jq
• HTTPie
• Rancher Desktop using docker

env.dev.my-org.zsh

#!/bin/echo Run: source

# export FIREBASE_API_KEY=AIzaSyA1sEvvjfTPDbjexJLeSm3UFDLM7Rm_v0k
export ONE_PASSWORD_ITEM=platform.dev.my-org
name=$(basename $0); name=$name:r:s/env.//

source env.platform.zsh $name

env.local.3x.zsh

#!/bin/echo Run: source

if [ ! -S /var/run/docker.sock ]; then 
  echo '/var/run/docker.sock not found. Run:'
  echo 'sudo ln -s $HOME/.rd/docker.sock /var/run/docker.sock'
  return 
fi

name=$(basename $0); name=$name:r:s/env.//
echo $name
source set-prompt-for-env.zsh $name 

export ONE_PASSWORD_ITEM=local.3x
export DPM_PORT=${DPM_PORT:-20631}
export DPM_PORT_ADMIN=$(($DPM_PORT + 1))
export DPM_URL=http://host.docker.internal:$DPM_PORT
export DPM_USER=$(op read "op://Employee/$ONE_PASSWORD_ITEM/username")
export DPM_PASSWORD=$(op read "op://Employee/$ONE_PASSWORD_ITEM/password")
export SDC_VERSION=5.8.0
export SDC_START_EXTRA_PARAMS="--stage-lib orchestrator jdbc"

export DPM_REPO=$HOME/src/streamsets/domainserver-3x
export JAVA_HOME=$(/usr/libexec/java_home -v1.8)

export DPM_CMD_SAMPLE_STF='stf \
  -v \
  --docker-extra-options="-p 5678:5678/tcp" \
test \
  -v -ra --capture=no \
  --sch-server-url $DPM_URL \
  --sch-username $DPM_USER --sch-password $DPM_PASSWORD \
  --sch-authoring-sdc $SCH_AUTHORING_SDC \
jobs/test_jobs.py::test_simple_job_lifecycle'

export DPM_CMD_START_SDC='stf \
  --env-var HOST_HOSTNAME=host.docker.internal \
start sdc \
  --enable-base-http-url private \
  --version $SDC_VERSION \
  --sch-server-url $DPM_URL \
  --sch-username $DPM_USER --sch-password $DPM_PASSWORD \
  --sch-executor-sdc-label $USER \
  $(echo $SDC_START_EXTRA_PARAMS)'
# This last echo is required. I do not get why

alias ch-set-dpm-dist='pushd $DPM_REPO/dist/target/dist/streamsets-dpm-*/ && export DPM_DIST=$(pwd) && popd'
alias setup.dpm-dist=ch-set-dpm-dist

# TO-DO: make DPM_LOG and DPM_CONF dependant on $DPM_PORT, to be able to run multi-instance
alias setup.dbs-and-rebuild='python ~/src/streamsets/dpm-scripts/dpm-utils.py --host host.docker.internal \
   --port $DPM_PORT --expiration=30 --java-version 1.8 --dpm-dir $DPM_REPO \
   --database postgres --database-version 14.6 --build-tool maven \
   install clean && \
 ch-set-dpm-dist && \
 $DPM_DIST/dev/00-initpostgresql.sh && \
 $DPM_DIST/dev/01-initdb-java.sh && \
 $DPM_DIST/dev/02-initsecurity-java.sh $DPM_PORT && \
 $DPM_DIST/bin/streamsets dpmcli security systemId -c && \
 $DPM_DIST/bin/streamsets dpmcli security activationKey --dev --devExpires=30 && \
 find $DPM_DIST/etc -name "*.properties" -exec sed -i "" "s/18631/$DPM_PORT/g" {} \; && \
 find $DPM_DIST/etc -name "*.properties" -exec sed -i "" "s/18632/$DPM_PORT_ADMIN/g" {} \; && \
 find $DPM_DIST/etc -name "*.properties" -exec sed -i "" "s/\(dpm\.componentId=[a-zA-Z_]*\)000/\1${DPM_PORT}/g" {} \; && \
 sed -i "" "s/#org.quartz.jobStore.driverDelegateClass/org.quartz.jobStore.driverDelegateClass/" $DPM_DIST/etc/scheduler-app.properties'
# We are modifying the componentId so that we can have a fake HA environment with multiple instances on different ports

alias ch-run='ch-set-dpm-dist && $DPM_DIST/bin/streamsets dpm'

alias setup.test-org='ch-set-dpm-dist && \
 python ~/src/streamsets/dpm-scripts/dpm-utils.py \
   --dpm-dir $DPM_REPO --host host.docker.internal --port $DPM_PORT \
   create-test-org && \
 export adminToken=$(ch-get-token $DPM_URL admin@admin admin@admin) && \
 echo -n '\''[{"id": "dpm.enable.events","value": "true"},{"id": "dpm.enforce.permissions", "value": "true"}]'\'' | \
   http POST $DPM_URL/security/rest/v1/organization/test/configs \
   X-Requested-By:SCH X-SS-REST-CALL:true X-SS-User-Auth-Token:$adminToken && \
 http --pretty=format GET $DPM_URL/security/rest/v1/organization/test/configs \
   X-Requested-By:SCH X-SS-REST-CALL:true X-SS-User-Auth-Token:$adminToken | grep -e subscriptions -e "Enforce permissions" -A 4'

source $HOME/py-envs/3.x-stf/bin/activate

alias echo.setup.sdc='non_redacted=$(eval echo $DPM_CMD_START_SDC) && echo ${non_redacted//$DPM_PASSWORD/xxxx}'
alias setup.sdc='echo.setup.sdc && eval $DPM_CMD_START_SDC'

function start_sdc_and_set_authoring_var() {
    echo.setup.sdc
    sdc_line=$(eval $DPM_CMD_START_SDC | tee /dev/tty | grep "can be followed along")
    export SCH_AUTHORING_SDC=$(echo $sdc_line | grep -o '\S*$')
    echo "SCH_AUTHORING_SDC is $SCH_AUTHORING_SDC"
}

alias setup.sdcs='setup.sdc && start_sdc_and_set_authoring_var'
alias ch-setup-sdc=setup.sdc
alias ch-setup-sdcs=setup.sdcs

alias setup.test-org-and-sdcs='setup.test-org && setup.sdc && start_sdc_and_set_authoring_var'
alias ch-setup-test-org-and-sdcs=setup.test-org-and-sdcs
unalias ch-rebuild-and-setup 2>/dev/null || true

export HELP="Helpful commands. 'echo \$HELP' if you need a reminder.
# Restart postgres, influxdb and rebuild
setup.dbs-and-rebuild
# Set \$DPM_DIST
setup.dpm-dist
# Run SCH
ch-run
# Create test org + start SDCs. Uses \$SDC_VERSION and \$SDC_START_EXTRA_PARAMS
setup.test-org-and-sdcs
# Start a single SDC
echo.setup.sdc && eval \$DPM_CMD_START_SDC
# Sample STF execution.
$DPM_CMD_SAMPLE_STF"

echo ---
echo $HELP

env.local.platform.zsh

#!/bin/echo Run: source

if [ ! -S /var/run/docker.sock ]; then 
  echo '/var/run/docker.sock not found. Run:'
  echo 'sudo ln -s $HOME/.rd/docker.sock /var/run/docker.sock'
  return 
fi

name=$(basename $0); name=$name:r:s/env.//
echo $name
source set-prompt-for-env.zsh $name 

# Build ####
export JAVA_HOME=$(/usr/libexec/java_home -v11)
export DPM_REPO=$HOME/src/streamsets/domainserver-master

# FIREBASE_API_KEY does not need to be kept safe
export FIREBASE_API_KEY=AIzaSyDRR6wl5XHAB3PfEXDDF2BmwonQcYAywIU

# Auth ####
export ASTER_LOGIN_URL=http://host.docker.internal:4200
export ASTER_SCH_URL=http://host.docker.internal:18631
export ASTER_EMAIL=$(op read "op://Employee/local.platform/username")
export ASTER_EMAIL_PWD=$(op read "op://Employee/local.platform/password")
export ASTER_URL=${ASTER_LOGIN_URL}
export ASTER_USER_EMAIL=${ASTER_EMAIL}
export ASTER_USER_PASSWORD=${ASTER_EMAIL_PWD}

# Tests ####
export DATAOPS_TEST_EMAIL_PASSWORD=${DATAOPS_TEST_EMAIL_PASSWORD:-UniterestingValue}
export SDC_VERSION=5.8.0
export SDC_START_EXTRA_PARAMS="--stage-lib orchestrator jdbc"

source $HOME/py-envs/4.x-stf/bin/activate

alias setup.test-org-and-sdcs="$HOME/src/streamsets/dpm-scripts/platform/setup-control-plane-testing.sh && source $HOME/src/streamsets/dpm-scripts/platform/.stf-env.sh"
alias setup.credentials="source $HOME/src/streamsets/dpm-scripts/platform/.stf-env.sh"

export HELP="Helpful commands. 'echo \$HELP' if you need a reminder.
# Start the services
python $HOME/src/streamsets/dpm-scripts/dpm-platform.py --batch --verbose run
# Create test org + start SDCs. Uses \$SDC_VERSION and \$SDC_START_EXTRA_PARAMS
setup.test-org-and-sdcs
# Set the environment vars needed for STF tests
setup.credentials"

echo $HELP

env.platform.zsh

#!/bin/echo Run: source

if [ ! -S /var/run/docker.sock ]; then 
  echo '/var/run/docker.sock not found. Run:'
  echo 'sudo ln -s $HOME/.rd/docker.sock /var/run/docker.sock'
  return 
fi

if [ -z "$ONE_PASSWORD_ITEM" ]; then
  echo 'The environment variable ONE_PASSWORD_ITEM needs to be set, and needs to specify a 1password item in the Employee vault.'
  echo 'The item needs to specify  `username`, `password`, `website`, `CRED_ID` and `CRED_TOKEN`'
  return
fi

name=${1:-$ONE_PASSWORD_ITEM}
echo $name
source set-prompt-for-env.zsh $name 

# Auth ####
export ASTER_EMAIL=$(op read "op://Employee/$ONE_PASSWORD_ITEM/username")
export ASTER_EMAIL_PWD=$(op read "op://Employee/$ONE_PASSWORD_ITEM/password")
export ASTER_LOGIN_URL=$(op read "op://Employee/$ONE_PASSWORD_ITEM/website")
export CRED_ID=$(op read "op://Employee/$ONE_PASSWORD_ITEM/CRED_ID")
export CRED_TOKEN=$(op read "op://Employee/$ONE_PASSWORD_ITEM/CRED_TOKEN")
export ASTER_URL=${ASTER_LOGIN_URL}
export ASTER_USER_EMAIL=${ASTER_EMAIL}
export ASTER_USER_PASSWORD=${ASTER_EMAIL_PWD}

# Tests ####
export DATAOPS_TEST_EMAIL_PASSWORD=${DATAOPS_TEST_EMAIL_PASSWORD:-UniterestingValue}
export SDC_VERSION=5.8.0
export SDC_START_EXTRA_PARAMS="--stage-lib orchestrator jdbc"

source $HOME/py-envs/4.x-stf/bin/activate

export DPM_CMD_START_SDC='stf \
  --env-var HOST_HOSTNAME=host.docker.internal \
start sdc \
  --enable-base-http-url private \
  --version $SDC_VERSION \
  --aster-server-url $ASTER_URL \
  --sch-credential-id $CRED_ID \
  --sch-token $CRED_TOKEN \
  --sch-executor-sdc-label $USER \
  $(echo $SDC_START_EXTRA_PARAMS)'
# This last echo is required. I do not get why

export DPM_CMD_SAMPLE_STF='stf \
  -v \
  --docker-extra-options="-p 5678:5678/tcp" \
  --env-var DATAOPS_TEST_EMAIL_PASSWORD \
test \
  -v -ra --capture=no \
  --sch-credential-id $CRED_ID \
  --sch-token $CRED_TOKEN \
  --aster-server-url $ASTER_LOGIN_URL \
  --aster-email $ASTER_EMAIL \
  --aster-email-password $ASTER_EMAIL_PWD \
  --sch-authoring-sdc $SCH_AUTHORING_SDC \
  --sch-executor-sdc-label $USER \
control_plane/jobs/test_jobs.py::test_simple_job_lifecycle'

alias echo.setup.sdc='non_redacted=$(eval echo $DPM_CMD_START_SDC) && echo ${non_redacted//$CRED_TOKEN/xxxx}'
alias setup.sdc='echo.setup.sdc && eval $DPM_CMD_START_SDC'

function start_sdc_and_set_authoring_var() {
    echo.setup.sdc
    sdc_line=$(eval $DPM_CMD_START_SDC | tee /dev/tty | grep "SDC ID:")
    export SCH_AUTHORING_SDC=$(echo $sdc_line | sed 's/.*\: //')
    echo "SCH_AUTHORING_SDC is $SCH_AUTHORING_SDC"
}

alias setup.sdcs='setup.sdc && start_sdc_and_set_authoring_var'
alias ch-setup-sdc=setup.sdc
alias ch-setup-sdcs=setup.sdcs

export HELP="Helpful commands. 'echo \$HELP' if you need a reminder.
# Start 2 SDCs and capture the SHC_AUTHORING_SDC
ch-setup-sdcs
# Start a single SDC
ch-setup.sdc
# Sample STF execution.
$DPM_CMD_SAMPLE_STF"

echo ---
echo $HELP