!opsman_tools_vm_helpers

# an opinionated Tools VM set up based on using an OpsMan VM from pivnet
# Intended to be used with BUCC, specifically https://github.com/matthewcosgrove/lab-ops

# Intructions for pivnet
# set env var
export PIVNET_TOKEN="grab a token from the GUI" # Deprecated approach but works for now

# Install
wget https://github.com/pivotal-cf/pivnet-cli/releases/download/v1.0.4/pivnet-linux-amd64-1.0.4
sudo chmod +x pivnet-linux-amd64-1.0.4
sudo mv pivnet-linux-amd64-1.0.4 /usr/local/bin/pivnet

# Download
pivnet login --api-token="$PIVNET_TOKEN"
pivnet download-product-files --product-slug='ops-manager' --release-version='2.9.5' --product-file-id=713268

# APPENDIX

# On Ubuntu these tools are useful
http://apt.starkandwayne.com/

# Alternative Windows binary via PowerShell started as admin
wget https://github.com/pivotal-cf/pivnet-cli/releases/download/v1.0.4/pivnet-windows-amd64-1.0.4 -OutFile pivnet
# stick it on the path

# All systems go on Windows..
# https://www.digitalocean.com/community/tutorials/how-to-install-go-and-set-up-a-local-programming-environment-on-windows-10
go get -u github.com/pivotal-cf/pivnet-cli

# And for extra fun
sudo vim /etc/sudoers
# After last line #includedir /etc/sudoers.d
username ALL=(ALL) NOPASSWD: ALL

.bash_aliases

alias gs="git s"
alias gp='git stash; git pull --rebase; git stash pop; git push'

.functions

function init-govc() {
  source /home/ubuntu/lab-ops/bin/init_govc.sh
}

function init_govc() {
  init-govc
}

function mgmt() {
  if [[ -f "$BUCC_WRAPPER_ROOT_DIR/bin/bucc_env" ]];then
    STATE_VARS_STORE="${BBL_STATE_DIR}"/vars/director-vars-store.yml
    if [[ ! -f "${STATE_VARS_STORE}" ]]; then
      echo "No BUCC state found in ${STATE_VARS_STORE}. Skipping set up on login"
      return 0
    fi

    echo "Preparing config for BUCC CLIs by sourcing bin/bucc_env.."
    echo "Please wait.."
    source <($BUCC_WRAPPER_ROOT_DIR/bin/bucc_env) > /dev/null
    echo "Done"
  else
    echo "BUCC env script not found in $BUCC_WRAPPER_ROOT_DIR/bin"
    ls $BUCC_WRAPPER_ROOT_DIR/bin
  fi
}

.gitconfig

[user]
	name = matthewcosgrove
	email = [email protected]
[credential]
	helper = /usr/share/doc/git/contrib/credential/libsecret/git-credential-libsecret
[push]
	default = simple
[core]
	editor = vim
[alias]
    # src: https://gist.github.com/mwhite/6887990
    # one-line log
    l = log --pretty=format:"%C(yellow)%h\\ %ad%Cred%d\\ %Creset%s%Cblue\\ [%cn]" --decorate --date=short

    a = add
    ap = add -p
    c = commit --verbose
    ca = commit -a --verbose
    cm = commit -m
    cam = commit -a -m
    m = commit --amend --verbose

    d = diff
    ds = diff --stat
    dc = diff --cached

    s = status # -s
    co = checkout
    cob = checkout -b
    # list branches sorted by last modified
    b = "!git for-each-ref --sort='-authordate' --format='%(authordate)%09%(objectname:short)%09%(refname)' refs/heads | sed -e 's-refs/heads/--'"

    # list aliases
    la = "!git config -l | grep alias | cut -c 7-"
    rao = remote add origin
    ac = !git add . && git commit -am
    pushitgood = push -u origin --all

.profile

export RAILS_ENV=production

alias uaac='BUNDLE_GEMFILE=/home/tempest-web/tempest/web/vendor/uaac/Gemfile bundle exec uaac'

export BUCC_WRAPPER_ROOT_DIR="/home/ubuntu/lab-ops"
export BBL_STATE_DIR="/home/ubuntu/lab-ops-state/state" # BBL_STATE_DIR is the convention use by BUCC https://github.com/starkandwayne/bucc/blob/2af7a2b47a151007b4db089f2349aa58bce8d1fc/bin/bucc#L8
. /home/ubuntu/.bash_aliases
if [ -f /home/ubuntu/.functions ]; then
    . /home/ubuntu/.functions
    mgmt # bin/bucc_env sourcing
fi

export GIT_EDITOR=vim
export PATH=/home/ubuntu/.local/bin:$PATH:/usr/local/go/bin:~/go/bin

00_install_via_stark_and_wayne.sh

#!/bin/bash
# From: http://apt.starkandwayne.com/

set -eu
apt-get update
apt install gnupg wget -y
wget -q -O - https://raw.githubusercontent.com/starkandwayne/homebrew-cf/master/public.key | apt-key add -
echo "deb http://apt.starkandwayne.com stable main" | tee /etc/apt/sources.list.d/starkandwayne.list
apt-get update

apt install govc
apt install spruce
apt install jq
apt install pivnet-cli

00_set_up.sh

#!/bin/bash

set -eu
# ssh-keygen -t rsa -q -f "$HOME/.ssh/id_rsa" -N ""
# Add key to settings https://github.com/settings/keys
# git clone https://gist.github.com/matthewcosgrove/9e77386991d77873ca6700acda9225bc
# or
# git clone [email protected]:9e77386991d77873ca6700acda9225bc.git

ln -nsf /home/ubuntu/gists/9e77386991d77873ca6700acda9225bc/.gitconfig ~/.gitconfig
ln -nsf /home/ubuntu/gists/9e77386991d77873ca6700acda9225bc/.bash_aliases ~/.bash_aliases
ln -nsf /home/ubuntu/gists/9e77386991d77873ca6700acda9225bc/.functions ~/.functions
ln -nsf /home/ubuntu/gists/9e77386991d77873ca6700acda9225bc/.vimrc ~/.vimrc

sudo add-apt-repository ppa:jonathonf/vim -y
sudo apt update
sudo apt install vim -y

urls=( https://github.com/luan/vim-concourse \
https://github.com/hashivim/vim-hashicorp-tools.git \
https://github.com/pearofducks/ansible-vim.git \
)

for url in "${urls[@]}"
do
   suffix_gone=${url%".git"}
   repo_name=${suffix_gone##h*/}
   dir="~/.vim/pack/$repo_name/start/$repo_name/"
   mkdir -p $dir
   pushd $dir
   git clone $url
   popd
done

sudo apt install bash-completion -y
sudo apt install shellcheck -y
sudo apt install tree -y

not-bosh-io-merge-result-minio.yml

fakeyamlrootforsprucetowork:
- path: /releases/name=minio
  type: replace
  value:
    name: minio
    sha1: f0f352f04b8d8bef3c390a720c3d1e733ed36668
    url: "https://dfdtmjpfvu0dy.cloudfront.net/ab2119b3-46ff-44fe-5630-dcc6690c21c7?response-content-disposition=attachment%3B%20filename%3Dminio-boshrelease-2020-07-31T03-39-05Z.tgz&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9kZmR0bWpwZnZ1MGR5LmNsb3VkZnJvbnQubmV0L2FiMjExOWIzLTQ2ZmYtNDRmZS01NjMwLWRjYzY2OTBjMjFjNz9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPWF0dGFjaG1lbnQlM0IlMjBmaWxlbmFtZSUzRG1pbmlvLWJvc2hyZWxlYXNlLTIwMjAtMDctMzFUMDMtMzktMDVaLnRneiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTU5NjQ1OTQ5M319fV19&Signature=I~W1mdgH6U-nTTmziav1j5GY~1vR407AS8IaIHnTfXo~3~Ea1VmXS4ctmbLuNpjrALp8N6Ni02ADv8dqdNNFquqbp~XZjqJTgTQ1yZfbJr9UTMqnc92Z19ogSOlIAkF1se2sSr~yqDAkuEZZWiwtAF4TNrF-b3xT5g3CG~k8srIGJlK2NjBhLzHP9BDXpPLSwNATMYcrZ3CESVLygPsXx3Cqx5ptCsC~7UNWnJdqiSFGwApG~lHkiP~cKKonkMUyRK8DdOcVjHWgvgNRk7WfaPbyFmw~rwdVcTwW2XWRKbsylA9nAY1Zld4GmbaiwTgMrMyrfMkcxYHhcVHQdqFaTQ__&Key-Pair-Id=APKAICIFS42OUZR7AVEQ\r"
    version: 2020-07-31T03-39-05Z

not-bosh-io-merge-result.yml

fakeyamlrootforsprucetowork:
- path: /releases/name=bosh-vsphere-cpi
  type: replace
  value:
    name: bosh-vsphere-cpi
    sha1: f1e9a08341229ff23ee08e85d7b6f559b9b8cdc2
    url: "https://dfdtmjpfvu0dy.cloudfront.net/0c603242-b933-40bb-547e-5d757170ec94?response-content-disposition=attachment%3B%20filename%3Dbosh-vsphere-cpi-release-53.0.9.tgz&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9kZmR0bWpwZnZ1MGR5LmNsb3VkZnJvbnQubmV0LzBjNjAzMjQyLWI5MzMtNDBiYi01NDdlLTVkNzU3MTcwZWM5ND9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPWF0dGFjaG1lbnQlM0IlMjBmaWxlbmFtZSUzRGJvc2gtdnNwaGVyZS1jcGktcmVsZWFzZS01My4wLjkudGd6IiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNTkzNTU0NjAzfX19XX0_&Signature=nx89mN6KNmXsDvQuyxTugVuzkwY2x8ImaB8UTFDXVidwIGSY3r~ZwUqBE~8vgMOoIeCJMUfFMjcyp8ruKu4FQZyLvhMzvTy28jt6b2q78N2CDgZ3OhnsEs8ZNCXxaOyFBrAX4K3g20ZQS7t2vsYah74gt7KYj7K58RTUBWImHC28VMboMICdRncv-wHczN6YRqtIwYVrO3OIFAKlBzAMxxl7uQ6Yq-Qs8SG2RfWtH9D5x9mYbzJKy0RyvBARgEhZTwt~n5o0CPXLpPGZwfAQQYENZHBoUKoiya6xOFhrDXhHh0jSEFXvaei0-LTMs6k7gXUU19tQ6Hm24fEn1cgglA__&Key-Pair-Id=APKAICIFS42OUZR7AVEQ\r"
    version: 53.0.9

not-bosh-io-result-minio.yml

- path: /releases/name=minio
  type: replace
  value:
    name: minio
    sha1: f0f352f04b8d8bef3c390a720c3d1e733ed36668
    url: "https://dfdtmjpfvu0dy.cloudfront.net/ab2119b3-46ff-44fe-5630-dcc6690c21c7?response-content-disposition=attachment%3B%20filename%3Dminio-boshrelease-2020-07-31T03-39-05Z.tgz&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9kZmR0bWpwZnZ1MGR5LmNsb3VkZnJvbnQubmV0L2FiMjExOWIzLTQ2ZmYtNDRmZS01NjMwLWRjYzY2OTBjMjFjNz9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPWF0dGFjaG1lbnQlM0IlMjBmaWxlbmFtZSUzRG1pbmlvLWJvc2hyZWxlYXNlLTIwMjAtMDctMzFUMDMtMzktMDVaLnRneiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTU5NjQ3NTgyNH19fV19&Signature=eokAe45wccwQQ4kSPZtf7EdyVVCMIgtF0EuZro7aTExcDhpcDGRflbunVar7hD85Fsxsrcw-Le8Pcf8Dd0lvkMyXC66CK8KzuZnC7AMrl4bnLoKLw-TWkEfcCV7M-WQDNYBZHCCKr5ERjv9U2wOfzMufjUnYp8t4QqGza0mvDNQ9mJQLYcsQNE-1Wc97gvI5yfC5sY4dKW-YrMc7PALxO~0MgbpB69ONW0ein4z0RAwVNlUucDJs6CraICloAxXgBMGy~ErRvjbtCT~qwaaLbJkUN02UaetcINyOtiJn32aabZG0PBtsMKtGIXyUi5bfp6iyPFzYHNIKkU1FvPpm6w__&Key-Pair-Id=APKAICIFS42OUZR7AVEQ"
    version: 2020-07-31T03-39-05Z

not-bosh-io-result.yml

- path: /releases/name=bosh-vsphere-cpi
  type: replace
  value:
    name: bosh-vsphere-cpi
    sha1: f1e9a08341229ff23ee08e85d7b6f559b9b8cdc2
    url: "https://dfdtmjpfvu0dy.cloudfront.net/0c603242-b933-40bb-547e-5d757170ec94?response-content-disposition=attachment%3B%20filename%3Dbosh-vsphere-cpi-release-53.0.9.tgz&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9kZmR0bWpwZnZ1MGR5LmNsb3VkZnJvbnQubmV0LzBjNjAzMjQyLWI5MzMtNDBiYi01NDdlLTVkNzU3MTcwZWM5ND9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPWF0dGFjaG1lbnQlM0IlMjBmaWxlbmFtZSUzRGJvc2gtdnNwaGVyZS1jcGktcmVsZWFzZS01My4wLjkudGd6IiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNTkzNTU0NjAzfX19XX0_&Signature=nx89mN6KNmXsDvQuyxTugVuzkwY2x8ImaB8UTFDXVidwIGSY3r~ZwUqBE~8vgMOoIeCJMUfFMjcyp8ruKu4FQZyLvhMzvTy28jt6b2q78N2CDgZ3OhnsEs8ZNCXxaOyFBrAX4K3g20ZQS7t2vsYah74gt7KYj7K58RTUBWImHC28VMboMICdRncv-wHczN6YRqtIwYVrO3OIFAKlBzAMxxl7uQ6Yq-Qs8SG2RfWtH9D5x9mYbzJKy0RyvBARgEhZTwt~n5o0CPXLpPGZwfAQQYENZHBoUKoiya6xOFhrDXhHh0jSEFXvaei0-LTMs6k7gXUU19tQ6Hm24fEn1cgglA__&Key-Pair-Id=APKAICIFS42OUZR7AVEQ\r"
    version: 53.0.9

not-bosh-io-template-vsphere.yml

# BUCC provides compiled release via s3 for everything except 
fakeyamlrootforsprucetowork:
- type: replace
  path: /releases/name=bosh-vsphere-cpi
  value:
    name: bosh-vsphere-cpi
    version: "53.0.9"
    url: (( grab $CLOUDFRONT_URL ))
    sha1: f1e9a08341229ff23ee08e85d7b6f559b9b8cdc2

not-bosh-io-template.yml

# BUCC provides compiled release via s3 but if not we need..
fakeyamlrootforsprucetowork:
- type: replace
  path: (( grab $RELEASE_PATH ))
  value:
    name: (( grab $RELEASE_NAME ))
    version: (( grab $RELEASE_VERSION ))
    url: (( grab $CLOUDFRONT_URL ))
    sha1: (( grab $RELEASE_SHA1 ))

not-bosh-io.yml

# BUCC provides compiled release via s3 for everything except 
- type: replace
  path: /releases/name=bosh-vsphere-cpi
  value:
    name: bosh-vsphere-cpi
    version: "53.0.9"
    url: https://dfdtmjpfvu0dy.cloudfront.net/0c603242-b933-40bb-547e-5d757170ec94?response-content-disposition=attachment%3B%20filename%3Dbosh-vsphere-cpi-release-53.0.9.tgz&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9kZmR0bWpwZnZ1MGR5LmNsb3VkZnJvbnQubmV0LzBjNjAzMjQyLWI5MzMtNDBiYi01NDdlLTVkNzU3MTcwZWM5ND9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPWF0dGFjaG1lbnQlM0IlMjBmaWxlbmFtZSUzRGJvc2gtdnNwaGVyZS1jcGktcmVsZWFzZS01My4wLjkudGd6IiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNTkzNTU1NDQyfX19XX0_&Signature=emV4lkd7w89DIfDUNjEzyXQOqxiRSbNLXEqz2fFa7bnExUoCDQOCRvwBxi6kbQ2nEppQw-OGH1S-7A-chwyyBSQOh87LXbu7ZMfw0-4esMIzOn63ZQFfYrfrEv56D~R-Qjr~nyrrLZsxoHGkacU5GtDiuYW6jiLlu9BNrT799F9wML5IAL0xwcFOcZzkpySeFbve-Dv8Qy67ycDaJTL8ltayFezcNI4yQZIaV9ogCNl0uz-KnPb34oH~rxx8Oy06BodQu8Tm-ktdih68-iAH1cP2q2CgSpMrZbwXtiPoP~WWsgtbyWykUFsyPMQBlxuSWoe6UkDsLvMNLogRYddD3w__&Key-Pair-Id=APKAICIFS42OUZR7AVEQ
    sha1: f1e9a08341229ff23ee08e85d7b6f559b9b8cdc2

not_bosh_io_minio.sh

#!/usr/bin/env bash

set -eu

export RELEASE_SHA1="f0f352f04b8d8bef3c390a720c3d1e733ed36668"
export RELEASE_VERSION="2020-07-31T03-39-05Z"
cloudfront_url_location=$(curl -I -sSLk https://bosh.io/d/github.com/minio/minio-boshrelease?v="${RELEASE_VERSION}" | grep ocation: --no-messages)
export CLOUDFRONT_URL=${cloudfront_url_location#* }
export RELEASE_NAME=minio
export RELEASE_PATH="/releases/name=${RELEASE_NAME}"
spruce merge not-bosh-io-template.yml > not-bosh-io-merge-result-"${RELEASE_NAME}".yml

sed -e '/^fakeyamlrootforsprucetowork:/d' not-bosh-io-merge-result-"${RELEASE_NAME}".yml > not-bosh-io-result-"${RELEASE_NAME}".yml
cat not-bosh-io-result-"${RELEASE_NAME}".yml

not_bosh_io_vsphere_cpi.sh

#!/usr/bin/env bash

set -eu

version="53.0.9"
cloudfront_url_location=$(curl -I -sSLk https://bosh.io/d/github.com/cloudfoundry/bosh-vsphere-cpi-release?v="${version}" | grep ocation: --no-messages)
export CLOUDFRONT_URL=${cloudfront_url_location#* }
spruce merge not-bosh-io-template-vsphere.yml > not-bosh-io-merge-result.yml

sed -e '/^fakeyamlrootforsprucetowork:/d' not-bosh-io-merge-result.yml > not-bosh-io-result.yml
cat not-bosh-io-result.yml

url_checker.sh

#!/usr/bin/env bash

set -u

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

source $SCRIPT_DIR/urls.sh

response_codes=()
for i in ${urls[@]}
do
  echo "[TEST] Can curl $i"
  response_code=$(curl -I $i -w '%{http_code}' -sSLko /dev/null)
  response_codes+=("$response_code")
done
echo "Results with response codes.."
echo "NOTE: 403 is considered a failure as it is likely this came from the proxy policy denying the url"
for i in "${!urls[@]}"; do
  printf "%s\t%s\n" "${response_codes[$i]}" "${urls[$i]}"
done
bad_response_codes=()
for i in "${!urls[@]}"; do
  # Amazon expected to return '405 - Method Not Allowed'
  if [[ ${response_codes[$i]} -eq 403 ]]; then
    bad_response_codes+=("${urls[$i]}")
    echo "Bad response found for ${urls[$i]}"
    set +e
    curl -sSLk -D - ${urls[$i]} | grep policy --no-messages
    set -e
    echo "For more details run 'curl -sSLk -D - ${urls[$i]}' from the same machine this script was run"
  fi
done
if [ ${#bad_response_codes[@]} -eq 0 ]; then
    echo "Success!"
else
    echo "Test failed. See above for details"
    exit 1
fi

urls.sh

#!/usr/bin/env bash

export urls=( \
  https://app.terraform.io \
  https://bosh.io \
  https://dfdtmjpfvu0dy.cloudfront.net \
  https://network.pivotal.io \
  https://github.com \
  # Amazon CloudFront Load Balancer for S3
  https://d26ekeud912fhb.cloudfront.net \
  # Amazon S3 for PCF tiles and stemcells
  https://bosh-core-stemcells.s3-accelerate.amazonaws.com \
  https://s3.amazonaws.com \
  https://s3-eu-west-1.amazonaws.com \
  # ADDITIONAL ENTRIES FOR DOCKER HUB (Source: https://forums.docker.com/t/list-of-docker-hub-mirror-sites-to-configure-proxy-whitelist/20845)
  # https://dseasb33srnrn.cloudfront.net \ # Returns 403 even without proxy so ignoring for the moment. TODO: Check if really needed
  https://auth.docker.io \
  https://elb-registry.us-east-1.aws.dckr.io \
  https://us-east-1-elbregis-10fucsvj1tcgy-133821800.us-east-1.elb.amazonaws.com \
  https://registry-1.docker.io \
  https://registry-origin.docker.io \
  https://index.docker.io/ \
  https://elb-io.us-east-1.aws.dckr.io \
  https://us-east-1-elbio-rm5bon1qaeo4-623296237.us-east-1.elb.amazonaws.com \
  https://registry.hub.docker.com \
  https://hub.docker.com/ \
  # ADDITIONAL ENTRIES FOR ANSIBLE 
  # CF CLI download
  https://packages.cloudfoundry.org \
  #Ruby Gems Package Repository CDN 
  #https://rubygems.org \
  #Debian Package Repositories 
  http://deb.debian.org \
  http://security.debian.org \
  http://cdn-fastly.deb.debian.org \
  http://stedolan.github.io \
  http://starkandwayne.com \
  https://raw.githubusercontent.com
)

vsphere-ops-file.yml

- type: replace
  path: /releases/name=bosh-vsphere-cpi
  value:
    name: bosh-vsphere-cpi
    version: "53.0.1"
    url: https://bosh.io/d/github.com/cloudfoundry/bosh-vsphere-cpi-release?v=53.0.1
    sha1: dc04b664641e95c045e93d84a0444ea8f7d0cae0