tt_upload.sh

#!/bin/bash

# curl -L https://gist.github.com/grenade/109bfd61a663902236e1d3f6530dec55/raw/tt_upload.sh | sh

tooltool_token_path=${HOME}/.tooltool.token
tooltool_url=https://tooltool.mozilla-releng.net
tooltool_client_url=https://raw.githubusercontent.com/mozilla-releng/tooltool/master/client/tooltool.py
wrk_dir=$(pwd)
tmp_dir=$(mktemp -d)

fg_black=`tput setaf 0`
fg_red=`tput setaf 1`
fg_green=`tput setaf 2`
fg_yellow=`tput setaf 3`
fg_blue=`tput setaf 4`
fg_magenta=`tput setaf 5`
fg_cyan=`tput setaf 6`
fg_white=`tput setaf 7`
reset=`tput sgr0`

cd ${tmp_dir}
if curl -s -o ${tmp_dir}/tooltool.py ${tooltool_client_url}; then
  echo "$(tput dim)[${current_script_name} $(date --utc +"%F %T.%3NZ")]${reset} tooltool client downloaded from ${tooltool_client_url} to ${tmp_dir}/tooltool.py"
else
  echo "$(tput dim)[${current_script_name} $(date --utc +"%F %T.%3NZ")]${reset} failed to download tooltool client from ${tooltool_client_url} to ${tmp_dir}/tooltool.py"
  exit 1
fi

curl -sL https://gist.githubusercontent.com/grenade/109bfd61a663902236e1d3f6530dec55/raw/tt_upload_manifest.json?$(uuidgen) | jq -r '.[] | @base64' | while read item; do
  _jq_decode() {
    echo ${item} | base64 --decode | jq -r ${1}
  }
  url=$(_jq_decode '.url')
  filename=$(_jq_decode '.filename')
  if curl -sL -o ${tmp_dir}/${filename} ${url} && [ -f "${tmp_dir}/${filename}" ]; then
    echo "$(tput dim)[${current_script_name} $(date --utc +"%F %T.%3NZ")]${reset} downloaded ${tmp_dir}/${filename} from ${url}"
    computed_sha512=$(sha512sum "${tmp_dir}/${filename}" | { read sha512 _; echo ${sha512}; })
    echo "$(tput dim)[${current_script_name} $(date --utc +"%F %T.%3NZ")]${reset} computed sha512 for ${tmp_dir}/${filename} as ${computed_sha512}"
    if curl --header "Authorization: Bearer $(cat ${tooltool_token_path})" --output /dev/null --silent --head --fail ${tooltool_url}/sha512/${computed_sha512} 2> /dev/null; then
      echo "$(tput dim)[${current_script_name} $(date --utc +"%F %T.%3NZ")]${reset} found ${filename} in tooltool with sha ${computed_sha512}"
    else
      echo "$(tput dim)[${current_script_name} $(date --utc +"%F %T.%3NZ")]${reset} failed to find ${filename} in tooltool with sha ${computed_sha512}"
      if python ${tmp_dir}/tooltool.py add --visibility internal "${tmp_dir}/${filename}" -m ${tmp_dir}/manifest.tt; then
        echo "$(tput dim)[${current_script_name} $(date --utc +"%F %T.%3NZ")]${reset} added ${tmp_dir}/${filename} to ${tmp_dir}/manifest.tt"
      else
        echo "$(tput dim)[${current_script_name} $(date --utc +"%F %T.%3NZ")]${reset} failed to add ${tmp_dir}/${filename} to ${tmp_dir}/manifest.tt"
      fi
    fi
  else
    echo "$(tput dim)[${current_script_name} $(date --utc +"%F %T.%3NZ")]${reset} failed to download ${tmp_dir}/${filename} from ${url}"
  fi
done

if [ -f ${tmp_dir}/manifest.tt ] && [ -s ${tmp_dir}/manifest.tt ] && python ${tmp_dir}/tooltool.py validate -m ${tmp_dir}/manifest.tt; then
  python ${tmp_dir}/tooltool.py upload --url ${tooltool_url} --authentication-file=${tooltool_token_path} --message "Bug 1342892 - OCC component: ${filename}" -m ${tmp_dir}/manifest.tt
  rm -f ${tmp_dir}/manifest.tt
else
  echo "$(tput dim)[${current_script_name} $(date --utc +"%F %T.%3NZ")]${reset} tooltool upload skipped due to manifest absence or validation failure for ${tmp_dir}/manifest.tt"
fi
cd ${wrk_dir}
#rm -rf ${tmp_dir}

tt_upload_manifest.json

[
  {
    "url": "https://windows-ami-builder.s3-us-west-2.amazonaws.com/package/generic-worker-windows-amd64.exe",
    "filename": "generic-worker-windows-amd64.exe"
  },
  {
    "url": "https://github.com/taskcluster/generic-worker/releases/download/v16.0.0/generic-worker-multiuser-windows-386.exe",
    "filename": "generic-worker-multiuser-windows-386.exe"
  },
  {
    "url": "https://github.com/taskcluster/generic-worker/releases/download/v16.0.0/generic-worker-multiuser-windows-amd64.exe",
    "filename": "generic-worker-multiuser-windows-amd64.exe"
  },
  {
    "url": "https://download.visualstudio.microsoft.com/download/pr/c4fef23e-cc45-4836-9544-70e213134bc8/ca949157f8a5d9dcd4be11fb32cebc04c03fdf401a48bf36ae31c289d8335ffb/vs_BuildTools.exe",
    "filename": "vs2019_BuildTools.exe"
  },
  {
    "url": "https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Configuration/GenericWorker/run-hw-generic-worker-10-and-reboot.bat",
    "filename": "run-hw-generic-worker-10-and-reboot.bat"
  },
  {
    "url": "https://dl.google.com/tag/s/dl/chrome/install/googlechromestandaloneenterprise64.msi",
    "filename": "googlechromestandaloneenterprise64.msi"
  },
  {
    "url": "https://dl.google.com/tag/s/dl/chrome/install/googlechromestandaloneenterprise.msi",
    "filename": "googlechromestandaloneenterprise.msi"
  },
  {
    "url": "https://packagecontrol.io/Package%20Control.sublime-package",
    "filename": "Package%20Control.sublime-package"
  },
  {
    "url": "https://github.com/stedolan/jq/releases/download/jq-1.5/jq-win32.exe",
    "filename": "jq-1-5-win32.exe"
  },
  {
    "url": "https://github.com/stedolan/jq/releases/download/jq-1.5/jq-win64.exe",
    "filename": "jq-1-5-win64.exe"
  },
  {
    "url": "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-win32.exe",
    "filename": "jq-1-6-win32.exe"
  },
  {
    "url": "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-win64.exe",
    "filename": "jq-1-6-win64.exe"
  },
  {
    "url": "https://raw.githubusercontent.com/mozilla-releng/OpenCloudConfig/master/userdata/Configuration/nxlog/hw-win10.conf",
    "filename": "hw-win10.conf"
  },
  {
    "url": "https://www.mercurial-scm.org/release/windows/mercurial-4.9.1-x64.msi",
    "filename": "mercurial-4.9.1-x64.msi"
  },
  {
    "url": "https://www.mercurial-scm.org/release/windows/mercurial-4.9.1-x86.msi",
    "filename": "mercurial-4.9.1-x86.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.6.3/python-3.6.3.exe",
    "filename": "python-3.6.3.exe"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/python-3.7.3.exe",
    "filename": "python-3.7.3.exe"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/core.msi",
    "filename": "python-3-7-3-win32-core.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/core_d.msi",
    "filename": "python-3-7-3-win32-core_d.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/core_pdb.msi",
    "filename": "python-3-7-3-win32-core_pdb.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/dev.msi",
    "filename": "python-3-7-3-win32-dev.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/dev_d.msi",
    "filename": "python-3-7-3-win32-dev_d.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/doc.msi",
    "filename": "python-3-7-3-win32-doc.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/exe.msi",
    "filename": "python-3-7-3-win32-exe.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/exe_d.msi",
    "filename": "python-3-7-3-win32-exe_d.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/exe_pdb.msi",
    "filename": "python-3-7-3-win32-exe_pdb.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/launcher.msi",
    "filename": "python-3-7-3-win32-launcher.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/lib.msi",
    "filename": "python-3-7-3-win32-lib.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/lib_d.msi",
    "filename": "python-3-7-3-win32-lib_d.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/lib_pdb.msi",
    "filename": "python-3-7-3-win32-lib_pdb.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/path.msi",
    "filename": "python-3-7-3-win32-path.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/pip.msi",
    "filename": "python-3-7-3-win32-pip.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/tcltk.msi",
    "filename": "python-3-7-3-win32-tcltk.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/tcltk_d.msi",
    "filename": "python-3-7-3-win32-tcltk_d.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/tcltk_pdb.msi",
    "filename": "python-3-7-3-win32-tcltk_pdb.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/test.msi",
    "filename": "python-3-7-3-win32-test.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/test_d.msi",
    "filename": "python-3-7-3-win32-test_d.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/test_pdb.msi",
    "filename": "python-3-7-3-win32-test_pdb.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/tools.msi",
    "filename": "python-3-7-3-win32-tools.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/win32/ucrt.msi",
    "filename": "python-3-7-3-win32-ucrt.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/python-3.7.3.exe",
    "filename": "python-3.7.3.exe"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/core.msi",
    "filename": "python-3-7-3-amd64-core.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/core_d.msi",
    "filename": "python-3-7-3-amd64-core_d.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/core_pdb.msi",
    "filename": "python-3-7-3-amd64-core_pdb.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/dev.msi",
    "filename": "python-3-7-3-amd64-dev.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/dev_d.msi",
    "filename": "python-3-7-3-amd64-dev_d.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/doc.msi",
    "filename": "python-3-7-3-amd64-doc.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/exe.msi",
    "filename": "python-3-7-3-amd64-exe.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/exe_d.msi",
    "filename": "python-3-7-3-amd64-exe_d.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/exe_pdb.msi",
    "filename": "python-3-7-3-amd64-exe_pdb.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/launcher.msi",
    "filename": "python-3-7-3-amd64-launcher.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/lib.msi",
    "filename": "python-3-7-3-amd64-lib.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/lib_d.msi",
    "filename": "python-3-7-3-amd64-lib_d.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/lib_pdb.msi",
    "filename": "python-3-7-3-amd64-lib_pdb.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/path.msi",
    "filename": "python-3-7-3-amd64-path.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/pip.msi",
    "filename": "python-3-7-3-amd64-pip.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/tcltk.msi",
    "filename": "python-3-7-3-amd64-tcltk.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/tcltk_d.msi",
    "filename": "python-3-7-3-amd64-tcltk_d.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/tcltk_pdb.msi",
    "filename": "python-3-7-3-amd64-tcltk_pdb.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/test.msi",
    "filename": "python-3-7-3-amd64-test.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/test_d.msi",
    "filename": "python-3-7-3-amd64-test_d.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/test_pdb.msi",
    "filename": "python-3-7-3-amd64-test_pdb.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/tools.msi",
    "filename": "python-3-7-3-amd64-tools.msi"
  },
  {
    "url": "https://www.python.org/ftp/python/3.7.3/amd64/ucrt.msi",
    "filename": "python-3-7-3-amd64-ucrt.msi"
  },
  {
    "url": "http://download.windowsupdate.com/d/msdownload/update/software/secu/2019/05/windows6.1-kb4499175-x86_6f1319c32d5bc4caf2058ae8ff40789ab10bf41b.msu",
    "filename": "windows6.1-kb4499175-x86_6f1319c32d5bc4caf2058ae8ff40789ab10bf41b.msu"
  }
]