bd.sh

#! /bin/bash
function usage() {
    echo "Usage: $0 [-r URL] [-k] (--projects|--scans|--purge-old-overlapping-maven-scans) [TOKEN|USER:PASS]

For example,

    \$ export BDSERVER="https://COMPANY.blackducksoftware.com"
    \$ read -rsp \"Password: \" BDPASS
    \$ export BDCRED="USER:\${BDPASS}"
    \$ $0 --projects
" >&2
    if (( $# )) ; then
        echo >&2
        echo "$*" >&2
    fi
    exit 1
}


if ! "${SHELL}" -c 'mapfile' < /dev/null > /dev/null 2>&1 ; then
    function mapfile() {
        local trim=0
        if [[ "$1" == "-t" ]] ; then
            trim=1
            shift
        fi
        local varname="$1"; shift
        source /dev/stdin <<<"${varname}=()"
        local line
        while read -r line ; do
            if (( trim )) ; then
                while [[ "${line}" =~ ^(.*)$'\n'$ ]] ; do
                    line="${BASH_REMATCH[1]}"
                done
            fi
            source /dev/stdin <<<"${varname}+=(\"\${line}\")"
        done
    }
    export -f mapfile
fi


set -e
bdserver="${BDSERVER}"

tmproot="${TMPDIR:-/tmp}"
tmproot="${tmproot%/}"
tmp="${tmproot}/bd"
rm -rf "${tmp}"
mkdir -p "${tmp}"
mitmcert="${tmp}/zscaler-ca.pem"
cat > "${mitmcert}" <<"EOT"
##
## The Zscaler's Certification Authority (self-signed) certificate.
##
## The following command from a workstation's registry generated the
## Zscaler part of the bundle.  The workstation appears to have received
## the CA via a group policy update.
##
## powershell "Get-ChildItem -Path Cert:\LocalMachine\Root | where {$_.Issuer -like \"*cn=zscaler*\" } | foreach {$content = \"-----BEGIN CERTIFICATE-----`r`n$([System.Convert]::ToBase64String($_.RawData, 'InsertLineBreaks'))`r`n-----END CERTIFICATE-----\"; $content | Out-File -FilePath \"zscaler-ca.pem\" -Encoding ascii}"
##
##
[email protected], CN=Zscaler Root CA, OU=Zscaler Inc., O=Zscaler Inc., L=San Jose, S=California, C=US
==========================================================================================================
-----BEGIN CERTIFICATE-----
MIIE0zCCA7ugAwIBAgIJANu+mC2Jt3uTMA0GCSqGSIb3DQEBCwUAMIGhMQswCQYDVQQGEwJVUzET
MBEGA1UECBMKQ2FsaWZvcm5pYTERMA8GA1UEBxMIU2FuIEpvc2UxFTATBgNVBAoTDFpzY2FsZXIg
SW5jLjEVMBMGA1UECxMMWnNjYWxlciBJbmMuMRgwFgYDVQQDEw9ac2NhbGVyIFJvb3QgQ0ExIjAg
BgkqhkiG9w0BCQEWE3N1cHBvcnRAenNjYWxlci5jb20wHhcNMTQxMjE5MDAyNzU1WhcNNDIwNTA2
MDAyNzU1WjCBoTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExETAPBgNVBAcTCFNh
biBKb3NlMRUwEwYDVQQKEwxac2NhbGVyIEluYy4xFTATBgNVBAsTDFpzY2FsZXIgSW5jLjEYMBYG
A1UEAxMPWnNjYWxlciBSb290IENBMSIwIAYJKoZIhvcNAQkBFhNzdXBwb3J0QHpzY2FsZXIuY29t
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqT7STSxZRTgEFFf6doHajSc1vk5jmzmM
6BWuOo044EsaTc9eVEV/HjH/1DWzZtcrfTj+ni205apMTlKBW3UYR+lyLHQ9FoZiDXYXK8poKSV5
+Tm0Vls/5Kb8mkhVVqv7LgYEmvEY7HPY+i1nEGZCa46ZXCOohJ0mBEtB9JVlpDIO+nN0hUMAYYdZ
1KZWCMNf5J/aTZiShsorN2A38iSOhdd+mcRM4iNL3gsLu99XhKnRqKoHeH83lVdfu1XBeoQzz5V6
gA3kbRvhDwoIlTBeMa5l4yRdJAfdpkbFzqiwSgNdhbxTHnYYorDzKfr2rEFMdsMU0DHdeAZf711+
1CunuQIDAQABo4IBCjCCAQYwHQYDVR0OBBYEFLm33UrNww4Mhp1d3+wcBGnFTpjfMIHWBgNVHSME
gc4wgcuAFLm33UrNww4Mhp1d3+wcBGnFTpjfoYGnpIGkMIGhMQswCQYDVQQGEwJVUzETMBEGA1UE
CBMKQ2FsaWZvcm5pYTERMA8GA1UEBxMIU2FuIEpvc2UxFTATBgNVBAoTDFpzY2FsZXIgSW5jLjEV
MBMGA1UECxMMWnNjYWxlciBJbmMuMRgwFgYDVQQDEw9ac2NhbGVyIFJvb3QgQ0ExIjAgBgkqhkiG
9w0BCQEWE3N1cHBvcnRAenNjYWxlci5jb22CCQDbvpgtibd7kzAMBgNVHRMEBTADAQH/MA0GCSqG
SIb3DQEBCwUAA4IBAQAw0NdJh8w3NsJu4KHuVZUrmZgIohnTm0j+RTmYQ9IKA/pvxAcA6K1i/LO+
Bt+tCX+C0yxqB8qzuo+4vAzoY5JEBhyhBhf1uK+P/WVWFZN/+hTgpSbZgzUEnWQG2gOVd24msex+
0Sr7hyr9vn6OueH+jj+vCMiAm5+ukd7lLvJsBu3AO3jGWVLyPkS3i6Gf+rwAp1OsRrv3WnbkYcFf
9xjuaf4z0hRCrLN2xFNjavxrHmsH8jPHVvgc1VD0Opja0l/BRVauTrUaoW6tE+wFG5rEcPGS80jj
HK4SpB5iDj2mUZH1T8lzYtuZy0ZPirxmtsk3135+CKNa2OCAhhFjE0xd
-----END CERTIFICATE-----
EOT
rm -rf "${tmp}/certs"
mkdir -p "${tmp}/certs"
certname=$(openssl x509 -hash -noout -in "${mitmcert}")
ln -s "${mitmcert}" "${tmp}/certs/${certname}.0"
copts=(--capath "/etc/ssl/certs:${tmp}/certs")
while (( $# )) ; do
    case "$1" in
        -r)
            shift
            bdserver="$1"; shift || usage
            ;;
        -k)
            copts+=($1)
            shift
            ;;
        *)
            break
            ;;
    esac
done
(( ${#bdserver} )) || usage "Missing the server URL"
command="$1" ; shift || usage
cred="$1"; shift || cred="${BDCRED}"
(( ! $# )) || usage
if [[ "${cred}" =~ : ]] ; then
    user="${cred%%:*}"
    pass="${cred#*:}"
    if (( ! ${#pass} )) ; then
        if (( ${#BDPASS} )) ; then
            pass="${BDPASS}"
        else
            read -rsp "Password: " pass
            echo
        fi
    fi
    echo "Logging in as ${user}..."
    http_code=$(curl -isS "${copts[@]}" --write-out "%{http_code}" -X POST -o "${tmp}/login-pass.txt" \
        --data-urlencode "j_username=${user}" \
        --data-urlencode "j_password=${pass}" \
        "${bdserver}/j_spring_security_check")
    if [[ "${http_code}" != "200" && "${http_code}" != "204" ]] ; then
        echo "HTTP response code ${http_code}"
        cat "${tmp}/login-pass.txt"
        echo
        exit 1
    fi
    bearertoken=$(sed -ne 's/^Set-Cookie: AUTHORIZATION_BEARER=\([^;[:space:]]*\).*/\1/p' "${tmp}/login-pass.txt")
    csrftoken=$(sed -ne 's/^X-CSRF-TOKEN: \([^;[:space:]]*\).*/\1/p' "${tmp}/login-pass.txt")
else
    token="${cred}"
    echo "Logging in with the token..."
    # https://github.com/blackducksoftware/blackduck-common/blob/master/src/main/java/com/synopsys/integration/blackduck/rest/ApiTokenBlackDuckHttpClient.java
    http_code=$(curl -isS "${copts[@]}" --write-out "%{http_code}" -X POST \
        -H "Authorization: token ${token}" -o "${tmp}/login-token.txt" \
        "${bdserver}/api/tokens/authenticate")
    if [[ "${http_code}" != "200" && "${http_code}" != "204" ]] ; then
        echo "HTTP response code ${http_code}"
        cat "${tmp}/login-token.txt"
        echo
        exit 1
    fi
    body=$(sed -e '1,/^[[:space:]]*$/d' "${tmp}/login-token.txt")
    bearertoken=$(jq -r ".bearerToken" <<< "${body}")
    csrftoken=$(sed -ne 's/^X-CSRF-TOKEN: \([^;[:space:]]*\).*/\1/p' "${tmp}/login-token.txt")
fi

copts+=(\
        -H "Cookie: AUTHORIZATION_BEARER=${bearertoken}" \
        # -H "Authorization: Bearer ${bearertoken}" \
        -H "X-CSRF-TOKEN: ${csrftoken}" \
    )

case "${command}" in
    --projects)
        projstart=0
        projlimit=100
        while : ; do
            echo "Getting a list of ${projlimit} projects from ${projstart}..."
            http_code=$(curl -sS "${copts[@]}" --write-out "%{http_code}" -o "${tmp}/projects-${projstart}.json" \
                    "${bdserver}/api/projects" \
                    -G --data-urlencode "offset=${projstart}" --data-urlencode "limit=${projlimit}")
            if [[ "${http_code}" != "200" ]] ; then
                echo "HTTP response code ${http_code}"
                echo
                jq . "${tmp}/projects-${projstart}.json" || cat "${tmp}/projects-${projstart}.json"
                echo
                exit 1
            fi
            # jq . "${tmp}/projects-${projstart}.json"
            numitems=$(jq ".items | length" "${tmp}/projects-${projstart}.json")
            echo "Got ${numitems} projects starting from ${projstart}."
            (( numitems )) || break
            mapfile -t projs < <(jq -r '.items[] | ._meta.href + " " + .name' "${tmp}/projects-${projstart}.json")
            for projrefname in "${projs[@]}" ; do
                projref="${projrefname%% *}"
                projname="${projrefname#* }"
                echo "  Project <<${projname}>>"
                mapfile -t versions < <(curl -sS "${copts[@]}" "${projref}/versions" | jq -r '.items[] | ._meta.href + " " + .versionName')
                for versionrefname in "${versions[@]}" ; do
                    verref="${versionrefname%% *}"
                    vername="${versionrefname#* }"
                    echo "  Version <<${vername}>>"
                    mapfile -t trees < <(curl -sS "${copts[@]}" "${verref}/source-trees" | jq -r '.items[] | .name')
                    for tree in "${trees[@]}" ; do
                        echo "    Source tree <<${tree}>>"
                    done
                done
            done
            ((projstart+=projlimit))
        done
        ;;
    --scans|--purge-old-overlapping-maven-scans)
        scanstart=0
        scanlimit=100
        while : ; do
            echo "Getting a list of ${scanlimit} scans from ${scanstart}..."
            http_code=$(curl -sS "${copts[@]}" --write-out "%{http_code}" -o "${tmp}/scans-${scanstart}.json" \
                    "${bdserver}/api/codelocations" \
                    -G --data-urlencode "offset=${scanstart}" --data-urlencode "limit=${scanlimit}" \
                    --data-urlencode "sort=updatedAt" --data-urlencode "order=asc" --data-urlencode "includeErrors=true")
            if [[ "${http_code}" != "200" ]] ; then
                echo "HTTP response code ${http_code}"
                echo
                jq . "${tmp}/scans-${scanstart}.json" || cat "${tmp}/scans-${scanstart}.json"
                echo
                exit 1
            fi
            # jq . "${tmp}/scans-${scanstart}.json"
            numitems=$(jq ".items | length" "${tmp}/scans-${scanstart}.json")
            echo "Got ${numitems} scans starting from ${scanstart}."
            (( numitems )) || break
            mapfile -t scans < <(jq -r \
                '.items[] | ._meta.href + " " + (.scanSize|tostring) + " " + .name + ": " + .updatedAt + " " + ([.status[].status] | join(" "))' \
                "${tmp}/scans-${scanstart}.json")
            for scanrefname in "${scans[@]}" ; do
                scanref="${scanrefname%% *}"
                scansizename="${scanrefname#* }"
                scansize="${scansizename%% *}"
                scannamest="${scansizename#* }"
                scanname="${scannamest%%: *}"
                scanst="${scannamest#*: }"
                laststatus="${scanst##* }"
                updatets="${scanst%% *}"
                scanst="${scanst#* }"
                case "${command}" in
                    --scans)
                        echo "  Scan <<${scanname}>>, size ${scansize}, date ${updatets}, statuses ${scanst}"
                        ;;
                    --purge-old-overlapping-maven-scans)
                        # https://github.com/blackducksoftware/synopsys-detect/issues/66
                        if [[ "${scanname}" =~ [[:space:]]BOM$ ]] && (( !scansize )) ; then
                            echo "  Suspicious scan ${scanname}, size ${scansize}, date ${updatets}, statuses ${scanst}"
                            case "${laststatus}" in
                                "COMPLETED"|"UNSTARTED")
                                    delref="${scanref//\/api\///api/v1/composite/}"
                                    echo "    DELETE ${delref}"
                                    http_code=$(curl -isS "${copts[@]}" --write-out "%{http_code}" -o "${tmp}/delete-scan-${updatets}-${scanref##*/}.txt" \
                                            "${delref}" \
                                            -X DELETE)
                                    if [[ "${http_code}" != "204" ]] ; then
                                        echo "HTTP response code ${http_code}"
                                        echo
                                        cat "${tmp}/delete-scan-${updatets}-${scanref##*/}.txt"
                                        echo
                                        # exit 1
                                    fi
                                    ;;
                            esac
                        fi
                        ;;
                esac
            done
            ((scanstart+=scanlimit))
        done
        ;;
    --notifications)
        # This returns 403
        notifstart=0
        notiflimit=100
        echo "Getting ${notiflimit} notifications starting from ${notifstart}..."
        echo "Bearer token ${bearertoken}"
        echo "CSRF token ${csrftoken}"
        http_code=$(curl -sS "${copts[@]}" --write-out "%{http_code}" -o "${tmp}/notifications-${notifstart}.json" \
                "${bdserver}/api/notifications" \
                -G --data-urlencode "offset=${notifstart}" --data-urlencode "limit=${notiflimit}")
        if [[ "${http_code}" != "200" ]] ; then
            echo "HTTP response code ${http_code}"
            echo
            jq . "${tmp}/notifications-${notifstart}.json" || cat "${tmp}/notifications-${notifstart}.json"
            echo
            exit 1
        fi
        jq . "${tmp}/notifications-${notifstart}.json"
        ;;
    *)
        usage
        ;;
esac