NVD download script - Linux bash

NVD-download_files

#!/usr/bin/env /bin/bash

#
# Available from:
# https://gist.github.com/dougluce/e59109adaf027d1b622039a79921e96f
#
# required: curl and jq tools before executing this script. Also an API key must be added, see API_KEY value in curl command
# Copyright 2024 Douglas Allen Luce
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License.  You
# may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.  See the License for the specific language governing
# permissions and limitations under the License.

#
# Download NVD CVE data in JSON format to the current directory. If
# there are a lot, multiple files will be created.
#
# Optional keyword search
#keywordSearch=IBM

baseUrl=https://services.nvd.nist.gov/rest/json/cves/2.0/


pubStartDate=1999-01-01T00:00:00.000
month_to_add=3
file_date_stamp="./start_date"
current_script_date=$(date -d "$pubStartDate" "+%Y%m%d")
current_date=$(date +%Y%m%d)


while (( $current_script_date < $current_date )); do

N=1
# Date range to retrieve, note: cannot exceed 120 days.
if [ -f $file_date_stamp ]; then
        pubStartDate=$(cat $file_date_stamp)
        current_script_date=$(date -d "$pubStartDate" "+%Y%m%d")
fi
pubEndDate=$(date -d "$pubStartDate" "+%Y-%m-%d")
pubEndDate=$(date -d "$pubEndDate + $month_to_add month" "+%Y-%m-%dT%H:%M:%S.%3N")

extra=
append () {
    if [[ -z $extra ]]; then
        extra="?$1"
    else
        extra="${extra}&$1"
    fi
}

if [[ ! -z $pubStartDate ]]; then
    append pubStartDate=$pubStartDate
fi
if [[ ! -z $pubEndDate ]]; then
    append pubEndDate=$pubEndDate
fi
if [[ ! -z $keywordSearch ]]; then
    append keywordSearch=$keywordSearch
fi

url=$baseUrl$extra
while true; do
    TEMPFILE=$(mktemp -u curl.output.XXXXXX)
    TEMPHEADERS=$(mktemp -u curl.headers.XXXXXX)
    OUTPUT=nvd-output-$current_script_date-${N}.json
    echo Downloading $url to $OUTPUT
    http_status=$(curl -s -H "Accept: application/json" -H "apiKey: <API_KEY>" $url -D $TEMPHEADERS -w '%{http_code}' -o $TEMPFILE)
    read resultsPerPage startIndex totalResults < <(cat $TEMPFILE |  jq -r '"\(.resultsPerPage) \(.startIndex) \(.totalResults)"')
    if [[ $http_status -ne 200 ]]; then
        perl -lne 'print if s/^message: //' $TEMPHEADERS
        rm -f $TEMPFILE $TEMPHEADERS
        exit 1
    fi
    rm -f $TEMPHEADERS
    # Success, move into sequence.
    mv $TEMPFILE $OUTPUT
    # Got all the results?
    if [[ $(($startIndex + $resultsPerPage)) -ge $totalResults ]]; then
        break
    fi
    if [[ -z $extra ]]; then
        paging="?"
    else
        paging="&"
    fi
    url=$baseUrl$extra$paging"startIndex=$(($startIndex + $resultsPerPage))"
    (( N++ ))
    sleep 10 # Cheesy simple rate limiting (https://nvd.nist.gov/developers/start-here)
done
echo $pubEndDate > $file_date_stamp
done