Meraki - [parse.sh] download/parse client list ; [compare.sh] compare 2 files (downloaded and parsed with parse.sh) and ask if you want to copy new data from the 2nd file to the 1st file (+create a json file).

compare.sh

#!/usr/bin/env bash

# This script compare 2 files previously parsed with the "parse.sh" script
# First file is the "base-file" the second one is the newer version of the first one
# It shows the differences between the 2 files,
# and you can choose what you want to copy from the newer file to the base file
# Finally, it format the (final) base-file in a new real JSON readable file.

# It overwirte the first base-file (for further comparison) with the new data, and create a new json file.

#TODO
# - [ ] check if user's numbers input are in the range of accepted numbers (see code at the end)
# - [ ] improve DIF command
# - [ ] improve formate() function

# Temp files
TMPDIR=$(mktemp -d /tmp/compare-XXX)
TMP5=$(mktemp -p ${TMPDIR})
TMP6=$(mktemp -p ${TMPDIR})
TMP7=$(mktemp -p ${TMPDIR})
TMP8=$(mktemp -p ${TMPDIR})
TMP9=$(mktemp -p ${TMPDIR})

usage () {
    echo
    echo "Usage:"
    echo "./compare.sh base-file.txt newer-file.txt"
    echo
    exit 1
}

compare_mac () {
    # function the check if there is new mac address is the new file
    # get only the mac address from the file (second last field)
    #AWK=$(awk -F'"' '/mac/ {print $(NF-1)}' ${NEWER})
    # check diff between base file and the newer one
    # and story only the new mac addresses, removing the + sign before
    #DIF=$(diff -c <(${AWK} ${BASEFILE}) <(${AWK} ${NEWER})|grep '^\+' | sed -E 's/^\+ //')
    DIF=$(diff -c <(awk '-F"' '/mac/ {print $(NF-1)}' ${BASEFILE}) <(awk '-F"' '/mac/ {print $(NF-1)}' ${NEWER})|grep '^\+'|sed -E 's/^\+ //')
    # iterate through each mac @ stored in DIF var
    while IFS= read -r MAC; do
        # store all the new mac @ in a tmp file
        COPY=$(awk 'NR==FNR{if (/'${MAC}'/) for (i=0;i<=6;i++) copy[NR+i]; next} (FNR in copy)' ${NEWER} ${NEWER} >> ${TMP5})
        # add a blank line after each group
        echo >> ${TMP5}
    done <<< ${DIF}

    # before each matching pattern line, we add a new line with ">)" and pipe the output to add the number before after each ">"
    # and finally pipe all of that with to display it and save it in a temp file
    awk -v patr="\"mac\"" '{$0 ~ patr gsub(patr, ">)\n "patr)}1' ${TMP5} | awk '/>/{$0 = ">" ++i substr($0, 2)}1' | tee ${TMP6}
    # get the last number in variable (to get the range and check the input from user)
    NUMC=$(grep -E -c "^>[0-9]+" ${TMP6})
    # next we ask for number(s)
    echo -e "Choose number between 1 and ${NUMC}.\nMultiple choice accepted (space separated) ex:\n  choice> 1 3 10\nOr for all numbers:\n  choice> @"
    # ask to add new group in terms of the number corresponding at the group
    # store result in a string
    read -p "choice> " MACNUM
    # check if array contains "*"
    # cannot check if contains ONLY "*"
    if [[ " ${MACNUM[*]} " == *"@"* ]]; then
        # copy all the mac @
        read -p "Are you sure, you want to copy all the mac @ to the ${BASEFILE} file? (y/n) " ANSWER
        [[ ! ${ANSWER} == "y" ]] && echo "Aborting..." && exit 1
        echo "Copying..."
        # check if last line of base-file is empty or not, if empty, add newline (to separate groups)
        [[ ! -z $(awk 'END{print}' ${BASEFILE}) ]] && echo >> ${BASEFILE}
        # read file content except the already existing mac @ and add it in the base file
        cat ${TMP5} >> ${BASEFILE}
        echo "Done."
    else
        # convert user string input in array
        read -a MACNUM2 <<< "${MACNUM}"
        # iterate on each element of the array
        for elt in ${MACNUM2[@]}; do
            # check if last line of basefile is empty or not, if empty, add newline (to separate groups)
            [[ ! -z $(awk 'END{print}' ${BASEFILE}) ]] && echo >> ${BASEFILE}
            # copy the group to the base file
            awk 'NR==FNR{if (/>'${elt}'\)/) for (i=1;i<=6;i++) copy[NR+i]; next} (FNR in copy)' ${TMP6} ${TMP6} >> ${BASEFILE}
            # add newline after copying each group
            echo >> ${BASEFILE}
        done
        echo "Done."
    fi
}

formate () {
    # function used to format in real JSON file the final newer file
    # suppress repeated empty output lines
    cat -s ${BASEFILE} > ${TMP7}
    # add "{" before each "mac" string
    sed -i 's#\"mac#{&#g' ${TMP7}
    # for each "recentDeviceMac" string found, add "}" before the last char (",")
    # first group \("..."\) match the first pattern, second group match the last char
    # then print the first group \1, print "}", and print the second group
    # and finally, reverse the lines of the file,
    # remove the only "," on the first occurence  and reverse again the lines
    sed 's/\("recentDeviceMac":.*\)\(.\)$/\1}\2/' ${TMP7} | tac | sed '0,/,/{s/,//}' | tac > ${TMP8}
    # merge all lines into one line, and add "[" and "]" in front of and at the end of the line
    paste -sd '' ${TMP8} | awk '{print "["$0"]"}' > ${TMP9}
    # paste -sd '\0' # could also have been a possiblity to merge all lines
    # moving final temp file to current folder
    mv ${TMP9} "${BF2}-$(date +%d-%m-%Y).json"
}

# add at the end of each line matching "recentDeviceMac" a "}" before the ","
#   ==> sed 's/\("recentDeviceMac":.*\)\(.\)$/\1}\2/'
#    (not optimized) ==> awk -F, '{ if(/recentDeviceMac/){print $(NF-1)"},"} else {print} }'
#    (if no empty lines at the EOF) ==> sed -e 's/\("recentDeviceMac":.*\)\(.\)$/\1}\2/' -e '$s/,$//'

# *** Main ***
# Store all arguments in a variable
NARGS=$#
# Check arguments
[[ ${NARGS} -ne 2 ]] && echo "Error, arguments must be 2 exactly." && usage
BASEFILE=$1
NEWER=$2
# removing suffix of base-file (if it exists)
BF2=${BASEFILE%.*}
compare_mac
echo "Formatting in real JSON readable file..."
formate
# removing all temp files
rm -rf ${TMPDIR}
echo "Done. Your final new file is ${BF2}."

# to check user's numbers input
### check if number is not out of range
##while ! ((${MACNUM}<=${NUMC} && ${MACNUM}>=1)); do
##    echo "number must be between 1 and ${NUMC}..."
##    read -p "choose again between 1 and ${NUMC}: " MACNUM
##done

parse.sh

#!/usr/bin/env bash

# This script either parse a JSON file with the fields we want
# or either get this file just with the Meraki API key, and parse it.
# the fields parsed are: mac, description, ip, user, lastSeen and recentDeviceMac

#TODO:
# - [ ] Change curl command to avoid output
#   * curl=$(curl --silent)
# - [ ] Remove all occurrence of cURL requests with a BASE variale (for headers and base URL)
# - [ ] Change default filename to add timespan something like "27-08-2022-[30d/2h/...]-request.json"
# The following are done in compare.sh
# - [x] change output file style, to either a real JSON file (so add [] and {}) or a txt file, yaml file, ...
# - [x] add option to choose field to keep)

# Temporary files used during the formatting of the JSON file.
TMPDIR=$(mktemp -d /tmp/parse-XXX)
TMP0=$(mktemp -p ${TMPDIR})
TMP1=$(mktemp -p ${TMPDIR})
TMP2=$(mktemp -p ${TMPDIR})
TMP3=$(mktemp -p ${TMPDIR})
TMP4=$(mktemp -p ${TMPDIR})
FINAL="./final-parsed.txt"

usage () {
    # Usage function, if something wrong with arguments
    echo
    echo "Usage:"
    echo "To just parse an existing JSON file (without making any Meraki API requests):"
    echo "./parse.sh input-file.json"
    echo
    echo "Or, to do the requests now, and optionally set a different output filename."
    echo "Default output filename: \"dd-mm-yyyy-request.json\""
    echo "./parse.sh [output-file.json] y0u#_mer@ki_ap1_k3y"
    echo
    # exit failure
    exit 1
}

my_requests () {
    # Execute a cURL requests to the Meraki API and save it in a JSON file
    # This can be done in the Meraki dashboard:
    #     1) Choose your Network
    #     2) Go to "Network-wide" > under "Monitor", click on "Clients"

    # get the organizationId
    ORG_ID=$(curl -L -H "X-Cisco-Meraki-API-Key:${MERAKI_API}" -H "Content-Type: application/json" -X GET "https://api.meraki.com/api/v1/organizations"|cut -d'"' -f4)

    # get the network(s)Id
    # the "awk" part, search for "?" and replace it by a number (start with 1) and increase it (depending the numbers of options)
    NETWORKS=$(curl -L -H "X-Cisco-Meraki-API-Key:${MERAKI_API}" -H "Content-Type: application/json" -X GET https://api.meraki.com/api/v1/organizations/${ORG_ID}/networks | jq '.[]|"?)"+.name+":"+.id' | awk 'sub(/?/,++c)')
    echo "Available networks:"
    echo ${NETWORKS}| tr ' ' "\n"

    # get the total number of available networks
    NUMNET=$(echo ${NETWORKS}|tee|tr ' ' "\n"|wc -l)
    read -p "Choose number between 1 and ${NUMNET}: " NETWORK_NUM
    # check if number is not out of range
    while ! ((${NETWORK_NUM}<=${NUMNET} && ${NETWORK_NUM}>=1)); do
        echo "number must be between 1 and ${NUMNET}..."
        read -p "choose again between 1 and ${NUMNET}: " NETWORK_NUM
    done
    # now we can grab the right network_id according to the number the user has given.
    # removing double quotes from output;separate each network one per line;get the network_id from the ${NETWORK_NUM} occurrence
    NETWORK_ID=$(echo ${NETWORKS}|tr -d '"'|tr ' ' "\n"|awk -F':' -v n=${NETWORK_NUM} '{l++} (l==n){print $2}')

    # timespan
    echo "Available timespan:"
    echo "  1) 2 hours"   # 7200s
    echo "  2) 24 hours"  # 86400s
    echo "  3) last week" # 604800s
    echo "  4) 30 days"   # 2592000s
    # if number is out of range
    ANSW="yes"
    while [ "${ANSW}" = "yes" ];do
        ANSW="no"
        read -p "Choose the timespan: " TIMENUM
        case ${TIMENUM} in
            1) TIMESPAN=7200; break;;
            2) TIMESPAN=86400; break;;
            3) TIMESPAN=604800; break;;
            4) TIMESPAN=2592000; break;;
            *) echo "Error, choose between 1 and 4."; ANSW="yes";;
        esac
    done

    # debug
    #echo
    #echo
    #echo "apikey = ${MERAKI_API}"
    #echo "org_ig = ${ORG_ID}"
    #echo "net id = ${NETWORK_ID}"
    #echo "timesp = ${TIMESPAN}"
    #echo "output = ${OUTPUT}"
    #echo "date   = $(date +%d-%m-%Y)-request.json"
    #echo "success" #after debug

    # set the default filename with the current date
    DEFAULT_FL="$(date +%d-%m-%Y)-request.json"

    # execute curl request and save it in a temp file
    LASTREQ=$(curl -L -H "X-Cisco-Meraki-API-Key:${MERAKI_API}" -H "Content-Type: application/json" -X GET "https://api.meraki.com/api/v1/networks/${NETWORK_ID}/clients?perPage=1000&getInternalData=true&timespan=${TIMESPAN}" | jq '.' >> ${DEFAULT_FL})

    # change filename if it was define in the arguments
    [[ -z ${OUTPUT} ]] && echo "JSON file save in the current directory with default filename (${DEFAULT_FL})" || (mv ${DEFAULT_FL} "${OUTPUT}.json" && echo "JSON file save in the current directory (${OUTPUT}.json)")
}

parse () {
    # parse function
    # keep only the fields we are looking for
    grep -E "mac|\"ip\"|\"description\"|\"user\"|lastSeen|recentDeviceMac" "${OUTPUT}.json" >> ${TMP0}
    echo "content grep!"
    # removing beginning spaces
    tr -s ' ' < ${TMP0} >> ${TMP1}
    echo "spaces removed!"
    echo
    echo "separating by groups..."
    # groups are always beginning with "mac" field, and compose of 7 lines
    # match lines containing pattern (here "mac), and add a newline before this one
    # -i to overwrite
    sed -i 's#\"mac#\n\n&#g' ${TMP1}
    echo "done"
    echo
    echo "removing groups we don't want (null, VM*, PVE*, ...)"
    echo "keeping only LGRP.*"
    # wait 2s only for reading time
    #sleep 2
    # removing one line before pattern, 5 lines after pattern, and pattern itself
    awk 'NR==FNR{if (/description[": ]*[ABCDEFGHIJKMNOPQRSTUVWXYZ]+/) for (i=-1;i<=5;i++) del[NR+i]; next} !(FNR in del)' ${TMP1} ${TMP1} > ${TMP2}
    awk 'NR==FNR{if (/description[": ]*[a-z]+/) for (i=-1;i<=5;i++) del[NR+i]; next} !(FNR in del)' ${TMP2} ${TMP2} > ${TMP3}
    awk 'NR==FNR{if (/description[": ]*[0-9]+/) for (i=-1;i<=5;i++) del[NR+i]; next} !(FNR in del)' ${TMP3} ${TMP3} > ${TMP4}
    # removing all blank/empty lines, and adding again a newline before "mac" fields
    awk 'NF' ${TMP4} | sed 's#\"mac#\n&#g' > ${FINAL}
    # removing temp files
    rm -rf ${TMPDIR}
}

# *** Main ***
# Store all arguments in a variable
NARGS=$#
# Checking arguments (3 possibilities)
# greater or equal than 1 and less or equal than 3
[[ ! (${NARGS} -ge 1 && ${NARGS} -le 3) ]] && echo "You should only put between 1 and 3 arguments." && usage

# check if "jq" package is installed
[[ -z $(dpkg -l | grep -wo "jq") ]] && echo -e "\"jq\" package not installed.\nYou must install it:\ntry: sudo apt install jq" && exit 1

# regex matching the api key (no upper case letters)
re="[a-z0-9]+"
# 1&2. check for ./parse.sh input-file.json or ./parse.sh API_KEY
if [[ ${NARGS} -eq 1 ]];then
    FILE=$1
    MERAKI_API=$1
    # check for input-file.json
    # check for existing and regular file
    if [[ -f ${FILE} ]];then
        # removing suffix
        OUTPUT=${FILE%.*}
        # just parsing the already existing JSON file
        echo "Parsing file "${FILE}"..."
        echo
        # call the function
        parse
        echo "Your final file is ${FINAL}."
        # exit success
        exit 0
    # check for non-zero string (matching regex for the api key)
    elif [[ ${MERAKI_API} =~ ${re} ]];then
        # call the function
        my_requests
        echo "Starting to parse the JSON file..."
        # creating the default filename, need for the parse function
        OUTPUT="$(date +%d-%m-%Y)-request"
        parse
        echo "Done. File parsed."
        mv ${FINAL} "${OUTPUT}.json"
        echo "Your final file is ${OUTPUT}."
        exit 0
    # else we do not match any of these conditions
    else
        echo "Error arguments."
        usage
    fi
# 3. check for ./parse.sh output.json api_key
elif [[ ${NARGS} -eq 2 ]];then
    FILE=$1
    # remove suffix of OUTPUT (it will be added after)
    # Example:
    #  if orginal file = /foo/bar/file.first.second
    #  then the output = /foo/bar/file.first
    #  To remove the long extension, write ${OUTPUT%%.*} instead of ${OUTPUT%.*}
    #  the output will be: /foo/bar/file
    OUTPUT=${FILE%.*}
    MERAKI_API=$2
    ## check if arg 1 is a file
    ##[[ ! -f ${FILE} ]] && echo "${FILE} is not a valid file" && usage
    # check if arg 2 match an api key
    [[ ! ${MERAKI_API} =~ ${re} ]] && echo "Check your API key, there must be a mistake." && usage
    my_requests
    echo
    echo "Starting to parse the JSON file..."
    parse
    echo "Done. File ${OUTPUT}.json parsed."
    # renaming final file
    mv ${FINAL} "${OUTPUT}.json"
    echo "Your final file is ${OUTPUT}.json."
    exit 0
fi