ZMap script to scan (SYN) IP addresses

scan_ips_zmap_public.sh

#!/usr/bin/env bash
set -euo pipefail

SCRIPT_ARGS=("$@")

zstd_check()
{
  if ! command -v zstd > /dev/null; then
    exit 1
  fi
}

gzip_check()
{
  if ! command -v gzip > /dev/null; then
    exit 1
  fi
}


go_check()
{
  echo "Checking for go..."
  if command -v go > /dev/null; then
    echo "Detected golang..."
  else
    echo "Installing golang..."
    sudo apt update
    sudo apt install -y golang git make
    if [ "$?" -ne "0" ]; then
     echo "Unable to install golang!"
     exit 1
    fi
  fi
}



zmap_check()
{
 echo "Checking zmap..."
 if command -v zmap > /dev/null; then
   echo "Detected zmap..."
 else
   echo "Installing zmap from source"
   sudo apt update
   sudo apt-get -y install build-essential cmake libgmp3-dev gengetopt libpcap-dev flex byacc libjson-c-dev pkg-config libunistring-dev libjudy-dev
   git clone https://github.com/zmap/zmap.git
   cd zmap && cmake . && make -j4 && sudo make install  && cd .. && rm -rf zmap
   #sudo apt install -y zmap
   if [ "$?" -ne "0" ]; then
    echo "Unable to install zmap"
   fi
 fi
}


script_check()
{
 if [ "${#SCRIPT_ARGS[@]}" -ne 3 ]; then
    #echo $#
    echo "Incorrect number of arguments. The script needs THREE (3) arguments:"
    echo -e "(1)\t the domain names list filename/folder path"
    echo -e "(2)\t the folder to store the result"
    echo -e "(3)\t script probe number"
    exit 1
 fi
}

input_check()
{
 if [[ ! -f "$1" && ! -d "$1"  ]]; then
  echo "File/Folder $1 is not accessible!"
  exit 1
 fi
}

output_check()
{
 if [[ ! -d "$1"  ]]; then
  echo "Folder $1 is not accessible!"
  exit 1
 else
  echo "Creating folder $1/$2"
  mkdir -p $1/$2
 fi
}

datediff() {

  local start_ts end_ts diff_ts
  start_ts=$(date -d "$1" +%s)
  end_ts=$(date -d "$2" +%s)


  diff_ts=$(( end_ts - start_ts ))
  #echo $diff_ts

  local days=$(( diff_ts / 86400 ))
  local hours=$(( diff_ts - ( days * 86400 ) ))
  local hours=$(( hours / 3600 ))
  local minutes=$(( diff_ts - ( days * 86400 ) - ( hours * 3600 ) ))
  local minutes=$(( ( minutes % 3600) / 60 ))

  echo "$(printf "%02d"  $days) days, $(printf "%02d" $hours) hours, $(printf "%02d" $minutes) minutes"
}


run_command()
{
   local out_file=$(basename "$1" ".txt")
   local ts=$(date +"%Y%m%d.%H%M")
   local rf=$RESULT/"$out_file.$ts.jsonl"
   local lg=$RESULT/"$out_file.$ts.log"
   local mt=$RESULT/"$out_file.$ts.metadata.json"
   local pf=$RESULT/"$out_file.$ts.progress.csv"
   local stok=$4
   local total_files=$5
   local rate=$6
   local probe=$7
   probe=$((probe+0))

   echo "[$ts] Collecting data for $(printf "%08d" ${cur_file_domains}) IPs at a rate of $rate (probe=$probe) in file $1"

   
   sudo $zm -p 443 --probe-module=tcp_synscan \
	   --probes=$probe \
	   --list-of-ips-file="$f" \
	   --rate=$rate \
	   --output-filter="" \
	   --output-module=json \
	   --output-fields="saddr,sport,daddr,dport,classification,success,rtt,ttl,cooldown,repeat,timestamp_str" \
	   --output-file=$rf \
	   --log-file=$lg \
	   --metadata-file=$mt \
	   --status-updates-file=$pf \
	   --quiet \
	   --verbosity=5 \

   ts=$(date +"%Y%m%d.%H%M")
   echo -e "[$ts] Result saved in file $rf"
   echo -e "[$ts] Process $(printf "%09d" ${total_domains}) domains from file no. ${total_files}/$2 ..."

   if [ $? -eq 0 ]; then
    touch "$stok"
   else
    rm -rf "$stok"
   fi
}


zmap_check
zm=`which zmap`
echo "zmap is vailable at $zm"
script_check
input_check $1
input_check $2


now=$(date +"%Y%m%d")
output_check $2 $now

INPUT=$1
RESULT="$2/$now"
RATE=1000
PROBE=$3
PROBE=$((PROBE + 0))
start=$(date +"%Y%m%d.%H%M")
start_ts=$(date +"%Y-%m-%d %H:%M")

echo "===================================================================="
echo "[$start] ============= Measurement starts ==================="
echo "[$start] ================ Testing TCP/SYN  =================="
echo "===================================================================="

total_domains=0
total_files=0
declare -A RESULT_MAP

if [[ -d "$INPUT"  ]]; then

 counter=0
 for f in $INPUT/*.txt; do
    ((counter += 1))
 done

 MAX_JOBS=2

 ## Shuffle files list
 # Create an array of shuffled paths
 mapfile -t FILES_SHUF < <(find $INPUT/*.txt  -type f | shuf)

 for f in "${FILES_SHUF[@]}"; do

  result_files=()
  result_status_files=()
  out_file=$(basename "$f" ".txt")
  ts=$(date +"%Y%m%d.%H%M")
  rf=$RESULT/"$out_file.$ts.jsonl"
  stok=$RESULT/"$out_file.$ts.ok"

  cur_file_domains=$(wc -l < $f)
  (( total_domains += $cur_file_domains ))

  RESULT_MAP["$f"]="$stok~$rf"
  (( total_files +=1 ))
  
  run_command "$f" $counter $ts $stok $total_files $RATE $PROBE &

  

  if (($total_files % $MAX_JOBS == 0)); then
     wait

    if gzip_check; then

     for ft in "${!RESULT_MAP[@]}"; do
       sok=$(echo ${RESULT_MAP[$ft]} | cut -d '~' -f 1)
       rff=$(echo ${RESULT_MAP[$ft]} | cut -d '~' -f 2)
       
       if [[  -f "$ft" && -f "$sok" && -f "$rff" ]]; then
         gzip $rff
         rm -rf $sok
         ts=$(date +"%Y%m%d.%H%M")
         echo -e "[$ts] Compressed $rff into $rff.gz"
       fi
     done

    fi
   fi

 done

fi


end=$(date +"%Y%m%d.%H%M")
end_ts=$(date +"%Y-%m-%d %H:%M")

diff_time=$(datediff "$start_ts" "$end_ts")

echo "===================================================================="
echo "[$end] ================ Measurement ends =================="
echo "[$end] ========= Collect records for $(printf "%09d" ${total_domains}) IPs ========"
echo "[$end] ========== ${diff_time} ==========="
echo "[$end] ===================================================="
echo "===================================================================="

exit 0