Convert IP to CIDR

convert_ip_to_cidr.sh

#!/bin/bash

# DOWNLOAD LIST FROM HERE
BLACKLIST_URL="http://www.squidblacklist.org/downloads/drop.malicious.rsc"

# THE LIST WITH IP ADDRESSES
BLACKLIST_FILE="drop.malicious.rsc"

# DO I HAVE TO DOWNLOAD IT
DOWNLOAD="yes"

# SHOW LINES COUNT BEFORE AND AFTER
RESULT="yes"

# CONVERT FILE TO ROS
CONVERT_TO_ROS="yes"

# ROS FILENAME
ROS_FILE_NAME="blocklist.rsc"

# ROS LISTNAME WITHOUT SPACES
ROS_LIST_NAME="blocklist"

# ROS COMMENT WITHOUT SPACES
ROS_COMMENT="blocklist"

# TEMP FILE FOR ALL IPS CONVERTED TO TWENTYFOUR NET IDS
twentyfour="./twentyfour.ips"

# TEMP FILE FOR SIXTEEN BIT
sixteen="./sixteen.ips"

# TEMP FILE FOR 24 BIT IDS
twentyfourlst1="./twentyfour1.txt"

# TEMP FILE FOR 24 BIT IDS FILTERED BY 16 BIT IDS THAT MATCH
twentyfourlst2="./twentyfour2.txt"

# TEMP FILE FOR PARSED SIXTEENBIT
sixteenlst="./sixteen.txt"

# MODIFY FOR YOUR OUTPUT OF CIDR ADDRESSES
# FINAL FILE POST-MERGE
finalfile="./blockips.list"

################################
# Do not modify anything below #
################################

get_blacklist () {
	wget -q $BLACKLIST_URL -O $BLACKLIST_FILE
}

check_blacklist_exist () {
	if [ ! -f $BLACKLIST_FILE ]; then
	echo "ERROR: BLACKLIST DO NOT EXIST!"
	exit 1
	fi
}

if [ $DOWNLOAD == yes ]; then
	get_blacklist
fi

check_blacklist_exist

extract_the_ips_from_blacklist () {
	grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' $BLACKLIST_FILE | sort -t. -k1,1n -k2,2n -k3,3n -k4,4n | uniq > origin.ips.txt
}

extract_the_ips_from_blacklist

###############################

cat origin.ips.txt | while read line; do
oc1=`echo "$line" | cut -d '.' -f 1`
oc2=`echo "$line" | cut -d '.' -f 2`
oc3=`echo "$line" | cut -d '.' -f 3`
oc4=`echo "$line" | cut -d '.' -f 4`
echo "$oc1.$oc2.$oc3.0/24" >> $twentyfour
echo "$oc1.$oc2.0.0/16" >> $sixteen
done
awk '{i=1;while(i <= NF){a[$(i++)]++}}END{for(i in a){if(a[i]>4){print i,a[i]}}}' $sixteen | sed 's/ [0-9]\| [0-9][0-9]\| [0-9][0-9][0-9]//g' > $sixteenlst
sort -u $twentyfour > twentyfour.txt

# THIS FINDS NEAR DUPLICATES MATCHING FIRST TWO OCTETS
cat $sixteenlst | while read line; do
   oc1=`echo "$line" | cut -d '.' -f 1`
   oc2=`echo "$line" | cut -d '.' -f 2`
   oc3=`echo "$line" | cut -d '.' -f 3`
   oc4=`echo "$line" | cut -d '.' -f 4`
   grep "\b$oc1.$oc2\b" twentyfour.txt >> duplicates.txt
done

# THIS REMOVES THE NEAR DUPLICATES FROM THE TWENTYFOUR FILE
fgrep -vw -f duplicates.txt twentyfour.txt > twentyfourfinal.txt

# THIS MERGES BOTH RESULTS
cat twentyfourfinal.txt > $finalfile
cat $sixteenlst >> $finalfile
sort -u $finalfile > finalfile1st.txt
cat finalfile1st.txt > $finalfile

# THIS SHOWS RESULT
if [ $RESULT == yes ]; then
	ori=`wc -l origin.ips.txt| awk '{print $1}'`
	new=`wc -l $finalfile| awk '{print $1}'`
	echo "LINES BEFORE CONVERSION $ori"
	echo "LINES AFTER CONVERSION $new"
fi

# LAST MIN CLEANUP
rm -f $twentyfour $twentyfourlst $sixteen $sixteenlst duplicates.txt twentyfourfinal.txt origin.ips.txt twentyfour.txt finalfile1st.txt

# BUGFIX CIDR CONVERSION
sed -i 's/160/16/g' $finalfile
sed -i 's/240/24/g' $finalfile

if [ $CONVERT_TO_ROS == yes ]; then

	# ROS HEADER
	echo "/system logging disable 0" > $ROS_FILE_NAME
	echo "/ip firewall address-list remove [find list=$ROS_LIST_NAME]" >> $ROS_FILE_NAME
	echo "/ip firewall address-list" >> $ROS_FILE_NAME

	cat $finalfile | while read line; do
		echo "add list=$ROS_LIST_NAME address=$line comment=$ROS_COMMENT" >> $ROS_FILE_NAME
	done

	# ROS FOOTER
	echo "/system logging enable 0" >> $ROS_FILE_NAME
fi