Skip to content

Instantly share code, notes, and snippets.

@ProBackup-nl
Last active February 14, 2024 19:34
Show Gist options
  • Save ProBackup-nl/f4914aef8c5f60d2d280c154dfc74da7 to your computer and use it in GitHub Desktop.
Save ProBackup-nl/f4914aef8c5f60d2d280c154dfc74da7 to your computer and use it in GitHub Desktop.
sub2rbl for padavan firmware
#!/bin/sh
#
# sub2rbl for padavan firmware @ProBackup-nl
# ============
# Subscribes to IP or CIDR (netblock) RBLs; auto-integrates into iptables
# Purely for reduced memory consumption it creates two IP sets:
# 1. one for unicast IPs (sub2rbl)
# 2. the other one for CIDR (sub2rbl-net) blocks
# Realtime block lists contain both types
#
# https://gist.github.com/ProBackup-nl/f4914aef8c5f60d2d280c154dfc74da7
#
# Dependencies
# ============
# entware(-ng-3x)
# awk + xargs + mktemp
# wget-ssl || curl (>=v7.73.0) + ca-certificates (186k) needed if using HTTPS RBLs
# kernel modules xt_set, ip_set_hash_ip, ip_set_hash_net
# built-in-firmware: ipset, iptables, grep, lsmod, grep, sed, mv, rm
# optional: iprange
# Installation
# ============
# 0. mkdir -p /opt/usr/sbin;NAME=sub2rbl;wget -O /opt/usr/sbin/$NAME https://gist.githubusercontent.com/ProBackup-nl/f4914aef8c5f60d2d280c154dfc74da7/raw/sub2rbl && chmod 755 /opt/usr/sbin/$NAME
# 1. Go to "Padavan Router > Customization > Scripts" web interface page and put following content to "Run After Router Started" field:
# modprobe ip_set_hash_ip
# modprobe ip_set_hash_net
# modprobe xt_set
# ipset create sub2rbl hash:ip maxelem 32768
# ipset create sub2rbl-net hash:net maxelem 32768
#
# 2. nano /opt/etc/init.d/S01system
# *) start
# [ -s '/opt/etc/ipset.conf' ] && ipset -file /opt/etc/ipset.conf restore
# func_start
#
# 3. Go to "Padavan Router > Administration > Services" web interface page and enable "Cron Daemon (Scheduler)"
#
# 4. Go to "Padavan Router > Administration > Services" web interface page and put following content to "Scheduler tasks (Crontab)" field:
# 1-59/13 * * * * /opt/usr/sbin/sub2rbl
#
# This will run the update process every 13 minutes past the hour + 1 minute
#
# 5. nano /opt/etc/init.d/S10iptables
# start|update)
# # add iptables sub2rbl rules
# [ -d '/opt/etc' ] || exit 0
# iptables -I INPUT 1 -m set --match-set sub2rbl src -j DROP
# iptables -I INPUT 1 -m set --match-set sub2rbl-net src -j DROP
# iptables -I FORWARD 1 -m set --match-set sub2rbl src -j DROP
# iptables -I FORWARD 1 -m set --match-set sub2rbl-net src -j DROP
# ;;
#
# 6. ssh into your router and run the update process once to generate the configuration file
# sub2rbl -l 3 -f stdout
#
# 7. enhance your configuration by excluding your own IP-adresses and/or CIDR blocks by whitelisting them
# nano /opt/etc/config/sub2rbl.conf
#
# 8. run the update process once more
# sub2rbl -l 3 -f stdout
# Improvements / ToDo
# ===================
# - remove (very) old cached downloaded files when upstream is gone (current: on reboot)
# - process rbl lists according to their individual update frequency (even skip if-modified check)
# - whitelist from url
#
# Advanced settings
#
setName=sub2rbl
logLevel=1
logFacility=authpriv.notice
firewallTarget=DROP
firewallHookChains='INPUT:1 FORWARD:1'
netSetName="$setName"-net
logTag="sub2rbl[$$]"
ipsetArgs="maxelem 32768"
webGetCmd=guess
umask 0002 # no write access for other
if [ ! -d /tmp/$setName ] ; then
mkdir -m 664 /tmp/$setName
fi
# if webGetCmd is set to guess (default), we'll try to autoselect
# between a few options for an TLS ready http client.
[ "$webGetCmd" = guess ] && {
if [ -x /opt/bin/wget-ssl ] ; then
# -q quiet
# -T timeout (seconds)
# -t tries
# -O output document
# -N time-stamping, only save newer files
# -P set directory prefix to prefix
webGetCmd="/opt/bin/wget-ssl -qT15 -t2 -N -P /tmp/$setName -O" # GNU wget
elif [ -x /opt/bin/wget ] ; then
webGetCmd="/opt/bin/wget -qT15 -t2 -N -P /tmp/$setName -O" # entware renamed wget-ssl to wget (wget became wget-nossl)
elif [ -x /opt/bin/curl ] ; then
# -s silent
# -S show error
# -R remote-time
# -O output file and name file to remote
# -m maximum transfer time (seconds)
# -z time-conditional requires file, thus skipped
webGetCmd="/opt/bin/curl -sSROm15 --output-dir /tmp/$setName -z" # curl with mbedTLS - most lightweight option
else
# lastly, busybox wget - bad option as it fails on some TLS connections like spamhaus
webGetCmd="/usr/bin/wget -qT15 -t2 -P /tmp/$setName -O"
fi
}
# Functions
#
logLine() {
[ $1 -gt $logLevel ] && return ; shift
if [ "$logFacility" = "stdout" ] ; then echo "$@"
elif [ "$logFacility" = "stderr" ] ; then echo "$@" >&2
else logger -t "$logTag" -p "$logFacility" "$@"
fi
}
# Arg: $1=logLevel, $2=setName, $3=setType (ip or net)
createSet() {
local useLogLevel="$1" set="$2" setType="$3"
ipset list -q -n "$set" >/dev/null || {
logLine $useLogLevel "Creating ipset $set"
ipset create "$set" hash:"$setType" $ipsetArgs
}
}
# Arg: $1=logLevel, $2=setName
removeSet() {
local useLogLevel=$1 set="$2"
ipset list -q -n "$set" >/dev/null && {
logLine $useLogLevel "Destroying ipset $set"
ipset destroy "$set"
} }
# Arg: 1=$setName
setupIPTables() {
local set="$1" x chain position
for x in $firewallHookChains ; do
chain="${x%:*}" ; position="${x#*:}"
if [ $position -ge 0 ] && ! iptables -C "$chain" -m set --match-set "$set" src -j "$firewallTarget" 2>/dev/null ; then
if [ $position = 0 ] ; then
logLine 1 "Appending iptables rule for $set into chain $chain"
iptables -A "$chain" -m set --match-set "$set" src -j "$firewallTarget"
else
logLine 1 "Inserting iptables rule at position $position for $set into chain $chain"
iptables -I "$chain" $position -m set --match-set "$set" src -j "$firewallTarget"
fi ; fi ; done
}
# Arg: 1=$setName
removeFromIPTables() {
local set="$1" x chain position
for x in $firewallHookChains ; do
chain="${x%:*}" ; position="${x#*:}"
if [ $position -ge 0 ] && iptables -C "$chain" -m set --match-set "$set" src -j "$firewallTarget" 2>/dev/null ; then
logLine 1 "Removing iptables rule for $set from chain $chain"
iptables -D "$chain" -m set --match-set "$set" src -j "$firewallTarget"
fi ; done
}
printUsage() {
cat <<-_EOF_
Usage: sub2rbl [-C ...] [-f ...] [-j ...] [-l #] [-s ...] [-w] [URL ...]
-C ... firewall hook chains:positions (def: $firewallHookChains)
-f ... log facility (syslog facility or stdout/stderr) (def: $logFacility)
-j ... firewall target when matching the set (def: $firewallTarget)
-l # log level - 0=off, 1=standard, 2=verbose (def: $logLevel)
-s ... ipset name
-w wipe ipset and iptables rule (do not process RBLs)
URL(s) RBL URLs - using command line RBL URLs will override uci entries
_EOF_
}
writeConfigFile() {
cat > /opt/etc/config/sub2rbl.conf <<'_EOF_'
#
# sub2rbl-padavan - https://github.com/ProBackup-nl/sub2rbl-padavan
#
# Save this file at /opt/etc/config/sub2rbl.conf
# Indent using tab-characters (TSV: tab separated value list)
# Values must be quoted
# Whitelist regex strings examples
# list whitelist '^10\.0\.[01]\..*$'
list whitelist '^127\.0\.0\.0/8$'
list whitelist '^192\.168\.0\.0/16$'
# RBL URLs - some (but not all) will also support http
# Prepend net: to the URL for CIDR (net) RBLs
# Order: net: URL's first (allows skipping IPs already blocked with net:)
### https://www.spamhaus.org/drop - Spamhaus (Extended) Don't Route Or Peer List - spammer/cyber-crime net based RBLs
### check frequency: 12 hours
list rbl 'net:https://www.spamhaus.org/drop/drop.txt'
list rbl 'net:https://www.spamhaus.org/drop/edrop.txt'
### Bogons and also IP space that has been allocated to an RIR, but not assigned by that RIR to an actual ISP or other end-user
### WARNING: Rigorously test this filter because it contains 127. 192. 172. and 10. non routable space.
list rbl 'net:https://team-cymru.org/Services/Bogons/fullbogons-ipv4.txt'
### http://ipdeny.com - provides lists of IPs associated with countries. Ex: CN
# check frequency: 1 day
# list rbl 'net:http://www.ipdeny.com/ipblocks/data/aggregated/cn-aggregated.zone'
### http://rbldata.interserver.net/ - malware posts, ssh/ftp/email/modsec scans, spam and more
### iprbl.txt = full (12K), standard ip.txt = last 7 days (2K), ipslim.txt = last 48 hours (1K)
### check frequency: 3 minutes
## list rbl 'https://sigs.interserver.net/iprbl.txt'
list rbl 'https://rbldata.interserver.net/ip.txt'
## list rbl 'https://rbldata.interserver.net/ipslim.txt'
### https://www.blocklist.de/en - blacklists generated from fail2ban feedback, last 48 hours
### check frequency: 15 minutes
list rbl 'https://lists.blocklist.de/lists/ssh.txt'
### All IPs which # are older then 2 month and have more then 5.000 attacks.
list rbl 'https://lists.blocklist.de/lists/strongips.txt'
_EOF_
}
if [ ! -f /opt/etc/config/sub2rbl.conf -o ! -s /opt/etc/config/sub2rbl.conf ] ; then
writeConfigFile
whitelist=''
rblList=''
else
# Read configuration from (uci style) configuration file /opt/etc/config/sub2rbl.conf
whiteList=$(awk '!(/^\t*#/) && $2 == "whitelist" {print substr($3, 2, length($3) - 2)}' /opt/etc/config/sub2rbl.conf | xargs | sed -e 's/ /|/g')
rblList=$(awk '!(/^\t*#/) && $2 == "rbl" {print substr($3, 2, length($3) - 2)}' /opt/etc/config/sub2rbl.conf)
fi
# Logic begins
#
wipe=false
# Pure ash getopts replacement by https://gist.github.com/jhurliman/ef18f8cd1880dc00365d
while [[ $# -gt 1 ]]; do
arg="$1"
case "$arg" in
-C) firewallHookChains="$2"; shift ;;
-f) logFacility="$2"; shift ;;
-j) firewallTarget="$2"; shift ;;
-l) logLevel="$2"; shift ;;
-s) setName="$2"; shift ;;
-w) wipe=true ;;
*) printUsage
exit 254
esac
shift
done
#before taking any action, first make sure that the required kernel modules are loaded:
modList='xt_set
ip_set_hash_ip
ip_set_hash_net'
#if not, load them using modprobe
#if loading fails, abort
for mod in $modList ; do
if [ -z "lsmod | grep '^$mod '" ] ; then
logLine 2 "Loading kernel module ($mod)"
modprobe "$mod"
wait 1
if [ -z "lsmod | grep '^$mod '" ] ; then
logLine 1 "Failure loading required kernel module ($mod)"
exit 78
fi
fi
done
if $wipe ; then
logLine 2 "Wiping sub2rbl from iptables and ipset"
for tempSet in "$setName" "$netSetName" ; do
removeFromIPTables "$tempSet"
removeSet 1 "$tempSet"
done
# remove ipset backup file too
[ -f /opt/etc/ipset.conf ] && rm /opt/etc/ipset.conf
exit
fi
# Download updates when available
# WARNING: each rbl MUST use a unique file name
newFile=false
for rbl in $rblList ; do
if [ ${rbl%%:*} = net ] ; then
rbl="${rbl##net:}"
fi
logLine 2 "Retrieving RBL ($rbl)"
fileName=/tmp/$setName/`basename $rbl`
if ! $newFile; then
[ -s $fileName ] && touch -r $fileName $fileName'.time'
fi
#logLine 3 "$webGetCmd $fileName $rbl"
if ! errorMsg=`$webGetCmd "$fileName" "$rbl" 2>&1`; then
logLine 1 "Error: retrieving RBL($rbl): $errorMsg"
# ToDo: remove really old files (current = cache forever)
continue
fi
if ! $newFile; then
if [ ! -e $fileName'.time' -a -s $fileName ] ; then
logLine 3 '= new file (not exists before and not empty after)'
newFile=true
elif [ -e $fileName'.time' -a $fileName'.time' -ot $fileName ] ; then
logLine 3 '= modified'
newFile=true
elif [ -e $fileName'.time' -a ! -e $fileName ] ; then
logLine 3 '= removed (exist before and not exists after)'
newFile=true
fi
[ -e $fileName'.time' ] && rm $fileName'.time'
fi
done
if ! $newFile; then
logLine 1 "No updates, exit."
exit 0
fi
swingSetName="${setName}_swing"
swingNetSetName="${netSetName}_swing"
intNetEntries=0
tmpFile="`/opt/bin/mktemp`"
tmpFile2="`/opt/bin/mktemp`"
tmpFile3="`/opt/bin/mktemp`"
# Process
# configure net based first then IP based to be able to remove overlaps
for rbl in $rblList ; do
if [ ${rbl%%:*} = net ] ; then
setType='net' # process net-based RBLs
rbl="${rbl##net:}"
tempSet="$netSetName"
else
setType='ip' # default is IP-based (default)
tempSet="$setName"
fi
fileName=/tmp/$setName/`basename $rbl`
createSet 1 "$tempSet" $setType
createSet 2 "$tempSet"_swing $setType
#if ! errorMsg=`$webGetCmd "$tmpFile" "$rbl" 2>&1`; then
# check that fileName exists and is not empty
if [ -s $fileName ] ; then
logLine 3 "Processing RBL ($rbl)"
if [ $setType = ip ] ; then
# autodetect the DRG sshpwauth report
grep -Ev '^#' "$fileName" | head -1 | grep -q '^.*|.*|.*|.*|.*sshpwauth'
if [ $? -eq 0 ] ; then
logLine 3 "Loading DRG pwauth"
sed -n 's/^.*|.*| *\([0-9\.][0-9\.]*\) .*$/\1/p' "$fileName" > "$tmpFile2"
else
logLine 3 "Loading plain IP list"
sed -n 's/^ *\([1-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\)[ ]*.*$/\1/p' "$fileName" > "$tmpFile2"
fi
if [ -x /opt/bin/iprange ] ; then
# remove IP's that match -net ranges
# ToDo move to end of processing (after ranges are removed that match whitelisted IP's)
ipset list $swingNetSetName | tail -n +9 > "$tmpFile"
if [ -s "$tmpFile" -a -s "$tmpFile2" ] ; then
logLine 3 "tmpFile (-net) and tmpFile2 not empty"
[ $logLevel -eq 3 ] && setSizeBefore=`grep -c ^ "$tmpFile2"`
/opt/bin/iprange -1 "$tmpFile2" --except "$tmpFile" > "$tmpFile3"
mv "$tmpFile3" "$tmpFile2"
[ $logLevel -eq 3 ] && setSize=`grep -c ^ "$tmpFile2"`
[ $logLevel -eq 3 ] && logLine 3 "RBL ($rbl) reduced by $((setSizeBefore - setSize)) entries, matching $netSetName"
fi
fi
elif [ $setType = net ] ; then
logLine 3 "Loading plain Net list from $fileName"
sed -n 's#^ *\([1-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*[/0-9]*\)[ ]*.*$#\1#p' "$fileName" > "$tmpFile2"
fi
if [ -n "$whiteList" ] ; then
logLine 3 "Applying whitelist"
# -E = extended regular expression (ERE) grep
# -v = invert match, to select non-matching lines
grep -Ev "$whiteList" "$tmpFile2" > "$tmpFile"
mv "$tmpFile" "$tmpFile2"
fi
# Prefix tmp file
sed -i "s/^/add "$tempSet"_swing /" "$tmpFile2"
[ $logLevel -gt 1 ] && setSizeBefore=`ipset save "$tempSet"_swing | grep -Ec '^add '`
ipset restore -f "$tmpFile2" -\!
setSize=`ipset save "$tempSet"_swing | grep -Ec '^add '`
[ $logLevel -gt 1 ] && logLine 2 "RBL ($rbl) added $((setSize - setSizeBefore)) entries"
[ $setType = net ] && intNetEntries=$((setSize))
fi
done
logLine 1 "Added $setSize entries (net:$intNetEntries)"
if [ -x /opt/bin/iprange ] ; then
logLine 3 '/opt/bin/iprange is executable'
if [[ $intNetEntries -gt 0 ]] ; then
logLine 2 'Optimizing '$netSetName'_swing ('$intNetEntries' entries)'
## Optimize, speed and memory (59% lower, Size in memory, in test case)
# ipset list sub2rbl-net | tail -n +9 | iprange --ipset-reduce 20 --print-prefix 'add sub2rbl-net ' > /tmp/tmp.lasIymKg
ipset list $netSetName'_swing' | tail -n +9 | /opt/bin/iprange --ipset-reduce 20 --print-prefix 'add '$netSetName'_swing ' > $tmpFile
# Flush ...-net_swing and Load from tmpFile
ipset flush $netSetName'_swing'
ipset restore -f "$tmpFile"
fi; fi
rm -f "$tmpFile" "$tmpFile2" "$tmpFile3"
for tempSet in "$netSetName" "$setName" ; do
if ipset list -q -n "$tempSet"_swing >/dev/null ; then
setupIPTables "$tempSet"
logLine 3 "Swinging set $tempSet"
ipset swap "$tempSet" "$tempSet"_swing
removeSet 2 "$tempSet"_swing
fi
done
if [ ! -z "$rblList" ] ; then
# save backup backup for restore on router restart
ipset -file /opt/etc/ipset.conf save
fi
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment