Last active
February 14, 2024 19:34
-
-
Save ProBackup-nl/f4914aef8c5f60d2d280c154dfc74da7 to your computer and use it in GitHub Desktop.
sub2rbl for padavan firmware
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/sh | |
# | |
# sub2rbl for padavan firmware @ProBackup-nl | |
# ============ | |
# Subscribes to IP or CIDR (netblock) RBLs; auto-integrates into iptables | |
# Purely for reduced memory consumption it creates two IP sets: | |
# 1. one for unicast IPs (sub2rbl) | |
# 2. the other one for CIDR (sub2rbl-net) blocks | |
# Realtime block lists contain both types | |
# | |
# https://gist.github.com/ProBackup-nl/f4914aef8c5f60d2d280c154dfc74da7 | |
# | |
# Dependencies | |
# ============ | |
# entware(-ng-3x) | |
# awk + xargs + mktemp | |
# wget-ssl || curl (>=v7.73.0) + ca-certificates (186k) needed if using HTTPS RBLs | |
# kernel modules xt_set, ip_set_hash_ip, ip_set_hash_net | |
# built-in-firmware: ipset, iptables, grep, lsmod, grep, sed, mv, rm | |
# optional: iprange | |
# Installation | |
# ============ | |
# 0. mkdir -p /opt/usr/sbin;NAME=sub2rbl;wget -O /opt/usr/sbin/$NAME https://gist.githubusercontent.com/ProBackup-nl/f4914aef8c5f60d2d280c154dfc74da7/raw/sub2rbl && chmod 755 /opt/usr/sbin/$NAME | |
# 1. Go to "Padavan Router > Customization > Scripts" web interface page and put following content to "Run After Router Started" field: | |
# modprobe ip_set_hash_ip | |
# modprobe ip_set_hash_net | |
# modprobe xt_set | |
# ipset create sub2rbl hash:ip maxelem 32768 | |
# ipset create sub2rbl-net hash:net maxelem 32768 | |
# | |
# 2. nano /opt/etc/init.d/S01system | |
# *) start | |
# [ -s '/opt/etc/ipset.conf' ] && ipset -file /opt/etc/ipset.conf restore | |
# func_start | |
# | |
# 3. Go to "Padavan Router > Administration > Services" web interface page and enable "Cron Daemon (Scheduler)" | |
# | |
# 4. Go to "Padavan Router > Administration > Services" web interface page and put following content to "Scheduler tasks (Crontab)" field: | |
# 1-59/13 * * * * /opt/usr/sbin/sub2rbl | |
# | |
# This will run the update process every 13 minutes past the hour + 1 minute | |
# | |
# 5. nano /opt/etc/init.d/S10iptables | |
# start|update) | |
# # add iptables sub2rbl rules | |
# [ -d '/opt/etc' ] || exit 0 | |
# iptables -I INPUT 1 -m set --match-set sub2rbl src -j DROP | |
# iptables -I INPUT 1 -m set --match-set sub2rbl-net src -j DROP | |
# iptables -I FORWARD 1 -m set --match-set sub2rbl src -j DROP | |
# iptables -I FORWARD 1 -m set --match-set sub2rbl-net src -j DROP | |
# ;; | |
# | |
# 6. ssh into your router and run the update process once to generate the configuration file | |
# sub2rbl -l 3 -f stdout | |
# | |
# 7. enhance your configuration by excluding your own IP-adresses and/or CIDR blocks by whitelisting them | |
# nano /opt/etc/config/sub2rbl.conf | |
# | |
# 8. run the update process once more | |
# sub2rbl -l 3 -f stdout | |
# Improvements / ToDo | |
# =================== | |
# - remove (very) old cached downloaded files when upstream is gone (current: on reboot) | |
# - process rbl lists according to their individual update frequency (even skip if-modified check) | |
# - whitelist from url | |
# | |
# Advanced settings | |
# | |
setName=sub2rbl | |
logLevel=1 | |
logFacility=authpriv.notice | |
firewallTarget=DROP | |
firewallHookChains='INPUT:1 FORWARD:1' | |
netSetName="$setName"-net | |
logTag="sub2rbl[$$]" | |
ipsetArgs="maxelem 32768" | |
webGetCmd=guess | |
umask 0002 # no write access for other | |
if [ ! -d /tmp/$setName ] ; then | |
mkdir -m 664 /tmp/$setName | |
fi | |
# if webGetCmd is set to guess (default), we'll try to autoselect | |
# between a few options for an TLS ready http client. | |
[ "$webGetCmd" = guess ] && { | |
if [ -x /opt/bin/wget-ssl ] ; then | |
# -q quiet | |
# -T timeout (seconds) | |
# -t tries | |
# -O output document | |
# -N time-stamping, only save newer files | |
# -P set directory prefix to prefix | |
webGetCmd="/opt/bin/wget-ssl -qT15 -t2 -N -P /tmp/$setName -O" # GNU wget | |
elif [ -x /opt/bin/wget ] ; then | |
webGetCmd="/opt/bin/wget -qT15 -t2 -N -P /tmp/$setName -O" # entware renamed wget-ssl to wget (wget became wget-nossl) | |
elif [ -x /opt/bin/curl ] ; then | |
# -s silent | |
# -S show error | |
# -R remote-time | |
# -O output file and name file to remote | |
# -m maximum transfer time (seconds) | |
# -z time-conditional requires file, thus skipped | |
webGetCmd="/opt/bin/curl -sSROm15 --output-dir /tmp/$setName -z" # curl with mbedTLS - most lightweight option | |
else | |
# lastly, busybox wget - bad option as it fails on some TLS connections like spamhaus | |
webGetCmd="/usr/bin/wget -qT15 -t2 -P /tmp/$setName -O" | |
fi | |
} | |
# Functions | |
# | |
logLine() { | |
[ $1 -gt $logLevel ] && return ; shift | |
if [ "$logFacility" = "stdout" ] ; then echo "$@" | |
elif [ "$logFacility" = "stderr" ] ; then echo "$@" >&2 | |
else logger -t "$logTag" -p "$logFacility" "$@" | |
fi | |
} | |
# Arg: $1=logLevel, $2=setName, $3=setType (ip or net) | |
createSet() { | |
local useLogLevel="$1" set="$2" setType="$3" | |
ipset list -q -n "$set" >/dev/null || { | |
logLine $useLogLevel "Creating ipset $set" | |
ipset create "$set" hash:"$setType" $ipsetArgs | |
} | |
} | |
# Arg: $1=logLevel, $2=setName | |
removeSet() { | |
local useLogLevel=$1 set="$2" | |
ipset list -q -n "$set" >/dev/null && { | |
logLine $useLogLevel "Destroying ipset $set" | |
ipset destroy "$set" | |
} } | |
# Arg: 1=$setName | |
setupIPTables() { | |
local set="$1" x chain position | |
for x in $firewallHookChains ; do | |
chain="${x%:*}" ; position="${x#*:}" | |
if [ $position -ge 0 ] && ! iptables -C "$chain" -m set --match-set "$set" src -j "$firewallTarget" 2>/dev/null ; then | |
if [ $position = 0 ] ; then | |
logLine 1 "Appending iptables rule for $set into chain $chain" | |
iptables -A "$chain" -m set --match-set "$set" src -j "$firewallTarget" | |
else | |
logLine 1 "Inserting iptables rule at position $position for $set into chain $chain" | |
iptables -I "$chain" $position -m set --match-set "$set" src -j "$firewallTarget" | |
fi ; fi ; done | |
} | |
# Arg: 1=$setName | |
removeFromIPTables() { | |
local set="$1" x chain position | |
for x in $firewallHookChains ; do | |
chain="${x%:*}" ; position="${x#*:}" | |
if [ $position -ge 0 ] && iptables -C "$chain" -m set --match-set "$set" src -j "$firewallTarget" 2>/dev/null ; then | |
logLine 1 "Removing iptables rule for $set from chain $chain" | |
iptables -D "$chain" -m set --match-set "$set" src -j "$firewallTarget" | |
fi ; done | |
} | |
printUsage() { | |
cat <<-_EOF_ | |
Usage: sub2rbl [-C ...] [-f ...] [-j ...] [-l #] [-s ...] [-w] [URL ...] | |
-C ... firewall hook chains:positions (def: $firewallHookChains) | |
-f ... log facility (syslog facility or stdout/stderr) (def: $logFacility) | |
-j ... firewall target when matching the set (def: $firewallTarget) | |
-l # log level - 0=off, 1=standard, 2=verbose (def: $logLevel) | |
-s ... ipset name | |
-w wipe ipset and iptables rule (do not process RBLs) | |
URL(s) RBL URLs - using command line RBL URLs will override uci entries | |
_EOF_ | |
} | |
writeConfigFile() { | |
cat > /opt/etc/config/sub2rbl.conf <<'_EOF_' | |
# | |
# sub2rbl-padavan - https://github.com/ProBackup-nl/sub2rbl-padavan | |
# | |
# Save this file at /opt/etc/config/sub2rbl.conf | |
# Indent using tab-characters (TSV: tab separated value list) | |
# Values must be quoted | |
# Whitelist regex strings examples | |
# list whitelist '^10\.0\.[01]\..*$' | |
list whitelist '^127\.0\.0\.0/8$' | |
list whitelist '^192\.168\.0\.0/16$' | |
# RBL URLs - some (but not all) will also support http | |
# Prepend net: to the URL for CIDR (net) RBLs | |
# Order: net: URL's first (allows skipping IPs already blocked with net:) | |
### https://www.spamhaus.org/drop - Spamhaus (Extended) Don't Route Or Peer List - spammer/cyber-crime net based RBLs | |
### check frequency: 12 hours | |
list rbl 'net:https://www.spamhaus.org/drop/drop.txt' | |
list rbl 'net:https://www.spamhaus.org/drop/edrop.txt' | |
### Bogons and also IP space that has been allocated to an RIR, but not assigned by that RIR to an actual ISP or other end-user | |
### WARNING: Rigorously test this filter because it contains 127. 192. 172. and 10. non routable space. | |
list rbl 'net:https://team-cymru.org/Services/Bogons/fullbogons-ipv4.txt' | |
### http://ipdeny.com - provides lists of IPs associated with countries. Ex: CN | |
# check frequency: 1 day | |
# list rbl 'net:http://www.ipdeny.com/ipblocks/data/aggregated/cn-aggregated.zone' | |
### http://rbldata.interserver.net/ - malware posts, ssh/ftp/email/modsec scans, spam and more | |
### iprbl.txt = full (12K), standard ip.txt = last 7 days (2K), ipslim.txt = last 48 hours (1K) | |
### check frequency: 3 minutes | |
## list rbl 'https://sigs.interserver.net/iprbl.txt' | |
list rbl 'https://rbldata.interserver.net/ip.txt' | |
## list rbl 'https://rbldata.interserver.net/ipslim.txt' | |
### https://www.blocklist.de/en - blacklists generated from fail2ban feedback, last 48 hours | |
### check frequency: 15 minutes | |
list rbl 'https://lists.blocklist.de/lists/ssh.txt' | |
### All IPs which # are older then 2 month and have more then 5.000 attacks. | |
list rbl 'https://lists.blocklist.de/lists/strongips.txt' | |
_EOF_ | |
} | |
if [ ! -f /opt/etc/config/sub2rbl.conf -o ! -s /opt/etc/config/sub2rbl.conf ] ; then | |
writeConfigFile | |
whitelist='' | |
rblList='' | |
else | |
# Read configuration from (uci style) configuration file /opt/etc/config/sub2rbl.conf | |
whiteList=$(awk '!(/^\t*#/) && $2 == "whitelist" {print substr($3, 2, length($3) - 2)}' /opt/etc/config/sub2rbl.conf | xargs | sed -e 's/ /|/g') | |
rblList=$(awk '!(/^\t*#/) && $2 == "rbl" {print substr($3, 2, length($3) - 2)}' /opt/etc/config/sub2rbl.conf) | |
fi | |
# Logic begins | |
# | |
wipe=false | |
# Pure ash getopts replacement by https://gist.github.com/jhurliman/ef18f8cd1880dc00365d | |
while [[ $# -gt 1 ]]; do | |
arg="$1" | |
case "$arg" in | |
-C) firewallHookChains="$2"; shift ;; | |
-f) logFacility="$2"; shift ;; | |
-j) firewallTarget="$2"; shift ;; | |
-l) logLevel="$2"; shift ;; | |
-s) setName="$2"; shift ;; | |
-w) wipe=true ;; | |
*) printUsage | |
exit 254 | |
esac | |
shift | |
done | |
#before taking any action, first make sure that the required kernel modules are loaded: | |
modList='xt_set | |
ip_set_hash_ip | |
ip_set_hash_net' | |
#if not, load them using modprobe | |
#if loading fails, abort | |
for mod in $modList ; do | |
if [ -z "lsmod | grep '^$mod '" ] ; then | |
logLine 2 "Loading kernel module ($mod)" | |
modprobe "$mod" | |
wait 1 | |
if [ -z "lsmod | grep '^$mod '" ] ; then | |
logLine 1 "Failure loading required kernel module ($mod)" | |
exit 78 | |
fi | |
fi | |
done | |
if $wipe ; then | |
logLine 2 "Wiping sub2rbl from iptables and ipset" | |
for tempSet in "$setName" "$netSetName" ; do | |
removeFromIPTables "$tempSet" | |
removeSet 1 "$tempSet" | |
done | |
# remove ipset backup file too | |
[ -f /opt/etc/ipset.conf ] && rm /opt/etc/ipset.conf | |
exit | |
fi | |
# Download updates when available | |
# WARNING: each rbl MUST use a unique file name | |
newFile=false | |
for rbl in $rblList ; do | |
if [ ${rbl%%:*} = net ] ; then | |
rbl="${rbl##net:}" | |
fi | |
logLine 2 "Retrieving RBL ($rbl)" | |
fileName=/tmp/$setName/`basename $rbl` | |
if ! $newFile; then | |
[ -s $fileName ] && touch -r $fileName $fileName'.time' | |
fi | |
#logLine 3 "$webGetCmd $fileName $rbl" | |
if ! errorMsg=`$webGetCmd "$fileName" "$rbl" 2>&1`; then | |
logLine 1 "Error: retrieving RBL($rbl): $errorMsg" | |
# ToDo: remove really old files (current = cache forever) | |
continue | |
fi | |
if ! $newFile; then | |
if [ ! -e $fileName'.time' -a -s $fileName ] ; then | |
logLine 3 '= new file (not exists before and not empty after)' | |
newFile=true | |
elif [ -e $fileName'.time' -a $fileName'.time' -ot $fileName ] ; then | |
logLine 3 '= modified' | |
newFile=true | |
elif [ -e $fileName'.time' -a ! -e $fileName ] ; then | |
logLine 3 '= removed (exist before and not exists after)' | |
newFile=true | |
fi | |
[ -e $fileName'.time' ] && rm $fileName'.time' | |
fi | |
done | |
if ! $newFile; then | |
logLine 1 "No updates, exit." | |
exit 0 | |
fi | |
swingSetName="${setName}_swing" | |
swingNetSetName="${netSetName}_swing" | |
intNetEntries=0 | |
tmpFile="`/opt/bin/mktemp`" | |
tmpFile2="`/opt/bin/mktemp`" | |
tmpFile3="`/opt/bin/mktemp`" | |
# Process | |
# configure net based first then IP based to be able to remove overlaps | |
for rbl in $rblList ; do | |
if [ ${rbl%%:*} = net ] ; then | |
setType='net' # process net-based RBLs | |
rbl="${rbl##net:}" | |
tempSet="$netSetName" | |
else | |
setType='ip' # default is IP-based (default) | |
tempSet="$setName" | |
fi | |
fileName=/tmp/$setName/`basename $rbl` | |
createSet 1 "$tempSet" $setType | |
createSet 2 "$tempSet"_swing $setType | |
#if ! errorMsg=`$webGetCmd "$tmpFile" "$rbl" 2>&1`; then | |
# check that fileName exists and is not empty | |
if [ -s $fileName ] ; then | |
logLine 3 "Processing RBL ($rbl)" | |
if [ $setType = ip ] ; then | |
# autodetect the DRG sshpwauth report | |
grep -Ev '^#' "$fileName" | head -1 | grep -q '^.*|.*|.*|.*|.*sshpwauth' | |
if [ $? -eq 0 ] ; then | |
logLine 3 "Loading DRG pwauth" | |
sed -n 's/^.*|.*| *\([0-9\.][0-9\.]*\) .*$/\1/p' "$fileName" > "$tmpFile2" | |
else | |
logLine 3 "Loading plain IP list" | |
sed -n 's/^ *\([1-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\)[ ]*.*$/\1/p' "$fileName" > "$tmpFile2" | |
fi | |
if [ -x /opt/bin/iprange ] ; then | |
# remove IP's that match -net ranges | |
# ToDo move to end of processing (after ranges are removed that match whitelisted IP's) | |
ipset list $swingNetSetName | tail -n +9 > "$tmpFile" | |
if [ -s "$tmpFile" -a -s "$tmpFile2" ] ; then | |
logLine 3 "tmpFile (-net) and tmpFile2 not empty" | |
[ $logLevel -eq 3 ] && setSizeBefore=`grep -c ^ "$tmpFile2"` | |
/opt/bin/iprange -1 "$tmpFile2" --except "$tmpFile" > "$tmpFile3" | |
mv "$tmpFile3" "$tmpFile2" | |
[ $logLevel -eq 3 ] && setSize=`grep -c ^ "$tmpFile2"` | |
[ $logLevel -eq 3 ] && logLine 3 "RBL ($rbl) reduced by $((setSizeBefore - setSize)) entries, matching $netSetName" | |
fi | |
fi | |
elif [ $setType = net ] ; then | |
logLine 3 "Loading plain Net list from $fileName" | |
sed -n 's#^ *\([1-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*[/0-9]*\)[ ]*.*$#\1#p' "$fileName" > "$tmpFile2" | |
fi | |
if [ -n "$whiteList" ] ; then | |
logLine 3 "Applying whitelist" | |
# -E = extended regular expression (ERE) grep | |
# -v = invert match, to select non-matching lines | |
grep -Ev "$whiteList" "$tmpFile2" > "$tmpFile" | |
mv "$tmpFile" "$tmpFile2" | |
fi | |
# Prefix tmp file | |
sed -i "s/^/add "$tempSet"_swing /" "$tmpFile2" | |
[ $logLevel -gt 1 ] && setSizeBefore=`ipset save "$tempSet"_swing | grep -Ec '^add '` | |
ipset restore -f "$tmpFile2" -\! | |
setSize=`ipset save "$tempSet"_swing | grep -Ec '^add '` | |
[ $logLevel -gt 1 ] && logLine 2 "RBL ($rbl) added $((setSize - setSizeBefore)) entries" | |
[ $setType = net ] && intNetEntries=$((setSize)) | |
fi | |
done | |
logLine 1 "Added $setSize entries (net:$intNetEntries)" | |
if [ -x /opt/bin/iprange ] ; then | |
logLine 3 '/opt/bin/iprange is executable' | |
if [[ $intNetEntries -gt 0 ]] ; then | |
logLine 2 'Optimizing '$netSetName'_swing ('$intNetEntries' entries)' | |
## Optimize, speed and memory (59% lower, Size in memory, in test case) | |
# ipset list sub2rbl-net | tail -n +9 | iprange --ipset-reduce 20 --print-prefix 'add sub2rbl-net ' > /tmp/tmp.lasIymKg | |
ipset list $netSetName'_swing' | tail -n +9 | /opt/bin/iprange --ipset-reduce 20 --print-prefix 'add '$netSetName'_swing ' > $tmpFile | |
# Flush ...-net_swing and Load from tmpFile | |
ipset flush $netSetName'_swing' | |
ipset restore -f "$tmpFile" | |
fi; fi | |
rm -f "$tmpFile" "$tmpFile2" "$tmpFile3" | |
for tempSet in "$netSetName" "$setName" ; do | |
if ipset list -q -n "$tempSet"_swing >/dev/null ; then | |
setupIPTables "$tempSet" | |
logLine 3 "Swinging set $tempSet" | |
ipset swap "$tempSet" "$tempSet"_swing | |
removeSet 2 "$tempSet"_swing | |
fi | |
done | |
if [ ! -z "$rblList" ] ; then | |
# save backup backup for restore on router restart | |
ipset -file /opt/etc/ipset.conf save | |
fi |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment