sub2rbl for padavan firmware

sub2rbl

#!/bin/sh
#
# sub2rbl for padavan firmware @ProBackup-nl
# ============
# Subscribes to IP or CIDR (netblock) RBLs; auto-integrates into iptables
# Purely for reduced memory consumption it creates two IP sets:
# 1. one for unicast IPs (sub2rbl)
# 2. the other one for CIDR (sub2rbl-net) blocks
# Realtime block lists contain both types
#
#   https://gist.github.com/ProBackup-nl/f4914aef8c5f60d2d280c154dfc74da7
#

# Dependencies
# ============
# entware(-ng-3x)
# awk + xargs + mktemp + nano
# wget-ssl || curl (>=v7.73.0) + ca-certificates (186k) needed if using HTTPS RBLs
# kernel modules xt_set, ip_set_hash_ip, ip_set_hash_net
# built-in-firmware: ipset, iptables, grep, lsmod, grep, sed, mv, rm
# optional: iprange

# Installation
# ============
# 1. Go to "Padavan Router > Customization > Scripts" web interface page and put following content to "Run After Router Started" field:
# modprobe ip_set_hash_ip
# modprobe ip_set_hash_net
# modprobe xt_set
# ipset create sub2rbl hash:ip maxelem 32768
# ipset create sub2rbl-net hash:net maxelem 32768
#
# 2. nano /opt/etc/init.d/S01system
# *) start
#   [ -s '/opt/etc/ipset.conf' ] && ipset -file /opt/etc/ipset.conf restore
#   func_start
#
# 3. Go to "Padavan Router > Administration > Services" web interface page and enable "Cron Daemon (Scheduler)"
#
# 4. Go to "Padavan Router > Administration > Services" web interface page and put following content to "Scheduler tasks (Crontab)" field:
# 1-59/13 * * * * /opt/usr/sbin/sub2rbl
#
# This will run the update process every 13 minutes past the hour + 1 minute
#
# 5. nano /opt/etc/init.d/S10iptables
# start|update)
#         # add iptables sub2rbl rules
#         [ -d '/opt/etc' ] || exit 0
#         iptables -I INPUT 1 -m set --match-set sub2rbl src -j DROP
#         iptables -I INPUT 1 -m set --match-set sub2rbl-net src -j DROP
#         iptables -I FORWARD 1 -m set --match-set sub2rbl src -j DROP
#         iptables -I FORWARD 1 -m set --match-set sub2rbl-net src -j DROP
#         ;;
#
# 6. ssh into your router and run the update process once to generate the configuration file
# sub2rbl -l 3 -f stdout
#
# 7. enhance your configuration by excluding your own IP-adresses and/or CIDR blocks by whitelisting them
# nano /opt/etc/config/sub2rbl.conf
#
# 8. run the update process once more
# sub2rbl -l 3 -f stdout

## Improvements:
# 2026.06: error number display, retry insecure on cert failures

# ToDo:
# ===================
# - remove (very) old cached downloaded files when upstream is gone (current: on reboot)
# - process rbl lists according to their individual update frequency (even skip if-modified check)
# - whitelist from url

#
# Advanced settings
#
setName=sub2rbl
logLevel=1
logFacility=authpriv.notice
firewallTarget=DROP
firewallHookChains='INPUT:1 FORWARD:1'
netSetName="$setName"-net
logTag="sub2rbl[$$]"
ipsetArgs="maxelem 32768"
webGetCmd=guess
ignoreCert="--no-check-certificate" #handle wget error 5
umask 0002 # no write access for other

if [ ! -d /tmp/$setName ] ; then
  mkdir -m 664 /tmp/$setName
fi

# if webGetCmd is set to guess (default), we'll try to autoselect
# between a few options for an TLS ready http client.
# baseWebGetCmd is here to prevent overwriting files with 0 byte on connection errors
[ "$webGetCmd" = guess ] && {
  if [ -x /opt/bin/wget-ssl ] ; then
    #                            -q quiet
    #                             -T timeout (seconds)
    #                                  -t tries
    #                                       -O output document
    #                                      -N time-stamping, only save newer files
    #                                         -P set directory prefix to prefix
    webGetCmd="/opt/bin/wget-ssl -qT15 -t2 -N -P /tmp/$setName --hsts-file /tmp -O"  # GNU wget
    baseWebGetCmd="/opt/bin/wget-ssl -qT8 --spider"
  elif [ -x /opt/bin/wget ] ; then
    webGetCmd="/opt/bin/wget -qT15 -t2 -N -P /tmp/$setName --hsts-file /tmp -O"  # entware renamed wget-ssl to wget (wget became wget-nossl)
    baseWebGetCmd="/opt/bin/wget -qT8 --spider"
  elif [ -x /opt/bin/curl ] ; then
    #                       -f do not touch the local file if there is an error fetching the file
    #                        -s silent
    #                         -S show error
    #                          -R remote-time
    #                           -O output file and name file to remote
    #                            -m maximum transfer time (seconds)
    #                                                            -z time-conditional requires file, thus skipped
    webGetCmd="/opt/bin/curl -fsSROm15 --output-dir /tmp/$setName -z"  # curl with mbedTLS - most lightweight option
    baseWebGetCmd="/opt/bin/curl -sSm8"
    ignoreCert="--insecure" #error 60
  else
    # lastly, busybox wget - bad option as it fails on some TLS connections like spamhaus
    webGetCmd="/usr/bin/wget -qT15 -t2 -P /tmp/$setName -O"
    baseWebGetCmd="/usr/bin/wget -qT8 --spider"
  fi
}

# Functions
#
logLine() {
  [ $1 -gt $logLevel ] && return ; shift
  if [ "$logFacility" = "stdout" ] ; then echo "$@"
  elif [ "$logFacility" = "stderr" ] ; then echo "$@" >&2
  else logger -t "$logTag" -p "$logFacility" "$@"
  fi
}

# Arg: $1=logLevel, $2=setName, $3=setType (ip or net)
createSet() {
  local useLogLevel="$1" set="$2" setType="$3"
  ipset list -q -n "$set" >/dev/null || {
    logLine $useLogLevel "Creating ipset $set"
    ipset create "$set" hash:"$setType" $ipsetArgs
  }
}

# Arg: $1=logLevel, $2=setName
removeSet() {
  local useLogLevel=$1 set="$2"
  ipset list -q -n "$set" >/dev/null && {
    logLine $useLogLevel "Destroying ipset $set"
    ipset destroy "$set"
} }

# Arg: 1=$setName
setupIPTables() {
  local set="$1" x chain position
  for x in $firewallHookChains ; do
    chain="${x%:*}" ; position="${x#*:}"
    if [ $position -ge 0 ] && ! iptables -C "$chain" -m set --match-set "$set" src -j "$firewallTarget" 2>/dev/null ; then
      if [ $position = 0 ] ; then
        logLine 1 "Appending iptables rule for $set into chain $chain"
        iptables -A "$chain" -m set --match-set "$set" src -j "$firewallTarget"
      else
        logLine 1 "Inserting iptables rule at position $position for $set into chain $chain"
        iptables -I "$chain" $position -m set --match-set "$set" src -j "$firewallTarget"
  fi ; fi ; done
}

# Arg: 1=$setName
removeFromIPTables() {
  local set="$1" x chain position
  for x in $firewallHookChains ; do
    chain="${x%:*}" ; position="${x#*:}"
    if [ $position -ge 0 ] && iptables -C "$chain" -m set --match-set "$set" src -j "$firewallTarget" 2>/dev/null ; then
      logLine 1 "Removing iptables rule for $set from chain $chain"
      iptables -D "$chain" -m set --match-set "$set" src -j "$firewallTarget"
  fi ; done
}

printUsage() {
  cat <<-_EOF_
  Usage: sub2rbl [-C ...] [-f ...] [-j ...] [-l #] [-s ...] [-w] [URL ...]
    -C ... firewall hook chains:positions (def: $firewallHookChains)
    -f ... log facility (syslog facility or stdout/stderr) (def: $logFacility)
    -j ... firewall target when matching the set (def: $firewallTarget)
    -l #   log level - 0=off, 1=standard, 2=verbose (def: $logLevel)
    -s ... ipset name
    -w     wipe ipset and iptables rule (do not process RBLs)
    URL(s) RBL URLs - using command line RBL URLs will override uci entries
  _EOF_
}

writeConfigFile() {
  cat > /opt/etc/config/sub2rbl.conf <<'_EOF_'
#
# sub2rbl-padavan - https://github.com/ProBackup-nl/sub2rbl-padavan
#
# Save this file at /opt/etc/config/sub2rbl.conf
# Indent using tab-characters (TSV: tab separated value list)
# Values must be quoted

  # Whitelist regex strings examples
  # list  whitelist '^10\.0\.[01]\..*$'
    list  whitelist '^127\.0\.0\.0/8$'
    list  whitelist '^192\.168\.0\.0/16$'

  # RBL URLs - some (but not all) will also support http
  # Prepend net: to the URL for CIDR (net) RBLs
  # Order: net: URL's first (allows skipping IPs already blocked with net:)

        ### https://www.spamhaus.org/drop - Spamhaus (Extended) Don't Route Or Peer List - spammer/cyber-crime net based RBLs
        ### check frequency: 12 hours
                list    rbl     'net:https://www.spamhaus.org/drop/drop.txt'
                list    rbl     'net:https://www.spamhaus.org/drop/edrop.txt'

        ### Bogons and also IP space that has been allocated to an RIR, but not assigned by that RIR to an actual ISP or other end-user
        ### WARNING: Rigorously test this filter because it contains 127. 192. 172. and 10. non routable space.
                list    rbl     'net:https://team-cymru.org/Services/Bogons/fullbogons-ipv4.txt'

        ### http://ipdeny.com - provides lists of IPs associated with countries.  Ex: CN
        # check frequency: 1 day
        # list  rbl 'net:http://www.ipdeny.com/ipblocks/data/aggregated/cn-aggregated.zone'

  ### http://rbldata.interserver.net/ - malware posts, ssh/ftp/email/modsec scans, spam and more
  ### iprbl.txt = full (12K), standard ip.txt = last 7 days (2K), ipslim.txt = last 48 hours (1K)
        ### check frequency: 3 minutes
  ## list rbl 'https://sigs.interserver.net/iprbl.txt'
    list  rbl 'https://rbldata.interserver.net/ip.txt'
  ## list rbl 'https://rbldata.interserver.net/ipslim.txt'

  ### https://www.blocklist.de/en - blacklists generated from fail2ban feedback, last 48 hours
        ### check frequency:  15 minutes
    list  rbl 'https://lists.blocklist.de/lists/ssh.txt'
  ### All IPs which # are older then 2 month and have more then 5.000 attacks.
    list  rbl 'https://lists.blocklist.de/lists/strongips.txt'

_EOF_
}

if [ ! -f /opt/etc/config/sub2rbl.conf -o ! -s /opt/etc/config/sub2rbl.conf ] ; then
  writeConfigFile
  whitelist=''
  rblList=''
else
  # Read configuration from (uci style) configuration file /opt/etc/config/sub2rbl.conf
  whiteList=$(awk '!(/^\t*#/) && $2 == "whitelist" {print substr($3, 2, length($3) - 2)}' /opt/etc/config/sub2rbl.conf | xargs | sed -e 's/ /|/g')
  rblList=$(awk '!(/^\t*#/) && $2 == "rbl" {print substr($3, 2, length($3) - 2)}' /opt/etc/config/sub2rbl.conf)
fi



# Logic begins
#
wipe=false
# Pure ash getopts replacement by https://gist.github.com/jhurliman/ef18f8cd1880dc00365d
while [[ $# -gt 1 ]]; do
  arg="$1"
  case "$arg" in
    -C) firewallHookChains="$2"; shift ;;
    -f) logFacility="$2"; shift ;;
    -j) firewallTarget="$2"; shift ;;
    -l) logLevel="$2"; shift ;;
    -s) setName="$2"; shift ;;
    -w) wipe=true ;;
    *) printUsage
      exit 254
  esac
  shift
done

#before taking any action, first make sure that the required kernel modules are loaded:
modList='xt_set
ip_set_hash_ip
ip_set_hash_net'
#if not, load them using modprobe
#if loading fails, abort
for mod in $modList ; do
  if [ -z "lsmod | grep '^$mod '" ] ; then
    logLine 2 "Loading kernel module ($mod)"
    modprobe "$mod"
    wait 1
    if [ -z "lsmod | grep '^$mod '" ] ; then
      logLine 1 "Failure loading required kernel module ($mod)"
      exit 78
    fi
  fi
done

if $wipe ; then
  logLine 2 "Wiping sub2rbl from iptables and ipset"
  for tempSet in "$setName" "$netSetName" ; do
    removeFromIPTables "$tempSet"
    removeSet 1 "$tempSet"
  done
  # remove ipset backup file too
  [ -f /opt/etc/ipset.conf ] && rm /opt/etc/ipset.conf
  exit
fi

# Download updates when available
# WARNING: each rbl MUST use a unique file name
newFile=false
for rbl in $rblList ; do
  if [ ${rbl%%:*} = net ] ; then
    rbl="${rbl##net:}"
  fi
  logLine 2 "Retrieving RBL ($rbl)"
  fileName=/tmp/$setName/`basename $rbl`
  if ! $newFile; then
    [ -s $fileName ] && touch -r $fileName $fileName'.time'
  fi

  #logLine 3 "$webGetCmd $fileName $rbl"
  #try using HEAD before actual GET
  #if ! errorMsg=`$baseWebGetCmd "$rbl" 2>&1 && $webGetCmd "$fileName" "$rbl" 2>&1`; then
  errorMsg=`$webGetCmd "$fileName" "$rbl" 2>&1`
  status=$?
  if [ $status -ne 0 ]; then # 2024 Test: $status inserted figure out which exit code corresponds to a blank error message
    if ( [[ $status -eq 5 && $ignoreCert == "--no-check-certificate" ]] || [[ $status -eq 60 && $ignoreCert == "--insecure" ]] ); then 
      # retry without checking certificate chain
      logLine 1 "Error: retrieving RBL($rbl): [$status] retry without checking certs"
      errorMsg=`$webGetCmd "$fileName" "$rbl" "$ignoreCert" 2>&1`
      status=$?
      if [ $status -ne 0 ]; then
        logLine 1 "Error: insecure retrieving RBL($rbl): [$status] $errorMsg"
        # ToDo: remove really old files (current = cache forever)
        continue
      fi
    else
      logLine 1 "Error: retrieving RBL($rbl): [$status] $errorMsg"
      #continue?
    fi
  fi

  if ! $newFile; then
    if [ ! -e $fileName'.time' -a -s $fileName ] ; then
      logLine 3 '= new file (not exists before and not empty after)'
      newFile=true
    elif [ -e $fileName'.time' -a $fileName'.time' -ot $fileName ] ; then
      logLine 3 '= modified'
      newFile=true
    elif [ -e $fileName'.time' -a ! -e $fileName ] ; then
      logLine 3 '= removed (exist before and not exists after)'
      newFile=true
    fi
    [ -e $fileName'.time' ] && rm $fileName'.time'
  fi
done

if ! $newFile; then
  logLine 1 "No updates, exit."
  exit 0
fi

swingSetName="${setName}_swing"
swingNetSetName="${netSetName}_swing"
intNetEntries=0
tmpFile="`/opt/bin/mktemp`"
tmpFile2="`/opt/bin/mktemp`"
tmpFile3="`/opt/bin/mktemp`"

# Process
# configure net based first then IP based to be able to remove overlaps
for rbl in $rblList ; do
  if [ ${rbl%%:*} = net ] ; then
    setType='net'     # process net-based RBLs
    rbl="${rbl##net:}"
    tempSet="$netSetName"
  else
    setType='ip'    # default is IP-based (default)
    tempSet="$setName"
  fi
  fileName=/tmp/$setName/`basename $rbl`
  createSet 1 "$tempSet" $setType
  createSet 2 "$tempSet"_swing $setType
  #if ! errorMsg=`$webGetCmd "$tmpFile" "$rbl" 2>&1`; then

  # check that fileName exists and is not empty
  if [ -s $fileName ] ; then
    logLine 3 "Processing RBL ($rbl)"

    if [ $setType = ip ] ; then
      # autodetect the DRG sshpwauth report
      grep -Ev '^#' "$fileName" | head -1 | grep -q '^.*|.*|.*|.*|.*sshpwauth'
      if [ $? -eq 0 ] ; then
        logLine 3 "Loading DRG pwauth"
        sed -n 's/^.*|.*| *\([0-9\.][0-9\.]*\) .*$/\1/p' "$fileName" > "$tmpFile2"
      else
        logLine 3 "Loading plain IP list"
        sed -n 's/^ *\([1-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\)[  ]*.*$/\1/p' "$fileName" > "$tmpFile2"
      fi
      if [ -x /opt/bin/iprange ] ; then
        # remove IP's that match -net ranges
        # ToDo move to end of processing (after ranges are removed that match whitelisted IP's)
        ipset list $swingNetSetName | tail -n +9 > "$tmpFile"
        if [ -s "$tmpFile" -a -s "$tmpFile2" ] ; then
          logLine 3 "tmpFile (-net) and tmpFile2 not empty"
          [ $logLevel -eq 3 ] && setSizeBefore=`grep -c ^ "$tmpFile2"`
          /opt/bin/iprange -1 "$tmpFile2" --except "$tmpFile" > "$tmpFile3"
          mv "$tmpFile3" "$tmpFile2"
          [ $logLevel -eq 3 ] && setSize=`grep -c ^ "$tmpFile2"`
          [ $logLevel -eq 3 ] && logLine 3 "RBL ($rbl) reduced by $((setSizeBefore - setSize)) entries, matching $netSetName"
        fi
      fi
    elif [ $setType = net ] ; then
        logLine 3 "Loading plain Net list from $fileName"
        sed -n 's#^ *\([1-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*[/0-9]*\)[   ]*.*$#\1#p' "$fileName" > "$tmpFile2"
    fi
    if [ -n "$whiteList" ] ; then
      logLine 3 "Applying whitelist"
      #    -E = extended regular expression (ERE) grep
      #     -v = invert match, to select non-matching lines
      grep -Ev "$whiteList" "$tmpFile2" > "$tmpFile"
      mv "$tmpFile" "$tmpFile2"
    fi
    # Prefix tmp file
    sed -i "s/^/add "$tempSet"_swing /" "$tmpFile2"
    [ $logLevel -gt 1 ] && setSizeBefore=`ipset save "$tempSet"_swing | grep -Ec '^add '`
    ipset restore -f "$tmpFile2" -\!
    setSize=`ipset save "$tempSet"_swing | grep -Ec '^add '`
    [ $logLevel -gt 1 ] && logLine 2 "RBL ($rbl) added $((setSize - setSizeBefore)) entries"
    [ $setType = net ] && intNetEntries=$((setSize))
  fi
done

logLine 1 "Added $setSize entries (net:$intNetEntries)"

if [ -x /opt/bin/iprange ] ; then
  logLine 3 '/opt/bin/iprange is executable'
  if [[ $intNetEntries -gt 0 ]] ; then
    logLine 2 'Optimizing '$netSetName'_swing ('$intNetEntries' entries)'
    ## Optimize, speed and memory (59% lower, Size in memory, in test case)
    # ipset list sub2rbl-net | tail -n +9 | iprange --ipset-reduce 20 --print-prefix 'add sub2rbl-net ' > /tmp/tmp.lasIymKg
    ipset list $netSetName'_swing' | tail -n +9 | /opt/bin/iprange --ipset-reduce 20 --print-prefix 'add '$netSetName'_swing ' > $tmpFile
    # Flush ...-net_swing and Load from tmpFile
    ipset flush $netSetName'_swing'
    ipset restore -f "$tmpFile"
fi; fi

rm -f "$tmpFile" "$tmpFile2" "$tmpFile3"

for tempSet in "$netSetName" "$setName" ; do
  if ipset list -q -n "$tempSet"_swing >/dev/null ; then
    setupIPTables "$tempSet"
    logLine 3 "Swinging set $tempSet"
    ipset swap "$tempSet" "$tempSet"_swing
    removeSet 2 "$tempSet"_swing
  fi
done

if [ ! -z "$rblList" ] ; then
  # save backup backup for restore on router restart
  ipset -file /opt/etc/ipset.conf save
fi