Block globally reported hack attempts using your local iptables firewall rules

sync-fail2ban.sh

#!/bin/bash

##  Update fail2ban iptables with globally known attackers.
##  Actually, runs 100% independently now, without needing fail2ban installed.
##
##  /etc/cron.daily/sync-fail2ban
##
## Author: Marcos Kobylecki <[email protected]>
## http://www.reddit.com/r/linux/comments/2nvzur/shared_blacklists_from_fail2ban/


## Quit if fail2ban is missing.  Maybe this fake requirement can be skipped? YES.
#PROGRAM=/etc/init.d/fail2ban
#[ -x $PROGRAM ] || exit 0

datadir=/etc/fail2ban
[[ -d "$datadir" ]] || datadir=/tmp

## Get default settings of fail2ban (optional?)
[ -r /etc/default/fail2ban ] && . /etc/default/fail2ban

umask 000
blacklistf=$datadir/blacklist.blocklist.de.txt

mv -vf  $blacklistf  $blacklistf.last

badlisturls="http://antivirus.neu.edu.cn/ssh/lists/base_30days.txt http://lists.blocklist.de/lists/ssh.txt  http://lists.blocklist.de/lists/bruteforcelogin.txt"


 iptables -vN fail2ban-ssh   # Create the chain if it doesn't exist. Harmless if it does.
  
# Grab list(s) at https://www.blocklist.de/en/export.html .  Block.
echo "Adding new blocks:"
 time  curl -s http://lists.blocklist.de/lists/ssh.txt  http://lists.blocklist.de/lists/bruteforcelogin.txt \
  |sort -u \
  |tee $blacklistf \
  |grep -v '^#\|:' \
  |while read IP; do iptables -I fail2ban-ssh 1 -s $IP -j DROP; done 



# Which listings had been removed since last time?  Unblock.
echo "Removing old blocks:"
if [[ -r  $blacklistf.diff ]]; then
  #       comm  is brittle, cannot use sort -rn 
 time  comm -23 $blacklistf.last  $blacklistf \
   |tee $blacklistf.delisted \
   |grep -v '^#\|:' \
   |while read IP; do  iptables -w -D fail2ban-ssh -s $IP -j DROP || iptables -wv -D fail2ban-ssh -s $IP -j LOGDROP; done 

fi


# prepare for next time.
	diff -wbay $blacklistf.last $blacklistf  > $blacklistf.diff 





# Saves a copy of current iptables rules, should you like to check them later.
(set -x; iptables -wnv -L --line-numbers; iptables -wnv -t nat -L --line-numbers) &> /tmp/iptables.fail2ban.log &


exit 

# iptables v1.4.21: host/network `2a00:1210:fffe:145::1' not found
# So weed out IPv6, try |grep -v ':' 

## http://ix.io/fpC

 
# Option:  actionban
# Notes.:  command executed when banning an IP. Take care that the
#          command is executed with Fail2Ban user rights.
# Tags:    See jail.conf(5) man page
# Values:  CMD
#
actionban = iptables -I fail2ban-<name> 1 -s <ip> -j <blocktype># Option:  actionunban
# Notes.:  command executed when unbanning an IP. Take care that the
#          command is executed with Fail2Ban user rights.
# Tags:    See jail.conf(5) man page
# Values:  CMD
#
actionunban = iptables -D fail2ban-<name> -s <ip> -j <blocktype>

Hi,

I'm using your script because ipset doesn't work in OpenVZ but i get the following errors:

/usr/local/bin/sync-blocklist: line 85: syntax error near unexpected token newline' /usr/local/bin/sync-blocklist: line 85:actionban = iptables -I fail2ban- 1 -s -j # Option: actionunban'
'/etc/fail2ban/blacklist.txt' -> '/etc/fail2ban/blacklist.txt.last'
/usr/local/bin/sync-blocklist: line 32: iptables: command not found
Adding new blocks:
/usr/local/bin/sync-blocklist: line 41: iptables: command not found
/usr/local/bin/sync-blocklist: line 41: iptables: command not found

The last lines repeat for every ip added.
The rules are added though.

joaomourato:

In a login shell, check
which iptables
you should get
/sbin/iptables

Then, in your script, explicitly change
iptables -> /sbin/iptables

Do the same for ip6tables and ipset.

I wish the ipset method worked, but it just does not on my Ubuntu 14.04.04 x64 VPS.

@d--j
Hi thanks for the code, I'm using your script, but sometimes I get some errors

/etc/cron.daily/sync_blocklist:
ipset v6.23: Set cannot be created: set with the same name already exists
ipset v6.23: Set cannot be created: set with the same name already exists
/etc/cron.daily/sync_blocklist: line 49: unexpected EOF while looking for matching `''
/etc/cron.daily/sync_blocklist: line 61: syntax error: unexpected end of file
run-parts: /etc/cron.daily/sync_blocklist exited with return code 2 

any ideas on how to fix it?

Hi @fwsl

sorry I haven't seen your question until now. I currently use the following version:

#!/bin/bash

# updates the block list once daily. Make this file executable and place it in
#   /etc/cron.daily/sync-blocklist
#
#  requires the ipset program, install it e.g. via
#  apt-get install ipset
#
# inspired by https://gist.github.com/klepsydra/ecf975984b32b1c8291a
# but uses ipset because directly using iptables / ip6tables does not scale

PATH="/sbin:/usr/sbin:/bin:/usr/bin"

if test ! -x /sbin/ipset && test ! -x /usr/sbin/ipset; then
  echo "ipset not installed"
  exit 1
fi

if test ! -x /usr/bin/sipcalc && test ! -x /bin/sipcalc; then
  echo "sipcalc not installed"
  exit 1
fi

# sleep for up to 30 seconds to not overload blocklist.de on midnight
sleep $(( ( RANDOM % 30 )  + 1 ))s

SET_TYPE="hash:ip"
SET_CONFIG="hashsize 16384 maxelem 131072"
IPTABLES_WAIT=""
if iptables --help | fgrep -q -- '--wait'; then
  IPTABLES_WAIT="-w" # use --wait when iptables supports it
fi

# initialize the iptables integration if it is not already present
ipset -exist create blacklist-ip4 $SET_TYPE family inet $SET_CONFIG
ipset -exist create blacklist-ip6 $SET_TYPE family inet6 $SET_CONFIG
iptables -n $IPTABLES_WAIT -L INPUT | fgrep -q 'match-set blacklist-ip4 src' || iptables $IPTABLES_WAIT -I INPUT -m set --match-set blacklist-ip4 src -j DROP
ip6tables -n $IPTABLES_WAIT -L INPUT | fgrep -q 'match-set blacklist-ip6 src' || ip6tables $IPTABLES_WAIT -I INPUT -m set --match-set blacklist-ip6 src -j DROP

# create the new lists
ipset create new-blacklist-ip4 $SET_TYPE family inet $SET_CONFIG
ipset create new-blacklist-ip6 $SET_TYPE family inet6 $SET_CONFIG

# fill the new lists
#   this does the following:
#    1. get the file https://lists.blocklist.de/lists/all.txt
#    2. stream every line to the grep command that finds lines
#       that only have a IPv4 or IPv6 address on them (actually something
#       like 999.999.999.999 will match, too but we do not mind) to filter out
#       comments and shell injection attacks
#    3. Canonicalize (v6) and validate (v4 and v6) IP adreses via sipcalc
#    4. removes duplicate IP addresses (with canonicalizing IPv6 addresses first)
#    5. Remove IP addresses that would block too much or unwanted targets
#    6. prefixes "add new-blacklist-ip6" or "add new-blacklist-ip4" to the line
#       depending on wheter there is a : in the line (IPv6 addresses always have one)
#    7. feed these to ipset in one single call
curl -s https://lists.blocklist.de/lists/all.txt \
  | grep -Pxe '((?:[0-9]{1,3}\.){3}[0-9]{1,3}|(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])))' \
  | xargs sipcalc \
  | awk -F '- ' '/Expanded Address/ { print $2 } /Host address\s+-/ { print $2 }' \
  | sort -u \
  | grep -ve '^0.0.0.0$' \
  | awk '/:/{ print "add new-blacklist-ip6 " $0 } !/:/{ print "add new-blacklist-ip4 " $0 }' \
  | ipset restore

# swap the new and the old lists – but only if there is something in the new lists
if [[ $(ipset list new-blacklist-ip4 | wc -l) -gt 10 ]]; then # header is 6 or 7, so list needs more than 3 entries
  ipset swap blacklist-ip4 new-blacklist-ip4
fi
if [[ $(ipset list new-blacklist-ip6 | wc -l) -gt 10 ]]; then
  ipset swap blacklist-ip6 new-blacklist-ip6
fi


# remove the old lists (they have the new lists names now)
ipset destroy new-blacklist-ip4
ipset destroy new-blacklist-ip6

It uses sipcalc (apt-get install sipcalc) to validate and canonicalize the IP-adresses before feeding them to ipset.

The code also does a very crude check if the new blacklist has any entries in it. If the new list is empty it does not replace it. Maybe that helps with your problem.

Hi,
firts of all thank you both for your scripts.
I have a problem i dont understand, in debian8 the script made by d__j starts then exit with the error
ipset v6.23: Error in line 1: The set with the given name does not exist
what can I investigate to fix it?
BR

That 2014 Server Fault post was removed. It's archived here: make fail2ban use public blacklists. The question and top-placed answer (score: -5) are by the author of this script, @klepsydra.

@klepsydra, check out this LLM-assisted revision, works on Ubuntu and adds logging and handles duplicates within the same set, not duplicate lines in general. The script now handles the first run gracefully by:

• Creating the main ipsets before trying to use them
• Not failing if iptables rules can't be added immediately
• Using a fallback copy method if the atomic swap fails
• Providing sensible defaults (0) when counting entries in non-existent sets

#!/bin/bash

# Daily blocklist updater script for ipset v7.19+
# Logs to /var/log/blocklist.log

PATH="/sbin:/usr/sbin:/bin:/usr/bin"
LOG_FILE="/var/log/blocklist.log"
TMP_RESTORE="/tmp/ipset_restore.$$"

exec >> "$LOG_FILE" 2>&1
echo "=== Blocklist update started at $(date '+%Y-%m-%d %H:%M:%S') ==="

# Check required commands
for cmd in ipset sipcalc curl; do
    if ! command -v "$cmd" >/dev/null 2>&1; then
        echo "ERROR: $cmd not installed"
        exit 1
    fi
done

# Random sleep to distribute load
DELAY=$((RANDOM % 30))
echo "Sleeping for $DELAY seconds..."
sleep "${DELAY}s"

# Function to create ipset
initialize_ipset() {
    local set_name=$1
    local family=$2
    echo "Creating set $set_name (family $family)"
    ipset create "$set_name" hash:ip family "$family" hashsize 16384 maxelem 131072 -exist
}

# Create main ipsets if they don't exist (first run)
initialize_ipset blacklist-ip4 inet || exit 1
initialize_ipset blacklist-ip6 inet6 || exit 1

# Create temporary ipsets for atomic swap
initialize_ipset new-blacklist-ip4 inet || exit 1
initialize_ipset new-blacklist-ip6 inet6 || exit 1

# Ensure iptables rules exist
initialize_iptables() {
    local cmd=$1
    local set=$2
    if ! $cmd -n -L INPUT 2>/dev/null | grep -q "match-set $set src"; then
        echo "Adding $cmd INPUT rule for $set"
        $cmd -I INPUT -m set --match-set "$set" src -j DROP 2>/dev/null || {
            echo "Warning: Failed to add $cmd rule for $set (may not be critical)"
        }
    else
        echo "$cmd rule for $set already exists"
    fi
}
initialize_iptables iptables blacklist-ip4
initialize_iptables ip6tables blacklist-ip6

# Download and process blocklist
process_blocklist() {
    echo "Downloading and processing blocklist..."

    OLD_IP4=$(ipset list blacklist-ip4 2>/dev/null | grep -c '^[0-9]' || echo 0)
    OLD_IP6=$(ipset list blacklist-ip6 2>/dev/null | grep -c '^[0-9]' || echo 0)

    # Clean temporary files
    TMP_IP4="/tmp/ip4.$$"
    TMP_IP6="/tmp/ip6.$$"
    > "$TMP_IP4"
    > "$TMP_IP6"

    # Download and normalize
    curl -s https://lists.blocklist.de/lists/all.txt \
        | grep -E '^([0-9]{1,3}\.){3}[0-9]{1,3}$|^([0-9a-fA-F:]+)$' \
        | xargs -n1 sipcalc 2>/dev/null \
        | awk -F '- ' '/Expanded Address/ {print $2} /Host address/ {print $2}' \
        | grep -v '^0\.0\.0\.0$' \
        | sort -u \
        | while read -r ip; do
            if [[ "$ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
                echo "$ip" >> "$TMP_IP4"
            elif [[ "$ip" =~ : ]]; then
                echo "$ip" >> "$TMP_IP6"
            fi
          done

    # Remove duplicates again, generate restore file
    {
        echo "flush new-blacklist-ip4"
        sort -u "$TMP_IP4" | awk '{ print "add new-blacklist-ip4 " $1 }'
        echo "flush new-blacklist-ip6"
        sort -u "$TMP_IP6" | awk '{ print "add new-blacklist-ip6 " $1 }'
    } > "$TMP_RESTORE"

    echo "Prepared $(wc -l < "$TMP_RESTORE") ipset entries for restore"
    ipset restore < "$TMP_RESTORE" 2>&1 | grep -v 'element already added'

    NEW_IP4=$(ipset list new-blacklist-ip4 2>/dev/null | grep -c '^[0-9]' || echo 0)
    NEW_IP6=$(ipset list new-blacklist-ip6 2>/dev/null | grep -c '^[0-9]' || echo 0)

    echo "Blocklist processed:"
    echo "IPv4: $OLD_IP4 (old) → $NEW_IP4 (new)"
    echo "IPv6: $OLD_IP6 (old) → $NEW_IP6 (new)"

    rm -f "$TMP_IP4" "$TMP_IP6"
}

process_blocklist

# Swap temporary sets into place
swap_sets() {
    local set=$1
    local temp="new-$set"
    local count
    count=$(ipset list "$temp" 2>/dev/null | grep -c '^[0-9]' || echo 0)
    if (( count >= 1 )); then
        echo "Swapping $set with $temp ($count entries)"
        if ipset swap "$set" "$temp" 2>/dev/null; then
            echo "Swap successful"
        else
            echo "Swap failed, copying entries instead..."
            # Fallback: flush main set and copy from temp
            ipset flush "$set" 2>/dev/null
            ipset list "$temp" | grep '^[0-9]' | while read -r ip; do
                ipset add "$set" "$ip" 2>/dev/null
            done
            echo "Entries copied to $set"
        fi
    else
        echo "Not enough entries in $temp ($count), skipping swap"
    fi
}

swap_sets blacklist-ip4
swap_sets blacklist-ip6

# Cleanup
ipset destroy new-blacklist-ip4 2>/dev/null
ipset destroy new-blacklist-ip6 2>/dev/null
rm -f "$TMP_RESTORE"

# Final report
echo "Blocklist update completed at $(date '+%Y-%m-%d %H:%M:%S')"
echo "IPv4 blocked: $(ipset list blacklist-ip4 2>/dev/null | grep -c '^[0-9]' || echo 0)"
echo "IPv6 blocked: $(ipset list blacklist-ip6 2>/dev/null | grep -c '^[0-9]' || echo 0)"
echo "======================================"