IPTables Blacklist Scanners and Threats

blacklist.sh

#!/usr/bin/env bash
# update-blacklist.sh — verbose IP blacklist updater using ipset + iptables
# - Mixed FireHOL .netset/.ipset feeds
# - Splits into IPv4 IPs vs IPv4 CIDRs
# - Atomically updates:
#     blacklist_ips  (hash:ip)
#     blacklist_nets (hash:net)
# - Enforces DROP in INPUT (all ports)
# Requirements: curl, grep, sed, sort, ipset, iptables, mktemp, flock

set -euo pipefail

BASE_URL="https://raw.githubusercontent.com/firehol/blocklist-ipsets/master"
FEEDS=(
  "iblocklist_abuse_spyeye.netset"
  "firehol_abusers_1d.netset"
  "firehol_abusers_30d.netset"
  "ciarmy.ipset"
  "bruteforceblocker.ipset"
  "et_spamhaus.netset"
  "feodo.ipset"
  "feodo_badips.ipset"
  "iblocklist_abuse_palevo.netset"
  "iblocklist_abuse_zeus.netset"
  "dshield.netset"
  "dshield_1d.netset"
  "dshield_30d.netset"
  "dshield_7d.netset"
)

IPSET_IPS="blacklist_ips"
IPSET_NETS="blacklist_nets"

# Unique temp names per run
RUN_ID="$$"
TMP_IPS="${IPSET_IPS}_tmp_${RUN_ID}"
TMP_NETS="${IPSET_NETS}_tmp_${RUN_ID}"

need() { command -v "$1" >/dev/null 2>&1 || { echo "Missing: $1" >&2; exit 1; }; }
need curl; need grep; need sed; need sort; need ipset; need iptables; need mktemp; need flock

log() { printf '[%s] %s\n' "$(date -Is)" "$*"; }

# Serialize runs
LOCKFILE="/var/lock/update-blacklist.lock"
mkdir -p "$(dirname "$LOCKFILE")"
exec 9>"$LOCKFILE"
if ! flock -n 9; then
  log "Another update-blacklist run is in progress. Exiting."
  exit 0
fi

# Workdir + cleanup
WORKDIR="$(mktemp -d)"
cleanup() {
  ipset destroy "$TMP_IPS"  2>/dev/null || true
  ipset destroy "$TMP_NETS" 2>/dev/null || true
  rm -rf "$WORKDIR"
}
trap cleanup EXIT

# Back-compat: remove legacy fixed temp names
ipset destroy "${IPSET_IPS}_tmp"  2>/dev/null || true
ipset destroy "${IPSET_NETS}_tmp" 2>/dev/null || true

log "Starting blacklist update"
log "Feeds: ${#FEEDS[@]}"

RAW_DIR="$WORKDIR/raw"; mkdir -p "$RAW_DIR"
ALL="$WORKDIR/all.txt"; : >"$ALL"

# Fetch feeds and show per-feed counts
for f in "${FEEDS[@]}"; do
  url="${BASE_URL}/${f}"
  dest="$RAW_DIR/$f"
  log "Downloading: ${f}"
  if curl -fsSL "$url" -o "$dest"; then
    feed_all="$WORKDIR/${f}.all"
    grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]{1,2})?' "$dest" \
      | sed 's/\r$//' \
      | sort -u > "$feed_all" || true
    ips_n="$(grep -vc '/' "$feed_all" || true)"
    nets_n="$(grep -c  '/' "$feed_all" || true)"
    total_n="$(wc -l < "$feed_all" | tr -d ' ')"
    log "  -> parsed: total=${total_n} IPs=${ips_n} CIDRs=${nets_n}"
    cat "$feed_all" >> "$ALL"
  else
    log "  !! FAILED: $url"
  fi
done

# Global dedupe then split IPs vs CIDRs
sort -u -o "$ALL" "$ALL" || true
IPS="$WORKDIR/ips.txt"; NETS="$WORKDIR/nets.txt"
grep -v '/' "$ALL" > "$IPS" || true
grep '/'   "$ALL" | grep -E '/(3[0-2]|[12]?[0-9])$' > "$NETS" || true

IPS_N="$(wc -l < "$IPS" | tr -d ' ')"
NETS_N="$(wc -l < "$NETS" | tr -d ' ')"
log "Combined unique: IPs=${IPS_N} CIDRs=${NETS_N}"

# Size sets with headroom to avoid "Hash is full"
calc_sizes() {
  # $1=count ; returns maxelem in $2, hashsize in $3 (by name)
  local count="$1"
  local -n maxref="$2"
  local -n hashref="$3"
  local cushion=$(( count / 2 + 65536 ))  # 1.5x + 64k
  maxref=$(( count + cushion ))
  local h=$(( count / 2 ))
  if (( h < 16384 )); then h=16384; fi
  if (( h > 1048576 )); then h=1048576; fi
  for p in 16384 32768 65536 131072 262144 524288 1048576; do
    if (( h <= p )); then hashref=$p; break; fi
  done
}

IPS_MAXELEM=0 IPS_HASH=0
NETS_MAXELEM=0 NETS_HASH=0
calc_sizes "$IPS_N"  IPS_MAXELEM  IPS_HASH
calc_sizes "$NETS_N" NETS_MAXELEM NETS_HASH
log "Sizing: ${TMP_IPS} maxelem=${IPS_MAXELEM} hashsize=${IPS_HASH} ; ${TMP_NETS} maxelem=${NETS_MAXELEM} hashsize=${NETS_HASH}"

# Build ipset-restore batch files
RESTORE_IPS="$WORKDIR/ips.restore"
RESTORE_NETS="$WORKDIR/nets.restore"

{
  printf 'create %s hash:ip family inet timeout 0 hashsize %s maxelem %s\n' "$TMP_IPS" "$IPS_HASH" "$IPS_MAXELEM"
  if [[ -s "$IPS" ]]; then
    while IFS= read -r ip; do
      [[ -n "$ip" ]] && printf 'add %s %s -exist\n' "$TMP_IPS" "$ip"
    done < "$IPS"
  fi
} > "$RESTORE_IPS"

{
  printf 'create %s hash:net family inet timeout 0 hashsize %s maxelem %s\n' "$TMP_NETS" "$NETS_HASH" "$NETS_MAXELEM"
  if [[ -s "$NETS" ]]; then
    while IFS= read -r net; do
      [[ -n "$net" ]] && printf 'add %s %s -exist\n' "$TMP_NETS" "$net"
    done < "$NETS"
  fi
} > "$RESTORE_NETS"

log "Loading temporary ipsets…"
ipset restore -! < "$RESTORE_IPS"
ipset restore -! < "$RESTORE_NETS"

# Ensure live sets exist
ipset list "$IPSET_IPS"  >/dev/null 2>&1 || ipset create "$IPSET_IPS"  hash:ip  family inet timeout 0
ipset list "$IPSET_NETS" >/dev/null 2>&1 || ipset create "$IPSET_NETS" hash:net family inet timeout 0

# Swap temp -> live
log "Swapping ${TMP_IPS} -> ${IPSET_IPS}"
ipset swap "$TMP_IPS" "$IPSET_IPS"  || true
ipset destroy "$TMP_IPS"            || true

log "Swapping ${TMP_NETS} -> ${IPSET_NETS}"
ipset swap "$TMP_NETS" "$IPSET_NETS" || true
ipset destroy "$TMP_NETS"            || true

# Ensure iptables DROP rules
if ! iptables -C INPUT -m set --match-set "$IPSET_IPS" src -j DROP 2>/dev/null; then
  log "Installing INPUT DROP rule for ${IPSET_IPS}"
  iptables -I INPUT 1 -m set --match-set "$IPSET_IPS" src -j DROP
else
  log "INPUT DROP rule already present for ${IPSET_IPS}"
fi
if ! iptables -C INPUT -m set --match-set "$IPSET_NETS" src -j DROP 2>/dev/null; then
  log "Installing INPUT DROP rule for ${IPSET_NETS}"
  iptables -I INPUT 1 -m set --match-set "$IPSET_NETS" src -j DROP
else
  log "INPUT DROP rule already present for ${IPSET_NETS}"
fi

LIVE_IPS_CNT="$(ipset list "$IPSET_IPS"  | grep -F 'Number of entries:' | sed 's/.*: //')"
LIVE_NETS_CNT="$(ipset list "$IPSET_NETS" | grep -F 'Number of entries:' | sed 's/.*: //')"
log "Live sets: ${IPSET_IPS}=${LIVE_IPS_CNT:-0} ${IPSET_NETS}=${LIVE_NETS_CNT:-0}"
log "Done."

sudo apt-get update
sudo apt-get install -y ipset iptables-persistent netfilter-persistent curl util-linux
sudo install -m 755 /usr/local/sbin/update-blacklist.sh /usr/local/sbin/update-blacklist.sh

# syntax check
sudo bash -n /usr/local/sbin/update-blacklist.sh

# run
sudo /usr/local/sbin/update-blacklist.sh

# persist firewall rules
sudo netfilter-persistent save