OpenWRT bogon-updater called from cron or on-boot

bogon-updater

#!/bin/sh

# Designed for OpenWRT and requires associated "bogons" UCI file.
#
# This is still pretty raw now and not formally published.

# Copyright 2018.
#
# Feel free to use, edit, and re-distribute under the BSD 3-clause License.
# https://opensource.org/licenses/BSD-3-Clause

. /lib/functions.sh

BIN_IPSET="/usr/sbin/ipset"

usage_exit() {
	cat <<_EOH
USAGE: $(basename "$0")
	[ -l | -u ]

ACTIONS:
	-l: load bogons from cache file
	-u: check for and perform an update if needed
_EOH
	exit 0
}

die() {
	log "$1"
	echo "ERROR(bogon-updater): $1" >&2
	exit "${2:-1}"
}

log() {
	logger -t "bogon-updater" "$1"
}

parse_config() {
	config_load bogons
	config_foreach load_settings settings

	if [ "$cache_mkdir" -eq 1 ]; then
		mkdir -p "$cache_dir" || die "failed to create cache_dir"
	fi
	[ -d "$cache_dir" ] || die "cache_dir does not exist: $cache_dir"

	TS="$cache_dir/$file_ts"
	GZ="$cache_dir/$file_gz"

	now=$(date +%s)
	last_update=0
	last_change=0
	[ -r "$TS" ] && {
		read last_update
		read last_change
	} < "$TS"

	next_update=$((last_update + valid_hours * 3600))
}

get_gz_time() {
	gz_time=0
	if [ -r "$GZ" ]; then
		gz_time=$(gzip -dc "$GZ" | awk '{print $4; exit 0;}')
		[ "$gz_time" -ge 0 ] || gz_time=0
	fi
}

load_settings() {
	local cfg="$1"

	config_get_bool enabled "$cfg" enabled 1
	[ "$enabled" -eq 1 ] || exit 0

	config_get_bool cache_mkdir "$cfg" cache_mkdir 1
	config_get cache_dir "$cfg" cache_dir '/tmp/bogons'
	config_get shm_dir "$cfg" shm_dir '/dev/shm'
	config_get valid_hours "$cfg" valid_hours 24
	config_get ipset_name "$cfg" ipset_name 'bogons'
	config_get ipset_options "$cfg" ipset_options ''
	config_get growth_size "$cfg" growth_size 0

	[ -d "$shm_dir" ] || die "shm_dir must exist"

	config_get uri_scheme "$cfg" uri_scheme
	config_get uri_domain "$cfg" uri_domain
	config_get uri_gz "$cfg" uri_gz
	config_get uri_ts "$cfg" uri_ts
	config_get uri_tries "$cfg" uri_tries 5
	config_get uri_delay "$cfg" uri_delay 60

	[ -n "$uri_scheme" ] || die "No uri_scheme defined"
	[ -n "$uri_domain" ] || die "No uri_domain defined"
	[ -n "$uri_ts" ] || die "No uri_ts defined"
	[ -n "$uri_gz" ] || die "No uri_gz defined"
	[ "$uri_tries" -ge 1 ] || die "uri_tries must be >= 1"
	[ "$uri_delay" -ge 0 ] || die "uri_delay must be >= 0"

	config_get file_gz "$cfg" file_gz "${uri_gz##*/}"
	config_get file_ts "$cfg" file_ts "${uri_ts##*/}"
}

bogon_loader() {
	[ -r "$GZ" ] || die "No bogons file availble to load"
	n_entries=$(gzip -dc "$GZ" | wc -l)
	n_entries=$((n_entries - 1 + growth_size))

	runfile=$(mktemp "$shm_dir/bogon-loader.XXXXXX") \
		|| die "mktemp failed to create runfile"
	trap cleanup INT TERM EXIT
	echo "$$" > "$runfile"
	tmp_set="loader_${runfile##*.}"

	"$BIN_IPSET" create "$tmp_set" 'hash:net' maxelem "$n_entries" $ipset_options \
		|| die "failed to create temp set: $tmp_set"

	gzip -dc "$GZ" \
		| sed -E "1d;s/^/add $tmp_set /" \
		| "$BIN_IPSET" restore \
		|| die "bogon temp ipset restore failed"

	"$BIN_IPSET" swap "$tmp_set" "$ipset_name" \
		|| die "bogon ipset swap failed"

	# sets $gz_time >= 0 based on comment line in $GZ
	get_gz_time

	log "Load successful (updated: $gz_time)"
}

update_bogons() {
	if [ "$now" -lt "$next_update" ]; then
		log "next update ($next_update) has not passed: nothing to do"
		return 0
	fi

	download "$uri_scheme" "$uri_domain" "$uri_ts" "$file_ts" \
		|| die "download of timestamp file failed"

	this_update=0
	this_change=0
	{
		read this_update
		read this_change
	} < "$TS"

	# sets $gz_time >= 0 based on comment line in $GZ
	get_gz_time

	if [ "$this_change" -le "$gz_time" ]; then
		log "Last download ($gz_time) includes current change ($this_change)"
		return 0
	fi

	download "$uri_scheme" "$uri_domain" "$uri_gz" "$file_gz" \
		|| die "download of bogons file failed"

	log "Loading new bogons (updated: $this_update) (changed: $this_change)"
	bogon_loader
}

download() {
	local uri="${1}${2}${3}"
	local file="$4"
	local try_num=0
	local dl_dir="$cache_dir/dl"

	if [ ! -d "$dl_dir" ]; then
		mkdir "$dl_dir" || {
			log "could not mkdir $dl_dir"
			return 1
		}
	fi

	while [ "$try_num" -lt "$uri_tries" ]; do
		if [ "$try_num" -gt 0 ]; then
			log "failed download attempt $try_num (sleeping $uri_delay s)"
			sleep "$uri_delay"
		fi
		try_num=$((try_num + 1))
		rm -f "$dl_dir/$file"
		wget -q -P "$dl_dir" "$uri" 2>/dev/null && break
	done
	rv=$?

	if [ "$rv" -eq 0 ]; then
		mv "$dl_dir/$file" "$cache_dir" || return 1
		return 0
	fi

	return "$rv"
}

cleanup() {
	if [ -n "$tmp_set" ]; then
		"$BIN_IPSET" destroy "$tmp_set"
		tmp_set=""
	fi

	[ -e "$runfile" ] && rm -f "$runfile"
}

INVOKE="usage_exit"
while getopts hlu opt
do
	case "$opt" in
		h)	;;
		l)	INVOKE="bogon_loader" ;;
		u)	INVOKE="update_bogons" ;;
		?)	die "Bad options"
	esac
done
shift $((OPTIND - 1))

parse_config
"$INVOKE" "$@"
exit 0

bogons.init

#!/bin/sh /etc/rc.common

# Copyright 2017 Josh Cepek <josh.cepek AT usa.net>
# Licensed under the 3-clause BSD License:
#
# https://opensource.org/licenses/BSD-3-Clause

NAME=bogons
START=42

bogon_startup() {
	local cfg="$1"
	config_get_bool enabled "$cfg" updater_enabled 1
	config_get loader_bin "$cfg" updater_path
	if [ "$enabled" -ne 1 ] || [ ! -x "$loader_bin" ]; then
		return 0
	fi

	"$loader_bin" -l
}

start() {
	config_load bogons
	config_foreach bogon_startup settings
}

restart() {
	start
}

bogons.uci

# Bogons loader configuration

config settings
	option cache_mkdir	1
	option cache_dir	'/tmp/bogons'
	option shm_dir		'/dev/shm'
	option valid_hours	24
	option ipset_name	'bogons'
	#option growth_size	0
	#option ipset_options	''

	# For optional on-boot initscript operation:
	#option updater_enabled	1
	#option updater_path	'/usr/sbin/bogon-updater'

	# Where to download files from.
	option uri_scheme	'https://'
	option uri_domain	'my-cdn.example.net'
	option uri_gz		'/bogons/fullbogons-ipv4.gz'
	option uri_ts		'/bogons/fullbogons-ipv4-ts.txt'