update_hostsfile_adblock.sh

#!/bin/bash
#
# Intended to be used as a cron script in order to
# update and manage a host-blocking conf-file for
# unbound DNS resolver.
#
#set -x

REMOTE_HOST_LISTS=(
	"http://winhelp2002.mvps.org/hosts.txt"
	"http://someonewhocares.org/hosts/hosts"
)

HASH_TRACKFILE="/tmp/unbound_hostfile_trackfile"
UNBOUND_ADBLOCK_CONFFILE="/etc/unbound/local.d/hosts_adblocker.conf"
DNS_RESOLVER_SERVICE="unbound.service"

curlHostFiles() {
	local argt="$#"
	while (( $# > 0 )); do
		local t="$(mktemp)"
		curl -so "$t" "$1"
		printf "%s\t%s\t%s\n" "$(( argt-$#+1 ))" "$1" "$t"
		shift
	done
}

getCurrentHashes() {
	local argt="$#"
	while (( $# > 0 )); do
		read -d'\t' -a a <<< "$1"
		local num="${a[0]}"
		local url="${a[1]}"
		local tfile="${a[2]}"
		local md5="$( md5sum "$tfile" )"
		printf "%s\t%s\t%s\n" "$num" "$url" "${md5:0:32}"  #output TSV list
		shift
	done | tee "$HASH_TRACKFILE"
}

getPrevHashes() { (
	local a="$1"
	if [[ -f "$a" ]]; then
		readarray -t b < "$a"
	else
		exit 1
	fi
	printf '%s\n' "${b[@]}"
)}

convertToUnboundConf() {
	awk '/^[0-9]{1,3}\.[0-9]{1,3}/{gsub("\r",""); print "	local-data: \""$2". A 127.0.0.2\""}' \
	| sort -fu \
	| awk 'NR==1{print "server:"}; {print}'
}

hupTheResolver() {
		systemctl kill -s HUP --kill-who=main "${DNS_RESOLVER_SERVICE}" \
		|| systemctl restart "${DNS_RESOLVER_SERVICE}"
}

compareHashes() { (
	local num_precord="${#phash_table[@]}"
	local num_crecord="${#chash_table[@]}"
	if [[ "$num_precord" != "$num_crecord" ]]; then
		exit 1
	else
		for (( i=0; i < num_crecord; i++ )); do
			read -d'\t' -a a <<< "${chash_table[i]}"
			read -d'\t' -a b <<< "${phash_table[i]}"
			local chash="${a[2]}"
			local phash="${b[2]}"
			if [[ "$chash" != "$phash" ]]; then
				exit 1
			elif [[ "$chash" == "$phash" ]]; then
				continue
			fi
		done
	fi
	exit 0
)}

_setup() {
	readarray -t file_table  < <( curlHostFiles "${REMOTE_HOST_LISTS[@]}" )
	readarray -t phash_table < <( getPrevHashes "${HASH_TRACKFILE}" )
	readarray -t chash_table < <( getCurrentHashes "${file_table[@]}" )
}

_breakdown() {
	for i in "${!file_table[@]}"; do
		read -d'\t' -a c <<< "${file_table[i]}"
		read_files+=( "${c[2]}" )
	done
	case "$@" in
		write*)
			cat "${read_files[@]}" | convertToUnboundConf > "$UNBOUND_ADBLOCK_CONFFILE" &&
			hupTheResolver
			;;
	esac
	rm -f "${read_files[@]}"

}

# keep line out of function to only compute once
line="$( eval printf "%.s-" {1..$(tput cols)} )"

wrapLine() {
	printf "\n%s:\n%s" "$1" "$line"
	shift
	printf "%s\n" "$@" "$line"
}

case "_$@" in
	_print*)
		_setup
		wrapLine "Curl Hosts" "${file_table[@]}"
		wrapLine "Previous Hashes" "${phash_table[@]}"
		wrapLine "Current Hashes" "${chash_table[@]}"
	        _breakdown
		;;
	_)
		_setup
		if ! compareHashes; then
			_breakdown write
		fi
		;;
esac

z_example_output.conf

# Unbound conf: this is NOT a full config, please use the script above.
# The full config is ~23,000 lines
	local-data: "ad18.checkm8.com. A 127.0.0.2"
	local-data: "ad18.focalink.com. A 127.0.0.2"
	local-data: "ad18digital.checkm8.com. A 127.0.0.2"
	local-data: "ad19.checkm8.com. A 127.0.0.2"
	local-data: "ad19.focalink.com. A 127.0.0.2"
	local-data: "ad19digital.checkm8.com. A 127.0.0.2"
	local-data: "ad1digital.checkm8.com. A 127.0.0.2"
	local-data: "ad2.adecn.com. A 127.0.0.2"
	local-data: "ad2.adfarm1.adition.com. A 127.0.0.2"
	local-data: "ad2.adnetwork.net. A 127.0.0.2"
	local-data: "ad2.bal.dotandad.com. A 127.0.0.2"
	local-data: "ad2.bannerbank.ru. A 127.0.0.2"
	local-data: "ad2.bannerhost.ru. A 127.0.0.2"
	local-data: "ad2.bbmedia.cz. A 127.0.0.2"
	local-data: "ad2.checkm8.com. A 127.0.0.2"
	local-data: "ad2.cooks.com. A 127.0.0.2"
	local-data: "ad2.doubleclick.net. A 127.0.0.2"
	local-data: "ad2.doublepimp.com. A 127.0.0.2"
	local-data: "ad2.emediate.se. A 127.0.0.2"
	local-data: "ad2.firehousezone.com. A 127.0.0.2"
	local-data: "ad2.gammae.com. A 127.0.0.2"
	local-data: "ad2.hotel.com. A 127.0.0.2"
	local-data: "ad2.hotels.com. A 127.0.0.2"
	local-data: "ad2.ip.ro. A 127.0.0.2"
	local-data: "ad2.ireklama.cz. A 127.0.0.2"
	local-data: "ad2.lbn.ru. A 127.0.0.2"
	local-data: "ad2.nationalreview.com. A 127.0.0.2"
	local-data: "ad2.neodatagroup.com. A 127.0.0.2"
	local-data: "ad2.netshelter.net. A 127.0.0.2"
	local-data: "ad2.pamedia.com. A 127.0.0.2"
	local-data: "ad2.parom.hu. A 127.0.0.2"
	local-data: "ad2.peel.com. A 127.0.0.2"
	local-data: "ad2.pl. A 127.0.0.2"
	local-data: "ad2.pl.mediainter.net. A 127.0.0.2"
	local-data: "ad2.rambler.ru. A 127.0.0.2"
	local-data: "ad2.sbisec.co.jp. A 127.0.0.2"
	local-data: "ad2.smni.com. A 127.0.0.2"
	local-data: "ad2.speedbit.com. A 127.0.0.2"
	local-data: "ad2.tr.mediainter.net. A 127.0.0.2"
	local-data: "ad2.turn.com. A 127.0.0.2"
	local-data: "ad2.vuiads.net. A 127.0.0.2"
	local-data: "ad2.yam.com. A 127.0.0.2"
	local-data: "ad2.ycasmd.info. A 127.0.0.2"
	local-data: "ad2.zapmedya.com. A 127.0.0.2"
	local-data: "ad2.zophar.net. A 127.0.0.2"
	local-data: "ad20.checkm8.com. A 127.0.0.2"
	local-data: "ad20.net. A 127.0.0.2"
	local-data: "ad20digital.checkm8.com. A 127.0.0.2"
	local-data: "ad21.checkm8.com. A 127.0.0.2"
	local-data: "ad21digital.checkm8.com. A 127.0.0.2"
	local-data: "ad22.checkm8.com. A 127.0.0.2"
	local-data: "ad22digital.checkm8.com. A 127.0.0.2"
	local-data: "ad23.checkm8.com. A 127.0.0.2"
	local-data: "ad234.prbn.ru. A 127.0.0.2"
	local-data: "ad23digital.checkm8.com. A 127.0.0.2"
	local-data: "ad24.checkm8.com. A 127.0.0.2"
	local-data: "ad24digital.checkm8.com. A 127.0.0.2"
	local-data: "ad25.checkm8.com. A 127.0.0.2"
	local-data: "ad25digital.checkm8.com. A 127.0.0.2"
	local-data: "ad26.checkm8.com. A 127.0.0.2"
	local-data: "ad26digital.checkm8.com. A 127.0.0.2"
	local-data: "ad27.checkm8.com. A 127.0.0.2"
	local-data: "ad27digital.checkm8.com. A 127.0.0.2"
	local-data: "ad28.checkm8.com. A 127.0.0.2"
	local-data: "ad28digital.checkm8.com. A 127.0.0.2"
	local-data: "ad29.checkm8.com. A 127.0.0.2"
	local-data: "ad29digital.checkm8.com. A 127.0.0.2"
	local-data: "ad2digital.checkm8.com. A 127.0.0.2"
	local-data: "ad2games.com. A 127.0.0.2"
	local-data: "ad2play.ftv-publicite.fr. A 127.0.0.2"
	local-data: "ad3.adfarm1.adition.com. A 127.0.0.2"