OpenWRT Update IPSet from Adguard

adguard_ipset.sh

#!/bin/sh

info='
    https://gist.github.com/levid0s/ed4b3ad883a183e0eda80b6c60b209e9

    Script for OpenWRT that queries the AdGuard Home API and retreives recent DNS query results, and updates an IPset file with the values.
    This way, firewall and DNS access can be centrally managed from AdGuard, perhaps with a block-all rule /.*/ and manually adding exception.
    Using this scirpt, those exceptions would propagate into OpenWRT firewall rules.
'

: ${HOST:=http://192.168.666.666} # Adguard server
: ${TOKEN:=cfbcxxxxe5} # token
: ${SEARCH:=} # adguard search query 
: ${EXCLUDE:=$^nevermatch} # grep exclude regex
: ${RESULTS_MIN:=200}
: ${RESULTS_MAX:=100000}
: ${IPSET_DST:=} # IPSET file to update. If not specified, the results are printed to the screen.

: ${LOG_FILE:=/varp/adg-ipset.log}
: ${TMP_DIR:=/varp/tmp-ipset}
: ${COUNTER_FILE:=/tmp/adguard-counter.txt}

TS="$(date +"%Y%m%d-%H%M%S")"
KEEP_TEMP='yes'
EXIT_CODE=0

# ANSI colors
DG="\033[1;30m"; NC="\033[0m"

[ "$0" = "-ash" ] && SOURCING=1 || SOURCING=0

! command -v jq  &>/dev/null && echo 'ERROR: please install jq.' && exit 1

log_msg() {
    local message prio
    message="${1-null}"
    prio="${2-notice}" # info, notice, warn, err, debug

    [[ "$prio" == debug && "$DEBUG" != 1 ]] && return 0 || true
    # echo "$message" | tee /dev/stderr | logger -t adguard-ipset -p "$prio"
    echo "$message" >&2
    echo "$message" | logger -t adguard-ipset -p "$prio"
    [ -z "$LOG_FILE" ] || echo "$(date +"%Y/%m/%d %T"): $message" >> "$LOG_FILE"
}

debug_var() {
    local var_name value cmd compressed
    [ "${DEBUG:-0}" = 1 ] || return 0
    for var_name in "$@"; do
        cmd="value=\"\$$var_name\""
        eval "$cmd"
        compressed="$(echo -n "$value" | awk '{print}' ORS="${DG}~${NC}")"
        echo -e "$var_name=$compressed" >&2
        echo '' >&2
    done
}

exec_cmd() {
    local cmd
    cmd="$1"
    [ -n "${cmd-}" ] || return
    [ ! "${DEBUG:-0}" = 1 ] || echo "EXEC: $cmd" >&2
    eval "$cmd"
    [ "$?" = 0 ] || false
}

delete_temp() {
	[[ "${KEEP_TEMP-no}" != "yes" ]] || rm -f "$ipset_tmp" "$aging_data_tmp" "$query_data_tmp" "$ipset_old_tmp" 2>/dev/null || true
}


curl_adg() {
    local url cmd result
    [ ! "${TOKEN-}" = 'cfbcxxxxe5' ] || { log_msg "Error: AdGuard token not set." err; return 1; } # Token is default value
    [ ! "${HOST-}" = 'http://192.168.666.666' ] || { log_msg "Error: AdGuard host not set." err; return 1; } # Host is default value
    [ -n "${HOST-}" ] || { log_msg "Error: AdGuard host not set." err; return 1; }
    url="$1" # relative URL path
    echo "Calling: /$url" >&2
    cmd="curl -sL --connect-timeout 1 --max-time 2 -H \"Cookie: agh_session=$TOKEN\" \"$HOST/$url\""
    result="$(exec_cmd "$cmd")"
    # [ ! "${DEBUG:-0}" = 1 ] || echo "EXEC: $cmd"
    # eval "$cmd"
    # curl -sL --connect-timeout 1 --max-time 2 -H "Cookie: agh_session=$TOKEN" "$HOST/$url" -f
    #[ "$?" -ne 0 ] && echo "Error connecting to AdGuard." | tee /dev/stderr | logger -t adguard-ipset -p err && exit 1
    [ "$?" = 0 ] || { log_msg "Error connecting to AdGuard: $url" err; return 2; }
    echo "$result" | jq -e . >/dev/null 2>&1 || { log_msg "Error response from AdGuard: $url: $result"; return 3; }
    echo "$result"
}

get_log() {
    # Fetch the last N processed + whitelisted queries (these will mostly be whitelisted because the default rule is deny all)
    : ${NLIMIT:=$LIMIT}
    
    curl_adg "control/querylog?limit=$NLIMIT&response_status=whitelisted&search=$SEARCH" || return 1
    curl_adg "control/querylog?limit=$NLIMIT&response_status=processed&search=$SEARCH" || return 1
}

get_stats() {
    local stats
    # NOT IN USE ATM
    stats=$(curl_adg "control/stats" | jq -r '.num_dns_queries, .num_blocked_filtering, (.num_dns_queries - .num_blocked_filtering) | @sh')
    echo "$stats"
    # returns stats in the format of: num_queries num_blocked num_allowed
}

# get_nlimit() {
#     # Function returns the number of DNS queries adguard received SINCE the last run
# 
#     [ -n "${COUNTER_FILE-}" ] || { log_msg "Error: Counter file not set." warn; return 1; }
#     [ -f "$COUNTER_FILE" ] || touch "$COUNTER_FILE"
# 
#     [ "$1" == "--save" ] && save_out='true'
# 
#     # This file has saved the old counter in format: $date $counter. eg: 2024-11-16 1392527
#     # This way we know that if the date has changed, the counter will reset
# 
#     # Get the number of queries today: all_queries, blocked_queries, allowed_queries
#     response="$(curl_adg "control/stats")" || return 1 
#     stats_list="$(echo "$response" | jq -r '.num_dns_queries, .num_blocked_filtering, (.num_dns_queries - .num_blocked_filtering) | @sh')"
#     set -- $(echo "$stats_list")
#         num_queries=$1
#         num_blocked=$2
#         num_allowed=$3
# 
#     c_date=$(date +%Y-%m-%d)
# 
#     debug_var c_date num_queries num_blocked num_allowed
# 
#     read old_date num_before 2>/dev/null < "$COUNTER_FILE" || true
#     : ${num_before:=0}
#     debug_var old_date num_before
# 
#     # If the date has not changed since the last run, calcualte the diff between queies
#     # ie. delta = the number of DNS queries since the last run
#     [ "$c_date" == "${old_date-}" ] || num_before=0
#     delta="$((num_queries - num_before))"
# 
#     # Check if delta is between the min/max boundaries, otherwise just use the MIN/MAX values
#     # ie. only fetch the last X queries from the log
#     delta="$(( $delta < $RESULTS_MAX ? $delta : $RESULTS_MAX ))" # min
#     delta="$(( $delta > $RESULTS_MIN ? $delta : $RESULTS_MIN ))" # max
# 
#     # Save the new counters
#     [ "$save_out" != 'true' ] || { echo "$c_date $num_queries" > "$COUNTER_FILE"; log_msg "Updated counters: $c_date $num_queries" debug; }
#     debug_var delta
#     echo "$c_date $num_queries $delta" # return
# }
 
get_nlimit() {
    # Function reads the counters from the COUNTER_FILE
    # Also reads the counters/stats from AdGuard
    # Checks how many changes were recently, so we know how many queries to fetch from AdGuard
    # Ensures the delta is within the limits

    local new_date new_counter new_blocked new_allowed

    new_date=$(date +%Y-%m-%d)
    set -- $(get_stats)
        new_counter=$1
        new_blocked=$2
        new_allowed=$3
    debug_var new_date new_counter new_blocked new_allowed

    local old_date old_counter
    set -- $(load_counter)
        old_date=$1
        old_counter=$2
    debug_var old_date old_counter
    
    [[ "$new_date" == "$old_date" ]] || old_counter=0

    local delta
    delta="$((new_counter - old_counter))";
    debug_var delta

    delta="$(( $delta < $RESULTS_MAX ? $delta : $RESULTS_MAX ))" # min
    delta="$(( $delta > $RESULTS_MIN ? $delta : $RESULTS_MIN ))" # max
    debug_var delta
    
    echo "$new_date $new_counter $delta" # return
}

load_counter() {
    local c_date counter
    [ -n "${COUNTER_FILE-}" ] || { log_msg "Error: Counter file not set." warn; return 1; }
    [ -f "$COUNTER_FILE" ] || { echo "null 0"; return 0; }

    read c_date counter 2>/dev/null < "$COUNTER_FILE" || true
    : ${c_date:=0}
    debug_var c_date counter
    echo "$c_date $counter"
}

save_counter() {
    local c_date counter
    counter="$1"
    c_date="$2"
    [ -n "${COUNTER_FILE-}" ] || { log_msg "Error: Counter file not set." warn; return 1; }
    [ -n "${counter}" ] || { log_msg "WARNING: Counter not set." warn; return 1; }

    [ -n "${c_date-}" ] || c_date="$(date +%Y-%m-%d)"

    echo "$c_date $counter" > "$COUNTER_FILE"
}

get_responses() {
    if [[ "$1" == "-v" ]]; then
        get_log | jq -sr 'map(select(.data) | .data[] | .time as $time | .question.name as $name | .answer[]? | select(.type == "A") | {ip: .value, time: $time, name: $name}) | unique_by(.ip) | sort_by(.time) | .[] | "\(.ip),\(.time),\(.name),"' | grep -vE "$EXCLUDE" | awk -v ts=$ts -F, "{ printf \"%-16s# %-35s  %s\n\", \$1, \$2, \$3 }" 
    else
        # get_log | jq -sr '.[].data[].answer[]? | select(.type == "A") | {ip: .value, time: $time, name: $name}' | grep -vE "$EXCLUDE" | sort -u
        get_log | jq -sr 'map(select(.data) | .data[] | .time as $time | .question.name as $name | .answer[]? | select(.type == "A") | {ip: .value, time: $time, name: $name}) | sort_by(.time) | .[] | "\(.ip),\(.time),\(.name),"'
    fi
    # Return data is deduplicated by keeping the most recent IP only
}

get_whitelist() {
    curl_adg "control/filtering/status" | \
    jq -r '.user_rules | .[]' | \
    grep -E '^@@\|\|' | \
    sed 's/^\@\@||//g' | \
    sed 's/\^\$.*//g' | \
    grep -vE '^\s*$' | \
    sort -u
}

## Main

# skip the following if the script is sourced
if [ "$SOURCING" = 0 ]; then

    trap 'EXIT_CODE="$?" && log_msg "FATAL: Line: $LINENO; Exit code: $EXIT_CODE" err;' ERR
    trap '[[ "$EXIT_CODE" ]] || delete_temp' EXIT

    # IPSET_DST is set, we're working with the file only,
    if [[ -n "$IPSET_DST" ]]; then
    
        # Create the IPSet file if not exists
        [ -f "$IPSET_DST" ] || touch "$IPSET_DST"
		
		[ -d "$TMP_DIR" ] || mkdir -p "$TMP_DIR"
		ipset_tmp="$(mktemp "${TMP_DIR}/${TS}-ipset-adg-XXXXXX")"; debug_var ipset_tmp
		aging_data_tmp="$(mktemp "${TMP_DIR}/${TS}-aging-data-XXXXXX")"; debug_var aging_data_tmp
		query_data_tmp="$(mktemp "${TMP_DIR}/${TS}-query-data-XXXXXX")"; debug_var query_data_tmp

        # All known IPs in IPSET_DST
        known_ips="$(cut -d' ' -f1 "$IPSET_DST")"; debug_var known_ips


        # NLIMIT = min/max(delta of adguard DNS queries since the last time); eg:
        # NLIMIT = 200
        # export NLIMIT="$(get_nlimit)"
        read c_date counter nlimit <<EOF 
        $(get_nlimit)
EOF
        debug_var counter_date counter nlimit
        export NLIMIT="$nlimit"

		###[ QUERY DATA ]###
        # Get the latest adguard DNS queries data in 3 column format, eg:
        # 142.250.200.3   # 2024-11-16T16:07:01.8012214Z         connectivitycheck.gstatic.com
		get_responses -v > "$query_data_tmp"
		query_ips="$(cut -f1 -d' ' "$query_data_tmp")"

		###[ AGING DATA ]###
        # List IPs (data) present in IPSET_DST that haven't been queried this time
        # We don't need to change the timestamp on these, but all other entries need their timestamps updated
		grep -Fvw -f <(echo "$query_ips") "$IPSET_DST" > "$aging_data_tmp"
		cat "$aging_data_tmp" "$query_data_tmp" > "$ipset_tmp"

		###[ LOGGING ]###
		query_new_ips="$(grep -Fvw -f <(echo "$known_ips") <(echo "$query_ips") || true)"

		set -- $(wc -l "$ipset_tmp" "$IPSET_DST" | awk '{print $1}')
			ipset_tmp_lines="$1"
			IPSET_DST_LINES="$2"
			delta="$((ipset_tmp_lines - IPSET_DST_LINES))"

        [ -n "$query_new_ips" ] && lastq_log_msg=": $(
			grep -Fw -f <(echo "$query_new_ips") "$query_data_tmp" | awk '{print $4, $1}' | sort | awk '
			{
				dns = $1  # DNS record
				ip = $2   # IP address
				grouped[dns] = grouped[dns] ? grouped[dns] "," ip : ip
			}
			END {
				output = ""
				for (dns in grouped) {
					output = output ? output "; " dns ":" grouped[dns] : dns ":" grouped[dns]
				}
				print output
			}'
		)"
		log_msg "Total: $ipset_tmp_lines. Added $delta IPs${lastq_log_msg}. Counter: $counter Nlimit: $NLIMIT"
        

		###[ UPDATE ]###
		if [ "$ipset_tmp_lines" -lt "$IPSET_DST_LINES" ]; then
			log_msg "ERROR: new ipset length ($ipset_tmp_lines) shorter than the original ($IPSET_DST_LINES)."
			trap -- ERR
			exit 1
		fi

        save_counter "$c_date" "$counter"

		if [ "$delta" == 0 ]; then
			exit 0
		fi

		! diff "$ipset_tmp" "$IPSET_DST" >/dev/null || { echo "No updates.." >&2; exit 0; }

		ipset_old_tmp="$(mktemp "${TMP_DIR}/${TS}-ipset-OLD-XXXXXX")"
		mv "$IPSET_DST" "$ipset_old_tmp"
		cp "$ipset_tmp" "$IPSET_DST"

		# The number of IPs didn't change, only the timestamps, so we're done.
		if [ "$delta" == 0 ]; then
		    exit 0
		fi

		log_msg "Restarting firewall service.." && service firewall restart

        rm -f "$ipset_tmp" "$aging_data_tmp" "$query_data_tmp" "$ipset_old_tmp" &>/dev/null
    else
        # export NLIMIT="$(get_nlimit)"
        NLIMIT=200
        get_responses -v
    fi
else
    echo "Sourcing complete.." >&2
fi

parse_ipsetentry.patch

--- /usr/share/ucode/fw4.uc 02:24:26.459445315 +0100
+++ /usr/share/ucode/fw4.uc 02:10:15.857472398 +0100
@@ -1489,6 +1489,7 @@
        },

        parse_ipsetentry: function(val, set) {
+               val = trim(split(val, "#")[0]);
                let values = split(val, /[ \t]+/);

                if (length(values) != length(set.types))

patch_apply.sh

cd /
patch -p1 < /root/parse_ipsetentry.patch
ACTION=reload-sets utpl -S /usr/share/firewall4/main.uc

upd_adg.sh

#!/bin/sh

LOCK_FILE="/tmp/upd_adg.lock"

[ -f "$LOCK_FILE" ] && echo "Script is already running." >&2 && exit

touch "$LOCK_FILE"
trap 'rm -f "$LOCK_FILE"; exit' INT TERM EXIT

cmd="$(crontab -l | grep adguard_ipset.sh | head -1 | awk '{for(i=6;i<=NF;++i)print $i}' | tr '\n' ' ')"
IPSET_DST=$(echo $cmd | grep -oE 'IPSET_DST=[^ ]+' | cut -d= -f2)

END=$(($(date +%s) + 1200))
today=$(date +%Y-%m-%d)

while [ $(date +%s) -lt $END ]; do
	eval $cmd 2>&1 | grep --color=auto -E "^($today|Added)" | tr '\n' '\t'
	total=$(wc -l "$IPSET_DST" | cut -d' ' -f1)
	echo "Total: $total"
	read -t 10 -n 1 -s
done

rm