Add a remote syslog server to Firewalla

addremotesyslog.sh

#!/bin/bash
# v 2.1.0
  
syslog=/etc/rsyslog.d/09-externalserver.conf
# this logs notice and above. use *.* log everything. 
filter=*.notice
server=192.168.0.19 # Change the server to the IP of your syslog server.
port=514
hostname=firewalla
valid=$(grep "$server:$port" $syslog 2>/dev/null)

create () {
	# To use TCP uncomment line 13 to use TCP and comment line 15
	# echo -e "# remote syslog server (TCP):\n$filter @@$server:$port" | sudo tee $syslog
  # Line 15 assumes UDP: to use TCP, comment the line 15 and uncomment line 13
	echo -e "# remote syslog server (UDP):\n\$LocalHostName $hostname\nfilter @$server:$port" | sudo tee $syslog
        echo "Restarting rsyslog..."
        sudo systemctl restart rsyslog
        echo "remote syslog added"
	exit
}

cleanup () {
	sudo rm -f $syslog
	sudo systemctl restart rsyslog
}

if [ -f "$syslog" ] ; then

	if [ -n "$valid" ] ; then

	echo "remote syslog already in place with $server:$port specified"

	case $1 in
		-c)
		echo -e "\nrecreating syslog configuration..." 
		cleanup
		create
		;;

  		-r|-restart|-force|-f)
                echo "Restarting rsyslog..."
                sudo systemctl restart rsyslog
		exit
		;;
		
		-u|-update)
			read -p "Are you sure you want to remove the syslog forwarder? type 'y' " -n 1 -r
			echo
	
			if [[ $REPLY =~ ^[Yy]$ ]] ; then
				ls $syslog 2>/dev/null && cleanup || echo -e "\n\nNo log found.\n"
			fi
			exit
		;;

		-h)
		echo -e "You can use:\n     - \`$0 -c\` recreate forwarding\n     - \`$0 -r\` restart the syslog service\
		\n     - \`$0 -u\` uninstall the settings to send to the remote syslog server\n\n"
    		exit
		;;
	esac

	else
		echo "The server is not configured correctly. On it." 
		cleanup
		create
	fi
else
	echo "There was no syslog forwarder in place."
	create
fi

@mbierman and @tsqrd

Here is my updated solution. I've tested all day, and so far so good. @mbierman I reused what you had already done, and built on that 👍

Script

#!/bin/bash
# v 2.1.0

script_location="/home/pi/.firewalla/config/post_main.d/" # script location
script="firewalla_rsyslog.sh" # script used to install firewalla syslog
cron_cmd="0 * * * * cd $script_location && sudo ./$script -c"
syslog="/etc/rsyslog.d/09-externalserver.conf" # rsyslog location
server="172.16.2.20" # Change the server to the IP of your syslog server.
port="514" # port used for forwarding logs to destination
protocol="tcp" #use tcp or udp
other_protocol="@@" # use @@ for TCP and @ for UDP
valid=$(grep "$server:$port" $syslog 2>/dev/null)



### Creating the syslog file

create() {
	sudo touch $syslog
	sudo cat > $syslog <<EOF
	
\$LocalHostName Firewalla
# deifne global workDirectory for saving the state file of log messages.
global(workDirectory="/var/spool/rsyslog")

# enable the Rsyslog imfile module processing text files or logs.
module(load="imfile" PollingInterval="10")


# define template for StandardSyslogFormat for processing log messages.
# that will be forwarded to rsyslog server
template(
    name="StandardSyslogFormat"
    type="string"
    string="<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%"
    )


# define ruleset "forwardSysLogs" with action object to send logs to rsyslog server
# define the queue
ruleset(name="forwardSysLogs") {
    action(
        type="omfwd"
        target="$server"  # set your Synology Syslog Server NAS IP
        port="$port"  # Specify port number
        protocol="$protocol"  # specify protocol UDP or TCP
        template="StandardSyslogFormat"  # specifies the template to use above

        queue.SpoolDirectory="/var/spool/rsyslog"
        queue.FileName="remote"
        queue.MaxDiskSpace="1g"
        queue.SaveOnShutdown="on"
        queue.Type="LinkedList"
        ResendLastMSGOnReconnect="on"
        )
        stop
}

# define input files forwardSysLogs logs to send to the rsyslog server
# and apply ruleset "forwardSysLogs" 
# in /bspool/manager

input(type="imfile" ruleset="forwardSysLogs" Tag="ConnLog" File="/bspool/manager/conn.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="ConnLongLog" File="/bspool/manager/conn_long.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="DNS" File="/bspool/manager/dns.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="Files" File="/bspool/manager/files.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="HeartBeat" File="/bspool/manager/heartbeat.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="NTP" File="/bspool/manager/ntp.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="OSCP" File="/bspool/manager/oscp.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="SSL" File="/bspool/manager/ssl.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="StdErr" File="/bspool/manager/stderr.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="StdOut" File="/bspool/manager/stdout.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="HTTP" File="/bspool/manager/http.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="Notice" File="/bspool/manager/notice.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="Weird" File="/bspool/manager/weird.log")

# define input files forwardSysLogs logs to send to the rsyslog server
# and apply ruleset "forwardSysLogs" 
# in /alog

input(type="imfile" ruleset="forwardSysLogs" Tag="ACL-Alarm" File="/alog/acl-alarm.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="ACL-Audit" File="/alog/acl-audit.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="DNS-Masq" File="/alog/dnsmasq-acl.log")

# define input files forwardSysLogs logs to send to the rsyslog server
# and apply ruleset "forwardSysLogs" 
# in /alog/firewalla

input(type="imfile" ruleset="forwardSysLogs" Tag="FireApi" File="/alog/firewalla/FireApi.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="FireKick" File="/alog/firewalla/FireKick.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="FireMain" File="/alog/firewalla/FireMain.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="FireMon" File="/alog/firewalla/FireMon.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="FireRouter" File="/alog/firewalla/FireRouter.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="Trace" File="/alog/firewalla/Trace.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="CleanLog" File="/alog/firewalla/clean_log.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="Firelog" File="/alog/firewalla/firelog.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="Node" File="/alog/firewalla/node.log")
input(type="imfile" ruleset="forwardSysLogs" Tag="SyncTime" File="/alog/firewalla/sync_time.log")

# Sending all other Syslog logs to Server (Synology) 
# @@IP is for TCP
# @IP is for UDP
*.* $other_protocol$server:$port
EOF

    echo "Restarting rsyslog..."
    sudo systemctl restart rsyslog
    echo "remote syslog added"
    echo "adding cron job for reliability"
    (crontab -u pi -l 2>/dev/null; echo "$cron_cmd") | crontab -u pi -
    sudo systemctl restart cron
    exit
}

cleanup() {
	sudo rm -f $syslog
	sudo systemctl restart rsyslog
	(crontab -u pi -l | grep -vF "$cron_cmd" | crontab -u pi -)	
}

if [ -f "$syslog" ] ; then
	if [ -n "$valid" ] ; then
	echo "remote syslog already in place with $server:$port specified"
	case $1 in
		-c)
		echo -e "\nrecreating syslog configuration..." 
		cleanup
		create
		;;
  		-r|-restart|-force|-f)
                echo "Restarting rsyslog..."
                sudo systemctl restart rsyslog
		exit
		;;
		-u|-update)
			read -p "Are you sure you want to remove the syslog forwarder? type 'y' " -n 1 -r
			echo
			if [[ $REPLY =~ ^[Yy]$ ]] ; then
				ls $syslog 2>/dev/null && cleanup || echo -e "\n\nNo log found.\n"
			fi
			exit
		;;
		-h)
		echo -e "You can use:\n     - \`$0 -c\` recreate forwarding\n     - \`$0 -r\` restart the syslog service\
		\n     - \`$0 -u\` uninstall the settings to send to the remote syslog server\n\n"
    		exit
		;;
	esac
	else
		echo "The server is not configured correctly. On it." 
		cleanup
		create
	fi
else
	echo "There was no syslog forwarder in place."
	create
fi

Fixes

• Hostname is now set to Firewalla
• Cron is now used to ensure persistent sending of all FW logs
• Implementation is now completely automated via script

Setting up the Directory

To send logs to a remote syslog server using UDP, do the following:

1. ssh to the Firewalla box.
2. Copy the script above.
3. If /home/pi/.firewalla/config/post_main.d/ doesn’t exist, create it first.
   sudo mkdir /home/pi/.firewalla/config/post_main.d/
4. Next, create the file:
   sudo vi /home/pi/.firewalla/config/post_main.d/firewalla_rsyslog.sh

Modifying the Variables in the Script and Executing

1. Edit the following variables in the script for your specific environment: 
   1. server to the IP address of your syslog server.
   2. port to the correct port being used for rsyslog
   3. protocol to specify TCP or UDP
   4. other_protocol uses a single @ for UDP and a double @@ for TCP
2. Paste this script into firewalla_rsyslog.sh. This is going to create rsyslog configs and the cron job that runs to ensure that the syslog setting remains in place, even if there's a firewalla update that wipes out the settings in the future.
3. Save the file :wq!
4. Give the script execute permissions.
   sudo chmod +x /home/pi/.firewalla/config/post_main.d/firewalla_rsyslog.sh
5. Execute the script.
   sudo /home/pi/.firewalla/config/post_main.d/firewalla_rsyslog.sh -c creates the file and restarts syslog

Additional Arguments

• /home/pi/.firewalla/config/post_main.d/firewalla_rsyslog.sh -r restarts syslog
• /home/pi/.firewalla/config/post_main.d/firewalla_rsyslog.sh -u uninstalls the forwarder and restarts syslog.

@mjaestewart Do you end up with an endless supply of imfile state files in /var/spool/rsyslog? I ended up adding a cronjob to delete files older than 5 minutes in that directory because otherwise it just fills up indefinitely. I'm assuming it has something to do with zeek truncating/rotating the log files because I also end up with these messages from rsyslogd in /var/log/syslog: imfile: internal error? inotify provided watch descriptor 3745 which we could not find in our tables.

You may also want to consider this post about persisting cron through reboots/restarts: https://help.firewalla.com/hc/en-us/articles/360054056754-Customized-Scripting

I notice the cronjob disappeared after a reload so I added it to the location described by that article.