Firewall Testing

test.sh

#Test all A entries for a DNS record to see if you can connect to port 443:
dig +short microsoft.com. | xargs -I {} nc -v -w2 {} 443

#Example of output
Connection to 191.239.213.197 443 port [tcp/https] succeeded!
Connection to 104.43.195.251 443 port [tcp/https] succeeded!
Connection to 104.40.211.35 443 port [tcp/https] succeeded!
Connection to 23.100.122.175 port 443 (tcp) timed out: Operation now in progress
Connection to 23.96.52.53 443 port [tcp/https] succeeded!


#Test using a string as input (printf prevents issue with \n from echo. echo -n also works)
printf %b '10.1.0.1 10.2.0.2 10.3.0.3 10.4.0.4' | xargs -d ' ' -I {} nc -v -w2 {} 443
echo -n test-{east,west,south,north}-server.example.com | xargs -d ' ' -I {} nc -v -w2 {} 443

#Example of output
nc: connect to 10.1.0.1 443 port [tcp/https] succeeded!
nc: connect to 10.2.0.2 port 443 (tcp) failed: Connection refused
nc: connect to 10.3.0.3 port 443 (tcp) timed out: Operation now in progress
nc: connect to 10.4.0.4 port 443 (tcp) timed out: Operation now in progress

#Run multiple commands. This brace expands to 10.x.0.1, .5, and .9 for x=1 thru 10
echo -n 10.{1..10}.0.{1,5,9} | xargs -d ' ' -I {} bash -c "echo -e '\n\n\nIP: {}'; curl -vsm 3 http://{}/"

#Can also test for local route details (example: see which local interface would be used for egress)
printf %b '192.168.0.4 192.168.76.36 192.168.4.100 10.120.100.1' | xargs -d ' ' -I {} ip route get {}
local 192.168.0.4 dev lo  src 192.168.0.4 
    cache <local> 
192.168.76.36 via 192.168.0.1 dev enp0s31f6  src 192.168.0.4 
    cache 
192.168.4.100 dev lxdbr0  src 192.168.4.1 
    cache <local> 
10.120.100.1 dev vpn0  src 172.16.2.1 
    cache 
    

#Use mtr for traceroute, send output to stdout so you can copy/paste. See also --tcp or --port options if ICMP is blocked
sudo mtr -n -r -c 5 microsoft.com
Start: Thu May 31 12:12:12 2018
HOST: gbmint02                    Loss%   Snt   Last   Avg  Best  Wrst StDev
  1.|-- 192.168.0.1                0.0%     5    0.4   0.4   0.4   0.4   0.0
  2.|-- 96.120.96.89               0.0%     5    8.1   7.7   7.3   8.1   0.0
  3.|-- 68.87.170.221              0.0%     5    7.6   7.7   7.6   7.9   0.0
  4.|-- 69.139.231.85              0.0%     5    7.8   7.9   7.6   8.3   0.0
  5.|-- 68.86.90.225               0.0%     5   20.0  20.4  19.9  21.1   0.0
  6.|-- 68.86.84.226               0.0%     5   20.6  29.6  20.2  64.6  19.6
  7.|-- 68.86.83.94                0.0%     5   20.0  19.7  19.0  21.3   0.9
  8.|-- 50.242.149.162             0.0%     5   19.0  19.7  19.0  21.1   0.5
  9.|-- 104.44.4.247               0.0%     5  138.1 138.0 136.0 140.1   1.4
 10.|-- 104.44.4.110               0.0%     5  138.6 137.9 135.6 140.1   1.6
 11.|-- 104.44.4.96                0.0%     5  136.2 136.9 136.2 137.8   0.5
 12.|-- 104.44.4.175               0.0%     5  135.9 137.4 135.9 140.8   1.9
 13.|-- 104.44.4.77                0.0%     5  136.7 136.9 136.0 137.6   0.5
 14.|-- 104.44.4.52                0.0%     5  143.8 138.0 135.7 143.8   3.2
 15.|-- 104.44.4.65                0.0%     5  137.7 136.8 135.6 137.7   0.0
 16.|-- 104.44.5.8                 0.0%     5  136.9 136.6 136.1 137.2   0.0
 17.|-- 104.44.7.71                0.0%     5  136.1 136.1 135.8 136.4   0.0
 18.|-- 104.44.7.39                0.0%     5  135.9 136.3 135.9 136.5   0.0
 19.|-- 104.44.11.82               0.0%     5  135.7 135.4 135.2 135.7   0.0
 20.|-- ???                       100.0     5    0.0   0.0   0.0   0.0   0.0

#If you use iptables to restrict outbound connections to specific user accounts you can test 
sudo -u username nmap -Pn 1.1.1.1 -p53
sudo -u username nc -v -w2 1.1.1.1 53

zz-lb-proxy-sockets.sh

#use awk to see connections to local port 9090 and show count of remote ips
sudo netstat -nap | awk '/:9090 / && ! /::1:| LISTEN / {match($5, /(.*):(.*)/, m); print m[1]}' |sort|uniq -c
#similar, but only include lines with local ip:80 or :443
sudo netstat -nap | awk "/$(hostname -i):(80|443) / && ! /::1:| LISTEN / {match(\$5, /(.*):(.*)/, m); print m[1]}" |sort|uniq -c
#Watch changes. Also wrap in $'command \'nested string\' |command' for better escaping of only nested single quotes
watch -d -n2 $'sudo netstat -nap | awk \'/:9090 / && ! /::1:| LISTEN / {match($5, /(.*):(.*)/, m); print m[1]}\' |sort|uniq -c'
#Watch local 80/443, excluding TIME_WAIT (rough estimate of active connections to local webserver). Change m[1] to \$0 to see raw data.
watch -d -n2 "sudo netstat -nap | awk '/$(hostname -i):(80|443) / && ! /::1:| LISTEN | TIME_WAIT/ {match(\$5, /(.*):(.*)/, m); print m[1]}' |sort|uniq -c"

#You can do more with gawk (sorting and counting in one pass)
#The commands below assume you are on a system that listens on 80/443 and then fetches responses 
#from other systems (using various ports)

#View client IP count by dst port. Limits to tcp, exludes listen state, requires local dst port of 80/443, 
#then counts src -> local port and displays any with a count more than 5 along with socket counts
uptime;netstat -na > /dev/shm/netstat; echo Total Established $(grep -P '(?<!127\.0\.0\.1):4444 .*ESTABLISHED' /dev/shm/netstat | wc -l); gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 ~ /:(80|443)$/ {split($4,dst,":");split($5,src,":");groups[(src[1] "-> :" dst[2])]++;states[(src[1] "-> :" dst[2])][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}' /dev/shm/netstat

13:05:09 up 1 day,  5:13,  1 user,  load average: 1.06, 0.66, 0.66
Total Established 106
378     10.100.0.11-> :443    ESTABLISHED=300 TIME_WAIT=78
365     10.100.0.12-> :443    TIME_WAIT=250 ESTABLISHED=115
363     10.100.0.13-> :443    ESTABLISHED=363
358     10.100.0.14-> :443    TIME_WAIT=358
357     10.100.0.15-> :443    ESTABLISHED=200 TIME_WAIT=157

#The above results show 350-380 connections from (in this case) an upstream WAF/LB cluster with 5 nodes 
#establishing pooled connections to our local port 443

# Another for loop to monitor a go server in a container (fixes issue with ipv6 source addresses):
for run in `seq 1 10`; do uptime;netstat -nat > /dev/shm/netstat; echo "Totals: $(curl -s 127.0.0.1:80/metrics|grep '^go_goroutines')       Established $(grep 'ESTABLISHED' /dev/shm/netstat | wc -l)"; gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 ~ /:(80|3000|3001)$/{ndst=split($4,dst,":");nsrc=split($5,src,":");groups[(src[nsrc-1] "-> :" dst[ndst])]++;states[(src[nsrc-1] "-> :" dst[ndst])][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}' /dev/shm/netstat; sleep 600;done;

#View fetches in flight (or recently finished). Limits to tcp, excludes listen state and local port 80/443 
#(rough filter for sockets where we are client not server), and again displays counts greater than 5
netstat -na | gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 !~ /:(80|443)$/ {groups[$5]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {print groups[g],g}}'

Count of fetches in flight
43      10.0.0.101:80
11      10.0.1.101:80
9       10.0.2.101:80
7       10.0.3.101:80
#Same as above but in format for watch or remote salt cmd.run
watch -d -n5 $'netstat -na | gawk \'BEGIN {OFS="\\t"} $1 ~ /tcp/ && ! ($6 ~ /LISTEN/) && ! ($4 ~ /:(80|443)$/) {groups[$5]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {print groups[g],g}}\''

#This shows the count of connections we have to port 80 on 4 different backend web servers

#Same as above but also break down connection state details
netstat -na | gawk 'BEGIN {OFS="\t"} $1 ~ /tcp/ && $6 !~ /LISTEN/ && $4 !~ /:(80|443)$/ {groups[$5]++;states[$5][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}'
Count of fetches in flight
43      10.0.0.101:80  TIME_WAIT=27 
9       10.0.1.101:80   ESTABLISHED=6 TIME_WAIT=3 
8       10.0.2.101:80  ESTABLISHED=6 TIME_WAIT=2 
7       10.0.3.101:80   ESTABLISHED=7 
#Same as above but in format for watch or remote salt cmd.run
watch -d -n5 $'netstat -na | gawk \'BEGIN {OFS="\\t"} $1 ~ /tcp/ && ! ($6 ~ /LISTEN/) && ! ($4 ~ /:(80|443)$/) {groups[$5]++;states[$5][$6]++}END{PROCINFO["sorted_in"] = "@val_num_desc";print "Count of fetches in flight";for (g in groups) if (groups[g]>5) {statelist=" ";for (s in states[g]) statelist=statelist s "=" states[g][s] " ";print groups[g],g,statelist}}\''

#Here TIME_WAIT represents closed connections waiting for any final packets 
#(hard coded at 60s... this is NOT tcp_fin_timeout, see https://stackoverflow.com/a/35000966/17373 )
#You can even see those countdown timers using -o on netstat or ss

# See countdown timer for all TIME_WAIT sockets in 10.0.0.0-10.255.255.255
ss --numeric -o state time-wait dst 10.0.0.0/8

NetidRecv-Q  Send-Q    Local Address:Port    Peer Address:Port                             
tcp  0       0         192.168.100.1:57516   10.0.0.101:80    timer:(timewait,55sec,0)   
tcp  0       0         192.168.100.1:57356   10.0.0.101:80    timer:(timewait,25sec,0)   
tcp  0       0         192.168.100.1:57334   10.0.0.101:80    timer:(timewait,6.536ms,0)  
tcp  0       0         192.168.100.1:57282   10.0.1.101:80    timer:(timewait,12sec,0)   
tcp  0       0         192.168.100.1:57418   10.0.2.101:80    timer:(timewait,38sec,0)

#TIME_WAIT does NOT mean there is any issue with the client or server. Especially if tcp_tw_reuse is enable,
#these sockets will get reused for new connections if there won't be any conflict in TCP Segment numbers.

#The above server with more TIME_WAIT may just be an API server that has a bunch of small request/responces,
#and those sockets don't get reused as much.

Interesting. I suppose this is a type of code or instructions for backend dev right?.