Suresh Kumar Ponnusamy surki

A realworld example showing how mixing ephemeral port range with server listen address can lead to port conflicts (due to unintuitive TCP Simulataneous Open (TSO) on same host, see RFC793)

In this example, we will run a "webserver" and a "health check client" in the same machine

Run this on Terminal1 (this is "webserver"):

$ while echo -e "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 6\r\n\r\nhello" | \
    socat -t 0.1 TCP-L:33000,reuseaddr STDIN; do echo "Served request"; done

deployments/
├── base
....
....
│   ├── logging ===============> THIS IS BASE logging SETUP
│   │   ├── filebeat
│   │   │   ├── filebeat-config.yml
│   │   │   ├── filebeat-daemonset.yaml
│   │   │   └── kustomization.yaml

Setup

Install rr

# Install rr dependencies
sudo yum install git python2 ccache cmake make gcc gcc-c++ gdb libgcc glibc-devel libstdc++-devel zlib-devel python27-pexpect man-pages capnproto

# Compile/install capnproto if no package available
curl -O https://capnproto.org/capnproto-c++-0.6.1.tar.gz
tar zxf capnproto-c++-0.6.1.tar.gz
pushd capnproto-c++-0.6.1

Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread 17327.17570]
0x000055a2009a52b2 in rb_get_alloc_func (klass=94154489136320) at vm_method.c:554
554             rb_alloc_func_t allocator = RCLASS_EXT(klass)->allocator;
 
 
(rr) bt
#0  0x000055a2009a52b2 in rb_get_alloc_func (klass=94154489136320) at vm_method.c:554
#1  0x000055a2008cb58f in rb_obj_alloc (klass=94154328666000) at object.c:1815

[root@ip-172-16-20-17 ec2-user]# curl http://instance-data/latest/meta-data/instance-type
i3.16xlarge
 
[root@ip-172-16-20-17 ec2-user]# cat /proc/cpuinfo | grep "model name"
model name      : Intel(R) Xeon(R) CPU E5-2686 v4 @ 2.30GHz
 
[root@ip-172-16-20-17 ec2-user]# dmesg | grep -i perf
[   22.307535] Performance Events: Broadwell events, core PMU driver.

[root@ip-172-16-20-17 ec2-user]# perf stat -e r5101c4 true
 
Performance counter stats for 'true':
 
           58,936      r5101c4
 
      0.000669296 seconds time elapsed

exec /usr/local/bin/rr -F record -n \
 --disable-cpuid-features 0x80050440,0x40140400 --disable-cpuid-features-ext 0xffffd854,0xffffffff,0xf3ffffff \
 --disable-cpuid-features-xsave 0xfffffffe \
 "/usr/local/bin/ruby" "$@"

# Run this from the directory where amzlinux_rootfs is present
 
# Once: mount the appropriate procfs
for f in /proc /sys /dev; do sudo mount --bind $f ./amzlinux_rootfs/$f; done
 
# Enter the rootfs and replay
sudo chroot ./amzlinux_rootfs/ /bin/bash -c "rr replay /rr_trace/ruby-0/ -- -ex 'handle SIGPIPE nostop noprint pass' -ex 'source /gdb_ruby.py'"
 
# now we should be in gdb prompt, use gdb commands to run/stop/inspect/watch/reverse-cont etc

# Start gdb-server in port 50505 using above instructions
sudo chroot ./amzlinux_rootfs/ /bin/bash -c "rr replay -s 50505 -k /rr_trace/ruby-0/ -- -ex 'handle SIGPIPE nostop noprint pass' -ex 'source /gdb_ruby.py'"
 
# In Emacs, start gdb client and connect to it
(gdb) set sysroot /home/surki/tmp/some_app_crash/rr/amzlinux_rootfs
(gdb) target remote :50505
(gdb) source ~/tmp/some_app_crash/rr/amzlinux_rootfs/gdb_ruby.py
(gdb) continue

	You can get even more detailed information about a socket from kernel's internal socket struct.
	For example, we will try to get nginx listening (on port 81) socket's backlog length

	NOTE: You may need to install kernel debug info if not already installed
	# yum-config-manager --enable "amzn-main-debuginfo" --enable "amzn-updates-debuginfo"
	# yum -y install kernel-debuginfo kernel-devel

	Or get socket info for listening socket on port 81
	# ss -len \| grep :81
	tcp LISTEN 0 511 :81 :* ino:29842919 sk:55 <->