Suresh Kumar Ponnusamy surki

# Start gdb-server in port 50505 using above instructions
sudo chroot ./amzlinux_rootfs/ /bin/bash -c "rr replay -s 50505 -k /rr_trace/ruby-0/ -- -ex 'handle SIGPIPE nostop noprint pass' -ex 'source /gdb_ruby.py'"
 
# In Emacs, start gdb client and connect to it
(gdb) set sysroot /home/surki/tmp/some_app_crash/rr/amzlinux_rootfs
(gdb) target remote :50505
(gdb) source ~/tmp/some_app_crash/rr/amzlinux_rootfs/gdb_ruby.py
(gdb) continue

# Run this from the directory where amzlinux_rootfs is present
 
# Once: mount the appropriate procfs
for f in /proc /sys /dev; do sudo mount --bind $f ./amzlinux_rootfs/$f; done
 
# Enter the rootfs and replay
sudo chroot ./amzlinux_rootfs/ /bin/bash -c "rr replay /rr_trace/ruby-0/ -- -ex 'handle SIGPIPE nostop noprint pass' -ex 'source /gdb_ruby.py'"
 
# now we should be in gdb prompt, use gdb commands to run/stop/inspect/watch/reverse-cont etc

exec /usr/local/bin/rr -F record -n \
 --disable-cpuid-features 0x80050440,0x40140400 --disable-cpuid-features-ext 0xffffd854,0xffffffff,0xf3ffffff \
 --disable-cpuid-features-xsave 0xfffffffe \
 "/usr/local/bin/ruby" "$@"

[root@ip-172-16-20-17 ec2-user]# perf stat -e r5101c4 true
 
Performance counter stats for 'true':
 
           58,936      r5101c4
 
      0.000669296 seconds time elapsed

[root@ip-172-16-20-17 ec2-user]# curl http://instance-data/latest/meta-data/instance-type
i3.16xlarge
 
[root@ip-172-16-20-17 ec2-user]# cat /proc/cpuinfo | grep "model name"
model name      : Intel(R) Xeon(R) CPU E5-2686 v4 @ 2.30GHz
 
[root@ip-172-16-20-17 ec2-user]# dmesg | grep -i perf
[   22.307535] Performance Events: Broadwell events, core PMU driver.

Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread 17327.17570]
0x000055a2009a52b2 in rb_get_alloc_func (klass=94154489136320) at vm_method.c:554
554             rb_alloc_func_t allocator = RCLASS_EXT(klass)->allocator;
 
 
(rr) bt
#0  0x000055a2009a52b2 in rb_get_alloc_func (klass=94154489136320) at vm_method.c:554
#1  0x000055a2008cb58f in rb_obj_alloc (klass=94154328666000) at object.c:1815

Setup

Install rr

# Install rr dependencies
sudo yum install git python2 ccache cmake make gcc gcc-c++ gdb libgcc glibc-devel libstdc++-devel zlib-devel python27-pexpect man-pages capnproto

# Compile/install capnproto if no package available
curl -O https://capnproto.org/capnproto-c++-0.6.1.tar.gz
tar zxf capnproto-c++-0.6.1.tar.gz
pushd capnproto-c++-0.6.1

deployments/
├── base
....
....
│   ├── logging ===============> THIS IS BASE logging SETUP
│   │   ├── filebeat
│   │   │   ├── filebeat-config.yml
│   │   │   ├── filebeat-daemonset.yaml
│   │   │   └── kustomization.yaml

A realworld example showing how mixing ephemeral port range with server listen address can lead to port conflicts (due to unintuitive TCP Simulataneous Open (TSO) on same host, see RFC793)

In this example, we will run a "webserver" and a "health check client" in the same machine

Run this on Terminal1 (this is "webserver"):

$ while echo -e "HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 6\r\n\r\nhello" | \
    socat -t 0.1 TCP-L:33000,reuseaddr STDIN; do echo "Served request"; done

	node:
	id: foo
	cluster: foo_envoy_test

	admin:
	access_log_path: "/dev/stdout"
	address:
	socket_address:
	protocol: TCP
	address: 0.0.0.0