Node.js Load Balancers Benchmark: HAProxy vs Nginx vs PM2

app-koa.js

const Koa = require('koa');
const app = new Koa();

app.use(ctx => {
    ctx.body = 'Hello World';
});

var listener = app.listen(process.env.PORT || 3000, () => {
    console.log(`Listening on port ${listener.address().port}`);
});

cat proc - cpuinfo

ubuntu@benchmark-machine:~/benchmark-v1/haproxy$ cat /proc/cpuinfo 
processor	: 0
vendor_id	: GenuineIntel
cpu family	: 6
model		: 86
model name	: Intel(R) Xeon(R) CPU D-1531 @ 2.20GHz
stepping	: 3
microcode	: 0x700000a
cpu MHz		: 801.195
cache size	: 9216 KB
physical id	: 0
siblings	: 12
core id		: 0
cpu cores	: 6
apicid		: 0
initial apicid	: 0
fpu		: yes
fpu_exception	: yes
cpuid level	: 20
wp		: yes
flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf eagerfpu pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch epb intel_pt tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm rdseed adx smap xsaveopt cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts
bugs		:
bogomips	: 4399.54
clflush size	: 64
cache_alignment	: 64
address sizes	: 46 bits physical, 48 bits virtual
power management:

etc - security - limits.conf

* soft   nofile 999999
* hard   nofile 999999

haproxy.cfg

defaults
    timeout connect 5000
    timeout client  50000
    timeout server  50000

frontend tcp-frontend
  mode tcp
  bind *:6001
  default_backend tcp-backend

backend tcp-backend
  mode tcp
  balance roundrobin
  server http-instance-0 localhost:8001
  server http-instance-1 localhost:8002
  server http-instance-2 localhost:8003
  server http-instance-3 localhost:8004

frontend http-frontend
  mode http
  bind *:6002
  default_backend http-backend

backend http-backend
  mode http
  balance roundrobin
  server http-instance-0 localhost:8001
  server http-instance-1 localhost:8002
  server http-instance-2 localhost:8003
  server http-instance-3 localhost:8004

haproxy.sh

sudo apt-get install -qq build-essential libssl-dev libev-dev libpcre3-dev
wget https://www.haproxy.org/download/1.7/src/haproxy-1.7.5.tar.gz
tar xzvf haproxy-1.7.5.tar.gz
cd haproxy-1.7.5
sudo make TARGET=generic USE_OPENSSL=1 USE_ZLIB=yes CPU=native USE_PCRE=1

nginx.conf

daemon off;
worker_processes  12;

events {
  worker_connections  4096;
}

http {
   server {
      server_name _;
      listen     7002;
      location / {
         proxy_redirect     off;
         proxy_pass http://apistream;
      }
   }

   upstream apistream {
      server localhost:8001;
      server localhost:8002;
      server localhost:8003;
      server localhost:8004;
    }
}

stream {
   server {
         listen     7001;
      proxy_pass apistream;
   }

   upstream apistream {
      server localhost:8001;
      server localhost:8002;
      server localhost:8003;
      server localhost:8004;
    }
}

nginx.sh

apt-get update
apt-get install libssl-dev gcc -y
wget http://nginx.org/download/nginx-1.11.13.tar.gz
tar zxvf nginx-1.11.13.tar.gz
./configure --without-http_rewrite_module --with-http_ssl_module --with-stream --with-http_stub_status_module --prefix=. --error-log-path=error.log --http-log-path=access.log --pid-path=nginx.pid --lock-path=nginx.lock --conf-path=nginx.conf
make -j12

pstree

systemd─┬─PM2 v2.4.6: God─┬─haproxy
        │                 ├─nginx───12*[nginx]
        │                 ├─5*[node /home/ubun─┬─4*[{V8 WorkerThread}]]
        │                 │                    └─5*[{node}]]
        │                 ├─4*[node /home/ubun─┬─4*[{V8 WorkerThread}]]
        │                 │                    ├─4*[{node /home/ubun}]]
        │                 │                    └─{node}]
        │                 ├─4*[{PM2 v2.4.6: God}]
        │                 ├─4*[{V8 WorkerThread}]
        │                 └─{node}
        ├─accounts-daemon─┬─{gdbus}
        │                 └─{gmain}
        ├─agetty
        ├─cron
        ├─dbus-daemon
        ├─dhclient
        ├─irqbalance
        ├─mdadm
        ├─named───14*[{named}]
        ├─ntpd───ntpd───ntpd
        ├─polkitd─┬─{gdbus}
        │         └─{gmain}
        ├─rsyslogd─┬─{in:imklog}
        │          ├─{in:imuxsock}
        │          └─{rs:main Q:Reg}
        ├─sshd─┬─sshd───sshd───bash───pstree
        │      └─sshd───sshd
        ├─systemd───(sd-pam)
        ├─systemd-journal
        ├─systemd-logind
        └─systemd-udevd

report.lua

f = io.open("result.csv", "a+"
)
done = function(summary, latency, requests)

  f:write(string.format("%s:%s,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d\n",
  wrk.host,
  wrk.port,
  latency.min,
  latency.max,
  latency.mean,
  latency.stdev,
  latency:percentile(99),
  summary["duration"],
  summary["requests"],
  summary["bytes"],
  summary["errors"].connect,
  summary["errors"].read,
  summary["errors"].write,
  summary["errors"].status,
  summary["errors"].timeout
  ))

  f:close()
end

sysctl.conf

# Increase number of incoming connections that can queue up                                                                                                                                                         
# before dropping                                                                                                                                                                                                   
net.core.somaxconn = 50000
# Increase the length of the network device input queue                                                                                                                                                             
net.core.netdev_max_backlog = 5000
# Widen the port range used for outgoing connections                                                                                                                                                                
net.ipv4.ip_local_port_range = 10000 65000

# Disable source routing and redirects                                                                                                                                                                              
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.all.accept_source_route = 0

# Disable TCP slow start on idle connections                                                                                                                                                                        
net.ipv4.tcp_slow_start_after_idle = 0

# Disconnect dead TCP connections after 1 minute                                                                                                                                                                    
net.ipv4.tcp_keepalive_time = 60

# Let the networking stack reuse TIME_WAIT connections when it thinks it's safe to do so                                                                                                                            
net.ipv4.tcp_tw_reuse = 1