benchmark-apisix with envoy

benchmark

对比 Apache APISIX 和 envoy 的一些性能参数：网关延迟、QPS 和是否线性扩展
系统环境：微软云 Linux (ubuntu 18.04)， Standard D13 v2 (8 vcpus, 56 GiB memory)
压力测试客户端：wrk

1 worker: apisix git:(master) ✗ bash -x ./benchmark/run.sh

• '[' -n '' ']'
• worker_cnt=1
• mkdir -p benchmark/server/logs
• mkdir -p benchmark/fake-apisix/logs
• trap onCtrlC INT
• openresty -p /home/rain/apisix/benchmark/server
• make init
  ./bin/apisix init
  ./bin/apisix init_etcd
  ++ uname
• [[ Linux == \D\a\r\w\i\n ]]
• sed -i 's/worker_processes .*/worker_processes 1;/g' conf/nginx.conf
• make run
  mkdir -p logs
  mkdir -p /tmp/apisix_cores/
  /usr/bin/openresty -p $PWD/ -c $PWD/conf/nginx.conf
• sleep 3
• echo -e '\n\napisix: 1 worker + 1 upstream + no plugin'

apisix: 1 worker + 1 upstream + no plugin

• curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
  {
  "uri": "/hello",
  "plugins": {
  },
  "upstream": {
  "type": "roundrobin",
  "nodes": {
  "127.0.0.1:1980": 1
  }
  }
  }'
  {"node":{"value":{"priority":0,"plugins":{},"upstream":{"nodes":{"127.0.0.1:1980":1},"hash_on":"vars","type":"roundrobin"},"id":"1","uri":"/hello"},"createdIndex":27,"key":"/apisix/routes/1","modifiedIndex":27},"prevNode":{"value":"{"priority":0,"plugins":{"limit-count":{"time_window":60,"count":2000000000000,"rejected_code":503,"key":"remote_addr","policy":"local"},"prometheus":{}},"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:1980":1},"type":"roundrobin"},"id":"1","uri":"\/hello"}","createdIndex":26,"key":"/apisix/routes/1","modifiedIndex":26},"action":"set"}
• sleep 1
• run_wrk
  ++ expr 1 '*' 18
• connections=18
• wrk -d 5 -t 1 -c 18 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  1 threads and 18 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 0.96ms 142.90us 4.15ms 84.74%
  Req/Sec 18.76k 624.45 20.08k 70.59%
  95164 requests in 5.10s, 380.08MB read
  Requests/sec: 18659.99
  Transfer/sec: 74.53MB
• sleep 1
• run_wrk
  ++ expr 1 '*' 18
• connections=18
• wrk -d 5 -t 1 -c 18 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  1 threads and 18 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 0.97ms 169.04us 5.08ms 88.50%
  Req/Sec 18.66k 664.09 19.95k 68.00%
  92794 requests in 5.00s, 370.61MB read
  Requests/sec: 18556.76
  Transfer/sec: 74.11MB
• sleep 1
• echo -e '\n\napisix: 1 worker + 1 upstream + 2 plugins (limit-count + prometheus)'

apisix: 1 worker + 1 upstream + 2 plugins (limit-count + prometheus)

• curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
  {
  "uri": "/hello",
  "plugins": {
  "limit-count": {
  "count": 2000000000000,
  "time_window": 60,
  "rejected_code": 503,
  "key": "remote_addr"
  },
  "prometheus": {}
  },
  "upstream": {
  "type": "roundrobin",
  "nodes": {
  "127.0.0.1:1980": 1
  }
  }
  }'
  {"node":{"value":{"priority":0,"plugins":{"limit-count":{"time_window":60,"count":2000000000000,"rejected_code":503,"key":"remote_addr","policy":"local"},"prometheus":{}},"upstream":{"nodes":{"127.0.0.1:1980":1},"hash_on":"vars","type":"roundrobin"},"id":"1","uri":"/hello"},"createdIndex":28,"key":"/apisix/routes/1","modifiedIndex":28},"prevNode":{"value":"{"priority":0,"plugins":{},"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:1980":1},"type":"roundrobin"},"id":"1","uri":"\/hello"}","createdIndex":27,"key":"/apisix/routes/1","modifiedIndex":27},"action":"set"}
• sleep 3
• run_wrk
  ++ expr 1 '*' 18
• connections=18
• wrk -d 5 -t 1 -c 18 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  1 threads and 18 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.14ms 220.44us 6.16ms 91.11%
  Req/Sec 15.86k 614.68 17.16k 74.00%
  78870 requests in 5.00s, 320.42MB read
  Requests/sec: 15773.19
  Transfer/sec: 64.08MB
• sleep 1
• run_wrk
  ++ expr 1 '*' 18
• connections=18
• wrk -d 5 -t 1 -c 18 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  1 threads and 18 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.13ms 169.34us 6.05ms 86.68%
  Req/Sec 15.93k 572.55 17.01k 60.00%
  79257 requests in 5.00s, 321.99MB read
  Requests/sec: 15849.75
  Transfer/sec: 64.39MB
• sleep 1
• make stop
  /usr/bin/openresty -p $PWD/ -c $PWD/conf/nginx.conf -s stop
• echo -e '\n\nfake empty apisix server: 1 worker'

fake empty apisix server: 1 worker

• sleep 1
• sed -i 's/worker_processes [0-9]*/worker_processes 1/g' benchmark/fake-apisix/conf/nginx.conf
• openresty -p /home/rain/apisix/benchmark/fake-apisix
• sleep 1
• run_wrk
  ++ expr 1 '*' 18
• connections=18
• wrk -d 5 -t 1 -c 18 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  1 threads and 18 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 0.88ms 147.20us 7.66ms 92.86%
  Req/Sec 20.53k 420.18 21.39k 64.00%
  102136 requests in 5.00s, 406.46MB read
  Requests/sec: 20425.68
  Transfer/sec: 81.29MB
• sleep 1
• run_wrk
  ++ expr 1 '*' 18
• connections=18
• wrk -d 5 -t 1 -c 18 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  1 threads and 18 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 0.86ms 226.03us 7.28ms 95.12%
  Req/Sec 20.85k 604.74 22.17k 68.00%
  103810 requests in 5.00s, 413.13MB read
  Requests/sec: 20760.66
  Transfer/sec: 82.62MB
• openresty -p /home/rain/apisix/benchmark/fake-apisix -s stop
• openresty -p /home/rain/apisix/benchmark/server -s stop
  2 worker: apisix git:(master) ✗ bash -x ./benchmark/run.sh 2
• '[' -n 2 ']'
• worker_cnt=2
• mkdir -p benchmark/server/logs
• mkdir -p benchmark/fake-apisix/logs
• trap onCtrlC INT
• openresty -p /home/rain/apisix/benchmark/server
• make init
  ./bin/apisix init
  ./bin/apisix init_etcd
  ++ uname
• [[ Linux == \D\a\r\w\i\n ]]
• sed -i 's/worker_processes .*/worker_processes 2;/g' conf/nginx.conf
• make run
  mkdir -p logs
  mkdir -p /tmp/apisix_cores/
  /usr/bin/openresty -p $PWD/ -c $PWD/conf/nginx.conf
• sleep 3
• echo -e '\n\napisix: 2 worker + 1 upstream + no plugin'

apisix: 2 worker + 1 upstream + no plugin

• curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
  {
  "uri": "/hello",
  "plugins": {
  },
  "upstream": {
  "type": "roundrobin",
  "nodes": {
  "127.0.0.1:1980": 1
  }
  }
  }'
  {"node":{"value":{"priority":0,"plugins":{},"upstream":{"nodes":{"127.0.0.1:1980":1},"hash_on":"vars","type":"roundrobin"},"id":"1","uri":"/hello"},"createdIndex":29,"key":"/apisix/routes/1","modifiedIndex":29},"prevNode":{"value":"{"priority":0,"plugins":{"limit-count":{"time_window":60,"count":2000000000000,"rejected_code":503,"key":"remote_addr","policy":"local"},"prometheus":{}},"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:1980":1},"type":"roundrobin"},"id":"1","uri":"\/hello"}","createdIndex":28,"key":"/apisix/routes/1","modifiedIndex":28},"action":"set"}
• sleep 1
• run_wrk
  ++ expr 2 '*' 18
• connections=36
• wrk -d 5 -t 2 -c 36 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  2 threads and 36 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.01ms 496.77us 10.56ms 72.12%
  Req/Sec 17.92k 1.75k 22.27k 78.00%
  178194 requests in 5.00s, 711.70MB read
  Requests/sec: 35634.71
  Transfer/sec: 142.32MB
• sleep 1
• run_wrk
  ++ expr 2 '*' 18
• connections=36
• wrk -d 5 -t 2 -c 36 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  2 threads and 36 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.03ms 660.18us 8.17ms 58.82%
  Req/Sec 17.41k 2.93k 33.44k 77.23%
  175029 requests in 5.10s, 699.06MB read
  Requests/sec: 34316.94
  Transfer/sec: 137.06MB
• sleep 1
• echo -e '\n\napisix: 2 worker + 1 upstream + 2 plugins (limit-count + prometheus)'

apisix: 2 worker + 1 upstream + 2 plugins (limit-count + prometheus)

• curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
  {
  "uri": "/hello",
  "plugins": {
  "limit-count": {
  "count": 2000000000000,
  "time_window": 60,
  "rejected_code": 503,
  "key": "remote_addr"
  },
  "prometheus": {}
  },
  "upstream": {
  "type": "roundrobin",
  "nodes": {
  "127.0.0.1:1980": 1
  }
  }
  }'
  {"node":{"value":{"priority":0,"plugins":{"limit-count":{"time_window":60,"count":2000000000000,"rejected_code":503,"key":"remote_addr","policy":"local"},"prometheus":{}},"upstream":{"nodes":{"127.0.0.1:1980":1},"hash_on":"vars","type":"roundrobin"},"id":"1","uri":"/hello"},"createdIndex":30,"key":"/apisix/routes/1","modifiedIndex":30},"prevNode":{"value":"{"priority":0,"plugins":{},"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:1980":1},"type":"roundrobin"},"id":"1","uri":"\/hello"}","createdIndex":29,"key":"/apisix/routes/1","modifiedIndex":29},"action":"set"}
• sleep 3
• run_wrk
  ++ expr 2 '*' 18
• connections=36
• wrk -d 5 -t 2 -c 36 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  2 threads and 36 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.19ms 755.77us 9.01ms 59.47%
  Req/Sec 15.12k 1.97k 21.49k 67.00%
  150392 requests in 5.00s, 610.98MB read
  Requests/sec: 30070.66
  Transfer/sec: 122.17MB
• sleep 1
• run_wrk
  ++ expr 2 '*' 18
• connections=36
• wrk -d 5 -t 2 -c 36 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  2 threads and 36 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.19ms 776.00us 7.47ms 54.86%
  Req/Sec 15.12k 2.46k 30.97k 82.18%
  151874 requests in 5.10s, 617.00MB read
  Requests/sec: 29782.74
  Transfer/sec: 121.00MB
• sleep 1
• make stop
  /usr/bin/openresty -p $PWD/ -c $PWD/conf/nginx.conf -s stop
• echo -e '\n\nfake empty apisix server: 2 worker'

fake empty apisix server: 2 worker

• sleep 1
• sed -i 's/worker_processes [0-9]*/worker_processes 2/g' benchmark/fake-apisix/conf/nginx.conf
• openresty -p /home/rain/apisix/benchmark/fake-apisix
• sleep 1
• run_wrk
  ++ expr 2 '*' 18
• connections=36
• wrk -d 5 -t 2 -c 36 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  2 threads and 36 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 0.91ms 186.65us 8.38ms 91.45%
  Req/Sec 19.82k 0.93k 24.85k 90.20%
  201062 requests in 5.10s, 800.15MB read
  Requests/sec: 39421.89
  Transfer/sec: 156.88MB
• sleep 1
• run_wrk
  ++ expr 2 '*' 18
• connections=36
• wrk -d 5 -t 2 -c 36 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  2 threads and 36 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 0.89ms 201.99us 8.35ms 90.44%
  Req/Sec 20.26k 2.19k 41.74k 99.01%
  203612 requests in 5.10s, 810.30MB read
  Requests/sec: 39922.56
  Transfer/sec: 158.88MB
• openresty -p /home/rain/apisix/benchmark/fake-apisix -s stop
• openresty -p /home/rain/apisix/benchmark/server -s stop
  3 workers: apisix git:(master) ✗ bash -x ./benchmark/run.sh 3
• '[' -n 3 ']'
• worker_cnt=3
• mkdir -p benchmark/server/logs
• mkdir -p benchmark/fake-apisix/logs
• trap onCtrlC INT
• openresty -p /home/rain/apisix/benchmark/server
• make init
  ./bin/apisix init
  ./bin/apisix init_etcd
  ++ uname
• [[ Linux == \D\a\r\w\i\n ]]
• sed -i 's/worker_processes .*/worker_processes 3;/g' conf/nginx.conf
• make run
  mkdir -p logs
  mkdir -p /tmp/apisix_cores/
  /usr/bin/openresty -p $PWD/ -c $PWD/conf/nginx.conf
• sleep 3
• echo -e '\n\napisix: 3 worker + 1 upstream + no plugin'

apisix: 3 worker + 1 upstream + no plugin

• curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
  {
  "uri": "/hello",
  "plugins": {
  },
  "upstream": {
  "type": "roundrobin",
  "nodes": {
  "127.0.0.1:1980": 1
  }
  }
  }'
  {"node":{"value":{"priority":0,"plugins":{},"upstream":{"nodes":{"127.0.0.1:1980":1},"hash_on":"vars","type":"roundrobin"},"id":"1","uri":"/hello"},"createdIndex":31,"key":"/apisix/routes/1","modifiedIndex":31},"prevNode":{"value":"{"priority":0,"plugins":{"limit-count":{"time_window":60,"count":2000000000000,"rejected_code":503,"key":"remote_addr","policy":"local"},"prometheus":{}},"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:1980":1},"type":"roundrobin"},"id":"1","uri":"\/hello"}","createdIndex":30,"key":"/apisix/routes/1","modifiedIndex":30},"action":"set"}
• sleep 1
• run_wrk
  ++ expr 3 '*' 18
• connections=54
• wrk -d 5 -t 3 -c 54 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  3 threads and 54 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.02ms 559.01us 8.16ms 68.25%
  Req/Sec 17.62k 1.70k 22.38k 72.67%
  262986 requests in 5.00s, 1.03GB read
  Requests/sec: 52580.67
  Transfer/sec: 210.00MB
• sleep 1
• run_wrk
  ++ expr 3 '*' 18
• connections=54
• wrk -d 5 -t 3 -c 54 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  3 threads and 54 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.02ms 688.37us 9.45ms 66.35%
  Req/Sec 17.68k 3.80k 55.88k 90.07%
  265647 requests in 5.10s, 1.04GB read
  Requests/sec: 52089.10
  Transfer/sec: 208.04MB
• sleep 1
• echo -e '\n\napisix: 3 worker + 1 upstream + 2 plugins (limit-count + prometheus)'

apisix: 3 worker + 1 upstream + 2 plugins (limit-count + prometheus)

• curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
  {
  "uri": "/hello",
  "plugins": {
  "limit-count": {
  "count": 2000000000000,
  "time_window": 60,
  "rejected_code": 503,
  "key": "remote_addr"
  },
  "prometheus": {}
  },
  "upstream": {
  "type": "roundrobin",
  "nodes": {
  "127.0.0.1:1980": 1
  }
  }
  }'
  {"node":{"value":{"priority":0,"plugins":{"limit-count":{"time_window":60,"count":2000000000000,"rejected_code":503,"key":"remote_addr","policy":"local"},"prometheus":{}},"upstream":{"nodes":{"127.0.0.1:1980":1},"hash_on":"vars","type":"roundrobin"},"id":"1","uri":"/hello"},"createdIndex":32,"key":"/apisix/routes/1","modifiedIndex":32},"prevNode":{"value":"{"priority":0,"plugins":{},"upstream":{"hash_on":"vars","nodes":{"127.0.0.1:1980":1},"type":"roundrobin"},"id":"1","uri":"\/hello"}","createdIndex":31,"key":"/apisix/routes/1","modifiedIndex":31},"action":"set"}
• sleep 3
• run_wrk
  ++ expr 3 '*' 18
• connections=54
• wrk -d 5 -t 3 -c 54 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  3 threads and 54 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.29ms 0.87ms 12.60ms 62.39%
  Req/Sec 13.96k 1.96k 21.02k 72.37%
  211139 requests in 5.10s, 857.78MB read
  Requests/sec: 41400.33
  Transfer/sec: 168.19MB
• sleep 1
• run_wrk
  ++ expr 3 '*' 18
• connections=54
• wrk -d 5 -t 3 -c 54 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  3 threads and 54 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.20ms 708.40us 10.50ms 65.05%
  Req/Sec 15.05k 1.83k 26.51k 78.29%
  227511 requests in 5.10s, 0.90GB read
  Requests/sec: 44613.47
  Transfer/sec: 181.25MB
• sleep 1
• make stop
  /usr/bin/openresty -p $PWD/ -c $PWD/conf/nginx.conf -s stop
• echo -e '\n\nfake empty apisix server: 3 worker'

fake empty apisix server: 3 worker

• sleep 1
• sed -i 's/worker_processes [0-9]*/worker_processes 3/g' benchmark/fake-apisix/conf/nginx.conf
• openresty -p /home/rain/apisix/benchmark/fake-apisix
• sleep 1
• run_wrk
  ++ expr 3 '*' 18
• connections=54
• wrk -d 5 -t 3 -c 54 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  3 threads and 54 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 0.86ms 211.68us 11.14ms 90.99%
  Req/Sec 20.96k 1.48k 31.82k 93.42%
  317004 requests in 5.10s, 1.23GB read
  Requests/sec: 62161.01
  Transfer/sec: 247.38MB
• sleep 1
• run_wrk
  ++ expr 3 '*' 18
• connections=54
• wrk -d 5 -t 3 -c 54 http://127.0.0.1:9080/hello
  Running 5s test @ http://127.0.0.1:9080/hello
  3 threads and 54 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 0.86ms 214.42us 9.05ms 91.26%
  Req/Sec 21.01k 1.40k 33.20k 95.39%
  317803 requests in 5.10s, 1.24GB read
  Requests/sec: 62316.06
  Transfer/sec: 248.00MB
• openresty -p /home/rain/apisix/benchmark/fake-apisix -s stop
• openresty -p /home/rain/apisix/benchmark/server -s stop
  envoy 测试脚本：https://gist.github.com/membphis/acafea9733147f465a73d4901827e3c0#gistcomment-3403431

➜ envoy ./run.sh 1

• '[' -n '' ']'
• worker_cnt=1
• trap onCtrlC INT
• openresty -p /home/rain/apisix/benchmark/server
  ++ pwd
• sudo docker run --name=envoy --rm -d --network=host -v /home/rain/envoy/config.yaml:/etc/envoy/envoy.yaml envoyproxy/envoy:v1.14-latest -c /etc/envoy/envoy.yaml --concurrency 1
  bd9ed3378879aec7f7c7f9494b9bc284fa146fc538120bf697cdad67e5fce302
• sleep 1
• curl http://127.0.0.1:10000/hello
  1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890+ sleep 1
• run_wrk
  ++ expr 1 '*' 16
• connections=16
• wrk -d 5 -t 1 -c 16 http://127.0.0.1:10000/hello
  Running 5s test @ http://127.0.0.1:10000/hello
  1 threads and 16 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.15ms 246.95us 9.98ms 96.47%
  Req/Sec 13.92k 459.51 14.72k 76.47%
  70650 requests in 5.10s, 281.03MB read
  Requests/sec: 13853.02
  Transfer/sec: 55.10MB
• sleep 1
• run_wrk
  ++ expr 1 '*' 16
• connections=16
• wrk -d 5 -t 1 -c 16 http://127.0.0.1:10000/hello
  Running 5s test @ http://127.0.0.1:10000/hello
  1 threads and 16 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.15ms 176.85us 6.54ms 90.02%
  Req/Sec 13.99k 467.18 14.83k 72.00%
  69608 requests in 5.00s, 276.89MB read
  Requests/sec: 13920.11
  Transfer/sec: 55.37MB
• sleep 1
• onCtrlC
• killall wrk
  wrk: no process found
• killall openresty
• openresty -p /home/rain/apisix/benchmark/server -s stop
  nginx: [error] open() "/home/rain/apisix/benchmark/server/logs/nginx.pid" failed (2: No such file or directory)
• sudo docker stop envoy
  envoy
  ➜ envoy ./run.sh 2
• '[' -n 2 ']'
• worker_cnt=2
• trap onCtrlC INT
• openresty -p /home/rain/apisix/benchmark/server
  ++ pwd
• sudo docker run --name=envoy --rm -d --network=host -v /home/rain/envoy/config.yaml:/etc/envoy/envoy.yaml envoyproxy/envoy:v1.14-latest -c /etc/envoy/envoy.yaml --concurrency 2
  70cb1b6142b9d5e02c541d63437b469c76b6bc3c773c2e97a8ab7de475780fb5
• sleep 1
• curl http://127.0.0.1:10000/hello
  1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890+ sleep 1
• run_wrk
  ++ expr 2 '*' 16
• connections=32
• wrk -d 5 -t 2 -c 32 http://127.0.0.1:10000/hello
  Running 5s test @ http://127.0.0.1:10000/hello
  2 threads and 32 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.21ms 650.84us 34.18ms 97.61%
  Req/Sec 13.19k 1.62k 27.31k 97.03%
  132583 requests in 5.10s, 527.39MB read
  Requests/sec: 25995.74
  Transfer/sec: 103.41MB
• sleep 1
• run_wrk
  ++ expr 2 '*' 16
• connections=32
• wrk -d 5 -t 2 -c 32 http://127.0.0.1:10000/hello
  Running 5s test @ http://127.0.0.1:10000/hello
  2 threads and 32 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.22ms 261.66us 10.69ms 95.19%
  Req/Sec 13.13k 427.13 14.25k 69.00%
  130601 requests in 5.00s, 519.50MB read
  Requests/sec: 26118.37
  Transfer/sec: 103.89MB
• sleep 1
• onCtrlC
• killall wrk
  wrk: no process found
• killall openresty
• openresty -p /home/rain/apisix/benchmark/server -s stop
  nginx: [error] open() "/home/rain/apisix/benchmark/server/logs/nginx.pid" failed (2: No such file or directory)
• sudo docker stop envoy
  envoy
  ➜ envoy ./run.sh 3
• '[' -n 3 ']'
• worker_cnt=3
• trap onCtrlC INT
• openresty -p /home/rain/apisix/benchmark/server
  ++ pwd
• sudo docker run --name=envoy --rm -d --network=host -v /home/rain/envoy/config.yaml:/etc/envoy/envoy.yaml envoyproxy/envoy:v1.14-latest -c /etc/envoy/envoy.yaml --concurrency 3
  7b74e0329d07b804a09531865b534c7b8fac796ee17f60ea8f07e360b1ebcdac
• sleep 1
• curl http://127.0.0.1:10000/hello
  1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890+ sleep 1
• run_wrk
  ++ expr 3 '*' 16
• connections=48
• wrk -d 5 -t 3 -c 48 http://127.0.0.1:10000/hello
  Running 5s test @ http://127.0.0.1:10000/hello
  3 threads and 48 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.22ms 670.84us 27.11ms 85.72%
  Req/Sec 13.15k 2.16k 36.42k 97.35%
  197670 requests in 5.10s, 786.29MB read
  Requests/sec: 38762.25
  Transfer/sec: 154.19MB
• sleep 1
• run_wrk
  ++ expr 3 '*' 16
• connections=48
• wrk -d 5 -t 3 -c 48 http://127.0.0.1:10000/hello
  Running 5s test @ http://127.0.0.1:10000/hello
  3 threads and 48 connections
  Thread Stats Avg Stdev Max +/- Stdev
  Latency 1.24ms 343.40us 10.03ms 66.01%
  Req/Sec 12.95k 560.28 14.00k 58.00%
  193226 requests in 5.00s, 768.61MB read
  Requests/sec: 38628.05
  Transfer/sec: 153.65MB
• sleep 1
• onCtrlC
• killall wrk
  wrk: no process found
• killall openresty
• openresty -p /home/rain/apisix/benchmark/server -s stop
  nginx: [error] open() "/home/rain/apisix/benchmark/server/logs/nginx.pid" failed (2: No such file or directory)
• sudo docker stop envoy
  envoy
  envoy cat run.sh
  #! /bin/bash -x

if [ -n "$1" ]; then
worker_cnt=$1
else
worker_cnt=1
fi

trap 'onCtrlC' INT
function onCtrlC () {
killall wrk
killall openresty
openresty -p /home/rain/apisix/benchmark/server -s stop
sudo docker stop envoy
}

function run_wrk() {
connections=expr $worker_cnt \* 16
wrk -d 5 -t $worke{r}_{c}nt-c${connections} http://127.0.0.1:10000/hello
}

openresty -p /home/rain/apisix/benchmark/server || exit 1

sudo docker run --name=envoy --rm -d
--network=host
-v $(pwd)/config.yaml:/etc/envoy/envoy.yaml
envoyproxy/envoy:v1.14-latest -c /etc/envoy/envoy.yaml --concurrency ${worker_cnt}

sleep 1

curl http://127.0.0.1:10000/hello

sleep 1

run_wrk

sleep 1

run_wrk

sleep 1

onCtrlC