Skip to content

Instantly share code, notes, and snippets.

@nickva

nickva/haproxyabrt.cfg

Created September 27, 2023 03:06
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save nickva/9d53e03e54fda24e77daa236b894a346 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save nickva/9d53e03e54fda24e77daa236b894a346 to your computer and use it in GitHub Desktop.
Download ZIP
CouchDB Haproxy With Quick Client Disconnect
Raw
haproxyabrt.cfg
global
maxconn 10000
spread-checks 5
defaults
mode http
log global
option httplog
log stdout format raw local0
balance roundrobin
option redispatch
retries 4
timeout client 150s
timeout server 360s
timeout connect 500
option abortonclose
frontend http-in
option clitcpka
clitcpka-cnt 3
clitcpka-intvl 1s
clitcpka-idle 3s
bind *:5984
default_backend servers
backend servers
option httpchk GET /_up
server node1 127.0.0.1:15984 check inter 600s
@nickva
Copy link
Copy Markdown
Author

nickva commented Sep 27, 2023 

% haproxy -f haproxyabrt.cfg
127.0.0.1:50640 [26/Sep/2023:23:03:17.639] http-in servers/node1 0/0/5/1/6 200 828 - - ---- 1/1/0/0/0 0/0 "GET /db HTTP/1.1"
127.0.0.1:50640 [26/Sep/2023:23:03:17.649] http-in servers/node1 0/0/0/66/66 200 242 - - ---- 1/1/0/0/0 0/0 "DELETE /db HTTP/1.1"
127.0.0.1:50640 [26/Sep/2023:23:03:22.722] http-in servers/node1 0/0/0/115/115 201 279 - - ---- 1/1/0/0/0 0/0 "PUT /db?q=1 HTTP/1.1"
127.0.0.1:50640 [26/Sep/2023:23:03:22.841] http-in servers/node1 0/0/0/79/79 201 47740 - - ---- 1/1/0/0/0 0/0 "POST /db/_bulk_docs HTTP/1.1"
127.0.0.1:50640 [26/Sep/2023:23:03:22.922] http-in servers/node1 0/0/0/76/76 201 47740 - - ---- 1/1/0/0/0 0/0 "POST /db/_bulk_docs HTTP/1.1"
127.0.0.1:50640 [26/Sep/2023:23:03:23.002] http-in servers/node1 0/0/0/73/73 201 47740 - - ---- 1/1/0/0/0 0/0 "POST /db/_bulk_docs HTTP/1.1"
127.0.0.1:50640 [26/Sep/2023:23:03:23.077] http-in servers/node1 0/0/0/74/74 201 47740 - - ---- 1/1/0/0/0 0/0 "POST /db/_bulk_docs HTTP/1.1"
127.0.0.1:50640 [26/Sep/2023:23:03:23.154] http-in servers/node1 0/0/0/-1/8137 400 206 - - CH-- 1/1/0/0/0 0/0 "POST /db/_find HTTP/1.1"

@nickva
Copy link
Copy Markdown
Author

nickva commented Sep 27, 2023

Issue slow running _find query then abort after a few seconds

% ./slowfindha.py
 * deleting http://127.0.0.1:5984/db
 * creating http://127.0.0.1:5984/db with q=1
 * created http://127.0.0.1:5984/db
 * creating 2000 docs
 * created docs in 0.3 seconds
 * calling _find with selector: {'x': {'$regex': '^(((a+)+)+)+$'}}

^C

@nickva
Copy link
Copy Markdown
Author

nickva commented Sep 27, 2023

Shorten client connection check a bit to avoid waiting too long and show when request process dies

diff --git a/src/chttpd/src/chttpd_util.erl b/src/chttpd/src/chttpd_util.erl
index d77e8d2aa..af7d598c0 100644
--- a/src/chttpd/src/chttpd_util.erl
+++ b/src/chttpd/src/chttpd_util.erl
@@ -31,8 +31,8 @@
 ]).

 -define(MOCHIWEB_CLIENT_REQ, mochiweb_client_req).
--define(DISCONNECT_CHECK_MSEC, 30000).
--define(DISCONNECT_CHECK_JITTER_MSEC, 15000).
+-define(DISCONNECT_CHECK_MSEC, 5000).
+-define(DISCONNECT_CHECK_JITTER_MSEC, 500).

 get_chttpd_config(Key) ->
     config:get("chttpd", Key, config:get("httpd", Key)).
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
index 8e4b72840..f6d502874 100644
--- a/src/mango/src/mango_httpd.erl
+++ b/src/mango/src/mango_httpd.erl
@@ -205,6 +205,8 @@ handle_explain_req(Req, _Db) ->
     chttpd:send_method_not_allowed(Req, "POST").

 handle_find_req(#httpd{method = 'POST'} = Req, Db) ->
+    couch_log:error(" ++++++++++TRACING _find++++++++ ~p:~p@~B REQPID:~p", [?MODULE, ?FUNCTION_NAME, ?LINE, self()]),
+    dbg:tracer(), dbg:p(self(), p),
     chttpd:validate_ctype(Req, "application/json"),
     Body = maybe_set_partition(Req),
     {ok, Opts0} = mango_opts:validate_find(Body),

@nickva
Copy link
Copy Markdown
Author

nickva commented Sep 27, 2023 •
edited
Loading

CouchDB logs show the request process is killed within 5 seconds or so

[notice] 2023-09-27T03:03:22.837658Z node1@127.0.0.1 <0.18737.0> 47b8291a4c 127.0.0.1:5984 127.0.0.1 adm PUT /db?q=1 201 ok 114
[notice] 2023-09-27T03:03:22.920781Z node1@127.0.0.1 <0.18737.0> e1ca592e21 127.0.0.1:5984 127.0.0.1 adm POST /db/_bulk_docs 201 ok 79
[notice] 2023-09-27T03:03:22.999434Z node1@127.0.0.1 <0.18737.0> fdbba7a0ee 127.0.0.1:5984 127.0.0.1 adm POST /db/_bulk_docs 201 ok 75
[notice] 2023-09-27T03:03:23.075838Z node1@127.0.0.1 <0.18737.0> f9af5e4d9a 127.0.0.1:5984 127.0.0.1 adm POST /db/_bulk_docs 201 ok 73
[notice] 2023-09-27T03:03:23.152235Z node1@127.0.0.1 <0.18737.0> 888ea8db06 127.0.0.1:5984 127.0.0.1 adm POST /db/_bulk_docs 201 ok 73
[error] 2023-09-27T03:03:23.155898Z node1@127.0.0.1 <0.18737.0> b311b36392  ++++++++++TRACING _find++++++++ mango_httpd:handle_find_req@208 REQPID:<0.18737.0>
(<0.18737.0>) spawn <0.19452.0> as erlang:apply(#Fun<rexi_monitor.1.77517421>,[])
(<0.18737.0>) link <0.19452.0>
(<0.18737.0>) spawn <0.19453.0> as erlang:apply(#Fun<fabric_streams.3.91837225>,[])
(<0.18737.0>) exit {shutdown,client_disconnected}

@nickva
Copy link
Copy Markdown
Author

nickva commented Sep 27, 2023

Capture traffic from both client <-> haproxy:5984 and haproxy <-> couchdb:15984 connections

tcp port 5984 or tcp port 15984

Screenshot 2023-09-26 at 11 10 04 PM

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment