Nginx / OpenResty segfault - ngx_http_lua_socket_tcp_finalize

backtrace

[root@my-server ~]# gdb /usr/local/nginx/sbin/nginx /tmp/core_dumps/99_27290_nginx
GNU gdb (GDB) Red Hat Enterprise Linux (7.2-60.el6_4.1)
Copyright (C) 2010 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/local/nginx/sbin/nginx...done.
[New Thread 27290]
Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done.
[Thread debugging using libthread_db enabled]
Loaded symbols for /lib64/libpthread.so.0
Reading symbols from /lib64/libcrypt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libcrypt.so.1
Reading symbols from /usr/local/luajit/lib/libluajit-5.1.so.2...done.
Loaded symbols for /usr/local/luajit/lib/libluajit-5.1.so.2
Reading symbols from /lib64/libm.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libm.so.6
Reading symbols from /usr/lib64/libssl.so.10...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libssl.so.10
Reading symbols from /usr/lib64/libcrypto.so.10...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libcrypto.so.10
Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libdl.so.2
Reading symbols from /lib64/libz.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libz.so.1
Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libc.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /lib64/libfreebl3.so...(no debugging symbols found)...done.
Loaded symbols for /lib64/libfreebl3.so
Reading symbols from /lib64/libgcc_s.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libgcc_s.so.1
Reading symbols from /lib64/libgssapi_krb5.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libgssapi_krb5.so.2
Reading symbols from /lib64/libkrb5.so.3...(no debugging symbols found)...done.
Loaded symbols for /lib64/libkrb5.so.3
Reading symbols from /lib64/libcom_err.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libcom_err.so.2
Reading symbols from /lib64/libk5crypto.so.3...(no debugging symbols found)...done.
Loaded symbols for /lib64/libk5crypto.so.3
Reading symbols from /lib64/libkrb5support.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib64/libkrb5support.so.0
Reading symbols from /lib64/libkeyutils.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libkeyutils.so.1
Reading symbols from /lib64/libresolv.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libresolv.so.2
Reading symbols from /lib64/libselinux.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libselinux.so.1
Reading symbols from /lib64/libnss_files.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libnss_files.so.2
Reading symbols from /usr/local/lualib/cjson.so...done.
Loaded symbols for /usr/local/lualib/cjson.so
Core was generated by `nginx: worker process                               '.
Program terminated with signal 11, Segmentation fault.
#0  0x0000000000515878 in ngx_http_lua_socket_tcp_finalize (r=0xddba80, u=0x41221f40) at ../ngx_lua-0.9.4/src/ngx_http_lua_socket_tcp.c:2461
2461	../ngx_lua-0.9.4/src/ngx_http_lua_socket_tcp.c: No such file or directory.
	in ../ngx_lua-0.9.4/src/ngx_http_lua_socket_tcp.c
Missing separate debuginfos, use: debuginfo-install nginx-1.5.8.1-1.x86_64
(gdb) backtrace full
#0  0x0000000000515878 in ngx_http_lua_socket_tcp_finalize (r=0xddba80, u=0x41221f40) at ../ngx_lua-0.9.4/src/ngx_http_lua_socket_tcp.c:2461
        spool = 0x7fff7a50fd20
        cl = 0xe08f58
        ll = 0x41222008
        ctx = 0xddcbb0
#1  0x0000000000515794 in ngx_http_lua_socket_tcp_cleanup (data=0x41221f40) at ../ngx_lua-0.9.4/src/ngx_http_lua_socket_tcp.c:2431
        u = 0x41221f40
        r = 0xddba80
#2  0x000000000046c4f1 in ngx_http_free_request (r=0xddba80, rc=0) at src/http/ngx_http_request.c:3395
        log = 0xdfabc0
        pool = 0x7fff7a50fcd0
        linger = {l_onoff = 2052123900, l_linger = 32767}
        cln = 0xe00068
        ctx = 0x0
        clcf = 0x7fff7a50ed13
#3  0x000000000046b208 in ngx_http_set_keepalive (r=0xddba80) at src/http/ngx_http_request.c:2857
        tcp_nodelay = 32767
        i = 4789529
        b = 0xe62ae8
        f = 0x7fff7a50fd78
        rev = 0xe21b90
        wev = 0xddba80
        c = 0x7fcedc45e4c0
        hc = 0xdfac08
        cscf = 0x0
        clcf = 0xe14840
#4  0x000000000046a69b in ngx_http_finalize_connection (r=0xddba80) at src/http/ngx_http_request.c:2500
        clcf = 0xe14840
#5  0x000000000046a303 in ngx_http_finalize_request (r=0xddba80, rc=0) at src/http/ngx_http_request.c:2401
        c = 0x7fcedc45e4c0
        pr = 0x1
        clcf = 0x0
#6  0x0000000000488421 in ngx_http_upstream_finalize_request (r=0xddba80, u=0xdddaf0, rc=0) at src/http/ngx_http_upstream.c:3563
        flush = 0
        tp = 0x7b5300
#7  0x000000000048753c in ngx_http_upstream_process_request (r=0xddba80) at src/http/ngx_http_upstream.c:3164
        tf = 0xdddfa0
        p = 0xdddfa0
        u = 0xdddaf0
#8  0x000000000048725a in ngx_http_upstream_process_upstream (r=0xddba80, u=0xdddaf0) at src/http/ngx_http_upstream.c:3095
        c = 0x7fcedc45e7e0
#9  0x000000000048257f in ngx_http_upstream_handler (ev=0xe21d30) at src/http/ngx_http_upstream.c:977
        c = 0x7fcedc45e4c0
        r = 0xddba80
        ctx = 0xe62ad0
        u = 0xdddaf0
#10 0x000000000044bd07 in ngx_epoll_process_events (cycle=0xddfa90, timer=4660, flags=1) at src/event/modules/ngx_epoll_module.c:691
        events = 1
        revents = 5
        instance = 0
        i = 0
        level = 14580648
        err = 0
        rev = 0xe21d30
        wev = 0xe3bd40
        queue = 0x7b70e0
        c = 0x7fcedc45e7e0
#11 0x000000000043c448 in ngx_process_events_and_timers (cycle=0xddfa90) at src/event/ngx_event.c:248
        flags = 1
        timer = 4660
        delta = 1395137731116
#12 0x0000000000449c3a in ngx_worker_process_cycle (cycle=0xddfa90, data=0x0) at src/os/unix/ngx_process_cycle.c:816
        worker = 0
        i = 0
        c = 0x417300
#13 0x0000000000446380 in ngx_spawn_process (cycle=0xddfa90, proc=0x449a34 <ngx_worker_process_cycle>, data=0x0, name=0x563d33 "worker process", respawn=1) at src/os/unix/ngx_process.c:198
        on = 1
        pid = 0
        s = 1
#14 0x0000000000449620 in ngx_reap_children (cycle=0xddfa90) at src/os/unix/ngx_process_cycle.c:627
        i = 1
        n = 2
        live = 0
        ch = {command = 2, pid = 27276, slot = 1, fd = -1}
        ccf = 0x7b688f
#15 0x0000000000448230 in ngx_master_process_cycle (cycle=0xddfa90) at src/os/unix/ngx_process_cycle.c:180
        title = 0xdfc353 ""
        p = 0xdfc396 ""
        size = 68
        i = 3
        n = 0
        sigio = 0
        set = {__val = {0 <repeats 16 times>}}
        itv = {it_interval = {tv_sec = 0, tv_usec = 1}, it_value = {tv_sec = 18, tv_usec = 18}}
        live = 1
        delay = 0
        ls = 0x0
        ccf = 0xde0050
#16 0x000000000041797b in main (argc=3, argv=0x7fff7a510588) at src/core/nginx.c:407
        i = 65
        log = 0x7b4ee0
        cycle = 0xddba80
        init_cycle = {conf_ctx = 0x0, pool = 0xddb530, log = 0x7b4ee0, new_log = {log_level = 0, file = 0x0, connection = 0, handler = 0, data = 0x0, action = 0x0, next = 0x0}, log_use_stderr = 0, files = 0x0,
          free_connections = 0x0, free_connection_n = 0, reusable_connections_queue = {prev = 0x0, next = 0x0}, listening = {elts = 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0}, paths = {elts = 0x0, nelts = 0, size = 0,
            nalloc = 0, pool = 0x0}, open_files = {last = 0x0, part = {elts = 0x0, nelts = 0, next = 0x0}, size = 0, nalloc = 0, pool = 0x0}, shared_memory = {last = 0x0, part = {elts = 0x0, nelts = 0, next = 0x0}, size = 0,
            nalloc = 0, pool = 0x0}, connection_n = 0, files_n = 0, connections = 0x0, read_events = 0x0, write_events = 0x0, old_cycle = 0x0, conf_file = {len = 21, data = 0x7fff7a511f5c ""}, conf_param = {len = 0, data = 0x0},
          conf_prefix = {len = 11, data = 0x7fff7a511f5c ""}, prefix = {len = 17, data = 0x560006 "/usr/local/nginx/"}, lock_file = {len = 0, data = 0x0}, hostname = {len = 0, data = 0x0}}
        ccf = 0xddca88

cache.lua

local memcached = require "resty.memcached"
local config = require 'config'
local cache = {}

local memc, err = memcached:new()
if not memc then
  ngx.log(ngx.ERR, "failed to instantiate memc: ", err)
  return ngx.exit(500)
end
memc:set_timeout(1000)

local function memcache_connect()
  local ok, err = memc:connect("127.0.0.1", 11211)
  if not ok then
    ngx.log(ngx.ERR, "failed to connect to memcached: ", err)
    return ngx.exit(500)
  end
end

function cache.get_user_details(token)
  memcache_connect()
  value, flags, err = memc:get(token)
  if value then
    local m, err = ngx.re.match(value, '^(\\S+) (\\S+) (\\S+)$')
    return m[1], m[2], m[3]
  else
    return false
  end
end

function cache.add_user_details(token, user, business, app)
  memcache_connect()
  memc:set(token, user..' '..business..' '..app, config.cache_timeout)
end

function cache.delete(token)
  memcache_connect()
  memc:delete(token)
end

return cache

nginx-error.log

get_user_details(): RETRIEVED FROM APP: f227be2fc5ff50d0f152cb353b015c69, bb27f4a451b9bd216b38c7fba64c5ce7
2014/03/11 11:32:05 [notice] 5045#0: signal 17 (SIGCHLD) received
2014/03/11 11:32:05 [notice] 5045#0: signal 17 (SIGCHLD) received
2014/03/11 11:32:05 [alert] 5045#0: worker process 26693 exited on signal 11
2014/03/11 11:32:05 [debug] 5045#0: shmtx forced unlock
2014/03/11 11:32:05 [debug] 5045#0: channel 3:7
2014/03/11 11:32:05 [notice] 5045#0: start worker process 27401
2014/03/11 11:32:05 [debug] 27401#0: setproctitle: "nginx: worker process"
2014/03/11 11:32:05 [debug] 27401#0: *37617 reusable connection: 1
2014/03/11 11:32:05 [debug] 27401#0: *37617 reusable connection: 0
2014/03/11 11:32:05 [debug] 27401#0: resolve: "app.example.com"
2014/03/11 11:32:05 [debug] 27401#0: resolve: "app.example.com" 61608
2014/03/11 11:32:05 [debug] 27401#0: resolver DNS response 61608 fl:8180 1/3/0/0
2014/03/11 11:32:05 [debug] 27401#0: resolver DNS response qt:1 cl:1
2014/03/11 11:32:05 [debug] 27401#0: resolver qs:app.example.com
2014/03/11 11:32:05 [debug] 27401#0: resolver naddrs:2 cname:00007FFFFA73849D ttl:59
2014/03/11 11:32:05 [debug] 27401#0: resolve name done: 0
2014/03/11 11:32:05 [debug] 27401#0: resolver expire
2014/03/11 11:32:05 [debug] 27401#0: *37617 chain writer buf fl:1 s:222
2014/03/11 11:32:05 [debug] 27401#0: *37617 chain writer in: 0000000000CDECE8
2014/03/11 11:32:05 [debug] 27401#0: *37617 chain writer out: 0000000000000000
2014/03/11 11:32:06 [debug] 27401#0: *37617 reusable connection: 0
2014/03/11 11:32:06 [debug] 27401#0: *37617 [lua] auth.lua:82:

nginx.conf

worker_processes  1;
error_log  /var/log/nginx-error.log  debug_core;
pid        /var/run/nginx.pid;

worker_rlimit_core 500M;
working_directory /tmp/core_dumps/;

events {
    worker_connections  1024;
}


http {
    lua_code_cache on;
    lua_package_path '/etc/nginx/lualib/?.lua;;';
    server {
        listen       80;
        server_name  app.example.com
		pre-app.example.biz;

	access_log /var/log/nginx-acces.log combined;

        location /_auth_check {
            resolver 8.8.8.8;
            access_by_lua_file "/etc/nginx/lualib/main.lua";
            proxy_pass_request_headers off;
            proxy_set_header Accept "application/json";
            proxy_set_header User-Agent "API Proxy";
            proxy_set_header X-API-PROXY-AUTH-DIGEST $secret_header_value;
            proxy_pass $app_location;
        }

        location /api/ {
            resolver 8.8.8.8;
            set $proxy_destination '';
            set $app_location '';
            set $secret_header_value '';
            set $user_guid_header_value '';
            set $business_guid_header_value '';
            access_by_lua_file "/etc/nginx/lualib/main.lua";
            proxy_set_header X-API-PROXY-AUTH-DIGEST $secret_header_value;
            proxy_set_header X-API-PROXY-USER-GUID $user_guid_header_value;
            proxy_set_header X-API-PROXY-BUSINESS-GUID $business_guid_header_value;
            proxy_pass $proxy_destination;
        }

        location / {
            deny all;
        }
    }
}