Skip to content

Instantly share code, notes, and snippets.

@chobits
Last active September 7, 2023 03:15
Show Gist options
  • Save chobits/33219d345710eb2d585fcb5295851dbc to your computer and use it in GitHub Desktop.
Save chobits/33219d345710eb2d585fcb5295851dbc to your computer and use it in GitHub Desktop.
trace function call chains while reading the lua source
-- Trace request context exclusively within Lua projects
-- executed by the Lua-nginx-module
--
-- The timer context is not available within functions triggered by
-- ngx.timer.at(). However, you can easily make adjustments to this file to
-- enable it.
--
-- In your lua source file:
-- require("bt").init_hook()
-- true : it displays the level number of the calling function.
-- false: it displays space-based indentation determined by the level number.
local DISPLAY_LEVEL_NUMBER = false
local _M = {}
function _M.init_hook()
if not ngx.ctx then
return nil
end
local _t = {
level = 0,
call_chains = {}, -- array[]
id = string.sub(ngx.var.request_id, 1, 8),
}
ngx.ctx._trace_info = _t
setmetatable(_M, {})
return _t
end
local function trace_skip(info)
if not info or not info.name then
return true
end
-- skip inline function, such as print()
if info.source == "[C]" or info.currentline == -1 then
return true
end
-- skip myself
if string.find(info.source, "/bt.lua") or info.name == "init_hook" then
return true
end
-- not kong source (maybe required library)
if string.find(info.source, "/openresty/")
or string.find(info.source, "/lualib/")
or string.find(info.source, "/lua/5%.[123]/")
then
return true
end
return false
end
-- This function will be called whenever a function is called or returns
local function trace_hook(event)
-- skip timer
-- request context
if not ngx.ctx then
return
end
local _t = ngx.ctx._trace_info
if not _t then
return
end
local info = debug.getinfo(2, "nSl")
if trace_skip(info) then
return
end
-- evaluate
local func_name = info.name
local line_info = info.short_src .. ":" .. (info.currentline or "unknown")
local prefix, s
local level = _t.level
local event_prefix = ">"
if event == "call" then
_t.level = level + 1
_t.call_chains[_t.level] = func_name
elseif event == "return" then
event_prefix = "< (no caller)"
-- fix that some function has no return event (e.g. __index [C]:-1)
local i = level
while i > 0 do
local orig_func_name = _t.call_chains[i]
i = i - 1
-- print("[DD] call chain pop:", orig_func_name, " return func:", func_name)
if orig_func_name == func_name then
_t.level = i
level = i
event_prefix = "<"
break
end
end
else
event_prefix = "?"
end
if DISPLAY_LEVEL_NUMBER then
prefix = tostring(level)
else
prefix = string.rep(" ", level)
end
s = string.format("%s: %s %s %s() %s",
_t.id, prefix, event_prefix, func_name, line_info)
-- show
ngx.log(ngx.ERR, "\nHook: [" .. s .. "]\n")
-- print(s)
end
local function init()
debug.sethook(trace_hook, "cr")
--[[
if jit then
jit.off()
end
]]
end
init()
return _M
@chobits
Copy link
Author

chobits commented Sep 3, 2023

For kong project, how to use

$ git clone https://github.com/kong/kong
$ cd kong/kong
$ wget https://gist.githubusercontent.com/chobits/33219d345710eb2d585fcb5295851dbc/raw/fe9270c5fb58d7c539615e892bf14bfea0a26e16/bt.lua

# modify init.lua: Kong:access() to start tracing the request context after Kong:access()
$ git diff
diff --git a/kong/init.lua b/kong/init.lua
index 5bb4aa68c..0c271e3a0 100644
--- a/kong/init.lua
+++ b/kong/init.lua
@@ -1016,6 +1016,8 @@ end


 function Kong.access()
+  require("kong.bt").init_hook()
+
   local ctx = ngx.ctx
   if not ctx.KONG_ACCESS_START then
     ctx.KONG_ACCESS_START = now() * 1000

Description of Log Fields

Hook: [0fc76e94:  > before() ./kong/runloop/handler.lua:1183]
      |           |  '- <function name> <file name>:<source line> 
      |           |
      |           '- > function call
      |           '- < function return
      |
      '- request id from nginx variable $request_id

result from nginx error.log

$ pwd
/Users/.../work/dev/kong/bazel-bin/build/kong-dev/kong/servroot
$ tail -f logs/error.log |grep Hook
Hook: [0fc76e94:  > before() ./kong/runloop/handler.lua:1183]
Hook: [0fc76e94:   > router() ./kong/tracing/instrumentation.lua:33]
Hook: [0fc76e94:   < router() ./kong/tracing/instrumentation.lua:33]
Hook: [0fc76e94:   > get_updated_router() ./kong/runloop/handler.lua:466]
Hook: [0fc76e94:    > __index() ./kong/pdk/init.lua:242]
Hook: [0fc76e94:    < __index() ./kong/pdk/init.lua:243]
Hook: [0fc76e94:   < get_updated_router() ./kong/runloop/handler.lua:475]
Hook: [0fc76e94:   > exec() ./kong/router/atc.lua:630]
Hook: [0fc76e94:    > is_empty_field() ./kong/router/atc.lua:92]
Hook: [0fc76e94:    < is_empty_field() ./kong/router/atc.lua:93]
Hook: [0fc76e94:    > is_empty_field() ./kong/router/atc.lua:92]
Hook: [0fc76e94:    < is_empty_field() ./kong/router/atc.lua:93]
Hook: [0fc76e94:    > strip_uri_args() ./kong/router/utils.lua:55]
Hook: [0fc76e94:     > strip_uri_args() ./kong/tools/uri.lua:78]
Hook: [0fc76e94:     < strip_uri_args() ./kong/tools/uri.lua:100]
Hook: [0fc76e94:     > route_match_stat() ./kong/router/utils.lua:240]
Hook: [0fc76e94:     < route_match_stat() ./kong/router/utils.lua:244]
Hook: [0fc76e94:     > add_debug_headers() ./kong/router/utils.lua:105]
Hook: [0fc76e94:     < add_debug_headers() ./kong/router/utils.lua:107]
Hook: [0fc76e94:   < exec() ./kong/router/atc.lua:697]
Hook: [0fc76e94:   > is_trusted() ./kong/pdk/ip.lua:64]
Hook: [0fc76e94:   < is_trusted() ./kong/pdk/ip.lua:64]
Hook: [0fc76e94:   > __index() ./kong/db/schema/init.lua:1474]
Hook: [0fc76e94:   < __index() ./kong/db/schema/init.lua:1477]
Hook: [0fc76e94:   > __index() ./kong/db/schema/init.lua:1474]
Hook: [0fc76e94:   < __index() ./kong/db/schema/init.lua:1477]
Hook: [0fc76e94:   > __index() ./kong/db/schema/init.lua:1474]
Hook: [0fc76e94:   < __index() ./kong/db/schema/init.lua:1477]
Hook: [0fc76e94:   > __index() ./kong/db/schema/init.lua:1474]
Hook: [0fc76e94:   < __index() ./kong/db/schema/init.lua:1477]
Hook: [0fc76e94:   > balancer_prepare() ./kong/runloop/handler.lua:774]
Hook: [0fc76e94:    > sleep_once_for_balancer_init() ./kong/runloop/handler.lua:71]
Hook: [0fc76e94:    < sleep_once_for_balancer_init() ./kong/runloop/handler.lua:71]
Hook: [0fc76e94:   < balancer_prepare() ./kong/runloop/handler.lua:878]
Hook: [0fc76e94:   > escape() ./kong/tools/uri.lua:69]
Hook: [0fc76e94:   < escape() ./kong/tools/uri.lua:74]
Hook: [0fc76e94:  < before() ./kong/runloop/handler.lua:1383]
Hook: [0fc76e94:  > get_plugins_iterator() ./kong/runloop/handler.lua:565]
Hook: [0fc76e94:  < get_plugins_iterator() ./kong/runloop/handler.lua:566]
Hook: [0fc76e94:  > execute_collecting_plugins_iterator() ./kong/init.lua:331]
Hook: [0fc76e94:   > get_collecting_iterator() ./kong/runloop/plugins_iterator.lua:357]
Hook: [0fc76e94:    > get_workspace() ./kong/runloop/plugins_iterator.lua:250]
Hook: [0fc76e94:     > get_workspace_id() ./kong/workspaces/init.lua:42]
Hook: [0fc76e94:     < get_workspace_id() ./kong/workspaces/init.lua:48]
Hook: [0fc76e94:    < get_workspace() ./kong/runloop/plugins_iterator.lua:255]
Hook: [0fc76e94:    > get_table_for_ctx() ./kong/runloop/plugins_iterator.lua:70]
Hook: [0fc76e94:    < get_table_for_ctx() ./kong/runloop/plugins_iterator.lua:85]
Hook: [0fc76e94:   < get_collecting_iterator() ./kong/runloop/plugins_iterator.lua:370]
Hook: [0fc76e94:   > (for generator)() ./kong/runloop/plugins_iterator.lua:317]
Hook: [0fc76e94:    > load_configuration_through_combos() ./kong/runloop/plugins_iterator.lua:238]
Hook: [0fc76e94:     > load_configuration_through_combos() ./kong/runloop/plugins_iterator.lua:174]
Hook: [0fc76e94:      > build_compound_key() ./kong/runloop/plugins_iterator.lua:66]
Hook: [0fc76e94:       > build_compound_key() ./kong/runloop/plugins_iterator.lua:66]
Hook: [0fc76e94:        > build_compound_key() ./kong/runloop/plugins_iterator.lua:66]
Hook: [0fc76e94:     < load_configuration_through_combos() ./kong/runloop/plugins_iterator.lua:220]
Hook: [0fc76e94:     > update() ./kong/pdk/vault.lua:1200]
Hook: [0fc76e94:      > update() ./kong/pdk/vault.lua:675]
Hook: [0fc76e94:       > update() ./kong/pdk/vault.lua:675]
Hook: [0fc76e94:       < update() ./kong/pdk/vault.lua:693]
Hook: [0fc76e94:      < update() ./kong/pdk/vault.lua:693]
Hook: [0fc76e94:   < (for generator)() ./kong/runloop/plugins_iterator.lua:349]
Hook: [0fc76e94:   > plugin_access() ./kong/tracing/instrumentation.lua:33]
Hook: [0fc76e94:   < plugin_access() ./kong/tracing/instrumentation.lua:33]
Hook: [0fc76e94:   > setup_plugin_context() ./kong/init.lua:254]
Hook: [0fc76e94:    > set_named_ctx() ./kong/global.lua:85]
Hook: [0fc76e94:     > set_namespace() ./kong/global.lua:45]
Hook: [0fc76e94:      > get_namespaces() ./kong/global.lua:29]
Hook: [0fc76e94:      < get_namespaces() ./kong/global.lua:41]
Hook: [0fc76e94:     < set_namespace() ./kong/global.lua:54]
Hook: [0fc76e94:    < set_named_ctx() ./kong/global.lua:107]
Hook: [0fc76e94:    > set_namespaced_log() ./kong/global.lua:131]
Hook: [0fc76e94:    < set_namespaced_log() ./kong/global.lua:147]
Hook: [0fc76e94:   < setup_plugin_context() ./kong/init.lua:262]
Hook: [0fc76e94:   > create() coroutine_api:7]
Hook: [0fc76e94:    > resume() coroutine_api:7]
Hook: [0fc76e94:     > do_authentication() ./kong/plugins/key-auth/handler.lua:103]
Hook: [0fc76e94:      > get_headers() ./kong/pdk/request.lua:680]
Hook: [0fc76e94:       > check_phase() ./kong/pdk/private/phases.lua:66]
Hook: [0fc76e94:       < check_phase() ./kong/pdk/private/phases.lua:84]
Hook: [0fc76e94:       > get_headers() ./kong/globalpatches.lua:148]
Hook: [0fc76e94:        > __index() ./kong/pdk/init.lua:242]
Hook: [0fc76e94:        < __index() ./kong/pdk/init.lua:243]
Hook: [0fc76e94:        > get_headers() ./kong/globalpatches.lua:140]
Hook: [0fc76e94:        < get_headers() ./kong/globalpatches.lua:145]
Hook: [0fc76e94:        > get_query() ./kong/pdk/request.lua:568]
Hook: [0fc76e94:         > check_phase() ./kong/pdk/private/phases.lua:66]
Hook: [0fc76e94:         < check_phase() ./kong/pdk/private/phases.lua:84]
Hook: [0fc76e94:         > get_uri_args() ./kong/globalpatches.lua:188]
Hook: [0fc76e94:          > __index() ./kong/pdk/init.lua:242]
Hook: [0fc76e94:          < __index() ./kong/pdk/init.lua:243]
Hook: [0fc76e94:          > get_uri_args() ./kong/globalpatches.lua:180]
Hook: [0fc76e94:          < get_uri_args() ./kong/globalpatches.lua:185]
Hook: [0fc76e94:        < get_query() ./kong/pdk/request.lua:607]
Hook: [0fc76e94:        > __index() ./kong/db/init.lua:29]
Hook: [0fc76e94:        < __index() ./kong/db/init.lua:30]
Hook: [0fc76e94:        > cache_key() ./kong/db/dao/init.lua:1497]
Hook: [0fc76e94:         > get_workspace_id() ./kong/workspaces/init.lua:42]
Hook: [0fc76e94:         < get_workspace_id() ./kong/workspaces/init.lua:48]
Hook: [0fc76e94:         > get() ./kong/cache/init.lua:158]
Hook: [0fc76e94:          > get() ./kong/resty/mlcache/init.lua:724]
Hook: [0fc76e94:          < get() ./kong/resty/mlcache/init.lua:737]
Hook: [0fc76e94:         < get() ./kong/cache/init.lua:168]
Hook: [0fc76e94:         > __index() ./kong/pdk/init.lua:275]
Hook: [0fc76e94:         < __index() ./kong/pdk/init.lua:280]
Hook: [0fc76e94:         > debug() ./kong/pdk/log.lua:286]
Hook: [0fc76e94:         < debug() ./kong/pdk/log.lua:298]
Hook: [0fc76e94:     < do_authentication() ./kong/plugins/key-auth/handler.lua:191]
Hook: [0fc76e94:     > check_phase() ./kong/pdk/private/phases.lua:66]
Hook: [0fc76e94:     < check_phase() ./kong/pdk/private/phases.lua:84]
Hook: [0fc76e94:     > is_grpc_request() ./kong/pdk/response.lua:660]
Hook: [0fc76e94:     < is_grpc_request() ./kong/pdk/response.lua:664]
Hook: [0fc76e94:     > get_headers() ./kong/globalpatches.lua:140]
Hook: [0fc76e94:     < get_headers() ./kong/globalpatches.lua:145]
Hook: [0fc76e94:     > get_response_type() ./kong/tools/utils.lua:1475]
Hook: [0fc76e94:      > strip() ./kong/tools/utils.lua:82]
Hook: [0fc76e94:       > get_mime_type() ./kong/tools/utils.lua:1513]
Hook: [0fc76e94:        > strip() ./kong/tools/utils.lua:82]
Hook: [0fc76e94:       < get_mime_type() ./kong/tools/utils.lua:1533]
Hook: [0fc76e94:     < get_response_type() ./kong/tools/utils.lua:1509]
Hook: [0fc76e94:     > get_error_template() ./kong/tools/utils.lua:1540]
Hook: [0fc76e94:     < get_error_template() ./kong/tools/utils.lua:1542]
Hook: [0fc76e94:     > yield() coroutine_api:7]
Hook: [0fc76e94:      > reset_plugin_context() ./kong/init.lua:265]
Hook: [0fc76e94:       > reset_log() ./kong/global.lua:150]
Hook: [0fc76e94:       < reset_log() ./kong/global.lua:156]
Hook: [0fc76e94:      < reset_plugin_context() ./kong/init.lua:271]
Hook: [0fc76e94:      > (for generator)() ./kong/runloop/plugins_iterator.lua:317]
Hook: [0fc76e94:      < (for generator)() ./kong/runloop/plugins_iterator.lua:322]
Hook: [0fc76e94:  < execute_collecting_plugins_iterator() ./kong/init.lua:382]
Hook: [0fc76e94:  > get_updated_now_ms() ./kong/tools/utils.lua:1799]
Hook: [0fc76e94:  < get_updated_now_ms() ./kong/tools/utils.lua:1801]
Hook: [0fc76e94:  > access() ./kong/init.lua:429]
Hook: [0fc76e94:   > delayed_response_callback() ./kong/pdk/response.lua:827]
Hook: [0fc76e94:    > delayed_response_callback() ./kong/pdk/response.lua:667]
Hook: [0fc76e94:     > normalize_multi_header() ./kong/pdk/private/checks.lua:13]
Hook: [0fc76e94:     < normalize_multi_header() ./kong/pdk/private/checks.lua:17]
Hook: [0fc76e94:     > normalize_multi_header() ./kong/pdk/private/checks.lua:13]
Hook: [0fc76e94:     < normalize_multi_header() ./kong/pdk/private/checks.lua:17]
Hook: [0fc76e94:     > header_filter() ./kong/init.lua:1356]
Hook: [0fc76e94:      > before() ./kong/runloop/handler.lua:1481]
Hook: [0fc76e94:       > runloop_before_header_filter() ./kong/tracing/instrumentation.lua:299]
Hook: [0fc76e94:       < runloop_before_header_filter() ./kong/tracing/instrumentation.lua:306]
Hook: [0fc76e94:      < before() ./kong/runloop/handler.lua:1484]
Hook: [0fc76e94:      > get_plugins_iterator() ./kong/runloop/handler.lua:565]
Hook: [0fc76e94:      < get_plugins_iterator() ./kong/runloop/handler.lua:566]
Hook: [0fc76e94:      > execute_collected_plugins_iterator() ./kong/init.lua:385]
Hook: [0fc76e94:       > get_collected_iterator() ./kong/runloop/plugins_iterator.lua:302]
Hook: [0fc76e94:       < get_collected_iterator() ./kong/runloop/plugins_iterator.lua:307]
Hook: [0fc76e94:      < execute_collected_plugins_iterator() ./kong/init.lua:392]
Hook: [0fc76e94:      > after() ./kong/runloop/handler.lua:1539]
Hook: [0fc76e94:       > __index() ./kong/pdk/init.lua:242]
Hook: [0fc76e94:       < __index() ./kong/pdk/init.lua:243]
Hook: [0fc76e94:      < after() ./kong/runloop/handler.lua:1572]
Hook: [0fc76e94:      > get_updated_now_ms() ./kong/tools/utils.lua:1799]
Hook: [0fc76e94:      < get_updated_now_ms() ./kong/tools/utils.lua:1801]
Hook: [0fc76e94:     < header_filter() ./kong/init.lua:1425]
Hook: [0fc76e94:     > body_filter() ./kong/init.lua:1428]
Hook: [0fc76e94:      > get_plugins_iterator() ./kong/runloop/handler.lua:565]
Hook: [0fc76e94:      < get_plugins_iterator() ./kong/runloop/handler.lua:566]
Hook: [0fc76e94:      > execute_collected_plugins_iterator() ./kong/init.lua:385]
Hook: [0fc76e94:       > get_collected_iterator() ./kong/runloop/plugins_iterator.lua:302]
Hook: [0fc76e94:       < get_collected_iterator() ./kong/runloop/plugins_iterator.lua:307]
Hook: [0fc76e94:      < execute_collected_plugins_iterator() ./kong/init.lua:392]
Hook: [0fc76e94:     < body_filter() ./kong/init.lua:1485]
Hook: [0fc76e94:     > body_filter() ./kong/init.lua:1428]
Hook: [0fc76e94:      > get_plugins_iterator() ./kong/runloop/handler.lua:565]
Hook: [0fc76e94:      < get_plugins_iterator() ./kong/runloop/handler.lua:566]
Hook: [0fc76e94:      > execute_collected_plugins_iterator() ./kong/init.lua:385]
Hook: [0fc76e94:       > get_collected_iterator() ./kong/runloop/plugins_iterator.lua:302]
Hook: [0fc76e94:       < get_collected_iterator() ./kong/runloop/plugins_iterator.lua:307]
Hook: [0fc76e94:      < execute_collected_plugins_iterator() ./kong/init.lua:392]
Hook: [0fc76e94:      > get_updated_now_ms() ./kong/tools/utils.lua:1799]
Hook: [0fc76e94:      < get_updated_now_ms() ./kong/tools/utils.lua:1801]
Hook: [0fc76e94:      > time_ns() ./kong/tools/utils.lua:1687]
Hook: [0fc76e94:      < time_ns() ./kong/tools/utils.lua:1692]
Hook: [0fc76e94:     < body_filter() ./kong/init.lua:1502]
Hook: [0fc76e94:     > log() ./kong/init.lua:1505]
Hook: [0fc76e94:      > before() ./kong/runloop/handler.lua:1575]
Hook: [0fc76e94:       > runloop_log_before() ./kong/tracing/instrumentation.lua:309]
Hook: [0fc76e94:        > balancer() ./kong/tracing/instrumentation.lua:33]
Hook: [0fc76e94:        < balancer() ./kong/tracing/instrumentation.lua:33]
Hook: [0fc76e94:        > active_span() ./kong/pdk/tracing.lua:34]
Hook: [0fc76e94:        < active_span() ./kong/pdk/tracing.lua:34]
Hook: [0fc76e94:       < runloop_log_before() ./kong/tracing/instrumentation.lua:320]
Hook: [0fc76e94:      < before() ./kong/runloop/handler.lua:1577]
Hook: [0fc76e94:      > get_plugins_iterator() ./kong/runloop/handler.lua:565]
Hook: [0fc76e94:      < get_plugins_iterator() ./kong/runloop/handler.lua:566]
Hook: [0fc76e94:      > execute_collected_plugins_iterator() ./kong/init.lua:385]
Hook: [0fc76e94:       > get_collected_iterator() ./kong/runloop/plugins_iterator.lua:302]
Hook: [0fc76e94:       < get_collected_iterator() ./kong/runloop/plugins_iterator.lua:307]
Hook: [0fc76e94:      < execute_collected_plugins_iterator() ./kong/init.lua:392]
Hook: [0fc76e94:      > release() ./kong/runloop/plugins_iterator.lua:88]
Hook: [0fc76e94:      < release() ./kong/runloop/plugins_iterator.lua:94]
Hook: [0fc76e94:      > after() ./kong/runloop/handler.lua:1578]
Hook: [0fc76e94:       > runloop_log_after() ./kong/tracing/instrumentation.lua:357]
Hook: [0fc76e94:       < runloop_log_after() ./kong/tracing/instrumentation.lua:373]
Hook: [0fc76e94:       > update_lua_mem() ./kong/runloop/handler.lua:140]
Hook: [0fc76e94:       < update_lua_mem() ./kong/runloop/handler.lua:153]
Hook: [0fc76e94:       > __index() ./kong/pdk/init.lua:242]
Hook: [0fc76e94:       < __index() ./kong/pdk/init.lua:243]
Hook: [0fc76e94:       > log() ./kong/reports.lua:475]
Hook: [0fc76e94:        > incr_counter() ./kong/reports.lua:236]
Hook: [0fc76e94:        < incr_counter() ./kong/reports.lua:241]
Hook: [0fc76e94:        > get_current_suffix() ./kong/reports.lua:250]
Hook: [0fc76e94:        < get_current_suffix() ./kong/reports.lua:272]
Hook: [0fc76e94:        > incr_counter() ./kong/reports.lua:236]
Hook: [0fc76e94:        < incr_counter() ./kong/reports.lua:241]
Hook: [0fc76e94:        > incr_counter() ./kong/reports.lua:236]
Hook: [0fc76e94:        < incr_counter() ./kong/reports.lua:241]
Hook: [0fc76e94:       < log() ./kong/reports.lua:502]
Hook: [0fc76e94:      < after() ./kong/runloop/handler.lua:1588]

@chobits
Copy link
Author

chobits commented Sep 5, 2023

  • TODO: Some inline C functions (or other situation) may not trigger a return event. We are actively working to address this issue within the call chain to ensure a balanced call/return depth level. However, especially when there are consecutive calls to inline functions, which makes call depth level increases.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment