Calls into skb:kfree_skb

README.md

Calls into skb:kfree_skb

As requested in netdev, this gist contains the stacks leading into skb:kfree_skb:

• https://lore.kernel.org/netdev/[email protected]/

The easiest way to look at the results is the flamegraphs, where reason is the top frame.

The results are from v6.1.38.

Normal (flamegraph.normal.svg + stacks.normal.txt)

We see ~10k calls per second during normal operation on this machine:

$ sudo perf record -a -g --kernel-callchains -e skb:kfree_skb -- sleep 1
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 8.470 MB perf.data (9373 samples) ]

$ sudo perf script | sed -e 's/skb:kfree_skb:.*reason:\(.*\)/\n\tfffff \1 (unknown)/' -e 's/^\(\w\+\)\s\+/kernel /' | /usr/bin/inferno-collapse-perf --all | perl -pe 's/.*?;//' | sed -e 's/.*irq_exit_rcu_\[k\];/irq_exit_rcu_[k];/' | /usr/bin/inferno-flamegraph --truncate-text-right --colors=java --hash --title=normal --width=1440 --minwidth=0.005 > flamegraph.normal.svg

$ sudo /usr/share/bcc/tools/stackcount -s -K -D 1 t:skb:kfree_skb > stacks.normal.txt

Spike (flamegraph.spike.svg + stacks.spike.txt)

Spikes up to 500k calls per second can happen from time to time:

$ sudo perf record -a -g --kernel-callchains -e skb:kfree_skb -- sleep 1
[ perf record: Woken up 172 times to write data ]
Warning:
4 out of order events recorded.
[ perf record: Captured and wrote 124.722 MB perf.data (398819 samples) ]

$ sudo perf script | sed -e 's/skb:kfree_skb:.*reason:\(.*\)/\n\tfffff \1 (unknown)/' -e 's/^\(\w\+\)\s\+/kernel /' | /usr/bin/inferno-collapse-perf --all | perl -pe 's/.*?;//' | sed -e 's/.*irq_exit_rcu_\[k\];/irq_exit_rcu_[k];/' | /usr/bin/inferno-flamegraph --truncate-text-right --colors=java --hash --title=spike --width=1440 --minwidth=0.005 > flamegraph.spike.svg

$ sudo /usr/share/bcc/tools/stackcount -s -K -D 1 t:skb:kfree_skb > stacks.spike.txt

flamegraph.normal.packet_rcv.aggregated.svg

flamegraph.normal.svg

flamegraph.spike.packet_rcv.aggregated.svg

flamegraph.spike.svg

<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg version="1.1" width="1440" height="982" onload="init(evt)" viewBox="0 0 1440 982" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:fg="http://github.com/jonhoo/inferno"><!--Flame graph stack visualization. See https://github.com/brendangregg/FlameGraph for latest version, and http://www.brendangregg.com/flamegraphs.html for examples.--><!--NOTES: --><defs><linearGradient id="background" y1="0" y2="1" x1="0" x2="0"><stop stop-color="#eeeeee" offset="5%"/><stop stop-color="#eeeeb0" offset="95%"/></linearGradient></defs><style type="text/css">
text { font-family:"Verdana"; font-size:12px; fill:rgb(0,0,0); }
#title { text-anchor:middle; font-size:17px; }
#search { opacity:0.1; cursor:pointer; }
#search:hover, #search.show { opacity:1; }
#subtitle { text-anchor:middle; font-color:rgb(160,160,160); }
#unzoom { cursor:pointer; }
#frames > *:hover { stroke:black; stroke-width:0.5; cursor:pointer; }
.hide { display:none; }
.parent { opacity:0.5; }
</style><script type="text/ecmascript"><![CDATA[
        var nametype = 'Function:';
        var fontsize = 12;
        var fontwidth = 0.59;
        var xpad = 10;
        var inverted = false;
        var searchcolor = 'rgb(230,0,230)';
        var fluiddrawing = false;
        var truncate_text_right = true;
    ]]><![CDATA["use strict";
var details, searchbtn, unzoombtn, matchedtxt, svg, searching, frames;
function init(evt) {
    details = document.getElementById("details").firstChild;
    searchbtn = document.getElementById("search");
    unzoombtn = document.getElementById("unzoom");
    matchedtxt = document.getElementById("matched");
    svg = document.getElementsByTagName("svg")[0];
    frames = document.getElementById("frames");
    total_samples = parseInt(frames.attributes.total_samples.value);
    searching = 0;

    // Use GET parameters to restore a flamegraph's state.
    var restore_state = function() {
        var params = get_params();
        if (params.x && params.y)
            zoom(find_group(document.querySelector('[*|x="' + params.x + '"][y="' + params.y + '"]')));
        if (params.s)
            search(params.s);
    };

    if (fluiddrawing) {
        // Make width dynamic so the SVG fits its parent's width.
        svg.removeAttribute("width");
        // Edge requires us to have a viewBox that gets updated with size changes.
        var isEdge = /Edge\/\d./i.test(navigator.userAgent);
        if (!isEdge) {
          svg.removeAttribute("viewBox");
        }
        var update_for_width_change = function() {
            if (isEdge) {
                svg.attributes.viewBox.value = "0 0 " + svg.width.baseVal.value + " " + svg.height.baseVal.value;
            }

            // Keep consistent padding on left and right of frames container.
            frames.attributes.width.value = svg.width.baseVal.value - xpad * 2;

            // Text truncation needs to be adjusted for the current width.
            var el = frames.children;
            for(var i = 0; i < el.length; i++) {
                update_text(el[i]);
            }

            // Keep search elements at a fixed distance from right edge.
            var svgWidth = svg.width.baseVal.value;
            searchbtn.attributes.x.value = svgWidth - xpad - 100;
            matchedtxt.attributes.x.value = svgWidth - xpad - 100;
        };
        window.addEventListener('resize', function() {
            update_for_width_change();
        });
        // This needs to be done asynchronously for Safari to work.
        setTimeout(function() {
            unzoom();
            update_for_width_change();
            restore_state();
        }, 0);
    } else {
        restore_state();
    }
}
// event listeners
window.addEventListener("click", function(e) {
    var target = find_group(e.target);
    if (target) {
        if (target.nodeName == "a") {
            if (e.ctrlKey === false) return;
            e.preventDefault();
        }
        if (target.classList.contains("parent")) unzoom();
        zoom(target);

        // set parameters for zoom state
        var el = target.querySelector("rect");
        if (el && el.attributes && el.attributes.y && el.attributes["fg:x"]) {
            var params = get_params()
            params.x = el.attributes["fg:x"].value;
            params.y = el.attributes.y.value;
            history.replaceState(null, null, parse_params(params));
        }
    }
    else if (e.target.id == "unzoom") {
        unzoom();

        // remove zoom state
        var params = get_params();
        if (params.x) delete params.x;
        if (params.y) delete params.y;
        history.replaceState(null, null, parse_params(params));
    }
    else if (e.target.id == "search") search_prompt();
}, false)
// mouse-over for info
// show
window.addEventListener("mouseover", function(e) {
    var target = find_group(e.target);
    if (target) details.nodeValue = nametype + " " + g_to_text(target);
}, false)
// clear
window.addEventListener("mouseout", function(e) {
    var target = find_group(e.target);
    if (target) details.nodeValue = ' ';
}, false)
// ctrl-F for search
window.addEventListener("keydown",function (e) {
    if (e.keyCode === 114 || (e.ctrlKey && e.keyCode === 70)) {
        e.preventDefault();
        search_prompt();
    }
}, false)
// functions
function get_params() {
    var params = {};
    var paramsarr = window.location.search.substr(1).split('&');
    for (var i = 0; i < paramsarr.length; ++i) {
        var tmp = paramsarr[i].split("=");
        if (!tmp[0] || !tmp[1]) continue;
        params[tmp[0]]  = decodeURIComponent(tmp[1]);
    }
    return params;
}
function parse_params(params) {
    var uri = "?";
    for (var key in params) {
        uri += key + '=' + encodeURIComponent(params[key]) + '&';
    }
    if (uri.slice(-1) == "&")
        uri = uri.substring(0, uri.length - 1);
    if (uri == '?')
        uri = window.location.href.split('?')[0];
    return uri;
}
function find_child(node, selector) {
    var children = node.querySelectorAll(selector);
    if (children.length) return children[0];
    return;
}
function find_group(node) {
    var parent = node.parentElement;
    if (!parent) return;
    if (parent.id == "frames") return node;
    return find_group(parent);
}
function orig_save(e, attr, val) {
    if (e.attributes["fg:orig_" + attr] != undefined) return;
    if (e.attributes[attr] == undefined) return;
    if (val == undefined) val = e.attributes[attr].value;
    e.setAttribute("fg:orig_" + attr, val);
}
function orig_load(e, attr) {
    if (e.attributes["fg:orig_"+attr] == undefined) return;
    e.attributes[attr].value = e.attributes["fg:orig_" + attr].value;
    e.removeAttribute("fg:orig_" + attr);
}
function g_to_text(e) {
    var text = find_child(e, "title").firstChild.nodeValue;
    return (text)
}
function g_to_func(e) {
    var func = g_to_text(e);
    // if there's any manipulation we want to do to the function
    // name before it's searched, do it here before returning.
    return (func);
}
function update_text(e) {
    var r = find_child(e, "rect");
    var t = find_child(e, "text");
    var w = parseFloat(r.attributes.width.value) * frames.attributes.width.value / 100 - 3;
    var txt = find_child(e, "title").textContent.replace(/\([^(]*\)$/,"");
    t.attributes.x.value = format_percent((parseFloat(r.attributes.x.value) + (100 * 3 / frames.attributes.width.value)));
    // Smaller than this size won't fit anything
    if (w < 2 * fontsize * fontwidth) {
        t.textContent = "";
        return;
    }
    t.textContent = txt;
    // Fit in full text width
    if (/^ *\$/.test(txt) || t.getComputedTextLength() < w)
        return;
    if (truncate_text_right) {
        // Truncate the right side of the text.
        for (var x = txt.length - 2; x > 0; x--) {
            if (t.getSubStringLength(0, x + 2) <= w) {
                t.textContent = txt.substring(0, x) + "..";
                return;
            }
        }
    } else {
        // Truncate the left side of the text.
        for (var x = 2; x < txt.length; x++) {
            if (t.getSubStringLength(x - 2, txt.length) <= w) {
                t.textContent = ".." + txt.substring(x, txt.length);
                return;
            }
        }
    }
    t.textContent = "";
}
// zoom
function zoom_reset(e) {
    if (e.tagName == "rect") {
        e.attributes.x.value = format_percent(100 * parseInt(e.attributes["fg:x"].value) / total_samples);
        e.attributes.width.value = format_percent(100 * parseInt(e.attributes["fg:w"].value) / total_samples);
    }
    if (e.childNodes == undefined) return;
    for(var i = 0, c = e.childNodes; i < c.length; i++) {
        zoom_reset(c[i]);
    }
}
function zoom_child(e, x, zoomed_width_samples) {
    if (e.tagName == "text") {
        var parent_x = parseFloat(find_child(e.parentNode, "rect[x]").attributes.x.value);
        e.attributes.x.value = format_percent(parent_x + (100 * 3 / frames.attributes.width.value));
    } else if (e.tagName == "rect") {
        e.attributes.x.value = format_percent(100 * (parseInt(e.attributes["fg:x"].value) - x) / zoomed_width_samples);
        e.attributes.width.value = format_percent(100 * parseInt(e.attributes["fg:w"].value) / zoomed_width_samples);
    }
    if (e.childNodes == undefined) return;
    for(var i = 0, c = e.childNodes; i < c.length; i++) {
        zoom_child(c[i], x, zoomed_width_samples);
    }
}
function zoom_parent(e) {
    if (e.attributes) {
        if (e.attributes.x != undefined) {
            e.attributes.x.value = "0.0%";
        }
        if (e.attributes.width != undefined) {
            e.attributes.width.value = "100.0%";
        }
    }
    if (e.childNodes == undefined) return;
    for(var i = 0, c = e.childNodes; i < c.length; i++) {
        zoom_parent(c[i]);
    }
}
function zoom(node) {
    var attr = find_child(node, "rect").attributes;
    var width = parseInt(attr["fg:w"].value);
    var xmin = parseInt(attr["fg:x"].value);
    var xmax = xmin + width;
    var ymin = parseFloat(attr.y.value);
    unzoombtn.classList.remove("hide");
    var el = frames.children;
    for (var i = 0; i < el.length; i++) {
        var e = el[i];
        var a = find_child(e, "rect").attributes;
        var ex = parseInt(a["fg:x"].value);
        var ew = parseInt(a["fg:w"].value);
        // Is it an ancestor
        if (!inverted) {
            var upstack = parseFloat(a.y.value) > ymin;
        } else {
            var upstack = parseFloat(a.y.value) < ymin;
        }
        if (upstack) {
            // Direct ancestor
            if (ex <= xmin && (ex+ew) >= xmax) {
                e.classList.add("parent");
                zoom_parent(e);
                update_text(e);
            }
            // not in current path
            else
                e.classList.add("hide");
        }
        // Children maybe
        else {
            // no common path
            if (ex < xmin || ex >= xmax) {
                e.classList.add("hide");
            }
            else {
                zoom_child(e, xmin, width);
                update_text(e);
            }
        }
    }
}
function unzoom() {
    unzoombtn.classList.add("hide");
    var el = frames.children;
    for(var i = 0; i < el.length; i++) {
        el[i].classList.remove("parent");
        el[i].classList.remove("hide");
        zoom_reset(el[i]);
        update_text(el[i]);
    }
}
// search
function reset_search() {
    var el = document.querySelectorAll("#frames rect");
    for (var i = 0; i < el.length; i++) {
        orig_load(el[i], "fill")
    }
    var params = get_params();
    delete params.s;
    history.replaceState(null, null, parse_params(params));
}
function search_prompt() {
    if (!searching) {
        var term = prompt("Enter a search term (regexp " +
            "allowed, eg: ^ext4_)", "");
        if (term != null) {
            search(term)
        }
    } else {
        reset_search();
        searching = 0;
        searchbtn.classList.remove("show");
        searchbtn.firstChild.nodeValue = "Search"
        matchedtxt.classList.add("hide");
        matchedtxt.firstChild.nodeValue = ""
    }
}
function search(term) {
    var re = new RegExp(term);
    var el = frames.children;
    var matches = new Object();
    var maxwidth = 0;
    for (var i = 0; i < el.length; i++) {
        var e = el[i];
        // Skip over frames which are either not visible, or below the zoomed-to frame
        if (e.classList.contains("hide") || e.classList.contains("parent")) {
            continue;
        }
        var func = g_to_func(e);
        var rect = find_child(e, "rect");
        if (func == null || rect == null)
            continue;
        // Save max width. Only works as we have a root frame
        var w = parseInt(rect.attributes["fg:w"].value);
        if (w > maxwidth)
            maxwidth = w;
        if (func.match(re)) {
            // highlight
            var x = parseInt(rect.attributes["fg:x"].value);
            orig_save(rect, "fill");
            rect.attributes.fill.value = searchcolor;
            // remember matches
            if (matches[x] == undefined) {
                matches[x] = w;
            } else {
                if (w > matches[x]) {
                    // overwrite with parent
                    matches[x] = w;
                }
            }
            searching = 1;
        }
    }
    if (!searching)
        return;
    var params = get_params();
    params.s = term;
    history.replaceState(null, null, parse_params(params));

    searchbtn.classList.add("show");
    searchbtn.firstChild.nodeValue = "Reset Search";
    // calculate percent matched, excluding vertical overlap
    var count = 0;
    var lastx = -1;
    var lastw = 0;
    var keys = Array();
    for (k in matches) {
        if (matches.hasOwnProperty(k))
            keys.push(k);
    }
    // sort the matched frames by their x location
    // ascending, then width descending
    keys.sort(function(a, b){
        return a - b;
    });
    // Step through frames saving only the biggest bottom-up frames
    // thanks to the sort order. This relies on the tree property
    // where children are always smaller than their parents.
    for (var k in keys) {
        var x = parseInt(keys[k]);
        var w = matches[keys[k]];
        if (x >= lastx + lastw) {
            count += w;
            lastx = x;
            lastw = w;
        }
    }
    // display matched percent
    matchedtxt.classList.remove("hide");
    var pct = 100 * count / maxwidth;
    if (pct != 100) pct = pct.toFixed(1);
    matchedtxt.firstChild.nodeValue = "Matched: " + pct + "%";
}
function format_percent(n) {
    return n.toFixed(4) + "%";
}
]]></script><rect x="0" y="0" width="100%" height="982" fill="url(#background)"/><text id="title" x="50.0000%" y="24.00">spike</text><text id="details" x="10" y="965.00"> </text><text id="unzoom" class="hide" x="10" y="24.00">Reset Zoom</text><text id="search" x="1330" y="24.00">Search</text><text id="matched" x="1330" y="965.00"> </text><svg id="frames" x="10" width="1420" total_samples="398819"><g><title>release_sock (24 samples, 0.01%)</title><rect x="0.0005%" y="805" width="0.0060%" height="15" fill="rgb(243,143,0)" fg:x="2" fg:w="24"/><text x="0.2088%" y="815.50"></text></g><g><title>__release_sock (24 samples, 0.01%)</title><rect x="0.0005%" y="789" width="0.0060%" height="15" fill="rgb(228,128,0)" fg:x="2" fg:w="24"/><text x="0.2088%" y="799.50"></text></g><g><title>tcp_v4_do_rcv (24 samples, 0.01%)</title><rect x="0.0005%" y="773" width="0.0060%" height="15" fill="rgb(237,137,0)" fg:x="2" fg:w="24"/><text x="0.2088%" y="783.50"></text></g><g><title>dev_queue_xmit_nit (81 samples, 0.02%)</title><rect x="0.0065%" y="709" width="0.0203%" height="15" fill="rgb(243,143,0)" fg:x="26" fg:w="81"/><text x="0.2149%" y="719.50"></text></g><g><title>packet_rcv (81 samples, 0.02%)</title><rect x="0.0065%" y="693" width="0.0203%" height="15" fill="rgb(234,134,0)" fg:x="26" fg:w="81"/><text x="0.2149%" y="703.50"></text></g><g><title>kfree_skb_reason (81 samples, 0.02%)</title><rect x="0.0065%" y="677" width="0.0203%" height="15" fill="rgb(229,129,0)" fg:x="26" fg:w="81"/><text x="0.2149%" y="687.50"></text></g><g><title>kfree_skb_reason (81 samples, 0.02%)</title><rect x="0.0065%" y="661" width="0.0203%" height="15" fill="rgb(229,129,0)" fg:x="26" fg:w="81"/><text x="0.2149%" y="671.50"></text></g><g><title>NOT_SPECIFIED (81 samples, 0.02%)</title><rect x="0.0065%" y="645" width="0.0203%" height="15" fill="rgb(90,237,90)" fg:x="26" fg:w="81"/><text x="0.2149%" y="655.50"></text></g><g><title>tcp_recvmsg (182 samples, 0.05%)</title><rect x="0.0003%" y="821" width="0.0456%" height="15" fill="rgb(237,137,0)" fg:x="1" fg:w="182"/><text x="0.2086%" y="831.50"></text></g><g><title>tcp_recvmsg_locked (157 samples, 0.04%)</title><rect x="0.0065%" y="805" width="0.0394%" height="15" fill="rgb(237,137,0)" fg:x="26" fg:w="157"/><text x="0.2149%" y="815.50"></text></g><g><title>__tcp_transmit_skb (157 samples, 0.04%)</title><rect x="0.0065%" y="789" width="0.0394%" height="15" fill="rgb(226,126,0)" fg:x="26" fg:w="157"/><text x="0.2149%" y="799.50"></text></g><g><title>__ip_queue_xmit (157 samples, 0.04%)</title><rect x="0.0065%" y="773" width="0.0394%" height="15" fill="rgb(225,125,0)" fg:x="26" fg:w="157"/><text x="0.2149%" y="783.50"></text></g><g><title>ip_finish_output2 (157 samples, 0.04%)</title><rect x="0.0065%" y="757" width="0.0394%" height="15" fill="rgb(230,130,0)" fg:x="26" fg:w="157"/><text x="0.2149%" y="767.50"></text></g><g><title>__dev_queue_xmit (157 samples, 0.04%)</title><rect x="0.0065%" y="741" width="0.0394%" height="15" fill="rgb(230,130,0)" fg:x="26" fg:w="157"/><text x="0.2149%" y="751.50"></text></g><g><title>dev_hard_start_xmit (157 samples, 0.04%)</title><rect x="0.0065%" y="725" width="0.0394%" height="15" fill="rgb(243,143,0)" fg:x="26" fg:w="157"/><text x="0.2149%" y="735.50"></text></g><g><title>vlan_dev_hard_start_xmit (76 samples, 0.02%)</title><rect x="0.0268%" y="709" width="0.0191%" height="15" fill="rgb(223,123,0)" fg:x="107" fg:w="76"/><text x="0.2352%" y="719.50"></text></g><g><title>__dev_queue_xmit (76 samples, 0.02%)</title><rect x="0.0268%" y="693" width="0.0191%" height="15" fill="rgb(230,130,0)" fg:x="107" fg:w="76"/><text x="0.2352%" y="703.50"></text></g><g><title>__qdisc_run (76 samples, 0.02%)</title><rect x="0.0268%" y="677" width="0.0191%" height="15" fill="rgb(229,129,0)" fg:x="107" fg:w="76"/><text x="0.2352%" y="687.50"></text></g><g><title>sch_direct_xmit (76 samples, 0.02%)</title><rect x="0.0268%" y="661" width="0.0191%" height="15" fill="rgb(236,136,0)" fg:x="107" fg:w="76"/><text x="0.2352%" y="671.50"></text></g><g><title>dev_hard_start_xmit (76 samples, 0.02%)</title><rect x="0.0268%" y="645" width="0.0191%" height="15" fill="rgb(243,143,0)" fg:x="107" fg:w="76"/><text x="0.2352%" y="655.50"></text></g><g><title>dev_queue_xmit_nit (76 samples, 0.02%)</title><rect x="0.0268%" y="629" width="0.0191%" height="15" fill="rgb(243,143,0)" fg:x="107" fg:w="76"/><text x="0.2352%" y="639.50"></text></g><g><title>packet_rcv (76 samples, 0.02%)</title><rect x="0.0268%" y="613" width="0.0191%" height="15" fill="rgb(234,134,0)" fg:x="107" fg:w="76"/><text x="0.2352%" y="623.50"></text></g><g><title>kfree_skb_reason (76 samples, 0.02%)</title><rect x="0.0268%" y="597" width="0.0191%" height="15" fill="rgb(229,129,0)" fg:x="107" fg:w="76"/><text x="0.2352%" y="607.50"></text></g><g><title>kfree_skb_reason (76 samples, 0.02%)</title><rect x="0.0268%" y="581" width="0.0191%" height="15" fill="rgb(229,129,0)" fg:x="107" fg:w="76"/><text x="0.2352%" y="591.50"></text></g><g><title>NOT_SPECIFIED (76 samples, 0.02%)</title><rect x="0.0268%" y="565" width="0.0191%" height="15" fill="rgb(90,237,90)" fg:x="107" fg:w="76"/><text x="0.2352%" y="575.50"></text></g><g><title>__napi_poll (20 samples, 0.01%)</title><rect x="0.0499%" y="725" width="0.0050%" height="15" fill="rgb(231,131,0)" fg:x="199" fg:w="20"/><text x="0.2582%" y="735.50"></text></g><g><title>mlx5e_napi_poll (20 samples, 0.01%)</title><rect x="0.0499%" y="709" width="0.0050%" height="15" fill="rgb(221,121,0)" fg:x="199" fg:w="20"/><text x="0.2582%" y="719.50"></text></g><g><title>net_rx_action (24 samples, 0.01%)</title><rect x="0.0499%" y="741" width="0.0060%" height="15" fill="rgb(244,144,0)" fg:x="199" fg:w="24"/><text x="0.2582%" y="751.50"></text></g><g><title>__sys_recvmsg (240 samples, 0.06%)</title><rect x="0.0000%" y="885" width="0.0602%" height="15" fill="rgb(227,127,0)" fg:x="0" fg:w="240"/><text x="0.2083%" y="895.50"></text></g><g><title>___sys_recvmsg (240 samples, 0.06%)</title><rect x="0.0000%" y="869" width="0.0602%" height="15" fill="rgb(223,123,0)" fg:x="0" fg:w="240"/><text x="0.2083%" y="879.50"></text></g><g><title>____sys_recvmsg (240 samples, 0.06%)</title><rect x="0.0000%" y="853" width="0.0602%" height="15" fill="rgb(223,123,0)" fg:x="0" fg:w="240"/><text x="0.2083%" y="863.50"></text></g><g><title>inet_recvmsg (239 samples, 0.06%)</title><rect x="0.0003%" y="837" width="0.0599%" height="15" fill="rgb(239,139,0)" fg:x="1" fg:w="239"/><text x="0.2086%" y="847.50"></text></g><g><title>udp_recvmsg (57 samples, 0.01%)</title><rect x="0.0459%" y="821" width="0.0143%" height="15" fill="rgb(234,134,0)" fg:x="183" fg:w="57"/><text x="0.2542%" y="831.50"></text></g><g><title>__skb_recv_udp (57 samples, 0.01%)</title><rect x="0.0459%" y="805" width="0.0143%" height="15" fill="rgb(227,127,0)" fg:x="183" fg:w="57"/><text x="0.2542%" y="815.50"></text></g><g><title>__local_bh_enable_ip (57 samples, 0.01%)</title><rect x="0.0459%" y="789" width="0.0143%" height="15" fill="rgb(233,133,0)" fg:x="183" fg:w="57"/><text x="0.2542%" y="799.50"></text></g><g><title>do_softirq (57 samples, 0.01%)</title><rect x="0.0459%" y="773" width="0.0143%" height="15" fill="rgb(243,143,0)" fg:x="183" fg:w="57"/><text x="0.2542%" y="783.50"></text></g><g><title>__do_softirq (57 samples, 0.01%)

stacks.normal.txt

stacks.spike.txt