https://stackoverflow.com/q/52185905/3448419

maxdepth.c

#include <jvmti.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>

#define INSIDE_MAIN 0x10000000
static volatile int max_depth = INSIDE_MAIN;

static intptr_t get_current_stack_depth(jvmtiEnv *jvmti) {
    intptr_t depth = 0;
    (*jvmti)->GetThreadLocalStorage(jvmti, NULL, (void**)&depth);
    return (int)depth;
}

static void set_current_stack_depth(jvmtiEnv *jvmti, int depth) {
    (*jvmti)->SetThreadLocalStorage(jvmti, NULL, (const void*)(intptr_t)depth);
}

static int is_main_method(jvmtiEnv *jvmti, jmethodID method) {
    int result = 0;
    char *name;
    char *signature;

    if ((*jvmti)->GetMethodName(jvmti, method, &name, &signature, NULL) == 0) {
        result = strcmp(name, "main") == 0 && strcmp(signature, "([Ljava/lang/String;)V") == 0;
        (*jvmti)->Deallocate(jvmti, (unsigned char*)signature);
        (*jvmti)->Deallocate(jvmti, (unsigned char*)name);
    }

    return result;
}

void JNICALL MethodEntry(jvmtiEnv *jvmti, JNIEnv* jni, jthread thread, jmethodID method) {
    int depth = get_current_stack_depth(jvmti) + 1;
    if (depth == 1 && is_main_method(jvmti, method)) {
        depth += INSIDE_MAIN;
    }
    set_current_stack_depth(jvmti, depth);

    if (depth > max_depth) {
        max_depth = depth;
    }
}

void JNICALL MethodExit(jvmtiEnv *jvmti, JNIEnv* jni, jthread thread, jmethodID method,
                        jboolean was_popped_by_exception, jvalue return_value) {
    int depth = get_current_stack_depth(jvmti) - 1;
    if (depth == INSIDE_MAIN) {
        depth = 0;
    }
    set_current_stack_depth(jvmti, depth);
}

JNIEXPORT jint JNICALL Agent_OnLoad(JavaVM *vm, char *options, void *reserved) {
    jvmtiEnv* jvmti;
    (*vm)->GetEnv(vm, (void**)&jvmti, JVMTI_VERSION_1_0);

    jvmtiCapabilities capabilities = {0};
    capabilities.can_generate_method_entry_events = 1;
    capabilities.can_generate_method_exit_events = 1;
    (*jvmti)->AddCapabilities(jvmti, &capabilities);

    jvmtiEventCallbacks callbacks = {0};
    callbacks.MethodEntry = MethodEntry;
    callbacks.MethodExit = MethodExit;
    (*jvmti)->SetEventCallbacks(jvmti, &callbacks, sizeof(callbacks));

    (*jvmti)->SetEventNotificationMode(jvmti, JVMTI_ENABLE, JVMTI_EVENT_METHOD_ENTRY, NULL);
    (*jvmti)->SetEventNotificationMode(jvmti, JVMTI_ENABLE, JVMTI_EVENT_METHOD_EXIT, NULL);

    return 0;
}

JNIEXPORT void JNICALL Agent_OnUnload(JavaVM *vm) {
    printf("Max stack depth = %d\n", max_depth - INSIDE_MAIN);
}