Demonstrates why hook_custom_theme() never seems to apply, and why to use hook_menu_get_item().

menu.inc-exerpt.php

<?php
    /**
     * Excerpt from menu.inc
     * Lines 1745-1755
     */
  
    // First allow modules to dynamically set a custom theme for the current
    // page. Since we can only have one, the last module to return a valid
    // theme takes precedence.
    $custom_themes = array_filter(module_invoke_all('custom_theme'), 'drupal_theme_access');
    if (!empty($custom_themes)) {
      $custom_theme = array_pop($custom_themes);
    }
    // If there is a theme callback function for the current page, execute it.
    // If this returns a valid theme, it will override any theme that was set
    // by a hook_custom_theme() implementation above.
    $router_item = menu_get_item();