Use AST to parse Sublime API files

output.txt

sublime
-------
sublime.version()
sublime.platform()
sublime.arch()
sublime.channel()
sublime.executable_path()
sublime.executable_hash()
sublime.packages_path()
sublime.installed_packages_path()
sublime.cache_path()
sublime.status_message()
sublime.error_message()
sublime.message_dialog()
sublime.ok_cancel_dialog()
sublime.run_command()
sublime.get_clipboard()
sublime.set_clipboard()
sublime.log_commands()
sublime.log_input()
sublime.log_result_regex()
sublime.score_selector()
sublime.load_settings()
sublime.save_settings()
sublime.set_timeout()
sublime.set_timeout_async()
sublime.active_window()
sublime.windows()
class sublime.Window()
    Window.__eq__()
    Window.__init__()
    Window.active_group()
    Window.active_view()
    Window.active_view_in_group()
    Window.create_output_panel()
    Window.find_open_file()
    Window.focus_group()
    Window.focus_view()
    Window.folders()
    Window.get_layout()
    Window.get_output_panel()
    Window.get_view_index()
    Window.hwnd()
    Window.id()
    Window.layout()
    Window.lookup_symbol_in_index()
    Window.lookup_symbol_in_open_files()
    Window.new_file()
    Window.num_groups()
    Window.open_file()
    Window.project_data()
    Window.project_file_name()
    Window.run_command()
    Window.set_layout()
    Window.set_project_data()
    Window.set_view_index()
    Window.settings()
    Window.show_input_panel()
    Window.show_quick_panel()
    Window.template_settings()
    Window.transient_view_in_group()
    Window.views()
    Window.views_in_group()
class sublime.Edit()
    Edit.__init__()
class sublime.Region()
    Region.__eq__()
    Region.__init__()
    Region.__len__()
    Region.__lt__()
    Region.__repr__()
    Region.__str__()
    Region.begin()
    Region.contains()
    Region.cover()
    Region.empty()
    Region.end()
    Region.intersection()
    Region.intersects()
    Region.size()
class sublime.Selection()
    Selection.__delitem__()
    Selection.__eq__()
    Selection.__getitem__()
    Selection.__init__()
    Selection.__len__()
    Selection.__lt__()
    Selection.add()
    Selection.add_all()
    Selection.clear()
    Selection.contains()
    Selection.subtract()
class sublime.View()
    View.__eq__()
    View.__init__()
    View.__len__()
    View.add_regions()
    View.assign_syntax()
    View.begin_edit()
    View.buffer_id()
    View.change_count()
    View.classify()
    View.em_width()
    View.encoding()
    View.end_edit()
    View.erase()
    View.erase_regions()
    View.erase_status()
    View.extract_completions()
    View.extract_scope()
    View.file_name()
    View.find()
    View.find_all()
    View.find_all_results()
    View.find_by_selector()
    View.fold()
    View.folded_regions()
    View.full_line()
    View.get_regions()
    View.get_status()
    View.get_symbols()
    View.has_non_empty_selection_region()
    View.id()
    View.indentation_level()
    View.indented_region()
    View.indexed_symbols()
    View.insert()
    View.is_dirty()
    View.is_folded()
    View.is_in_edit()
    View.is_loading()
    View.is_read_only()
    View.is_scratch()
    View.layout_extent()
    View.layout_to_text()
    View.line()
    View.line_endings()
    View.line_height()
    View.lines()
    View.match_selector()
    View.meta_info()
    View.name()
    View.replace()
    View.retarget()
    View.rowcol()
    View.run_command()
    View.scope_name()
    View.sel()
    View.set_encoding()
    View.set_line_endings()
    View.set_name()
    View.set_read_only()
    View.set_scratch()
    View.set_status()
    View.set_syntax_file()
    View.set_viewport_position()
    View.settings()
    View.show()
    View.show_at_center()
    View.size()
    View.split_by_newlines()
    View.substr()
    View.symbols()
    View.text_point()
    View.text_to_layout()
    View.unfold()
    View.viewport_extent()
    View.viewport_position()
    View.visible_region()
    View.window()
    View.word()
class sublime.Settings()
    Settings.__init__()
    Settings.add_on_change()
    Settings.clear_on_change()
    Settings.erase()
    Settings.get()
    Settings.has()
    Settings.set()



sublime_plugin
--------------
class sublime_plugin.ApplicationCommand()
    ApplicationCommand.description()
    ApplicationCommand.is_checked()
    ApplicationCommand.is_enabled()
    ApplicationCommand.is_visible()
    ApplicationCommand.name()
    ApplicationCommand.run()
class sublime_plugin.WindowCommand()
    WindowCommand.__init__()
    WindowCommand.description()
    WindowCommand.is_checked()
    WindowCommand.is_enabled()
    WindowCommand.is_visible()
    WindowCommand.name()
    WindowCommand.run()
class sublime_plugin.TextCommand()
    TextCommand.__init__()
    TextCommand.description()
    TextCommand.is_checked()
    TextCommand.is_enabled()
    TextCommand.is_visible()
    TextCommand.name()
    TextCommand.run()
class sublime_plugin.EventListener()
    EventListener.on_activated()
    EventListener.on_activated_async()
    EventListener.on_clone()
    EventListener.on_clone_async()
    EventListener.on_close()
    EventListener.on_deactivated()
    EventListener.on_deactivated_async()
    EventListener.on_load()
    EventListener.on_load_async()
    EventListener.on_modified()
    EventListener.on_modified_async()
    EventListener.on_new()
    EventListener.on_new_async()
    EventListener.on_post_save()
    EventListener.on_post_save_async()
    EventListener.on_pre_save()
    EventListener.on_pre_save_async()
    EventListener.on_query_completions()
    EventListener.on_query_context()
    EventListener.on_selection_modified()
    EventListener.on_selection_modified_async()

parser.py

import os
import ast
import zipfile

# Classes that are part of the API in general; the source files contain some
# helper classes that are just noise
_classes = [
    # From the sublime module
    "Edit", "Phantom", "PhantomSet", "Region", "Selection", "Settings", "View",
    "Window", "Sheet",

    # From the sublime_plugin module
   "ApplicationCommand", "EventListener", "ListInputHandler", "TextCommand",
   "TextInputHandler", "ViewEventListener", "WindowCommand",

   # These are special base classes; we don't want to spit them out directly
   # but we do want to remember they exist because they provide the API to
   # their base classes. The members for event listeners comes from the
   # 'all_callbacks' dictionary at the top level of sublime_plugin.py
   # "Command", "CommandInputHandler",
]


def _get_class_methods(cls_node, base=None):
    """
    Return a sorted list of the methods in a given class node. The methods in
    the base list (if any) are also included. Private methods are excluded and
    the list is sorted lexically.
    """
    def keep(method):
        n = method.name
        if n.startswith("__"):
            return True
        return not (n.startswith("_") or n.endswith("_"))

    methods = list(base or [])
    for node in ast.iter_child_nodes(cls_node):
        if isinstance(node, ast.FunctionDef) and keep(node):
            methods.append(node.name)

    return sorted(methods)


def get_plugin_specials(module):
    """
    Capture the methods for special classes that are not fully defined in the
    module files themselves. This returns empty lists if invoked from a file
    that doesn't contain the appropriate classes.
    """
    cmd_methods = []
    input_methods = []
    events = []
    for node in ast.iter_child_nodes(module):
        # Pick up command class and input handler methods; these classes are
        # not directly exposed in the API (except to use as base classes) but
        # subclasses inherit them.
        if isinstance(node, ast.ClassDef):
            if node.name == "Command":
                cmd_methods = _get_class_methods(node)
            elif node.name == "CommandInputHandler":
                input_methods = _get_class_methods(node)

        # Find event handlers by getting the keys from the global all_callbacks
        # variable. NOTE: Not all of these apply to ViewEventListener (e.g.
        # on_new) so external work is needed to cull those after the fact.
        elif isinstance(node, ast.Assign):
            for target in [n for n in node.targets if isinstance(n, ast.Name)]:
                if target.id == "all_callbacks":
                    events = [key.s for key in node.value.keys]

    return cmd_methods, input_methods, events


def module_report(module, name, inc_funcs, inc_class):
    """
    Display a report containing the functions and/or classes in the given
    module. Parameters indicate if module level functions and/or classes should
    be extracted for the report.
    """
    cmd_methods, input_methods, events = get_plugin_specials(module)

    def base_methods(cls_name):
        """
        Get the methods from known base classes
        """
        if cls_name.endswith("Command"):
            return cmd_methods
        elif cls_name.endswith("InputHandler"):
            return input_methods
        elif cls_name.endswith("EventListener"):
            return events

        return []

    print("%s\n%s" % (name, "-" * len(name)))
    for node in ast.iter_child_nodes(module):
        # Display module functions if requested
        if isinstance(node, ast.FunctionDef) and inc_funcs:
            print("%s.%s()" % (name, node.name))

        # Display module classes if requested
        elif isinstance(node, ast.ClassDef) and inc_class:
            if node.name in _classes:
                methods = _get_class_methods(node, base_methods(node.name))
                print("class %s.%s()" % (name, node.name))
                for method in methods:
                    print("    %s.%s()" % (node.name, method))


def load_module(handle, filename, inc_funcs=True, inc_class=True):
    """
    Given an open handle to the named file, load the file into an AST tree and
    generate a report of the contents, including functions or classes as
    requested.
    """
    module_name = os.path.splitext(os.path.basename(filename))[0]
    tree = ast.parse(handle.read(), filename)
    module_report(tree, module_name, inc_funcs, inc_class)

    print("\n")


def handle_raw_file(filename, inc_funcs=True, inc_class=True):
    """
    Handle a single input file by loading it, parsing it into an AST, and then
    dumping the module it contains out.
    """
    with open(filename) as handle:
        load_module(handle, filename, inc_funcs, inc_class)


def handle_archive(archive_name):
    """
    Handle both of the appropriate files from the given zip file version of a
    portable install of Sublime Text.
    """
    with zipfile.ZipFile(archive_name, "r") as zFile:
        load_module(zFile.open("sublime.py"), "sublime.py")
        load_module(zFile.open("sublime_plugin.py"), "sublime_plugin.py", inc_funcs=False)


if __name__ == "__main__":
    handle_raw_file("sublime.py")
    handle_raw_file("sublime_plugin.py", inc_funcs=False)
    # handle_archive("Sublime Text Build 3176.zip")

The initial proof of concept for loading up the sublime.py and sublime_plugin.py files, parsing them into an AST, and using that to come up with the appropriate classes and API methods. The sample output is from the files in Sublime build 3006.

This is close but not without it's issues, since some of the relevant information is hidden in the core and not exposed in the plugin files (or at least that I can see in this build).

One example of this is that the EventListener class is empty, but a list of all events is provided as a global dictionary. This works for now but will not work when we hit where ViewEventListener appearing in the API. A quick cursory examination shows that there is no list of events that ViewEventListener responds to directly, which is unfortunate because it does not respond to on_new or on_clone (per the official docs) which means that tracking extensions to the view listener API is more manual than one might like.

Another example is that all of the command classes get their default behaviour from the hidden Command class, but only ApplicationCommand will ever have is_checked() invoked for it. The output is thus technically correct but perhaps misleading. This is less of a problem than the above, though.

The example also needs to be extended to pull the files directly from the source zip files and generate output in a nicer tabulated format annotated by the build ID (taken from the source zip) so that the results can be collated to see where various API endpoints first appeared.