async_edits.py

import sublime, sublime_plugin
import re
import threading


# This is an example plugin that simulates a completion interaction with an
# LSP server, while correctly apply any edits that are received from the
# server. This is trickier than you'd expect, due to the async nature the
# server communication, the async auto complete system in Sublime Text, and
# the use of row+col offsets within the LSP protocol.
#
# Note this uses APIs that are only available in Sublime Text 4069+
#
# For reference, this plugin adds a single completion with a trigger of
# "hello", that when run replaces all "#include" occurrences in the buffer
# with smiley faces



# Apply a sequence of edits to the text buffer.
#
# Each edit is specified as a tuple of (begin_offset, end_offset, text)
#
# `when` is a token that indicates which version of the text buffer the regions
# are in relation to.
# `transform_region_from` translates these regions into the coordinate space
# of the current buffer.
class ApplyEditsCommand(sublime_plugin.TextCommand):
    def run(self, edit_token, when, edits):
        for e in edits:
            begin, end, text = e
            r = self.view.transform_region_from(sublime.Region(begin, end), when)
            self.view.replace(edit_token, r, text)


# Represents a point in a TextBuffer in row+col format, as opposed to Sublime
# Text's native character offset
class RcPoint():
    __slots__ = ['row', 'col']

    def __init__(self, row, col):
        self.row = row
        self.col = col

    def __repr__(self):
        return str(self.row) + ":" + str(self.col)

    def __eq__(x, y):
        return x.row == y.row and x.col == y.col

    def __lt__(x, y):
        if x.row == y.row:
            return x.col < y.col
        else:
            return x.row < y.row

    def __le__(x, y):
        if x.row == y.row:
            return x.col <= y.col
        else:
            return x.row <= y.row

    def __gt__(x, y):
        if x.row == y.row:
            return x.col > y.col
        else:
            return x.row > y.row

    def __ge__(x, y):
        if x.row == y.row:
            return x.col >= y.col
        else:
            return x.row >= y.row


# Equivalent to a sublime.Region, but using row+col offsets, rather than
# character offsets
class RcRegion():
    __slots__ = ['a', 'b']

    def __init__(self, a: RcPoint, b: RcPoint):
        self.a = a
        self.b = b

    def __repr__(self):
        return str(self.a) + ", " + str(self.b)

    def __eq__(x, y):
        return x.a == y.a and x.b == y.b


# Convert an offset into a string into a row+column offset into the string.
# Used by simulate_server() only
def offset_to_rowcol(s: str, offset: int) -> RcPoint:
    row = 0
    col = 0
    for c in s[:offset]:
        if c == '\n':
            row += 1
            col = 0
        else:
            col += 1

    return RcPoint(row,col)


# Acts in a similar manner to a LSP server: takes the current contents of the
# text buffer, spends some time thinking, and produces a sequence of edits to
# that buffer
def simulate_server(contents: str, request):
    # calculate all the edits
    edits = []
    for m in re.finditer("#include", contents):
        begin = offset_to_rowcol(contents, m.start())
        end = offset_to_rowcol(contents, m.end())
        edits.append((RcRegion(begin, end), "😀"))

    # send the edits back to the client with a simulated delay
    sublime.set_timeout(lambda: request.fill_completions(edits), 500)


# Given an RcRegion, and a change to the text buffer, transform the region
# into the coordinate space after that change has been applied.
#
# For example, if the r.a is row 10, column 5, and a newline is inserted at
# the beginning of the file, then the resulting region has its r.a set to row
# 11, column 5.
#
# Note that this mirrors the algorithm that View.transform_region_from uses
# internally
def transform_region(r: RcRegion, change: sublime.TextChange) -> RcRegion:
    begin = RcPoint(change.a.row, change.a.col)
    end = RcPoint(change.b.row, change.b.col)

    num_added_newlines = change.str.count("\n")

    if num_added_newlines > 0:
        num_chars_on_last_line = len(change.str) - change.str.rfind("\n")
    else:
        num_chars_on_last_line = len(change.str)

    if r.a >= begin and r.a <= end:
        r.a = begin
    elif r.a > end:
        # handle the erased region
        if r.a.row == end.row:
            if begin.row == end.row:
                r.a.col -= end.col - begin.col
            else:
                r.a.col -= end.col
        else:
            r.a.row -= (end.row - begin.row)

        # handle the added text
        if r.a.row == end.row:
            r.a.row += num_added_newlines
            r.a.col += num_chars_on_last_line
        else:
            r.a.row += num_added_newlines

    if r.b >= begin and r.b < end:
        r.b = begin
    elif r.b >= end:
        # handle the erased region
        if r.b.row == end.row:
            if begin.row == end.row:
                r.b.col -= end.col - begin.col
            else:
                r.b.col -= end.col
        else:
            r.b.row -= (end.row - begin.row)

        # handle the added text
        if r.b.row == end.row:
            r.b.row += num_added_newlines
            r.b.col += num_chars_on_last_line
        else:
            r.b.row += num_added_newlines

    return r


def check_transform_region():
    assert(transform_region(RcRegion(RcPoint(10, 0), RcPoint(10, 0)), sublime.TextChange(
        sublime.HistoricPosition(-1, 5, 0, 0, 0),
        sublime.HistoricPosition(-1, 6, 0, 0, 0),
        "")) == RcRegion(RcPoint(9, 0), RcPoint(9, 0)))

    assert(transform_region(RcRegion(RcPoint(10, 0), RcPoint(10, 0)), sublime.TextChange(
        sublime.HistoricPosition(-1, 5, 0, 0, 0),
        sublime.HistoricPosition(-1, 6, 0, 0, 0),
        "abc")) == RcRegion(RcPoint(9, 0), RcPoint(9, 0)))

    assert(transform_region(RcRegion(RcPoint(10, 0), RcPoint(10, 0)), sublime.TextChange(
        sublime.HistoricPosition(-1, 5, 0, 0, 0),
        sublime.HistoricPosition(-1, 6, 0, 0, 0),
        "\n")) == RcRegion(RcPoint(10, 0), RcPoint(10, 0)))

    assert(transform_region(RcRegion(RcPoint(10, 0), RcPoint(10, 0)), sublime.TextChange(
        sublime.HistoricPosition(-1, 15, 0, 0, 0),
        sublime.HistoricPosition(-1, 16, 0, 0, 0),
        "")) == RcRegion(RcPoint(10, 0), RcPoint(10, 0)))

    assert(transform_region(RcRegion(RcPoint(10, 0), RcPoint(10, 0)), sublime.TextChange(
        sublime.HistoricPosition(-1, 10, 0, 0, 0),
        sublime.HistoricPosition(-1, 10, 0, 0, 0),
        "xx")) == RcRegion(RcPoint(10, 0), RcPoint(10, 2)))

check_transform_region()


# Takes an edit in row+col format, and returns an edit in character offset
# format
def row_col_edit_to_offset_edit(view: sublime.View, e):
    r, text = e

    # convert from row_col into a text point
    begin = view.text_point(r.a.row, r.a.col)
    end = view.text_point(r.b.row, r.b.col)

    return (begin, end, text)


# Represents an outstanding request to the 'server' (a thread running
# simulate_server() in this case).
#
# The important aspect here is self.changes_since_sent, which represents all
# the changes to the text buffer since the request was sent. This allows the
# response from the server to be adjusted to the current coordinate space of
# the buffer
class Request():
    def __init__(self, listener, view, completions):
        self.listener = listener
        self.view = view
        self.changes_since_sent = []
        self.completions = sublime.CompletionList()

    # Called when we get a response from the 'server'
    def fill_completions(self, edits):
        # The server has sent us a series of edits it wants to be made when
        # the completion command is run, however the user may have edited the
        # buffer since the request want sent. Adjust the response from the
        # server to account for these edits
        for c in self.changes_since_sent:
            edits = [(transform_region(r, c), text) for r, text in edits]

        # The edits are now relative to the current document, and could be
        # applied now. However, we don't want to apply them now, we want to
        # apply them only when the user selects the completion. Because
        # further edits to the buffer may occur between now and when the user
        # selects the completion, grab a token that represents the current
        # state of the buffer, so the edits can be transformed a second time
        # when the user does select the completion.
        change_id = self.view.change_id()

        # Transform the edits from row+col representation, to the simpler
        # character offset representation
        normal_edits = [row_col_edit_to_offset_edit(self.view, e) for e in edits]

        # Create the completion item, indicating which command to run, and its
        # arguments. Note that `change_id` and `normal_edits` are both simple
        # types that can be JSON encoded
        item = sublime.CompletionItem.command_completion(
                trigger="hello",
                command="apply_edits",
                args={"when":change_id, "edits": normal_edits})

        # The request has completed, so remove ourself from
        # HelloCompletions.requests
        self.listener.requests.remove(self)

        self.completions.set_completions([item], 0)


class AsyncEditsListener(sublime_plugin.EventListener):
    def __init__(self):
        self.requests = []

    def on_query_completions(self, view, prefix, locations):
        req = Request(self, view, sublime.CompletionList())

        # Send the contents of the current view to the simulated server. When
        # the response is ready, Request.fill_completions()) will be called
        contents = view.substr(sublime.Region(0, view.size()))
        t = threading.Thread(
            target=simulate_server,
            args=(contents, req))
        t.start()

        self.requests.append(req)

        return req.completions

    def on_text_changed(self, view, changes):
        # Record the changes in all pending requests
        for r in self.requests:
            if r.view.view_id == view.view_id:
                r.changes_since_sent.extend(changes)

I would really love to skip all the calculating of the offsets(transform_region, row_col_edit_to_offset_edit)
just by moving the call to view.now() from the fill_completionsmethod to the on_query_completion? Would something like that be possible.

Would the following work?

    def on_query_completions(self, view, prefix, locations):
+        edit_regions = []
+        for point in locations:
+            line = self.view.line(locations[poin]) 
+            line_contents = view.substr(line)
+            edit = (line.a, line.b, line_contents)
+            edit_regions.append(edit)

+        now_token = self.view.now()
+        req = Request(self, view, sublime.CompletionList(), now_token, edit_regions)

...

class Request():
+    def __init__(self, listener, view, completions, now_token, edit_regions):
        self.listener = listener
        self.view = view
        self.completions = sublime.CompletionList()
+        self.now_token = now_token
+        self.edit_regions = edit_regions

    # Called when we get a response from the 'server'
    def fill_completions(self, edits):

-        # The server has sent us a series of edits it wants to be made when
-        # the completion command is run, however the user may have edited the
-        # buffer since the request want sent. Adjust the response from the
-        # server to account for these edits
-        for c in self.changes_since_sent:
-            edits = [(transform_region(r, c), text) for r, text in edits]

-        # The edits are now relative to the current document, and could be
-        # applied now. However, we don't want to apply them now, we want to
-        # apply them only when the user selects the completion. Because
-        # further edits to the buffer may occur between now and when the user
-        # selects the completion, grab a token that represents the current
-        # state of the buffer, so the edits can be transformed a second time
-        # when the user does select the completion.
-        now = self.view.now()

-        # Transform the edits from row+col representation, to the simpler
-        # character offset representation
-        normal_edits = [row_col_edit_to_offset_edit(self.view, e) for e in edits]

-        # Create the completion item, indicating which command to run, and its
-        # arguments. Note that `now` and `normal_edits` are both simple types
-        # that can be JSON encoded
        item = sublime.CompletionItem.command_completion(
                trigger="hello",
                command="apply_edits",
+                args={"when":self.now_token, "edits": self.edit_regions})

        # The request has completed, so remove ourself from
        # HelloCompletions.requests
        self.listener.requests.remove(self)

        self.completions.set_completions([item], 0)

...

class ApplyEditsCommand(sublime_plugin.TextCommand):
    def run(self, edit_token, when, edits):
        for e in edits:
            begin, end, text = e
            r = self.view.transform_region_from(sublime.Region(begin, end), when)
            self.view.replace(edit_token, r, text)

As written, I would expect that change to work just fine, yes.

However, in the LSP protocol TextEdits are specified with row+col offsets (https://microsoft.github.io/language-server-protocol/specifications/specification-current/#textEdit), so they have to be converted into character offsets to be applied in Sublime Text, and that requires first bringing them up-to-date to the current TextBuffer state, and then converting them via View.text_point()

If View.text_point() is just called directly without transforming the TextEdits into the current state of the buffer then any edits to the current line won't apply correctly if the user has been able to type while the LSP server is thinking about the completions (more generally, they also won't apply correctly if any newlines have been inserted into the buffer, which is unlikely during auto complete, but still technically possible)

This is all implemented as suggested: https://github.com/sublimelsp/LSP/blob/st4000-exploration/plugin/completion.py

Feedback:

I don't know how we should handle completion items like this one:

{
    'insertTextFormat': 1,
    'label': "End with '</xml>'",
    'filterText': '</xml>',
    'kind': 10,
    'textEdit':
    {
        'newText': '</xml>',
        'range':
        {
            'end': {'character': 5, 'line': 0},
            'start': {'character': 5, 'line': 0}
        }
    }
}

This completion item is provided after typing <xml> in the view (with an XML language server). > is apparently a trigger character. The cursor is at col 5. According to the LSP spec, the textEdit.range must contain the cursor, which it does if we consider ranges to be closed intervals. Because the textEdit.range is empty, I call this an insertion.

What is supposed to happen is that we end up with <xml></xml> in the view.

I've made gifs on what happens (replace <xml> with <asdf> everywhere)

What actually happens is that ST removes <asdf> from the buffer. After that we end up in the run() of the command completion. Now what should self.view.transform_region_from do in this case? We're giving it an ST historic region of length 0. What ST region can possibly account for this?

I tried categorizing all valid edits, I hope it can be of some use:

...prefix... (ST prefix)
..xxxxxxx... (edits behind prefix: OK)

...prefix... (ST prefix)
...xxxxxx... (edits coincide with prefix: OK)

...prefix... (ST prefix)
....xxxxxx.. (edits after prefix and including part of prefix: OK)

...prefix... (ST prefix)
.........xxx (pure insertion edit: doesn't work)

(cc @deathaxe)

Perhaps also of note are the label, filterText and textEdit.newText:

label        End with '</xml>'
filterText   </xml>
newText      </xml>

We're putting the label into the ST trigger, but it doesn't really "fit". The label serves a different purpose than being a trigger.
We could use filterText as trigger, but then the user wouldn't see the label anymore.

Here's another example for comparing label, filterText and the ST trigger. It's a completion item from clangd when typing abo:

{
    'sortText': '3f651eb8abort',
    'kind': 3,
    'insertText': 'abort()',  # ignore insertText: only provided for legacy reasons
    'label': ' abort()',
    'filterText': 'abort',
    'textEdit':
    {
        'range':
        {
            'start': {'character': 4, 'line': 28},
            'end': {'character': 7, 'line': 28}
        },
        'newText': 'abort()'
    },
    'documentation': 'Abort execution and generate a core-dump.  ',
    'detail': 'void\n<cstdlib>',
    'insertTextFormat': 2
}

Here we're now putting ' abort()' into the trigger (note the whitespace prefix in ' abort()'). But as we noticed this confuses the internal ST algorithm for prefix removal somewhat. Again we could put the filterText into the trigger, but then the user could be confused: Is abort a variable or a function?

Leading or trailing whitespace in a label is dump and should be trimmed. But anyway a label might start/end with nearly anything with the trigger being located at the beginning, somewhere in the middle or at the end.

Static sublime-completions suffer from the label vs. trigger issue as well. The trigger may be used as label to present additional information about the contents of the completion as it would look like after committing, e.g.: the parameter list or at least the (). Adding such additional characters causes dedupping with normal word completions to break.

The kind container provides some info about the completion. A f at the left hand side indicates a function. But it might not be enough to distinguish overloaded functions of a C++ class for instance or different variants of a snippet beginning with the same name.

Example

[
	{
		"trigger": "LM(NAME)",
		"annotation": "load mask",
		"kind": "function",
		"contents": "LM(\"${1:NAME}\")"
	},
	{
		"trigger": "LM(NAME,,CHILD)",
		"annotation": "load mask",
		"kind": "function",
		"contents": "LM(\"${1:NAME}\"${2:,,${3:TRUE}})"
	},
]

The same might apply to plugin driven completions in the one way or the other.

I'd therefore suggest to add an optional "label" field to the completions. If it is present and not empty it was used to display the completion in the autocomplete popup, while trigger is used to do filtering and dedupping. If it is not present or empty, just display the trigger.

---

In the case of the XML language server: What it provides as label (Ends with ...) in this special situation should be the annotation.

In the about() example the detail field should be the annotation while documentation could be used to fill ST's details field.

Created an issue about it at sublimehq/sublime_text#3296

Thanks for creating an issue and explaining the situation very clearly.

I came up with the following idea:

        label = item["label"]
        filter_text = item.get("filterText")
        if filter_text:
            trigger = filter_text
            annotation = label
        else:
            trigger = label
            annotation = item.get("detail", "")

        documentation = item.get("documentation")
        if isinstance(documentation, str):
            details = documentation
        elif isinstance(documentation, dict):
            value = documentation.get("value", "")
            if documentation.get("kind") == "markdown":
                details = mdpopups.md2html(self.view, value)
            else:
                details = value
        else:
            details = ""

        return sublime.CompletionItem.command_completion(
            trigger=trigger,
            command="lsp_select_completion_item",
            args=item,
            annotation=annotation,
            kind=kind,
            details=details
        )

So: if there's LSP-filterText, use that as ST-trigger and put the LSP-label in ST-annotations. Otherwise, use the LSP-label as ST-trigger and put the LSP-detail in the ST-annotations.

This would not show the LSP-detail in case we have LSP-filterText.
Perhaps we can also put the LSP-detail into ST-details, and with a keybinding show the LSP-documentation in an ST-popup.

Didn't the json lsp server send long documentation string? The general idea sounds not too bad, but maybe we'd need some language-server-specific handling for such things. But this feels like it becomes off topic here.

I'd therefore suggest to add an optional "label" field to the completions. If it is present and not empty it was used to display the completion in the autocomplete popup, while trigger is used to do filtering and dedupping. If it is not present or empty, just display the trigger.

The problem is that filterText is not guaranteed to be a substring of label. I don't know why.
Perhaps it is safe to assume that it is.

But maybe we'd need some language-server-specific handling for such things.

Perhaps, but it should be a last resort :)

But this feels like it becomes off topic here.

I'll stop here.

I've touched on some of the items raised here in sublimehq/sublime_text#3296