Python Console Execution for SuperFences

readme.md

Transform Python to Executed Pycon Output

facelessuser/pymdown-extensions#1690

mkdocs/mkdocs#2835

Register a custom Python block (here we use py). For a project, you can specify global code to execute before every block if desired, here we are importing markdown.

  - pymdownx.superfences:
      preserve_tabs: true
      custom_fences:
        - name: py
          class: 'highlight'
          format: !!python/object/apply:tools.pycon.py_command_formatter
            kwds:
              init: |
                import markdown
          validator: !!python/name:tools.pycon.py_command_validator

When we specify a python block normally, it'll highlight like Python:

```py
text = "A link https://google.com"
markdown.markdown(text, extensions=['pymdownx.magiclink'])
```

But when we add the run argument, the code is evaluated and run given us a Python Console output:

```py run
text = "A link https://google.com"
markdown.markdown(text, extensions=['pymdownx.magiclink'])
```

We can even do some per block setup and hide the setup if we don't want it to be part of the example. We just have to wrap code at the very beginning of the block with # pragma: init

```py run
# pragma: init
text = "A link https://google.com"
# pragma: init
markdown.markdown(text, extensions=['pymdownx.magiclink'])
```

pycon.py

"""
Execute Python code in code blocks and construct a interactive Python console output.

This allows you to write code examples, but then execute them, showing the results.

https://github.com/facelessuser/pymdown-extensions/issues/1690

---

MIT License

Copyright (c) 2023 Isaac Muse <[email protected]>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from functools import partial
import ast
import re
from io import StringIO
import sys
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import find_formatter_class
import code

PY310 = (3, 10) <= sys.version_info
PY311 = (3, 11) <= sys.version_info

RE_INIT = re.compile(r'^\s*#\s*pragma:\s*init\n(.*?)#\s*pragma:\s*init\n', re.DOTALL | re.I)

AST_BLOCKS = (
    ast.If,
    ast.For,
    ast.While,
    ast.Try,
    ast.With,
    ast.FunctionDef,
    ast.ClassDef,
    ast.AsyncFor,
    ast.AsyncWith,
    ast.AsyncFunctionDef
)

if PY310:
    AST_BLOCKS = AST_BLOCKS + (ast.Match,)


if PY311:
    AST_BLOCKS = AST_BLOCKS + (ast.TryStar,)


class IPY(code.InteractiveInterpreter):
    """Handle code."""

    def __init__(self, show_except=True, locals=None):
        """Initialize."""

        super().__init__(locals=locals)

        self.show_except = show_except

    def set_exceptions(self, enable):
        """Set exceptions handling."""

        self.show_except = enable

    def write(self, data):
        """Write."""

        if not self.show_except:
            raise RuntimeError(data)

        sys.stdout.write(data)


class StreamOut:
    """Override the standard out."""

    def __init__(self):
        """Initialize."""
        self.old = sys.stdout
        self.stdout = StringIO()
        sys.stdout = self.stdout

    def read(self):
        """Read the stringIO buffer."""

        value = ''
        if self.stdout is not None:
            self.stdout.flush()
            value = self.stdout.getvalue()
            self.stdout = StringIO()
            sys.stdout = self.stdout
        return value

    def __enter__(self):
        """Enter."""
        return self

    def __exit__(self, type, value, traceback):
        """Exit."""

        sys.stdout = self.old
        self.old = None
        self.stdout = None


def execute(cmd, no_except=True, init='', ipy=None):
    """Execute color commands."""

    # Setup global initialization
    if ipy is None:
        ipy = IPY(show_except=not no_except)
    if init:
        ipy.set_exceptions(False)
        execute(init.strip(), ipy=ipy)
        ipy.set_exceptions(not no_except)

    console = ''

    # Build AST tree
    m = RE_INIT.match(cmd)
    if m:
        block_init = m.group(1)
        src = cmd[m.end():]
        ipy.set_exceptions(False)
        execute(block_init, ipy=ipy)
        ipy.set_exceptions(not no_except)
    else:
        src = cmd
    lines = src.split('\n')
    try:
        tree = ast.parse(src)
    except Exception as e:
        if no_except:
            from pymdownx.superfences import SuperFencesException
            raise SuperFencesException from e
        import traceback
        return '{}'.format(traceback.format_exc())

    for node in tree.body:
        result = []

        # Format source as Python console statements
        start = node.lineno
        end = node.end_lineno
        stmt = lines[start - 1: end]
        command = ''
        payload = '\n'.join(stmt)
        for i, line in enumerate(stmt, 0):
            if i == 0:
                stmt[i] = '>>> ' + line
            else:
                stmt[i] = '... ' + line
        command += '\n'.join(stmt)
        if isinstance(node, AST_BLOCKS):
            command += '\n... '
            payload += '\n'

        try:
            # Capture anything sent to standard out
            with StreamOut() as s:
                # Execute code
                ipy.runsource(payload)

                # Output captured standard out after statements
                text = s.read()
                if text:
                    result.append(text)

                # Execution went well, so append command
                console += command

        except Exception as e:
            if no_except:
                from pymdownx.superfences import SuperFencesException
                raise SuperFencesException from e
            import traceback
            console += '{}\n{}'.format(command, traceback.format_exc())
            # Failed for some reason, so quit
            break

        # If we got a result, output it as well
        console += '\n{}'.format(''.join(result))

    return console


def colorize(src, lang, **options):
    """Colorize."""

    HtmlFormatter = find_formatter_class('html')
    lexer = get_lexer_by_name(lang, **options)
    formatter = HtmlFormatter(cssclass="highlight", wrapcode=True)
    return highlight(src, lexer, formatter).strip()


def py_command_validator(language, inputs, options, attrs, md):
    """Python validator."""

    valid_inputs = set(['exceptions', 'run'])

    for k, v in inputs.items():
        if k in valid_inputs:
            options[k] = True
            continue
        attrs[k] = v
    return True


def _py_command_formatter(
    src="",
    language="",
    class_name=None,
    options=None,
    md="",
    init='',
    **kwargs
):
    """Formatter wrapper."""

    from pymdownx.superfences import SuperFencesException

    try:
        # Check if we should allow exceptions
        exceptions = options.get('exceptions', False) if options is not None else False
        run = options.get('run', False) if options is not None else False

        if run:
            console = execute(src.strip(), not exceptions, init=init)
            language = 'pycon'
        else:
            console = src
            language = 'py'

        el = md.preprocessors['fenced_code_block'].extension.superfences[0]['formatter'](
            src=console,
            class_name="class_name",
            language=language,
            md=md,
            options=options,
            **kwargs
        )
    except SuperFencesException:
        raise
    except Exception:
        from pymdownx import superfences
        import traceback
        print(traceback.format_exc())
        return superfences.fence_code_format(src, 'text', class_name, options, md, **kwargs)
    return el


def py_command_formatter(init='', interactive=False):
    """Return a Python command formatter with the provided imports."""

    return partial(_py_command_formatter, init=init, interactive=interactive)

Can I please ask where should I put the pycon.py file? I tried to put it at root directory, docs directory and overrides directory, but they all complains:

Error: MkDocs encountered an error parsing the configuration file: while constructing a Python object
cannot find module 'pycon' (No module named 'pycon')

Wherever you'd like. From the example config, you can infer the location: !!python/object/apply:tools.pycon.py_command_formatter

project
    tools
        __init__.py
        pycon.py

Notice that tools requires us to have a __init__.py to treat it as a module.

Thank you for your quick reply!
I have tried this at first, but doesn't seem to work.
My code is at https://github.com/ZhiyuanChen/CHANfiG, would you mind to help me figure out what have I done wrong?
Here in GitHub Action outputs:

  mkdocs build
  shell: /usr/bin/bash -e {0}
  env:
    pythonLocation: /opt/hostedtoolcache/Python/3.11.3/x64
    PKG_CONFIG_PATH: /opt/hostedtoolcache/Python/3.11.3/x64/lib/pkgconfig
    Python_ROOT_DIR: /opt/hostedtoolcache/Python/3.11.3/x64
    Python2_ROOT_DIR: /opt/hostedtoolcache/Python/3.11.3/x64
    Python3_ROOT_DIR: /opt/hostedtoolcache/Python/3.11.3/x64
    LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.11.3/x64/lib
Error: MkDocs encountered an error parsing the configuration file: while constructing a Python object
cannot find module 'tools.pycon' (No module named 'tools')
  in "/home/runner/work/CHANfiG/CHANfiG/mkdocs.yml", line 152, column 19
Error: Process completed with exit code 1.

In the example above, are your running mkdocs from project? The path would be relative to your current working directory.

In the example above, are your running mkdocs from project? The path would be relative to your current working directory.

Yes

Okay, I don't have time right now, but I'll take a look later today and get back to you.

Okay, I don't have time right now, but I'll take a look later today and get back to you.

Thank you so much, please take your time.

@ZhiyuanChen Here is a simple working example: https://file.io/KHr29MpxZ2gf

Hmmmm, I tried on my PC but it raises the same Error, I'll try later on my Mac

@ZhiyuanChen I think I know what the issue might be.

If you run mkdocs serve it will not work, but if you do something like python -m mkdocs serve it will work.

Using -m will load local modules as the current working directory will be added to the system path. That is how I run it if I need local modules to work.

It is possible to modify this gist to add support for inspect.getsource() and pyplot.show() by monkey-patching some functions. I had to do this to make my code work (it relies on inspect).

def execute(cmd, no_except=True, init='', ipy=None):
    """Execute color commands."""

    # Setup global initialization
    if ipy is None:
        ipy = IPY(show_except=not no_except)
    if init:
        ipy.set_exceptions(False)
        execute(init.strip(), ipy=ipy)
        ipy.set_exceptions(not no_except)

    console = ''

    # Build AST tree
    m = RE_INIT.match(cmd)
    if m:
        block_init = m.group(1)
        src = cmd[m.end():]
        ipy.set_exceptions(False)
        execute(block_init, ipy=ipy)
        ipy.set_exceptions(not no_except)
    else:
        src = cmd
    lines = src.split('\n')
    try:
        tree = ast.parse(src)
    except Exception as e:
        if no_except:
            from pymdownx.superfences import SuperFencesException
            raise SuperFencesException from e
        import traceback
        return '{}'.format(traceback.format_exc())

    pyplot_buffers: List[BytesIO] = []

    for node in tree.body:
        result = []

        # Format source as Python console statements
        start = node.lineno
        end = node.end_lineno
        stmt = lines[start - 1: end]
        command = ''
        payload = '\n'.join(stmt)
        for i, line in enumerate(stmt, 0):
            if i == 0:
                stmt[i] = '>>> ' + line
            else:
                stmt[i] = '... ' + line
        command += '\n'.join(stmt)
        if isinstance(node, AST_BLOCKS):
            command += '\n... '
            payload += '\n'

        getlines = linecache.getlines

        # Monkey patch linecache to return our payload
        # This makes inspect.getsource() work
        # See https://stackoverflow.com/a/69668959
        def getlines_monkey_patch(filename, module_globals=None):
            if filename == '<string>':
                return src.splitlines(keepends=True)
            else:
                return getlines(filename, module_globals)

        linecache.getlines = getlines_monkey_patch

        pyplot_show = plt.show

        # Monkey patch pyplot.show() to save the figure to a buffer
        def pyplot_show_monkey_patch() -> None:
            buf = BytesIO()
            plt.savefig(buf, format='png', bbox_inches='tight')
            plt.close()
            buf.seek(0)
            # The buffer will be closed after the image is output
            pyplot_buffers.append(buf)

        plt.show = pyplot_show_monkey_patch

        try:
            # Capture anything sent to standard out
            with StreamOut() as s:
                # Execute code
                ipy.runsource(payload, '<string>')

                # Output captured standard out after statements
                text = s.read()
                if text:
                    result.append(text)

                # Execution went well, so append command
                console += command

        except Exception as e:
            if no_except:
                print("PAYLOAD: ", payload)
                from pymdownx.superfences import SuperFencesException
                raise SuperFencesException from e
            import traceback
            console += '{}\n{}'.format(command, traceback.format_exc())
            # Failed for some reason, so quit
            break
        finally:
            linecache.getlines = getlines
            plt.show = pyplot_show

        # If we got a result, output it as well
        console += '\n{}'.format(''.join(result))

    return console, pyplot_buffers

You'll also have to update _py_code_formatter to include the images.

def _py_command_formatter(
    src="",
    language="",
    class_name=None,
    options=None,
    md="",
    init='',
    **kwargs
):
    """Formatter wrapper."""

    from pymdownx.superfences import SuperFencesException

    try:
        # Check if we should allow exceptions
        exceptions = options.get('exceptions', False) if options is not None else False
        run = options.get('run', False) if options is not None else False

        if run:
            console, pyplot_buffers = execute(src.strip(), not exceptions, init=init)
            language = 'pycon'
        else:
            console = src
            pyplot_buffers = []
            language = 'py'

        el = md.preprocessors['fenced_code_block'].extension.superfences[0]['formatter'](
            src=console,
            class_name="class_name",
            language=language,
            md=md,
            options=options,
            **kwargs
        )
        # Populate el with pyplot buffers

        for buf in pyplot_buffers:
            el += "<img src='data:image/png;base64,"
            # Decode the png buffer
            el += base64.b64encode(buf.read()).decode('utf-8')
            el += "' />"
            # We can now safely close the buffer
            buf.close()
    except SuperFencesException:
        raise
    except Exception:
        from pymdownx import superfences
        import traceback
        print(traceback.format_exc())
        return superfences.fence_code_format(src, 'text', class_name, options, md, **kwargs)
    return el

It is certainly possible to modify it to do whatever you want it to do, but you would have to experiment. I provide the gist as is. I don't necessarily have the time to personally tweak it for individual user needs.

I understand - here is a modified gist with the linecache and matplotlib changes. I have run it locally and it works with multiple matplotlib plots.: https://gist.github.com/WolfByttner/e0b855e4b80b9742647411c51c8dfe55

It is a cool modification, thanks for sharing.