Custom processor for markedapp

codrspace_processor.py

#!/usr/bin/env python

"""
Simplified stand-alone script to mimic codrspace.com short code processing.

Script takes 1 optional argument, the path of your python environment.
The rest of the input is read directly from stdin and expected to be codrspace
markdown text.  The output is codrspace html after all markdown and codrspace
specific short codes have been processed.

To use:
    - git clone git://gist.github.com/3815902.git codrspace_processor
    - cd codrspace_processor
    - pip install -r requirements.txt
    - Test script installation:
        - echo '[code]import this[/code]' | codrspace_processor.py <PYTHONPATH>
        - PYTHONPATH should be path to where your pip install from above put
          the requirements
        - Should see output like the following (without pretty whitespace)
            <p>
                <div class="highlight">
                    <pre>
                      <span class="n">import</span> <span class="n">this</span>
                    </pre>
                </div>
            </p>

You can use this script with the fantastic Marked (http://markedapp.com)
markdown preview tool by doing the following after installating the processor
(above steps):
    - Open Marked app
    - Go to preferences
    - Select the 'behavior' tab
    - Select 'Custom Markdown Processor'
    - Type the FULL path to the codrspace_processor.py script in the 'Path'
      field
    - Type the FULL path to the Python library path (site-packages) if you
      installed the processor script in a virtual environment.  This argument
      is not needed if you installed the script into the global Python
      installation.
    - Click 'Save'
"""


from hashlib import md5
import re
import types
import json


def get_lexer_list():
    """Get list of all available lexers"""

    lexers = []

    for lexer in get_all_lexers():
        name, aliases, filetypes, mimetypes = lexer
        lexers.extend([alias for alias in aliases])

    return lexers


def get_lexer(value, lang):
    """Find a lexer for the given text and language"""

    if lang:
        if '.' in lang:

            # possibly a filename, poor detection for now
            return get_lexer_for_filename(lang)
        elif lang in get_lexer_list():

            # guess it by specific language
            return get_lexer_by_name(lang)

    # try and guess the lexer by content
    return guess_lexer(value)


def explosivo(value):
    """
    Search text for any references to supported short codes and explode them
    """

    # Round-robin through all functions as if they are filter methods so we
    # don't have to update some silly list of available ones when they are
    # added
    module = sys.modules[__name__]
    all_replacements = []

    # get the replacement values and content with replacement hashes
    for name, var in vars(module).items():
        if type(var) == types.FunctionType and name.startswith('filter_'):
            replacements, value, match = var(value)
            if match:
                all_replacements.extend(replacements)

    # convert to markdown
    value = markdown.markdown(value)

    # replace the hash values with the replacement values
    for rep in all_replacements:
        _hash, text = rep
        value = value.replace(_hash, text)

    return value


def filter_gitstyle(value):
    replacements = []
    pattern = re.compile("```(?P<lang>[^\\n\\s`]+)+?(?P<code>[^```]+)+?```",
                         re.I | re.S | re.M)

    if len(re.findall(pattern, value)) == 0:
        return (replacements, value, None,)

    git_styles = re.finditer(pattern, value)

    for gs in git_styles:
        try:
            lang = gs.group('lang')
        except IndexError:
            lang = None

        text = colorize(gs.group('code'), lang=lang)
        text_hash = md5(text.encode('utf-8')).hexdigest()

        replacements.append([text_hash, text])
        value = re.sub(pattern, text_hash, value, count=1)

    return (replacements, value, True,)


def filter_inline(value):
    """
    Look for any [code][/code] blocks and replace with syntax highlighted html

    Return hash of text that needs to be replaced
    """

    replacements = []
    regx = '\\[code(\\s+lang=\"(?P<lang>[\\w]+)\")*\\](?P<code>.*?)\\[/code\\]'
    pattern = re.compile(regx, re.I | re.S | re.M)

    if len(re.findall(pattern, value)) == 0:
        return (replacements, value, None,)

    inlines = re.finditer(pattern, value)

    for inline_code in inlines:
        try:
            lang = inline_code.group('lang')
        except IndexError:
            lang = None

        text = colorize(inline_code.group('code'), lang=lang)
        text_hash = md5(text.encode('utf-8')).hexdigest()

        replacements.append([text_hash, text])
        value = re.sub(pattern, text_hash, value, count=1)

    return (replacements, value, True,)


def filter_gist(value):
    """
    Look for any [gist][gist] blocks, fetch gist code (if applicable) and
    syntax highlight it

    Return hash of text that needs to be replaced
    """

    gist_base_url = 'https://api.github.com/gists/'
    replacements = []
    pattern = re.compile('\[gist (\d+) *\]', flags=re.IGNORECASE)

    ids = re.findall(pattern, value)
    if not len(ids):
        return (replacements, value, None,)

    for gist_id in ids:
        gist_text = ""
        lang = None
        resp = requests.get('%s%d' % (gist_base_url, int(gist_id)))

        if resp.status_code != 200:
            return (replacements, value, None,)

        content = json.loads(resp.content)

        # Go through all files in gist and smash 'em together
        for name in content['files']:
            _file = content['files'][name]

            # try and get the language of the file either
            # by passing filename or by passing the language
            # specified
            if 'filename' in _file:
                lang = _file['filename']
            elif 'language' in _file:
                lang = _file['language']

            gist_text += "%s" % (colorize(_file['content'], lang=lang))

        if content['comments'] > 0:
            gist_text += """
                        <hr>
                        <p class="github_convo">
                            Join the conversation on
                            <a href="%s#comments">github</a>
                            (%d comments)
                        </p>""" % (content['html_url'], content['comments'])

        text_hash = md5(gist_text.encode('utf-8')).hexdigest()

        replacements.append([text_hash, gist_text])
        value = re.sub(pattern, text_hash, value, count=1)

    return (replacements, value, True,)


def colorize(value, lang=None):
    """
    Syntax highlight given text based on the language of choice

    If no language is provided, try to guess (not reliable).
    """

    return highlight(value, get_lexer(value, lang), HtmlFormatter())


if __name__ == "__main__":
    import sys

    # Hack path with first argument
    if len(sys.argv) > 1:
        sys.path.append(sys.argv[1])

    import requests
    import markdown

    from pygments import highlight
    from pygments.formatters import HtmlFormatter
    from pygments.lexers import get_lexer_by_name, get_lexer_for_filename, \
        guess_lexer, get_all_lexers

    print explosivo(sys.stdin.read())

requirements.txt

Markdown==2.2.0
Pygments==1.5
requests==0.14.1