test_passwd_string_tokenization.py

import io
import re
import pytest

test_cases = [
    (r'eggs:spam:bacon', ['eggs', 'spam', 'bacon']),
    (r'eggs\:spam:bacon', ['eggs:spam', 'bacon']),
    (r'eggs\\:spam:bacon', ['eggs\\', 'spam', 'bacon']),
    (r'eggs\\\:spam:bacon', ['eggs\\:spam', 'bacon']),
    (r'eggs::spam:bacon', ['eggs', '', 'spam', 'bacon']),
    (r':eggs:spam:bacon', ['', 'eggs', 'spam', 'bacon']),
    (r'eggs:spam:bacon:', ['eggs', 'spam', 'bacon', '']),
    (r'::eggs:spam:bacon', ['', '', 'eggs', 'spam', 'bacon']),
    (r'eggs:spam:bacon::', ['eggs', 'spam', 'bacon', '', '']),
]


def split_string(s):
    ## any character preceded by backslash, or anything
    ## that's not a backslash or a colon

    buf = io.BytesIO()
    for ch in re.findall(r'\\.|.', s):
        if ch == ':':
            yield buf.getvalue()
            buf.seek(0)
            buf.truncate()
        else:
            if len(ch) == 2:
                assert ch[0] == '\\'
                ch = ch[1]
            assert len(ch) == 1
            buf.write(ch)
    yield buf.getvalue()


@pytest.mark.parametrize('instring,output', test_cases)
def test_string_tokenization(instring, output):
    assert list(split_string(instring)) == output