tokenize.py

def skip_ahead(code, index):
	return index + (len(code[index:]) - len(code[index:].strip()))

def match_letters(code, index):
	value = ''

	while index < len(code):
		if code[index].lower() in 'abcdefghijklmnopqrstuvwxyz':
			value += code[index]
		else:
			break

		index += 1

	return value, index if value else None

class BaseToken(object):
	@staticmethod
	def match(code, index):
		raise NotImplemented()

class SetVariableToken(BaseToken):
	def __init__(self, left, right):
		self.left = left
		self.right = right

	def __repr__(self):
		return '<SetVariable {} = {}>'.format(self.left, self.right)

	@staticmethod
	def match(code, index):
		left = match_letters(code, index)

		if left is None:
			return None

		left, index = left
		index = skip_ahead(code, index)

		if code[index] != '=':
			return None

		index += 1
		index = skip_ahead(code, index)

		right = match_letters(code, index)

		if right is None:
			return None

		right, index = right
		return SetVariableToken(left, right), index

class PrintVariableToken(BaseToken):
	def __init__(self, right):
		self.right = right

	def __repr__(self):
		return 'PrintVariable {}>'.format(self.right)
	
	@staticmethod
	def match(code, index):
		if code[index:index+6] != 'print ':
			return None

		index += 6
		right = match_letters(code, index)

		if right is None:
			raise SyntaxError('missing variable name')

		right, index = right
		return PrintVariableToken(right), index

TOKENS = [
	SetVariableToken,
	PrintVariableToken
]

def scan(code, index):
	index = skip_ahead(code, index)

	print code
	print ' ' * index + '^'

	if index >= len(code):
		return None

	for token in TOKENS:
		try:
			result = token.match(code, index)
		except IndexError:
			continue

		if result is not None:
			return result

	raise SyntaxError('invalid token at {}'.format(index))

def tokenize(code):
	tokens = []
	index = 0

	while index < len(code):
		result = scan(code, index)

		if result is None:
			break

		tokens.append(result[0])
		index = result[1]

	return tokens