scanner_excerpt.go

func scanIdentifier(s string) (string, int) {
	byteLen := 0
	runeLen := 0
	for {
		if byteLen >= len(s) {
			break
		}

		nextRune, size := utf8.DecodeRuneInString(s[byteLen:])
		if (!(nextRune == '_' ||
			nextRune == '-' ||
			nextRune == '.' ||
			nextRune == '*' ||
			unicode.IsNumber(nextRune) ||
			unicode.IsLetter(nextRune) ||
			unicode.IsMark(nextRune)) && nextRune != '\\') {
			if byteLen > 0 && s[byteLen-1] == '\\' {
				// Escape this character, so allow to continue
				runeLen--
			}	else {
				break
			}
		}

		// If we reach a star, it must be between periods to be part
		// of the same identifier.
		if nextRune == '*' && s[byteLen-1] != '.' {
			break
		}

		// If our previous character was a star, then the current must
		// be period. Otherwise, undo that and exit.
		if byteLen > 0 && s[byteLen-1] == '*' && nextRune != '.' {
			byteLen--
			if s[byteLen-1] == '.' {
				byteLen--
			}

			break
		}

		byteLen = byteLen + size
		runeLen = runeLen + 1
	}
	t := strings.Replace(s[:byteLen], "\\", "", -1)
	return t, runeLen
}

Input:

foo ${bar.foo\\[bar\\].baz}

Output:

foo ${bar.foo[bar].baz}

runeLen: 18
t: "bar.foo[bar].baz"
ident: "bar.foo[bar].baz"
runeLen: 2
t: "az"
ident: "az"
nested = %!q(bool=false) -- len(s) = '\x00' -- litLen = '\x00' -- --- FAIL: TestScanner (0.00s)
        scanner_test.go:30:
                Input: foo ${bar.foo\[bar\].baz}
                Bad:   []string{"LITERAL", "BEGIN", "IDENTIFIER", "IDENTIFIER", "END", "EOF"}
                Want:  []string{"LITERAL", "BEGIN", "IDENTIFIER", "END", "EOF"}
=== RUN   TestScannerPos
--- PASS: TestScannerPos (0.00s)
=== RUN   TestTokenString
--- PASS: TestTokenString (0.00s)
FAIL
FAIL    _/Users/Chase/Sites/Charity/hil/scanner 0.007s