Split code into chunks (processing args, coloring syntax, etc..) based on if strings exist while taking escape characters into account. Golang rune parsing supports UTF8.

splitByStrings.go

// Run at: https://go.dev/play/p/oFUYPNRZiBp
package main

import "fmt"

// Suppose you have a piece of code you want to split into chunks based on if you're in a string or not.
// You also want to respect escape characters.

var input = `fmt.Println("Hello, \"世界\"")`

func main() {
	for part := range splitByStrings(input) {
		fmt.Printf("%v\n", part)
	}
}

type Token struct {
	String bool
	Value  string
}

func splitByStrings(input string) chan Token {

	c := make(chan Token)
	go func() {
		skipNext := false
		chunk := []rune{}
		inString := false
		for _, r := range input {

			chunk = append(chunk, r)
			if skipNext {
				skipNext = false
				continue
			}

			if r == '\\' && inString {
				skipNext = true
			} else if r == '"' {
				c <- Token{String: inString, Value: string(chunk)}
				chunk = []rune{}
				inString = !inString
				continue
			}
		}
		c <- Token{String: inString, Value: string(chunk)}
		close(c)
	}()

	return c
}