The little tokeniser that could

the_little_tokeniser.php

<?php

	function tokeniseString( $string ) {
        
        $regex  = "/([[:space:]]+)|([[:punct:]])/";    
        $opts   = PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY;
        $toks   = preg_split( $regex, $string, -1,  $opts );
        
        return $toks;
    }

	$string = "[various words!] 2 + (thirty thirsty_badgers), '9^7' | 6 >=2 ";
	
	var_dump( tokeniseString( $string ) );

?>