rust: better tokenizer

tokenizer2.rs

fn update_state<'a>(s :&'a str, tokens :&mut Vec<String>, match_point : &Captures<'a>) -> &'a str {
    let new_s = &s[match_point.get(1).unwrap().end()..];
    tokens.push(match_point.get(1).unwrap().as_str().to_string());
    new_s
}

pub fn tokenizer(s_in: &str) -> Reader {
    let mut s = &s_in[0..];
    let brackets = regex!(r###"^[\s,]*([\(\)\{\}\[\]])[\s,]*"###);
    let digits = regex!(r"^[\s,]*(-?\d+)");
    let operands = regex!(r"^[\s,]*(\*{1,2}|[\+\-\\])"); //{} is greedy to detect ** instead of: *
    let alphas = regex!(r###"^[\s,]*([\w\d:"-]+)"###);
    let mut tokens = vec![];
    let all_regexs = vec![&brackets, &digits, &operands, &alphas] ;

    let empty = regex!(r"^[\s,]+$");

    while s.len() > 0 && !empty.is_match(s) {

        for regex in &all_regexs {
            let rb = regex.captures(s);

            if !rb.is_none() {
                s = update_state(s, &mut tokens, &rb.unwrap());
                break
            }
        }
    }
    Reader{tokens: tokens, position: 0}
}

good spot.