tokenizer snippets

tokenizer.fs

    let idents tokens =
        //reverse the tokens
        let tokens = tokens |> Array.rev |> List.ofArray
        let rec loop tokens result =
            match tokens with
            | [] -> result
            | (name, token: TokenInformation) :: tail ->
                match token.TokenName with
                | "IDENT" -> loop tail (name :: result)
                | "DOT" -> loop tail result
                | "WHITESPACE" -> loop tail result
                | "COMMA" -> loop tail result
                | "LPAREN" -> loop tail result
                | "RPAREN" -> loop tail result
                | _ -> result
        loop tokens []

    let getIdentIslands tokens =
        let tok = tokens |> Array.rev |> List.ofArray
        let rec loop tokens current result =
            match tokens with
            | [] -> match result, current with
                    | [],_ -> [current] //if the result is empty but we have a 'current', return that.
                    | _,[] -> result
                    | _,_ -> (current::result)
            | (name, token: TokenInformation) as head::tail ->
                match token.TokenName with
                | "IDENT"      -> loop tail (head::current) result
                | "DOT"        -> loop tail current result
                | "WHITESPACE" -> loop tail current result
                | _ -> match current with
                       | [] -> loop tail [] result
                       | _ -> loop tail [] (current :: result)//accumulate the current result with the runing result.
        loop tok [] []

    let getIslandWithinCursor islands cursor =
        islands |> List.find
         (fun xs -> let filtered = 
                        xs |> List.filter (fun (name: string, token: TokenInformation) -> 
                                               cursor >= token.LeftColumn && cursor <= token.RightColumn)
                    match filtered with
                    | [] -> false
                    | _ -> true)
    
    let getIdentAndResidue(doc:Mono.TextEditor.TextDocument) lineStr =
        let lines = doc.Lines 
                    |> Seq.map (fun line -> line.ToString()) 
                    |> Seq.toList
        let currentLineTokens = getLineTokens (doc.FileName.ToString()) lines lineStr
        let currentLineEndsWithDot = lineStr.TrimEnd().EndsWith(".")
        let ids = idents currentLineTokens

        match ids with
        | H::[] -> if currentLineEndsWithDot then Some("",[H]) 
                   else Some(H,[])
        | H::T  -> if currentLineEndsWithDot then Some("", ids) 
                   else match ids |> List.rev with
                        | H::T -> Some(H,T)
                        | _ -> failwith "Dunno"
        | [] -> None