Add Attribute Tokens

stackoverflow_import_ser.erl

add_attribute_tokens(Id, AttributeName, AttributeValue) ->
    add_facet( Id, AttributeName, AttributeValue ),
    Tokens = string:tokens(string:to_lower(AttributeValue), " ,.:;-"),
    TokensAndPositions = lists:zip( Tokens, lists:seq( 1, length(Tokens))),
    lists:foreach(
        fun({Token, Position}) ->
            keyword_ser:add_keyword_position( Token, Id, Position)
        end,
        TokensAndPositions
    ).