Set of regular expressions that (might) tokenize julia input

julia_tokenizer.jl

function _julia_re(name, re)
    sym = symbol("julia_$(string(name))")
    quote
        $(esc(sym)) = $re
        $(esc(sym)) = "(?<"*$(string(name))*">"*$(esc(sym))*")"
    end
end

macro julia_re(name, re)
   _julia_re(name, re)
end

julia_op =(L"~ ! -> ' .' ... = := += -= *= /= //= .//= .*= ./= \= " *
           L".\= ^= .^= %= |= &= $= => <<= >>= >>>= ~ .+= .-= ? " *
           L"|| && <- -- --> > < >= <= == === != .> .< .>= .<= .== " *
           L".!= .= .! <: >: &> &< : .. + - .+ .- | $ << >> >>> .<< " *
           L".>> &>> &<< * / ./ % & .* \ .\ // .// ^ .^ :: ." ) |
          split | 
          (x -> sort!((op1, op2) -> length(op1) > length(op2), x)) |
          (x -> [replace(op, Regex(L"([\^.$|*+?])"), c -> "\\$c") for op in x]) | 
          (x -> join(x, "|"))

julia_op = "(<op>$julia_op)"
          
julia_paren = L"( ) [ ] { }" | 
               (x -> replace(x, r"\S", c->"\\$c")) | 
               (x -> replace(x, " ", "|"))
julia_paren = "(<paren>$julia_paren)"

@julia_re(hex, L"0x[0-9A-Fa-f]+")
@julia_re(oct, L"0o[0-7]+")
@julia_re(bin, L"0b[01]+")
@julia_re(float, L"\d+?\.\d+|\d+\.")
@julia_re(int, L"\d+")

@julia_re(num, join([julia_hex, julia_oct, julia_bin, julia_float, julia_int], "|"))

@julia_re(mac, L"@[A-Za-z]\w*")
@julia_re(ident, L"[A-Za-z]\w*")

@julia_re(space, L"\s+")

@julia_re(string, L"\"(?:\\?.)*\"")

@julia_re(comment, L"#.*\n")

julia_regex=Regex(I"$julia_space|$julia_comment|$julia_string|$julia_op|$julia_paren|$julia_num|$julia_mac|$julia_ident")