advanced_tokenize_example.rebol

REBOL []

string "say 'hello world!'"

action-rule:   [ some alphanumer ] 
action-coerce: func [match][ to-lit-word match ] ;; coercion function to apply to match
action:        [ action-rule action-coerce ]

apostext: [ delim "'" ]

tokens: tokenize string [
  save/coerce action     ;; expects second word in block to be a coercion function
  space
  save/annotate apostext ;; wraps match in block with name of rule prefixed
]

print mold tokens
;;== ['say ['apostext "hello world!"]]

simple_tokenize_example.rebol

REBOL []

string "say 'hello world!'"

word:     [ some alphanumer ] ;; alphanumer is a built in TOKENIZE charset
apostext: [ delim "'" ]       ;; delim is a TOKENIZE function that matches everything between given delimiter, which can be any valid PARSE rule

tokens: tokenize string [
  save word
  space
  save apostext
]

print mold tokens
;;== ["say" "hello world!"]