Setup scripts for MarkLogic custom tokenization tutorial at http://developer.marklogic.com/learn/custom-tokenization

setup1.xqy

xquery version "1.0-ml";
(: Create the database and forest on current host :)
import module namespace admin="http://marklogic.com/xdmp/admin" 
  at "/MarkLogic/admin.xqy";

let $config := admin:get-configuration() 
let $config := 
  admin:database-create($config, "tweets", 
    xdmp:database("Security"), xdmp:database("Schemas"))
let $config := admin:forest-create($config, "tweetsF1", xdmp:host(), ())
return admin:save-configuration($config)
;
xquery version "1.0-ml";
(: Associate forest with the database :)
import module namespace admin="http://marklogic.com/xdmp/admin" 
  at "/MarkLogic/admin.xqy";

let $config := admin:get-configuration()
let $dbid := admin:database-get-id($config, "tweets")
let $asid := admin:forest-get-id($config, "tweetsF1")
let $config := admin:database-attach-forest($config, $dbid, $asid)
return admin:save-configuration($config)     
;
xquery version "1.0-ml";
(: Create the fields 'tweet' and 'phone' with their overrides and
add a field word lexicon to 'tweet' :)
import module namespace admin = "http://marklogic.com/xdmp/admin" 
  at "/MarkLogic/admin.xqy";

let $config := admin:get-configuration()
let $dbid := xdmp:database("tweets")
let $f1 := admin:database-field("tweet", fn:false())
let $config := admin:database-add-field($config, $dbid, $f1)
let $e1 := admin:database-included-element("", "tweet", 1.0, "", "", "")
let $config := 
  admin:database-add-field-included-element($config, $dbid, "tweet", $e1)
let $config := 
  admin:database-add-field-tokenizer-override(
    $config, $dbid, "tweet",
    (admin:database-tokenizer-override("@","word"),
     admin:database-tokenizer-override("#","symbol"))
  )
let $l1 := 
  admin:database-word-lexicon("http://marklogic.com/collation/codepoint")
let $config := 
  admin:database-add-field-word-lexicon($config, $dbid, "tweet", $lexicon)
let $f2 := admin:database-field("phone", fn:false())
let $config := admin:database-add-field($config, $dbid, $f2)
let $e2 := admin:database-included-element("", "phone", 1.0, "", "", "")
let $config := 
  admin:database-add-field-included-element($config, $dbid, "tweet", $e2)
let $config := 
  admin:database-add-field-tokenizer-override(
    $config, $dbid, "phone",
    (admin:database-tokenizer-override("-","remove"),
     admin:database-tokenizer-override("(","remove"),
     admin:database-tokenizer-override(")","remove"),
     admin:database-tokenizer-override("+","remove"),
     admin:database-tokenizer-override(" ","remove"))
  )
return admin:save-configuration($config)
;
xquery version "1.0-ml";
(: Create a REST server and associate it with the database :)
import module namespace boot-util="http://marklogic.com/rest-api/bootstrap-util"
  at "/MarkLogic/rest-api/lib/bootstrap-util.xqy";

boot-util:bootstrap-rest-server("tweets", "Modules", "Default", "TweetServer", 8003)
;

setup2.sh

# make sure tweet_options.xml is in your current directory and
# replace user:password with your credentials and replace localhost with hostname as needed

curl --anyauth --user user:password -X POST -d@'./tweet_options.xml' \
  -i -H "Content-type:application/xml" \
  http://localhost:8003/v1/config/query/tweet

tweet_options.xml

<?xml version="1.0" encoding="UTF-8"?>
<options xmlns="http://marklogic.com/appservices/search">
  <search-option>unfiltered</search-option>
  <quality-weight>0</quality-weight>
  <return-plan>true</return-plan>
  <debug>true</debug>
  <constraint name="tweet">
    <word>
      <field name="tweet"/>
    </word>
  </constraint>
  <constraint name="phone">
    <word>
      <field name="phone"/>
    </word>
  </constraint>
  <default-suggestion-source>
    <word>
      <field name="tweet" collation="http://marklogic.com/collation/codepoint"/>
    </word>
  </default-suggestion-source>
</options>