stdlib.js

var M = (
  { set: (o, ...things) => Object.assign({}, o, ...things)

  , copy: (o) => M.set(o)

  , modify: (o, diff) =>
    { var result = M.copy(o)
      for (var key in o)
        if (diff[key])
          result[key] = diff[key](o[key])
      return result
    }
  
  , times: (n, f, o) => {
      for (var i = 0; i < n; i++)
        o = f(o)
      return o
    }
  , loop: (cont, ...state) => {
      while (true) {
        try {
          return cont((...newState) => {
            throw newState
          }, ...state)
        }
        catch (newState) {
          state = newState
        }
      }
    }
  }
)

module.exports = M

tokenizer.js

"use strict";

var assert = require('assert')

var { modify, loop, times } = require('./stdlib.js');

var Cursor = (
  { fromString: (text) =>
      ({ line: 1
       , col:  1
       , char: 0
       , text
       })

  , currentChar: (cursor) => cursor.text[cursor.char]


  , accountChar: (cursor) => {
    var isLineBreak = Cursor.currentChar(cursor) == "\n"
    return modify(cursor,
      { line: (it) => isLineBreak? it + 1 : it
      , col:  (it) => isLineBreak?      1 : it + 1
      , char: (it) => it + 1
      }
    )
  }
  
  , take: (length, i) => (
      { token:  i.text.substr(i.char, i.char + length)
      , cursor: times(length, Cursor.accountChar, i)
      }
    )
  
  , between: (from, to) => (
      { token:  from.text.substr(from.char, to.char)
      , cursor: to
      }
    )
  
  , takeWhile: (predicate, start) => (
      Cursor.between
        ( start
        , loop
          ( (recure, i) => {
              if (Cursor.currentChar(i) && predicate(Cursor.currentChar(i)))
                recure(Cursor.accountChar(i))
              else
                return i
            }
          , start
          )
        )
    )
  }
)

var isIn = (str) => {
  var set = []
  for (var i = 0; i < str.length; i++)
    set[str[i]] = true;
  return (c) => set[c]
}

var Predicates = (
  { isSpace:    isIn(' \n')
  , isSingular: isIn("({;,})")
  }
)

var Tokenizers = (
  { spaces: (i) => Cursor.takeWhile(isSpace)
  , fail:   (_) => undefined
  }
)

var i = Cursor.fromString(" \nadasd ad asdas ")

console.log(Cursor.takeWhile(c => c != 'd', i));