composingTokenizers.js

const initialState = {
  source: `this is a text
spread (in many) lines,
- not so many though`
}

const identityParser = state => token => state

const characterNumber = parser => state => token => {
  const currentCharacterNumber = state.currentCharacterNumber == null
    ? 0
    : state.currentCharacterNumber

  return parser({
    ...state,
    currentCharacterNumber: token.value === '\n'
      ? 0
      : currentCharacterNumber + 1
  })(token)
}

const lineNumber = parser => state => token => {
  const currentLineNumber = state.currentLineNumber == null
    ? 0
    : state.currentLineNumber

  const newState = {
    ...state,
    currentLineNumber: token.value === '\n'
      ? currentLineNumber + 1
      : currentLineNumber
  }

  return parser(newState)(token)
}

const charactersParser = parser => state => () => {
  const characters = state.source
    .split('')
    .reduce((characters, character, index) => [
      ...characters,
      {
        value: character,
        index,
      }
    ], [])

  const newState = {
    ...state,
    characters
  }

  return characters.reduce((
    state, character) => parser(state)(character), newState
  )
}

const parser = charactersParser(lineNumber(characterNumber(identityParser)))

console.log(parser(initialState)())