Peekerate json tokenizer

json-tokenizer.js

import peekerate from '@iter-tools/peekerate';

export function* tokenize(input) {
  const peekr = peekerate(input);

  while (!peekr.done) {
    const char = peekr.value;
    if (char === '"') {
      // yields three tokens: " content "
      yield* tokenizeString(peekr);
    } else if ('{}[]:,'.includes(char)) {
      yield t.token(char);
    } else {
      throw new SyntaxError();
    }
    peekr.advance();
  }
}

function* tokenizeString(peekr) {
  let escaped = false;
  let literal = '';

  peekr.advance(); // "

  while(!peekr.done) {
    const char = peekr.value;
    if (escaped) {
      if (escapes.hasOwnProperty(char)) {
        literal += escapes[char];
      } else if (char === 'u') {
        // TODO handle unicode escapes : (
      } else {
        literal += char;
      }
    } else if (char === '\\') {
      escaped = true;
    } else if (char === '"') {
      yield t.literal(literal);
      yield '"';
      return;
    } else {
      literal += char;
    }
    peekr.advance();
  }
  peekr.advance(); // "
}