tokentools temporary repository

.gitignore

*.exe
*.dll
tags*
objs

lua_tokens.c

#define TT_LEXER
#include "lua_tokens.h"
#include <ctype.h>
#include <locale.h> /* for localeconv */


static const char *ltokens[] = {
#define X(a, b) b,
    lua_tokens
#undef X
};


/* helpers */

static void trydecpoint (tt_Lexer *L) {
  /*
  ** in case of format error, try to change decimal point separator to
  ** the one defined in the current locale and check again
  */
  lua_Lexer *ll = (lua_Lexer*)L;
  char old = ll->decpoint;
  ll->decpoint = localeconv()->decimal_point[0];
  ttL_replacebuffer(L, old, ll->decpoint);  /* try new decimal separator */
  if (!ttL_buffer2d(L)) {
    /* format error with correct decimal point: no more options */
    ttL_replacebuffer(L, ll->decpoint, '.');  /* undo change (for error message) */
    ttL_lexerror(L, "malformed number", TTK_NUMBER);
  }
}

static void read_numeral (tt_Lexer *L) {
  tt_assert(isdigit(ttL_current(L)));
  do {
    ttL_save_and_next(L);
    if (ttL_check_next(L, "EePp"))  /* exponent part? */
      ttL_check_next(L, "+-");  /* optional exponent sign */
  } while (isalnum(ttL_current(L)) || ttL_current(L) == '.');
  ttL_save(L, '\0');
  ttL_replacebuffer(L, '.', ((lua_Lexer*)L)->decpoint);  /* follow locale for decimal point */
  if (!ttL_buffer2d(L))  /* format error? */
    trydecpoint(L); /* try to update decimal point separator */
}

static int skip_sep (tt_Lexer *L) {
  /*
  ** skip a sequence '[=*[' or ']=*]' and return its number of '='s or
  ** -1 if sequence is malformed
  */
  int count = 0;
  int s = ttL_current(L);
  tt_assert(s == '[' || s == ']');
  ttL_save_and_next(L);
  while (ttL_current(L) == '=') {
    ttL_save_and_next(L);
    count++;
  }
  return (ttL_current(L) == s) ? count : (-count) - 1;
}

static void read_long_string (tt_Lexer *L, int isstring, int sep) {
  ttL_save_and_next(L);  /* skip 2nd `[' */
  if (ttL_isnewline(L))  /* string starts with a newline? */
    ttL_inclinenumber(L);  /* skip it */
  for (;;) {
    switch (ttL_current(L)) {
      case TTL_EOZ:
        ttL_lexerror(L, isstring ? "unfinished long string" :
                                 "unfinished long comment", TTK_EOS);
        break;  /* to avoid warnings */
      case ']': {
        if (skip_sep(L) == sep) {
          ttL_save_and_next(L);  /* skip 2nd `]' */
          goto endloop;
        }
        break;
      }
      case '\n': case '\r': {
        ttL_save(L, '\n');
        ttL_inclinenumber(L);
        if (!isstring) ttL_resetbuffer(L);  /* avoid wasting space */
        break;
      }
      default: {
        if (isstring) ttL_save_and_next(L);
        else ttL_next(L);
      }
    }
  } endloop:
  ttL_movebuffer(L, ttL_buffer(L)+2+sep,
                    ttL_bufflen(L)-2*(2+sep));
}

static void escerror (tt_Lexer *L, int *c, int n, const char *msg) {
  int i;
  ttL_resetbuffer(L);  /* prepare error message */
  ttL_save(L, '\\');
  for (i = 0; i < n && c[i] != TTL_EOZ; i++)
    ttL_save(L, c[i]);
  ttL_lexerror(L, msg, TTK_STRING);
}

static int hexavalue(int ch) {
    if (ch >= '0' && ch <= '9')
        return ch-'0';
    else {
        ch = tolower(ch);
        if (ch >= 'a' && ch <= 'f')
            return 10+ch-'a';
    }
    return 0;
}

static int readhexaesc (tt_Lexer *L) {
  int c[3], i;  /* keep input for error message */
  int r = 0;  /* result accumulator */
  c[0] = 'x';  /* for error message */
  for (i = 1; i < 3; i++) {  /* read two hexa digits */
    c[i] = ttL_next(L);
    if (!isxdigit(c[i]))
      escerror(L, c, i + 1, "hexadecimal digit expected");
    r = (r << 4) + hexavalue(c[i]);
  }
  return r;
}

static int readdecesc (tt_Lexer *L) {
  int c[3], i;
  int r = 0;  /* result accumulator */
  for (i = 0; i < 3 && isdigit(ttL_current(L)); i++) {  /* read up to 3 digits */
    c[i] = ttL_current(L);
    r = 10*r + c[i] - '0';
    ttL_next(L);
  }
  if (r > UCHAR_MAX)
    escerror(L, c, i, "decimal escape too large");
  return r;
}

static void read_string (tt_Lexer *L, int del) {
  ttL_save_and_next(L);  /* keep delimiter (for error messages) */
  while (ttL_current(L) != del) {
    switch (ttL_current(L)) {
      case TTL_EOZ:
        ttL_lexerror(L, "unfinished string", TTK_EOS);
        break;  /* to avoid warnings */
      case '\n':
      case '\r':
        ttL_lexerror(L, "unfinished string", TTK_STRING);
        break;  /* to avoid warnings */
      case '\\': {  /* escape sequences */
        int c;  /* final character to be saved */
        ttL_next(L);  /* do not ttL_save the `\' */
        switch (ttL_current(L)) {
          case 'a': c = '\a'; goto read_save;
          case 'b': c = '\b'; goto read_save;
          case 'f': c = '\f'; goto read_save;
          case 'n': c = '\n'; goto read_save;
          case 'r': c = '\r'; goto read_save;
          case 't': c = '\t'; goto read_save;
          case 'v': c = '\v'; goto read_save;
          case 'x': c = readhexaesc(L); goto read_save;
          case '\n': case '\r':
            ttL_inclinenumber(L); c = '\n'; goto only_save;
          case '\\': case '\"': case '\'':
            c = ttL_current(L); goto read_save;
          case TTL_EOZ: goto no_save;  /* will raise an error ttL_next loop */
          case 'z': {  /* zap following span of spaces */
            ttL_next(L);  /* skip the 'z' */
            while (isspace(ttL_current(L))) {
              if (ttL_isnewline(L)) ttL_inclinenumber(L);
              else ttL_next(L);
            }
            goto no_save;
          }
          default: {
            if (!isdigit(ttL_current(L))) {
              int ch = ttL_current(L);
              escerror(L, &ch, 1, "invalid escape sequence");
            }
            /* digital escape \ddd */
            c = readdecesc(L);
            goto only_save;
          }
        }
       read_save: ttL_next(L);  /* read next character */
       only_save: ttL_save(L, c);  /* save 'c' */
       no_save: break;
      }
      default:
        ttL_save_and_next(L);
    }
  }
  ttL_save_and_next(L);  /* skip delimiter */
  ttL_movebuffer(L, ttL_buffer(L)+1, ttL_bufflen(L)-2);
}


/* implements */

static int llex (tt_Lexer *L) {
  for (;;) {
    switch (ttL_current(L)) {
      case '\n': case '\r': {  /* line breaks */
        ttL_inclinenumber(L);
        break;
      }
      case ' ': case '\f': case '\t': case '\v': {  /* spaces */
        ttL_next(L);
        break;
      }
      case '-': {  /* '-' or '--' (comment) */
        ttL_next(L);
        if (ttL_current(L) != '-') return '-';
        /* else is a comment */
        ttL_next(L);
        if (ttL_current(L) == '[') {  /* long comment? */
          int sep = skip_sep(L);
          ttL_resetbuffer(L);  /* `skip_sep' may dirty the buffer */
          if (sep >= 0) {
            read_long_string(L, 0, sep);  /* skip long comment */
            ttL_resetbuffer(L);  /* previous call may dirty the buff. */
            break;
          }
        }
        /* else short comment */
        while (!ttL_isnewline(L) && ttL_current(L) != TTL_EOZ)
          ttL_next(L);  /* skip until end of line (or end of file) */
        break;
      }
      case '[': {  /* long string or simply '[' */
        int sep = skip_sep(L);
        if (sep >= 0) {
          read_long_string(L, 1, sep);
          return TTK_STRING;
        }
        else if (sep == -1) return '[';
        else ttL_lexerror(L, "invalid long string delimiter", TTK_STRING);
      }
      case '=': {
        ttL_next(L);
        if (ttL_current(L) != '=') return '=';
        else { ttL_next(L); return TTK_EQ; }
      }
      case '<': {
        ttL_next(L);
        if (ttL_current(L) != '=') return '<';
        else { ttL_next(L); return TTK_LE; }
      }
      case '>': {
        ttL_next(L);
        if (ttL_current(L) != '=') return '>';
        else { ttL_next(L); return TTK_GE; }
      }
      case '~': {
        ttL_next(L);
        if (ttL_current(L) != '=') return '~';
        else { ttL_next(L); return TTK_NE; }
      }
      case ':': {
        ttL_next(L);
        if (ttL_current(L) != ':') return ':';
        else { ttL_next(L); return TTK_DBCOLON; }
      }
      case '"': case '\'': {  /* short literal strings */
        read_string(L, ttL_current(L));
        return TTK_STRING;
      }
      case '.': {  /* '.', '..', '...', or number */
        ttL_save_and_next(L);
        if (ttL_check_next(L, ".")) {
          if (ttL_check_next(L, "."))
            return TTK_DOTS;   /* '...' */
          else return TTK_CONCAT;   /* '..' */
        }
        else if (!isdigit(ttL_current(L))) return '.';
        /* else go through */
      }
      case '0': case '1': case '2': case '3': case '4':
      case '5': case '6': case '7': case '8': case '9': {
        read_numeral(L);
        return TTK_NUMBER;
      }
      case TTL_EOZ: {
        return TTK_EOS;
      }
      default: {
        if (isalpha(ttL_current(L)) || ttL_current(L) == '_') {  /* identifier or reserved word? */
          int reversed;
          do {
            ttL_save_and_next(L);
          } while (isalnum(ttL_current(L)) || ttL_current(L) == '_');
          if ((reversed = ttL_iskeyword(L, ltokens, TTK_LUA_NUM_RESERVED)) != 0)
            return reversed + TTK_FIRST_REVERSED;
          return TTK_NAME;
        }
        else {  /* single-char tokens (+ - / ...) */
          int c = ttL_current(L);
          ttL_next(L);
          return c;
        }
      }
    }
  }
}

static int linitlexer(tt_Lexer *t, size_t lexersize) {
    lua_Lexer *ll = (lua_Lexer*)t;
    tt_assert(lexersize == sizeof(lua_Lexer));
    ll->decpoint = 0;
    return ttL_lexer_init(t);
}

static const char *ltostring(tt_Lexer *L, tt_TokenId tokenid) {
  if (tokenid > TTK_LUA_BEFORE_RESERVED && tokenid < TTK_LUA_AFTER_RESERVED)
    return ltokens[tokenid+1 - TTK_LUA_BEFORE_RESERVED];
  return ttL_tostring(L, tokenid);
}

static ttL_Reg reg = {
    TTL_LEXER(lua)
    /* .init = */       linitlexer,
    /* .free = */       NULL,
    /* .lexer = */      llex,
    /* .tostring = */   ltostring,
};

TT_LIBAPI int ttlexeropen_lua(tt_State *S) {
    ttL_register(S, &reg);
    return 1;
}

/* cc: flags+='-pedantic -I../src' input="*.c ../src/tt*.c" */

lua_tokens.h

#ifndef lua_tokens_h
#define lua_tokens_h


#include <ttlexer.h>


#define lua_tokens \
  X(AND,      "and"      ) \
  X(BREAK,    "break"    ) \
  X(DO,       "do"       ) \
  X(ELSE,     "else"     ) \
  X(ELSEIF,   "elseif"   ) \
  X(END,      "end"      ) \
  X(FALSE,    "false"    ) \
  X(FOR,      "for"      ) \
  X(FUNCTION, "function" ) \
  X(GOTO,     "goto"     ) \
  X(IF,       "if"       ) \
  X(IN,       "in"       ) \
  X(LOCAL,    "local"    ) \
  X(NIL,      "nil"      ) \
  X(NOT,      "not"      ) \
  X(OR,       "or"       ) \
  X(REPEAT,   "repeat"   ) \
  X(RETURN,   "return"   ) \
  X(THEN,     "then"     ) \
  X(TRUE,     "true"     ) \
  X(UNTIL,    "until"    ) \
  X(WHILE,    "while"    ) \
  \
  /* other terminal symbols */ \
  X(CONCAT,  ".."  ) \
  X(DOTS,    "..." ) \
  X(EQ,      "=="  ) \
  X(GE,      ">="  ) \
  X(LE,      "<="  ) \
  X(NE,      "~="  ) \
  X(DBCOLON, "::"  )


enum TTL_LUA_RESERVED {
    TTK_LUA_BEFORE_RESERVED = TTK_FIRST_REVERSED-1,
#define X(a, b) TTK_ ## a,
    lua_tokens
#undef X
    TTK_LUA_AFTER_RESERVED
};

#define TTK_LUA_NUM_RESERVED (TTK_WHILE-TTK_FIRST_REVERSED+1)

typedef struct {
    tt_Lexer base;
    int decpoint;
} lua_Lexer;

TT_LIBAPI int tt_lexer_lua(tt_State *S);
TT_LIBAPI int tt_lexer_lua_init (lua_Lexer *L, tt_Reader *reader, void *ud);

#endif /* lua_tokens_h */

Makefile

tokentools.h

#ifndef tokentools_h
#define tokentools_h


#include "ttconfig.h"
#include <stddef.h>


#define TTK_EOS            (TTK_FIRST_TOKEN+0)
#define TTK_NAME           (TTK_FIRST_TOKEN+1)
#define TTK_STRING         (TTK_FIRST_TOKEN+2)
#define TTK_NUMBER         (TTK_FIRST_TOKEN+3)
#define TTK_INTEGER        (TTK_FIRST_TOKEN+4)
#define TTK_FIRST_REVERSED (TTK_FIRST_TOKEN+10)


typedef       void *tt_Alloc  (void *ud, void *mem, size_t nsize, size_t osize);
typedef const char *tt_Reader (tt_State *S, void *ud, size_t *sz);


TT_API tt_State *tt_new   (tt_Alloc *allocf, void *ud);
TT_API void      tt_close (tt_State *S);

TT_API void *tt_uservalue    (tt_State *S);
TT_API void  tt_setuservalue (tt_State *S, void *ud);

TT_API tt_Alloc *tt_allocf    (tt_State *S, void **ud);
TT_API void      tt_setallocf (tt_State *S, tt_Alloc *allocf, void *ud);

TT_API tt_Lexer *tt_lexer_new      (tt_State *S, const char *lexername);
TT_API tt_Lexer *tt_lexer_setinput (tt_State *S, tt_Lexer *L, tt_Reader *reader, void *ud);
TT_API void      tt_lexer_close    (tt_State *S, tt_Lexer *L);

TT_API tt_Lexer *tt_lexer    (tt_State *S);
TT_API int       tt_setlexer (tt_State *S, tt_Lexer *lexer);

TT_API const char *tt_tokename (tt_State *S, tt_Token *t);

TT_API tt_TokenId tt_next      (tt_State *S);
TT_API tt_TokenId tt_lookahead (tt_State *S);

TT_API tt_Token *tt_current (tt_State *S);
TT_API tt_Token *tt_ahead   (tt_State *S);

TT_API const char *tt_filename (tt_State *S);

TT_API int tt_linenumber (tt_State *S);
TT_API int tt_linecolume (tt_State *S);

TT_API void tt_lexerror (tt_State *S, const char *msg);


TT_API tt_TokenId tt_token   (tt_Token *t);
TT_API tt_String  tt_string  (tt_Token *t);
TT_API tt_Number  tt_number  (tt_Token *t);
TT_API tt_Integer tt_integer (tt_Token *t);


#endif /* tokentools.h */

ttbuffer.c

#define TT_CORE
#include <ttlexer.h>

#undef ttL_buffer
#undef ttL_bufflen
#undef ttL_resetbuffer
#undef ttL_resizebuffer
#undef ttL_copybuffer
#undef ttL_movebuffer

char *tt_buffer(tt_Lexer *L) {
    return L->buffer->buffer;
}

size_t ttL_bufflen(tt_Lexer *L) {
    return L->buffer->bufflen;
}

void ttL_resetbuffer(tt_Lexer *L) {
    L->buffer->bufflen = 0;
}

void ttL_resizebuffer(tt_Lexer *L, size_t size) {
    ttM_reallocvector(L->S, L->buffer->buffer, L->buffer->buffsize,
            size, char);
    L->buffer->buffsize = size;
}

char *ttL_copybuffer(tt_Lexer *L, const char *s, size_t len) {
    if (len > L->buffer->bufflen)
        ttL_resizebuffer(L, len);
    memcpy(L->buffer->buffer, s, len);
    L->buffer->bufflen = len;
    return L->buffer->buffer;
}

char *ttL_movebuffer(tt_Lexer *L, const char *s, size_t len) {
    if (len > L->buffer->bufflen)
        ttL_resizebuffer(L, len);
    memmove(L->buffer->buffer, s, len);
    L->buffer->bufflen = len;
    return L->buffer->buffer;
}

char *ttL_replacebuffer(tt_Lexer *L, int oldchar, int newchar) {
    int i;
    struct tt_Buffer *buff = L->buffer;
    for (i = 0; i < buff->bufflen; ++i) {
        if (buff->buffer[i] == oldchar)
            buff->buffer[i] = newchar;
    }
    return buff->buffer;
}

ttconfig.h

#ifndef ttconfig_h
#define ttconfig_h


#include <limits.h> /* for UCHAR_MAX */


#ifndef TT_API
#  ifdef __WIN32__
#    if !defined(TT_CORE) && !defined(TT_LEXER)
#      define TT_API __declspec(dllimport)
#    else
#      define TT_API __declspec(dllexport)
#    endif 
#  else
#    define TT_API
#  endif 
#endif 
#define TT_LIBAPI TT_API

#ifndef TTI_FUNC
#define TTI_FUNC
#endif


#ifndef TTK_FIRST_TOKEN
#define TTK_FIRST_TOKEN UCHAR_MAX
#endif

#ifndef TT_TOKENID
#define TT_TOKENID int
#endif

#ifndef TT_STRING_T 
#define TT_STRING_T const char*
#endif

#ifndef TT_NUMBER_T
#define TT_NUMBER_T double
#endif

#ifndef TT_INTEGER_T
#define TT_INTEGER_T unsigned long
#endif

#ifndef MAX_SIZE_T
#define MAX_SIZE_T ((size_t)(~(size_t)0-2))
#endif

typedef TT_TOKENID tt_TokenId;
typedef TT_STRING_T tt_String;
typedef TT_NUMBER_T tt_Number;
typedef TT_INTEGER_T tt_Integer;
typedef struct tt_State tt_State;
typedef struct tt_Token tt_Token;
typedef struct tt_Lexer tt_Lexer;
typedef struct ttL_Reg ttL_Reg;

#endif /* ttconfig_h */

ttinit.c

#include "tokentools.h"

ttlexer.c

#define TT_CORE
#include <ttlexer.h>
#include <ttzio.h>

#undef ttL_next

int ttL_next(tt_Lexer *L) {
    L->current = tt_zio_getc(L->z);
    return L->current;
}

void ttL_save(tt_Lexer *L, int ch) {
    tt_Buffer *b = L->buffer;
    if (b->bufflen + 1 > b->buffsize) {
        size_t newsize;
        if (b->buffsize >= MAX_SIZE_T/2)
            ttL_lexerror(L, "lexical element too loog", 0);
        newsize = b->buffsize * 2;
        (void)ttL_resizebuffer(L, newsize);
    }
    b->buffer[b->bufflen++] = (char)ch;
}

int ttL_check_next(tt_Lexer *L, const char *setchars) {
  if (L->current == '\0' || !strchr(setchars, L->current))
    return 0;
  ttL_save_and_next(L);
  return 1;
}

ttlexer.h

#ifndef tt_h
#define tt_h


#include "ttconfig.h"
#include <stddef.h>


#define TTL_EOZ (-1)


struct ttL_Reg {
    const char *name;
    size_t lexersize;

    /* methods of lexer */
    int  (*init_lexer) (tt_Lexer *L, size_t lsize);
    void (*free_lexer) (tt_Lexer *L);
    int  (*lexer) (tt_Lexer *L);
    const char *(*tostring) (tt_Lexer *L, tt_TokenId tokenid);
};


#ifndef tt_assert
#  include <assert.h>
#  define tt_assert(e) assert(e)
#endif

int ttL_lexer_init (tt_Lexer *L);

int ttL_inclinenumber (tt_Lexer *L);

int  ttL_current    (tt_Lexer *L);
int  ttL_isnewline  (tt_Lexer *L);
int  ttL_next       (tt_Lexer *L);
void ttL_save       (tt_Lexer *L, int ch);
int  ttL_check_next (tt_Lexer *L, const char *setchars);
#define ttL_save_and_next(L) (ttL_save(L, ttL_current(L)), ttL_next(L))

const char *ttL_tostring (tt_Lexer *L, tt_TokenId tokenid);

char  *ttL_buffer        (tt_Lexer *L);
size_t ttL_bufflen       (tt_Lexer *L);
void   ttL_resetbuffer   (tt_Lexer *L);
void   ttL_resizebuffer  (tt_Lexer *L, size_t nsize);
char  *ttL_replacebuffer (tt_Lexer *L, int oldchar, int newchar);
char  *ttL_copybuffer    (tt_Lexer *L, const char *s, size_t len);
char  *ttL_movebuffer    (tt_Lexer *L, const char *s, size_t len);
#define ttL_freebuffer(L) ttL_resetbuffer(L, 0)

tt_Number  ttL_buffer2d (tt_Lexer *L);
tt_Integer ttL_buffer2i (tt_Lexer *L);
int ttL_iskeyword (tt_Lexer *L, const char **tokens, size_t tokenlen);
int ttL_lexerror (tt_Lexer *L, const char *msg, tt_TokenId token);

int ttL_register (tt_State *S, ttL_Reg *reg);


#define TTL_LEXER(name) #name, sizeof(name##_Lexer),

#if !defined(TT_CORE) || !defined(TT_RESTRICT_API)
#  include <ttprivate.h>
#  define ttL_current(L)       ((L)->current)
#  define ttL_isnewline(L)     ((L)->current == '\r' || (L)->current == '\n')
#  define ttL_inclinenumber(L) ((L)->linenumber++, (L)->linecolume = 0)
#  define ttL_buffer(L)        ((L)->buffer->buffer)
#  define ttL_bufflen(L)       ((L)->buffer->bufflen)
#  define ttL_resetbuffer(L)   ((L)->buffer->bufflen = 0)
#  include "ttmem.h"
#  define ttL_resizebuffer(L,size) \
    (ttM_reallocvector((L)->S, (L)->buffer->buffer, (L)->buffer->buffsize, size, char), \
     (L)->buffer->buffsize = (size), (L)->buffer->buffer)
#  include <string.h>
#  define ttL_copybuffer(L,s,len) \
    ((len) > ttL_bufflen(L) ? (void)ttL_resizebuffer(L, len) : \
     (void)(ttL_bufflen(L) = len), \
     memcpy(ttL_buffer(L), s, len))
#  define ttL_movebuffer(L,s,len) \
    ((len) > ttL_bufflen(L) ? (void)ttL_resizebuffer(L, len) : \
     (void)(ttL_bufflen(L) = len), \
     memmove(ttL_buffer(L), s, len))

#  include <ttzio.h>
#  define ttL_next(L) (L->current = tt_zio_getc(L->z))
#endif

#endif /* tt_h */

ttmem.h

#include "ttprivate.h"

#define ttM_reallocv(L,b,on,n,e) \
        (((size_t)(n)+1) > MAX_SIZE_T/(e) ?  /* +1 to avoid warnings */ \
                (ttM_toobig(L), (void *)0) : \
                 ttM_realloc_(L, (b), (on)*(e), (n)*(e)))

#define ttM_reallocvector(L, v,oldn,n,t) \
   ((v)=(t*)ttM_reallocv(L, v, oldn, n, sizeof(t)))

void ttM_toobig (tt_State *L);

TTI_FUNC void *ttM_realloc_ (tt_State *L, void *block, size_t oldsize, size_t size);
TTI_FUNC void *ttM_growaux_ (tt_State *L, void *block, int *size,
                              size_t size_elem, int limit,
                              const char *what);

ttprivate.h

#ifndef ttpriv_h
#define ttpriv_h


#if !defined(TT_CORE) && !defined(TT_LEXER)
#error "you mustn't use restricted api!"
#endif

#include "tokentools.h"
#include <stddef.h>

typedef struct tt_Buffer tt_Buffer;

struct tt_Buffer {
    char *buffer;
    size_t bufflen;
    size_t buffsize;
};

struct tt_SemInfo {
    tt_Integer i;
    tt_Number n;
    struct tt_Buffer s;
};

struct tt_Token {
    tt_TokenId tokenid;
    struct tt_SemInfo seminfo;
};

struct tt_State {
    tt_Alloc *allocf;
    void *ud;
    void *uservalue;
    tt_Lexer *curlexer;
    ttL_Reg *lexers;
};

struct tt_Lexer {
    ttL_Reg *vt;
    tt_State *S;
    struct tt_Zio *z;
    int linenumber;
    int linecolume;
    int current;
    struct tt_Buffer *buffer;
    tt_Token token;
    tt_Token ahead;
};


#endif /* ttpriv_h */

ttstate.c

#define TT_CORE
#include <ttprivate.h>

ttzio.c

#define TT_CORE
#include <ttlexer.h>
#include <ttzio.h>


void tt_zio_init(tt_State *S, tt_ZIO *z, tt_Reader *reader, void *ud) {
    z->S = S;
    z->reader = reader;
    z->ud = ud;
    z->n = 0;
    z->p = NULL;
}

size_t tt_zio_read(tt_ZIO *z, void *b, size_t n) {
  while (n) {
    size_t m;
    if (z->n == 0) {  /* no bytes in buffer? */
      if (tt_zio_fill(z) == TTL_EOZ)  /* try to read more */
        return n;  /* no more input; return number of missing bytes */
      else {
        z->n++;  /* luaZ_fill consumed first byte; put it back */
        z->p--;
      }
    }
    m = (n <= z->n) ? n : z->n;  /* min. between n and z->n */
    memcpy(b, z->p, m);
    z->n -= m;
    z->p += m;
    b = (char *)b + m;
    n -= m;
  }
  return 0;
}

int tt_zio_fill(tt_ZIO *z) {
    size_t size;
    tt_State *S = z->S;
    const char *buff;
    /* tt_unlock(S); */
    buff = z->reader(S, z->ud, &size);
    /* tt_lock(S); */
    if (buff == NULL || size == 0)
        return TTL_EOZ;
    z->n = size - 1;
    z->p = buff;
    return (unsigned char)(*(z->p++));
}

ttzio.h

#ifndef ttzio_h
#define ttzio_h


#include <tokentools.h>


typedef struct tt_Zio tt_ZIO;

#define tt_zio_getc(z) (((z)->n--)>0 ? (unsigned char)(*(z)->p++) : tt_zio_fill(z))

TTI_FUNC void   tt_zio_init (tt_State *S, tt_ZIO *z, tt_Reader *reader, void *ud);
TTI_FUNC size_t tt_zio_read (tt_ZIO *z, void *b, size_t n);

/* private part */

struct tt_Zio {
    size_t n;
    const char *p;
    tt_Reader *reader;
    void *ud;
    tt_State *S; /* used by tt_Reader */
};

TTI_FUNC int tt_zio_fill (tt_ZIO *z);

#endif /* ttzio_h */