RE: parser

[Luiz Henrique de Figueiredo <lhf@...>]

>From lua-l@tecgraf.puc-rio.br  Fri Jun  2 14:25:52 2000
>From: "Martin Dvorak" <mdvorak@ninell.cz>

>So if you are so kind to send me the code to reuse Lua's lexer

The program below reuses Lua's lexer. It "compress" Lua source code,
removing comments and multiple blanks, but preserves line breaks.
This particular application is not really important (we just needed it here at
TeCGraf), but I hope the code shows how to use the lexer and some inside
information from Lua's core.
Of course, all internal details are subject to change. The code below compiles
with Lua 4.0 alpha, but similar code should work for 3.2.

One thing that must be clear is that comments are lost, and so it might not
be the exact solution needed for editors and beautifiers.

--lhf

/* cc ltokens.c -I../lua ../lua/liblua.a */

#include <stdio.h>

#define LUA_REENTRANT
#include "lauxlib.h"
#include "llex.h"
#include "lzio.h"

static void quote(const char* s)
{
 putchar('"');
 for (; *s; s++)
 {
  switch (*s)
  {
   case '"': printf("\\\""); break;
   case '\a': printf("\\a"); break;
   case '\b': printf("\\b"); break;
   case '\f': printf("\\f"); break;
   case '\n': printf("\\n"); break;
   case '\r': printf("\\r"); break;
   case '\t': printf("\\t"); break;
   case '\v': printf("\\v"); break;
   default: putchar(*s); break;
  }
 }
 putchar('"');
}

#define NEEDSPACE(t)	(((t)>=TK_AND && (t)<=TK_NAME) || (t)==TK_NUMBER)

static void tokens(FILE* f, const char* filename)
{
 ZIO z;
 int ln=1;
 int lt=0;
 LexState lexstate;
 lua_State* L=lua_newstate(NULL);
 luaZ_Fopen(&z,f,filename);
 luaX_setinput(L,&lexstate,&z);
 while (1)
 {
  char b[TOKEN_LEN];
  int t=luaX_lex(&lexstate);
  if (lexstate.linenumber!=ln)
  {
   while (lexstate.linenumber!=ln++) printf("\n");
   ln=lexstate.linenumber;
   lt=0;
  }
  if (NEEDSPACE(lt) && NEEDSPACE(t)) printf(" ");
  switch (t)
  {
   case TK_EOS:
    return;
   case TK_STRING:
    quote(lexstate.seminfo.ts->str);
    break;
   case TK_NAME:
    printf("%s",lexstate.seminfo.ts->str);
    break;
   case TK_NUMBER:
    printf("%.16g",lexstate.seminfo.r);		/* use text read by lex? */
#if 0
    luaL_addchar(L,0);
    printf("-- %s\n",luaL_buffer(L));
#endif
    break;
   case TK_CONCAT:
    if (lt==TK_NUMBER) printf(" ");
   default:
    luaX_token2str(t,b);
    printf("%s",b);
  }
  lt=t;
 }
}

int main(void)
{
 tokens(stdin,"(stdin)");
 return 0;
}