Bug in cpcall

["Patrick Donnelly" <batrick.donnelly@...>]

I believe cpcall does some improper memory handling somewhere because
it attempts to free an invalid pointer. The attached c code provides a
reproducible case. The basic idea is to limit the memory usage of a
lua_State. (Looking at the code) In my application, I don't actually
have to call cpcall multiple times (it actually happens the first
time).

It should compile as:
gcc -shared -o test.so test_alloc.c
And run as:
batrick@waterdeep:~/lua$ lua -e "require'test'"
Segmentation fault

This may happen in lua_pcall() as well (I haven't checked). There is a
bug in LuaTask using pcalls that cause a segfault and I get the
feeling this may be it.

-- 
-Patrick Donnelly

"One of the lessons of history is that nothing is often a good thing
to do and always a clever thing to say."

-Will Durant

#include <lua.h>
#include <lauxlib.h>
#include <stdio.h>

static size_t limit = 10000;
static size_t mem_used = 0;

static void *my_Alloc(void *ud, void *ptr, size_t osize, size_t nsize)
{
  (void) ud;
    if (nsize == 0)
    {
      mem_used -= osize;
      free(ptr);
      return NULL;
    }
    else if (nsize + mem_used > limit)
      return NULL;
    else
    {
      mem_used += nsize;
      return realloc(ptr, nsize);
    }
}

static int some_func(lua_State *L)
{
  return 0;
}

static int do_stuff(lua_State *L)
{
  lua_createtable(L, 0, 1);
  lua_pushcfunction(L, some_func);
  lua_setfield(L, -2, "__gc");
  lua_setfield(L, LUA_REGISTRYINDEX, "thread_value_gc_m");
  lua_createtable(L, 0, 100);
  lua_createtable(L, 0, 1);
  lua_pushstring(L, "__mode");
  lua_pushstring(L, "k");
  lua_settable(L, -3);
  lua_setmetatable(L, -2);
  lua_setfield(L, LUA_REGISTRYINDEX, "other_worldly_references");
  lua_createtable(L, 0, 100);
  lua_setfield(L, LUA_REGISTRYINDEX, "my_worldly_references");
  return 0;
}

int luaopen_test(lua_State *L)
{
  int i;
  int status;
  for (i = 0; i < 20000; i++)
  {
    int j;
    lua_State *LL;
    limit += 1;
    mem_used = 0;
    LL = lua_newstate(my_Alloc, NULL);
    if (LL == NULL)
      continue;
    for (j = 1; j<1000;j++)
     status = lua_cpcall(LL, do_stuff, NULL); /* does not panic */
    switch(status)
    {
      case 0:
        printf("success!\n");
        break;
      case LUA_ERRRUN:
      case LUA_ERRERR:
      case LUA_ERRMEM:
        printf("error = %d\n", status);
    }
    lua_close(LL);
  }
  return 0;
}