[BUG] call hook again when lua yield in count hook

["G.k Ray" <gzlkylin@...>]

Bug Summary(GPT-4):

In Lua 5.4.6, there may be a bug that causes the 'CALL' hook to be
called again when a Lua yield occurs in the 'COUNT' hook. This issue
happens when running Lua code with local functions that are not
variadic functions.

Example Code:

The example code provided demonstrates the issue by setting up a
'CALL' hook and a 'COUNT' hook with a yield. The Lua code contains a
local function that is not a variadic function. When running the code,
the output shows that the 'CALL' hook is called twice, which is
unexpected behavior.

This issue does not occur when calling a variadic function or when not
yielding in the 'COUNT' hook.

Possible Cause:

The issue seems to be caused by the luaD_hookcall function not being
discarded after a yield and resume, causing it to be called again.

Recommendation:

A possible fix for this issue would be to ensure that the
luaD_hookcall function is properly discarded after a yield and resume,
preventing it from being called again.

----------------------------------------------------------

Lua Version 5.4.6

Example Code:

#include <lua.hpp>

#define QUOTE(...) #__VA_ARGS__

static void call_hook(lua_State *L, lua_Debug *ar);
static void pause_hook(lua_State *L, lua_Debug *ar) {
 lua_sethook(L, call_hook, LUA_MASKCALL, 0);
 lua_yield(L, 0);
}

static void call_hook(lua_State *L, lua_Debug *ar) {
  lua_getinfo(L, "S", ar);
  printf("call_hook event %d addr %d what %s\n",
ar->event,ar->linedefined, ar->what);
  lua_sethook(L, pause_hook, LUA_MASKCOUNT, 1);
}

int main(int argc, char* argv[]) {
  lua_State *L = luaL_newstate();
  lua_State *l = lua_newthread(L);
  luaL_openlibs(l);

  luaL_loadstring(l, QUOTE(
    local f = function(...)
    end
    f(1)
  ));

  lua_sethook(l, call_hook, LUA_MASKCALL, 0);
  int rc = 1;
  int nres;
  while (rc) {
    rc = lua_resume(l, NULL, 0, &nres);
    printf("ret = %d\n", rc);
  }
  lua_close(L);
}
```

lua code:
```
local f = function()
end
f(1)
```

the output:
```
call_hook event 0 addr 0 what main
ret = 1
call_hook event 0 addr 1 what Lua
ret = 1
call_hook event 0 addr 1 what Lua
ret = 0
```
When running the code, the output shows that the 'CALL' hook is called
twice, which is unexpected behavior.


SPECIAL, when call a variadic function
```
local f = function(...)
end
f(1)
```

the output is
```
call_hook event 0 addr 0 what main
ret = 1
call_hook event 0 addr 1 what Lua
ret = 1
ret = 0
```

OR, do not yield
```
static void pause_hook(lua_State *L, lua_Debug *ar) {
 lua_sethook(L, call_hook, LUA_MASKCALL, 0);
 //lua_yield(L, 0);
}
```

the output is
```
call_hook event 0 addr 0 what main
call_hook event 0 addr 1 what Lua
ret = 0
```

Different function types lead to different results.
Don't yield in 'COUNT' hook lead to different results.
So，I think this should not be the expected behavior, there may be a bug.

----------------------------------------------------------
----------------------------------------------------------

WHY this happen in example code, detailed process
0. Not critical, skip the first call event "call_hook event 0 addr 0 what main"
1. luaD_hookcall call 'CALL' hook
        output "call_hook event 0 addr 1 what lua" & set hook mask MASKCOUNT
```
//lvm.c
1148 void luaV_execute (lua_State *L, CallInfo *ci) {
     ...
1163   if (l_unlikely(trap)) {
1164     if (pc == cl->p->code) {  /* first instruction (not resuming)? */
1165       if (cl->p->is_vararg)
1166         trap = 0;  /* hooks will start after VARARGPREP instruction */
1167       else  /* check 'call' hook */
1168         luaD_hookcall(L, ci);// call 'CALL' hook <---***------***-
1169     }
1170     ci->u.l.trap = 1;  /* assume trap is on, for now */
1171   }
```

2. into main loop of interpreter after luaD_hookcall
```
//lvm.c
1173   /* main loop of interpreter */
1174   for (;;) {
1175     Instruction i;  /* instruction being executed */
1176     vmfetch();
```

3. vmfetch->luaG_traceexec call 'COUNT' hook(with lua_yield) and throw yield.
   notice, luaD_throw(L, LUA_YIELD) in luaG_traceexec and mark
callstatus CIST_HOOKYIELD.
```
//ldebug.c
880 int luaG_traceexec (lua_State *L, const Instruction *pc) {
    ...
902   if (counthook)
903     luaD_hook(L, LUA_HOOKCOUNT, -1, 0, 0);  /* call count hook */
    ...
915   if (L->status == LUA_YIELD) {  /* did hook yield? */
916     if (counthook)
917       L->hookcount = 1;  /* undo decrement to zero */
918     ci->u.l.savedpc--;  /* undo increment (resume will increment
it again) */
919     ci->callstatus |= CIST_HOOKYIELD;  /* mark that it yielded */
//3. MAKR CIST_HOOKYIELD  <---***------***-
920     luaD_throw(L, LUA_YIELD);
921   }
```

```
//ldo.c
871 LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx,
872                         lua_KFunction k) {
    ...
885   ci->u2.nyield = nresults;  /* save number of results */
886   if (isLua(ci)) {  /* inside a hook? */ //3. INSIDE A HOOK
<---***------***-
887     lua_assert(!isLuacode(ci));
888     api_check(L, nresults == 0, "hooks cannot yield values");
889     api_check(L, k == NULL, "hooks cannot continue after yielding");
890   }
```

4.resume, luaV_execute `luaD_hookcall` call 'CALL' hook again when
called hook last after VM yielded.
   output "call_hook event 0 addr 1 what lua" twice
```
//lvm.c
1148 void luaV_execute (lua_State *L, CallInfo *ci) {
     ...
1163   if (l_unlikely(trap)) {
1164     if (pc == cl->p->code) {  /* first instruction (not resuming)? */
1165       if (cl->p->is_vararg)
1166         trap = 0;  /* hooks will start after VARARGPREP instruction */
1167       else  /* check 'call' hook */
1168         luaD_hookcall(L, ci); // call 'CALL' hook <---***------***-
1169     }
1170     ci->u.l.trap = 1;  /* assume trap is on, for now */
1171   }
```

5. into main loop of interpreter vmfetch->luaG_traceexec erase
CIST_HOOKYIELD mark.
```
880 int luaG_traceexec (lua_State *L, const Instruction *pc) {
    ...
896   if (ci->callstatus & CIST_HOOKYIELD) {  /* called hook last time? */
897     ci->callstatus &= ~CIST_HOOKYIELD;  /* erase mark */
898     return 1;  /* do not call hook again (VM yielded, so it did not move) */
899   }
   ...
```

----------------------------------------------------------
----------------------------------------------------------

A possible fix for this issue would be to ensure that the
luaD_hookcall function is properly discarded when VM yielded,
preventing it from being called again.

diff --git a/lvm.c b/lvm.c
index 2b437bdf..fd52127c 100644
--- a/lvm.c
+++ b/lvm.c
@@ -1165,7 +1165,8 @@ void luaV_execute (lua_State *L, CallInfo *ci) {
       if (cl->p->is_vararg)
         trap = 0;  /* hooks will start after VARARGPREP instruction */
       else  /* check 'call' hook */
-        luaD_hookcall(L, ci);
+        if (!(ci->callstatus & CIST_HOOKYIELD))
+          luaD_hookcall(L, ci);
     }
     ci->u.l.trap = 1;  /* assume trap is on, for now */
   }