Re: lua-5.0 performance problems

["Tom Spilman" <tom@...>]

    Isn't the context switch between OS threads fairly costly?  Are you sure
this isn't a OS thread performance issue?

    Tom

----- Original Message ----- 
From: "Simon at the Threshold" <dweller@the-threshold.org>
To: <lua@bazar2.conectiva.com.br>
Sent: Thursday, October 02, 2003 3:11 AM
Subject: lua-5.0 performance problems


> Lo there,
>
> I searched the internet for a small, fast embedible scripting/language
> to integrate in with my carrier grade web application (millions of
> users) to replace php (which is very slow).
>
> Not wanting to rewrite the entire interface in c/c++ I figured it would
> be worth looking at other languages. I've evaluated (and rejected) the
> likes of Java, Python, PHP, Perl, oCaml and finally came to rest on lua.
>
> In multi-threaded C tests with a single process connecting to a socket
> and driving a ascii based command line I can get 400 TPS (transactions
> per seconds) across a 100 MBS network.
>
> I wrappered in lua (pretty easy all things considered) directly into the
> core code and wrote myself a multi-threaded c program to run anything up
> to 300 threads each running a lua parser.
>
> I got 2 TPS
>
> I figured maybe I was being dim and optimised my code so that it ran
> compiled byte-code and got 4 TPS.  I then figured it was a waste of time
>   each iteration of the lua script loading the same shared library so I
> linked it in staticly and only call the registration once for each
> thread and got 6 TPS.
>
> On the plus side lua doesn't seem to thrash the machine (80% idle at 100
> threads) but the performance is dire.
>
> Am I being dim or is lua really this so..
>
> I've attached my lua test script (test.lua) my C lua wrapper
> (lua_mmcore.c) and my multi-threaded test program (luatest.c).Apols if
> they are a little rough around the edges but this was a test bed not a
> long-term maintainable code base.
>
> Thanks in advance for any advice..
>
> Regards
> Simon
>
> -- 
> If you want to know what god thinks of money, just look at the people he
> gave
> it to.
> -- Dorthy Parker
> -- 
> This email and any attachments hereto are strictly confidential and
> intended solely for the addressee. It may contain information which is
> covered by legal, professional or other privilege. If you are not the
> indended addressee, you must not disclose, forward, copy or take any
> action in reliance of this email or attachments. If you have received
> this email in error, please notify us as soon as possible.
>


----------------------------------------------------------------------------
----


> //
> //  LUA interface layer, this file defines all of the Core Mediator Lua
> //  wrappers which allow script language to resources and protocols like
> // HPI.
> //
> //  Revision History:-
> //
> //  $Log$
> //
> static const char* const cvsId = "@(#) $Header$";
>
> #include <stdio.h>
> #include <stdarg.h>
> #include <errno.h>
> #include <unistd.h>
> #include <string.h>
> #include <strings.h>
> #include <sys/types.h>
> #include <sys/stat.h>
> #include <fcntl.h>
> #include <signal.h>
> #include <pthread.h>
> #include "common.h"
> #include "hpi.h"
> #include "lua_mmcore.h"
>
> // Internal/Private method (single hpi call point)
> //
> static HPIRequest doHPI(lua_State* lua, char cmd[])
> {
> HPIRequest hpi = NULL;
>
> // Attempt to obtain a HPI handle from the registery
> //
> lua_getglobal(lua, "hpi");
> hpi = lua_touserdata(lua, -1);
>
> // Make the parse attempt even if we don't have a HPI handle
> //
> hpi = parseHPIRequest(hpi, cmd);
>
> // Now stick the latest socket/hpi values onto the registary so we
> // can make use of them in other lua/c calls.
> //
> lua_pushlightuserdata(lua, hpi);
> lua_setglobal(lua, "hpi");
>
> return hpi;
> }
>
> // Wrapper action for attempting a HPI command
> //
> static int lua_doHPI(lua_State* lua)
> {
> HPIRequest  hpi = lua_touserdata(lua, lua_upvalueindex(1));
> char* cmd = (char*)luaL_checkstring(lua, 1);
>
> hpi = doHPI(lua, cmd);
> if(hpi == NULL)
> {
> lua_pushnil(lua);
> }
> else
> {
> HPIToken tok;
> Profile prof;
>
> lua_newtable(lua);
>
> lua_pushstring(lua, "RESP");
> lua_pushnumber(lua, (lua_Number)hpi->errcode);
> lua_settable(lua, -3);
>
> lua_pushstring(lua, "ERROR");
> if(hpi->retstr)
> {
> lua_pushstring(lua, hpi->retstr);
> }
> else
> {
> lua_pushnil(lua);
> }
> lua_settable(lua, -3);
>
> // Convert the info fields into the table return
> //
> for(tok=hpi->argout; tok; tok=tok->next)
> {
> lua_pushstring(lua, tok->name);
> if(tok->numdata > 0)
> {
> lua_pushstring(lua, tok->data[0]);
> }
> else
> {
> lua_pushnil(lua);
> }
> lua_settable(lua, -3);
> }
>
> // Convert the profile data (persist/io data) into the table return
> //
> for(prof=hpi->out; prof; prof=prof->next)
> {
> char value[MAXSTRING+1];
>
> lua_pushstring(lua, prof->feature->name);
> if(numProfileData(hpi->prop, prof) > 0
> && getProfileData(hpi->prop, prof, value, 0, STRING_O))
> {
> lua_pushstring(lua, value);
> }
> else
> {
> lua_pushnil(lua);
> }
> lua_settable(lua, -3);
> }
> }
>
> return 1;
> }
>
> // Register the defined lua wrapper methods to MMLIBNAME (mmcore)
> //
> LUALIB_API int luaopen_mmcore(lua_State* lua)
> {
> static struct luaL_reg reg[] =
> {
> { "hpi", lua_doHPI },
> { NULL, NULL }
> };
>
> // Now register the methods
> //
> luaL_openlib(lua, "mmcore", reg, 0);
>
> return 1;
> }
>
> // End of File
>


----------------------------------------------------------------------------
----


> // My attempt at multi-threading some lua parsers
> //
> #include <stdio.h>
> #include <stdlib.h>
> #include <stdarg.h>
> #include <unistd.h>
> #include <string.h>
> #include <fcntl.h>
> #include <signal.h>
> #include <sys/socket.h>
> #include <sys/time.h>
> #include <sys/types.h>
> #include <sys/stat.h>
> #include <pthread.h>
> #include <hpi.h>
> #include <lua.h>
> #include <lualib.h>
> #include <lauxlib.h>
> #include <lua_mmcore.h>
>
> extern char* optarg;
> extern int optind;
>
> /*
>     We have been asked to shutdown gracefully
> */
> int shutdownModule(int sig)
> {
>     ModuleShutdown = TRUE;
>     return sig;
> }
>
>
> /*
>     We have been asked to Reconfigure
> */
> int configModule(int sig)
> {
>     if(loadConfig(SERVICE_DATA) < 0)
>     {
>         return -1;
>     }
>
>     if(loadConfig(QUEUE_DATA) < 0)
>     {
>         return -1;
>     }
>
>     if(loadConfig(DB_DATA) < 0)
>     {
>         return -1;
>     }
>
>     if(loadConfig(PROPERTY_DATA) < 0)
>     {
>         return -1;
>     }
>
>     if(loadConfig(ELEMENT_DATA) < 0)
>     {
>         return -1;
>     }
>
>     if(loadConfig(USER_DATA) < 0)
>     {
>         return -1;
>     }
>
>     if(loadConfig(LOGIN_DATA) < 0)
>     {
>         return -1;
>     }
>
>     if(loadConfig(RESPONSE_DATA) < 0)
>     {
>         return -1;
>     }
>
>     if(loadConfig(MAPPING_DATA) < 0)
>     {
>         return -1;
>     }
>
>     if(loadConfig(ROUTING_DATA) < 0)
>     {
>         return -1;
>     }
>
>     if(loadConfig(REMOTE_DATA) < 0)
>     {
>         return -1;
>     }
>
>     return 0;
> }
>
>
> void report(FILE* fp, char format[], ...)
> {
> struct timeval tyme;
> va_list ap;
>
> if(gettimeofday(&tyme, NULL) == 0)
> {
> fprintf(fp, "%d %d ", (int)tyme.tv_sec, (int)tyme.tv_usec);
> va_start(ap, format);
> vfprintf(fp, format, ap);
> va_end(ap);
> }
> }
>
> static char* luaBinary = NULL;
> static int luaBinarySize = 0;
>
> static char* luaScript = "test.lub";
> static int loopCount = 3;
> static unsigned startNum = 1000;
> static int numThreads = 51;
>
> void loadluabin()
> {
> struct stat st;
>
> if(stat(luaScript, &st) == 0)
> {
> luaBinarySize = st.st_size;
> FILE* fp = fopen(luaScript, "r");
> if(fp)
> {
> luaBinary = calloc(st.st_size, sizeof(char));
> fread(luaBinary, sizeof(char), st.st_size, fp);
> fclose(fp);
> }
> }
> }
>
>
> void* luaThread(void* arg)
> {
> int id = (int)arg;
> int i;
> char strid[64];
> char file[64];
> FILE* fp;
> lua_State* lua = lua_open();
>
> luaopen_base(lua);
> luaopen_table(lua);
> luaopen_loadlib(lua);
> luaopen_mmcore(lua);
>
> sprintf(strid, "%d", startNum+id);
> sprintf(file, "log/%d-log-%d.out", startNum+id, id);
> printf("Thread %d loging to %s\n", id, file);
>
> fp = fopen(file,"w+");
> if(fp == NULL)
> {
> printf("Failed to open %s\n", file);
> }
>
> //
> // Set the subid in the lua script..
> lua_pushstring(lua, strid);
> lua_setglobal(lua, "subid");
>
> for(i=0; i < loopCount; ++i)
> {
> report(fp, "START LUA %s ", luaScript);
> if(luaBinary)
> {
> lua_dobuffer(lua, luaBinary, luaBinarySize, "script");
> }
> else
> {
> lua_dofile(lua, luaScript);
> }
> report(fp, "END LUA %s\n", luaScript);
> }
>
> // Free and close off the output file
> //
> fclose(fp);
>
> // Shutdown lua
> //
> lua_close(lua);
>
> return arg;
> }
>
> int main(int argc, char*argv[])
> {
> int opt;
>
> if(initModule("luatest", 0) < 0)
> {
> printf("Failed to init\n");
> exit(-1);
> }
>
> while((opt = getopt(argc, argv, "lz:t:n:s:S:c:")) != -1)
> {
> switch(opt)
> {
> case 'l': // lua loop test
> {
> int i;
> pthread_t tid[1024];
> pthread_attr_t  tattr;
>
>
> loadluabin();
>
> printf("LUA SCRIPT TEST\n");
>
> pthread_attr_init(&tattr);
>
> // Create the threads
> //
> for(i=0; i < numThreads; ++i)
> {
> if(pthread_create(&tid[i], &tattr, luaThread, (void*)i) < 0)
> {
> printf("Failed to create thread %d\n", i);
> }
> else
> {
> printf("Created thread %d\n", i);
> }
> }
>
> // Wait for the threads to terminate and cleanup
> //
> for(i=0; i < numThreads; ++i)
> {
> pthread_join(tid[i], NULL);
> }
> }
> break;
>
> case 'z': // Sleep
> {
> int winks = atoi(optarg);
> printf("Sleeping %d winks\n", winks);
> sleep(winks);
> }
> break;
>
> case 'c': // Number of iterations
> {
> loopCount = atoi(optarg);
> printf("loopCount %d\n", loopCount);
> }
> break;
>
> case 't': // Number of threads
> {
> numThreads = atoi(optarg);
> printf("numThreads %d\n", numThreads);
> }
> break;
>
> case 's': // Start number
> {
> startNum = atoi(optarg);
> printf("startNum %d\n", startNum);
> }
> break;
>
> case 'S': // Lua script file to run
> {
> luaScript = optarg;
> printf("luaScript %s\n", luaScript);
> }
> break;
>
> default:
> printf("Useless usage\n");
> break;
> }
> }
>
> // this should terminate the process
> printf("Calling deinitModule\n");
>
>
> deinitModule(SIGTERM);
> // should not get this far
> exit(0);
> }
>


----------------------------------------------------------------------------
----


> print ("About to login subid "..subid)
>
> resp = mmcore.hpi("LOGIN:SUBID,"..subid..":PIN,1234:type,subscriber;")
> if(resp)
> then
> sid = resp["SID"]
>
> print ("Dumping response "..sid);
> table.foreach(resp, print);
>
> print ("About to info")
> resp = mmcore.hpi("INFO;");
> if(resp)
> then
> print ("Dumping response");
> table.foreach(resp, print);
> end
>
> print ("About to info")
> resp = mmcore.hpi("GET:VIRTUAL,120:ERROR,0;");
> if(resp)
> then
> print ("Dumping response");
> table.foreach(resp, print);
> end
>
> print ("About to logout ")
> resp = mmcore.hpi("LOGOUT;");
> if(resp)
> then
> print ("Dumping response");
> table.foreach(resp, print);
> end
> else
> print("No response");
> end
>
>