Benchmark Module

wiki

Now, here is my benchmarking module. I wrote it to find out, which is the fastest way to do some things in Lua. It requires "HiResTimer".

Benchmark.lua

---------------------------
--- Benchmark provides a set of functions, which measures and compares execution time of different code fragments
---------------------------

module("Benchmark",package.seeall)
local hrt=require"HiResTimer"


local bench_results={}
local total_diff=0.0

---------------------------
--- Call a function for several times and measure execution time.
-- @param name (string) literal to identify test in result table
-- @param func (function) function to be benchmarked
-- @param loops (number) how often this function should be called
----------------------------
function Bench(name,func,loops)
	loops=loops or 1000
	local q0=hrt.clock()
	for i=1,loops do
		func()
	end
	local q1=hrt.clock()
	local result=bench_results[name] or {count=0,diff=0}
	local diff=(q1-q0)
	result.count=result.count+loops
	result.diff=result.diff+diff
	total_diff=total_diff+diff
	bench_results[name]=result
end

---------------------------
--- Do Benchmark over a table of functions
-- @param functions (table) table of functions to check
-- @param loops (number) how often to call the function (optional, default 1000)
---------------------------
function BenchTable(functions,loops)
	loops=loops or 1000
	for name,func in pairs(functions) do
		Bench(name,func,loops)
	end
end

----------------------------
--- Printout benchmark results.
-- @param Output (function) to receive values (optional, default=io.write)
----------------------------
function Results(Output)

	--
	-- prepare the output
	--
	Output=Output or io.write
	local function printf(form,...)
		Output(string.format(form,...))
	end

	--
	-- calculate mean values
	-- create a table of names
	--
	local names={}
	local namlen=0
	for name,result in pairs(bench_results) do
		result.mean=result.diff/result.count
--~ 		printf("diff=%8.3g cnt=%d mean=%8.3g\n",result.diff,result.count,result.mean)
		names[#names+1]=name
		if #name>namlen then namlen=#name end
	end

	--
	-- sort table by mean value
	--
	local function comp(na,nb)
		return bench_results[na].mean<bench_results[nb].mean
	end
	table.sort(names,comp)

	--
	-- derive some reasonable output scaling
	--
	local max=bench_results[names[#names]].mean
	local fac,unit=1,"sec"
	if max<0.001 then
		fac,unit=1000000,"µs"
	elseif max<1.0 then
		fac,unit=1000,"ms"
	end

	--
	-- create a format string (due "%-*s" is missing in string.format)
	--
	local form=string.gsub("-- %-#s : %8.3f %s = %6d loops/s [%6.2f %%] %5.3f times faster\n","#",tostring(namlen))

	--
	-- now print the result
	--
	printf("-----------------------------------\n")
	printf("-- MAX = %8.3f %s\n",max*fac,unit)
	for i=1,#names do
		local name=names[i]
		result=bench_results[name]
		local ratio=result.mean*100/max
		local times=max/result.mean
		local loops=1/result.mean
		printf(form,name,result.mean*fac,unit,loops,ratio,times)
	end
	printf("-----------------------------------\n")
end

return Benchmark

Example : string concatenation

The question is, how to combine some literals and some variables to a string.

require"Benchmark"

local sf=string.format

local TestCases=
{

	TextConcat=function()
		local t,a,s='A','#',5
		local n=""
		for i=1,1000 do
			n="\symbol{circled"..s.."}="..t..a..i
		end
	end,

	-- extreme slow
	TableConcat=function()
		local t,a,s='A','#',5
		local n=""
		for i=1,1000 do
			n=table.concat{"\symbol{circled",s,"}=",t,a,i}
		end
	end,

	StringFormat=function ()
		local t,a,s='A','#',5
		local n=""
		for i=1,1000 do
			n=string.format("\symbol{circled%d}=%s%s%d",s,t,a,i)
		end
	end,

	FunctionLocalStringFormat=function()
		local t,a,s='A','#',5
		local n=""
		local sf=string.format
		for i=1,1000 do
			n=sf("\symbol{circled%d}=%s%s%d",s,t,a,i)
		end
	end,

	ModuleLocalStringFormat=function()
		local t,a,s='A','#',5
		local n=""
		for i=1,1000 do
			n=sf("\symbol{circled%d}=%s%s%d",s,t,a,i)
		end
	end
}
Benchmark.BenchTable(TestCases,100)
Benchmark.Results()

Result

-----------------------------------
-- MAX =    4.951 ms
-- ModuleLocalStringFormat   :    1.665 ms =    600 loops/s [ 33.63 %] 2.974 times faster
-- FunctionLocalStringFormat :    1.696 ms =    589 loops/s [ 34.26 %] 2.919 times faster
-- StringFormat              :    1.728 ms =    578 loops/s [ 34.90 %] 2.865 times faster
-- TextConcat                :    2.754 ms =    363 loops/s [ 55.64 %] 1.797 times faster
-- TableConcat               :    4.951 ms =    201 loops/s [100.00 %] 1.000 times faster
-----------------------------------

Example loop over array

-----------------------------------
-- setup test data
-----------------------------------

local array={}

for i=1,100 do
	local x={}
	for j=1,100 do
		x[j]=tostring(i)..tostring(j)
	end
	array[i]=x
end

local TestCases=
{
	for_i_array=function()
		local count=0
		for i=1,#array do
			local x=array[i]
			for j=1,#x do
				local y=x[j]
				-- do something with y
			end
		end
		return count
	end,

	for_ipairs=function()
		local count=0
		for i,x in ipairs(array) do
			for j,y in ipairs(x) do
				-- do something with y
			end
		end
		return count
	end
}

local Benchmark=require"Benchmark"
Benchmark.BenchTable(TestCases,1000)
Benchmark.Results()

Result

-----------------------------------
-- MAX =    1.127 ms
-- for_i_array :    0.689 ms =   1451 loops/s [ 61.12 %] 1.636 times faster
-- for_ipairs  :    1.127 ms =    887 loops/s [100.00 %] 1.000 times faster
-----------------------------------