Re: Wanted: Lua source code cruncher

[Tony Finch <dot@...>]

On Sun, 9 Mar 2008, David Given wrote:
>
> I've got this build tool of which you have heard me speak before, Prime
> Mover. It ships a small but reasonably verbose Lua program as part of
> its binary. As it makes no sense to ship comments in a binary, I'd
> rather like to make this smaller.

Compile it with luac -s and compress the result. I've attached some
utility code for loading Lua chunks (source or compiled, since Lua doesn't
care) that have been compressed in zlib or gzip format. I also have a
little deflate utility for making zlib format files, which are a few bytes
shorter than gzip.

This is probably only a win if your app is dynamically linked to Lua,
since you'd need quite a lot of Lua code to get near the size of liblua.a
(200K on my machine).

Tony.
-- 
f.a.n.finch  <dot@dotat.at>  http://dotat.at/
LUNDY FASTNET: SOUTHWESTERLY 5 TO 7 INCREASING GALE 8 TO STORM 10 THEN VEERING
NORTHWESTERLY, OCCASIONALLY VIOLENT STORM 11 IN FASTNET, PERHAPS VIOLENT STORM
11 LATER IN LUNDY. ROUGH OR VERY ROUGH BECOMING HIGH OR VERY HIGH,
OCCASIONALLY PHENOMENAL LATER IN FASTNET. SHOWERS. GOOD BECOMING POOR.

/*
 * Load Lua chunks in gzip or zlib format.
 *
 * Written by Tony Finch <dot@dotat.at> <fanf2@cam.ac.uk>
 * at the University of Cambridge Computing Service.
 * You may do anything with this, at your own risk.
 *
 * $Cambridge: users/fanf2/sdist/lunzip.h,v 1.3 2008/03/05 11:52:06 fanf2 Exp $
 */

/*
 * A lua_Reader for loading a Lua chunk from a buffer.
 *	void *lunbuf = lunbuf_data(buffer, size);
 *	ret = lua_load(L, lunbuf_reader, lunbuf, name);
 *	lunbuf_free(lunbuf);
 * is equivalent to
 *	luaL_loadbuffer(L, buffer, size, name);
 * is equivalent to
 *	lunbuf_load(L, buffer, size, name);
 */
void *lunbuf_data(const char *buffer, size_t size);
const char *lunbuf_reader(lua_State *L, void *opaque, size_t *size);
void lunbuf_free(void *opaque);

/*
 * A lua_Reader for loading a compressed Lua chunk. It acts as a
 * wrapper that inflates the data returned by another lua_Reader.
 * For example, see the source of lunzip_load().
 */
void *lunzip_data(lua_Reader reader, void *opaque);
const char *lunzip_reader(lua_State *L, void *opaque, size_t *size);
void lunzip_free(void *opaque);

/*
 * Wrappers in the manner of the luaL_load...() functions.
 */
int lunbuf_load(lua_State *L,
    const char *buffer, size_t size, const char *name);
int lunzip_load(lua_State *L,
    lua_Reader reader, void *opaque, const char *name);
int lunzip_loadbuffer(lua_State *L,
    const char *buffer, size_t size, const char *name);

/* eof */

/*
 * Load Lua chunks in gzip or zlib format.
 *
 * Written by Tony Finch <dot@dotat.at> <fanf2@cam.ac.uk>
 * at the University of Cambridge Computing Service.
 * You may do anything with this, at your own risk.
 *
 * $Cambridge: users/fanf2/sdist/lunzip.c,v 1.3 2008/03/05 11:52:06 fanf2 Exp $
 */

#include <stdlib.h>
#include <zlib.h>
#include "lua.h"
#include "lunzip.h"

#define BUFFER 4096

/* lunbuf */

typedef struct lunbuf_state {
	const char *buffer;
	size_t size;
} lunbuf_state;

void *
lunbuf_data(const char *buffer, size_t size) {
	lunbuf_state *state = malloc(sizeof(*state));
	if(state == NULL) return(NULL);
	state->buffer = buffer;
	state->size = size;
	return(state);
}

const char *
lunbuf_reader(lua_State *L, void *opaque, size_t *size) {
	lunbuf_state *state = opaque;
	L = L;
	if(state->size == 0) return(NULL);
	*size = state->size;
	state->size = 0;
	return(state->buffer);
}

void
lunbuf_free(void *opaque) {
	free(opaque);
}

/* lunzip */

typedef struct lunzip_state {
	lua_Reader reader;
	void *opaque;
	z_stream z;
	union {
		unsigned char b[BUFFER];
		const char c[BUFFER];
	} buffer;
} lunzip_state;

void *
lunzip_data(lua_Reader reader, void *opaque) {
	int ret;
	lunzip_state *state = malloc(sizeof(*state));
	if(state == NULL) return(NULL);
	state->reader = reader;
	state->opaque = opaque;
	state->z.zalloc = NULL;
	state->z.zfree = NULL;
	state->z.opaque = NULL;
	state->z.next_in = NULL;
	state->z.avail_in = 0;
	ret = inflateInit2(&state->z, 47);
	if(ret == Z_OK) return(state);
	if(ret != Z_MEM_ERROR) abort(); /* programming error */
	free(state);
	return(NULL);
}

const char *
lunzip_reader(lua_State *L, void *opaque, size_t *size) {
	lunzip_state *state = opaque;
	int ret;
	if(state->z.avail_in == 0)
		state->z.next_in = state->reader(L, state->opaque,
		    &state->z.avail_in);
	if(state->z.next_in == NULL)
		state->z.avail_in = 0;
	state->z.next_out = state->buffer.b;
	state->z.avail_out = sizeof(state->buffer);
	ret = inflate(&state->z, Z_SYNC_FLUSH);
	*size = state->z.next_out - state->buffer.b;
	if(*size != 0) return(state->buffer.c);
	return(NULL);
}

void
lunzip_free(void *opaque) {
	lunzip_state *state = opaque;
	if(state == NULL) return;
	inflateEnd(&state->z);
	free(state);
}

/* canned loaders */

int
lunbuf_load(lua_State *L,
    const char *buffer, size_t size, const char *name) {
	int ret;
	void *lunbuf = lunbuf_data(buffer, size);
	if(lunbuf == NULL) return(LUA_ERRMEM);
	ret = lua_load(L, lunbuf_reader, lunbuf, name);
	lunbuf_free(lunbuf);
	return(ret);
}

int
lunzip_load(lua_State *L,
    lua_Reader reader, void *opaque, const char *name) {
	int ret;
	void *lunzip = lunzip_data(reader, opaque);
	if(lunzip == NULL) return(LUA_ERRMEM);
	ret = lua_load(L, lunzip_reader, lunzip, name);
	lunzip_free(lunzip);
	return(ret);
}

int
lunzip_loadbuffer(lua_State *L,
    const char *buffer, size_t size, const char *name) {
	int ret;
	void *lunbuf = lunbuf_data(buffer, size);
	if(lunbuf == NULL) return(LUA_ERRMEM);
	ret = lunzip_load(L, lunbuf_reader, lunbuf, name);
	lunbuf_free(lunbuf);
	return(ret);
}

/* eof */

/*
 * Compress a stream in zlib format (less rubric than gzip).
 *
 * Written by Tony Finch <dot@dotat.at> <fanf2@cam.ac.uk>
 * at the University of Cambridge Computing Service.
 * You may do anything with this, at your own risk.
 *
 * $Cambridge: users/fanf2/sdist/deflate.c,v 1.1 2008/03/04 22:33:07 fanf2 Exp $
 */

#include <err.h>
#include <stdio.h>
#include <zlib.h>

#define BUFFER 4096

int main(void) {
	unsigned char in[BUFFER], out[BUFFER];
	unsigned char *inp = in;
	z_stream z;
	size_t n;
	int mode, ret;

	z.zalloc = NULL;
	z.zfree = NULL;
	z.opaque = NULL;
	ret = deflateInit(&z, 9);
	if(ret != Z_OK) errx(1, "init: %s (%d)", z.msg, ret);
	mode = Z_NO_FLUSH;
	z.avail_in = 0;
	for(;;) {
		if(z.avail_in == 0) z.next_in = inp = in;
		n = fread(inp, 1, sizeof(in)-(inp-in), stdin);
		if(n == 0 && feof(stdin)) mode = Z_FINISH;
		if(n == 0 && ferror(stdin)) err(1, "read");
		inp += n;
		z.avail_in += n;
		z.next_out = out;
		z.avail_out = sizeof(out);
		ret = deflate(&z, mode);
		if(ret != Z_OK && ret != Z_STREAM_END)
			errx(1, "decompress: %s (%d)", z.msg, ret);
		n = z.next_out - out;
		if(fwrite(out, 1, n, stdout) < n) err(1, "write");
		if(ret == Z_STREAM_END) break;
	}
	ret = deflateEnd(&z);
	if(ret != Z_OK) errx(1, "finish: %s (%d)", z.msg, ret);
	return(0);
}

/* eof */

/*
 * Convert a file to a C array so that it can be embedded in a program.
 *
 * Written by Tony Finch <dot@dotat.at> <fanf2@cam.ac.uk>
 * at the University of Cambridge Computing Service.
 * You may do anything with this, at your own risk.
 *
 * $Cambridge: users/fanf2/sdist/data2c.c,v 1.4 2007/10/27 12:55:52 fanf2 Exp $
 */

#include <stdio.h>
#include <string.h>

int
main(int argc, char *argv[])
{
	if(argc != 2) {
		fprintf(stderr, "usage: data2c arrayname <data >data.c\n");
		return(1);
	}
	printf("/* auto-generated by data2c */\n");
	printf("static const unsigned %s[] = {\n", argv[1]);
	for(;;) {
		unsigned line[8], count;
		memset(line, 0, sizeof(line));
		count = fread(line, 1, sizeof(line), stdin);
		for(unsigned i = 0; sizeof(*line) * i < count; i++)
			printf(" %u,", line[i]);
		if(count < sizeof(line))
			break;
		else
			printf("\n");
	}
	if(ferror(stdin)) {
		perror("error reading input");
		return(1);
	} else {
		printf("\n};\n");
		return(0);
	}
}

/* eof */