Add first cut of VCL compiler to the tree.

The Makefile is a temporary shim until I get the auto* stuff working.

The sample.vcl is a small mock-up to test the compiler.

Some of the .h files needs to move other places in the fullness of time.

But other than that...



git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@43 d4fa192b-c00b-0410-8231-f00ffab90ce4

lib/Makefile.am

@@ -2,4 +2,5 @@
 
 SUBDIRS = \
 	libvarnish \
-	libvarnishapi
+	libvarnishapi \
+	libvcl

lib/libvcl/Makefile

+PROG	=	vpc
+
+SRCS	+=	vcl_compile.c
+SRCS	+=	vcl_fixed_token.c
+
+NO_MAN	=	yes
+
+WARNS	?=	5
+
+LDADD	+=	-lsbuf
+
+.include <bsd.prog.mk>
+
+
+test:	${PROG}
+	./${PROG} ${.CURDIR}/sample.vcl
+	cc -Wall -c _.c
+
+flint:	
+	flint flint.lnt -I/usr/include -I. ${SRCS}

lib/libvcl/sample.vcl

+
+acl rfc1918 {
+	10.0.0.0/8;
+	172.16.0.0/12;
+	192.168.0.0/16;
+}
+
+sub request_policy {
+
+	if (client.ip == 10.1.2.3) {
+		no_cache;
+		finish;
+	}
+
+	if (client.ip ~ rfc1918) {
+		no_cache;
+		finish;
+	}
+
+	if (req.url.host ~ "cnn.no$") {
+		rewrite "cnn.no$" "vg.no";
+	}
+
+	if (req.url.path ~ "cgi-bin") {
+		no_cache;
+	}
+
+	if (req.useragent ~ "spider") {
+		no_new_cache;
+	}
+
+# comment
+
+	if (backend.response_time <= 0.8s) {
+		set req.ttlfactor = 1.5;
+	} elseif (backend.response_time > 1.5s) {
+		set req.ttlfactor = 2.0;
+	} elseif (backend.response_time > 2.5m) {
+		set req.ttlfactor = 5.0;
+	}
+
+	/*
+	 * the program contains no references to
+	 * maxage, s-maxage and expires, so the
+	 * default handling (RFC2616) applies
+	 */
+}
+
+backend vg {
+	set backend.ip = 10.0.0.100;
+	set backend.timeout = 4s;
+	set backend.bandwidth = 2000Mb/s;
+}
+
+backend chat {
+	set backend.ip = 10.0.0.4;
+	set backend.timeout = 4s;
+	set backend.bandwidth = 2000Mb/s;
+}
+
+sub bail {
+	error 404 "Bailing out";
+	finish;
+}
+
+sub fetch_policy {
+
+	if (!req.url.host ~ "/vg.no$/") {
+		set req.backend = vg;
+	} else {
+		/* XXX: specify 404 page url ? */
+		error 404;
+	}
+
+	if (backend.response_time > 2.0s) {
+		if (req.url.path ~ "/landbrugspriser/") {
+			call bail;
+		}
+	}
+	fetch;
+	if (backend.down) {
+		if (obj.exist) {
+			set obj.ttl += 10m;
+			finish;
+		}
+		switch_config ohhshit;
+	}
+	if (obj.result == 404) {
+		error 300 "http://www.vg.no";
+	}
+	if (obj.result != 200) {
+		finish;
+	}
+	if (obj.size > 256kb) {
+		no_cache;
+	} else if (obj.size > 32kb && obj.ttl < 2m) {
+		set obj.ttl = 5m;
+	}
+	if (backend.response_time > 2.0s) {
+		set obj.ttl *= 2.0;
+	}
+}
+
+sub prefetch_policy {
+
+	if (obj.usage < 10 && obj.ttl < 5m) {
+		fetch;
+	}
+}

lib/libvcl/vcl_compile.c

+/*
+ * $Id$
+ */
+
+/*
+ * XXX:
+ *	generate interface structure
+ *
+ * XXX:
+ *	Better error messages, throughout.
+ *	>It also accured to me that we could link the errors to the error 
+ *	>documentation.
+ *	>
+ *	>Unreferenced  function 'request_policy', first mention is
+ *	>         Line 8 Pos 4
+ *	>         sub request_policy {
+ *	>         ----##############--
+ *	>Read more about this type of error:
+ *	>http://varnish/doc/error.html#Unreferenced%20function
+ *	>
+ *	>
+ *	>         Unknown variable 'obj.bandwidth'
+ *	>         At: Line 88 Pos 12
+ *	>                 if (obj.bandwidth < 1 kb/h) {
+ *	>         ------------#############------------
+ *	>Read more about this type of error:
+ *	>http://varnish/doc/error.html#Unknown%20variable
+ * 
+ * XXX:
+ *	Create proper tmp filenames for .h, .c and .o
+ *
+ * XXX:
+ *	and all the rest...
+ */
+
+#include <assert.h>
+#include <string.h>
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/sbuf.h>
+#include <sys/queue.h>
+#include "vcl_priv.h"
+
+#define ERRCHK(tl)	do { if ((tl)->err) return; } while (0)
+
+#define INDENT		2
+
+struct token {
+	unsigned		tok;
+	const char		*b;
+	const char		*e;
+	TAILQ_ENTRY(token)	list;
+	unsigned		cnt;
+};
+
+struct tokenlist {
+	TAILQ_HEAD(, token)	tokens;
+	const char		*b;
+	const char		*e;
+	struct token		*t;
+	int			indent;
+	unsigned		cnt;
+	FILE			*fc, *fh;
+	TAILQ_HEAD(, ref)	refs;
+	struct sbuf		*sb;
+	int			err;
+};
+
+enum var_type {
+	BACKEND,
+	BOOL,
+	INT,
+	FLOAT,
+	SIZE,
+	RATE,
+	TIME,
+	STRING,
+	IP
+};
+
+struct var {
+	const char		*name;
+	enum var_type		fmt;
+	int			len;
+	const char		*cname;
+	
+};
+
+enum ref_type {
+	R_FUNC,
+	R_ACL,
+	R_BACKEND
+};
+
+struct ref {
+	enum ref_type		type;
+	struct token		*name;
+	unsigned		defcnt;
+	unsigned		refcnt;
+	TAILQ_ENTRY(ref)	list;
+};
+
+
+static struct var vars[] = {
+	{ "req.ttlfactor",		FLOAT, 0,   "req->ttlfactor" },
+	{ "req.url.host",		STRING, 0,  "req->url.host" },
+	{ "req.url.path",		STRING, 0,  "req->url.path" },
+	{ "req.useragent",		STRING, 0,  "req->useragent" },
+	{ "req.backend",		BACKEND, 0, "req->backend"   },
+	{ "client.ip",			IP, 0,	    "client->ip"     },
+	{ "backend.response_time",	TIME, 0,    "backend->responsetime" },
+	{ "backend.ip",			IP, 0,      "backend->ip"    },
+	{ "backend.down",		BOOL, 0,    "backend->down"  },
+	{ "backend.timeout",		TIME, 0,    "backend->timeout" },
+	{ "backend.bandwidth",		RATE, 0,    "backend->bandwidth" },
+	{ "obj.exist",			BOOL, 0,    "obj->exists" },
+	{ "obj.ttl",			TIME, 0,    "obj->ttl" },
+	{ "obj.result",			INT, 0,     "obj->result" },
+	{ "obj.size",			SIZE, 0,    "obj->size" },
+	{ "obj.usage",			INT, 0,     "obj->usage" },
+	{ NULL,				INT, 0,	    "NULL" }
+};
+
+static void Compound(struct tokenlist *tl);
+static void Cond_0(struct tokenlist *tl);
+
+/*--------------------------------------------------------------------*/
+
+static void
+ErrToken(struct tokenlist *tl, struct token *t)
+{
+
+	if (t->tok == EOI)
+		sbuf_printf(tl->sb, "end of input");
+	else
+		sbuf_printf(tl->sb, "'%*.*s'", t->e - t->b, t->e - t->b, t->b);
+}
+
+static void
+_ErrInternal(struct tokenlist *tl, const char *func, unsigned line)
+{
+
+	sbuf_printf(tl->sb, "VCL compiler internal error at %s():%u\n",
+	    func, line);
+	tl->err = 1;
+}
+
+#define ErrInternal(tl) _ErrInternal(tl, __func__, __LINE__)
+
+static void
+ErrWhere(struct tokenlist *tl, struct token *t)
+{
+	unsigned lin, pos, x, y;
+	const char *p, *l;
+	
+	lin = 1;
+	pos = 0;
+	for (l = p = tl->b; p < t->b; p++) {
+		if (*p == '\n') {
+			lin++;
+			pos = 0;
+			l = p + 1;
+		} else if (*p == '\t') {
+			pos &= ~7;
+			pos += 8;
+		} else
+			pos++;
+	}
+	sbuf_printf(tl->sb, "Line %d Pos %d\n", lin, pos);
+	x = y = 0;
+	for (p = l; p < tl->e && *p != '\n'; p++) {
+		if (*p == '\t') {
+			y &= ~7;
+			y += 8;
+			while (x < y) {
+				sbuf_bcat(tl->sb, " ", 1);
+				x++;
+			}
+		} else {
+			x++;
+			y++;
+			sbuf_bcat(tl->sb, p, 1);
+		}
+	}
+	sbuf_cat(tl->sb, "\n");
+	x = y = 0;
+	for (p = l; p < tl->e && *p != '\n'; p++) {
+		if (p >= t->b && p < t->e) {
+			sbuf_bcat(tl->sb, "#", 1);
+			x++;
+			y++;
+			continue;
+		}
+		if (*p == '\t') {
+			y &= ~7;
+			y += 8;
+		} else
+			y++;
+		while (x < y) {
+			sbuf_bcat(tl->sb, "-", 1);
+			x++;
+		}
+	}
+	sbuf_cat(tl->sb, "\n");
+	tl->err = 1;
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+NextToken(struct tokenlist *tl)
+{
+	tl->t = TAILQ_NEXT(tl->t, list);
+	if (tl->t == NULL) {
+		sbuf_printf(tl->sb,
+		    "Ran out of input, something is missing or"
+		    " maybe unbalanced (...) or {...}\n");
+		tl->err = 1;
+		return;
+	}
+}
+
+static void
+_Expect(struct tokenlist *tl, unsigned tok, int line)
+{
+	if (tl->t->tok == tok)
+		return;
+	sbuf_printf(tl->sb, "Expected %s got ", tnames[tok]);
+	ErrToken(tl, tl->t);
+	sbuf_printf(tl->sb, "\n(program line %u), at\n", line);
+	ErrWhere(tl, tl->t);
+}
+
+#define Expect(a, b) _Expect(a, b, __LINE__)
+#define ExpectErr(a, b) do { _Expect(a, b, __LINE__); ERRCHK(a);} while (0)
+
+#define I(tl)	do { 		\
+	fprintf(tl->fc, "/* %-11s */ ", __func__); \
+	fprintf(tl->fc, "%*.*s", tl->indent, tl->indent, ""); \
+} while (0)
+
+#define L(tl, foo)	do {	\
+	tl->indent += INDENT;	\
+	foo;			\
+	tl->indent -= INDENT;	\
+} while (0)
+
+#define C(tl, sep)	do {				\
+	I(tl);						\
+	fprintf(tl->fc, "VCL_count(%u)%s\n", ++tl->cnt, sep);	\
+	tl->t->cnt = tl->cnt; 				\
+} while (0)
+	
+/*--------------------------------------------------------------------
+ * Compare ID token to string, return true of match
+ */
+
+static int
+IdIs(struct token *t, const char *p)
+{
+	const char *q;
+
+	assert(t->tok == ID);
+	for (q = t->b; q < t->e && *p != '\0'; p++, q++)
+		if (*q != *p)
+			return (0);
+	if (q != t->e || *p != '\0')
+		return (0);
+	return (1);
+}
+
+/*--------------------------------------------------------------------
+ * Keep track of definitions and references
+ */
+
+static struct ref *
+FindRef(struct tokenlist *tl, struct token *t, enum ref_type type)
+{
+	struct ref *r;
+
+	TAILQ_FOREACH(r, &tl->refs, list) {
+		if (r->type != type)
+			continue;
+		if (r->name->e - r->name->b != t->e - t->b)
+			continue;
+		if (memcmp(t->b, r->name->b, t->e - t->b))
+			continue;
+		return (r);
+	}
+	r = calloc(sizeof *r, 1);
+	assert(r != NULL);
+	r->name = t;
+	r->type = type;
+	TAILQ_INSERT_TAIL(&tl->refs, r, list);
+	return (r);
+}
+
+static void
+AddRef(struct tokenlist *tl, struct token *t, enum ref_type type)
+{
+
+	FindRef(tl, t, type)->refcnt++;
+}
+
+static void
+AddDef(struct tokenlist *tl, struct token *t, enum ref_type type)
+{
+
+	FindRef(tl, t, type)->defcnt++;
+}
+
+/*--------------------------------------------------------------------
+ * Recognize and convert units of time, return seconds.
+ */
+
+static double
+TimeUnit(struct tokenlist *tl)
+{
+	double sc = 1.0;
+
+	assert(tl->t->tok == ID);
+	if (IdIs(tl->t, "ms"))
+		sc = 1e-3;
+	else if (IdIs(tl->t, "s"))
+		sc = 1.0;
+	else if (IdIs(tl->t, "m"))
+		sc = 60.0;
+	else if (IdIs(tl->t, "h"))
+		sc = 60.0 * 60.0;
+	else if (IdIs(tl->t, "d"))
+		sc = 60.0 * 60.0 * 24.0;
+	else {
+		sbuf_printf(tl->sb, "Unknown time unit ");
+		ErrToken(tl, tl->t);
+		sbuf_printf(tl->sb, ".  Legal are 's', 'm', 'h' and 'd'\n");
+		ErrWhere(tl, tl->t);
+		return (1.0);
+	}
+	NextToken(tl);
+	return (sc);
+}
+
+/*--------------------------------------------------------------------
+ * Recognize and convert units of size, return bytes.
+ */
+
+static double
+SizeUnit(struct tokenlist *tl)
+{
+	double sc = 1.0;
+
+	assert(tl->t->tok == ID);
+	if (IdIs(tl->t, "b"))
+		sc = 1.0;
+	else if (IdIs(tl->t, "kb"))
+		sc = 1024.0;
+	else if (IdIs(tl->t, "mb") || IdIs(tl->t, "Mb"))
+		sc = 1024.0 * 1024.0;
+	else if (IdIs(tl->t, "gb") || IdIs(tl->t, "Gb"))
+		sc = 1024.0 * 1024.0 * 1024.0;
+	else {
+		sbuf_printf(tl->sb, "Unknown size unit ");
+		ErrToken(tl, tl->t);
+		sbuf_printf(tl->sb, ".  Legal are 'kb', 'mb' and 'gb'\n");
+		ErrWhere(tl, tl->t);
+		return (1.0);
+	}
+	NextToken(tl);
+	return (sc);
+}
+
+/*--------------------------------------------------------------------
+ * Recognize and convert units of rate as { space '/' time }
+ */
+
+static double
+RateUnit(struct tokenlist *tl)
+{
+	double sc = 1.0;
+
+	assert(tl->t->tok == ID);
+	sc = SizeUnit(tl);
+	Expect(tl, '/');
+	NextToken(tl);
+	sc /= TimeUnit(tl);
+	return (sc);
+}
+
+/*--------------------------------------------------------------------
+ * Recognize and convert { CNUM } to unsigned value
+ */
+
+static unsigned
+UintVal(struct tokenlist *tl)
+{
+	unsigned d = 0;
+	const char *p;
+
+	Expect(tl, CNUM);
+	for (p = tl->t->b; p < tl->t->e; p++) {
+		d *= 10;
+		d += digittoint(*p);
+	}
+	NextToken(tl);
+	return (d);
+}
+
+/*--------------------------------------------------------------------
+ * Recognize and convert { CNUM [ '.' [ CNUM ] ] } to double value
+ */
+
+static double
+DoubleVal(struct tokenlist *tl)
+{
+	double d = 0.0, e = 0.1;
+	const char *p;
+
+	Expect(tl, CNUM);
+	for (p = tl->t->b; p < tl->t->e; p++) {
+		d *= 10;
+		d += digittoint(*p);
+	}
+	NextToken(tl);
+	if (tl->t->tok != '.') 
+		return (d);
+	NextToken(tl);
+	if (tl->t->tok != CNUM)
+		return (d);
+	for (p = tl->t->b; p < tl->t->e; p++) {
+		d += digittoint(*p) * e;
+		e *= 0.1;
+	}
+	NextToken(tl);
+	return (d);
+}
+
+/*--------------------------------------------------------------------*/
+
+static unsigned
+IpVal(struct tokenlist *tl)
+{
+	unsigned u, v;
+	struct token *t;
+
+	t = tl->t;
+	u = UintVal(tl);
+	if (u < 256) {
+		v = u << 24;
+		Expect(tl, '.');
+		NextToken(tl);
+		t = tl->t;
+		u = UintVal(tl);
+		if (u < 256) {
+			v |= u << 16;
+			Expect(tl, '.');
+			NextToken(tl);
+			t = tl->t;
+			u = UintVal(tl);
+			if (u < 256) {
+				v |= u << 8;
+				Expect(tl, '.');
+				NextToken(tl);
+				t = tl->t;
+				u = UintVal(tl);
+				if (u < 256) {
+					v |= u;
+					return (v);
+				}
+			}
+		}
+	}
+	sbuf_printf(tl->sb, "Illegal octet in IP number\n");
+	ErrWhere(tl, t);
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
+static struct var *
+FindVar(struct tokenlist *tl, struct token *t)
+{
+	struct var *v;
+
+	for (v = vars; v->name != NULL; v++) {
+		if (t->e - t->b != v->len)
+			continue;
+		if (!memcmp(t->b, v->name, v->len))
+			return (v);
+	}
+	sbuf_printf(tl->sb, "Unknown variable ");
+	ErrToken(tl, t);
+	sbuf_cat(tl->sb, "\nAt: ");
+	ErrWhere(tl, t);
+	return (NULL);
+}
+
+
+/*--------------------------------------------------------------------*/
+
+static void
+TimeVal(struct tokenlist *tl)
+{
+	double v, sc;
+
+	v = DoubleVal(tl);
+	ExpectErr(tl, ID);
+	sc = TimeUnit(tl);
+	fprintf(tl->fc, "(%g * %g)", v, sc);
+}
+
+static void
+SizeVal(struct tokenlist *tl)
+{
+	double v, sc;
+
+	v = DoubleVal(tl);
+	ExpectErr(tl, ID);
+	sc = SizeUnit(tl);
+	fprintf(tl->fc, "(%g * %g)", v, sc);
+}
+
+static void
+RateVal(struct tokenlist *tl)
+{
+	double v, sc;
+
+	v = DoubleVal(tl);
+	ExpectErr(tl, ID);
+	sc = RateUnit(tl);
+	fprintf(tl->fc, "(%g * %g)", v, sc);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+Cond_Ip(struct var *vp, struct tokenlist *tl)
+{
+	unsigned u;
+
+	switch (tl->t->tok) {
+	case '~':
+		NextToken(tl);
+		ExpectErr(tl, ID);
+		I(tl);
+		AddRef(tl, tl->t, R_ACL);
+		fprintf(tl->fc, "ip_match(%s, acl_%*.*s)\n",
+		    vp->cname,
+		    tl->t->e - tl->t->b,
+		    tl->t->e - tl->t->b, tl->t->b);
+		NextToken(tl);
+		break;
+	case T_EQ:
+	case T_NEQ:
+		I(tl);
+		fprintf(tl->fc, "%s %*.*s ",
+		    vp->cname,
+		    tl->t->e - tl->t->b,
+		    tl->t->e - tl->t->b, tl->t->b);
+		NextToken(tl);
+		u = IpVal(tl);
+		fprintf(tl->fc, "%uU /* %u.%u.%u.%u */\n", u,
+		    (u >> 24) & 0xff, (u >> 16) & 0xff,
+		    (u >> 8) & 0xff, (u) & 0xff);
+		break;
+	default:
+		sbuf_printf(tl->sb, "Illegal condition ");
+		ErrToken(tl, tl->t);
+		sbuf_printf(tl->sb, " on IP number variable\n");
+		sbuf_printf(tl->sb, "  only '==', '!=' and '~' are legal\n");
+		ErrWhere(tl, tl->t);
+		break;
+	}
+}
+
+static void
+Cond_String(struct var *vp __unused, struct tokenlist *tl)
+{
+
+	switch (tl->t->tok) {
+	case '~':
+		I(tl); fprintf(tl->fc, "string_match(%s, ", vp->cname);
+		NextToken(tl);
+		ExpectErr(tl, CSTR);
+		fprintf(tl->fc, "%*.*s)\n",
+			tl->t->e - tl->t->b,
+			tl->t->e - tl->t->b, tl->t->b);
+		NextToken(tl);
+		break;
+	default:
+		sbuf_printf(tl->sb, "Illegal condition ");
+		ErrToken(tl, tl->t);
+		sbuf_printf(tl->sb, " on string variable\n");
+		sbuf_printf(tl->sb, "  only '~' is legal\n");
+		ErrWhere(tl, tl->t);
+		break;
+	}
+}
+
+static void
+Cond_Int(struct var *vp, struct tokenlist *tl)
+{
+
+	I(tl);
+	fprintf(tl->fc, "%s ", vp->cname);
+	switch (tl->t->tok) {
+	case T_EQ:
+	case T_NEQ:
+	case T_LEQ:
+	case T_GEQ:
+	case '>':
+	case '<':
+		fprintf(tl->fc, "%*.*s ", 
+			tl->t->e - tl->t->b,
+			tl->t->e - tl->t->b, tl->t->b);
+		NextToken(tl);
+		switch(vp->fmt) {
+		case TIME:
+			TimeVal(tl);
+			break;
+		case INT:
+			ExpectErr(tl, CNUM);
+			fprintf(tl->fc, "%*.*s ", 
+				tl->t->e - tl->t->b,
+				tl->t->e - tl->t->b, tl->t->b);
+			NextToken(tl);
+			break;
+		case SIZE:
+			SizeVal(tl);
+			break;
+		default:
+			sbuf_printf(tl->sb,
+			    "No conditions available for variable '%s'\n",
+			    vp->name);
+			ErrWhere(tl, tl->t);
+			return;
+		}
+		fprintf(tl->fc, "\n");
+		break;
+	default:
+		sbuf_printf(tl->sb, "Illegal condition ");
+		ErrToken(tl, tl->t);
+		sbuf_printf(tl->sb, " on integer variable\n");
+		sbuf_printf(tl->sb,
+		    "  only '==', '!=', '<', '>', '<=' and '>=' are legal\n");
+		ErrWhere(tl, tl->t);
+		break;
+	}
+}
+
+static void
+Cond_Bool(struct var *vp, struct tokenlist *tl)
+{
+
+	I(tl);
+	fprintf(tl->fc, "%s\n", vp->cname);
+}
+
+static void
+Cond_2(struct tokenlist *tl)
+{
+	struct var *vp;
+
+	C(tl, ",");
+	I(tl);
+	if (tl->t->tok == '!') {
+		fprintf(tl->fc, "!");
+		NextToken(tl);
+	}
+	fprintf(tl->fc, "(\n");
+	if (tl->t->tok == '(') {
+		NextToken(tl);
+		Cond_0(tl);
+		ExpectErr(tl, ')');
+		NextToken(tl);
+	} else if (tl->t->tok == VAR) {
+		vp = FindVar(tl, tl->t);
+		ERRCHK(tl);
+		assert(vp != NULL);
+		NextToken(tl);
+		switch (vp->fmt) {
+		case INT:	L(tl, Cond_Int(vp, tl)); break;
+		case SIZE:	L(tl, Cond_Int(vp, tl)); break;
+		case BOOL:	L(tl, Cond_Bool(vp, tl)); break;
+		case IP:	L(tl, Cond_Ip(vp, tl)); break;
+		case STRING:	L(tl, Cond_String(vp, tl)); break;
+		case TIME:	L(tl, Cond_Int(vp, tl)); break;
+		/* XXX backend == */
+		default:	
+			sbuf_printf(tl->sb,
+			    "Variable '%s'"
+			    " has no conditions that can be checked\n",
+			    vp->name);
+			ErrWhere(tl, tl->t);
+			return;
+		}
+	} else {
+		sbuf_printf(tl->sb,
+		    "Syntax error in condition, expected '(', '!' or"
+		    " variable name, found ");
+		ErrToken(tl, tl->t);
+		sbuf_printf(tl->sb, "\n");
+		ErrWhere(tl, tl->t);
+		return;
+	}
+	I(tl);
+	fprintf(tl->fc, ")\n");
+}
+
+static void
+Cond_1(struct tokenlist *tl)
+{
+
+	I(tl); fprintf(tl->fc, "(\n");
+	L(tl, Cond_2(tl));
+	while (tl->t->tok == T_CAND) {
+		NextToken(tl);
+		I(tl); fprintf(tl->fc, ") && (\n");
+		L(tl, Cond_2(tl));
+	}
+	I(tl); fprintf(tl->fc, ")\n");
+}
+
+static void
+Cond_0(struct tokenlist *tl)
+{
+
+	I(tl); fprintf(tl->fc, "(\n");
+	L(tl, Cond_1(tl));
+	while (tl->t->tok == T_COR) {
+		NextToken(tl);
+		I(tl); fprintf(tl->fc, ") || (\n");
+		L(tl, Cond_1(tl));
+	}
+	I(tl); fprintf(tl->fc, ")\n");
+}
+
+static void
+Conditional(struct tokenlist *tl)
+{
+
+	ExpectErr(tl, '(');
+	NextToken(tl);
+	I(tl); fprintf(tl->fc, "(\n");
+	L(tl, Cond_0(tl));
+	ERRCHK(tl);
+	I(tl); fprintf(tl->fc, ")\n");
+	ExpectErr(tl, ')');
+	NextToken(tl);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+IfStmt(struct tokenlist *tl)
+{
+
+	ExpectErr(tl, T_IF);
+	I(tl); fprintf(tl->fc, "if \n");
+	NextToken(tl);
+	L(tl, Conditional(tl));
+	ERRCHK(tl);
+	L(tl, Compound(tl));
+	ERRCHK(tl);
+	while (1) {
+		switch (tl->t->tok) {
+		case T_ELSE:
+			NextToken(tl);
+			if (tl->t->tok != T_IF) {
+				I(tl); fprintf(tl->fc, "else \n");
+				L(tl, Compound(tl));
+				ERRCHK(tl);
+				return;
+			}
+			/* FALLTHROUGH */
+		case T_ELSEIF:
+		case T_ELSIF:
+			I(tl); fprintf(tl->fc, "else if \n");
+			NextToken(tl);
+			L(tl, Conditional(tl));
+			ERRCHK(tl);
+			L(tl, Compound(tl));
+			ERRCHK(tl);
+			break;
+		default:
+			return;
+		}
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+Action(struct tokenlist *tl)
+{
+	unsigned a, u;
+	struct var *vp;
+	struct token *at;
+
+	at = tl->t;
+	NextToken(tl);
+	switch (at->tok) {
+	case T_NO_NEW_CACHE:
+		I(tl);
+		fprintf(tl->fc, "VCL_no_new_cache();\n");
+		return;
+	case T_NO_CACHE:
+		I(tl);
+		fprintf(tl->fc, "VCL_no_cache();\n");
+		return;
+	case T_FINISH:
+		I(tl);
+		fprintf(tl->fc, "return;\n");
+		return;
+	case T_FETCH:
+		I(tl);
+		fprintf(tl->fc, "VCL_fetch();\n");
+		return;
+	case T_ERROR:
+		a = UintVal(tl);
+		I(tl);
+		fprintf(tl->fc, "VCL_error(%u, ", a);
+		if (tl->t->tok == CSTR) {
+			fprintf(tl->fc, "%*.*s);\n",
+			    tl->t->e - tl->t->b,
+			    tl->t->e - tl->t->b, tl->t->b);
+			NextToken(tl);
+		} else
+			fprintf(tl->fc, "(const char *)0);\n");
+		return;
+	case T_SWITCH_CONFIG:
+		ExpectErr(tl, ID);
+		I(tl);
+		fprintf(tl->fc, "VCL_switch_config(\"%*.*s\");\n",
+		    tl->t->e - tl->t->b,
+		    tl->t->e - tl->t->b, tl->t->b);
+		NextToken(tl);
+		return;
+	case T_CALL:
+		ExpectErr(tl, ID);
+		AddRef(tl, tl->t, R_FUNC);
+		I(tl);
+		fprintf(tl->fc, "VCL_function_%*.*s(VCL_PASS_ARGS);\n",
+		    tl->t->e - tl->t->b,
+		    tl->t->e - tl->t->b, tl->t->b);
+		/* XXX: check if function finished request */
+		NextToken(tl);
+		return;
+	case T_REWRITE:
+		ExpectErr(tl, CSTR);
+		I(tl);
+		fprintf(tl->fc, "VCL_rewrite(%*.*s",
+		    tl->t->e - tl->t->b,
+		    tl->t->e - tl->t->b, tl->t->b);
+		NextToken(tl);
+		ExpectErr(tl, CSTR);
+		fprintf(tl->fc, ", %*.*s);\n",
+		    tl->t->e - tl->t->b,
+		    tl->t->e - tl->t->b, tl->t->b);
+		NextToken(tl);
+		return;
+	case T_SET:
+		ExpectErr(tl, VAR);
+		vp = FindVar(tl, tl->t);
+		ERRCHK(tl);
+		assert(vp != NULL);
+		I(tl);
+		fprintf(tl->fc, "%s ", vp->cname);
+		NextToken(tl);
+		switch (vp->fmt) {
+		case INT:
+		case SIZE:
+		case RATE:
+		case TIME:
+		case FLOAT:
+			fprintf(tl->fc, "%*.*s ",
+			    tl->t->e - tl->t->b,
+			    tl->t->e - tl->t->b, tl->t->b);
+			a = tl->t->tok;
+			NextToken(tl);
+			if (a == T_MUL || a == T_DIV)
+				fprintf(tl->fc, "%g", DoubleVal(tl));
+			else if (vp->fmt == TIME)
+				TimeVal(tl);
+			else if (vp->fmt == SIZE)
+				SizeVal(tl);
+			else if (vp->fmt == RATE)
+				RateVal(tl);
+			else 
+				fprintf(tl->fc, "%g", DoubleVal(tl));
+			fprintf(tl->fc, ";\n");
+			break;
+		case IP:
+			if (tl->t->tok == '=') {
+				NextToken(tl);
+				u = IpVal(tl);
+				fprintf(tl->fc, "= %uU; /* %u.%u.%u.%u */\n",
+				    u,
+				    (u >> 24) & 0xff,
+				    (u >> 16) & 0xff,
+				    (u >> 8) & 0xff,
+				    u & 0xff);
+				break;
+			}
+			sbuf_printf(tl->sb, "Illegal assignment operator ");
+			ErrToken(tl, tl->t);
+			sbuf_printf(tl->sb,
+			    " only '=' is legal for IP numbers\n");
+			ErrWhere(tl, tl->t);
+			return;
+		case BACKEND:
+			if (tl->t->tok == '=') {
+				NextToken(tl);
+				fprintf(tl->fc, "= &VCL_backend_%*.*s;\n",
+				    tl->t->e - tl->t->b,
+				    tl->t->e - tl->t->b, tl->t->b);
+				NextToken(tl);
+				break;
+			}
+			sbuf_printf(tl->sb, "Illegal assignment operator ");
+			ErrToken(tl, tl->t);
+			sbuf_printf(tl->sb,
+			    " only '=' is legal for backend\n");
+			ErrWhere(tl, tl->t);
+			return;
+		default:
+			sbuf_printf(tl->sb,
+			    "Assignments not possible for '%s'\n", vp->name);
+			ErrWhere(tl, tl->t);
+			return;
+		}
+		return;
+	default:
+		sbuf_printf(tl->sb, "Expected action, 'if' or '}'\n");
+		ErrWhere(tl, at);
+		return;
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+Acl(struct tokenlist *tl)
+{
+	unsigned u, m;
+
+	NextToken(tl);
+
+	ExpectErr(tl, ID);
+	AddDef(tl, tl->t, R_ACL);
+	fprintf(tl->fh, "static struct vcl_acl acl_%*.*s[];\n",
+	    tl->t->e - tl->t->b,
+	    tl->t->e - tl->t->b, tl->t->b);
+	I(tl);
+	fprintf(tl->fc, "static struct vcl_acl acl_%*.*s[] = {\n",
+	    tl->t->e - tl->t->b,
+	    tl->t->e - tl->t->b, tl->t->b);
+	NextToken(tl);
+
+	tl->indent += INDENT;
+
+	ExpectErr(tl, '{');
+	NextToken(tl);
+
+	while (tl->t->tok == CNUM) {
+		u = IpVal(tl);
+		if (tl->t->tok == '/') {
+			NextToken(tl);
+			ExpectErr(tl, CNUM);
+			m = UintVal(tl);
+		} else
+			m = 32;
+		ExpectErr(tl, ';');
+		NextToken(tl);
+		I(tl);
+		fprintf(tl->fc, "{ %11uU, %3uU }, /* %u.%u.%u.%u/%u */\n",
+		    u, m,
+		    (u >> 24) & 0xff, (u >> 16) & 0xff,
+		    (u >> 8) & 0xff, (u) & 0xff, m);
+	}
+	ExpectErr(tl, '}');
+	I(tl);
+	fprintf(tl->fc, "{ %11uU, %3uU }\n", 0, 0);
+
+	tl->indent -= INDENT;
+
+	I(tl);
+	fprintf(tl->fc, "};\n\n");
+	NextToken(tl);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+Compound(struct tokenlist *tl)
+{
+
+	ExpectErr(tl, '{');
+	I(tl); fprintf(tl->fc, "{\n");
+	tl->indent += INDENT;
+	C(tl, ";");
+	NextToken(tl);
+	while (1) {
+		ERRCHK(tl);
+		switch (tl->t->tok) {
+		case '{':
+			Compound(tl);
+			break;
+		case T_IF:
+			IfStmt(tl);
+			break;
+		case '}':
+			NextToken(tl);
+			tl->indent -= INDENT;
+			I(tl); fprintf(tl->fc, "}\n");
+			return;
+		case EOI:
+			sbuf_printf(tl->sb,
+			    "End of input while in compound statement\n");
+			tl->err = 1;
+			return;
+		default:
+			Action(tl);
+			ERRCHK(tl);
+			ExpectErr(tl, ';');
+			NextToken(tl);
+			break;
+		}
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+Backend(struct tokenlist *tl)
+{
+
+	NextToken(tl);
+	ExpectErr(tl, ID);
+	AddDef(tl, tl->t, R_BACKEND);
+	I(tl);
+	fprintf(tl->fh, "static struct backend VCL_backend_%*.*s;\n",
+	    tl->t->e - tl->t->b,
+	    tl->t->e - tl->t->b, tl->t->b);
+	fprintf(tl->fc, "static struct backend VCL_backend_%*.*s;\n",
+	    tl->t->e - tl->t->b,
+	    tl->t->e - tl->t->b, tl->t->b);
+	fprintf(tl->fc, "static void\n");
+	I(tl);
+	fprintf(tl->fc, "VCL_init_backend_%*.*s (struct backend *backend)\n",
+	    tl->t->e - tl->t->b,
+	    tl->t->e - tl->t->b, tl->t->b);
+	NextToken(tl);
+	L(tl, Compound(tl));
+	fprintf(tl->fc, "\n");
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+Function(struct tokenlist *tl)
+{
+
+	NextToken(tl);
+	ExpectErr(tl, ID);
+	AddDef(tl, tl->t, R_FUNC);
+	fprintf(tl->fh, "static void VCL_function_%*.*s (VCL_FARGS);\n",
+	    tl->t->e - tl->t->b,
+	    tl->t->e - tl->t->b, tl->t->b);
+	I(tl);
+	fprintf(tl->fc, "static void\n");
+	I(tl);
+	fprintf(tl->fc, "VCL_function_%*.*s (VCL_FARGS)\n",
+	    tl->t->e - tl->t->b,
+	    tl->t->e - tl->t->b, tl->t->b);
+	NextToken(tl);
+	L(tl, Compound(tl));
+	fprintf(tl->fc, "\n");
+}
+
+/*--------------------------------------------------------------------
+ * Top level of parser, recognize:
+ *	Function definitions
+ *	Backend definitions
+ *	End of input
+ */
+
+static void
+Parse(struct tokenlist *tl)
+{
+
+	while (tl->t->tok != EOI) {
+		ERRCHK(tl);
+		switch (tl->t->tok) {
+		case T_ACL:
+			Acl(tl);
+			break;
+		case T_SUB:
+			Function(tl);
+			break;
+		case T_BACKEND:
+			Backend(tl);
+			break;
+		case EOI:
+			break;
+		default:
+			sbuf_printf(tl->sb,
+			    "Expected 'acl', 'sub' or 'backend', found ");
+			ErrToken(tl, tl->t);
+			sbuf_printf(tl->sb, " at\n");
+			ErrWhere(tl, tl->t);
+			return;
+		}
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Add a token to the token list.
+ */
+
+static void
+AddToken(struct tokenlist *tl, unsigned tok, const char *b, const char *e)
+{
+	struct token *t;
+
+	t = calloc(sizeof *t, 1);
+	assert(t != NULL);
+	t->tok = tok;
+	t->b = b;
+	t->e = e;
+	TAILQ_INSERT_TAIL(&tl->tokens, t, list);
+	tl->t = t;
+}
+
+/*--------------------------------------------------------------------
+ * Lexical analysis and token generation
+ */
+
+static void
+Lexer(struct tokenlist *tl, const char *b, const char *e)
+{
+	const char *p, *q;
+	unsigned u;
+
+	for (p = b; p < e; ) {
+
+		/* Skip any whitespace */
+		if (isspace(*p)) {
+			p++;
+			continue;
+		}
+
+		/* Skip '#.*\n' comments */
+		if (*p == '#') {
+			while (p < e && *p != '\n')
+				p++;
+			continue;
+		}
+
+		/* Skip C-style comments */
+		if (*p == '/' && p[1] == '*') {
+			p += 2;
+			for (p += 2; p < e; p++) {
+				if (*p == '*' && p[1] == '/') {
+					p += 2;
+					break;
+				}
+			}
+			continue;
+		}
+
+		/* Match for the fixed tokens (see token.tcl) */
+		u = fixed_token(p, &q);
+		if (u != 0) {
+			AddToken(tl, u, p, q);
+			p = q;
+			continue;
+		}
+
+		/* Match strings, with \\ and \" escapes */
+		if (*p == '"') {
+			for (q = p + 1; q < e; q++) {
+				if (*q == '\\' && q[1] == '\\')
+					q++;
+				else if (*q == '\\' && q[1] == '"')
+					q++;
+				else if (*q == '"') {
+					q++;
+					break;
+				}
+			}
+			AddToken(tl, CSTR, p, q);
+			p = q;
+			continue;
+		}
+
+		/* Match Identifiers */
+		if (isident1(*p)) {
+			for (q = p; q < e; q++)
+				if (!isident(*q))
+					break;
+			if (isvar(*q)) {
+				for (; q < e; q++)
+					if (!isvar(*q))
+						break;
+				AddToken(tl, VAR, p, q);
+			} else {
+				AddToken(tl, ID, p, q);
+			}
+			p = q;
+			continue;
+		}
+
+		/* Match numbers { [0-9]+ } */
+		if (isdigit(*p)) {
+			for (q = p; q < e; q++)
+				if (!isdigit(*q))
+					break;
+			AddToken(tl, CNUM, p, q);
+			p = q;
+			continue;
+		}
+		AddToken(tl, EOI, p, p + 1);
+		sbuf_printf(tl->sb, "Syntax error at\n");
+		ErrWhere(tl, tl->t);
+		return;
+	}
+	/* Add End Of Input token */
+	AddToken(tl, EOI, p, p);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+CheckRefs(struct tokenlist *tl)
+{
+	struct ref *r;
+	const char *bug;
+
+	TAILQ_FOREACH(r, &tl->refs, list) {
+		if (r->defcnt == 0)
+			bug = "Undefined ";
+		else if (r->refcnt == 0)
+			bug = "Unreferenced ";
+		else
+			continue;
+		switch(r->type) {
+		case R_FUNC:
+			sbuf_printf(tl->sb, "%s function ", bug);
+			break;
+		case R_ACL:
+			sbuf_printf(tl->sb, "%s acl ", bug);
+			break;
+		case R_BACKEND:
+			sbuf_printf(tl->sb, "%s backend ", bug);
+			break;
+		default:
+			ErrInternal(tl);
+			sbuf_printf(tl->sb, "Ref ");
+			ErrToken(tl, r->name);
+			sbuf_printf(tl->sb, " has unknown type %d\n",
+			    r->type);
+			return;
+		}
+		ErrToken(tl, r->name);
+		sbuf_cat(tl->sb, ", first mention is\n");
+		ErrWhere(tl, r->name);
+		return;
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+LocTable(struct tokenlist *tl)
+{
+	struct token *t;
+	unsigned lin, pos;
+	const char *p;
+	
+	fprintf(tl->fh, "static struct vcl_ref VCL_ref[%u];\n", tl->cnt + 1);
+	fprintf(tl->fc, "static struct vcl_ref VCL_ref[%u] = {\n", tl->cnt + 1);
+	lin = 1;
+	pos = 0;
+	p = tl->b;
+	TAILQ_FOREACH(t, &tl->tokens, list) {
+		if (t->cnt == 0)
+			continue;
+		for (;p < t->b; p++) {
+			if (*p == '\n') {
+				lin++;
+				pos = 0;
+			} else if (*p == '\t') {
+				pos &= ~7;
+				pos += 8;
+			} else
+				pos++;
+		
+		}
+		fprintf(tl->fc,
+		    "%*.*s[%3u] = { %4u, %3u, 0, \"%*.*s\" },\n",
+		    INDENT, INDENT, "",
+		    t->cnt, lin, pos + 1,
+		    t->e - t->b,
+		    t->e - t->b, t->b);
+	}
+	fprintf(tl->fc, "};\n");
+}
+
+
+/*--------------------------------------------------------------------*/
+
+static void
+Compile(struct sbuf *sb, const char *b, const char *e)
+{
+	struct tokenlist	tokens;
+
+	memset(&tokens, 0, sizeof tokens);
+	TAILQ_INIT(&tokens.tokens);
+	TAILQ_INIT(&tokens.refs);
+	tokens.sb = sb;
+
+	tokens.fc = fopen("_.c", "w");
+	assert(tokens.fc != NULL);
+
+	tokens.fh = fopen("_.h", "w");
+	assert(tokens.fh != NULL);
+
+	fprintf(tokens.fc, "#include \"vcl_lang.h\"\n");
+	fprintf(tokens.fc, "#include \"_.h\"\n");
+	tokens.b = b;
+	tokens.e = e;
+	Lexer(&tokens, b, e);
+	ERRCHK(&tokens);
+	tokens.t = TAILQ_FIRST(&tokens.tokens);
+	Parse(&tokens);
+	ERRCHK(&tokens);
+	CheckRefs(&tokens);
+	ERRCHK(&tokens);
+	LocTable(&tokens);
+}
+
+/*--------------------------------------------------------------------*/
+
+#include <err.h>
+
+#define MYSPACE	(128 * 1024)
+
+int
+main(int argc, char **argv)
+{
+	char *p;
+	size_t z;
+	FILE *fi;
+	struct sbuf *sb;
+
+	setbuf(stdout, NULL);
+	{
+	struct var *v;
+
+	for (v = vars; v->name != NULL; v++) {
+		v->len = strlen(v->name);
+	}
+	}
+	if (argc != 2)
+		err(1, "Usage: %s file", argv[0]);
+	fi = fopen(argv[1], "r");
+	if (fi == NULL)
+		err(1, "Cannot open %s", argv[1]);
+	p = malloc(MYSPACE);
+	assert(p != NULL);
+
+	z = fread(p, 1, MYSPACE - 1, fi);
+	if (z == 0)
+		err(1, "Nothing read from %s", argv[1]);
+	p[z] = '\0';
+	sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND);
+	Compile(sb, p, p + z);
+	sbuf_finish(sb);
+	if (sbuf_len(sb))
+		printf("<%s>\n", sbuf_data(sb));
+	return (0);
+}

lib/libvcl/vcl_fixed_token.c

+/*
+ * NB:  This file is machine generated, DO NOT EDIT!
+ * instead, edit the Tcl script vcl_gen_fixed_token.tcl and run it by hand
+ */
+
+#include "vcl_priv.h"
+
+unsigned
+fixed_token(const char *p, const char **q)
+{
+
+	switch (p[0]) {
+	case '!':
+		if (p[0] == '!' && p[1] == '=') {
+			*q = p + 2;
+			return (T_NEQ);
+		}
+		if (p[0] == '!') {
+			*q = p + 1;
+			return ('!');
+		}
+		return (0);
+	case '%':
+		if (p[0] == '%') {
+			*q = p + 1;
+			return ('%');
+		}
+		return (0);
+	case '&':
+		if (p[0] == '&' && p[1] == '&') {
+			*q = p + 2;
+			return (T_CAND);
+		}
+		if (p[0] == '&') {
+			*q = p + 1;
+			return ('&');
+		}
+		return (0);
+	case '(':
+		if (p[0] == '(') {
+			*q = p + 1;
+			return ('(');
+		}
+		return (0);
+	case ')':
+		if (p[0] == ')') {
+			*q = p + 1;
+			return (')');
+		}
+		return (0);
+	case '*':
+		if (p[0] == '*' && p[1] == '=') {
+			*q = p + 2;
+			return (T_MUL);
+		}
+		if (p[0] == '*') {
+			*q = p + 1;
+			return ('*');
+		}
+		return (0);
+	case '+':
+		if (p[0] == '+' && p[1] == '=') {
+			*q = p + 2;
+			return (T_INCR);
+		}
+		if (p[0] == '+' && p[1] == '+') {
+			*q = p + 2;
+			return (T_INC);
+		}
+		if (p[0] == '+') {
+			*q = p + 1;
+			return ('+');
+		}
+		return (0);
+	case ',':
+		if (p[0] == ',') {
+			*q = p + 1;
+			return (',');
+		}
+		return (0);
+	case '-':
+		if (p[0] == '-' && p[1] == '-') {
+			*q = p + 2;
+			return (T_DEC);
+		}
+		if (p[0] == '-') {
+			*q = p + 1;
+			return ('-');
+		}
+		return (0);
+	case '.':
+		if (p[0] == '.') {
+			*q = p + 1;
+			return ('.');
+		}
+		return (0);
+	case '/':
+		if (p[0] == '/' && p[1] == '=') {
+			*q = p + 2;
+			return (T_DECR);
+		}
+		if (p[0] == '/' && p[1] == '=') {
+			*q = p + 2;
+			return (T_DIV);
+		}
+		if (p[0] == '/') {
+			*q = p + 1;
+			return ('/');
+		}
+		return (0);
+	case ';':
+		if (p[0] == ';') {
+			*q = p + 1;
+			return (';');
+		}
+		return (0);
+	case '<':
+		if (p[0] == '<' && p[1] == '=') {
+			*q = p + 2;
+			return (T_LEQ);
+		}
+		if (p[0] == '<' && p[1] == '<') {
+			*q = p + 2;
+			return (T_SHL);
+		}
+		if (p[0] == '<') {
+			*q = p + 1;
+			return ('<');
+		}
+		return (0);
+	case '=':
+		if (p[0] == '=' && p[1] == '=') {
+			*q = p + 2;
+			return (T_EQ);
+		}
+		if (p[0] == '=') {
+			*q = p + 1;
+			return ('=');
+		}
+		return (0);
+	case '>':
+		if (p[0] == '>' && p[1] == '>') {
+			*q = p + 2;
+			return (T_SHR);
+		}
+		if (p[0] == '>' && p[1] == '=') {
+			*q = p + 2;
+			return (T_GEQ);
+		}
+		if (p[0] == '>') {
+			*q = p + 1;
+			return ('>');
+		}
+		return (0);
+	case 'a':
+		if (p[0] == 'a' && p[1] == 'c' && p[2] == 'l'
+		     && !isvar(p[3])) {
+			*q = p + 3;
+			return (T_ACL);
+		}
+		return (0);
+	case 'b':
+		if (p[0] == 'b' && p[1] == 'a' && p[2] == 'c' && 
+		    p[3] == 'k' && p[4] == 'e' && p[5] == 'n' && 
+		    p[6] == 'd' && !isvar(p[7])) {
+			*q = p + 7;
+			return (T_BACKEND);
+		}
+		return (0);
+	case 'c':
+		if (p[0] == 'c' && p[1] == 'a' && p[2] == 'l' && 
+		    p[3] == 'l' && !isvar(p[4])) {
+			*q = p + 4;
+			return (T_CALL);
+		}
+		return (0);
+	case 'e':
+		if (p[0] == 'e' && p[1] == 'r' && p[2] == 'r' && 
+		    p[3] == 'o' && p[4] == 'r' && !isvar(p[5])) {
+			*q = p + 5;
+			return (T_ERROR);
+		}
+		if (p[0] == 'e' && p[1] == 'l' && p[2] == 's' && 
+		    p[3] == 'i' && p[4] == 'f' && !isvar(p[5])) {
+			*q = p + 5;
+			return (T_ELSIF);
+		}
+		if (p[0] == 'e' && p[1] == 'l' && p[2] == 's' && 
+		    p[3] == 'e' && p[4] == 'i' && p[5] == 'f'
+		     && !isvar(p[6])) {
+			*q = p + 6;
+			return (T_ELSEIF);
+		}
+		if (p[0] == 'e' && p[1] == 'l' && p[2] == 's' && 
+		    p[3] == 'e' && !isvar(p[4])) {
+			*q = p + 4;
+			return (T_ELSE);
+		}
+		return (0);
+	case 'f':
+		if (p[0] == 'f' && p[1] == 'u' && p[2] == 'n' && 
+		    p[3] == 'c' && !isvar(p[4])) {
+			*q = p + 4;
+			return (T_FUNC);
+		}
+		if (p[0] == 'f' && p[1] == 'i' && p[2] == 'n' && 
+		    p[3] == 'i' && p[4] == 's' && p[5] == 'h'
+		     && !isvar(p[6])) {
+			*q = p + 6;
+			return (T_FINISH);
+		}
+		if (p[0] == 'f' && p[1] == 'e' && p[2] == 't' && 
+		    p[3] == 'c' && p[4] == 'h' && !isvar(p[5])) {
+			*q = p + 5;
+			return (T_FETCH);
+		}
+		return (0);
+	case 'i':
+		if (p[0] == 'i' && p[1] == 'f' && !isvar(p[2])) {
+			*q = p + 2;
+			return (T_IF);
+		}
+		return (0);
+	case 'n':
+		if (p[0] == 'n' && p[1] == 'o' && p[2] == '_' && 
+		    p[3] == 'n' && p[4] == 'e' && p[5] == 'w' && 
+		    p[6] == '_' && p[7] == 'c' && p[8] == 'a' && 
+		    p[9] == 'c' && p[10] == 'h' && p[11] == 'e'
+		     && !isvar(p[12])) {
+			*q = p + 12;
+			return (T_NO_NEW_CACHE);
+		}
+		if (p[0] == 'n' && p[1] == 'o' && p[2] == '_' && 
+		    p[3] == 'c' && p[4] == 'a' && p[5] == 'c' && 
+		    p[6] == 'h' && p[7] == 'e' && !isvar(p[8])) {
+			*q = p + 8;
+			return (T_NO_CACHE);
+		}
+		return (0);
+	case 'p':
+		if (p[0] == 'p' && p[1] == 'r' && p[2] == 'o' && 
+		    p[3] == 'c' && !isvar(p[4])) {
+			*q = p + 4;
+			return (T_PROC);
+		}
+		return (0);
+	case 'r':
+		if (p[0] == 'r' && p[1] == 'e' && p[2] == 'w' && 
+		    p[3] == 'r' && p[4] == 'i' && p[5] == 't' && 
+		    p[6] == 'e' && !isvar(p[7])) {
+			*q = p + 7;
+			return (T_REWRITE);
+		}
+		return (0);
+	case 's':
+		if (p[0] == 's' && p[1] == 'w' && p[2] == 'i' && 
+		    p[3] == 't' && p[4] == 'c' && p[5] == 'h' && 
+		    p[6] == '_' && p[7] == 'c' && p[8] == 'o' && 
+		    p[9] == 'n' && p[10] == 'f' && p[11] == 'i' && 
+		    p[12] == 'g' && !isvar(p[13])) {
+			*q = p + 13;
+			return (T_SWITCH_CONFIG);
+		}
+		if (p[0] == 's' && p[1] == 'u' && p[2] == 'b'
+		     && !isvar(p[3])) {
+			*q = p + 3;
+			return (T_SUB);
+		}
+		if (p[0] == 's' && p[1] == 'e' && p[2] == 't'
+		     && !isvar(p[3])) {
+			*q = p + 3;
+			return (T_SET);
+		}
+		return (0);
+	case '{':
+		if (p[0] == '{') {
+			*q = p + 1;
+			return ('{');
+		}
+		return (0);
+	case '|':
+		if (p[0] == '|' && p[1] == '|') {
+			*q = p + 2;
+			return (T_COR);
+		}
+		if (p[0] == '|') {
+			*q = p + 1;
+			return ('|');
+		}
+		return (0);
+	case '}':
+		if (p[0] == '}') {
+			*q = p + 1;
+			return ('}');
+		}
+		return (0);
+	case '~':
+		if (p[0] == '~') {
+			*q = p + 1;
+			return ('~');
+		}
+		return (0);
+	default:
+		return (0);
+	}
+}
+
+const char *tnames[256] = {
+	['!'] = "'!'" /* t2 '!' T! */,
+	['%'] = "'%'" /* t2 '%' T% */,
+	['&'] = "'&'" /* t2 '&' T& */,
+	['('] = "'('" /* t2 '(' T( */,
+	[')'] = "')'" /* t2 ')' T) */,
+	['*'] = "'*'" /* t2 '*' T* */,
+	['+'] = "'+'" /* t2 '+' T+ */,
+	[','] = "','" /* t2 ',' T, */,
+	['-'] = "'-'" /* t2 '-' T- */,
+	['.'] = "'.'" /* t2 '.' T. */,
+	['/'] = "'/'" /* t2 '/' T/ */,
+	['<'] = "'<'" /* t2 '<' T< */,
+	['='] = "'='" /* t2 '=' T= */,
+	['>'] = "'>'" /* t2 '>' T> */,
+	['{'] = "'{'" /* t2 '\{' T\{ */,
+	['}'] = "'}'" /* t2 '\}' T\} */,
+	['|'] = "'|'" /* t2 '|' T| */,
+	['~'] = "'~'" /* t2 '~' T~ */,
+	[';'] = "';'" /* t2 {';'} {T;} */,
+	[CNUM] = "CNUM" /* t CNUM CNUM */,
+	[CSTR] = "CSTR" /* t CSTR CSTR */,
+	[EOI] = "EOI" /* t EOI EOI */,
+	[ID] = "ID" /* t ID ID */,
+	[T_ACL] = "acl" /* t T_ACL acl */,
+	[T_BACKEND] = "backend" /* t T_BACKEND backend */,
+	[T_CALL] = "call" /* t T_CALL call */,
+	[T_CAND] = "&&" /* t T_CAND && */,
+	[T_COR] = "||" /* t T_COR || */,
+	[T_DEC] = "--" /* t T_DEC -- */,
+	[T_DECR] = "/=" /* t T_DECR /= */,
+	[T_DIV] = "/=" /* t T_DIV /= */,
+	[T_ELSE] = "else" /* t T_ELSE else */,
+	[T_ELSEIF] = "elseif" /* t T_ELSEIF elseif */,
+	[T_ELSIF] = "elsif" /* t T_ELSIF elsif */,
+	[T_EQ] = "==" /* t T_EQ == */,
+	[T_ERROR] = "error" /* t T_ERROR error */,
+	[T_FETCH] = "fetch" /* t T_FETCH fetch */,
+	[T_FINISH] = "finish" /* t T_FINISH finish */,
+	[T_FUNC] = "func" /* t T_FUNC func */,
+	[T_GEQ] = ">=" /* t T_GEQ >= */,
+	[T_IF] = "if" /* t T_IF if */,
+	[T_INC] = "++" /* t T_INC ++ */,
+	[T_INCR] = "+=" /* t T_INCR += */,
+	[T_LEQ] = "<=" /* t T_LEQ <= */,
+	[T_MUL] = "*=" /* t T_MUL *= */,
+	[T_NEQ] = "!=" /* t T_NEQ != */,
+	[T_NO_CACHE] = "no_cache" /* t T_NO_CACHE no_cache */,
+	[T_NO_NEW_CACHE] = "no_new_cache" /* t T_NO_NEW_CACHE no_new_cache */,
+	[T_PROC] = "proc" /* t T_PROC proc */,
+	[T_REWRITE] = "rewrite" /* t T_REWRITE rewrite */,
+	[T_SET] = "set" /* t T_SET set */,
+	[T_SHL] = "<<" /* t T_SHL << */,
+	[T_SHR] = ">>" /* t T_SHR >> */,
+	[T_SUB] = "sub" /* t T_SUB sub */,
+	[T_SWITCH_CONFIG] = "switch_config" /* t T_SWITCH_CONFIG switch_config */,
+	[VAR] = "VAR" /* t VAR VAR */,
+};

lib/libvcl/vcl_gen_fixed_token.tcl

+#!/usr/local/bin/tclsh8.4
+#
+# Generate a C source file to recognize a set of tokens for the
+# Varnish 
+
+set keywords {
+	if else elseif elsif
+
+	func proc sub
+
+	acl
+
+	backend
+
+	error
+	fetch
+	call
+	no_cache
+	no_new_cache
+	set
+	rewrite
+	finish
+	switch_config
+}
+
+set magic {
+	{"++"	INC}
+	{"--"	DEC}
+	{"&&"	CAND}
+	{"||"	COR}
+	{"<="	LEQ}
+	{"=="	EQ}
+	{"!="	NEQ}
+	{">="	GEQ}
+	{">>"	SHR}
+	{"<<"	SHL}
+	{"+="	INCR}
+	{"/="	DECR}
+	{"*="	MUL}
+	{"/="	DIV}
+}
+
+set char {{}()*+-/%><=;!&.|~,}
+
+set extras {ID VAR CNUM CSTR EOI}
+
+set fo [open "vcl_fixed_token.c" w]
+
+puts $fo {/*
+ * NB:  This file is machine generated, DO NOT EDIT!
+ * instead, edit the Tcl script vcl_gen_fixed_token.tcl and run it by hand
+ */
+}
+
+set foh [open "vcl_token_defs.h" w]
+puts $foh {/*
+ * NB:  This file is machine generated, DO NOT EDIT!
+ * instead, edit the Tcl script vcl_gen_fixed_token.tcl and run it by hand
+ */
+}
+
+puts $fo "#include \"vcl_priv.h\""
+
+set tn 128
+puts $foh "#define LOW_TOKEN $tn"
+
+foreach k $keywords {
+	set t T_[string toupper $k]
+	lappend tokens [list $t $k]
+	puts $foh "#define $t $tn"
+	incr tn
+	lappend fixed [list $k $t 1]
+}
+foreach k $magic {
+	set t T_[string toupper [lindex $k 1]]
+	lappend tokens [list $t [lindex $k 0]]
+	puts $foh "#define $t $tn"
+	incr tn
+	lappend fixed [list [lindex $k 0] $t 0]
+}
+foreach k $extras {
+	set t [string toupper $k]
+	lappend tokens [list $t $t]
+	puts $foh "#define [string toupper $k] $tn"
+	incr tn
+}
+for {set i 0} {$i < [string length $char]} {incr i} {
+	set t [string index $char $i]
+	lappend token2 [list '$t' T$t]
+	lappend fixed [list "$t" '$t' 0]
+}
+
+set tokens [lsort $tokens]
+set token2 [lsort $token2]
+
+# We want to output in ascii order: create sorted first char list
+foreach t $fixed {
+	set xx([string index [lindex $t 0] 0]) 1
+}
+set seq [lsort [array names xx]]
+
+set ll 0
+
+puts $fo {
+unsigned
+fixed_token(const char *p, const char **q)}
+puts $fo "{"
+puts $fo ""
+puts $fo "	switch (p\[0\]) {"
+
+foreach ch "$seq" {
+	# Now find all tokens starting with ch
+	set l ""
+	foreach t $fixed {
+		if {[string index [lindex $t 0] 0] == $ch} {
+			lappend l $t
+		}
+	}
+	# And do then in reverse order to match longest first
+	set l [lsort -index 0 -decreasing $l]
+	scan "$ch" "%c" cx
+	if {$cx != $ll} {
+		if {$ll} {
+			puts $fo "		return (0);"
+		}
+	
+		puts $fo "	case '$ch':"
+		set ll $cx
+	}
+	foreach tt $l {
+		set k [lindex $tt 0]
+		puts -nonewline $fo "		if ("
+		for {set i 0} {$i < [string length $k]} {incr i} {
+			if {$i > 0} {
+				puts -nonewline $fo " && "
+				if {![expr $i % 3]} {
+					puts -nonewline $fo "\n\t\t    "
+				}
+			}
+			puts -nonewline $fo "p\[$i\] == '[string index $k $i]'"
+		}
+		if {[lindex $tt 2]} {
+			if {![expr $i % 3]} {
+				puts -nonewline $fo "\n\t\t    "
+			}
+			puts -nonewline $fo " && !isvar(p\[$i\])"
+		}
+		puts $fo ") {"
+		puts $fo "			*q = p + [string length $k];"
+		puts $fo "			return ([lindex $tt 1]);"
+		puts $fo "		}"
+	}
+} 
+puts $fo "		return (0);"
+puts $fo "	default:"
+puts $fo "		return (0);"
+puts $fo "	}"
+puts $fo "}"
+
+puts $fo ""
+puts $fo "const char *tnames\[256\] = {"
+foreach i $token2 {
+	puts $fo "\t\[[lindex $i 0]\] = \"[lindex $i 0]\" /* t2 $i */,"
+}
+foreach i $tokens {
+	puts $fo "\t\[[lindex $i 0]\] = \"[lindex $i 1]\" /* t $i */,"
+}
+puts $fo "};"
+
+close $foh
+close $fo

lib/libvcl/vcl_lang.h

+/*
+ * Stuff necessary to compile a VCL programs C code
+ */
+
+
+struct vcl_ref {
+	unsigned	line;
+	unsigned	pos;
+	unsigned	count;
+	const char	*token;
+};
+
+struct vcl_acl {
+	unsigned	ip;
+	unsigned	mask;
+};
+
+struct client {
+	unsigned	ip;
+};
+
+struct req {
+	char		*req;
+	char		*useragent;
+	struct {
+		char		*path;
+		char		*host;
+	}		url;
+	double		ttlfactor;
+	struct backend	*backend;
+};
+
+struct backend {
+	unsigned	ip;
+	double		responsetime;
+	double		timeout;
+	double		bandwidth;
+	int		down;
+};
+
+struct obj {
+	int		exists;
+	double		ttl;
+	unsigned	result;
+	unsigned	size;
+	unsigned	usage;
+};
+
+#define VCL_FARGS	struct client *client, struct obj *obj, struct req *req, struct backend *backend
+#define VCL_PASS_ARGS	client, obj, req, backend
+
+void VCL_count(unsigned);
+void VCL_no_cache();
+void VCL_no_new_cache();
+int ip_match(unsigned, struct vcl_acl *);
+int string_match(const char *, const char *);
+int VCL_rewrite(const char *, const char *);
+int VCL_error(unsigned, const char *);
+int VCL_fetch(void);
+int VCL_switch_config(const char *);
+
+

lib/libvcl/vcl_priv.h

+/*
+ * Stuff shared between main.c and fixed_token.c
+ */
+
+#include "vcl_token_defs.h"
+#include <ctype.h>
+
+#define isident1(c) (isalpha(c))
+#define isident(c) (isalpha(c) || isdigit(c) || (c) == '_')
+#define isvar(c) (isident(c) || (c) == '.')
+unsigned fixed_token(const char *p, const char **q);
+extern const char *tnames[256];

lib/libvcl/vcl_token_defs.h

+/*
+ * NB:  This file is machine generated, DO NOT EDIT!
+ * instead, edit the Tcl script vcl_gen_fixed_token.tcl and run it by hand
+ */
+
+#define LOW_TOKEN 128
+#define T_IF 128
+#define T_ELSE 129
+#define T_ELSEIF 130
+#define T_ELSIF 131
+#define T_FUNC 132
+#define T_PROC 133
+#define T_SUB 134
+#define T_ACL 135
+#define T_BACKEND 136
+#define T_ERROR 137
+#define T_FETCH 138
+#define T_CALL 139
+#define T_NO_CACHE 140
+#define T_NO_NEW_CACHE 141
+#define T_SET 142
+#define T_REWRITE 143
+#define T_FINISH 144
+#define T_SWITCH_CONFIG 145
+#define T_INC 146
+#define T_DEC 147
+#define T_CAND 148
+#define T_COR 149
+#define T_LEQ 150
+#define T_EQ 151
+#define T_NEQ 152
+#define T_GEQ 153
+#define T_SHR 154
+#define T_SHL 155
+#define T_INCR 156
+#define T_DECR 157
+#define T_MUL 158
+#define T_DIV 159
+#define ID 160
+#define VAR 161
+#define CNUM 162
+#define CSTR 163
+#define EOI 164