Improve the VCL compiler in various ways:

Generate the methods and their legal returns with the tcl script.

Add consistency checks to make sure methods don't use illegal returns,
and also check called subrourtines.

Add consistency check to complain about recursive subroutine calls.

Add consistency check to complain about unused or undefined subroutines.



git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@222 d4fa192b-c00b-0410-8231-f00ffab90ce4

bin/varnishd/cache.h

@@ -78,13 +78,7 @@ struct object {
 	TAILQ_HEAD(, storage)	store;
 };
 
-#define	HND_Error	(1 << 0)
-#define	HND_Pipe	(1 << 1)
-#define	HND_Pass	(1 << 2)
-#define	HND_Lookup	(1 << 3)
-#define	HND_Fetch	(1 << 4)
-#define	HND_Insert	(1 << 5)
-#define	HND_Deliver	(1 << 6)
+#include "vcl_returns.h"
 
 struct sess {
 	int			fd;

bin/varnishd/cache_pool.c

@@ -109,23 +109,23 @@ CacheWorker(void *priv)
 
 		for (done = 0; !done; ) {
 			switch(sp->handling) {
-			case HND_Lookup:
+			case VCL_RET_LOOKUP:
 				VSL(SLT_Handling, sp->fd, "Lookup");
 				done = LookupSession(&w, sp);
 				break;
-			case HND_Fetch:
+			case VCL_RET_FETCH:
 				done = FetchSession(&w, sp);
 				break;
-			case HND_Deliver:
+			case VCL_RET_DELIVER:
 				VSL(SLT_Handling, sp->fd, "Deliver");
 				done = DeliverSession(&w, sp);
 				break;
-			case HND_Pipe:
+			case VCL_RET_PIPE:
 				VSL(SLT_Handling, sp->fd, "Pipe");
 				PipeSession(&w, sp);
 				done = 1;
 				break;
-			case HND_Pass:
+			case VCL_RET_PASS:
 				VSL(SLT_Handling, sp->fd, "Pass");
 				PassSession(&w, sp);
 				done = 1;

bin/varnishd/cache_vcl.c

@@ -195,13 +195,11 @@ HandlingName(unsigned u)
 {
 
 	switch (u) {
-	case HND_Error:		return ("Error");
-	case HND_Pass:		return ("Pass");
-	case HND_Pipe:		return ("Pipe");
-	case HND_Lookup:	return ("Lookup");
-	case HND_Fetch:		return ("Fetch");
-	case HND_Insert:	return ("Insert");
-	case HND_Deliver:	return ("Deliver");
+#define VCL_RET_MAC(a, b, c)	case VCL_RET_##b: return(#a);
+#define VCL_RET_MAC_E(a, b, c)	case VCL_RET_##b: return(#a);
+#include "vcl_returns.h"
+#undef VCL_RET_MAC
+#undef VCL_RET_MAC_E
 	default:		return (NULL);
 	}
 }
@@ -226,7 +224,7 @@ CheckHandling(struct sess *sp, const char *func, unsigned bitmap)
 		    "Wrong handling after %s function: 0x%x", func, u);
 	else
 		return;
-	sp->handling = HND_Error;
+	sp->handling = VCL_RET_ERROR;
 }
 
 #define VCL_method(func, bitmap) 		\
@@ -239,7 +237,8 @@ VCL_##func##_method(struct sess *sp)		\
 	CheckHandling(sp, #func, (bitmap));	\
 }
 
-VCL_method(recv,  HND_Error|HND_Pass|HND_Pipe|HND_Lookup)
-VCL_method(miss,  HND_Error|HND_Pass|HND_Pipe|HND_Fetch)
-VCL_method(hit,	  HND_Error|HND_Pass|HND_Pipe|HND_Deliver)
-VCL_method(fetch, HND_Error|HND_Pass|HND_Pipe|HND_Insert)
+#define VCL_RET_MAC(l,u,b)
+#define VCL_MET_MAC(l,u,b) VCL_method(l, b)
+#include "vcl_returns.h"
+#undef VCL_MET_MAC
+#undef VCL_RET_MAC

bin/varnishd/cache_vrt.c

@@ -75,19 +75,7 @@ VRT_handling(struct sess *sp, unsigned hand)
 {
 
 	assert(!(hand & (hand -1)));	/* must be power of two */
-	switch (hand) {
-#define FOO(a,b)  case VRT_H_##a: sp->handling = HND_##b; break;
-	FOO(error, Error);
-	FOO(pipe, Pipe);
-	FOO(pass, Pass);
-	FOO(lookup, Lookup);
-	FOO(fetch, Fetch);
-	FOO(insert, Insert);
-	FOO(deliver, Deliver);
-#undef FOO
-	default:
-		assert(hand == 0);
-	}
+	sp->handling = hand;
 }
 
 int

include/vcl.h

 /*
  * $Id$
  *
- * Interface to a compiled VCL program.
+ * NB:  This file is machine generated, DO NOT EDIT!
  *
- * XXX: When this file is changed, lib/libvcl/vcl_gen_fixed_token.tcl
- * XXX: *MUST* be rerun.
+ * Edit vcl_gen_fixed_token.tcl instead
  */
 
 struct sess;
@@ -13,16 +12,19 @@ typedef void vcl_init_f(void);
 typedef int vcl_func_f(struct sess *sp);
 
 struct VCL_conf {
-	unsigned	magic;
-#define VCL_CONF_MAGIC	0x7406c509	/* from /dev/random */
-	vcl_init_f	*init_func;
+	unsigned        magic;
+#define VCL_CONF_MAGIC  0x7406c509      /* from /dev/random */
+
+        struct backend  **backend;
+        unsigned        nbackend;
+        struct vrt_ref  *ref;
+        unsigned        nref;
+        unsigned        busy;
+
+        vcl_init_f      *init_func;
+
 	vcl_func_f	*recv_func;
-	vcl_func_f	*hit_func;
 	vcl_func_f	*miss_func;
+	vcl_func_f	*hit_func;
 	vcl_func_f	*fetch_func;
-	struct backend	**backend;
-	unsigned	nbackend;
-	struct vrt_ref	*ref;
-	unsigned	nref;
-	unsigned	busy;
 };

include/vcl_returns.h

+/*
+ * $Id$
+ *
+ * NB:  This file is machine generated, DO NOT EDIT!
+ *
+ * Edit vcl_gen_fixed_token.tcl instead
+ */
+
+#ifdef VCL_RET_MAC
+#ifdef VCL_RET_MAC_E
+VCL_RET_MAC_E(error, ERROR, 0)
+#endif
+VCL_RET_MAC(lookup, LOOKUP, (1 << 1))
+VCL_RET_MAC(pipe, PIPE, (1 << 2))
+VCL_RET_MAC(pass, PASS, (1 << 3))
+VCL_RET_MAC(fetch, FETCH, (1 << 4))
+VCL_RET_MAC(insert, INSERT, (1 << 5))
+VCL_RET_MAC(deliver, DELIVER, (1 << 6))
+#else
+#define VCL_RET_ERROR  (1 << 0)
+#define VCL_RET_LOOKUP  (1 << 1)
+#define VCL_RET_PIPE  (1 << 2)
+#define VCL_RET_PASS  (1 << 3)
+#define VCL_RET_FETCH  (1 << 4)
+#define VCL_RET_INSERT  (1 << 5)
+#define VCL_RET_DELIVER  (1 << 6)
+#define VCL_RET_MAX 7
+#endif
+
+#ifdef VCL_MET_MAC
+VCL_MET_MAC(recv,RECV,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_LOOKUP))
+VCL_MET_MAC(miss,MISS,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_FETCH))
+VCL_MET_MAC(hit,HIT,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_DELIVER))
+VCL_MET_MAC(fetch,FETCH,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_INSERT))
+#endif

include/vrt.h

@@ -7,14 +7,6 @@
  * XXX: *MUST* be rerun.
  */
 
-#define	VRT_H_error	(1 << 0)
-#define	VRT_H_pipe	(1 << 1)
-#define	VRT_H_pass	(1 << 2)
-#define	VRT_H_lookup	(1 << 3)
-#define	VRT_H_fetch	(1 << 4)
-#define	VRT_H_insert	(1 << 5)
-#define	VRT_H_deliver	(1 << 6)
-
 struct sess;
 struct backend;
 struct VCL_conf;

lib/libvcl/vcl_compile.c

@@ -50,6 +50,7 @@
 #include <unistd.h>
 
 #include "vcl_priv.h"
+#include "vcl_returns.h"
 
 #include "libvcl.h"
 
@@ -77,6 +78,8 @@ struct tokenlist {
 	struct sbuf		*sb;
 	int			err;
 	int			nbackend;
+	TAILQ_HEAD(, proc)	procs;
+	struct proc		*curproc;
 };
 
 enum var_type {
@@ -117,6 +120,41 @@ struct var {
 	const char		*lname;
 };
 
+/*--------------------------------------------------------------------
+ * Consistency check
+ */
+
+static struct method {
+	const char		*name;
+	unsigned		returns;
+} method_tab[] = {
+#define VCL_RET_MAC(a,b,c)
+#define VCL_MET_MAC(a,b,c)	{ "vcl_"#a, c },
+#include "vcl_returns.h"
+#undef VCL_MET_MAC
+#undef VCL_RET_MAC
+	{ NULL, 0U }
+};
+
+struct proccall {
+	TAILQ_ENTRY(proccall)	list;
+	struct proc		*p;
+	struct token		*t;
+};
+
+struct proc {
+	TAILQ_ENTRY(proc)	list;
+	TAILQ_HEAD(,proccall)	calls;
+	struct token		*name;
+	unsigned		returns;
+	unsigned		exists;
+	unsigned		called;
+	unsigned		active;
+	struct token		*returnt[VCL_RET_MAX];
+};
+
+/*--------------------------------------------------------------------*/
+
 static struct var be_vars[] = {
 	{ "backend.host",
 		HOSTNAME, 0,  NULL, "VRT_set_backend_hostname(backend, %s)" },
@@ -152,6 +190,8 @@ static struct var vars[] = {
 
 static void Compound(struct tokenlist *tl);
 static void Cond_0(struct tokenlist *tl);
+static struct proc *AddProc(struct tokenlist *tl, struct token *t, int def);
+static void AddCall(struct tokenlist *tl, struct token *t);
 
 /*--------------------------------------------------------------------*/
 
@@ -925,15 +965,14 @@ Action(struct tokenlist *tl)
 		I(tl);
 		sbuf_printf(tl->fc, "VCL_no_cache(sp);\n");
 		return;
-	case T_DELIVER:
-	case T_LOOKUP:
-	case T_PASS:
-	case T_FETCH:
-	case T_INSERT:
-		I(tl); sbuf_printf(tl->fc, "VRT_done(sp, VRT_H_%*.*s);\n",
-		    at->e - at->b,
-		    at->e - at->b, at->b);
+#define VCL_RET_MAC(a,b,c) case T_##b: \
+		I(tl); \
+		sbuf_printf(tl->fc, "VRT_done(sp, VCL_RET_%s);\n", #b); \
+		tl->curproc->returns |= VCL_RET_##b; \
+		tl->curproc->returnt[c] = at; \
 		return;
+#include "vcl_returns.h"
+#undef VCL_RET_MAC
 	case T_ERROR:
 		if (tl->t->tok == CNUM)
 			a = UintVal(tl);
@@ -948,7 +987,7 @@ Action(struct tokenlist *tl)
 			NextToken(tl);
 		} else
 			sbuf_printf(tl->fc, "(const char *)0);\n");
-		I(tl); sbuf_printf(tl->fc, "VRT_done(sp, VRT_H_error);\n");
+		I(tl); sbuf_printf(tl->fc, "VRT_done(sp, VCL_RET_ERROR);\n");
 		return;
 	case T_SWITCH_CONFIG:
 		ExpectErr(tl, ID);
@@ -960,6 +999,7 @@ Action(struct tokenlist *tl)
 		return;
 	case T_CALL:
 		ExpectErr(tl, ID);
+		AddCall(tl, tl->t);
 		AddRef(tl, tl->t, R_FUNC);
 		I(tl); sbuf_printf(tl->fc,
 		    "if (VGC_function_%*.*s(sp))\n",
@@ -1294,8 +1334,11 @@ Function(struct tokenlist *tl)
 
 	NextToken(tl);
 	ExpectErr(tl, ID);
+	tl->curproc = AddProc(tl, tl->t, 1);
+	tl->curproc->exists++;
 	AddDef(tl, tl->t, R_FUNC);
-	sbuf_printf(tl->fh, "static int VGC_function_%*.*s (struct sess *sp);\n",
+	sbuf_printf(tl->fh,
+	    "static int VGC_function_%*.*s (struct sess *sp);\n",
 	    tl->t->e - tl->t->b,
 	    tl->t->e - tl->t->b, tl->t->b);
 	I(tl); sbuf_printf(tl->fc, "static int\n");
@@ -1460,6 +1503,125 @@ Lexer(struct tokenlist *tl, const char *b, const char *e)
 	AddToken(tl, EOI, p, p);
 }
 
+/*--------------------------------------------------------------------
+ * Consistency check
+ */
+
+static struct proc *
+AddProc(struct tokenlist *tl, struct token *t, int def)
+{
+	struct proc *p;
+
+	TAILQ_FOREACH(p, &tl->procs, list) {
+		if (p->name->e - p->name->b != t->e - t->b)
+			continue;
+		if (!memcmp(p->name->b, t->b, t->e - t->b)) {
+			if (def)
+				p->name = t;
+			return (p);
+		}
+	}
+	p = calloc(sizeof *p, 1);
+	assert(p != NULL);
+	p->name = t;
+	TAILQ_INIT(&p->calls);
+	TAILQ_INSERT_TAIL(&tl->procs, p, list);
+	return (p);
+}
+
+static void
+AddCall(struct tokenlist *tl, struct token *t)
+{
+	struct proccall *pc;
+	struct proc *p;
+
+	p = AddProc(tl, t, 0);
+	TAILQ_FOREACH(pc, &tl->curproc->calls, list) {
+		if (pc->p == p)
+			return;
+	}
+	pc = calloc(sizeof *pc, 1);
+	assert(pc != NULL);
+	pc->p = p;
+	pc->t = t;
+	TAILQ_INSERT_TAIL(&tl->curproc->calls, pc, list);
+}
+
+static int
+Consist_Decend(struct tokenlist *tl, struct proc *p, unsigned returns)
+{
+	unsigned u;
+	struct proccall *pc;
+
+	if (!p->exists) {
+		sbuf_printf(tl->sb, "Function %*.*s does not exist\n",
+			p->name->e - p->name->b,
+			p->name->e - p->name->b,
+			p->name->b);
+		return (1);
+	}
+	if (p->active) {
+		sbuf_printf(tl->sb, "Function recurses on\n");
+		ErrWhere(tl, p->name);
+		return (1);
+	}
+	u = p->returns & ~returns;
+	if (u) {
+#define VCL_RET_MAC(a, b, c) \
+		if (u & VCL_RET_##b) { \
+			sbuf_printf(tl->sb, "Illegal return for method\n"); \
+			ErrWhere(tl, p->returnt[c]); \
+		} 
+#include "vcl_returns.h"
+#undef VCL_RET_MAC
+		sbuf_printf(tl->sb, "In function\n");
+		ErrWhere(tl, p->name);
+		return (1);
+	}
+	p->active = 1;
+	TAILQ_FOREACH(pc, &p->calls, list) {
+		if (Consist_Decend(tl, pc->p, returns)) {
+			sbuf_printf(tl->sb, "\nCalled from\n");
+			ErrWhere(tl, p->name);
+			sbuf_printf(tl->sb, "at\n");
+			ErrWhere(tl, pc->t);
+			return (1);
+		}
+	}
+	p->active = 0;
+	p->called++;
+	return (0);
+}
+
+static int
+Consistency(struct tokenlist *tl)
+{
+	struct proc *p;
+	struct method *m;
+
+	TAILQ_FOREACH(p, &tl->procs, list) {
+		for(m = method_tab; m->name != NULL; m++) {
+			if (IdIs(p->name, m->name))
+				break;
+		}
+		if (m->name == NULL) 
+			continue;
+		if (Consist_Decend(tl, p, m->returns)) {
+			sbuf_printf(tl->sb,
+			    "\nwhich is a %s method\n", m->name);
+			return (1);
+		}
+	}
+	TAILQ_FOREACH(p, &tl->procs, list) {
+		if (p->called)
+			continue;
+		sbuf_printf(tl->sb, "Function unused\n");
+		ErrWhere(tl, p->name);
+		return (1);
+	}
+	return (0);
+}
+
 /*--------------------------------------------------------------------*/
 
 static void
@@ -1614,6 +1776,7 @@ VCC_Compile(struct sbuf *sb, const char *b, const char *e)
 	memset(&tokens, 0, sizeof tokens);
 	TAILQ_INIT(&tokens.tokens);
 	TAILQ_INIT(&tokens.refs);
+	TAILQ_INIT(&tokens.procs);
 	tokens.sb = sb;
 
 	tokens.fc = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND);
@@ -1636,6 +1799,7 @@ VCC_Compile(struct sbuf *sb, const char *b, const char *e)
 	Parse(&tokens);
 	if (tokens.err)
 		goto done;
+	Consistency(&tokens);
 if (0)
 	CheckRefs(&tokens);
 	if (tokens.err)

lib/libvcl/vcl_fixed_token.c

 /*
+ * $Id$
+ *
  * NB:  This file is machine generated, DO NOT EDIT!
- * instead, edit the Tcl script vcl_gen_fixed_token.tcl and run it by hand
+ *
+ * Edit vcl_gen_fixed_token.tcl instead
  */
 
 #include <stdio.h>
@@ -261,6 +264,11 @@ vcl_fixed_token(const char *p, const char **q)
 			*q = p + 4;
 			return (T_PROC);
 		}
+		if (p[0] == 'p' && p[1] == 'i' && p[2] == 'p' && 
+		    p[3] == 'e' && !isvar(p[4])) {
+			*q = p + 4;
+			return (T_PIPE);
+		}
 		if (p[0] == 'p' && p[1] == 'a' && p[2] == 's' && 
 		    p[3] == 's' && !isvar(p[4])) {
 			*q = p + 4;
@@ -384,6 +392,7 @@ vcl_init_tnames(void)
 	vcl_tnames[T_NO_CACHE] = "no_cache";
 	vcl_tnames[T_NO_NEW_CACHE] = "no_new_cache";
 	vcl_tnames[T_PASS] = "pass";
+	vcl_tnames[T_PIPE] = "pipe";
 	vcl_tnames[T_PROC] = "proc";
 	vcl_tnames[T_REWRITE] = "rewrite";
 	vcl_tnames[T_SET] = "set";
@@ -397,13 +406,19 @@ vcl_init_tnames(void)
 void
 vcl_output_lang_h(FILE *f)
 {
+	fputs("#define VCL_RET_ERROR  (1 << 0)\n", f);
+	fputs("#define VCL_RET_LOOKUP  (1 << 1)\n", f);
+	fputs("#define VCL_RET_PIPE  (1 << 2)\n", f);
+	fputs("#define VCL_RET_PASS  (1 << 3)\n", f);
+	fputs("#define VCL_RET_FETCH  (1 << 4)\n", f);
+	fputs("#define VCL_RET_INSERT  (1 << 5)\n", f);
+	fputs("#define VCL_RET_DELIVER  (1 << 6)\n", f);
 	fputs("/*\n", f);
 	fputs(" * $Id$\n", f);
 	fputs(" *\n", f);
-	fputs(" * Interface to a compiled VCL program.\n", f);
+	fputs(" * NB:  This file is machine generated, DO NOT EDIT!\n", f);
 	fputs(" *\n", f);
-	fputs(" * XXX: When this file is changed, lib/libvcl/vcl_gen_fixed_token.tcl\n", f);
-	fputs(" * XXX: *MUST* be rerun.\n", f);
+	fputs(" * Edit vcl_gen_fixed_token.tcl instead\n", f);
 	fputs(" */\n", f);
 	fputs("\n", f);
 	fputs("struct sess;\n", f);
@@ -412,18 +427,21 @@ vcl_output_lang_h(FILE *f)
 	fputs("typedef int vcl_func_f(struct sess *sp);\n", f);
 	fputs("\n", f);
 	fputs("struct VCL_conf {\n", f);
-	fputs("	unsigned	magic;\n", f);
-	fputs("#define VCL_CONF_MAGIC	0x7406c509	/* from /dev/random */\n", f);
-	fputs("	vcl_init_f	*init_func;\n", f);
+	fputs("	unsigned        magic;\n", f);
+	fputs("#define VCL_CONF_MAGIC  0x7406c509      /* from /dev/random */\n", f);
+	fputs("\n", f);
+	fputs("        struct backend  **backend;\n", f);
+	fputs("        unsigned        nbackend;\n", f);
+	fputs("        struct vrt_ref  *ref;\n", f);
+	fputs("        unsigned        nref;\n", f);
+	fputs("        unsigned        busy;\n", f);
+	fputs("\n", f);
+	fputs("        vcl_init_f      *init_func;\n", f);
+	fputs("\n", f);
 	fputs("	vcl_func_f	*recv_func;\n", f);
-	fputs("	vcl_func_f	*hit_func;\n", f);
 	fputs("	vcl_func_f	*miss_func;\n", f);
+	fputs("	vcl_func_f	*hit_func;\n", f);
 	fputs("	vcl_func_f	*fetch_func;\n", f);
-	fputs("	struct backend	**backend;\n", f);
-	fputs("	unsigned	nbackend;\n", f);
-	fputs("	struct vrt_ref	*ref;\n", f);
-	fputs("	unsigned	nref;\n", f);
-	fputs("	unsigned	busy;\n", f);
 	fputs("};\n", f);
 	fputs("/*\n", f);
 	fputs(" * $Id$ \n", f);
@@ -434,14 +452,6 @@ vcl_output_lang_h(FILE *f)
 	fputs(" * XXX: *MUST* be rerun.\n", f);
 	fputs(" */\n", f);
 	fputs("\n", f);
-	fputs("#define	VRT_H_error	(1 << 0)\n", f);
-	fputs("#define	VRT_H_pipe	(1 << 1)\n", f);
-	fputs("#define	VRT_H_pass	(1 << 2)\n", f);
-	fputs("#define	VRT_H_lookup	(1 << 3)\n", f);
-	fputs("#define	VRT_H_fetch	(1 << 4)\n", f);
-	fputs("#define	VRT_H_insert	(1 << 5)\n", f);
-	fputs("#define	VRT_H_deliver	(1 << 6)\n", f);
-	fputs("\n", f);
 	fputs("struct sess;\n", f);
 	fputs("struct backend;\n", f);
 	fputs("struct VCL_conf;\n", f);

lib/libvcl/vcl_gen_fixed_token.tcl

 #!/usr/local/bin/tclsh8.4
 #
-# Generate a C source file to recognize a set of tokens for the
-# Varnish 
+# Generate various .c and .h files for the VCL compiler and the interfaces
+# for it.
 
+# These are the metods which can be called in the VCL program. 
+# Second element is list of valid return actions.
+#
+set methods {
+	{recv	{error pass pipe lookup}}
+	{miss	{error pass pipe fetch}}
+	{hit	{error pass pipe deliver}}
+	{fetch	{error pass pipe insert}}
+}
+
+# These are the return actions
+#
+set returns {
+	error
+	lookup
+	pipe
+	pass
+	fetch
+	insert
+	deliver
+}
+
+# Language keywords
+#
 set keywords {
 	if else elseif elsif
 
@@ -12,13 +36,6 @@ set keywords {
 
 	backend
 
-	error
-	lookup
-	pass
-	fetch
-	insert
-	deliver
-
 	call
 	no_cache
 	no_new_cache
@@ -27,6 +44,8 @@ set keywords {
 	switch_config
 }
 
+# Non-word tokens
+#
 set magic {
 	{"++"	INC}
 	{"--"	DEC}
@@ -44,24 +63,107 @@ set magic {
 	{"/="	DIV}
 }
 
+# Single char tokens
+#
 set char {{}()*+-/%><=;!&.|~,}
 
+# Other token identifiers
+#
 set extras {ID VAR CNUM CSTR EOI}
 
-set fo [open "vcl_fixed_token.c" w]
+#----------------------------------------------------------------------
+# Boilerplate warning for all generated files.
 
-puts $fo {/*
- * NB:  This file is machine generated, DO NOT EDIT!
- * instead, edit the Tcl script vcl_gen_fixed_token.tcl and run it by hand
- */
+proc warns {fd} {
+
+	puts $fd "/*"
+	puts $fd { * $Id$}
+	puts $fd " *"
+	puts $fd " * NB:  This file is machine generated, DO NOT EDIT!"
+	puts $fd " *"
+	puts $fd " * Edit vcl_gen_fixed_token.tcl instead"
+	puts $fd " */"
+	puts $fd ""
 }
 
-set foh [open "vcl_token_defs.h" w]
-puts $foh {/*
- * NB:  This file is machine generated, DO NOT EDIT!
- * instead, edit the Tcl script vcl_gen_fixed_token.tcl and run it by hand
- */
+#----------------------------------------------------------------------
+# Build the vcl.h #include file
+
+set fo [open ../../include/vcl.h w]
+warns $fo
+puts $fo {struct sess;
+
+typedef void vcl_init_f(void);
+typedef int vcl_func_f(struct sess *sp);
+}
+puts $fo "struct VCL_conf {"
+puts $fo {	unsigned        magic;
+#define VCL_CONF_MAGIC  0x7406c509      /* from /dev/random */
+
+        struct backend  **backend;
+        unsigned        nbackend;
+        struct vrt_ref  *ref;
+        unsigned        nref;
+        unsigned        busy;
+
+        vcl_init_f      *init_func;
+}
+foreach m $methods {
+	puts $fo "\tvcl_func_f\t*[lindex $m 0]_func;"
+}
+puts $fo "};"
+
+close $fo
+
+#----------------------------------------------------------------------
+# Build the vcl_returns.h #include file
+
+set for [open "../../include/vcl_returns.h" w]
+warns $for
+puts $for "#ifdef VCL_RET_MAC"
+set i 0
+foreach k $returns {
+	if {$k == "error"} {
+		puts $for "#ifdef VCL_RET_MAC_E"
+		puts $for "VCL_RET_MAC_E($k, [string toupper $k], $i)"
+		puts $for "#endif"
+	} else {
+		puts $for "VCL_RET_MAC($k, [string toupper $k], (1 << $i))"
+	}
+	incr i
+}
+puts $for "#else"
+set i 0
+foreach k $returns {
+	puts $for "#define VCL_RET_[string toupper $k]  (1 << $i)"
+	incr i
+}
+puts $for "#define VCL_RET_MAX $i"
+puts $for "#endif"
+puts $for ""
+puts $for "#ifdef VCL_MET_MAC"
+foreach m $methods {
+	puts -nonewline $for "VCL_MET_MAC([lindex $m 0]"
+	puts -nonewline $for ",[string toupper [lindex $m 0]]"
+	set l [lindex $m 1]
+	puts -nonewline $for ",(VCL_RET_[string toupper [lindex $l 0]]"
+	foreach r [lrange $l 1 end] {
+		puts -nonewline $for "|VCL_RET_[string toupper $r]"
+	}
+	puts -nonewline $for ")"
+	puts $for ")"
 }
+puts $for "#endif"
+close $for
+
+#----------------------------------------------------------------------
+# Build the compiler token table and recognizers
+
+set fo [open "vcl_fixed_token.c" w]
+warns $fo
+
+set foh [open "vcl_token_defs.h" w]
+warns $foh
 
 puts $fo "#include <stdio.h>"
 puts $fo "#include <ctype.h>"
@@ -70,20 +172,24 @@ puts $fo "#include \"vcl_priv.h\""
 set tn 128
 puts $foh "#define LOW_TOKEN $tn"
 
-foreach k $keywords {
-	set t T_[string toupper $k]
-	lappend tokens [list $t $k]
-	puts $foh "#define $t $tn"
+
+proc add_token {tok str alpha} {
+	global tokens tn fixed foh
+
+	lappend tokens [list $tok $str]
+	puts $foh "#define $tok $tn"
 	incr tn
-	lappend fixed [list $k $t 1]
+	lappend fixed [list $str $tok $alpha]
 }
-foreach k $magic {
-	set t T_[string toupper [lindex $k 1]]
-	lappend tokens [list $t [lindex $k 0]]
-	puts $foh "#define $t $tn"
-	incr tn
-	lappend fixed [list [lindex $k 0] $t 0]
+
+proc mk_token {tok str alpha} {
+	set tok T_[string toupper $tok]
+	add_token $tok $str $alpha
 }
+
+foreach k $keywords { mk_token $k $k 1 }
+foreach k $returns { mk_token $k $k 1 }
+foreach k $magic { mk_token [lindex $k 1] [lindex $k 0] 0 }
 foreach k $extras {
 	set t [string toupper $k]
 	lappend tokens [list $t $t]
@@ -176,7 +282,9 @@ foreach i $tokens {
 }
 puts $fo "}"
 
-
+#----------------------------------------------------------------------
+# Create the C-code which emits the boilerplate definitions for the
+# generated C code output
 
 proc copy_include {n} {
 	global fo
@@ -193,6 +301,12 @@ puts $fo ""
 puts $fo "void"
 puts $fo "vcl_output_lang_h(FILE *f)"
 puts $fo "{"
+set i 0
+foreach k $returns {
+	puts $fo "\tfputs(\"#define VCL_RET_[string toupper $k]  (1 << $i)\\n\", f);"
+	incr i
+}
+
 copy_include ../../include/vcl.h
 copy_include ../../include/vrt.h
 

lib/libvcl/vcl_token_defs.h

 /*
+ * $Id$
+ *
  * NB:  This file is machine generated, DO NOT EDIT!
- * instead, edit the Tcl script vcl_gen_fixed_token.tcl and run it by hand
+ *
+ * Edit vcl_gen_fixed_token.tcl instead
  */
 
 #define LOW_TOKEN 128
@@ -13,34 +16,35 @@
 #define T_SUB 134
 #define T_ACL 135
 #define T_BACKEND 136
-#define T_ERROR 137
-#define T_LOOKUP 138
-#define T_PASS 139
-#define T_FETCH 140
-#define T_INSERT 141
-#define T_DELIVER 142
-#define T_CALL 143
-#define T_NO_CACHE 144
-#define T_NO_NEW_CACHE 145
-#define T_SET 146
-#define T_REWRITE 147
-#define T_SWITCH_CONFIG 148
-#define T_INC 149
-#define T_DEC 150
-#define T_CAND 151
-#define T_COR 152
-#define T_LEQ 153
-#define T_EQ 154
-#define T_NEQ 155
-#define T_GEQ 156
-#define T_SHR 157
-#define T_SHL 158
-#define T_INCR 159
-#define T_DECR 160
-#define T_MUL 161
-#define T_DIV 162
-#define ID 163
-#define VAR 164
-#define CNUM 165
-#define CSTR 166
-#define EOI 167
+#define T_CALL 137
+#define T_NO_CACHE 138
+#define T_NO_NEW_CACHE 139
+#define T_SET 140
+#define T_REWRITE 141
+#define T_SWITCH_CONFIG 142
+#define T_ERROR 143
+#define T_LOOKUP 144
+#define T_PIPE 145
+#define T_PASS 146
+#define T_FETCH 147
+#define T_INSERT 148
+#define T_DELIVER 149
+#define T_INC 150
+#define T_DEC 151
+#define T_CAND 152
+#define T_COR 153
+#define T_LEQ 154
+#define T_EQ 155
+#define T_NEQ 156
+#define T_GEQ 157
+#define T_SHR 158
+#define T_SHL 159
+#define T_INCR 160
+#define T_DECR 161
+#define T_MUL 162
+#define T_DIV 163
+#define ID 164
+#define VAR 165
+#define CNUM 166
+#define CSTR 167
+#define EOI 168