Implement a facility for source file modularization in the VCL

compiler.  The syntax is:

	include "filename" ;

Unlike the C preprocessors #include directive, a VCL include can
appear anywhere in the sourcefile:

	if {req.Cookie == include "cookie.vcl" ; || !req.Host } {
	}

and have cookie.vcl contain just:

	"8435398475983275293759843"


Technically this results in a change to how we account for source
code references in the counter/profile table as well, and as a result
the entire source code of the VCL program is now compiled into the
shared library for easy reference.



git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@1281 d4fa192b-c00b-0410-8231-f00ffab90ce4

include/vcl.h

@@ -22,6 +22,10 @@ struct VCL_conf {
         unsigned        nref;
         unsigned        busy;
 
+	unsigned	nsrc;
+	const char	**srcname;
+	const char	**srcbody;
+
         void            *priv;
 
         vcl_init_f      *init_func;

include/vrt.h

@@ -40,7 +40,8 @@ struct backend;
 struct VCL_conf;
 
 struct vrt_ref {
-	unsigned	file;
+	unsigned	source;
+	unsigned	offset;
 	unsigned	line;
 	unsigned	pos;
 	unsigned	count;

lib/libvcl/vcc_acl.c

@@ -120,14 +120,14 @@ vcc_Acl(struct tokenlist *tl)
 			mask = UintVal(tl);
 		}
 		Fc(tl, 1, "{ %u, %u, %u, ", not, mask, para);
-		EncString(tl->fc, t);
+		EncToken(tl->fc, t);
 		Fc(tl, 0, ", \"");
 		if (para)
 			Fc(tl, 0, "(");
 		if (not)
 			Fc(tl, 0, "!");
 		Fc(tl, 0, "\\\"\" ");
-		EncString(tl->fc, t);
+		EncToken(tl->fc, t);
 		Fc(tl, 0, " \"\\\"");
 		if (mask)
 			Fc(tl, 0, "/%u", mask);

lib/libvcl/vcc_compile.c

@@ -86,8 +86,8 @@
 #include "libvcl.h"
 
 static struct method method_tab[] = {
-#define VCL_RET_MAC(a,b,c,d)
-#define VCL_MET_MAC(a,b,c)	{ "vcl_"#a, "default_vcl_"#a, c },
+#define VCL_RET_MAC(l,U,b,n)
+#define VCL_MET_MAC(l,U,m)	{ "vcl_"#l, "default_vcl_"#l, m },
 #include "vcl_returns.h"
 #undef VCL_MET_MAC
 #undef VCL_RET_MAC
@@ -170,23 +170,42 @@ Ff(struct tokenlist *tl, int indent, const char *fmt, ...)
 /*--------------------------------------------------------------------*/
 
 void
-EncString(struct vsb *sb, struct token *t)
+EncString(struct vsb *sb, const char *b, const char *e)
 {
-	const char *p;
 
-	assert(t->tok == CSTR);
+	if (e == NULL)
+		e = strchr(b, '\0');
+
 	vsb_cat(sb, "\"");
-	for (p = t->dec; *p != '\0'; p++) {
-		if (*p == '\\' || *p == '"')
-			vsb_printf(sb, "\\%c", *p);
-		else if (isgraph(*p))
-			vsb_printf(sb, "%c", *p);
-		else
-			vsb_printf(sb, "\\%03o", *p);
+	for (; b < e; b++) {
+		switch (*b) {
+		case '\\':
+		case '"':
+			vsb_printf(sb, "\\%c", *b);
+			break;
+		case '\n': vsb_printf(sb, "\\n"); break;
+		case '\t': vsb_printf(sb, "\\t"); break;
+		case '\r': vsb_printf(sb, "\\r"); break;
+		case ' ': vsb_printf(sb, " "); break;
+		default:
+			if (isgraph(*b))
+				vsb_printf(sb, "%c", *b);
+			else
+				vsb_printf(sb, "\\%03o", *b);
+			break;
+		}
 	}
 	vsb_cat(sb, "\"");
 }
 
+void
+EncToken(struct vsb *sb, struct token *t)
+{
+
+	assert(t->tok == CSTR);
+	EncString(sb, t->dec, NULL);
+}
+
 /*--------------------------------------------------------------------*/
 
 static int
@@ -505,7 +524,7 @@ vcc_re(struct tokenlist *tl, const char *str, struct token *re)
 	Fc(tl, 1, "VRT_re_match(%s, %s)\n", str, buf);
 	Fh(tl, 0, "void *%s;\n", buf);
 	Fi(tl, 0, "\tVRT_re_init(&%s, ",buf);
-	EncString(tl->fi, re);
+	EncToken(tl->fi, re);
 	Fi(tl, 0, ");\n");
 	Ff(tl, 0, "\tVRT_re_fini(%s);\n", buf);
 }
@@ -530,7 +549,7 @@ Cond_String(struct var *vp, struct tokenlist *tl)
 		    tl->t->tok == T_EQ ? "!" : "", vp->rname);
 		vcc_NextToken(tl);
 		ExpectErr(tl, CSTR);
-		EncString(tl->fc, tl->t);
+		EncToken(tl->fc, tl->t);
 		Fc(tl, 0, ")\n");
 		vcc_NextToken(tl);
 		break;
@@ -974,7 +993,7 @@ Backend(struct tokenlist *tl)
 			ExpectErr(tl, CSTR);
 			t_host = tl->t;
 			Fc(tl, 1, "\t%s ", vp->lname);
-			EncString(tl->fc, t_host);
+			EncToken(tl->fc, t_host);
 			Fc(tl, 0, ");\n");
 			vcc_NextToken(tl);
 			break;
@@ -982,7 +1001,7 @@ Backend(struct tokenlist *tl)
 			ExpectErr(tl, CSTR);
 			t_port = tl->t;
 			Fc(tl, 1, "\t%s ", vp->lname);
-			EncString(tl->fc, t_port);
+			EncToken(tl->fc, t_port);
 			Fc(tl, 0, ");\n");
 			vcc_NextToken(tl);
 			break;
@@ -1282,32 +1301,39 @@ CheckRefs(struct tokenlist *tl)
 	return (nerr);
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Output the location/profiling table.  For each counted token, we
+ * record source+line+charpos for the first character in the token.
+ */
 
 static void
 LocTable(struct tokenlist *tl)
 {
 	struct token *t;
-	unsigned fil, lin, pos;
+	unsigned lin, pos;
+	struct source *sp;
 	const char *p;
 
 	Fh(tl, 0, "#define VGC_NREFS %u\n", tl->cnt + 1);
 	Fh(tl, 0, "static struct vrt_ref VGC_ref[VGC_NREFS];\n");
 	Fc(tl, 0, "static struct vrt_ref VGC_ref[VGC_NREFS] = {\n");
-	fil = 0;
 	lin = 1;
 	pos = 0;
-	p = vcc_default_vcl_b;
+	sp = 0;
+	p = NULL;
 	TAILQ_FOREACH(t, &tl->tokens, list) {
 		if (t->cnt == 0)
 			continue;
+		assert(t->src != NULL);
+		if (t->src != sp) {
+			lin = 1;
+			pos = 0;
+			sp = t->src;
+			p = sp->b;
+		}
+		assert(sp != NULL);
+		assert(p != NULL);
 		for (;p < t->b; p++) {
-			if (p == vcc_default_vcl_e) {
-				p = tl->b;
-				fil = 1;
-				lin = 1;
-				pos = 0;
-			}
 			if (*p == '\n') {
 				lin++;
 				pos = 0;
@@ -1318,13 +1344,12 @@ LocTable(struct tokenlist *tl)
 				pos++;
 
 		}
-		Fc(tl, 0, "  [%3u] = { %d, %4u, %3u, 0, \"%.*s\" },\n",
-		    t->cnt, fil, lin, pos + 1, PF(t));
+		Fc(tl, 0, "  [%3u] = { %d, %8u, %4u, %3u, 0, \"%.*s\" },\n",
+		    t->cnt, sp->idx, t->b - sp->b, lin, pos + 1, PF(t));
 	}
 	Fc(tl, 0, "};\n");
 }
 
-
 /*--------------------------------------------------------------------*/
 
 static void
@@ -1352,6 +1377,23 @@ EmitFiniFunc(struct tokenlist *tl)
 static void
 EmitStruct(struct tokenlist *tl)
 {
+	struct source *sp;
+
+	Fc(tl, 0, "\nconst char *srcname[%u] = {\n", tl->nsources);
+	TAILQ_FOREACH(sp, &tl->sources, list) {
+		Fc(tl, 0, "\t");
+		EncString(tl->fc, sp->name, NULL);
+		Fc(tl, 0, ",\n");
+	}
+	Fc(tl, 0, "};\n");
+	
+	Fc(tl, 0, "\nconst char *srcbody[%u] = {\n", tl->nsources);
+	TAILQ_FOREACH(sp, &tl->sources, list) {
+		Fc(tl, 0, "\t");
+		EncString(tl->fc, sp->b, sp->e);
+		Fc(tl, 0, ",\n");
+	}
+	Fc(tl, 0, "};\n");
 
 	Fc(tl, 0, "\nstruct VCL_conf VCL_conf = {\n");
 	Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n");
@@ -1360,6 +1402,9 @@ EmitStruct(struct tokenlist *tl)
 	Fc(tl, 0, "\t.nbackend = %d,\n", tl->nbackend);
 	Fc(tl, 0, "\t.ref = VGC_ref,\n");
 	Fc(tl, 0, "\t.nref = VGC_NREFS,\n");
+	Fc(tl, 0, "\t.nsrc = %u,\n", tl->nsources);
+	Fc(tl, 0, "\t.srcname = srcname,\n");
+	Fc(tl, 0, "\t.srcbody = srcbody,\n");
 #define VCL_RET_MAC(l,u,b,n)
 #define VCL_MET_MAC(l,u,b) \
 	if (FindRefStr(tl, "vcl_" #l, R_FUNC)) { \
@@ -1377,10 +1422,112 @@ EmitStruct(struct tokenlist *tl)
 
 /*--------------------------------------------------------------------*/
 
-char *
-VCC_Compile(struct vsb *sb, const char *b, const char *e)
+static struct source *
+vcc_new_source(const char *b, const char *e, const char *name)
+{
+	struct source *sp;
+
+	if (e == NULL)
+		e = strchr(b, '\0');
+	sp = calloc(sizeof *sp, 1);
+	assert(sp != NULL);
+	sp->name = strdup(name);
+	sp->b = b;
+	sp->e = e;
+	return (sp);
+}
+
+static void
+vcc_destroy_source(struct source *sp)
+{
+
+	free(sp->name);	
+	free(sp);
+}
+
+/*--------------------------------------------------------------------*/
+
+static struct source *
+vcc_file_source(struct vsb *sb, const char *fn)
+{
+	char *f;
+	int fd, i;
+	struct stat st;
+
+	fd = open(fn, O_RDONLY);
+	if (fd < 0) {
+		vsb_printf(sb, "Cannot open file '%s': %s\n",
+		    fn, strerror(errno));
+		return (NULL);
+	}
+	assert(0 == fstat(fd, &st));
+	f = malloc(st.st_size + 1);
+	assert(f != NULL);
+	i = read(fd, f, st.st_size);
+	assert(i == st.st_size);
+	close(fd);
+	f[i] = '\0';
+	return (vcc_new_source(f, f + i, fn));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+vcc_resolve_includes(struct tokenlist *tl)
 {
-	struct tokenlist tokens;
+	struct token *t, *t1, *t2;
+	struct source *sp;
+
+	TAILQ_FOREACH(t, &tl->tokens, list) {
+		if (t->tok != T_INCLUDE)
+			continue;
+
+		t1 = TAILQ_NEXT(t, list);
+		assert(t1 != NULL);	/* There's always an EOI */
+		if (t1->tok != CSTR) {
+			vsb_printf(tl->sb,
+			    "include not followed by string constant.\n");
+			vcc_ErrWhere(tl, t1);
+			return;
+		}
+		t2 = TAILQ_NEXT(t1, list);
+		assert(t2 != NULL);	/* There's always an EOI */
+		if (t2->tok != ';') {
+			vsb_printf(tl->sb,
+			    "include <string> not followed by semicolon.\n");
+			vcc_ErrWhere(tl, t1);
+			return;
+		}
+		assert(t2 != NULL);
+
+		sp = vcc_file_source(tl->sb, t1->dec);
+		if (sp == NULL) {
+			vcc_ErrWhere(tl, t1);
+			return;
+		}
+		TAILQ_INSERT_TAIL(&tl->sources, sp, list);
+		sp->idx = tl->nsources++;
+		tl->t = t2;
+		vcc_Lexer(tl, sp);
+
+		TAILQ_REMOVE(&tl->tokens, t, list);
+		TAILQ_REMOVE(&tl->tokens, t1, list);
+		TAILQ_REMOVE(&tl->tokens, t2, list);
+		vcc_FreeToken(t);
+		vcc_FreeToken(t1);
+		vcc_FreeToken(t2);
+		if (!tl->err)
+			vcc_resolve_includes(tl);
+		return;
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static char *
+vcc_CompileSource(struct vsb *sb, struct source *sp)
+{
+	struct tokenlist tokens, *tl;
 	struct ref *r;
 	struct token *t;
 	FILE *fo;
@@ -1389,11 +1536,15 @@ VCC_Compile(struct vsb *sb, const char *b, const char *e)
 	int i;
 
 	memset(&tokens, 0, sizeof tokens);
-	TAILQ_INIT(&tokens.tokens);
-	TAILQ_INIT(&tokens.refs);
-	TAILQ_INIT(&tokens.procs);
+	tl = &tokens;
+	TAILQ_INIT(&tl->tokens);
+	TAILQ_INIT(&tl->refs);
+	TAILQ_INIT(&tl->procs);
+	TAILQ_INIT(&tl->sources);
 	tokens.sb = sb;
 
+	tl->nsources = 0;
+
 	tokens.fc = vsb_new(NULL, NULL, 0, VSB_AUTOEXTEND);
 	assert(tokens.fc != NULL);
 
@@ -1406,38 +1557,52 @@ VCC_Compile(struct vsb *sb, const char *b, const char *e)
 	tokens.ff = vsb_new(NULL, NULL, 0, VSB_AUTOEXTEND);
 	assert(tokens.ff != NULL);
 
-	Fh(&tokens, 0, "extern struct VCL_conf VCL_conf;\n");
+#define VCL_MET_MAC(l,U,m) \
+		tokens.fm_##l = vsb_new(NULL, NULL, 0, VSB_AUTOEXTEND); \
+		assert(tokens.fm_##l != NULL);
+#include "vcl_returns.h"
+#undef VCL_MET_MAC
 
-	Fi(&tokens, 0, "\tVRT_alloc_backends(&VCL_conf);\n");
+	Fh(tl, 0, "extern struct VCL_conf VCL_conf;\n");
 
-	tokens.b = b;
-	if (e == NULL)
-		e = strchr(b, '\0');
-	assert(e != NULL);
-	tokens.e = e;
-	vcc_Lexer(&tokens, vcc_default_vcl_b, vcc_default_vcl_e);
-	vcc_Lexer(&tokens, b, e);
-	vcc_AddToken(&tokens, EOI, e, e);
+	Fi(tl, 0, "\tVRT_alloc_backends(&VCL_conf);\n");
+
+	TAILQ_INSERT_TAIL(&tl->sources, sp, list);
+	sp->idx = tl->nsources++;
+	vcc_Lexer(tl, sp);
 	if (tokens.err)
 		goto done;
-	tokens.t = TAILQ_FIRST(&tokens.tokens);
-	Parse(&tokens);
+
+	sp = vcc_new_source(vcc_default_vcl_b, vcc_default_vcl_e, "Default");
+	TAILQ_INSERT_TAIL(&tl->sources, sp, list);
+	sp->idx = tl->nsources++;
+	vcc_Lexer(tl, sp);
+	vcc_AddToken(tl, EOI, sp->e, sp->e);
 	if (tokens.err)
 		goto done;
-	Consistency(&tokens);
+
+	vcc_resolve_includes(tl);
 	if (tokens.err)
 		goto done;
-	LocTable(&tokens);
 
-	Ff(&tokens, 0, "\tVRT_free_backends(&VCL_conf);\n");
+	tokens.t = TAILQ_FIRST(&tl->tokens);
+	Parse(tl);
+	if (tokens.err)
+		goto done;
+	Consistency(tl);
+	if (tokens.err)
+		goto done;
+	LocTable(tl);
 
-	EmitInitFunc(&tokens);
+	Ff(tl, 0, "\tVRT_free_backends(&VCL_conf);\n");
 
-	EmitFiniFunc(&tokens);
+	EmitInitFunc(tl);
 
-	EmitStruct(&tokens);
+	EmitFiniFunc(tl);
 
-	if (CheckRefs(&tokens))
+	EmitStruct(tl);
+
+	if (CheckRefs(tl))
 		goto done;
 
 	of = strdup("/tmp/vcl.XXXXXXXX");
@@ -1472,45 +1637,55 @@ VCC_Compile(struct vsb *sb, const char *b, const char *e)
 	}
 done:
 
+#define VCL_MET_MAC(l,U,m) vsb_delete(tokens.fm_##l);
+#include "vcl_returns.h"
+#undef VCL_MET_MAC
+
 	/* Free References */
-	while (!TAILQ_EMPTY(&tokens.refs)) {
-		r = TAILQ_FIRST(&tokens.refs);
-		TAILQ_REMOVE(&tokens.refs, r, list);
+	while (!TAILQ_EMPTY(&tl->refs)) {
+		r = TAILQ_FIRST(&tl->refs);
+		TAILQ_REMOVE(&tl->refs, r, list);
 		free(r);
 	}
 
 	/* Free Tokens */
-	while (!TAILQ_EMPTY(&tokens.tokens)) {
-		t = TAILQ_FIRST(&tokens.tokens);
-		TAILQ_REMOVE(&tokens.tokens, t, list);
-		free(t);
+	while (!TAILQ_EMPTY(&tl->tokens)) {
+		t = TAILQ_FIRST(&tl->tokens);
+		TAILQ_REMOVE(&tl->tokens, t, list);
+		vcc_FreeToken(t);
 	}
 	return (of);
 }
 
 /*--------------------------------------------------------------------*/
 
+char *
+VCC_Compile(struct vsb *sb, const char *b, const char *e)
+{
+	struct source *sp;
+	char *r;
+
+	sp = vcc_new_source(b, e, "input");
+	if (sp == NULL)
+		return (NULL);
+	r = vcc_CompileSource(sb, sp);
+	vcc_destroy_source(sp);
+	return (r);
+}
+
+/*--------------------------------------------------------------------*/
+
 char *
 VCC_CompileFile(struct vsb *sb, const char *fn)
 {
-	char *f, *r;
-	int fd, i;
-	struct stat st;
+	struct source *sp;
+	char *r;
 
-	fd = open(fn, O_RDONLY);
-	if (fd < 0) {
-		vsb_printf(sb, "Cannot open file '%s': %s",
-		    fn, strerror(errno));
+	sp = vcc_file_source(sb, fn);
+	if (sp == NULL)
 		return (NULL);
-	}
-	assert(0 == fstat(fd, &st));
-	f = malloc(st.st_size + 1);
-	assert(f != NULL);
-	i = read(fd, f, st.st_size);
-	assert(i == st.st_size);
-	f[i] = '\0';
-	r = VCC_Compile(sb, f, NULL);
-	free(f);
+	r = vcc_CompileSource(sb, sp);
+	vcc_destroy_source(sp);
 	return (r);
 }
 

lib/libvcl/vcc_compile.h

@@ -34,23 +34,38 @@
 
 #define INDENT		2
 
+struct source {
+	TAILQ_ENTRY(source)	list;
+	char			*name;
+	const char		*b;
+	const char		*e;
+	unsigned		idx;
+};
+
 struct token {
 	unsigned		tok;
 	const char		*b;
 	const char		*e;
+	struct source		*src;
 	TAILQ_ENTRY(token)	list;
 	unsigned		cnt;
 	char			*dec;
 };
 
+TAILQ_HEAD(tokenhead, token);
+
 struct tokenlist {
-	TAILQ_HEAD(, token)	tokens;
-	const char		*b;
-	const char		*e;
+	struct tokenhead	tokens;
+	TAILQ_HEAD(, source)	sources;
+	unsigned		nsources;
+	struct source		*src;
 	struct token		*t;
 	int			indent;
 	unsigned		cnt;
 	struct vsb		*fc, *fh, *fi, *ff;
+#define VCL_MET_MAC(l,U,m) struct vsb *fm_##l;
+#include "vcl_returns.h"
+#undef VCL_MET_MAC
 	TAILQ_HEAD(, ref)	refs;
 	struct vsb		*sb;
 	int			err;
@@ -138,7 +153,8 @@ void Ff(struct tokenlist *tl, int indent, const char *fmt, ...);
 unsigned UintVal(struct tokenlist *tl);
 void AddDef(struct tokenlist *tl, struct token *t, enum ref_type type);
 void AddRef(struct tokenlist *tl, struct token *t, enum ref_type type);
-void EncString(struct vsb *sb, struct token *t);
+void EncToken(struct vsb *sb, struct token *t);
+void EncString(struct vsb *sb, const char *b, const char *e);
 
 
 /* vcc_obj.c */
@@ -153,10 +169,11 @@ void vcc_ErrWhere(struct tokenlist *tl, struct token *t);
 void vcc__Expect(struct tokenlist *tl, unsigned tok, int line);
 int vcc_Teq(struct token *t1, struct token *t2);
 int vcc_IdIs(struct token *t, const char *p);
-void vcc_Lexer(struct tokenlist *tl, const char *b, const char *e);
+void vcc_Lexer(struct tokenlist *tl, struct source *sp);
 void vcc_NextToken(struct tokenlist *tl);
 void vcc__ErrInternal(struct tokenlist *tl, const char *func, unsigned line);
 void vcc_AddToken(struct tokenlist *tl, unsigned tok, const char *b, const char *e);
+void vcc_FreeToken(struct token *t);
 
 #define ERRCHK(tl)      do { if ((tl)->err) return; } while (0)
 #define ErrInternal(tl) vcc__ErrInternal(tl, __func__, __LINE__)

lib/libvcl/vcc_fixed_token.c

@@ -242,6 +242,12 @@ vcl_fixed_token(const char *p, const char **q)
 			*q = p + 6;
 			return (T_INSERT);
 		}
+		if (p[0] == 'i' && p[1] == 'n' && p[2] == 'c' && 
+		    p[3] == 'l' && p[4] == 'u' && p[5] == 'd' && 
+		    p[6] == 'e' && !isvar(p[7])) {
+			*q = p + 7;
+			return (T_INCLUDE);
+		}
 		if (p[0] == 'i' && p[1] == 'f' && !isvar(p[2])) {
 			*q = p + 2;
 			return (T_IF);
@@ -399,6 +405,7 @@ vcl_init_tnames(void)
 	vcl_tnames[T_HASH] = "hash";
 	vcl_tnames[T_IF] = "if";
 	vcl_tnames[T_INC] = "++";
+	vcl_tnames[T_INCLUDE] = "include";
 	vcl_tnames[T_INCR] = "+=";
 	vcl_tnames[T_INSERT] = "insert";
 	vcl_tnames[T_LEQ] = "<=";
@@ -455,6 +462,10 @@ vcl_output_lang_h(FILE *f)
 	fputs("        unsigned        nref;\n", f);
 	fputs("        unsigned        busy;\n", f);
 	fputs("\n", f);
+	fputs("	unsigned	nsrc;\n", f);
+	fputs("	const char	**srcname;\n", f);
+	fputs("	const char	**srcbody;\n", f);
+	fputs("\n", f);
 	fputs("        void            *priv;\n", f);
 	fputs("\n", f);
 	fputs("        vcl_init_f      *init_func;\n", f);
@@ -511,7 +522,8 @@ vcl_output_lang_h(FILE *f)
 	fputs("struct VCL_conf;\n", f);
 	fputs("\n", f);
 	fputs("struct vrt_ref {\n", f);
-	fputs("	unsigned	file;\n", f);
+	fputs("	unsigned	source;\n", f);
+	fputs("	unsigned	offset;\n", f);
 	fputs("	unsigned	line;\n", f);
 	fputs("	unsigned	pos;\n", f);
 	fputs("	unsigned	count;\n", f);

lib/libvcl/vcc_gen_fixed_token.tcl

@@ -61,6 +61,8 @@ set returns {
 # Language keywords
 #
 set keywords {
+	include 
+
 	if else elseif elsif
 
 	func proc sub
@@ -140,6 +142,10 @@ puts $fo {	unsigned        magic;
         unsigned        nref;
         unsigned        busy;
 
+	unsigned	nsrc;
+	const char	**srcname;
+	const char	**srcbody;
+
         void            *priv;
 
         vcl_init_f      *init_func;

lib/libvcl/vcc_token.c

@@ -70,20 +70,16 @@ vcc_ErrWhere(struct tokenlist *tl, struct token *t)
 {
 	unsigned lin, pos, x, y;
 	const char *p, *l, *f, *b, *e;
+	struct source *sp;
 
 	lin = 1;
 	pos = 0;
 	if (t->tok == METHOD)
 		return;
-	if (t->b >= vcc_default_vcl_b && t->b < vcc_default_vcl_e) {
-		f = "Default VCL code (compiled in)";
-		b = vcc_default_vcl_b;
-		e = vcc_default_vcl_e;
-	} else {
-		f = "VCL code";
-		b = tl->b;
-		e = tl->e;
-	}
+	sp = t->src;
+	f = sp->name;
+	b = sp->b;
+	e = sp->e;
 	for (l = p = b; p < t->b; p++) {
 		if (*p == '\n') {
 			lin++;
@@ -266,27 +262,44 @@ vcc_AddToken(struct tokenlist *tl, unsigned tok, const char *b, const char *e)
 	t->tok = tok;
 	t->b = b;
 	t->e = e;
-	TAILQ_INSERT_TAIL(&tl->tokens, t, list);
+	t->src = tl->src;
+	if (tl->t != NULL)
+		TAILQ_INSERT_AFTER(&tl->tokens, tl->t, t, list);
+	else
+		TAILQ_INSERT_TAIL(&tl->tokens, t, list);
 	tl->t = t;
 	if (0) {
 		fprintf(stderr, "[%s %.*s] ",
-		    vcl_tnames[tok],(int)(e - b), b);
+		    vcl_tnames[tok], PF(t));
 		if (tok == EOI)
 			fprintf(stderr, "\n");
 	}
 }
 
+/*--------------------------------------------------------------------
+ * Free a token
+ */
+
+void
+vcc_FreeToken(struct token *t)
+{
+
+	/* XXX: more */
+	free(t);
+}
+
 /*--------------------------------------------------------------------
  * Lexical analysis and token generation
  */
 
 void
-vcc_Lexer(struct tokenlist *tl, const char *b, const char *e)
+vcc_Lexer(struct tokenlist *tl, struct source *sp)
 {
 	const char *p, *q;
 	unsigned u;
 
-	for (p = b; p < e; ) {
+	tl->src = sp;
+	for (p = sp->b; p < sp->e; ) {
 
 		/* Skip any whitespace */
 		if (isspace(*p)) {
@@ -296,7 +309,7 @@ vcc_Lexer(struct tokenlist *tl, const char *b, const char *e)
 
 		/* Skip '#.*\n' comments */
 		if (*p == '#') {
-			while (p < e && *p != '\n')
+			while (p < sp->e && *p != '\n')
 				p++;
 			continue;
 		}
@@ -304,7 +317,7 @@ vcc_Lexer(struct tokenlist *tl, const char *b, const char *e)
 		/* Skip C-style comments */
 		if (*p == '/' && p[1] == '*') {
 			p += 2;
-			for (p += 2; p < e; p++) {
+			for (p += 2; p < sp->e; p++) {
 				if (*p == '*' && p[1] == '/') {
 					p += 2;
 					break;
@@ -315,7 +328,7 @@ vcc_Lexer(struct tokenlist *tl, const char *b, const char *e)
 
 		/* Skip C++-style comments */
 		if (*p == '/' && p[1] == '/') {
-			while (p < e && *p != '\n')
+			while (p < sp->e && *p != '\n')
 				p++;
 			continue;
 		}
@@ -330,7 +343,7 @@ vcc_Lexer(struct tokenlist *tl, const char *b, const char *e)
 
 		/* Match strings, with \\ and \" escapes */
 		if (*p == '"') {
-			for (q = p + 1; q < e; q++) {
+			for (q = p + 1; q < sp->e; q++) {
 				if (*q == '"') {
 					q++;
 					break;
@@ -352,11 +365,11 @@ vcc_Lexer(struct tokenlist *tl, const char *b, const char *e)
 
 		/* Match Identifiers */
 		if (isident1(*p)) {
-			for (q = p; q < e; q++)
+			for (q = p; q < sp->e; q++)
 				if (!isident(*q))
 					break;
 			if (isvar(*q)) {
-				for (; q < e; q++)
+				for (; q < sp->e; q++)
 					if (!isvar(*q))
 						break;
 				vcc_AddToken(tl, VAR, p, q);
@@ -369,7 +382,7 @@ vcc_Lexer(struct tokenlist *tl, const char *b, const char *e)
 
 		/* Match numbers { [0-9]+ } */
 		if (isdigit(*p)) {
-			for (q = p; q < e; q++)
+			for (q = p; q < sp->e; q++)
 				if (!isdigit(*q))
 					break;
 			vcc_AddToken(tl, CNUM, p, q);

lib/libvcl/vcc_token_defs.h

@@ -7,47 +7,48 @@
  */
 
 #define LOW_TOKEN 128
-#define T_IF 128
-#define T_ELSE 129
-#define T_ELSEIF 130
-#define T_ELSIF 131
-#define T_FUNC 132
-#define T_PROC 133
-#define T_SUB 134
-#define T_ACL 135
-#define T_BACKEND 136
-#define T_CALL 137
-#define T_NO_CACHE 138
-#define T_NO_NEW_CACHE 139
-#define T_SET 140
-#define T_REWRITE 141
-#define T_SWITCH_CONFIG 142
-#define T_ERROR 143
-#define T_LOOKUP 144
-#define T_HASH 145
-#define T_PIPE 146
-#define T_PASS 147
-#define T_FETCH 148
-#define T_INSERT 149
-#define T_DELIVER 150
-#define T_DISCARD 151
-#define T_INC 152
-#define T_DEC 153
-#define T_CAND 154
-#define T_COR 155
-#define T_LEQ 156
-#define T_EQ 157
-#define T_NEQ 158
-#define T_GEQ 159
-#define T_SHR 160
-#define T_SHL 161
-#define T_INCR 162
-#define T_DECR 163
-#define T_MUL 164
-#define T_DIV 165
-#define ID 166
-#define VAR 167
-#define CNUM 168
-#define CSTR 169
-#define EOI 170
-#define METHOD 171
+#define T_INCLUDE 128
+#define T_IF 129
+#define T_ELSE 130
+#define T_ELSEIF 131
+#define T_ELSIF 132
+#define T_FUNC 133
+#define T_PROC 134
+#define T_SUB 135
+#define T_ACL 136
+#define T_BACKEND 137
+#define T_CALL 138
+#define T_NO_CACHE 139
+#define T_NO_NEW_CACHE 140
+#define T_SET 141
+#define T_REWRITE 142
+#define T_SWITCH_CONFIG 143
+#define T_ERROR 144
+#define T_LOOKUP 145
+#define T_HASH 146
+#define T_PIPE 147
+#define T_PASS 148
+#define T_FETCH 149
+#define T_INSERT 150
+#define T_DELIVER 151
+#define T_DISCARD 152
+#define T_INC 153
+#define T_DEC 154
+#define T_CAND 155
+#define T_COR 156
+#define T_LEQ 157
+#define T_EQ 158
+#define T_NEQ 159
+#define T_GEQ 160
+#define T_SHR 161
+#define T_SHL 162
+#define T_INCR 163
+#define T_DECR 164
+#define T_MUL 165
+#define T_DIV 166
+#define ID 167
+#define VAR 168
+#define CNUM 169
+#define CSTR 170
+#define EOI 171
+#define METHOD 172