Implement a facility for source file modularization in the VCL

compiler.  The syntax is:

	include "filename" ;

Unlike the C preprocessors #include directive, a VCL include can
appear anywhere in the sourcefile:

	if {req.Cookie == include "cookie.vcl" ; || !req.Host } {
	}

and have cookie.vcl contain just:

	"8435398475983275293759843"


Technically this results in a change to how we account for source
code references in the counter/profile table as well, and as a result
the entire source code of the VCL program is now compiled into the
shared library for easy reference.



git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@1281 d4fa192b-c00b-0410-8231-f00ffab90ce4

include/vcl.h

@@ -22,6 +22,10 @@ struct VCL_conf {
         unsigned        nref;
         unsigned        busy;
 
+	unsigned	nsrc;
+	const char	**srcname;
+	const char	**srcbody;
+
         void            *priv;
 
         vcl_init_f      *init_func;

include/vrt.h

@@ -40,7 +40,8 @@ struct backend;
 struct VCL_conf;
 
 struct vrt_ref {
-	unsigned	file;
+	unsigned	source;
+	unsigned	offset;
 	unsigned	line;
 	unsigned	pos;
 	unsigned	count;

lib/libvcl/vcc_acl.c

@@ -120,14 +120,14 @@ vcc_Acl(struct tokenlist *tl)
 			mask = UintVal(tl);
 		}
 		Fc(tl, 1, "{ %u, %u, %u, ", not, mask, para);
-		EncString(tl->fc, t);
+		EncToken(tl->fc, t);
 		Fc(tl, 0, ", \"");
 		if (para)
 			Fc(tl, 0, "(");
 		if (not)
 			Fc(tl, 0, "!");
 		Fc(tl, 0, "\\\"\" ");
-		EncString(tl->fc, t);
+		EncToken(tl->fc, t);
 		Fc(tl, 0, " \"\\\"");
 		if (mask)
 			Fc(tl, 0, "/%u", mask);

lib/libvcl/vcc_compile.h

@@ -34,23 +34,38 @@
 
 #define INDENT		2
 
+struct source {
+	TAILQ_ENTRY(source)	list;
+	char			*name;
+	const char		*b;
+	const char		*e;
+	unsigned		idx;
+};
+
 struct token {
 	unsigned		tok;
 	const char		*b;
 	const char		*e;
+	struct source		*src;
 	TAILQ_ENTRY(token)	list;
 	unsigned		cnt;
 	char			*dec;
 };
 
+TAILQ_HEAD(tokenhead, token);
+
 struct tokenlist {
-	TAILQ_HEAD(, token)	tokens;
-	const char		*b;
-	const char		*e;
+	struct tokenhead	tokens;
+	TAILQ_HEAD(, source)	sources;
+	unsigned		nsources;
+	struct source		*src;
 	struct token		*t;
 	int			indent;
 	unsigned		cnt;
 	struct vsb		*fc, *fh, *fi, *ff;
+#define VCL_MET_MAC(l,U,m) struct vsb *fm_##l;
+#include "vcl_returns.h"
+#undef VCL_MET_MAC
 	TAILQ_HEAD(, ref)	refs;
 	struct vsb		*sb;
 	int			err;
@@ -138,7 +153,8 @@ void Ff(struct tokenlist *tl, int indent, const char *fmt, ...);
 unsigned UintVal(struct tokenlist *tl);
 void AddDef(struct tokenlist *tl, struct token *t, enum ref_type type);
 void AddRef(struct tokenlist *tl, struct token *t, enum ref_type type);
-void EncString(struct vsb *sb, struct token *t);
+void EncToken(struct vsb *sb, struct token *t);
+void EncString(struct vsb *sb, const char *b, const char *e);
 
 
 /* vcc_obj.c */
@@ -153,10 +169,11 @@ void vcc_ErrWhere(struct tokenlist *tl, struct token *t);
 void vcc__Expect(struct tokenlist *tl, unsigned tok, int line);
 int vcc_Teq(struct token *t1, struct token *t2);
 int vcc_IdIs(struct token *t, const char *p);
-void vcc_Lexer(struct tokenlist *tl, const char *b, const char *e);
+void vcc_Lexer(struct tokenlist *tl, struct source *sp);
 void vcc_NextToken(struct tokenlist *tl);
 void vcc__ErrInternal(struct tokenlist *tl, const char *func, unsigned line);
 void vcc_AddToken(struct tokenlist *tl, unsigned tok, const char *b, const char *e);
+void vcc_FreeToken(struct token *t);
 
 #define ERRCHK(tl)      do { if ((tl)->err) return; } while (0)
 #define ErrInternal(tl) vcc__ErrInternal(tl, __func__, __LINE__)

lib/libvcl/vcc_fixed_token.c

@@ -242,6 +242,12 @@ vcl_fixed_token(const char *p, const char **q)
 			*q = p + 6;
 			return (T_INSERT);
 		}
+		if (p[0] == 'i' && p[1] == 'n' && p[2] == 'c' && 
+		    p[3] == 'l' && p[4] == 'u' && p[5] == 'd' && 
+		    p[6] == 'e' && !isvar(p[7])) {
+			*q = p + 7;
+			return (T_INCLUDE);
+		}
 		if (p[0] == 'i' && p[1] == 'f' && !isvar(p[2])) {
 			*q = p + 2;
 			return (T_IF);
@@ -399,6 +405,7 @@ vcl_init_tnames(void)
 	vcl_tnames[T_HASH] = "hash";
 	vcl_tnames[T_IF] = "if";
 	vcl_tnames[T_INC] = "++";
+	vcl_tnames[T_INCLUDE] = "include";
 	vcl_tnames[T_INCR] = "+=";
 	vcl_tnames[T_INSERT] = "insert";
 	vcl_tnames[T_LEQ] = "<=";
@@ -455,6 +462,10 @@ vcl_output_lang_h(FILE *f)
 	fputs("        unsigned        nref;\n", f);
 	fputs("        unsigned        busy;\n", f);
 	fputs("\n", f);
+	fputs("	unsigned	nsrc;\n", f);
+	fputs("	const char	**srcname;\n", f);
+	fputs("	const char	**srcbody;\n", f);
+	fputs("\n", f);
 	fputs("        void            *priv;\n", f);
 	fputs("\n", f);
 	fputs("        vcl_init_f      *init_func;\n", f);
@@ -511,7 +522,8 @@ vcl_output_lang_h(FILE *f)
 	fputs("struct VCL_conf;\n", f);
 	fputs("\n", f);
 	fputs("struct vrt_ref {\n", f);
-	fputs("	unsigned	file;\n", f);
+	fputs("	unsigned	source;\n", f);
+	fputs("	unsigned	offset;\n", f);
 	fputs("	unsigned	line;\n", f);
 	fputs("	unsigned	pos;\n", f);
 	fputs("	unsigned	count;\n", f);

lib/libvcl/vcc_gen_fixed_token.tcl

@@ -61,6 +61,8 @@ set returns {
 # Language keywords
 #
 set keywords {
+	include 
+
 	if else elseif elsif
 
 	func proc sub
@@ -140,6 +142,10 @@ puts $fo {	unsigned        magic;
         unsigned        nref;
         unsigned        busy;
 
+	unsigned	nsrc;
+	const char	**srcname;
+	const char	**srcbody;
+
         void            *priv;
 
         vcl_init_f      *init_func;

lib/libvcl/vcc_token.c

@@ -70,20 +70,16 @@ vcc_ErrWhere(struct tokenlist *tl, struct token *t)
 {
 	unsigned lin, pos, x, y;
 	const char *p, *l, *f, *b, *e;
+	struct source *sp;
 
 	lin = 1;
 	pos = 0;
 	if (t->tok == METHOD)
 		return;
-	if (t->b >= vcc_default_vcl_b && t->b < vcc_default_vcl_e) {
-		f = "Default VCL code (compiled in)";
-		b = vcc_default_vcl_b;
-		e = vcc_default_vcl_e;
-	} else {
-		f = "VCL code";
-		b = tl->b;
-		e = tl->e;
-	}
+	sp = t->src;
+	f = sp->name;
+	b = sp->b;
+	e = sp->e;
 	for (l = p = b; p < t->b; p++) {
 		if (*p == '\n') {
 			lin++;
@@ -266,27 +262,44 @@ vcc_AddToken(struct tokenlist *tl, unsigned tok, const char *b, const char *e)
 	t->tok = tok;
 	t->b = b;
 	t->e = e;
-	TAILQ_INSERT_TAIL(&tl->tokens, t, list);
+	t->src = tl->src;
+	if (tl->t != NULL)
+		TAILQ_INSERT_AFTER(&tl->tokens, tl->t, t, list);
+	else
+		TAILQ_INSERT_TAIL(&tl->tokens, t, list);
 	tl->t = t;
 	if (0) {
 		fprintf(stderr, "[%s %.*s] ",
-		    vcl_tnames[tok],(int)(e - b), b);
+		    vcl_tnames[tok], PF(t));
 		if (tok == EOI)
 			fprintf(stderr, "\n");
 	}
 }
 
+/*--------------------------------------------------------------------
+ * Free a token
+ */
+
+void
+vcc_FreeToken(struct token *t)
+{
+
+	/* XXX: more */
+	free(t);
+}
+
 /*--------------------------------------------------------------------
  * Lexical analysis and token generation
  */
 
 void
-vcc_Lexer(struct tokenlist *tl, const char *b, const char *e)
+vcc_Lexer(struct tokenlist *tl, struct source *sp)
 {
 	const char *p, *q;
 	unsigned u;
 
-	for (p = b; p < e; ) {
+	tl->src = sp;
+	for (p = sp->b; p < sp->e; ) {
 
 		/* Skip any whitespace */
 		if (isspace(*p)) {
@@ -296,7 +309,7 @@ vcc_Lexer(struct tokenlist *tl, const char *b, const char *e)
 
 		/* Skip '#.*\n' comments */
 		if (*p == '#') {
-			while (p < e && *p != '\n')
+			while (p < sp->e && *p != '\n')
 				p++;
 			continue;
 		}
@@ -304,7 +317,7 @@ vcc_Lexer(struct tokenlist *tl, const char *b, const char *e)
 		/* Skip C-style comments */
 		if (*p == '/' && p[1] == '*') {
 			p += 2;
-			for (p += 2; p < e; p++) {
+			for (p += 2; p < sp->e; p++) {
 				if (*p == '*' && p[1] == '/') {
 					p += 2;
 					break;
@@ -315,7 +328,7 @@ vcc_Lexer(struct tokenlist *tl, const char *b, const char *e)
 
 		/* Skip C++-style comments */
 		if (*p == '/' && p[1] == '/') {
-			while (p < e && *p != '\n')
+			while (p < sp->e && *p != '\n')
 				p++;
 			continue;
 		}
@@ -330,7 +343,7 @@ vcc_Lexer(struct tokenlist *tl, const char *b, const char *e)
 
 		/* Match strings, with \\ and \" escapes */
 		if (*p == '"') {
-			for (q = p + 1; q < e; q++) {
+			for (q = p + 1; q < sp->e; q++) {
 				if (*q == '"') {
 					q++;
 					break;
@@ -352,11 +365,11 @@ vcc_Lexer(struct tokenlist *tl, const char *b, const char *e)
 
 		/* Match Identifiers */
 		if (isident1(*p)) {
-			for (q = p; q < e; q++)
+			for (q = p; q < sp->e; q++)
 				if (!isident(*q))
 					break;
 			if (isvar(*q)) {
-				for (; q < e; q++)
+				for (; q < sp->e; q++)
 					if (!isvar(*q))
 						break;
 				vcc_AddToken(tl, VAR, p, q);
@@ -369,7 +382,7 @@ vcc_Lexer(struct tokenlist *tl, const char *b, const char *e)
 
 		/* Match numbers { [0-9]+ } */
 		if (isdigit(*p)) {
-			for (q = p; q < e; q++)
+			for (q = p; q < sp->e; q++)
 				if (!isdigit(*q))
 					break;
 			vcc_AddToken(tl, CNUM, p, q);

lib/libvcl/vcc_token_defs.h

@@ -7,47 +7,48 @@
  */
 
 #define LOW_TOKEN 128
-#define T_IF 128
-#define T_ELSE 129
-#define T_ELSEIF 130
-#define T_ELSIF 131
-#define T_FUNC 132
-#define T_PROC 133
-#define T_SUB 134
-#define T_ACL 135
-#define T_BACKEND 136
-#define T_CALL 137
-#define T_NO_CACHE 138
-#define T_NO_NEW_CACHE 139
-#define T_SET 140
-#define T_REWRITE 141
-#define T_SWITCH_CONFIG 142
-#define T_ERROR 143
-#define T_LOOKUP 144
-#define T_HASH 145
-#define T_PIPE 146
-#define T_PASS 147
-#define T_FETCH 148
-#define T_INSERT 149
-#define T_DELIVER 150
-#define T_DISCARD 151
-#define T_INC 152
-#define T_DEC 153
-#define T_CAND 154
-#define T_COR 155
-#define T_LEQ 156
-#define T_EQ 157
-#define T_NEQ 158
-#define T_GEQ 159
-#define T_SHR 160
-#define T_SHL 161
-#define T_INCR 162
-#define T_DECR 163
-#define T_MUL 164
-#define T_DIV 165
-#define ID 166
-#define VAR 167
-#define CNUM 168
-#define CSTR 169
-#define EOI 170
-#define METHOD 171
+#define T_INCLUDE 128
+#define T_IF 129
+#define T_ELSE 130
+#define T_ELSEIF 131
+#define T_ELSIF 132
+#define T_FUNC 133
+#define T_PROC 134
+#define T_SUB 135
+#define T_ACL 136
+#define T_BACKEND 137
+#define T_CALL 138
+#define T_NO_CACHE 139
+#define T_NO_NEW_CACHE 140
+#define T_SET 141
+#define T_REWRITE 142
+#define T_SWITCH_CONFIG 143
+#define T_ERROR 144
+#define T_LOOKUP 145
+#define T_HASH 146
+#define T_PIPE 147
+#define T_PASS 148
+#define T_FETCH 149
+#define T_INSERT 150
+#define T_DELIVER 151
+#define T_DISCARD 152
+#define T_INC 153
+#define T_DEC 154
+#define T_CAND 155
+#define T_COR 156
+#define T_LEQ 157
+#define T_EQ 158
+#define T_NEQ 159
+#define T_GEQ 160
+#define T_SHR 161
+#define T_SHL 162
+#define T_INCR 163
+#define T_DECR 164
+#define T_MUL 165
+#define T_DIV 166
+#define ID 167
+#define VAR 168
+#define CNUM 169
+#define CSTR 170
+#define EOI 171
+#define METHOD 172