The new string concatenation syntax is STRING + STRING + STRING [...]

For now we assemble the string in the worker threads workspace.

I will reintroduce the optimization to pass it to VRT uncollected
where appropriate, once I get the new expression system pushed through.

Add a hacked up "regsub" function call for now, once functions gets
introduced it should be treated like any other function.

Add "true", "false" BOOL constants, REAL constants

Move the "set" statement over to the new expressions.

Fix testcases to use new string concat syntax where applicable.

"set req.url = 1;" was impossible before, now it works.


git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@5131 d4fa192b-c00b-0410-8231-f00ffab90ce4

bin/varnishd/cache_vrt.c

@@ -143,13 +143,13 @@ VRT_GetHdr(const struct sess *sp, enum gethdr_e where, const char *n)
 
 /*lint -e{818} ap,hp could be const */
 static char *
-vrt_assemble_string(struct http *hp, const char *h, const char *p, va_list ap)
+vrt_build_string(struct ws *ws, const char *h, const char *p, va_list ap)
 {
 	char *b, *e;
 	unsigned u, x;
 
-	u = WS_Reserve(hp->ws, 0);
-	e = b = hp->ws->f;
+	u = WS_Reserve(ws, 0);
+	e = b = ws->f;
 	e += u;
 	if (h != NULL) {
 		x = strlen(h);
@@ -173,16 +173,45 @@ vrt_assemble_string(struct http *hp, const char *h, const char *p, va_list ap)
 		*b = '\0';
 	b++;
 	if (b > e) {
-		WS_Release(hp->ws, 0);
+		WS_Release(ws, 0);
 		return (NULL);
 	} else {
 		e = b;
-		b = hp->ws->f;
-		WS_Release(hp->ws, e - b);
+		b = ws->f;
+		WS_Release(ws, e - b);
 		return (b);
 	}
 }
 
+/*--------------------------------------------------------------------
+ * XXX: Optimize the single element case ?
+ */
+
+/*lint -e{818} ap,hp could be const */
+static char *
+vrt_assemble_string(struct http *hp, const char *h, const char *p, va_list ap)
+{
+
+	return (vrt_build_string(hp->ws, h, p, ap));
+}
+
+/*--------------------------------------------------------------------
+ * Build a string on the worker threads workspace
+ */
+
+const char *
+VRT_String(const struct sess *sp, const char *p, ...)
+{
+	va_list ap;
+	char *b;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	va_start(ap, p);
+	b = vrt_build_string(sp->wrk->ws, NULL, p, ap);
+	va_end(ap);
+	return (b);
+}
+
 /*--------------------------------------------------------------------*/
 
 void
@@ -892,12 +921,14 @@ VRT_time_string(const struct sess *sp, double t)
 }
 
 const char *
-VRT_backend_string(struct sess *sp)
+VRT_backend_string(struct sess *sp, const struct director *d)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	if (sp->director == NULL)
+	if (d == NULL)
+		d = sp->director;
+	if (d == NULL)
 		return (NULL);
-	return (sp->director->vcl_name);
+	return (d->vcl_name);
 }
 
 /*--------------------------------------------------------------------*/

bin/varnishd/default.vcl

@@ -43,7 +43,7 @@ sub vcl_recv {
     if (req.restarts == 0) {
 	if (req.http.x-forwarded-for) {
 	    set req.http.X-Forwarded-For =
-		req.http.X-Forwarded-For ", " client.ip;
+		req.http.X-Forwarded-For + ", " + client.ip;
 	} else {
 	    set req.http.X-Forwarded-For = client.ip;
 	}

bin/varnishtest/tests/r00251.vtc

@@ -13,8 +13,8 @@ server s1 {
 varnish v1 -vcl+backend { 
 	sub vcl_fetch {
 		set beresp.http.Snafu1 =
-		    "zoom"
-		    regsub(beresp.http.Foomble, "ar", "\0\0")
+		    "zoom" +
+		    regsub(beresp.http.Foomble, "ar", "\0\0") +
 		    "box";
 	}
 } -start 

bin/varnishtest/tests/r00693.vtc

@@ -24,37 +24,37 @@ varnish v1 -arg "-p sess_workspace=1024" -vcl+backend {
 
 	sub vcl_recv {
 		set req.http.foo = 
-			req.http.bar
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" "0123456789abcdef"
-			"0123456789abcdef" 
+			req.http.bar +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" +
 			"01234567";
 		set req.http.baz = "BAZ";
 		return (pass);

bin/varnishtest/tests/v00001.vtc

@@ -13,8 +13,8 @@ server s1 -start
 varnish v1 -vcl+backend {
 	sub vcl_recv {
 		set req.http.foobar =
-		    req.url
-		    req.request
+		    req.url +
+		    req.request +
 		    req.proto;
 		set req.url = "/";
 		set req.proto = "HTTP/1.2";
@@ -22,7 +22,7 @@ varnish v1 -vcl+backend {
 	}
 	sub vcl_miss {
 		set bereq.http.foobar =
-		    bereq.url
+		    bereq.url +
 		    bereq.proto;
 		set bereq.url = "/";
 		set bereq.proto = "HTTP/1.2";
@@ -30,7 +30,7 @@ varnish v1 -vcl+backend {
 	}
 	sub vcl_fetch {
 		set beresp.http.foobar =
-		    beresp.proto beresp.response beresp.status;
+		    beresp.proto + beresp.response + beresp.status;
 		set beresp.proto = "HTTP/1.2";
 		set beresp.response = "For circular files";
 		set beresp.status = 903;
@@ -42,9 +42,7 @@ varnish v1 -vcl+backend {
 		# XXX should be moved to it's own test
 		set resp.http.x-served-by-hostname = server.hostname;
 		set resp.http.x-served-by-identity = server.identity;
-		set resp.http.foobar =
-		    resp.proto
-		    resp.status;
+		set resp.http.foobar = resp.proto + resp.status;
 	}
 } -start
 

bin/varnishtest/tests/v00018.vtc

@@ -56,7 +56,7 @@ varnish v1 -badvcl {
 	sub vcl_hash { hash_data(req.hash); }
 }
 
-varnish v1 -badvcl {
+varnish v1 -vcl {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_recv { set req.url = 1; }
 }

include/vrt.h

@@ -189,10 +189,12 @@ char *VRT_IP_string(const struct sess *sp, const struct sockaddr *sa);
 char *VRT_int_string(const struct sess *sp, int);
 char *VRT_double_string(const struct sess *sp, double);
 char *VRT_time_string(const struct sess *sp, double);
-const char *VRT_backend_string(struct sess *sp);
+const char *VRT_backend_string(struct sess *sp, const struct director *d);
 
 #define VRT_done(sp, hand)			\
 	do {					\
 		VRT_handling(sp, hand);		\
 		return (1);			\
 	} while (0)
+
+const char *VRT_String(const struct sess *sp, const char *p, ...);

lib/libvcl/vcc_action.c

@@ -101,126 +101,64 @@ parse_error(struct vcc *tl)
 
 /*--------------------------------------------------------------------*/
 
-#if 1
-static void
-illegal_assignment(const struct vcc *tl, const char *type)
-{
-
-	vsb_printf(tl->sb, "Invalid assignment operator ");
-	vcc_ErrToken(tl, tl->t);
-	vsb_printf(tl->sb,
-	    " only '=' is legal for %s\n", type);
-}
-#endif
+static const struct arith {
+	enum var_type		type;
+	unsigned		oper;
+	enum var_type		want;
+} arith[] = {
+	{ INT,		T_INCR,		INT },
+	{ INT,		T_DECR,		INT },
+	{ INT,		T_MUL,		INT },
+	{ INT,		T_DIV,		INT },
+	{ INT, 		'=',		INT },
+	{ INT, 		0,		INT },
+	{ TIME,		T_INCR,		DURATION },
+	{ TIME,		T_DECR,		DURATION },
+	{ TIME,		T_MUL,		REAL },
+	{ TIME,		T_DIV,		REAL },
+	{ TIME,		'=',		TIME },
+	{ TIME,		0,		TIME },
+	{ DURATION,	T_INCR,		DURATION },
+	{ DURATION,	T_DECR,		DURATION },
+	{ DURATION,	T_MUL,		REAL },
+	{ DURATION,	T_DIV,		REAL },
+	{ DURATION,	'=',		DURATION },
+	{ DURATION,	0,		DURATION },
+	{ VOID, 	'=',		VOID }
+};
 
 static void
 parse_set(struct vcc *tl)
 {
 	const struct var *vp;
-	struct token *vt;
-	struct token *at;
+	const struct arith *ap;
+	enum var_type fmt;
 
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	vt = tl->t;
 	vp = vcc_FindVar(tl, tl->t, 1, "cannot be set");
 	ERRCHK(tl);
 	assert(vp != NULL);
 	Fb(tl, 1, "%s", vp->lname);
-#if 0
 	vcc_NextToken(tl);
-	SkipToken(tl, '=');
-	vcc_Expr(tl, vp->fmt);
-#else
-	vcc_NextToken(tl);
-	switch (vp->fmt) {
-	case INT:
-	case TIME:
-	case DURATION:
-		if (tl->t->tok != '=')
+	fmt = vp->fmt;
+	for (ap = arith; ap->type != VOID; ap++) {
+		if (ap->type != fmt)
+			continue;
+		if (ap->oper != tl->t->tok)
+			continue;
+		if (ap->oper != '=') 
 			Fb(tl, 0, "%s %c ", vp->rname, *tl->t->b);
-		at = tl->t;
-		vcc_NextToken(tl);
-		switch (at->tok) {
-		case T_MUL:
-		case T_DIV:
-			Fb(tl, 0, "%g", vcc_DoubleVal(tl));
-			break;
-		case T_INCR:
-		case T_DECR:
-		case '=':
-			vcc_VarVal(tl, vp, vt);
-			ERRCHK(tl);
-			break;
-		default:
-			vsb_printf(tl->sb, "Invalid assignment operator.\n");
-			vcc_ErrWhere(tl, at);
-			return;
-		}
-		Fb(tl, 0, ");\n");
-		break;
-	case BACKEND:
-		if (tl->t->tok != '=') {
-			illegal_assignment(tl, "backend");
-			return;
-		}
-		vcc_NextToken(tl);
-		vcc_ExpectCid(tl);
-		ERRCHK(tl);
-		vcc_AddRef(tl, tl->t, R_BACKEND);
-		Fb(tl, 0, "VGCDIR(_%.*s)", PF(tl->t));
-		vcc_NextToken(tl);
-		Fb(tl, 0, ");\n");
-		break;
-	case STRING:
-		if (tl->t->tok != '=') {
-			illegal_assignment(tl, "strings");
-			return;
-		}
-		vcc_NextToken(tl);
-		if (!vcc_StringVal(tl)) {
-			ERRCHK(tl);
-			vcc_ExpectedStringval(tl);
-			return;
-		}
-		do
-			Fb(tl, 0, ", ");
-		while (vcc_StringVal(tl));
-		if (tl->t->tok != ';') {
-			ERRCHK(tl);
-			vsb_printf(tl->sb,
-			    "Expected variable, string or semicolon\n");
-			vcc_ErrWhere(tl, tl->t);
-			return;
-		}
-		Fb(tl, 0, "vrt_magic_string_end);\n");
-		break;
-	case BOOL:
-		if (tl->t->tok != '=') {
-			illegal_assignment(tl, "boolean");
-			return;
-		}
-		vcc_NextToken(tl);
-		ExpectErr(tl, ID);
-		if (vcc_IdIs(tl->t, "true")) {
-			Fb(tl, 0, " 1);\n", vp->lname);
-		} else if (vcc_IdIs(tl->t, "false")) {
-			Fb(tl, 0, " 0);\n", vp->lname);
-		} else {
-			vsb_printf(tl->sb,
-			    "Expected true or false\n");
-			vcc_ErrWhere(tl, tl->t);
-			return;
-		}
 		vcc_NextToken(tl);
+		fmt = ap->want;
 		break;
-	default:
-		vsb_printf(tl->sb,
-		    "Assignments not possible for type of '%s'\n", vp->name);
-		vcc_ErrWhere(tl, tl->t);
-		return;
 	}
-#endif
+	if (ap->type == VOID)
+		SkipToken(tl, ap->oper);
+	vcc_Expr(tl, fmt);
+	if (vp->fmt == STRING)
+		Fb(tl, 1, ", vrt_magic_string_end");
+	Fb(tl, 0, ");\n");
 }
 
 /*--------------------------------------------------------------------*/

lib/libvcl/vcc_expr.c

@@ -273,8 +273,10 @@ vcc_expr_edit(enum var_type fmt, const char *p, struct expr *e1, struct expr *e2
 				vsb_cat(e->vsb, "\v+\n");
 			if (*p == '1')
 				vsb_cat(e->vsb, vsb_data(e1->vsb));
-			else
+			else {
+				AN(e2);
 				vsb_cat(e->vsb, vsb_data(e2->vsb));
+			}
 			if (q != NULL) 
 				vsb_cat(e->vsb, "\v-\n");
 			break;
@@ -326,6 +328,35 @@ vcc_expr_fmt(struct vsb *d, int ind, const struct expr *e1)
 	}
 }
 
+/*--------------------------------------------------------------------
+ */
+
+static void
+hack_regsub(struct vcc *tl, struct expr **e, int all)
+{
+	struct expr *e2;
+	char *p;
+	char buf[128];
+
+	SkipToken(tl, ID);
+	SkipToken(tl, '(');
+
+	vcc_expr0(tl, &e2, STRING);
+
+	SkipToken(tl, ',');
+	ExpectErr(tl, CSTR);
+	p = vcc_regexp(tl);
+	vcc_NextToken(tl);
+
+	bprintf(buf, "VRT_regsub(sp, %d,\n\v1,\n%s\n", all, p);
+	*e = vcc_expr_edit(STRING, buf, e2, *e);
+
+	SkipToken(tl, ',');
+	vcc_expr0(tl, &e2, STRING);
+	*e = vcc_expr_edit(STRING, "\v1, \v2)", *e, e2);
+	SkipToken(tl, ')');
+}
+
 /*--------------------------------------------------------------------
  * SYNTAX:
  *    Expr4:
@@ -354,10 +385,42 @@ vcc_expr4(struct vcc *tl, struct expr **e, enum var_type fmt)
 	e1 = vcc_new_expr();
 	switch(tl->t->tok) {
 	case ID:
+		if (vcc_IdIs(tl->t, "regsub")) {
+			vcc_delete_expr(e1);
+			hack_regsub(tl, e, 0);
+			return;
+		}
+		if (vcc_IdIs(tl->t, "regsuball")) {
+			vcc_delete_expr(e1);
+			hack_regsub(tl, e, 1);
+			return;
+		}
+		if (vcc_IdIs(tl->t, "true")) {
+			vcc_NextToken(tl);
+			vsb_printf(e1->vsb, "(1==1)");
+			e1->fmt = BOOL;
+			break;
+		}
+		if (vcc_IdIs(tl->t, "false")) {
+			vcc_NextToken(tl);
+			vsb_printf(e1->vsb, "(0!=0)");
+			e1->fmt = BOOL;
+			break;
+		}
+		if (fmt == BACKEND) {
+			vcc_ExpectCid(tl);
+			vcc_AddRef(tl, tl->t, R_BACKEND);
+			vsb_printf(e1->vsb, "VGCDIR(_%.*s)", PF(tl->t));
+			e1->fmt = BACKEND;
+			vcc_NextToken(tl);
+			break;
+		}
 		sym = VCC_FindSymbol(tl, tl->t);
 		if (sym == NULL) {
 			vsb_printf(tl->sb, "Symbol not found: ");
 			vcc_ErrToken(tl, tl->t);
+			vsb_printf(tl->sb, " (expected type %s):\n",
+			    vcc_Type(fmt));
 			vcc_ErrWhere(tl, tl->t);
 			return;
 		}
@@ -385,6 +448,9 @@ vcc_expr4(struct vcc *tl, struct expr **e, enum var_type fmt)
 			ERRCHK(tl);
 			vsb_printf(e1->vsb, "%g", d);
 			e1->fmt = DURATION;
+		} else if (fmt == REAL) {
+			vsb_printf(e1->vsb, "%g", vcc_DoubleVal(tl));
+			e1->fmt = REAL;
 		} else {
 			vsb_printf(e1->vsb, "%.*s", PF(tl->t));
 			vcc_NextToken(tl);
@@ -422,7 +488,7 @@ vcc_expr_mul(struct vcc *tl, struct expr **e, enum var_type fmt)
 
 	switch(f2) {
 	case INT:	f2 = INT; break;
-	case DURATION:	f2 = INT; break; 	/* XXX: should be Double */
+	case DURATION:	f2 = REAL; break;
 	default:
 		return;
 	}
@@ -454,6 +520,19 @@ vcc_expr_add(struct vcc *tl, struct expr **e, enum var_type fmt)
 	ERRCHK(tl);
 	f2 = (*e)->fmt;
 
+	if (f2 == STRING && tl->t->tok == '+') {
+		*e = vcc_expr_edit(STRING, "\v+VRT_String(sp,\n\v1", *e, NULL);
+		while (tl->t->tok == '+') {
+			vcc_NextToken(tl);
+			vcc_expr0(tl, &e2, STRING);
+			assert(e2->fmt == STRING);
+			*e = vcc_expr_edit(STRING, "\v1, \v2", *e, e2);
+		}
+		*e = vcc_expr_edit(STRING, "\v1, vrt_magic_string_end)",
+		    *e, NULL);
+		return;
+	}
+
 	switch(f2) {
 	case INT:	break;
 	case TIME:	f2 = DURATION; break;
@@ -517,6 +596,7 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, enum var_type fmt)
 	char buf[256];
 	char *re;
 	const char *not;
+	const char *p;
 	struct token *tk;
 
 	*e = NULL;
@@ -602,6 +682,20 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, enum var_type fmt)
 			break;
 		}
 	}
+	if (fmt == STRING) {
+		p = NULL;
+		switch((*e)->fmt) {
+		case BACKEND:	p = "VRT_backend_string(sp, \v1)"; break;
+		case INT:	p = "VRT_int_string(sp, \v1)"; break;
+		case IP:	p = "VRT_IP_string(sp, \v1)"; break;
+		case TIME:	p = "VRT_time_string(sp, \v1)"; break;
+		default:	break;
+		}
+		if (p != NULL) {
+			*e = vcc_expr_edit(STRING, p, *e, NULL);
+			return;
+		}
+	}
 	if (fmt == VOID || fmt != (*e)->fmt) {
 		vsb_printf(tl->sb, "WANT: %s has %s next %.*s (%s)\n",
 		    vcc_Type(fmt), vcc_Type((*e)->fmt),

lib/libvcl/vcc_string.c

@@ -185,7 +185,7 @@ vcc_StringVal(struct vcc *tl)
 			Fb(tl, 0, "VRT_double_string(sp, %s)", vp->rname);
 			break;
 		case BACKEND:
-			Fb(tl, 0, "VRT_backend_string(sp)");
+			Fb(tl, 0, "VRT_backend_string(sp, NULL)");
 			break;
 		default:
 			vsb_printf(tl->sb, "String representation of '%s'"

lib/libvcl/vcc_types.h

@@ -37,3 +37,4 @@ VCC_TYPE(DURATION)
 VCC_TYPE(STRING)
 VCC_TYPE(IP)
 VCC_TYPE(HEADER)
+VCC_TYPE(REAL)