Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache

bin/varnishd/cache_center.c

@@ -430,12 +430,12 @@ cnt_error(struct sess *sp)
 }
 
 /*--------------------------------------------------------------------
- * Fetch an object from the backend, either for pass or for caching.
+ * Fetch response headers from the backend
  *
 DOT subgraph xcluster_fetch {
 DOT	fetch [
 DOT		shape=ellipse
-DOT		label="fetch from backend\n(find obj.ttl)"
+DOT		label="fetch hdr\nfrom backend\n(find obj.ttl)"
 DOT	]
 DOT	vcl_fetch [
 DOT		shape=record
@@ -446,11 +446,10 @@ DOT	fetch_pass [
 DOT		shape=ellipse
 DOT		label="obj.pass=true"
 DOT	]
-DOT	vcl_fetch -> fetch_pass [label="pass",style=bold,color=red]
+DOT	vcl_fetch -> fetch_pass [label="hit_for_pass",style=bold,color=red]
 DOT }
-DOT fetch_pass -> deliver [style=bold,color=red]
-DOT vcl_fetch -> deliver [label="deliver",style=bold,color=blue,weight=2]
-DOT vcl_fetch -> recv [label="restart"]
+DOT fetch_pass -> fetchbody [style=bold,color=red]
+DOT vcl_fetch -> fetchbody [label="deliver",style=bold,color=blue,weight=2]
 DOT vcl_fetch -> rstfetch [label="restart",color=purple]
 DOT rstfetch [label="RESTART",shape=plaintext]
 DOT fetch -> errfetch
@@ -462,11 +461,6 @@ static int
 cnt_fetch(struct sess *sp)
 {
 	int i;
-	struct http *hp, *hp2;
-	char *b;
-	unsigned l, nhttp;
-	int varyl = 0, pass;
-	struct vsb *vary = NULL;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
@@ -489,21 +483,6 @@ cnt_fetch(struct sess *sp)
 		i = FetchHdr(sp);
 	}
 
-	/*
-	 * These two headers can be spread over multiple actual headers
-	 * and we rely on their content outside of VCL, so collect them
-	 * into one line here.
-	 */
-	http_CollectHdr(sp->wrk->beresp, H_Cache_Control);
-	http_CollectHdr(sp->wrk->beresp, H_Vary);
-
-	/*
-	 * Figure out how the fetch is supposed to happen, before the
-	 * headers are adultered by VCL
-	 * Also sets other sp->wrk variables 
-	 */ 
-	sp->wrk->body_status = RFC2616_Body(sp);
-
 	if (i) {
 		if (sp->objcore != NULL) {
 			CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
@@ -520,6 +499,22 @@ cnt_fetch(struct sess *sp)
 		return (0);
 	}
 
+	/*
+	 * These two headers can be spread over multiple actual headers
+	 * and we rely on their content outside of VCL, so collect them
+	 * into one line here.
+	 */
+	http_CollectHdr(sp->wrk->beresp, H_Cache_Control);
+	http_CollectHdr(sp->wrk->beresp, H_Vary);
+
+	/*
+	 * Figure out how the fetch is supposed to happen, before the
+	 * headers are adultered by VCL
+	 * Also sets other sp->wrk variables 
+	 */ 
+	sp->wrk->body_status = RFC2616_Body(sp);
+
+
 	sp->err_code = http_GetStatus(sp->wrk->beresp);
 
 	/*
@@ -541,6 +536,75 @@ cnt_fetch(struct sess *sp)
 
 	VCL_fetch_method(sp);
 
+	switch (sp->handling) {
+	case VCL_RET_HIT_FOR_PASS:
+		if (sp->objcore != NULL)
+			sp->objcore->flags |= OC_F_PASS;
+		sp->step = STP_FETCHBODY;
+		return (0);
+	case VCL_RET_DELIVER:
+		sp->step = STP_FETCHBODY;
+		return (0);
+	default:
+		break;
+	}
+
+	/*
+	 * We are not going to fetch the body
+	 * Close the connection and clean up...
+	 */
+
+	VDI_CloseFd(sp);
+	if (sp->objcore != NULL) {
+		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
+		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
+		sp->objcore = NULL;
+	}
+	http_Setup(sp->wrk->bereq, NULL);
+	http_Setup(sp->wrk->beresp, NULL);
+	sp->wrk->h_content_length = NULL;
+	sp->director = NULL;
+	sp->wrk->storage_hint = NULL;
+
+	switch (sp->handling) {
+	case VCL_RET_RESTART:
+		sp->restarts++;
+		sp->step = STP_RECV;
+		return (0);
+	case VCL_RET_ERROR:
+		sp->step = STP_ERROR;
+		return (0);
+	default:
+		WRONG("Illegal action in vcl_fetch{}");
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Fetch response body from the backend
+ *
+DOT subgraph xcluster_body {
+DOT	fetchbody [
+DOT		shape=ellipse
+DOT		label="fetch body\nfrom backend\n"
+DOT	]
+DOT }
+DOT fetchbody -> deliver [style=bold,color=red]
+ */
+
+
+static int
+cnt_fetchbody(struct sess *sp)
+{
+	int i;
+	struct http *hp, *hp2;
+	char *b;
+	unsigned l, nhttp;
+	struct vsb *vary = NULL;
+	int varyl = 0, pass;
+
+	assert(sp->handling == VCL_RET_HIT_FOR_PASS ||
+	    sp->handling == VCL_RET_DELIVER);
+
 	if (sp->objcore == NULL) {
 		/* This is a pass from vcl_recv */
 		pass = 1;
@@ -550,7 +614,6 @@ cnt_fetch(struct sess *sp)
 		/* pass from vcl_fetch{} -> hit-for-pass */
 		/* XXX: the bereq was not filtered pass... */
 		pass = 1;
-		sp->objcore->flags |= OC_F_PASS;
 	} else {
 		/* regular object */
 		pass = 0;
@@ -698,24 +761,6 @@ cnt_fetch(struct sess *sp)
 		return (0);
 	}
 
-	switch (sp->handling) {
-	case VCL_RET_RESTART:
-		HSH_Drop(sp);
-		sp->director = NULL;
-		sp->restarts++;
-		sp->step = STP_RECV;
-		return (0);
-	case VCL_RET_HIT_FOR_PASS:
-	case VCL_RET_DELIVER:
-		break;
-	case VCL_RET_ERROR:
-		HSH_Drop(sp);
-		sp->step = STP_ERROR;
-		return (0);
-	default:
-		WRONG("Illegal action in vcl_fetch{}");
-	}
-
 	if (sp->obj->objcore != NULL) {
 		EXP_Insert(sp->obj);
 		AN(sp->obj->objcore);
@@ -1334,7 +1379,6 @@ CNT_Session(struct sess *sp)
 		CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
 		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
 		WS_Assert(w->ws);
-		AZ(sp->wrk->storage_hint);
 
 		switch (sp->step) {
 #define STEP(l,u) \

bin/varnishd/steps.h

@@ -40,6 +40,7 @@ STEP(lookup,	LOOKUP)
 STEP(miss,	MISS)
 STEP(hit,	HIT)
 STEP(fetch,	FETCH)
+STEP(fetchbody,	FETCHBODY)
 STEP(deliver,	DELIVER)
 STEP(error,	ERROR)
 STEP(done,	DONE)

bin/varnishtest/tests/b00019.vtc

@@ -5,14 +5,19 @@ test "Check that max_restarts works and that we don't fall over"
 server s1 {
 	rxreq
 	txresp -body "012345\n"
+	accept
 	rxreq
 	txresp -body "012345\n"
+	accept
 	rxreq
 	txresp -body "012345\n"
+	accept
 	rxreq
 	txresp -body "012345\n"
+	accept
 	rxreq
 	txresp -body "012345\n"
+	accept
 	rxreq
 	txresp -body "012345\n"
 } -start

bin/varnishtest/tests/c00029.vtc

@@ -18,12 +18,18 @@ server s1 {
 	rxreq
 	txresp -hdr "X-Saint: yes"
 
+	accept
+
 	rxreq
 	txresp -hdr "X-Saint: yes"
 
+	accept
+
 	rxreq
 	txresp -hdr "X-Saint: yes"
 
+	accept
+
 	rxreq
 	txresp -hdr "X-Saint: yes"
 } -start

bin/varnishtest/tests/c00030.vtc

@@ -18,12 +18,18 @@ server s1 {
 	rxreq
 	txresp -hdr "X-Saint: yes"
 
+	accept
+
 	rxreq
 	txresp -hdr "X-Saint: yes"
 
+	accept
+
 	rxreq
 	txresp -hdr "X-Saint: yes"
 
+	accept
+
 	rxreq
 	txresp -hdr "X-Saint: yes"
 } -start

bin/varnishtest/tests/c00032.vtc

@@ -8,6 +8,7 @@ server s1 {
 	expect req.url == "/foo"
 	expect req.http.foobar == "harck-coff"
 	txresp -status 400
+	accept
 	rxreq
 	expect req.url == "/bar"
 	expect req.http.foobar == "snark"

bin/varnishtest/tests/r00412.vtc

@@ -6,6 +6,7 @@ server s1 {
 	rxreq
 	expect req.url == "/"
 	txresp -status 303 -hdr "Location: /foo" 
+	accept
 	rxreq
 	expect req.url == "/foo"
 	txresp -body "12345"

bin/varnishtest/tests/s00003.vtc

@@ -11,6 +11,7 @@ server s1 {
 	rxreq
 	expect req.url == "/"
 	txresp -status 200 -hdr "foo: 2"
+	accept
 	rxreq
 	expect req.url == "/"
 	txresp -status 200 -hdr "foo: 3"

bin/varnishtest/vtc.c

@@ -76,6 +76,8 @@ struct macro {
 
 static VTAILQ_HEAD(,macro) macro_list = VTAILQ_HEAD_INITIALIZER(macro_list);
 
+struct _extmacro_list extmacro_list = VTAILQ_HEAD_INITIALIZER(extmacro_list);
+
 static pthread_mutex_t		macro_mtx;
 
 static void
@@ -479,6 +481,7 @@ exec_file(const char *fn, const char *script, const char *tmpdir,
 	char *cwd, *p;
 	char topbuild[BUFSIZ];
 	FILE *f;
+	struct extmacro *m;
 
 	vtc_loginit(logbuf, loglen);
 	vltop = vtc_logopen("top");
@@ -487,6 +490,10 @@ exec_file(const char *fn, const char *script, const char *tmpdir,
 	init_macro();
 	init_sema();
 
+	/* Apply extmacro definitions */
+	VTAILQ_FOREACH(m, &extmacro_list, list)
+		macro_def(vltop, NULL, m->name, m->val);
+
 	/* We are still in bin/varnishtest at this point */
 	cwd = getcwd(NULL, PATH_MAX);
 	bprintf(topbuild, "%s/%s", cwd, TOP_BUILDDIR);

bin/varnishtest/vtc.h

@@ -34,6 +34,7 @@
 #ifdef HAVE_PTHREAD_NP_H
 #include <pthread_np.h>
 #endif
+#include "vqueue.h"
 
 struct vsb;
 struct vtclog;
@@ -49,6 +50,15 @@ struct cmds {
 	cmd_f		*cmd;
 };
 
+struct extmacro {
+	VTAILQ_ENTRY(extmacro)	list;
+	char			*name;
+	char			*val;
+};
+
+VTAILQ_HEAD(_extmacro_list, extmacro);
+extern struct _extmacro_list extmacro_list;
+
 void parse_string(char *buf, const struct cmds *cmd, void *priv,
     struct vtclog *vl);
 

bin/varnishtest/vtc_main.c

@@ -91,6 +91,31 @@ static int vtc_good;
 static int vtc_fail;
 static int leave_temp;
 
+/**********************************************************************
+ * Parse a -D option argument into a name/val pair, and insert
+ * into extmacro list
+ */
+
+static int
+parse_D_opt(char *arg)
+{
+	char *p, *q;
+	struct extmacro *m;
+
+	p = arg;
+	q = strchr(p, '=');
+	if (!q)
+		return (0);
+	*q++ = '\0';
+	m = calloc(sizeof *m, 1);
+	AN(m);
+	REPLACE(m->name, p);
+	REPLACE(m->val, q);
+	VTAILQ_INSERT_TAIL(&extmacro_list, m, list);
+
+	return (1);
+}
+
 /**********************************************************************
  * Read a file into memory
  */
@@ -130,6 +155,7 @@ usage(void)
 {
 	fprintf(stderr, "usage: varnishtest [options] file ...\n");
 #define FMT "    %-28s # %s\n"
+	fprintf(stderr, FMT, "-D name=val", "Define macro for use in scripts");
 	fprintf(stderr, FMT, "-j jobs", "Run this many tests in parallel");
 	fprintf(stderr, FMT, "-k", "Continue on test failure");
 	fprintf(stderr, FMT, "-l", "Leave /tmp/vtc.* if test fails");
@@ -206,7 +232,7 @@ tst_cb(const struct vev *ve, int what)
 			    jp->tst->filename, t);
 			if (!vtc_continue) {
 				/* XXX kill -9 other jobs ? */
-				exit (2);
+				exit(2);
 			}
 		} else if (vtc_verbosity) {
 			printf("#     top  TEST %s passed (%.3f)\n",
@@ -310,8 +336,15 @@ main(int argc, char * const *argv)
 
 	setbuf(stdout, NULL);
 	setbuf(stderr, NULL);
-	while ((ch = getopt(argc, argv, "j:klLn:qt:v")) != -1) {
+	while ((ch = getopt(argc, argv, "D:j:klLn:qt:v")) != -1) {
 		switch (ch) {
+		case 'D':
+			if (!parse_D_opt(optarg)) {
+				fprintf(stderr, "Cannot parse D opt '%s'\n", 
+					optarg);
+				exit(2);
+			}
+			break;
 		case 'j':
 			npar = strtoul(optarg, NULL, 0);
 			break;
@@ -352,7 +385,7 @@ main(int argc, char * const *argv)
 			    *argv, strerror(errno));
 			if (vtc_continue)
 				continue;
-			exit (2);
+			exit(2);
 		}
 		ALLOC_OBJ(tp, TST_MAGIC);
 		AN(tp);