vtc: add User-Agent: cNAME header by default

by default, add "User-Agent: cNAME" header to all client txreq requests
if not already present (via -hdr). This can be disabled with -nouseragent

bin/varnishtest/tests/a00024.vtc

-varnishtest "test -noserver"
+varnishtest "test -nouseragent and -noserver"
 
 server s1  {
     rxreq
+    # by default, User-Agent header is set to cNAME
+    expect req.http.User-Agent == "c101"
     txresp
     rxreq
+    # when specified with -hdr, it overrides the default
+    expect req.http.User-Agent == "not-c101"
     txresp -hdr "Server: not-s1"
 } -start
 
 server s2  {
     rxreq
+    expect req.http.User-Agent == "c202"
     txresp
     rxreq
+    # default User-Agent header is not included when -nouseragent is specified
+    expect req.http.User-Agent == <undef>
     txresp -noserver
 } -start
 
@@ -35,7 +42,7 @@ client c101 {
     rxresp
     # by default, Server header is set to sNAME
     expect resp.http.Server == "s1"
-    txreq  -url "/home"
+    txreq  -url "/home" -hdr "User-Agent: not-c101"
     rxresp
     # when specified with -hdr, it overrides the default
     expect resp.http.Server == "not-s1"
@@ -45,7 +52,7 @@ client c202 {
 	txreq
 	rxresp
 	expect resp.http.Server == "s2"
-    txreq
+    txreq -nouseragent
 	rxresp
 	# default Server header is not included when -noserver is specified
 	expect resp.http.Server == <undef>

bin/varnishtest/tests/e00003.vtc

@@ -50,7 +50,7 @@ varnish v1 -vcl+backend {
 
 logexpect l1 -v v1 -g request {
 	expect 0 1001   Begin   "^req .* rxreq"
-	expect * =	ReqAcct	"^29 0 29 182 75 257$"
+	expect * =	ReqAcct	"^45 0 45 182 75 257$"
 	expect 0 =      End
 } -start
 
@@ -73,7 +73,7 @@ logexpect l4 -v v1 -g request {
 logexpect l5 -v v1 -g request {
 	expect * 1006   Begin   "^req .* rxreq"
 	# Header bytes is 5 larger than in l1 due to two item X-Varnish hdr
-	expect * =	ReqAcct	"^29 0 29 187 75 262$"
+	expect * =	ReqAcct	"^45 0 45 187 75 262$"
 	expect 0 =      End
 } -start
 

bin/varnishtest/tests/l00002.vtc

@@ -26,8 +26,9 @@ varnish v1 -vcl+backend {
 # POST /1 HTTP/1.1\r\n		 18 bytes
 # Host: foo\r\n			 11 bytes
 # Content-Length: 4\r\n		 19 bytes
+# User-Agent: c1\r\n		 16 bytes
 # \r\n				  2 bytes
-# Total:			 50 bytes
+# Total:			 66 bytes
 # Response:
 # HTTP/1.1 200 OK\r\n		 17 bytes
 # Content-Length: 1000\r\n	 22 bytes
@@ -75,7 +76,7 @@ varnish v1 -vcl+backend {
 # Total:			 28 bytes
 logexpect l1 -v v1 -g session {
 	expect * 1001	Begin	"^req .* rxreq"
-	expect * =	ReqAcct	"^50 4 54 99 1000 1099$"
+	expect * =	ReqAcct	"^66 4 70 99 1000 1099$"
 	expect 0 =	End
 	expect * 1003	Begin	"^req .* rxreq"
 	expect * =	ReqAcct "^30 0 30 99 2000 2099$"
@@ -111,7 +112,7 @@ client c1 {
 
 logexpect l1 -wait
 
-varnish v1 -expect s_req_hdrbytes == 117
+varnish v1 -expect s_req_hdrbytes == 133
 varnish v1 -expect s_req_bodybytes == 4
 varnish v1 -expect s_resp_hdrbytes == 325
 varnish v1 -expect s_resp_bodybytes == 5000

bin/varnishtest/tests/l00003.vtc

@@ -31,8 +31,9 @@ varnish v1 -vcl+backend {
 # Request (1001):
 # GET / HTTP/1.1\r\n			16 bytes
 # Host: foo\r\n				11 bytes
+# User-Agent: c1\r\n			16 bytes
 # \r\n					 2 bytes
-# Total:				29 bytes
+# Total:				45 bytes
 
 # Reponse:
 # HTTP/1.1 200 OK\r\n			17 bytes
@@ -64,7 +65,7 @@ varnish v1 -vcl+backend {
 
 logexpect l1 -v v1 -g request {
 	expect 0 1001	Begin	"^req .* rxreq"
-	expect * =	ReqAcct		"^29 0 29 105 15 120$"
+	expect * =	ReqAcct		"^45 0 45 105 15 120$"
 	expect 0 =	End
 	expect * 1003	Begin		"^req .* esi"
 	expect * =	ReqAcct		"^0 0 0 0 12 12$"
@@ -86,7 +87,7 @@ client c1 {
 
 logexpect l1 -wait
 
-varnish v1 -expect s_req_hdrbytes == 29
+varnish v1 -expect s_req_hdrbytes == 45
 varnish v1 -expect s_req_bodybytes == 0
 varnish v1 -expect s_resp_hdrbytes == 105
 varnish v1 -expect s_resp_bodybytes == 15

bin/varnishtest/tests/l00004.vtc

@@ -26,15 +26,18 @@ varnish v1 -vcl+backend {
 # POST / HTTP/1.1\r\n			17 bytes
 # Host: foo\r\n				11 bytes
 # Content-Length: 4\r\n			19 bytes
+# User-Agent: c1\r\n			16 bytes
 # \r\n					 2 bytes
-# Total:				49 bytes
+# Total:				65 bytes
 
 # bereq:
 # POST / HTTP/1.1\r\n			17 bytes
 # Content-Length: 4\r\n			19 bytes
+# Host: foo\r\n				11 bytes
+# User-Agent: c1\r\n			16 bytes
 # test: yes\r\n				11 bytes
 # \r\n					 2 bytes
-# Total:				49 bytes
+# Total:				76 bytes
 
 # reqbody
 # asdf					 4 bytes
@@ -49,7 +52,7 @@ varnish v1 -vcl+backend {
 
 logexpect l1 -v v1 -g request {
 	expect 0 1001	Begin		"^req .* rxreq"
-	expect * =	PipeAcct	"^49 60 4 54$"
+	expect * =	PipeAcct	"^65 76 4 54$"
 	expect 0 =	End
 } -start
 
@@ -62,6 +65,6 @@ client c1 {
 
 logexpect l1 -wait
 
-varnish v1 -expect s_pipe_hdrbytes == 49
+varnish v1 -expect s_pipe_hdrbytes == 65
 varnish v1 -expect s_pipe_in == 4
 varnish v1 -expect s_pipe_out == 54

bin/varnishtest/tests/l00005.vtc

@@ -28,9 +28,10 @@ varnish v1 -vcl+backend {
 # Request (1002):
 # POST /1 HTTP/1.1\r\n		 18 bytes
 # Content-Length: 4\r\n		 19 bytes
+# User-Agent: c1\r\n		 16 bytes
 # Host: foo.bar\r\n		 15 bytes
 # \r\n				  2 bytes
-# Total:			 54 bytes
+# Total:			 70 bytes
 # Response:
 # HTTP/1.1 200 OK\r\n		 17 bytes
 # Content-Length: 1000\r\n	 22 bytes
@@ -41,9 +42,10 @@ varnish v1 -vcl+backend {
 # Request (1004):
 # POST /2 HTTP/1.1\r\n		 18 bytes
 # Content-Length: 4\r\n		 19 bytes
+# User-Agent: c1\r\n		 16 bytes
 # Host: foo.bar\r\n		 15 bytes
 # \r\n				  2 bytes
-# Total:			 54 bytes
+# Total:			 70 bytes
 # Reponse:
 # HTTP/1.1\r\n			 10 bytes
 # \r\n				  2 bytes
@@ -55,10 +57,10 @@ logexpect l1 -v v1 -g session {
 	expect 0 1003	Begin		"^req .* rxreq"
 	expect * =	End
 	expect 0 1002   Begin		"^bereq "
-	expect * =      BereqAcct       "^54 4 58 53 1000 1053$"
+	expect * =      BereqAcct       "^70 4 74 53 1000 1053$"
 	expect 0 =      End
 	expect 0 1004	Begin		"^bereq"
-	expect * =	BereqAcct	"^54 4 58 12 0 12$"
+	expect * =	BereqAcct	"^70 4 74 12 0 12$"
 	expect * =	End
 
 } -start

bin/varnishtest/tests/r01637.vtc

@@ -25,7 +25,7 @@ varnish v1  -arg "-sdefault,1M" -arg "-p nuke_limit=0 -p gzip_level=0" \
 } -start
 
 logexpect l1 -v v1 -g vxid -q "vxid == 1004" {
-	expect 25 1004	VCL_call        {^BACKEND_RESPONSE}
+	expect 26 1004	VCL_call        {^BACKEND_RESPONSE}
 	expect 0  =	BerespHeader    {^free:}
 	expect 0  =	VCL_return      {^deliver}
 	expect 0  =	Timestamp       {^Process}

bin/varnishtest/tests/u00003.vtc

@@ -34,9 +34,9 @@ process p1 -winsz 25 200 {varnishncsa -n ${v1_name}} -start
 delay 1
 
 client c1 {
-	txreq -url /1?foo=bar -hdr "authorization: basic dXNlcjpwYXNz"
+	txreq -nouseragent -url /1?foo=bar -hdr "authorization: basic dXNlcjpwYXNz"
 	rxresp
-	txreq -url /1?foo=bar -hdr "baz: qux"
+	txreq -nouseragent -url /1?foo=bar -hdr "baz: qux"
 	rxresp
 } -run
 
@@ -46,7 +46,7 @@ shell "mv ${tmpdir}/ncsa.log ${tmpdir}/ncsa.old.log"
 shell "kill -HUP `cat ${tmpdir}/ncsa.pid`"
 
 client c2 {
-	txreq -url /2
+	txreq -nouseragent -url /2
 	rxresp
 } -run
 

bin/varnishtest/tests/v00051.vtc

@@ -188,6 +188,7 @@ logexpect l1012 -v v1 -g vxid -q "vxid == 1012" {
 	expect 0 =	BereqProtocol	{^HTTP/1.1}
 	expect 0 =	BereqHeader	{^foo: pipe}
 	expect 0 =	BereqHeader	{^Host: }
+	expect 0 =	BereqHeader	{^User-Agent: c1}
 	expect 0 =	BereqHeader	{^X-Forwarded-For: }
 	expect 0 =	BereqHeader	{^Via: }
 	expect 0 =	BereqHeader	{^X-Varnish: 1011}
@@ -207,6 +208,7 @@ logexpect l1011 -v v1 -g vxid -q "vxid == 1011" {
 	expect 0 =	ReqProtocol     {^HTTP/1.1}
 	expect 0 =	ReqHeader       {^foo: pipe}
 	expect 0 =	ReqHeader       {^Host: }
+	expect 0 =	reqHeader	{^User-Agent: c1}
 	expect 0 =	ReqHeader       {^X-Forwarded-For: }
 	expect 0 =	ReqHeader	{^Via: }
 	expect 0 =	VCL_call        {^RECV}

bin/varnishtest/vtc_http.c

@@ -769,7 +769,7 @@ cmd_http_gunzip(CMD_ARGS)
 
 static char* const *
 http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp,
-    char *body, unsigned nohost, unsigned nodate, unsigned noserver)
+    char *body, unsigned nohost, unsigned nodate, unsigned noserver, unsigned nouseragent)
 {
 	long bodylen = 0;
 	char *b, *c;
@@ -797,6 +797,8 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp,
 				nodate = 1;
 			if (!strncasecmp(av[1], "Server:", 7))
 				noserver = 1;
+			if (!strncasecmp(av[1], "User-Agent:", 11))
+				nouseragent = 1;
 			VSB_printf(hp->vsb, "%s%s", av[1], nl);
 			av++;
 		} else if (!strcmp(*av, "-hdrlen")) {
@@ -863,6 +865,8 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp,
 	}
 	if (!noserver)
 		VSB_printf(hp->vsb, "Server: %s%s", hp->sess->name, nl);
+	if (!nouseragent)
+		VSB_printf(hp->vsb, "User-Agent: %s%s", hp->sess->name, nl);
 	if (body != NULL && !nolen)
 		VSB_printf(hp->vsb, "Content-Length: %ld%s", bodylen, nl);
 	VSB_cat(hp->vsb, nl);
@@ -906,6 +910,9 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp,
  *         \-noserver (txresp only)
  *                 Don't include a Server header with the id of the server.
  *
+ *         \-nouseragent (txreq only)
+ *                 Don't include a User-Agent header with the id of the client.
+ *
  *         These three switches can appear in any order but must come before the
  *         following ones.
  *
@@ -1000,7 +1007,7 @@ cmd_http_txresp(CMD_ARGS)
 	/* send a "Content-Length: 0" header unless something else happens */
 	REPLACE(body, "");
 
-	av = http_tx_parse_args(av, vl, hp, body, 1, 0, noserver);
+	av = http_tx_parse_args(av, vl, hp, body, 1, 0, noserver, 1);
 	if (*av != NULL)
 		vtc_fatal(hp->vl, "Unknown http txresp spec: %s\n", *av);
 
@@ -1196,6 +1203,7 @@ cmd_http_txreq(CMD_ARGS)
 	const char *proto = "HTTP/1.1";
 	const char *up = NULL;
 	unsigned nohost;
+	unsigned nouseragent = 0;
 
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
@@ -1221,6 +1229,8 @@ cmd_http_txreq(CMD_ARGS)
 		} else if (!hp->sfd && !strcmp(*av, "-up")) {
 			up = av[1];
 			av++;
+		} else if (!strcmp(*av, "-nouseragent")) {
+			nouseragent = 1;
 		} else
 			break;
 	}
@@ -1232,7 +1242,7 @@ cmd_http_txreq(CMD_ARGS)
 				"HTTP2-Settings: %s%s", nl, nl, up, nl);
 
 	nohost = strcmp(proto, "HTTP/1.1") != 0;
-	av = http_tx_parse_args(av, vl, hp, NULL, nohost, 1, 1);
+	av = http_tx_parse_args(av, vl, hp, NULL, nohost, 1, 1, nouseragent);
 	if (*av != NULL)
 		vtc_fatal(hp->vl, "Unknown http txreq spec: %s\n", *av);
 	http_write(hp, 4, "txreq");