Big bang rewrite of m00049.vtc

Now that vtc.typesize() is a bit more reliable, it's easier to deal with
architecture differences for the sizeof struct vrt_blob. For a much more
reliable failure check, it now looks at logs.

There may also have been a time when depending on the error we trigger
we could fail with a 500 or a 503, this is no longer the case.

For the nitty-gritty of the new test case, there are some comments to
hopefully help (at least my future self) decipher this beast.

This new test case remains stable even with the changes from #3123.

bin/varnishtest/tests/m00049.vtc

 varnishtest "VMOD blob workspace overflow conditions"
 
-varnish v1 -vcl {
+# This test case is tricky to get right and is somewhat repetitive. The VCL
+# below provides off-the-shelves subs to abstract away some of the complexity.
+# Since not all of them may be called by all VCLs we also need to ensure this
+# will not result in a compilation failure.
+
+varnish v1 -cliok "param.set vcc_err_unref off"
+
+shell {
+	cat >vrt_blob.vcl <<-EOF
 	import blob;
+	import std;
 	import vtc;
-	backend b None;
+
+	backend be none;
 
 	sub vcl_recv {
-		if (req.url == "/1") {
-			# Too small for the vmod_priv object.
-			vtc.workspace_alloc(client, -8);
-		}
-		elsif (req.url == "/2") {
-			# Likely large enough for the vmod_priv object,
-			# but not enough left over for the decode.
-			vtc.workspace_alloc(client, -33);
-		}
-		elsif (req.url == "/3") {
-			# Enough for the decode, but not enough left
-			# over for the encode.
-			vtc.workspace_alloc(client, -50);
-		}
-		set req.http.Decode
-			= blob.encode(blob=blob.decode(encoded="1234567890"));
+		set req.http.vrt_blob = vtc.typesize("uzp");
 	}
 
-	sub vcl_miss {
-		if (req.url == "/4") {
-			# Enough for the req.hash BLOB (vmod_priv + 32
-			# bytes), but not enough left over for the
-			# encoding.
-			vtc.workspace_alloc(client, -65);
-			set req.http.Encode = blob.encode(blob=req.hash);
-		}
-		return( synth(200) );
+	sub shrink {
+		std.log("shrink");
+		vtc.workspace_alloc(client, -std.integer(req.http.leave));
 	}
-} -start
 
-client c1 {
-	txreq -url "/1"
+	sub leave_struct {
+		set req.http.leave = std.integer(req.http.vrt_blob);
+		call shrink;
+	}
+
+	sub leave_half_struct {
+		set req.http.leave = std.integer(req.http.vrt_blob) / 2;
+		call shrink;
+	}
+
+	sub leave_blob {
+		set req.http.leave = std.integer(req.http.vrt_blob) +
+		    std.integer(req.http.blob);
+		call shrink;
+	}
+	EOF
+}
+
+# For each type of BLOB codec we may use some of the prepared clients. It is
+# important not to run them concurently to ensure we can predictably peek at
+# logs.
+
+client c-struct {
+	txreq -url "/struct"
 	rxresp
-	expect resp.status >= 500
-	expect resp.status <= 503
-	expect resp.reason ~ {(?i)^VCL Failed$|^Internal Server Error$}
-} -run
+	expect resp.status == 503
+	expect resp.reason == "VCL failed"
+	expect_close
+}
 
-client c2 {
-	txreq -url "/2"
+client c-encode {
+	txreq -url "/encode"
 	rxresp
-	expect resp.status >= 500
-	expect resp.status <= 503
-	expect resp.reason ~ {(?i)^VCL Failed$|^Internal Server Error$}
-} -run
+	expect resp.status == 503
+	expect resp.reason == "VCL failed"
+	expect_close
+}
 
-client c3 {
-	txreq -url "/3"
+client c-decode {
+	txreq -url "/decode"
 	rxresp
-	expect resp.status >= 500
-	expect resp.status <= 503
-	expect resp.reason ~ {(?i)^VCL Failed$|^Internal Server Error$}
-} -run
+	expect resp.status == 503
+	expect resp.reason == "VCL failed"
+	expect_close
+}
 
-client c4 {
-	txreq -url "/4"
+client c-req-hash {
+	txreq -url "/req.hash"
 	rxresp
-	expect resp.status >= 500
-	expect resp.status <= 503
-	expect resp.reason ~ {(?i)^VCL Failed$|^Internal Server Error$}
-} -run
+	expect resp.status == 503
+	expect resp.reason == "VCL failed"
+	expect_close
+}
+
+client c-sub {
+	txreq -url "/sub"
+	rxresp
+	expect resp.status == 503
+	expect resp.reason == "VCL failed"
+	expect_close
+}
+
+# For each type of BLOB codec we will check failure conditions from the logs
+# so we provision a log reader that will only deal with meaningful log records
+# for this job. This means that all logexpect expect commands must use 0 as
+# their first argument. Meaning we make a comprehensive check of all the log
+# records that aren't filtered out.
+
+logexpect l1 -v v1 -i ReqURL,VCL_Error,VCL_Log,VCL_use
+
+# IDENTITY codec
 
-# Similar tests with BASE64 encoding
 varnish v1 -vcl {
-	import blob;
-	import vtc;
-	backend b None;
+	include "${tmpdir}/vrt_blob.vcl";
 
 	sub vcl_recv {
-		if (req.url == "/1") {
-			vtc.workspace_alloc(client, -33);
+		if (req.url ~ "decode") {
+			# Not enough space to collect the string.
+			set req.http.leave = 5;
+			call shrink;
+		}
+		if (req.url ~ "struct") {
+			# Enough space to collect the decoded string.
+			# Not enough space to allocate a blob (aligned).
+			set req.http.leave = 16;
+			call shrink;
 		}
-		elsif (req.url == "/2") {
-			vtc.workspace_alloc(client, -50);
+		if (req.url ~ "encode") {
+			# Enough space to decode the string.
+			# Not enough space to encode the blob.
+			set req.http.blob = 16;
+			call leave_blob;
 		}
-		set req.http.Decode
-			= blob.encode(blob=blob.decode(BASE64,
-						encoded="MTIzNDU2Nzg5MA=="));
+		blob.encode(blob=blob.decode(encoded="1234567890"));
+		return (synth(200));
 	}
+} -start
 
-	sub vcl_miss {
-		if (req.url == "/3") {
-			vtc.workspace_alloc(client, -65);
-			set req.http.Encode
-				= blob.encode(BASE64, blob=req.hash);
+logexpect l1 {
+	expect 0 * VCL_use	vcl1
+	expect 0 = ReqURL	decode
+	expect 0 = VCL_Log	shrink
+	expect 0 = VCL_Error	"cannot decode, out of space"
+	expect 0 * VCL_use	vcl1
+	expect 0 = ReqURL	struct
+	expect 0 = VCL_Log	shrink
+	expect 0 = VCL_Error	"Workspace overflow .blob.decode."
+	expect 0 * VCL_use	vcl1
+	expect 0 = ReqURL	encode
+	expect 0 = VCL_Log	shrink
+	expect 0 = VCL_Error	"cannot encode, out of space"
+} -start
+
+client c-decode -run
+client c-struct -run
+client c-encode -run
+
+logexpect l1 -wait
+
+# BASE64 codec
+
+varnish v1 -vcl {
+	include "${tmpdir}/vrt_blob.vcl";
+
+	sub vcl_recv {
+		if (req.url ~ "decode") {
+			# Not enough space to collect the string.
+			set req.http.leave = 5;
+			call shrink;
+		}
+		if (req.url ~ "struct") {
+			# Enough space to collect the decoded string.
+			# Not enough space to allocate a blob.
+			set req.http.leave = 16;
+			call shrink;
 		}
-		return( synth(200) );
+		if (req.url ~ "encode") {
+			# Enough space to decode the string.
+			# Not enough space to encode the blob.
+			set req.http.blob = 16;
+			call leave_blob;
+		}
+		blob.encode(
+		    blob=blob.decode(BASE64, encoded="MTIzNDU2Nzg5MA=="));
+		return (synth(200));
 	}
 }
 
-client c1 -run
-client c2 -run
-client c3 -run
+logexpect l1 {
+	expect 0 * VCL_use	vcl2
+	expect 0 = ReqURL	decode
+	expect 0 = VCL_Log	shrink
+	expect 0 = VCL_Error	"cannot decode, out of space"
+	expect 0 * VCL_use	vcl2
+	expect 0 = ReqURL	struct
+	expect 0 = VCL_Log	shrink
+	expect 0 = VCL_Error	"Workspace overflow .blob.decode."
+	expect 0 * VCL_use	vcl2
+	expect 0 = ReqURL	encode
+	expect 0 = VCL_Log	shrink
+	expect 0 = VCL_Error	"cannot encode, out of space"
+} -start
+
+client c-decode -run
+client c-struct -run
+client c-encode -run
+
+logexpect l1 -wait
+
+# URL codec
 
-# URL encoding
 varnish v1 -vcl {
-	import blob;
-	import vtc;
-	backend b None;
+	include "${tmpdir}/vrt_blob.vcl";
 
 	sub vcl_recv {
-		if (req.url == "/1") {
-			vtc.workspace_alloc(client, -33);
+		if (req.url ~ "decode") {
+			# Not enough space to collect the string.
+			set req.http.leave = 5;
+			call shrink;
 		}
-		elsif (req.url == "/2") {
-			vtc.workspace_alloc(client, -50);
+		if (req.url ~ "struct") {
+			# Enough space to collect the decoded string.
+			# Not enough space to allocate a blob (aligned).
+			set req.http.leave = 16;
+			call shrink;
 		}
-		set req.http.Decode
-			= blob.encode(blob=blob.decode(URL,
-						encoded="1234567890"));
-	}
-
-	sub vcl_miss {
-		if (req.url == "/3") {
-			vtc.workspace_alloc(client, -65);
-			set req.http.Encode = blob.encode(URL, blob=req.hash);
+		if (req.url ~ "encode") {
+			# Enough space to decode the string.
+			# Not enough space to encode the blob.
+			set req.http.blob = 16;
+			call leave_blob;
 		}
-		return( synth(200) );
+		blob.encode(blob=blob.decode(URL, encoded="1234567890"));
+		return (synth(200));
 	}
 }
 
-client c1 -run
-client c2 -run
-client c3 -run
+logexpect l1 {
+	expect 0 * VCL_use	vcl3
+	expect 0 = ReqURL	decode
+	expect 0 = VCL_Log	shrink
+	expect 0 = VCL_Error	"cannot decode, out of space"
+	expect 0 * VCL_use	vcl3
+	expect 0 = ReqURL	struct
+	expect 0 = VCL_Log	shrink
+	expect 0 = VCL_Error	"Workspace overflow .blob.decode."
+	expect 0 * VCL_use	vcl3
+	expect 0 = ReqURL	encode
+	expect 0 = VCL_Log	shrink
+	expect 0 = VCL_Error	"cannot encode, out of space"
+} -start
+
+client c-decode -run
+client c-struct -run
+client c-encode -run
+
+logexpect l1 -wait
+
+# HEX codec
 
-# HEX encoding
 varnish v1 -vcl {
-	import blob;
-	import vtc;
-	backend b None;
+	include "${tmpdir}/vrt_blob.vcl";
 
 	sub vcl_recv {
-		if (req.url == "/1") {
-			vtc.workspace_alloc(client, -33);
+		if (req.url ~ "decode") {
+			# Not enough space to collect the string.
+			set req.http.leave = 5;
+			call shrink;
 		}
-		elsif (req.url == "/2") {
-			vtc.workspace_alloc(client, -50);
+		if (req.url ~ "struct") {
+			# Enough space to collect the decoded string.
+			# Not enough space to allocate a blob (aligned).
+			set req.http.leave = 20;
+			call shrink;
 		}
-		set req.http.Decode
-			= blob.encode(blob=blob.decode(HEX,
-					encoded="31323334353637383930"));
-	}
-
-	sub vcl_miss {
-		if (req.url == "/3") {
-			vtc.workspace_alloc(client, -65);
-			set req.http.Encode = blob.encode(HEX, blob=req.hash);
+		if (req.url ~ "encode") {
+			# Enough space to decode the string.
+			# Not enough space to encode the blob.
+			set req.http.blob = 20;
+			call leave_blob;
 		}
-		return( synth(200) );
+		blob.encode(
+		    blob=blob.decode(HEX, encoded="31323334353637383930"));
+		return (synth(200));
 	}
 }
 
-client c1 -run
-client c2 -run
-client c3 -run
+logexpect l1 {
+	expect 0 * VCL_use	vcl4
+	expect 0 = ReqURL	decode
+	expect 0 = VCL_Log	shrink
+	expect 0 = VCL_Error	"cannot decode, out of space"
+	expect 0 * VCL_use	vcl4
+	expect 0 = ReqURL	struct
+	expect 0 = VCL_Log	shrink
+	expect 0 = VCL_Error	"Workspace overflow .blob.decode."
+	expect 0 * VCL_use	vcl4
+	expect 0 = ReqURL	encode
+	expect 0 = VCL_Log	shrink
+	expect 0 = VCL_Error	"cannot encode, out of space"
+} -start
+
+client c-decode -run
+client c-struct -run
+client c-encode -run
+
+logexpect l1 -wait
+
+# blob.sub() function
 
-# sub() function
 varnish v1 -vcl {
-	import blob;
-	import vtc;
-	backend b None;
+	include "${tmpdir}/vrt_blob.vcl";
 
 	sub vcl_miss {
-		if (req.url == "/1") {
-			# Not enough for req.hash + vmod_priv
-			vtc.workspace_alloc(client, -65);
+		if (req.url ~ "req.hash") {
+			# Not enough to create the req.hash blob.
+			call leave_half_struct;
 		}
-		elsif (req.url == "/2") {
-			# Not enough for req.hash + vmod_priv + 30 bytes
-			vtc.workspace_alloc(client, -90);
+		if (req.url ~ "struct") {
+			# Enough for the req.hash blob.
+			# Not enough for the sub-blob.
+			call leave_struct;
 		}
-		set req.http.Encode = blob.encode(blob=blob.sub(req.hash, 30B));
-		return( synth(200) );
+		blob.encode(blob=blob.sub(req.hash, 30B));
+		return (synth(200));
 	}
 }
 
-client c1 -run
-client c2 -run
+logexpect l1 {
+	expect 0 * VCL_use	vcl5
+	expect 0 = ReqURL	req.hash
+	expect 0 = VCL_Log	shrink
+	expect 0 = VCL_Error	"Workspace overflow .req.hash."
+	expect 0 * VCL_use	vcl5
+	expect 0 = ReqURL	struct
+	expect 0 = VCL_Log	shrink
+	expect 0 = VCL_Error	"Workspace overflow .blob.sub."
+} -start
+
+client c-req-hash -run
+client c-struct -run
+
+logexpect l1 -wait

lib/libvmod_blob/vmod_blob.c

@@ -534,6 +534,6 @@ vmod_sub(VRT_CTX, VCL_BLOB b, VCL_BYTES n, VCL_BYTES off)
 		return null_blob;
 
 
-	return (VRT_blob(ctx, "blob.sub()",
+	return (VRT_blob(ctx, "blob.sub",
 	    (const char *)b->blob + off, n, b->type));
 }