Add a flag to mark 304 backend response processing (aka Backend IMS/INM)

This is a compromise discussed at VDD15Q1 after a previous suggestion
to expose the 304 response status directly to VCL.

This way, VCL can differenciate between 200 and 304 responses, but simpler
processing logic just checking for status 200 can be preserved where
differenciating is not required.

bin/varnishd/cache/cache_fetch.c

@@ -254,7 +254,7 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo)
 static enum fetch_step
 vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 {
-	int i, do_ims = 0;
+	int i;
 	double now;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -385,6 +385,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 		bo->fetch_objcore->exp.ttl = -1.;
 
 	AZ(bo->do_esi);
+	AZ(bo->was_304);
 
 	if (http_IsStatus(bo->beresp, 304)) {
 		if (bo->stale_oc != NULL &&
@@ -401,7 +402,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 			http_Unset(bo->beresp, H_Content_Length);
 			HTTP_Merge(bo->wrk, bo->stale_oc, bo->beresp);
 			assert(http_IsStatus(bo->beresp, 200));
-			do_ims = 1;
+			bo->was_304 = 1;
 		} else if (!bo->do_pass) {
 			/*
 			 * Backend sent unallowed 304
@@ -451,7 +452,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	assert(wrk->handling == VCL_RET_DELIVER);
 
-	return (do_ims ? F_STP_CONDFETCH : F_STP_FETCH);
+	return (bo->was_304 ? F_STP_CONDFETCH : F_STP_FETCH);
 }
 
 /*--------------------------------------------------------------------

bin/varnishtest/tests/b00039.vtc

@@ -13,6 +13,7 @@ varnish v1 -vcl+backend {
 		set beresp.ttl = 2s;
 		set beresp.grace = 20s;
 		set beresp.keep = 1m;
+		set beresp.http.was-304 = beresp.was_304;
 	}
 } -start
 
@@ -21,6 +22,7 @@ client c1 {
 	rxresp
 	expect resp.status == 200
 	expect resp.body == "Geoff Rules"
+	expect resp.http.was-304 == "false"
 } -run
 
 delay 3
@@ -30,6 +32,7 @@ client c1 {
 	rxresp
 	expect resp.status == 200
 	expect resp.body == "Geoff Rules"
+	expect resp.http.was-304 == "false"
 } -run
 
 client c1 {
@@ -37,4 +40,5 @@ client c1 {
 	rxresp
 	expect resp.status == 200
 	expect resp.body == "Geoff Rules"
+	expect resp.http.was-304 == "true"
 } -run

doc/sphinx/users-guide/vcl-built-in-subs.rst

@@ -277,6 +277,25 @@ vcl_backend_response
 Called after the response headers have been successfully retrieved from
 the backend.
 
+For a 304 response, varnish core code amends ``beresp`` before calling
+`vcl_backend_response`:
+
+* If the gzip status changed, ``Content-Encoding`` is unset and any
+  ``Etag`` is weakened
+
+* Any headers not present in the 304 response are copied from the
+  existing cache object. ``Content-Length`` is copied if present in
+  the existing cache object and discarded otherwise.
+
+* The status gets set to 200.
+
+`beresp.was_304` marks that this conditional response processing has
+happened.
+
+Note: Backend conditional requests are independend of client
+conditional requests, so clients may receive 304 responses no matter
+if a backend request was conditional.
+
 The `vcl_backend_response` subroutine may terminate with calling
 ``return()`` with one of the following keywords:
 

include/tbl/bo_flags.h

@@ -39,5 +39,6 @@ BO_FLAG(uncacheable,	0, 0, "")
 BO_FLAG(abandon,	0, 0, "")
 BO_FLAG(is_gzip,	0, 0, "")
 BO_FLAG(is_gunzip,	0, 0, "")
+BO_FLAG(was_304,	1, 0, "")
 
 /*lint -restore */

lib/libvcc/generate.py

@@ -461,6 +461,15 @@ sp_variables = [
 		cache.  Defaults to false.
 		"""
 	),
+	('beresp.was_304',
+		'BOOL',
+		( 'backend_response', 'backend_error'),
+		( ), """
+		Boolean. If this is a successful 304 response to a
+		backend conditional request refreshing an existing
+		cache object.
+		"""
+	),
 	('beresp.uncacheable',
 		'BOOL',
 		( 'backend_response', 'backend_error'),