Change the way we calculate the hash key for the cache.

This lifts internal restrictions on the hashkey production,
but does not change how things work from a VCL perspective.

bin/varnishd/cache/cache_hash.c

@@ -183,19 +183,6 @@ HSH_DeleteObjHead(const struct worker *wrk, struct objhead *oh)
 	FREE_OBJ(oh);
 }
 
-void
-HSH_AddString(struct req *req, void *ctx, const char *str)
-{
-
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	AN(ctx);
-	if (str != NULL) {
-		VSHA256_Update(ctx, str, strlen(str));
-		VSLb(req->vsl, SLT_Hash, "%s", str);
-	} else
-		VSHA256_Update(ctx, &str, 1);
-}
-
 /*---------------------------------------------------------------------
  * This is a debugging hack to enable testing of boundary conditions
  * in the hash algorithm.

bin/varnishd/cache/cache_objhead.h

@@ -71,7 +71,6 @@ int HSH_DerefObjCore(struct worker *, struct objcore **, int rushmax);
 
 enum lookup_e HSH_Lookup(struct req *, struct objcore **, struct objcore **);
 void HSH_Ref(struct objcore *o);
-void HSH_AddString(struct req *, void *ctx, const char *str);
 unsigned HSH_Purge(struct worker *, struct objhead *, vtim_real ttl_now,
     vtim_dur ttl, vtim_dur grace, vtim_dur keep);
 struct objcore *HSH_Private(const struct worker *wrk);

bin/varnishd/cache/cache_req_fsm.c

@@ -50,7 +50,6 @@
 #include "storage/storage.h"
 #include "common/heritage.h"
 #include "vcl.h"
-#include "vsha256.h"
 #include "vtim.h"
 
 #define REQ_STEPS \
@@ -886,7 +885,6 @@ static enum req_fsm_nxt v_matchproto_(req_state_f)
 cnt_recv(struct worker *wrk, struct req *req)
 {
 	unsigned recv_handling;
-	struct VSHA256Context sha256ctx;
 	const char *ci, *cp, *endpname;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -957,13 +955,12 @@ cnt_recv(struct worker *wrk, struct req *req)
 		}
 	}
 
-	VSHA256_Init(&sha256ctx);
-	VCL_hash_method(req->vcl, wrk, req, NULL, &sha256ctx);
+	memset(req->digest, 0, sizeof req->digest);
+	VCL_hash_method(req->vcl, wrk, req, NULL, NULL);
 	if (wrk->handling == VCL_RET_FAIL)
 		recv_handling = wrk->handling;
 	else
 		assert(wrk->handling == VCL_RET_LOOKUP);
-	VSHA256_Final(req->digest, &sha256ctx);
 
 	switch (recv_handling) {
 	case VCL_RET_VCL:

bin/varnishd/cache/cache_vrt.c

@@ -689,19 +689,29 @@ VRT_fail(VRT_CTX, const char *fmt, ...)
 VCL_VOID
 VRT_hashdata(VRT_CTX, VCL_STRANDS s)
 {
+	struct VSHA256Context sha256ctx;
 	int i;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-	AN(ctx->specific);
+	AZ(ctx->specific);
+	VSHA256_Init(&sha256ctx);
+	VSHA256_Update(&sha256ctx, ctx->req->digest, sizeof ctx->req->digest);
 	AN(s);
-	for (i = 0; i < s->n; i++)
-		HSH_AddString(ctx->req, ctx->specific, s->p[i]);
+	for (i = 0; i < s->n; i++) {
+		if (s->p[i] != NULL) {
+			VSHA256_Update(&sha256ctx, s->p[i], strlen(s->p[i]));
+			VSLb(ctx->req->vsl, SLT_Hash, "%s", s->p[i]);
+		} else {
+			VSHA256_Update(&sha256ctx, "", 1);
+		}
+	}
 	/*
 	 * Add a 'field-separator' to make it more difficult to
 	 * manipulate the hash.
 	 */
-	HSH_AddString(ctx->req, ctx->specific, NULL);
+	VSHA256_Update(&sha256ctx, "", 1);
+	VSHA256_Final(ctx->req->digest, &sha256ctx);
 }
 
 /*--------------------------------------------------------------------*/

bin/varnishtest/tests/b00051.vtc

@@ -22,5 +22,5 @@ client c1 {
 	rxresp
 	expect resp.http.req_hash ~ "[[:xdigit:]]{64}"
 	expect resp.http.req_hash == resp.http.bereq_hash
-	expect resp.http.req_hash-sf == ":3k0f0yRKtKt7akzkyNsTGSDOJAZOQowTwKWhu5+kIu0=:"
+	expect resp.http.req_hash-sf == ":0jkH41nfmD0PRFsKpM1m7ucOApnxFc62B//mQWxOnmQ=:"
 } -run

bin/varnishtest/tests/d00020.vtc

@@ -218,9 +218,9 @@ client c1 {
 client c2 {
 	txreq -url /b/def
 	rxresp
-	expect resp.http.hash		== "93d1c4ad76396c91dd97fa310f7f26445332662c89393dbeeb77fe49f9111ee4"
+	expect resp.http.hash		== "2e03d4fee2722154a84036faa3e4b6851e003830eadbe0d2874e3df09c8efe55"
 	expect resp.http.by		== "HASH"
-	expect resp.http.key		-eq 0x93d1c4ad
+	expect resp.http.key		-eq 0x2e03d4fe
 	expect resp.http.alt		== 0
 	expect resp.http.warmup	== "-1.000"
 	expect resp.http.rampup	== "true"
@@ -228,9 +228,9 @@ client c2 {
 
 	txreq -url /b/hash
 	rxresp
-	expect resp.http.hash		== "e47da20ea4db49d4f22acdadc69f02f445002be520a2865cd3351272add62540"
+	expect resp.http.hash		== "49f33e9019891091d3dcf6edab6d9433b756678bdf5202dd41b8a667c89887b1"
 	expect resp.http.by		== "HASH"
-	expect resp.http.key		-eq 0xe47da20e
+	expect resp.http.key		-eq 0x49f33e90
 	expect resp.http.alt		== 1
 	expect resp.http.warmup	== "-1.000"
 	expect resp.http.rampup	== "true"
@@ -268,9 +268,9 @@ client c2 {
 client c3 {
 	txreq -url /b/c/hash/def
 	rxresp
-	expect resp.http.hash		== "df9a465f8a0455c334b24c1638d3adda0f6e64fbe759029ab83602e3b9138884"
+	expect resp.http.hash		== "dd70dcbbf385c398ee3b53a849f12d0d846fc21292349fb45d37b2d9d8eca25e"
 	expect resp.http.by		== "HASH"
-	expect resp.http.key		-eq 0xdf9a465f
+	expect resp.http.key		-eq 0xdd70dcbb
 	expect resp.http.alt		== 7
 	expect resp.http.warmup	== "-1.000"
 	expect resp.http.rampup	== "true"
@@ -278,9 +278,9 @@ client c3 {
 
 	txreq -url /b/c/hash/hash
 	rxresp
-	expect resp.http.hash		== "0eb35bc1fab5aad5902fd1bac86540bd13d43aa31c6c46f54e776b43392e66e6"
+	expect resp.http.hash		== "41d09b9877cd0ac0eab888359b0ad54f0bf41da0ac03dc1b8ae12aff18465a8d"
 	expect resp.http.by		== "HASH"
-	expect resp.http.key		-eq 0x0eb35bc1
+	expect resp.http.key		-eq 0x41d09b98
 	expect resp.http.alt		== 8
 	expect resp.http.warmup	== "-1.000"
 	expect resp.http.rampup	== "true"
@@ -288,9 +288,9 @@ client c3 {
 
 	txreq -url /b/c/hash/url
 	rxresp
-	expect resp.http.hash		== "1eb67b701ea07151cac5bea1f11b6267b9de15a3ff83cec995590480cbc2c750"
+	expect resp.http.hash		== "dcac849e02b3322f5fd3dddf9b9f5fc26d295733e6f1c51b190dfb7239a56e28"
 	expect resp.http.by		== "HASH"
-	expect resp.http.key		-eq 0x1eb67b70
+	expect resp.http.key		-eq 0xdcac849e
 	expect resp.http.alt		== 9
 	expect resp.http.warmup	== "0.500"
 	expect resp.http.rampup	== "true"
@@ -298,9 +298,9 @@ client c3 {
 
 	txreq -url /b/c/hash/key
 	rxresp
-	expect resp.http.hash		== "a11b617e21aa7db22b6205d7612002e595b1b00d8c11602017f65456a1be3a35"
+	expect resp.http.hash		== "112393761506e85f0c700a5d669a48b54001c870eb2e8d95f4d2f6fdccbe80a3"
 	expect resp.http.by		== "HASH"
-	expect resp.http.key		-eq 0xa11b617e
+	expect resp.http.key		-eq 0x11239376
 	expect resp.http.alt		== 10
 	expect resp.http.warmup	== "-1.000"
 	expect resp.http.rampup	== "false"
@@ -308,9 +308,9 @@ client c3 {
 
 	txreq -url /b/c/hash/blob
 	rxresp
-	expect resp.http.hash		== "d7eecc0ac83e1727332dcd8c7c8ae9f3114123abb2bf7e3fb15ecea8c84bb239"
+	expect resp.http.hash		== "5ef050c1185ac02a66d9f79703b8cd5f0636abf3b1f15b9f22e0fe64df985d28"
 	expect resp.http.by		== "HASH"
-	expect resp.http.key		-eq 0xd7eecc0a
+	expect resp.http.key		-eq 0x5ef050c1
 	expect resp.http.alt		== 11
 	expect resp.http.warmup	== "-1.000"
 	expect resp.http.rampup	== "true"

include/vrt.h

@@ -53,6 +53,7 @@
  * binary/load-time compatible, increment MAJOR version
  *
  * 13.0 (2021-03-15)
+ *	VRT_hashdata() produces different hash-keys.
  *	Move VRT_synth_page() to deprecated status
  *	Add VRT_synth_strands() and VRT_synth_blob()
  *	struct vrt_type now produced by generate.py