Instrument and test adding Key header support

bin/varnishd/Makefile.am

@@ -31,6 +31,7 @@ varnishd_SOURCES = \
 	cache/cache_http1_fetch.c \
 	cache/cache_http1_fsm.c \
 	cache/cache_http1_proto.c \
+	cache/cache_key.c \
 	cache/cache_lck.c \
 	cache/cache_main.c \
 	cache/cache_mempool.c \

bin/varnishd/cache/cache.h

@@ -514,6 +514,7 @@ struct busyobj {
 	struct req		*req;
 
 	uint8_t			*vary;
+	uint8_t			*key;
 	unsigned		is_gzip;
 	unsigned		is_gunzip;
 
@@ -576,6 +577,7 @@ struct object {
 	struct ws		ws_o[1];
 
 	uint8_t			*vary;
+	uint8_t			*key;
 	unsigned		hits;
 	uint16_t		response;
 
@@ -633,6 +635,11 @@ struct req {
 	struct objhead		*hash_objhead;
 	struct busyobj		*busyobj;
 
+	/* Built Key string */
+	uint8_t			*key_b;
+	uint8_t			*key_l;
+	uint8_t			*key_e;
+
 	/* Built Vary string */
 	uint8_t			*vary_b;
 	uint8_t			*vary_l;

bin/varnishd/cache/cache_fetch.c

@@ -217,10 +217,13 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 static enum fetch_step
 vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 {
+	printf("    vbf_stp_fetch(wrk: %p, bo: %p)\n", wrk, bo);
 	struct http *hp, *hp2;
 	char *b;
 	uint16_t nhttp;
 	unsigned l;
+	struct vsb *key = NULL;
+	int keyl = 0;
 	struct vsb *vary = NULL;
 	int varyl = 0;
 	struct object *obj;
@@ -297,24 +300,30 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	/* Create Vary instructions */
 	if (bo->fetch_objcore->objhead != NULL) {
-		varyl = VRY_Create(bo, &vary);
-		if (varyl > 0) {
-			AN(vary);
-			assert(varyl == VSB_len(vary));
-			l += varyl;
-		} else if (varyl < 0) {
-			/*
-			 * Vary parse error
-			 * Complain about it, and make this a pass.
-			 */
-			VSLb(bo->vsl, SLT_Error,
-			    "Illegal 'Vary' header from backend, "
-			    "making this a pass.");
-			bo->uncacheable = 1;
-			AZ(vary);
-		} else
-			/* No vary */
-			AZ(vary);
+		keyl = KEY_Create(bo, &key);
+		if (keyl <= 0) {
+			varyl = VRY_Create(bo, &vary);
+			if (varyl > 0) {
+				AN(vary);
+				assert(varyl == VSB_len(vary));
+				l += varyl;
+			} else if (varyl < 0) {
+				/*
+				 * Vary parse error
+				 * Complain about it, and make this a pass.
+				 */
+				VSLb(bo->vsl, SLT_Error,
+				    "Illegal 'Vary' header from backend, "
+				    "making this a pass.");
+				bo->uncacheable = 1;
+				AZ(vary);
+			} else
+				/* No vary */
+				AZ(vary);
+		} else {
+			vary = NULL;
+			varyl = 0;
+		}
 	}
 
 	if (bo->uncacheable)
@@ -360,6 +369,14 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
 		obj->gziped = 1;
 
+	if (key != NULL) {
+		obj->key = (void *)WS_Copy(obj->http->ws,
+		    VSB_data(key), keyl);
+		AN(obj->key);
+		//KEY_Validate(obj->key);
+		VSB_delete(key);
+	}
+
 	if (vary != NULL) {
 		obj->vary = (void *)WS_Copy(obj->http->ws,
 		    VSB_data(vary), varyl);
@@ -417,6 +434,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	}
 
 	VBO_DerefBusyObj(wrk, &bo);	// XXX ?
+	printf("    vbf_stp_fetch(wrk: %p, bo: %p) = F_STP_DONE\n", wrk, bo);
 	return (F_STP_DONE);
 }
 
@@ -509,6 +527,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 void
 VBF_Fetch(struct worker *wrk, struct req *req)
 {
+	printf("  VBF_Fetch(wrk: %p, req: %p)\n", wrk, req);
 	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -536,4 +555,5 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 		printf("XXX\n");
 		(void)usleep(100000);
 	}
+	printf("  VBF_Fetch(wrk: %p, req: %p) = void\n", wrk, req);
 }

bin/varnishd/cache/cache_hash.c

@@ -305,6 +305,7 @@ enum lookup_e
 HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
     int wait_for_busy, int always_insert)
 {
+	printf("  HSH_Lookup(req: %p, *ocp: %p, *bocp: %p, wait_for_busy: %d, always_insert: %d)\n", req, *ocp, *bocp, wait_for_busy, always_insert);
 	struct worker *wrk;
 	struct objhead *oh;
 	struct objcore *oc;
@@ -353,6 +354,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 		*bocp = hsh_insert_busyobj(wrk, oh);
 		/* NB: no deref of objhead, new object inherits reference */
 		Lck_Unlock(&oh->mtx);
+		printf("  HSH_Lookup(req: %p, *ocp: %p, *bocp: %p, wait_for_busy: %d, always_insert: %d) = HSH_MISS\n", req, *ocp, *bocp, wait_for_busy, always_insert);
 		return (HSH_MISS);
 	}
 
@@ -373,6 +375,11 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 			if (req->hash_ignore_busy)
 				continue;
 
+			if (oc->busyobj != NULL &&
+			    oc->busyobj->key != NULL &&
+			    !KEY_Match(req, oc->busyobj->key))
+				continue;
+
 			if (oc->busyobj != NULL &&
 			    oc->busyobj->vary != NULL &&
 			    !VRY_Match(req, oc->busyobj->vary))
@@ -389,6 +396,8 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 			continue;
 		if (BAN_CheckObject(o, req))
 			continue;
+		if (o->key != NULL && !KEY_Match(req, o->key))
+			continue;
 		if (o->vary != NULL && !VRY_Match(req, o->vary))
 			continue;
 
@@ -402,6 +411,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 			if (!cache_param->obj_readonly && o->hits < INT_MAX)
 				o->hits++;
 			*ocp = oc;
+			printf("  HSH_Lookup(req: %p, *ocp: %p, *bocp: %p, wait_for_busy: %d, always_insert: %d) = HSH_HIT\n", req, *ocp, *bocp, wait_for_busy, always_insert);
 			return (HSH_HIT);
 		}
 		if (o->exp.entered > exp_entered) {
@@ -440,6 +450,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 		*bocp = hsh_insert_busyobj(wrk, oh);
 		/* NB: no deref of objhead, new object inherits reference */
 		Lck_Unlock(&oh->mtx);
+		printf("  HSH_Lookup(req: %p, *ocp: %p, *bocp: %p, wait_for_busy: %d, always_insert: %d) = HSH_MISS\n", req, *ocp, *bocp, wait_for_busy, always_insert);
 		return (HSH_MISS);
 	}
 
@@ -471,6 +482,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 	 */
 	req->hash_objhead = oh;
 	Lck_Unlock(&oh->mtx);
+	printf("  HSH_Lookup(req: %p, *ocp: %p, *bocp: %p, wait_for_busy: %d, always_insert: %d) = HSH_BUSY\n", req, *ocp, *bocp, wait_for_busy, always_insert);
 	return (HSH_BUSY);
 }
 

bin/varnishd/cache/cache_key.c

+#include "cache.h"
+
+int
+KEY_Create(struct busyobj *bo, struct vsb **psb)
+{
+	printf("      KEY_Create(bo: %p, psb: %p)\n", bo, *psb);
+	char *v;
+	struct vsb *sb;
+
+	if (!http_GetHdr(bo->beresp, "\4Key:", &v))
+		return (0);
+
+	sb = VSB_new_auto();
+	AN(sb);
+	VSB_printf(sb, "%s", v);
+	AZ(VSB_finish(sb));
+	*psb = sb;
+	printf("      KEY_Create(bo: %p, psb: %p) = %d\n", bo, *psb, VSB_len(sb));
+	return (VSB_len(sb));
+}
+
+void
+KEY_Prep(struct req *req)
+{
+	printf("  KEY_Prep(req: %p)\n", req);
+	printf("   - key_b %p\n", req->key_b);
+	printf("   - key_l %p\n", req->key_l);
+	printf("   - key_e %p\n", req->key_e);
+	req->key_b = req->vary_b;
+	req->key_l = req->vary_l;
+	req->key_e = req->vary_e;
+	printf("   - key_b %p\n", req->key_b);
+	printf("   - key_l %p\n", req->key_l);
+	printf("   - key_e %p\n", req->key_e);
+	printf("  KEY_Prep(req: %p) = void\n", req);
+}
+
+void
+KEY_Finish(struct req *req, struct busyobj *bo)
+{
+	printf("  KEY_Finish(req: %p, bo: %p)\n", req, bo);
+	printf("   - key_b %p\n", req->key_b);
+	printf("   - key_l %p\n", req->key_l);
+	printf("   - key_e %p\n", req->key_e);
+	if (bo != NULL) {
+		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+		VRY_Validate(req->key_b);
+		if (req->key_l != NULL) {
+			bo->key = WS_Copy(bo->ws,
+			    req->key_b, req->key_l - req->key_b);
+			AN(bo->key);
+			//VRY_Validate(bo->key);
+		} else
+			bo->key = NULL;
+	}
+	//WS_Release(req->ws, 0);
+	req->key_b = NULL;
+	req->key_l = NULL;
+	req->key_e = NULL;
+	printf("   - key_b %p\n", req->key_b);
+	printf("   - key_l %p\n", req->key_l);
+	printf("   - key_e %p\n", req->key_e);
+	printf("  KEY_Finish(req: %p, bo: %p) = void\n", req, bo);
+}
+
+int
+KEY_Match(struct req *req, const uint8_t *vary)
+{
+	printf("    KEY_Match()\n");
+	return 0;
+}

bin/varnishd/cache/cache_req_fsm.c

@@ -418,6 +418,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	AZ(req->busyobj);
 
 	VRY_Prep(req);
+	KEY_Prep(req);
 
 	AZ(req->objcore);
 	lr = HSH_Lookup(req, &oc, &boc,
@@ -492,6 +493,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	req->obj = o;
 
+	KEY_Finish(req, NULL);
 	VRY_Finish(req, NULL);
 
 	if (oc->flags & OC_F_PASS)
@@ -574,6 +576,7 @@ cnt_miss(struct worker *wrk, struct req *req)
 	bo = VBO_GetBusyObj(wrk, req);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	req->busyobj = bo;
+	KEY_Finish(req, bo);
 	VRY_Finish(req, bo);
 
 	VCL_miss_method(req->vcl, wrk, req, NULL, req->http->ws);
@@ -887,12 +890,14 @@ cnt_purge(struct worker *wrk, struct req *req)
 	AZ(req->busyobj);
 
 	VRY_Prep(req);
+	KEY_Prep(req);
 
 	AZ(req->objcore);
 	lr = HSH_Lookup(req, &oc, &boc, 1, 1);
 	assert (lr == HSH_MISS);
 	AZ(oc);
 	CHECK_OBJ_NOTNULL(boc, OBJCORE_MAGIC);
+	KEY_Finish(req, NULL);
 	VRY_Finish(req, NULL);
 
 	HSH_Purge(wrk, boc->objhead, 0, 0);
@@ -925,6 +930,7 @@ cnt_diag(struct req *req, const char *state)
 enum req_fsm_nxt
 CNT_Request(struct worker *wrk, struct req *req)
 {
+	printf("CNT_Request(wrk: %p, req: %p)\n", wrk, req);
 	enum req_fsm_nxt nxt;
 	struct storage *st;
 
@@ -1007,6 +1013,7 @@ CNT_Request(struct worker *wrk, struct req *req)
 	req->wrk = NULL;
 
 	assert(WRW_IsReleased(wrk));
+	printf("CNT_Request(wrk: %p, req: %p) = %d\n", wrk, req, nxt);
 	return (nxt);
 }
 

bin/varnishd/cache/cache_session.c

@@ -344,6 +344,7 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now)
 struct req *
 SES_GetReq(struct worker *wrk, struct sess *sp)
 {
+	printf("SES_GetReq(wrk: %p, sp: %p)\n", wrk, sp);
 	struct sesspool *pp;
 	struct req *req;
 	uint16_t nhttp;
@@ -401,6 +402,12 @@ SES_GetReq(struct worker *wrk, struct sess *sp)
 
 	VTAILQ_INIT(&req->body);
 
+	printf(" - Start:   %p\n", req->ws->s);
+	printf(" - Free:    %p\n", req->ws->f);
+	printf(" - Reserve: %p\n", req->ws->r);
+	printf(" - End:     %p\n", req->ws->e);
+	printf("            %dKB\n", (req->ws->e - req->ws->s) / 1024);
+	printf("SES_GetReq(wrk: %p, sp: %p) = %p\n", wrk, sp, req);
 	return (req);
 }
 

bin/varnishd/cache/cache_vary.c

@@ -73,6 +73,7 @@
 int
 VRY_Create(struct busyobj *bo, struct vsb **psb)
 {
+	printf("      VRY_Create(bo: %p, psb: %p)\n", bo, *psb);
 	char *v, *p, *q, *h, *e;
 	struct vsb *sb, *sbh;
 	unsigned l;
@@ -165,6 +166,7 @@ VRY_Create(struct busyobj *bo, struct vsb **psb)
 	VSB_delete(sbh);
 	AZ(VSB_finish(sb));
 	*psb = sb;
+	printf("      VRY_Create(bo: %p, psb: %p) = %d\n", bo, *psb, VSB_len(sb));
 	return (VSB_len(sb));
 }
 
@@ -218,6 +220,10 @@ vry_cmp(const uint8_t *v1, const uint8_t *v2)
 void
 VRY_Prep(struct req *req)
 {
+	printf("  VRY_Prep(req: %p)\n", req);
+	printf("   - vary_b %p\n", req->vary_b);
+	printf("   - vary_l %p\n", req->vary_l);
+	printf("   - vary_e %p\n", req->vary_e);
 	if (req->hash_objhead == NULL) {
 		/* Not a waiting list return */
 		AZ(req->vary_b);
@@ -231,6 +237,10 @@ VRY_Prep(struct req *req)
 	req->vary_e = (void*)req->ws->r;
 	if (req->vary_b + 2 < req->vary_e)
 		req->vary_b[2] = '\0';
+	printf("   - vary_b %p\n", req->vary_b);
+	printf("   - vary_l %p\n", req->vary_l);
+	printf("   - vary_e %p\n", req->vary_e);
+	printf("  VRY_Prep(req: %p) = void\n", req);
 }
 
 /**********************************************************************
@@ -240,7 +250,10 @@ VRY_Prep(struct req *req)
 void
 VRY_Finish(struct req *req, struct busyobj *bo)
 {
-
+	printf("  VRY_Finish(req: %p, bo: %p)\n", req, bo);
+	printf("   - vary_b %p\n", req->vary_b);
+	printf("   - vary_l %p\n", req->vary_l);
+	printf("   - vary_e %p\n", req->vary_e);
 	if (bo != NULL) {
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 		VRY_Validate(req->vary_b);
@@ -256,6 +269,10 @@ VRY_Finish(struct req *req, struct busyobj *bo)
 	req->vary_b = NULL;
 	req->vary_l = NULL;
 	req->vary_e = NULL;
+	printf("   - vary_b %p\n", req->vary_b);
+	printf("   - vary_l %p\n", req->vary_l);
+	printf("   - vary_e %p\n", req->vary_e);
+	printf("  VRY_Finish(req: %p, bo: %p) = void\n", req, bo);
 }
 
 /**********************************************************************
@@ -268,7 +285,9 @@ VRY_Finish(struct req *req, struct busyobj *bo)
 int
 VRY_Match(struct req *req, const uint8_t *vary)
 {
+	printf("    VRY_Match(req: %p, vary: %p)\n", req, vary);
 	uint8_t *vsp = req->vary_b;
+	printf("     - vsp: %p\n", vsp);
 	char *h, *e;
 	unsigned lh, ln;
 	int i, oflo = 0;