Add the skeleton of a Conditional Fetch facility.

This comes with a big apology to Geoff Simmons:  I should have
tackled Conditional Fetch before even thinking about streaming
and therefore this has taken much longer than it should.

bin/varnishd/cache/cache.h

@@ -544,9 +544,10 @@ struct busyobj {
 	struct http		*bereq0;
 	struct http		*bereq;
 	struct http		*beresp;
-	struct objcore		*ims_objcore;
+	struct object		*ims_obj;
 	struct objcore		*fetch_objcore;
 	struct object		*fetch_obj;
+
 	struct exp		exp;
 	struct http_conn	htc;
 
@@ -918,7 +919,7 @@ enum vbf_fetch_mode_e {
 	VBF_BACKGROUND = 2,
 };
 void VBF_Fetch(struct worker *wrk, struct req *req,
-    struct objcore *oc, struct objcore *oldoc, enum vbf_fetch_mode_e);
+    struct objcore *oc, struct object *oldobj, enum vbf_fetch_mode_e);
 
 /* cache_fetch_proc.c */
 struct storage *VFP_GetStorage(struct busyobj *, ssize_t sz);
@@ -1127,6 +1128,7 @@ void VRY_Validate(const uint8_t *vary);
 void VRY_Prep(struct req *);
 enum vry_finish_flag { KEEP, DISCARD };
 void VRY_Finish(struct req *req, enum vry_finish_flag);
+unsigned VRY_Len(const uint8_t *);
 
 /* cache_vcl.c */
 void VCL_Init(void);

bin/varnishd/cache/cache_fetch.c

@@ -60,6 +60,7 @@ vbf_release_req(struct busyobj *bo)
 static enum fetch_step
 vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 {
+	char *p;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -86,6 +87,17 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 			http_SetHeader(bo->bereq0, "Accept-Encoding: gzip");
 		}
 	}
+	if (bo->ims_obj != NULL) {
+		if (http_GetHdr(bo->ims_obj->http, H_Last_Modified, &p)) {
+			http_PrintfHeader(bo->bereq0,
+			    "If-Modified-Since: %s", p);
+		} else if (http_GetHdr(bo->ims_obj->http, H_ETag, &p)) {
+			http_PrintfHeader(bo->bereq0,
+			    "If-None-Match: %s", p);
+		} else {
+			WRONG("Shouldn't have bo->ims_obj");
+		}
+	}
 
 	return (F_STP_STARTFETCH);
 }
@@ -131,7 +143,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 static enum fetch_step
 vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 {
-	int i;
+	int i, do_ims;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -203,6 +215,14 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 
 	AZ(bo->do_esi);
 
+	if (bo->ims_obj != NULL && bo->beresp->status == 304) {
+		bo->beresp->status = 200;
+		http_PrintfHeader(bo->beresp, "Content-Length: %jd",
+		    bo->ims_obj->len);
+		do_ims = 1;
+	} else 
+		do_ims = 0;
+
 	VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws);
 
 	if (bo->do_esi)
@@ -211,7 +231,7 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 		bo->fetch_objcore->flags |= OC_F_PASS;
 
 	if (wrk->handling == VCL_RET_DELIVER)
-		return (F_STP_FETCH);
+		return (do_ims ? F_STP_CONDFETCH : F_STP_FETCH);
 	if (wrk->handling == VCL_RET_RETRY) {
 		assert(bo->state == BOS_REQ_DONE);
 		bo->retries++;
@@ -239,7 +259,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	int varyl = 0;
 	struct object *obj;
 
-
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
@@ -438,6 +457,99 @@ vbf_stp_done(void)
 	WRONG("Just plain wrong");
 }
 
+/*--------------------------------------------------------------------
+ */
+
+static enum fetch_step
+vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
+{
+	unsigned l;
+	uint16_t nhttp;
+	struct object *obj;
+	struct objiter *oi;
+	void *sp;
+	ssize_t sl, al, tl;
+	struct storage *st;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+
+	l = 0;
+	if (bo->ims_obj->vary != NULL)
+		l += VRY_Len(bo->ims_obj->vary);
+	l += http_EstimateWS(bo->ims_obj->http, 0, &nhttp);
+
+	bo->stats = &wrk->stats;
+	obj = STV_NewObject(bo, bo->storage_hint, l, nhttp);
+	if (obj == NULL) {
+		(void)VFP_Error(bo, "Could not get storage");
+		VDI_CloseFd(&bo->vbc);
+		return (F_STP_DONE);
+	}
+	bo->stats = NULL;
+
+	AZ(bo->fetch_obj);
+	bo->fetch_obj = obj;
+
+	obj->gziped = bo->ims_obj->gziped;
+	obj->gzip_start = bo->ims_obj->gzip_start;
+	obj->gzip_last = bo->ims_obj->gzip_last;
+	obj->gzip_stop = bo->ims_obj->gzip_stop;
+
+	/* XXX: ESI */
+
+	if (bo->ims_obj->vary != NULL)
+		obj->vary = (void *)WS_Copy(obj->http->ws,
+		    bo->ims_obj->vary, VRY_Len(bo->ims_obj->vary));
+
+	obj->vxid = bo->vsl->wid;
+
+	obj->http->logtag = HTTP_Obj;
+	/* XXX: we should have our own HTTP_A_CONDFETCH */
+	http_FilterResp(bo->ims_obj->http, obj->http, HTTPH_A_INS);
+	http_CopyHome(obj->http);
+
+
+	if (!(bo->fetch_obj->objcore->flags & OC_F_PRIVATE)) {
+		EXP_Insert(obj);
+		AN(obj->objcore->ban);
+	}
+
+	AZ(bo->ws_o->overflow);
+	VBO_setstate(bo, BOS_FETCHING);
+	HSH_Unbusy(&wrk->stats, obj->objcore);
+
+	st = NULL;
+	al = 0;
+
+	oi = ObjIterBegin(wrk, bo->ims_obj);
+	while (1 == ObjIter(oi, &sp, &sl)) {
+		while (sl > 0) {
+			if (st == NULL) {
+				st = VFP_GetStorage(bo, bo->ims_obj->len - al);
+				XXXAN(st);
+			}
+			tl = sl;
+			if (tl > st->space - st->len)
+				tl = st->space - st->len;
+			memcpy(st->ptr + st->len, sp, tl);
+			st->len += tl;
+			al += tl;
+			sp = (char *)sp + tl;
+			sl -= tl;
+			VBO_extend(bo, al);
+			if (st->len == st->space)
+				st = NULL;
+		}
+	}
+	assert(al == bo->ims_obj->len);
+	assert(obj->len == al);
+	if (bo->state != BOS_FAILED)
+		VBO_setstate(bo, BOS_FINISHED);
+	HSH_Complete(obj->objcore);
+	return (F_STP_DONE);
+}
+
 /*--------------------------------------------------------------------
  */
 
@@ -490,8 +602,8 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	if (bo->state == BOS_FAILED)
 		assert(bo->fetch_objcore->flags & OC_F_FAILED);
 
-	if (bo->ims_objcore != NULL)
-		(void)HSH_DerefObjCore(&wrk->stats, &bo->ims_objcore);
+	if (bo->ims_obj != NULL)
+		(void)HSH_DerefObj(&wrk->stats, &bo->ims_obj);
 
 	VBO_DerefBusyObj(wrk, &bo);
 	THR_SetBusyobj(NULL);
@@ -502,14 +614,14 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 
 void
 VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc,
-    struct objcore *oldoc, enum vbf_fetch_mode_e mode)
+    struct object *oldobj, enum vbf_fetch_mode_e mode)
 {
 	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	CHECK_OBJ_ORNULL(oldoc, OBJCORE_MAGIC);
+	CHECK_OBJ_ORNULL(oldobj, OBJECT_MAGIC);
 
 	bo = VBO_GetBusyObj(wrk, req);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -528,9 +640,12 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc,
 	HSH_Ref(oc);
 	bo->fetch_objcore = oc;
 
-	if (oldoc != NULL) {
-		HSH_Ref(oldoc);
-		bo->ims_objcore = oldoc;
+	if (oldobj != NULL) {
+		if (http_GetHdr(oldobj->http, H_Last_Modified, NULL) ||
+		   http_GetHdr(oldobj->http, H_ETag, NULL)) {
+			HSH_Ref(oldobj->objcore);
+			bo->ims_obj = oldobj;
+		}
 	}
 
 	AZ(bo->req);

bin/varnishd/cache/cache_req_fsm.c

@@ -416,7 +416,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	switch (wrk->handling) {
 	case VCL_RET_DELIVER:
 		if (boc != NULL) {
-			VBF_Fetch(wrk, req, boc, oc, VBF_BACKGROUND);
+			VBF_Fetch(wrk, req, boc, o, VBF_BACKGROUND);
 		} else {
 			(void)HTTP1_DiscardReqBody(req);// XXX: handle err
 		}

bin/varnishd/cache/cache_vary.c

@@ -173,8 +173,8 @@ VRY_Create(struct busyobj *bo, struct vsb **psb)
 /*
  * Find length of a vary entry
  */
-static unsigned
-vry_len(const uint8_t *p)
+unsigned
+VRY_Len(const uint8_t *p)
 {
 	unsigned l = vbe16dec(p);
 
@@ -189,7 +189,7 @@ vry_cmp(const uint8_t *v1, const uint8_t *v2)
 {
 	unsigned retval = 0;
 
-	if (!memcmp(v1, v2, vry_len(v1))) {
+	if (!memcmp(v1, v2, VRY_Len(v1))) {
 		/* Same same */
 		retval = 0;
 	} else if (memcmp(v1 + 2, v2 + 2, v1[2] + 2)) {
@@ -331,8 +331,8 @@ VRY_Match(struct req *req, const uint8_t *vary)
 		}
 		if (i == 0) {
 			/* Same header, same contents*/
-			vsp += vry_len(vsp);
-			vary += vry_len(vary);
+			vsp += VRY_Len(vsp);
+			vary += VRY_Len(vary);
 		} else if (i == 2) {
 			/* Same header, different contents, cannot match */
 			return (0);
@@ -358,6 +358,6 @@ VRY_Validate(const uint8_t *vary)
 
 	while (vary[2] != 0) {
 		assert(strlen((const char*)vary+3) == vary[2]);
-		vary += vry_len(vary);
+		vary += VRY_Len(vary);
 	}
 }

include/tbl/steps.h

@@ -52,6 +52,7 @@ REQ_STEP(error,		ERROR,		(wrk, req))
 FETCH_STEP(mkbereq,	MKBEREQ,	(wrk, bo))
 FETCH_STEP(startfetch,	STARTFETCH,	(wrk, bo))
 FETCH_STEP(fetchhdr,	FETCHHDR,	(wrk, bo))
+FETCH_STEP(condfetch,	CONDFETCH,	(wrk, bo))
 FETCH_STEP(fetch,	FETCH,		(wrk, bo))
 FETCH_STEP(done,	DONE,		())
 #endif