Move 'age' and 'entered' into struct exp.

Simplify the move from sp->wrk to sp->obj accordingly.

Add a diagram that explains how ttl, grace & keep related to
each other.

Fix VCL's obj.ttl variable to appear to be relative to "now" rather
than obj->entered. (#956)

Also log SLT_TTL when we change obj.grace and obj.keep

Make SLT_TTL always have the same format:
	XID
	"RFC" or "VCL"
	obj.ttl
	obj.grace
	obj.keep
	obj.entered
	obj.age
In addition "RFC" has:
	obj.date
	obj.expires
	obj.max-age

Fixes	#956

Thanks to:	DocWilco

bin/varnishd/cache.h

@@ -242,6 +242,8 @@ struct exp {
 	double			ttl;
 	double			grace;
 	double			keep;
+	double			age;
+	double			entered;
 };
 
 /*--------------------------------------------------------------------*/
@@ -309,8 +311,6 @@ struct worker {
 	struct http		*beresp;
 	struct http		*resp;
 
-	double			age;
-	double			entered;
 	struct exp		exp;
 
 	/* This is only here so VRT can find it */
@@ -507,8 +507,6 @@ struct object {
 
 	ssize_t			len;
 
-	double			age;
-	double			entered;
 	struct exp		exp;
 
 	double			last_modified;

bin/varnishd/cache_center.c

@@ -428,7 +428,7 @@ cnt_error(struct sess *sp)
 		}
 		AN(sp->obj);
 		sp->obj->xid = sp->xid;
-		sp->obj->entered = sp->t_req;
+		sp->obj->exp.entered = sp->t_req;
 	} else {
 		/* XXX: Null the headers ? */
 	}
@@ -554,9 +554,8 @@ cnt_fetch(struct sess *sp)
 		/*
 		 * What does RFC2616 think about TTL ?
 		 */
-		sp->wrk->entered = TIM_real();
-		sp->wrk->age = 0;
 		EXP_Clr(&sp->wrk->exp);
+		sp->wrk->exp.entered = TIM_real();
 		sp->wrk->exp.ttl = RFC2616_Ttl(sp);
 
 		/* pass from vclrecv{} has negative TTL */
@@ -782,8 +781,6 @@ cnt_fetchbody(struct sess *sp)
 
 	sp->obj->xid = sp->xid;
 	sp->obj->response = sp->err_code;
-	sp->obj->age = sp->wrk->age;
-	sp->obj->entered = sp->wrk->entered;
 	WS_Assert(sp->obj->ws_o);
 
 	/* Filter into object */
@@ -799,7 +796,7 @@ cnt_fetchbody(struct sess *sp)
 	if (http_GetHdr(hp, H_Last_Modified, &b))
 		sp->obj->last_modified = TIM_parse(b);
 	else
-		sp->obj->last_modified = floor(sp->wrk->entered);
+		sp->obj->last_modified = floor(sp->wrk->exp.entered);
 
 	assert(WRW_IsReleased(sp->wrk));
 

bin/varnishd/cache_expire.c

@@ -35,6 +35,18 @@
  *
  * We hold a single object reference for both data structures.
  *
+ * An attempted overview:
+ *
+ *	                        EXP_Ttl()      EXP_Grace()   EXP_Keep()
+ *				   |                |            |
+ *      entered                    v                v            |
+ *         |                       +--------------->+            |
+ *         v                       |      grace                  |
+ *         +---------------------->+                             |
+ *                  ttl            |                             v
+ *                                 +---------------------------->+
+ *                                     keep
+ *				   
  */
 
 #include "config.h"
@@ -68,6 +80,8 @@ EXP_Clr(struct exp *e)
 	e->ttl = -1;
 	e->grace = -1;
 	e->keep = -1;
+	e->age = 0;
+	e->entered = 0;
 }
 
 #define EXP_ACCESS(fld, low_val, extra)				\
@@ -93,8 +107,8 @@ EXP_ACCESS(grace, 0., )
 EXP_ACCESS(keep, 0.,)
 
 /*--------------------------------------------------------------------
- * Calculate when an object is out of ttl or grace, possibly constrained
- * by per-session limits.
+ * Calculate an objects effective keep, grace or ttl time, suitably
+ * adjusted for defaults and by per-session limits.
  */
 
 static double
@@ -131,7 +145,7 @@ EXP_Ttl(const struct sess *sp, const struct object *o)
 	r = o->exp.ttl;
 	if (sp != NULL && sp->exp.ttl > 0. && sp->exp.ttl < r)
 		r = sp->exp.ttl;
-	return (o->entered + r);
+	return (o->exp.entered + r);
 }
 
 /*--------------------------------------------------------------------
@@ -214,8 +228,8 @@ EXP_Insert(struct object *o)
 	AssertObjBusy(o);
 	HSH_Ref(oc);
 
-	assert(o->entered != 0 && !isnan(o->entered));
-	o->last_lru = o->entered;
+	assert(o->exp.entered != 0 && !isnan(o->exp.entered));
+	o->last_lru = o->exp.entered;
 
 	lru = oc_getlru(oc);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);

bin/varnishd/cache_hash.c

@@ -383,9 +383,9 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 		 */
 		if (EXP_Grace(sp, o) >= sp->t_req) {
 			if (grace_oc == NULL ||
-			    grace_ttl < o->entered + o->exp.ttl) {
+			    grace_ttl < o->exp.entered + o->exp.ttl) {
 				grace_oc = oc;
-				grace_ttl = o->entered + o->exp.ttl;
+				grace_ttl = o->exp.entered + o->exp.ttl;
 			}
 		}
 	}

bin/varnishd/cache_response.c

@@ -226,7 +226,7 @@ RES_BuildHttp(struct sess *sp)
 		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
 		    "X-Varnish: %u", sp->xid);
 	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Age: %.0f",
-	    sp->obj->age + sp->t_resp - sp->obj->entered);
+	    sp->obj->exp.age + sp->t_resp - sp->obj->exp.entered);
 	http_SetHeader(sp->wrk, sp->fd, sp->wrk->resp, "Via: 1.1 varnish");
 	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Connection: %s",
 	    sp->doclose ? "close" : "keep-alive");

bin/varnishd/cache_vrt_var.c

@@ -360,15 +360,20 @@ VRT_r_req_restarts(const struct sess *sp)
 	return (sp->restarts);
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * NB: TTL is relative to when object was created, whereas grace and
+ * keep are relative to ttl.
+ */
 
-#define VRT_DO_EXP(which, exp, fld, extra)			\
+#define VRT_DO_EXP(which, exp, fld, offset, extra)		\
 								\
 void __match_proto__()						\
 VRT_l_##which##_##fld(struct sess *sp, double a)		\
 {								\
 								\
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
+	if (a > 0.)						\
+		a += offset;					\
 	EXP_Set_##fld(&exp, a);					\
 	extra;							\
 }								\
@@ -378,21 +383,37 @@ VRT_r_##which##_##fld(struct sess *sp)				\
 {								\
 								\
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	return(EXP_Get_##fld(&exp));				\
-}
-
-VRT_DO_EXP(req, sp->exp, ttl, )
-VRT_DO_EXP(req, sp->exp, grace, )
-VRT_DO_EXP(req, sp->exp, keep, )
-VRT_DO_EXP(obj, sp->obj->exp, grace, EXP_Rearm(sp->obj))
-VRT_DO_EXP(obj, sp->obj->exp, ttl,
-	   EXP_Rearm(sp->obj);
-	   WSP(sp, SLT_TTL, "%u VCL %.0f %.0f", sp->obj->xid, a, sp->t_req))
-VRT_DO_EXP(obj, sp->obj->exp, keep, EXP_Rearm(sp->obj))
-VRT_DO_EXP(beresp, sp->wrk->exp, grace, )
-VRT_DO_EXP(beresp, sp->wrk->exp, ttl,
-	   WSP(sp, SLT_TTL, "%u VCL %.0f %.0f", sp->xid, a, sp->t_req))
-VRT_DO_EXP(beresp, sp->wrk->exp, keep, )
+	return(EXP_Get_##fld(&exp) - offset);			\
+}
+
+static void
+vrt_wsp_exp(const struct sess *sp, unsigned xid, const struct exp *e)
+{
+	WSP(sp, SLT_TTL, "%u VCL %.0f %.0f %.0f %.0f %.0f",
+	    xid, e->ttl - (sp->t_req - e->entered), e->grace, e->keep,
+	    sp->t_req, e->age + (sp->t_req - e->entered));
+}
+
+VRT_DO_EXP(req, sp->exp, ttl, 0, )
+VRT_DO_EXP(req, sp->exp, grace, 0, )
+VRT_DO_EXP(req, sp->exp, keep, 0, )
+
+VRT_DO_EXP(obj, sp->obj->exp, grace, 0,
+   EXP_Rearm(sp->obj);
+   vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
+VRT_DO_EXP(obj, sp->obj->exp, ttl, (sp->t_req - sp->obj->exp.entered),
+   EXP_Rearm(sp->obj);
+   vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
+VRT_DO_EXP(obj, sp->obj->exp, keep, 0,
+   EXP_Rearm(sp->obj);
+   vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
+
+VRT_DO_EXP(beresp, sp->wrk->exp, grace, 0,
+   vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
+VRT_DO_EXP(beresp, sp->wrk->exp, ttl, 0,
+   vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
+VRT_DO_EXP(beresp, sp->wrk->exp, keep, 0,
+   vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
 
 /*--------------------------------------------------------------------
  * req.xid

bin/varnishd/rfc2616.c

@@ -76,7 +76,7 @@ RFC2616_Ttl(const struct sess *sp)
 
 	hp = sp->wrk->beresp;
 
-	assert(sp->wrk->entered != 0.0 && !isnan(sp->wrk->entered));
+	assert(sp->wrk->exp.entered != 0.0 && !isnan(sp->wrk->exp.entered));
 	/* If all else fails, cache using default ttl */
 	ttl = params->default_ttl;
 
@@ -116,7 +116,7 @@ RFC2616_Ttl(const struct sess *sp)
 				max_age = strtoul(p, NULL, 0);
 			if (http_GetHdr(hp, H_Age, &p)) {
 				age = strtoul(p, NULL, 0);
-				sp->wrk->age = age;
+				sp->wrk->exp.age = age;
 			}
 
 			if (age > max_age)
@@ -145,16 +145,16 @@ RFC2616_Ttl(const struct sess *sp)
 		}
 
 		if (h_date == 0 ||
-		    fabs(h_date - sp->wrk->entered) < params->clock_skew) {
+		    fabs(h_date - sp->wrk->exp.entered) < params->clock_skew) {
 			/*
 			 * If we have no Date: header or if it is
 			 * sufficiently close to our clock we will
 			 * trust Expires: relative to our own clock.
 			 */
-			if (h_expires < sp->wrk->entered)
+			if (h_expires < sp->wrk->exp.entered)
 				ttl = 0;
 			else
-				ttl = h_expires - sp->wrk->entered;
+				ttl = h_expires - sp->wrk->exp.entered;
 			break;
 		} else {
 			/*
@@ -168,8 +168,10 @@ RFC2616_Ttl(const struct sess *sp)
 	}
 
 	/* calculated TTL, Our time, Date, Expires, max-age, age */
-	WSP(sp, SLT_TTL, "%u RFC %g %.0f %.0f %.0f %u %u", sp->xid,
-	    ttl, sp->wrk->entered, h_date, h_expires, max_age, age);
+	WSP(sp, SLT_TTL,
+	    "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u %u",
+	    sp->xid, ttl, -1. -1., sp->wrk->exp.entered, sp->wrk->exp.age,
+	     h_date, h_expires, max_age);
 
 	return (ttl);
 }

bin/varnishd/stevedore.c

@@ -36,7 +36,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <math.h>
 
 #include "cache.h"
 #include "stevedore.h"
@@ -248,7 +247,6 @@ STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
 
 	http_Setup(o->http, o->ws_o);
 	o->http->magic = HTTP_MAGIC;
-	o->entered = NAN;
 	o->exp = *soc->exp;
 	VTAILQ_INIT(&o->store);
 	sp->wrk->stats.n_object++;

bin/varnishtest/tests/r00956.vtc

+varnishtest "obj.ttl relative/absolute"
+
+server s1 {
+	rxreq
+	txresp -hdr "Cache-Control: max-age=23" -hdr "Age: 4" -bodylen 40
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		set beresp.ttl = 10s;
+		set req.http.foo = beresp.ttl;
+		set req.http.bar = "xxx";
+	}
+	sub vcl_hit {
+		set req.http.foo = obj.ttl;
+		set obj.ttl = 7s;
+		set obj.grace = 120s;
+		set obj.keep = 1h;
+		set req.http.bar = obj.ttl;
+	}
+	sub vcl_deliver {
+		set resp.http.foo = req.http.foo;
+		set resp.http.bar = req.http.bar;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.bodylen == 40
+	expect resp.http.foo == 10.000
+	expect resp.http.bar == "xxx"
+
+	delay 2
+	txreq
+	rxresp
+	expect resp.bodylen == 40
+	# XXX: should be: < 8
+	expect resp.http.foo != 10.000
+	expect resp.http.bar == 7.000
+
+	delay 2
+	txreq
+	rxresp
+	expect resp.bodylen == 40
+	# XXX: should be: < 5
+	expect resp.http.foo != 7.000
+	expect resp.http.bar == 7.000
+} -run