Add {req,obj,beresp}.keep and req.ttl variables.

Keep is for future use by conditional backend requests.

req.ttl provides a way to force a fetch of a not yet stale object,
see how in testcase c00042.

bin/varnishd/cache.h

@@ -243,6 +243,7 @@ extern struct vfp vfp_esi;
 struct exp {
 	double			ttl;
 	double			grace;
+	double			keep;
 };
 
 /*--------------------------------------------------------------------*/
@@ -642,13 +643,16 @@ extern pthread_t cli_thread;
 
 /* cache_expiry.c */
 void EXP_Clr(struct exp *e);
-double EXP_Get_grace(const struct exp *e);
 double EXP_Get_ttl(const struct exp *e);
-void EXP_Set_grace(struct exp *e, double v);
+double EXP_Get_grace(const struct exp *e);
+double EXP_Get_keep(const struct exp *e);
 void EXP_Set_ttl(struct exp *e, double v);
+void EXP_Set_grace(struct exp *e, double v);
+void EXP_Set_keep(struct exp *e, double v);
 
-double EXP_Grace(const struct sess *, const struct object*);
 double EXP_Ttl(const struct sess *, const struct object*);
+double EXP_Grace(const struct sess *, const struct object*);
+double EXP_Keep(const struct sess *, const struct object*);
 void EXP_Insert(struct object *o);
 void EXP_Inject(struct objcore *oc, struct lru *lru, double when);
 void EXP_Init(void);

bin/varnishd/cache_expire.c

@@ -68,51 +68,51 @@ void
 EXP_Clr(struct exp *e)
 {
 
-	e->grace = -1;
 	e->ttl = -1;
+	e->grace = -1;
+	e->keep = -1;
 }
 
-void
-EXP_Set_grace(struct exp *e, double v)
-{
-
-	if (v > 0.)
-		e->grace = v;
-	else
-		e->grace = -1.;
-}
-
-double
-EXP_Get_grace(const struct exp *e)
-{
-
-	return (e->grace > 0. ? e->grace : -1.);
-}
-
-void
-EXP_Set_ttl(struct exp *e, double v)
-{
+#define EXP_ACCESS(fld, extra)					\
+	double							\
+	EXP_Get_##fld(const struct exp *e)			\
+	{							\
+		return (e->fld > 0. ? e->fld : -1.);		\
+	}							\
+								\
+	void							\
+	EXP_Set_##fld(struct exp *e, double v)			\
+	{							\
+		if (v > 0.)					\
+			e->fld = v;				\
+		else {						\
+			e->fld = -1.;				\
+			extra;					\
+		}						\
+	}							\
+
+EXP_ACCESS(ttl, (e->grace = e->keep = -1.))
+EXP_ACCESS(grace,)
+EXP_ACCESS(keep,)
 
-	if (v > 0.)
-		e->ttl = v;
-	else {
-		e->ttl = -1.;
-		e->grace = -1.;
-	}
-}
+/*--------------------------------------------------------------------
+ * Calculate when an object is out of ttl or grace, possibly constrained
+ * by per-session limits.
+ */
 
 double
-EXP_Get_ttl(const struct exp *e)
+EXP_Keep(const struct sess *sp, const struct object *o)
 {
+	double r;
 
-	return (e->ttl > 0. ? e->ttl : -1.);
+	r = (double)params->default_keep;
+	if (o->exp.keep > 0.)
+		r = o->exp.keep;
+	if (sp != NULL && sp->exp.keep > 0. && sp->exp.keep < r)
+		r = sp->exp.keep;
+	return (EXP_Grace(sp, o) + r);
 }
 
-/*--------------------------------------------------------------------
- * Calculate when an object is out of ttl or grace, possibly constrained
- * by per-session limits.
- */
-
 double
 EXP_Grace(const struct sess *sp, const struct object *o)
 {
@@ -152,7 +152,7 @@ update_object_when(const struct object *o)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	Lck_AssertHeld(&exp_mtx);
 
-	when = EXP_Grace(NULL, o);
+	when = EXP_Keep(NULL, o);
 	assert(!isnan(when));
 	if (when == oc->timer_when)
 		return (0);

bin/varnishd/cache_vrt_var.c

@@ -383,11 +383,15 @@ VRT_r_##which##_##fld(struct sess *sp)				\
 	return(EXP_Get_##fld(&exp));				\
 }
 
+VRT_DO_EXP(req, sp->exp, ttl, )
 VRT_DO_EXP(req, sp->exp, grace, )
+VRT_DO_EXP(req, sp->exp, keep, )
 VRT_DO_EXP(obj, sp->obj->exp, grace, EXP_Rearm(sp->obj))
 VRT_DO_EXP(obj, sp->obj->exp, ttl, EXP_Rearm(sp->obj))
+VRT_DO_EXP(obj, sp->obj->exp, keep, EXP_Rearm(sp->obj))
 VRT_DO_EXP(beresp, sp->wrk->exp, grace, )
 VRT_DO_EXP(beresp, sp->wrk->exp, ttl, )
+VRT_DO_EXP(beresp, sp->wrk->exp, keep, )
 
 /*--------------------------------------------------------------------
  * req.xid

bin/varnishd/heritage.h

@@ -73,6 +73,12 @@ struct params {
 	/* TTL used for lack of anything better */
 	double			default_ttl;
 
+	/* Default grace period */
+	double			default_grace;
+
+	/* Default keep period */
+	double			default_keep;
+
 	/* Maximum concurrent sessions */
 	unsigned		max_sess;
 
@@ -162,9 +168,6 @@ struct params {
 	/* Control diagnostic code */
 	unsigned		diag_bitmap;
 
-	/* Default grace period */
-	double			default_grace;
-
 	/* Log hash string to shm */
 	unsigned		log_hash;
 

bin/varnishd/mgt_param.c

@@ -583,6 +583,15 @@ static const struct parspec input_parspec[] = {
 		"made until they are fetched from the backend again.\n",
 		DELAYED_EFFECT,
 		"10", "seconds" },
+	{ "default_keep", tweak_timeout_double, &master.default_keep,
+		0, UINT_MAX,
+		"Default keep period.  We will keep a useless object "
+		"around this long, making it available for conditional "
+		"backend fetches.  "
+		"That means that the object will be removed from the "
+		"cache at the end of ttl+grace+keep.",
+		DELAYED_EFFECT,
+		"0", "seconds" },
 	{ "sess_timeout", tweak_timeout, &master.sess_timeout, 0, 0,
 		"Idle timeout for persistent sessions. "
 		"If a HTTP request has not been received in this many "

bin/varnishtest/tests/c00042.vtc

+# $Id$
+
+test "using req.ttl to force fetch"
+
+server s1 {
+	rxreq
+	expect req.url == "/1"
+	txresp -bodylen 1
+	rxreq
+	expect req.url == "/1"
+	txresp -bodylen 2
+
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		if (req.http.short) {
+			set req.ttl = 1s;
+			set req.grace = 0s;
+		}
+	}
+} -start
+
+client c1 {
+	txreq -url "/1" 
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 1
+
+	txreq -url "/1" 
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 1
+
+	delay 2
+
+	txreq -url "/1" -hdr "short: yes"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 2
+
+	txreq -url "/1" 
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 2
+
+} -run

lib/libvcl/generate.py

@@ -178,12 +178,24 @@ sp_variables = (
 		( ),
 		'const struct sess *'
 	),
+	('req.ttl',
+		'DURATION',
+		( 'all',),
+		( 'all',),
+		'struct sess *'
+	),
 	('req.grace',
 		'DURATION',
 		( 'all',),
 		( 'all',),
 		'struct sess *'
 	),
+	('req.keep',
+		'DURATION',
+		( 'all',),
+		( 'all',),
+		'struct sess *'
+	),
 	('req.xid',
 		'STRING',
 		( 'all',),
@@ -322,6 +334,12 @@ sp_variables = (
 		( 'fetch',),
 		'struct sess *'
 	),
+	('beresp.keep',
+		'DURATION',
+		( 'fetch',),
+		( 'fetch',),
+		'struct sess *'
+	),
 	('beresp.backend.name',
 		'STRING',
 		( 'fetch',),
@@ -388,6 +406,12 @@ sp_variables = (
 		( 'hit', 'error',),
 		'struct sess *'
 	),
+	('obj.keep',
+		'DURATION',
+		( 'hit', 'error',),
+		( 'hit', 'error',),
+		'struct sess *'
+	),
 	('obj.lastuse',
 		'DURATION',
 		( 'hit', 'deliver', 'error',),