Add a callback facility to be notified about events in the expiry module.

This is necessary for VMODs which implement secondary hash-keys.

Patches from martin@ with some minor tweaking by me.

bin/varnishd/cache/cache.h

@@ -778,14 +778,25 @@ void EXP_Clr(struct exp *e);
 
 double EXP_Ttl(const struct req *, const struct exp*);
 double EXP_When(const struct exp *exp);
-void EXP_Insert(struct objcore *oc);
-void EXP_Inject(struct objcore *oc, struct lru *lru);
+void EXP_Insert(struct worker *wrk, struct objcore *oc);
+void EXP_Inject(struct worker *wrk, struct objcore *oc, struct lru *lru);
 void EXP_Init(void);
 void EXP_Rearm(struct objcore *, double now, double ttl, double grace,
     double keep);
 void EXP_Touch(struct objcore *oc, double now);
 int EXP_NukeOne(struct worker *wrk, struct lru *lru);
 
+enum exp_event_e {
+	EXP_INSERT,
+	EXP_INJECT,
+	EXP_REMOVE,
+};
+typedef void exp_callback_f(struct worker *, struct objcore *,
+    enum exp_event_e, void *priv);
+
+uintptr_t EXP_Register_Callback(exp_callback_f *func, void *priv);
+void EXP_Deregister_Callback(uintptr_t*);
+
 /* cache_fetch.c */
 enum vbf_fetch_mode_e {
 	VBF_NORMAL = 0,

bin/varnishd/cache/cache_expire.c

@@ -41,6 +41,14 @@
 #include "hash/hash_slinger.h"
 #include "vtim.h"
 
+struct exp_callback {
+	unsigned			magic;
+#define EXP_CALLBACK_MAGIC		0xab956eb1
+	exp_callback_f			*func;
+	void				*priv;
+	VTAILQ_ENTRY(exp_callback)	list;
+};
+
 struct exp_priv {
 	unsigned			magic;
 #define EXP_PRIV_MAGIC			0x9db22482
@@ -52,10 +60,33 @@ struct exp_priv {
 	VTAILQ_HEAD(,objcore)		inbox;
 	struct binheap			*heap;
 	pthread_cond_t			condvar;
+
+	VTAILQ_HEAD(,exp_callback)	ecb_list;
+	pthread_rwlock_t		cb_rwl;
 };
 
 static struct exp_priv *exphdl;
 
+static void
+exp_event(struct worker *wrk, struct objcore *oc, enum exp_event_e e)
+{
+	struct exp_callback *cb;
+
+	/*
+	 * Strictly speaking this is not atomic, but neither is VMOD
+	 * loading in general, so this is a fair optimization
+	 */
+	if (VTAILQ_EMPTY(&exphdl->ecb_list))
+		return;
+
+	AZ(pthread_rwlock_rdlock(&exphdl->cb_rwl));
+	VTAILQ_FOREACH(cb, &exphdl->ecb_list, list) {
+		CHECK_OBJ_NOTNULL(cb, EXP_CALLBACK_MAGIC);
+		cb->func(wrk, oc, e, cb->priv);
+	}
+	AZ(pthread_rwlock_unlock(&exphdl->cb_rwl));
+}
+
 /*--------------------------------------------------------------------
  * struct exp manipulations
  */
@@ -130,9 +161,10 @@ exp_mail_it(struct objcore *oc)
  */
 
 void
-EXP_Inject(struct objcore *oc, struct lru *lru)
+EXP_Inject(struct worker *wrk, struct objcore *oc, struct lru *lru)
 {
 
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
 	AZ(oc->exp_flags & (OC_EF_OFFLRU | OC_EF_INSERT | OC_EF_MOVE));
@@ -146,6 +178,8 @@ EXP_Inject(struct objcore *oc, struct lru *lru)
 	oc->timer_when = EXP_When(&oc->exp);
 	Lck_Unlock(&lru->mtx);
 
+	exp_event(wrk, oc, EXP_INJECT);
+
 	exp_mail_it(oc);
 }
 
@@ -157,10 +191,11 @@ EXP_Inject(struct objcore *oc, struct lru *lru)
  */
 
 void
-EXP_Insert(struct objcore *oc)
+EXP_Insert(struct worker *wrk, struct objcore *oc)
 {
 	struct lru *lru;
 
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	HSH_Ref(oc);
 
@@ -177,6 +212,8 @@ EXP_Insert(struct objcore *oc)
 	oc->exp_flags |= OC_EF_MOVE;
 	Lck_Unlock(&lru->mtx);
 
+	exp_event(wrk, oc, EXP_INSERT);
+
 	exp_mail_it(oc);
 }
 
@@ -344,6 +381,45 @@ EXP_NukeOne(struct worker *wrk, struct lru *lru)
 	return (1);
 }
 
+/*--------------------------------------------------------------------*/
+
+uintptr_t
+EXP_Register_Callback(exp_callback_f *func, void *priv)
+{
+	struct exp_callback *ecb;
+
+	AN(func);
+
+	ALLOC_OBJ(ecb, EXP_CALLBACK_MAGIC);
+	AN(ecb);
+	ecb->func = func;
+	ecb->priv = priv;
+	AZ(pthread_rwlock_wrlock(&exphdl->cb_rwl));
+	VTAILQ_INSERT_TAIL(&exphdl->ecb_list, ecb, list);
+	AZ(pthread_rwlock_unlock(&exphdl->cb_rwl));
+	return ((uintptr_t)ecb);
+}
+
+void
+EXP_Deregister_Callback(uintptr_t *handle)
+{
+	struct exp_callback *ecb;
+
+	AN(handle);
+	AN(*handle);
+	AZ(pthread_rwlock_wrlock(&exphdl->cb_rwl));
+	VTAILQ_FOREACH(ecb, &exphdl->ecb_list, list) {
+		CHECK_OBJ_NOTNULL(ecb, EXP_CALLBACK_MAGIC);
+		if ((uintptr_t)ecb == *handle)
+			break;
+	}
+	AN(ecb);
+	VTAILQ_REMOVE(&exphdl->ecb_list, ecb, list);
+	AZ(pthread_rwlock_unlock(&exphdl->cb_rwl));
+	FREE_OBJ(ecb);
+	*handle = 0;
+}
+
 /*--------------------------------------------------------------------
  * Handle stuff in the inbox
  */
@@ -385,6 +461,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
 			binheap_delete(ep->heap, oc->timer_idx);
 		}
 		assert(oc->timer_idx == BINHEAP_NOIDX);
+		exp_event(ep->wrk, oc, EXP_REMOVE);
 		(void)HSH_DerefObjCore(ep->wrk, &oc);
 		return;
 	}
@@ -464,6 +541,7 @@ exp_expire(struct exp_priv *ep, double now)
 	CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
 	VSLb(&ep->vsl, SLT_ExpKill, "EXP_Expired x=%u t=%.0f",
 	    ObjGetXID(ep->wrk, oc), EXP_Ttl(NULL, &oc->exp) - now);
+	exp_event(ep->wrk, oc, EXP_REMOVE);
 	(void)HSH_DerefObjCore(ep->wrk, &oc);
 	return (0);
 }
@@ -545,6 +623,8 @@ EXP_Init(void)
 	Lck_New(&ep->mtx, lck_exp);
 	AZ(pthread_cond_init(&ep->condvar, NULL));
 	VTAILQ_INIT(&ep->inbox);
+	AZ(pthread_rwlock_init(&ep->cb_rwl, NULL));
+	VTAILQ_INIT(&ep->ecb_list);
 	exphdl = ep;
 	WRK_BgThread(&pt, "cache-timeout", exp_thread, ep);
 }

bin/varnishd/cache/cache_hash.c

@@ -693,7 +693,7 @@ HSH_Unbusy(struct worker *wrk, struct objcore *oc)
 
 	if (!(oc->flags & OC_F_PRIVATE)) {
 		BAN_NewObjCore(oc);
-		EXP_Insert(oc);
+		EXP_Insert(wrk, oc);
 		AN(oc->exp_flags & OC_EF_EXP);
 		AN(oc->ban);
 	}

bin/varnishd/flint.lnt

@@ -111,6 +111,7 @@
 -emacro(527, NEEDLESS_RETURN)	// unreachable code
 
 -sem(EXP_Inject, custodial(1))
+-sem(HSH_Insert, custodial(3))
 -sem(WS_Init, custodial(2))
 -sem(http_Setup, custodial(2))
 -sem(vfp_esi_end, custodial(2))

bin/varnishd/storage/storage_persistent_silo.c

@@ -166,7 +166,7 @@ smp_load_seg(struct worker *wrk, const struct smp_sc *sc,
 		oc->ban = BAN_RefBan(oc, so->ban, sc->tailban);
 		HSH_Insert(wrk, so->hash, oc);
 		oc->exp = so->exp;
-		EXP_Inject(oc, sg->lru);
+		EXP_Inject(wrk, oc, sg->lru);
 		sg->nobj++;
 	}
 	Pool_Sumstat(wrk);

bin/varnishtest/tests/m00021.vtc

+varnishtest "Test expiry callbacks"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+varnish v1 -cliok "param.set debug +vclrel"
+
+logexpect l1 -v v1 -g raw {
+	expect * 0 Debug "exp_cb: registered"
+	expect * 0 Debug "exp_cb: event insert 0x[0-9a-f]+"
+	expect * 0 Debug "exp_cb: event remove 0x[0-9a-f]+"
+	expect * 0 Debug "exp_cb: deregistered"
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_debug};
+
+	sub vcl_init {
+		debug.register_exp_callback();
+	}
+
+	sub vcl_recv {
+		if (req.method == "PURGE") {
+			return (purge);
+		}
+	}
+}
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+
+	txreq -req PURGE
+	rxresp
+} -run
+varnish v1 -expect n_object == 0
+
+varnish v1 -vcl+backend {}
+varnish v1 -cliok "vcl.discard vcl2"
+varnish v1 -cliok "debug.vmod"
+varnish v1 -cliok "vcl.list"
+varnish v1 -expect vmods == 0
+
+logexpect l1 -wait

lib/libvmod_debug/vmod.vcc

@@ -93,3 +93,7 @@ Encrypt the HTTP header with quad-ROT13 encryption,
 $Function STRING argtest(STRING one, REAL two=2, STRING three="3")
 
 $Function INT vre_limit()
+
+$Function VOID register_exp_callback(PRIV_VCL)
+
+Register the vmod to receive expiry callbacks

lib/libvmod_debug/vmod_debug.c

@@ -36,6 +36,13 @@
 #include "vrt.h"
 #include "vcc_if.h"
 
+struct priv_vcl {
+	unsigned		magic;
+#define PRIV_VCL_MAGIC		0x8E62FA9D
+	char			*foo;
+	uintptr_t		exp_cb;
+};
+
 VCL_VOID __match_proto__(td_debug_panic)
 vmod_panic(VRT_CTX, const char *str, ...)
 {
@@ -65,16 +72,6 @@ vmod_author(VRT_CTX, VCL_ENUM id)
 	WRONG("Illegal VMOD enum");
 }
 
-int
-init_function(struct vmod_priv *priv, const struct VCL_conf *cfg)
-{
-	(void)cfg;
-
-	priv->priv = strdup("FOO");
-	priv->free = free;
-	return (0);
-}
-
 VCL_VOID __match_proto__(td_debug_test_priv_call)
 vmod_test_priv_call(VRT_CTX, struct vmod_priv *priv)
 {
@@ -103,9 +100,13 @@ vmod_test_priv_task(VRT_CTX, struct vmod_priv *priv, VCL_STRING s)
 VCL_VOID __match_proto__(td_debug_test_priv_vcl)
 vmod_test_priv_vcl(VRT_CTX, struct vmod_priv *priv)
 {
+	struct priv_vcl *priv_vcl;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	assert(!strcmp(priv->priv, "FOO"));
+	AN(priv);
+	CAST_OBJ_NOTNULL(priv_vcl, priv->priv, PRIV_VCL_MAGIC);
+	AN(priv_vcl->foo);
+	assert(!strcmp(priv_vcl->foo, "FOO"));
 }
 
 VCL_BLOB
@@ -175,3 +176,65 @@ vmod_vre_limit(VRT_CTX)
 	(void)ctx;
 	return (cache_param->vre_limits.match);
 }
+
+static void __match_proto__(exp_callback_f)
+exp_cb(struct worker *wrk, struct objcore *oc, enum exp_event_e ev, void *priv)
+{
+	const struct priv_vcl *priv_vcl;
+	const char *what;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	CAST_OBJ_NOTNULL(priv_vcl, priv, PRIV_VCL_MAGIC);
+	switch (ev) {
+	case EXP_INSERT: what = "insert"; break;
+	case EXP_INJECT: what = "inject"; break;
+	case EXP_REMOVE: what = "remove"; break;
+	default: WRONG("Wrong exp_event");
+	}
+	VSL(SLT_Debug, 0, "exp_cb: event %s %p", what, oc);
+}
+
+VCL_VOID __match_proto__()
+vmod_register_exp_callback(VRT_CTX, struct vmod_priv *priv)
+{
+	struct priv_vcl *priv_vcl;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CAST_OBJ_NOTNULL(priv_vcl, priv->priv, PRIV_VCL_MAGIC);
+	AZ(priv_vcl->exp_cb);
+	priv_vcl->exp_cb = EXP_Register_Callback(exp_cb, priv_vcl);
+	VSL(SLT_Debug, 0, "exp_cb: registered");
+}
+
+static void __match_proto__(vmod_priv_free_f)
+priv_vcl_free(void *priv)
+{
+	struct priv_vcl *priv_vcl;
+
+	CAST_OBJ_NOTNULL(priv_vcl, priv, PRIV_VCL_MAGIC);
+	AN(priv_vcl->foo);
+	free(priv_vcl->foo);
+	if (priv_vcl->exp_cb != 0) {
+		EXP_Deregister_Callback(&priv_vcl->exp_cb);
+		VSL(SLT_Debug, 0, "exp_cb: deregistered");
+	}
+	FREE_OBJ(priv_vcl);
+	AZ(priv_vcl);
+}
+
+int __match_proto__(vmod_init_f)
+init_function(struct vmod_priv *priv, const struct VCL_conf *cfg)
+{
+	struct priv_vcl *priv_vcl;
+
+	(void)cfg;
+
+	ALLOC_OBJ(priv_vcl, PRIV_VCL_MAGIC);
+	AN(priv_vcl);
+	priv_vcl->foo = strdup("FOO");
+	AN(priv_vcl->foo);
+	priv->priv = priv_vcl;
+	priv->free = priv_vcl_free;
+	return (0);
+}