Introduce vmod-purge

This module offers a finer-grained control over purges, in the old
pre-vcl 4.0 style. For that, the signatures of VRT_purge and HSH_Purge
were updated to inform about the number of rearmed objects.

bin/varnishd/cache/cache_hash.c

@@ -586,7 +586,7 @@ hsh_rush2(struct worker *wrk, struct rush *r)
  * Purge an entire objhead
  */
 
-void
+unsigned
 HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace,
 double keep)
 {
@@ -667,6 +667,7 @@ double keep)
 	} while (more);
 	WS_Release(wrk->aws, 0);
 	Pool_PurgeStat(n_tot);
+	return (n_tot);
 }
 
 /*---------------------------------------------------------------------

bin/varnishd/cache/cache_vrt.c

@@ -538,7 +538,7 @@ VRT_CacheReqBody(VRT_CTX, VCL_BYTES maxsize)
  * purges
  */
 
-void
+unsigned
 VRT_purge(VRT_CTX, double ttl, double grace, double keep)
 {
 
@@ -548,12 +548,13 @@ VRT_purge(VRT_CTX, double ttl, double grace, double keep)
 		VSLb(ctx->vsl, SLT_VCL_Error,
 		    "purge can only happen in vcl_hit{} or vcl_miss{}");
 		VRT_handling(ctx, VCL_RET_FAIL);
-		return;
+		return (0);
 	}
 
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req->wrk, WORKER_MAGIC);
-	HSH_Purge(ctx->req->wrk, ctx->req->objcore->objhead, ttl, grace, keep);
+	return (HSH_Purge(ctx->req->wrk, ctx->req->objcore->objhead,
+	    ttl, grace, keep));
 }
 
 /*--------------------------------------------------------------------

bin/varnishd/hash/hash_slinger.h

@@ -69,7 +69,7 @@ void HSH_Init(const struct hash_slinger *slinger);
 void HSH_AddString(struct req *, void *ctx, const char *str);
 void HSH_Insert(struct worker *, const void *hash, struct objcore *,
     struct ban *);
-void HSH_Purge(struct worker *, struct objhead *, double ttl, double grace,
+unsigned HSH_Purge(struct worker *, struct objhead *, double ttl, double grace,
     double keep);
 void HSH_config(const char *h_arg);
 struct boc *HSH_RefBoc(const struct objcore *);

bin/varnishtest/tests/m00031.vtc

+varnishtest "Test purge vmod"
+
+server s1 {
+	rxreq
+	expect req.http.Accept-Language == fr
+	txresp -hdr "Vary: Accept-Language" -body fr
+
+	rxreq
+	expect req.http.Accept-Language == fr
+	txresp -hdr "Vary: Accept-Language" -body fr
+
+	rxreq
+	expect req.http.Accept-Language == en
+	txresp -hdr "Vary: Accept-Language" -body en
+} -start
+
+varnish v1 -vcl+backend {
+	import purge;
+
+	sub vcl_recv {
+		if (req.method ~ "PURGE") {
+			return (hash);
+		}
+	}
+
+	sub my_purge {
+		if (req.method == "PURGE") {
+			set req.http.purged = purge.hard();
+		}
+		else {
+			set req.http.purged = purge.soft();
+		}
+		if (req.http.purged == "0") {
+			return (synth(404));
+		}
+		else {
+			return (synth(200));
+		}
+	}
+
+	sub vcl_hit {
+		if (req.method ~ "PURGE") { call my_purge; }
+	}
+
+	sub vcl_miss {
+		if (req.method ~ "PURGE") { call my_purge; }
+	}
+
+	sub vcl_synth {
+		if (req.method ~ "PURGE") {
+			set resp.http.purged = req.http.purged;
+			return (deliver);
+		}
+	}
+
+	sub vcl_backend_response {
+		set beresp.http.X-Varnish-Be = bereq.xid;
+	}
+} -start
+
+logexpect l1 -v v1 -q Hit -i Hit {
+	expect * * Hit "^1002 -.+ 10.000000 0.000000$"
+} -start
+
+logexpect l2 -v v1 -q "Begin ~ bgfetch" {
+	expect * * BerespHeader "X-Varnish-Be: 1005"
+} -start
+
+logexpect l3 -v v1 -g raw -q "vxid == 0" {
+	expect * * ExpKill x=1008
+} -start
+
+client c1 {
+	txreq -hdr "Accept-Language: fr"
+	rxresp
+	expect resp.body == fr
+	expect resp.http.X-Varnish-Be == 1002
+
+	txreq -req SOFTPURGE
+	rxresp
+	expect resp.status == 200
+	expect resp.http.purged == 1
+
+	txreq -hdr "Accept-Language: fr"
+	rxresp
+	expect resp.body == fr
+	expect resp.http.X-Varnish == "1004 1002"
+	expect resp.http.X-Varnish-Be == 1002
+} -run
+
+# Wait until bgfetch triggered by 1004 is done
+logexpect l2 -wait
+
+client c1 {
+	txreq -hdr "Accept-Language: en"
+	rxresp
+	expect resp.body == en
+	expect resp.http.X-Varnish-Be == 1008
+
+	txreq -req PURGE
+	rxresp
+	expect resp.status == 200
+	expect resp.http.purged == 2
+} -run
+
+# Wait until the expire thread kicks in
+logexpect l3 -wait
+
+client c1 {
+	txreq -req PURGE
+	rxresp
+	expect resp.status == 404
+	expect resp.http.purged == 0
+} -run
+
+logexpect l1 -wait

bin/varnishtest/vmods.h

@@ -30,4 +30,5 @@
 VTC_VMOD(std)
 VTC_VMOD(debug)
 VTC_VMOD(directors)
+VTC_VMOD(purge)
 VTC_VMOD(vtc)

configure.ac

@@ -732,6 +732,7 @@ AC_CONFIG_FILES([
     lib/libvmod_debug/Makefile
     lib/libvmod_std/Makefile
     lib/libvmod_directors/Makefile
+    lib/libvmod_purge/Makefile
     lib/libvmod_vtc/Makefile
     man/Makefile
     varnishapi.pc

doc/sphinx/Makefile.am

@@ -205,6 +205,10 @@ reference/vmod_directors.generated.rst: reference $(top_builddir)/lib/libvmod_di
 	cp $(top_builddir)/lib/libvmod_directors/vmod_directors.rst $@ || true
 BUILT_SOURCES += reference/vmod_directors.generated.rst
 
+reference/vmod_purge.generated.rst: reference $(top_builddir)/lib/libvmod_purge/vmod_purge.rst
+	cp $(top_builddir)/lib/libvmod_purge/vmod_purge.rst $@ || true
+BUILT_SOURCES += reference/vmod_purge.generated.rst
+
 reference/vmod_vtc.generated.rst: reference $(top_builddir)/lib/libvmod_vtc/vmod_vtc.rst
 	cp $(top_builddir)/lib/libvmod_vtc/vmod_vtc.rst $@ || true
 BUILT_SOURCES += reference/vmod_vtc.generated.rst

include/vrt.h

@@ -41,7 +41,7 @@
  *
  * 6.1 (unreleased):
  *	http_CollectHdrSep added
- *	VRT_purge modified (may fail a transaction)
+ *	VRT_purge modified (may fail a transaction, signature changed)
  * 6.0 (2017-03-15):
  *	VRT_hit_for_pass added
  *	VRT_ipcmp added
@@ -300,7 +300,7 @@ int VRT_re_match(VRT_CTX, const char *, void *re);
 const char *VRT_regsub(VRT_CTX, int all, const char *, void *, const char *);
 
 void VRT_ban_string(VRT_CTX, const char *);
-void VRT_purge(VRT_CTX, double ttl, double grace, double keep);
+unsigned VRT_purge(VRT_CTX, double ttl, double grace, double keep);
 
 void VRT_count(VRT_CTX, unsigned);
 void VRT_synth(VRT_CTX, unsigned, const char *);

lib/Makefile.am

@@ -8,4 +8,5 @@ SUBDIRS = \
 	libvmod_debug \
 	libvmod_std \
 	libvmod_directors \
+	libvmod_purge \
 	libvmod_vtc

lib/libvmod_purge/Makefile.am

+#
+
+AM_LDFLAGS  = $(AM_LT_LDFLAGS)
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_srcdir)/bin/varnishd \
+	-I$(top_builddir)/include
+
+vmoddir = $(pkglibdir)/vmods
+vmod_srcdir = $(top_srcdir)/lib/libvmod_purge
+vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
+vmodtoolargs = --strict
+vmod_LTLIBRARIES = libvmod_purge.la
+
+libvmod_purge_la_CFLAGS = \
+	@SAN_CFLAGS@
+
+libvmod_purge_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version -shared \
+	@SAN_LDFLAGS@
+
+libvmod_purge_la_SOURCES = vmod_purge.c
+
+nodist_libvmod_purge_la_SOURCES = \
+	vcc_if.c \
+	vcc_if.h
+
+# BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
+$(libvmod_purge_la_OBJECTS): vcc_if.h
+
+vcc_if.h vmod_purge.rst vmod_purge.man.rst: vcc_if.c
+
+vcc_if.c: $(vmodtool) $(vmod_srcdir)/vmod.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(vmod_srcdir)/vmod.vcc
+
+EXTRA_DIST = vmod.vcc
+
+CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
+	$(builddir)/vmod_purge.rst \
+	$(builddir)/vmod_purge.man.rst

lib/libvmod_purge/flint.sh

+#!/bin/sh
+
+FLOPS='
+	-I../../bin/varnishd
+	*.c
+'
+
+. ../../tools/flint_skel.sh

lib/libvmod_purge/vmod.vcc

+#-
+# Copyright (c) 2017 Varnish Software AS
+# All rights reserved.
+#
+# Author: Dridi Boukelmoune <dridi.boukelmoune@gmail.com>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+$Module purge 3 Varnish Purge Module
+$ABI strict
+
+DESCRIPTION
+===========
+
+`vmod_purge` contains functions that offer a finer-grained control than the
+``purge`` transition in ``vcl_recv``. The functions can only be called from
+``vcl_hit`` or ``vcl_miss`` and they should in general be used in both to
+ensure that all variants of a same object are taken care of.
+
+EXAMPLE
+=======
+
+::
+
+    sub vcl_recv {
+        if (req.method == "PURGE") {
+            if (client.ip !~ purge_acl) {
+                return (synth(405));
+            }
+            return (hash);
+        }
+    }
+
+    sub my_purge {
+        set req.http.purged = purge.hard();
+        if (req.http.purged == "0") {
+            return (synth(404));
+        }
+        else {
+            return (synth(200));
+        }
+    }
+
+    sub vcl_hit {
+        if (req.method == "PURGE") {
+            call my_purge;
+        }
+    }
+
+    sub vcl_miss {
+        if (req.method == "PURGE") {
+            call my_purge;
+        }
+    }
+
+    sub vcl_synth {
+        if (req.method == "PURGE") {
+            if (req.http.purged) {
+                set resp.http.purged = req.http.purged;
+            }
+            return (deliver);
+        }
+    }
+
+$Function INT hard()
+
+Description
+	This is equivalent to ``return(purge)`` but explicitly called from
+        ``vcl_hit`` and ``vcl_miss``. It returns the number of purged objects.
+Example
+	set req.http.purged = purge.hard();
+
+$Function INT soft(DURATION ttl = 0, DURATION grace = -1, DURATION keep = -1)
+
+Description
+        Sets the TTL, grace and keep. By default, TTL is set to 0 with grace
+        and keep periods left untouched. Setting a negative value for any of
+        the parameters leaves them untouched. Setting all three parameters to
+        0 is equivalent to a hard purge. It can only be called from ``vcl_hit``
+        or ``vcl_miss``. It returns the number of soft-purged objects.
+
+SEE ALSO
+========
+
+* :ref:`vcl(7)`

lib/libvmod_purge/vmod_purge.c

+/*-
+ * Copyright (c) 2017 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Dridi Boukelmoune <dridi.boukelmoune@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include "math.h"
+
+#include "cache/cache.h"
+
+#include "vrt.h"
+
+#include "vcc_if.h"
+
+VCL_INT __match_proto__(td_purge_hard)
+vmod_hard(VRT_CTX)
+{
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	return (VRT_purge(ctx, 0, 0, 0));
+}
+
+VCL_INT __match_proto__(td_purge_soft)
+vmod_soft(VRT_CTX, VCL_DURATION ttl, VCL_DURATION grace, VCL_DURATION keep)
+{
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	if (ttl < 0)
+		ttl = NAN;
+	if (grace < 0)
+		grace = NAN;
+	if (keep < 0)
+		keep = NAN;
+	return (VRT_purge(ctx, ttl, grace, keep));
+}

man/Makefile.am

@@ -16,6 +16,7 @@ dist_man_MANS = \
 	vtc.7 \
 	varnishtop.1 \
 	vmod_directors.3 \
+	vmod_purge.3 \
 	vmod_std.3 \
 	vmod_vtc.3
 
@@ -87,6 +88,9 @@ varnishhist.1: \
 vmod_directors.3: $(top_builddir)/lib/libvmod_directors/vmod_directors.man.rst
 	${RST2MAN} $(RST2ANY_FLAGS) $? $@
 
+vmod_purge.3: $(top_builddir)/lib/libvmod_purge/vmod_purge.man.rst
+	${RST2MAN} $(RST2ANY_FLAGS) $? $@
+
 vmod_std.3: $(top_builddir)/lib/libvmod_std/vmod_std.man.rst
 	${RST2MAN} $(RST2ANY_FLAGS) $? $@