Split ban_lurker into separate source file.

No functional changes.

bin/varnishd/Makefile.am

@@ -15,6 +15,7 @@ varnishd_SOURCES = \
 	cache/cache_backend_probe.c \
 	cache/cache_backend_tcp.c \
 	cache/cache_ban.c \
+	cache/cache_ban_lurker.c \
 	cache/cache_busyobj.c \
 	cache/cache_cli.c \
 	cache/cache_deliver_proc.c \
@@ -99,6 +100,7 @@ varnishd_SOURCES = \
 
 noinst_HEADERS = \
 	builtin_vcl.h \
+	cache/cache_ban.h \
 	cache/cache_esi.h \
 	cache/cache_pool.h \
 	cache/cache_priv.h \

bin/varnishd/cache/cache_ban.h

+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Ban processing
+ *
+ * A ban consists of a number of conditions (or tests), all of which must be
+ * satisfied.  Here are some potential bans we could support:
+ *
+ *	req.url == "/foo"
+ *	req.url ~ ".iso" && obj.size > 10MB
+ *	req.http.host ~ "web1.com" && obj.http.set-cookie ~ "USER=29293"
+ *
+ * We make the "&&" mandatory from the start, leaving the syntax space
+ * for latter handling of "||" as well.
+ *
+ * Bans are compiled into bytestrings as follows:
+ *	8 bytes	- double: timestamp		XXX: Byteorder ?
+ *	4 bytes - be32: length
+ *	1 byte - flags: 0x01: BAN_F_{REQ|OBJ|COMPLETED}
+ *	N tests
+ * A test have this form:
+ *	1 byte - arg (see ban_vars.h col 3 "BANS_ARG_XXX")
+ *	(n bytes) - http header name, canonical encoding
+ *	lump - comparison arg
+ *	1 byte - operation (BANS_OPER_)
+ *	(lump) - compiled regexp
+ * A lump is:
+ *	4 bytes - be32: length
+ *	n bytes - content
+ *
+ * In a perfect world, we should vector through VRE to get to PCRE,
+ * but since we rely on PCRE's ability to encode the regexp into a
+ * byte string, that would be a little bit artificial, so this is
+ * the exception that confirms the rule.
+ *
+ */
+
+/*--------------------------------------------------------------------
+ * BAN string defines & magic markers
+ */
+
+#define BANS_TIMESTAMP		0
+#define BANS_LENGTH		8
+#define BANS_FLAGS		12
+#define BANS_HEAD_LEN		16
+
+#define BANS_FLAG_REQ		(1<<0)
+#define BANS_FLAG_OBJ		(1<<1)
+#define BANS_FLAG_COMPLETED	(1<<2)
+#define BANS_FLAG_HTTP		(1<<3)
+#define BANS_FLAG_ERROR		(1<<4)
+
+#define BANS_OPER_EQ		0x10
+#define BANS_OPER_NEQ		0x11
+#define BANS_OPER_MATCH		0x12
+#define BANS_OPER_NMATCH	0x13
+
+#define BANS_ARG_URL		0x18
+#define BANS_ARG_REQHTTP	0x19
+#define BANS_ARG_OBJHTTP	0x1a
+#define BANS_ARG_OBJSTATUS	0x1b
+
+/*--------------------------------------------------------------------*/
+
+struct ban {
+	unsigned		magic;
+#define BAN_MAGIC		0x700b08ea
+	VTAILQ_ENTRY(ban)	list;
+	VTAILQ_ENTRY(ban)	l_list;
+	int			refcount;
+	unsigned		flags;		/* BANS_FLAG_* */
+
+	VTAILQ_HEAD(,objcore)	objcore;
+	struct vsb		*vsb;
+	uint8_t			*spec;
+};
+
+VTAILQ_HEAD(banhead_s,ban);
+
+struct ban_test {
+	uint8_t			arg1;
+	const char		*arg1_spec;
+	uint8_t			oper;
+	const char		*arg2;
+	const void		*arg2_spec;
+};
+
+bgthread_t ban_lurker;
+extern struct lock ban_mtx;
+extern int ban_shutdown;
+extern struct banhead_s ban_head;
+extern struct ban * volatile ban_start;
+
+void ban_mark_completed(struct ban *b);
+unsigned ban_len(const uint8_t *banspec);
+void ban_info(enum baninfo event, const uint8_t *ban, unsigned len);
+int ban_evaluate(struct worker *wrk, const uint8_t *bs, struct objcore *oc,
+    const struct http *reqhttp, unsigned *tests);
+double ban_time(const uint8_t *banspec);

bin/varnishd/cache/cache_ban_lurker.c

+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include "cache.h"
+#include "cache_ban.h"
+
+#include "hash/hash_slinger.h"
+#include "vtim.h"
+
+static struct objcore oc_marker = { .magic = OBJCORE_MAGIC, };
+static unsigned ban_batch;
+
+static void
+ban_cleantail(void)
+{
+	struct ban *b;
+
+	do {
+		Lck_Lock(&ban_mtx);
+		b = VTAILQ_LAST(&ban_head, banhead_s);
+		if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
+			if (b->flags & BANS_FLAG_COMPLETED)
+				VSC_C_main->bans_completed--;
+			if (b->flags & BANS_FLAG_OBJ)
+				VSC_C_main->bans_obj--;
+			if (b->flags & BANS_FLAG_REQ)
+				VSC_C_main->bans_req--;
+			VSC_C_main->bans--;
+			VSC_C_main->bans_deleted++;
+			VTAILQ_REMOVE(&ban_head, b, list);
+			VSC_C_main->bans_persisted_fragmentation +=
+			    ban_len(b->spec);
+			ban_info(BI_DROP, b->spec, ban_len(b->spec));
+		} else {
+			b = NULL;
+		}
+		Lck_Unlock(&ban_mtx);
+		if (b != NULL)
+			BAN_Free(b);
+	} while (b != NULL);
+}
+
+/*--------------------------------------------------------------------
+ * Our task here is somewhat tricky:  The canonical locking order is
+ * objhead->mtx first, then ban_mtx, because that is the order which
+ * makes most sense in HSH_Lookup(), but we come the other way.
+ * We optimistically try to get them the other way, and get out of
+ * the way if that fails, and retry again later.
+ */
+
+static struct objcore *
+ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt)
+{
+	struct objhead *oh;
+	struct objcore *oc;
+
+	while (1) {
+		Lck_Lock(&ban_mtx);
+		oc = VTAILQ_FIRST(&bt->objcore);
+		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+		if (oc == &oc_marker) {
+			VTAILQ_REMOVE(&bt->objcore, oc, ban_list);
+			Lck_Unlock(&ban_mtx);
+			return (NULL);
+		}
+		oh = oc->objhead;
+		CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+		if (!Lck_Trylock(&oh->mtx)) {
+			if (oc->refcnt == 0) {
+				Lck_Unlock(&oh->mtx);
+			} else {
+				/*
+				 * We got the lock, and the oc is not being
+				 * dismantled under our feet, run with it...
+				 */
+				AZ(oc->flags & OC_F_BUSY);
+				oc->refcnt += 1;
+				VTAILQ_REMOVE(&bt->objcore, oc, ban_list);
+				VTAILQ_INSERT_TAIL(&bt->objcore, oc, ban_list);
+				Lck_Unlock(&oh->mtx);
+				Lck_Unlock(&ban_mtx);
+				break;
+			}
+		}
+
+		/* Try again, later */
+		Lck_Unlock(&ban_mtx);
+		VSC_C_main->bans_lurker_contention++;
+		VSL_Flush(vsl, 0);
+		VTIM_sleep(cache_param->ban_lurker_sleep);
+	}
+	return (oc);
+}
+
+static void
+ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
+    struct banhead_s *obans)
+{
+	struct ban *bl, *bln;
+	struct objcore *oc;
+	unsigned tests;
+	int i;
+
+	/*
+	 * First see if there is anything to do, and if so, insert marker
+	 */
+	Lck_Lock(&ban_mtx);
+	oc = VTAILQ_FIRST(&bt->objcore);
+	if (oc != NULL)
+		VTAILQ_INSERT_TAIL(&bt->objcore, &oc_marker, ban_list);
+	Lck_Unlock(&ban_mtx);
+	if (oc == NULL)
+		return;
+
+	while (1) {
+		if (++ban_batch > cache_param->ban_lurker_batch) {
+			VTIM_sleep(cache_param->ban_lurker_sleep);
+			ban_batch = 0;
+		}
+		oc = ban_lurker_getfirst(vsl, bt);
+		if (oc == NULL)
+			return;
+		i = 0;
+		VTAILQ_FOREACH_REVERSE_SAFE(bl, obans, banhead_s, l_list, bln) {
+			if (bl->flags & BANS_FLAG_COMPLETED) {
+				/* Ban was overtaken by new (dup) ban */
+				VTAILQ_REMOVE(obans, bl, l_list);
+				continue;
+			}
+			tests = 0;
+			i = ban_evaluate(wrk, bl->spec, oc, NULL, &tests);
+			VSC_C_main->bans_lurker_tested++;
+			VSC_C_main->bans_lurker_tests_tested += tests;
+			if (i)
+				break;
+		}
+		if (i) {
+			VSLb(vsl, SLT_ExpBan, "%u banned by lurker",
+			    ObjGetXID(wrk, oc));
+
+			EXP_Rearm(oc, oc->exp.t_origin, 0, 0, 0);
+					// XXX ^ fake now
+			VSC_C_main->bans_lurker_obj_killed++;
+		}
+		(void)HSH_DerefObjCore(wrk, &oc);
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Ban lurker thread
+ */
+
+static int
+ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
+{
+	struct ban *b, *bt;
+	struct banhead_s obans;
+	double d;
+	int i;
+
+	/* Make a list of the bans we can do something about */
+	VTAILQ_INIT(&obans);
+	Lck_Lock(&ban_mtx);
+	b = ban_start;
+	Lck_Unlock(&ban_mtx);
+	i = 0;
+	d = VTIM_real() - cache_param->ban_lurker_age;
+	while (b != NULL) {
+		if (b->flags & BANS_FLAG_COMPLETED) {
+			;
+		} else if (b->flags & BANS_FLAG_REQ) {
+			;
+		} else if (b == VTAILQ_LAST(&ban_head, banhead_s)) {
+			;
+		} else if (ban_time(b->spec) > d) {
+			;
+		} else {
+			VTAILQ_INSERT_TAIL(&obans, b, l_list);
+			i++;
+		}
+		b = VTAILQ_NEXT(b, list);
+	}
+	if (DO_DEBUG(DBG_LURKER))
+		VSLb(vsl, SLT_Debug, "lurker: %d actionable bans", i);
+	if (i == 0)
+		return (0);
+
+	/* Go though all the bans to test the objects */
+	VTAILQ_FOREACH_REVERSE(bt, &ban_head, banhead_s, list) {
+		if (bt == VTAILQ_LAST(&obans, banhead_s)) {
+			if (DO_DEBUG(DBG_LURKER))
+				VSLb(vsl, SLT_Debug,
+				    "Lurk bt completed %p", bt);
+			Lck_Lock(&ban_mtx);
+			/* We can be raced by a new ban */
+			if (!(bt->flags & BANS_FLAG_COMPLETED))
+				ban_mark_completed(bt);
+			Lck_Unlock(&ban_mtx);
+			VTAILQ_REMOVE(&obans, bt, l_list);
+			if (VTAILQ_EMPTY(&obans))
+				break;
+		}
+		if (DO_DEBUG(DBG_LURKER))
+			VSLb(vsl, SLT_Debug, "Lurk bt %p", bt);
+		ban_lurker_test_ban(wrk, vsl, bt, &obans);
+		if (VTAILQ_EMPTY(&obans))
+			break;
+	}
+	return (1);
+}
+
+void * __match_proto__(bgthread_t)
+ban_lurker(struct worker *wrk, void *priv)
+{
+	struct vsl_log vsl;
+	volatile double d;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	AZ(priv);
+
+	VSL_Setup(&vsl, NULL, 0);
+
+	while (!ban_shutdown) {
+		d = cache_param->ban_lurker_sleep;
+		if (d <= 0.0 || !ban_lurker_work(wrk, &vsl))
+			d = 0.609;	// Random, non-magic
+		ban_cleantail();
+		VTIM_sleep(d);
+	}
+	pthread_exit(0);
+	NEEDLESS_RETURN(NULL);
+}