Give the srcaddr stats an overhaul.

Make the number of buckets a parameter (needs restart though).

Make the ttl a parameter and have zero disable srcaddr accounting.

Give each hash bucket its own mutex.


git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@1033 d4fa192b-c00b-0410-8231-f00ffab90ce4

bin/varnishd/cache.h

@@ -213,22 +213,6 @@ struct objhead {
 
 /* -------------------------------------------------------------------*/
 
-struct srcaddr {
-	unsigned		magic;
-#define SRCADDR_MAGIC		0x375111db
-
-	unsigned		hash;
-	TAILQ_ENTRY(srcaddr)	list;
-	struct srcaddrhead	*sah;
-
-	char			addr[TCP_ADDRBUFSIZE];
-	unsigned		nref;
-
-	time_t			ttl;
-
-	struct acct		acct;
-};
-
 struct sess {
 	unsigned		magic;
 #define SESS_MAGIC		0x2c2f9c5a

bin/varnishd/cache_session.c

@@ -30,9 +30,6 @@
 #include "shmlog.h"
 #include "cache.h"
 
-#define CLIENT_HASH			1024
-#define CLIENT_TTL			30
-
 /*--------------------------------------------------------------------*/
 
 struct sessmem {
@@ -46,20 +43,41 @@ struct sessmem {
 	TAILQ_ENTRY(sessmem)	list;
 };
 
-/*--------------------------------------------------------------------*/
-
 static TAILQ_HEAD(,sessmem)	ses_free_mem[2] = {
     TAILQ_HEAD_INITIALIZER(ses_free_mem[0]),
     TAILQ_HEAD_INITIALIZER(ses_free_mem[1]),
 };
 
 static unsigned ses_qp;
+static MTX			ses_mem_mtx;
+
+/*--------------------------------------------------------------------*/
+
+struct srcaddr {
+	unsigned		magic;
+#define SRCADDR_MAGIC		0x375111db
+
+	unsigned		hash;
+	TAILQ_ENTRY(srcaddr)	list;
+	struct srcaddrhead	*sah;
+
+	char			addr[TCP_ADDRBUFSIZE];
+	unsigned		nref;
+
+	time_t			ttl;
+
+	struct acct		acct;
+};
 
-TAILQ_HEAD(srcaddrhead ,srcaddr);
-static struct srcaddrhead	srcaddr_hash[CLIENT_HASH];
-static MTX			ses_mtx;
+static struct srcaddrhead {
+	unsigned		magic;
+#define SRCADDRHEAD_MAGIC	0x38231a8b
+	TAILQ_HEAD(,srcaddr)	head;
+	MTX			mtx;
+} *srchash;
+	
+unsigned			nsrchash;
 static MTX			stat_mtx;
-static MTX			ses_mem_mtx;
 
 /*--------------------------------------------------------------------
  * Assign a srcaddr to this session.
@@ -77,29 +95,34 @@ SES_RefSrcAddr(struct sess *sp)
 	struct srcaddrhead *ch;
 	time_t now;
 
+	if (params->srcaddr_ttl == 0) {
+		sp->srcaddr = NULL;
+		return;
+	}
 	AZ(sp->srcaddr);
 	u = crc32_2s(sp->addr, "");
-	v = u % CLIENT_HASH;
-	ch = &srcaddr_hash[v];
+	v = u % nsrchash;
+	ch = &srchash[v];
+	CHECK_OBJ(ch, SRCADDRHEAD_MAGIC);
 	now = sp->t_open.tv_sec;
 
-	LOCK(&ses_mtx);
+	LOCK(&ch->mtx);
 	c3 = NULL;
-	TAILQ_FOREACH_SAFE(c, ch, list, c2) {
+	TAILQ_FOREACH_SAFE(c, &ch->head, list, c2) {
 		if (c->hash == u && !strcmp(c->addr, sp->addr)) {
 			if (c->nref == 0)
 				VSL_stats->n_srcaddr_act++;
 			c->nref++;
-			c->ttl = now + CLIENT_TTL;
+			c->ttl = now + params->srcaddr_ttl;
 			sp->srcaddr = c;
-			TAILQ_REMOVE(ch, c, list);
-			TAILQ_INSERT_TAIL(ch, c, list);
+			TAILQ_REMOVE(&ch->head, c, list);
+			TAILQ_INSERT_TAIL(&ch->head, c, list);
 			if (0 && c3 != NULL) {
-				TAILQ_REMOVE(ch, c3, list);
+				TAILQ_REMOVE(&ch->head, c3, list);
 				VSL_stats->n_srcaddr--;
 				free(c3);
 			}
-			UNLOCK(&ses_mtx);
+			UNLOCK(&ch->mtx);
 			return;
 		}
 		if (c->nref > 0 || c->ttl > now)
@@ -108,7 +131,7 @@ SES_RefSrcAddr(struct sess *sp)
 			c3 = c;
 			continue;
 		}
-		TAILQ_REMOVE(ch, c, list);
+		TAILQ_REMOVE(&ch->head, c, list);
 		free(c);
 		VSL_stats->n_srcaddr--;
 	}
@@ -118,23 +141,45 @@ SES_RefSrcAddr(struct sess *sp)
 		if (c3 != NULL)
 			VSL_stats->n_srcaddr++;
 	} else
-		TAILQ_REMOVE(ch, c3, list);
+		TAILQ_REMOVE(&ch->head, c3, list);
 	AN(c3);
 	if (c3 != NULL) {
 		memset(c3, 0, sizeof *c3);
 		strcpy(c3->addr, sp->addr);
 		c3->hash = u;
 		c3->acct.first = now;
-		c3->ttl = now + CLIENT_TTL;
+		c3->ttl = now + params->srcaddr_ttl;
 		c3->nref = 1;
 		c3->sah = ch;
 		VSL_stats->n_srcaddr_act++;
-		TAILQ_INSERT_TAIL(ch, c3, list);
+		TAILQ_INSERT_TAIL(&ch->head, c3, list);
 		sp->srcaddr = c3;
 	}
-	UNLOCK(&ses_mtx);
+	UNLOCK(&ch->mtx);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+ses_relsrcaddr(struct sess *sp)
+{
+	struct srcaddrhead *ch;
+
+	if (sp->srcaddr == NULL)
+		return;
+	ch = sp->srcaddr->sah;
+	CHECK_OBJ(ch, SRCADDRHEAD_MAGIC);
+	LOCK(&ch->mtx);
+	assert(sp->srcaddr->nref > 0);
+	sp->srcaddr->nref--;
+	if (sp->srcaddr->nref == 0)
+		VSL_stats->n_srcaddr_act--;
+	sp->srcaddr = NULL;
+	UNLOCK(&ch->mtx);
 }
 
+/*--------------------------------------------------------------------*/
+
 static void
 ses_sum_acct(struct acct *sum, struct acct *inc)
 {
@@ -152,16 +197,19 @@ void
 SES_Charge(struct sess *sp)
 {
 	struct acct *a = &sp->wrk->acct;
-	struct acct *b = &sp->srcaddr->acct;
+	struct acct *b;
 
 	ses_sum_acct(&sp->acct, a);
 	
 	LOCK(&stat_mtx);
-	ses_sum_acct(b, a);
-	VSL(SLT_StatAddr, 0, "%s 0 %d %ju %ju %ju %ju %ju %ju %ju",
-	    sp->srcaddr->addr, sp->t_end.tv_sec - b->first,
-	    b->sess, b->req, b->pipe, b->pass,
-	    b->fetch, b->hdrbytes, b->bodybytes);
+	if (sp->srcaddr != NULL) {
+		b = &sp->srcaddr->acct;
+		ses_sum_acct(b, a);
+		VSL(SLT_StatAddr, 0, "%s 0 %d %ju %ju %ju %ju %ju %ju %ju",
+		    sp->srcaddr->addr, sp->t_end.tv_sec - b->first,
+		    b->sess, b->req, b->pipe, b->pass,
+		    b->fetch, b->hdrbytes, b->bodybytes);
+	}
 	VSL_stats->s_sess += a->sess;
 	VSL_stats->s_req += a->req;
 	VSL_stats->s_pipe += a->pipe;
@@ -173,24 +221,6 @@ SES_Charge(struct sess *sp)
 	memset(a, 0, sizeof *a);
 }
 
-static void
-ses_relsrcaddr(struct sess *sp)
-{
-
-	if (sp->srcaddr == NULL) {
-		/* If we never get to work pool (illegal req) */
-		return;
-	}
-	AN(sp->srcaddr);
-	LOCK(&ses_mtx);
-	assert(sp->srcaddr->nref > 0);
-	sp->srcaddr->nref--;
-	if (sp->srcaddr->nref == 0)
-		VSL_stats->n_srcaddr_act--;
-	sp->srcaddr = NULL;
-	UNLOCK(&ses_mtx);
-}
-
 /*--------------------------------------------------------------------*/
 
 struct sess *
@@ -286,9 +316,14 @@ SES_Init()
 {
 	int i;
 
-	for (i = 0; i < CLIENT_HASH; i++)
-		TAILQ_INIT(&srcaddr_hash[i]);
-	MTX_INIT(&ses_mtx);
+	nsrchash = params->srcaddr_hash;
+	srchash = calloc(sizeof *srchash, nsrchash);
+	XXXAN(srchash);
+	for (i = 0; i < nsrchash; i++) {
+		srchash[i].magic = SRCADDRHEAD_MAGIC;
+		TAILQ_INIT(&srchash[i].head);
+		MTX_INIT(&srchash[i].mtx);
+	}
 	MTX_INIT(&stat_mtx);
 	MTX_INIT(&ses_mem_mtx);
 }

bin/varnishd/heritage.h

@@ -65,6 +65,10 @@ struct params {
 
 	/* Listen depth */
 	unsigned		listen_depth;
+
+	/* Srcaddr hash */
+	unsigned		srcaddr_hash;
+	unsigned		srcaddr_ttl;
 };
 
 extern struct params *params;

bin/varnishd/mgt_param.c

@@ -282,6 +282,24 @@ tweak_listen_depth(struct cli *cli, struct parspec *par, const char *arg)
 
 /*--------------------------------------------------------------------*/
 
+static void
+tweak_srcaddr_hash(struct cli *cli, struct parspec *par, const char *arg)
+{
+	(void)par;
+	tweak_generic_uint(cli, &params->srcaddr_hash, arg, 64, UINT_MAX);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+tweak_srcaddr_ttl(struct cli *cli, struct parspec *par, const char *arg)
+{
+	(void)par;
+	tweak_generic_uint(cli, &params->srcaddr_ttl, arg, 0, UINT_MAX);
+}
+
+/*--------------------------------------------------------------------*/
+
 /*
  * Make sure to end all lines with either a space or newline of the
  * formatting will go haywire.
@@ -372,6 +390,13 @@ static struct parspec parspec[] = {
 		"Listen(2) queue depth.\n"
 		MUST_RESTART,
 		"1024", "connections" },
+	{ "srcaddr_hash", tweak_srcaddr_hash,
+		"Number of source address hash buckets.\n"
+		MUST_RESTART,
+		"1024", "buckets" },
+	{ "srcaddr_ttl", tweak_srcaddr_ttl,
+		"Lifetime of srcaddr entries.\n",
+		"30", "seconds" },
 	{ NULL, NULL, NULL }
 };