Implement runtime part of VCL controlled hashing.

The vcl_hash() is now used to control which fields go
into the hash algorithm, and the default is stil,
as previously, the URL + Host: header.

But now it is controlled by the vcl code, with the
default vcl_hash() being:

	sub vcl_hash {
		req.hash += req.url;
		req.hash += req.http.host;
		hash;
	}

Once I get a bit further, this will be changed to

	sub vcl_hash {
		req.hash += req.url;
		if (req.http.host) {
			req.hash += req.http.host;
		} else {
			req.hash += server.ip;
		}
		hash;
	}

So that we correctly hash HTTP requests without Host:
headers, that go to a machine with multiple IP numbers.

If you want to add fields to the hash, just write
a vcl_hash that does not end in "hash;":

	sub vcl_hash {
		req.hash += req.http.cookie;
	}

If you want to override the default vcl_hash, just
say so:

	sub vcl_hash {
		req.hash += req.url;
		hash;	// do not continue into default vcl_hash
	}




git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@1398 d4fa192b-c00b-0410-8231-f00ffab90ce4

bin/varnishd/cache.h

@@ -291,6 +291,9 @@ struct sess {
 
 	struct workreq		workreq;
 	struct acct		acct;
+
+	char			*hash_b;	/* Start of hash string */
+	char			*hash_e;	/* End of hash string */
 };
 
 struct backend {

bin/varnishd/cache_center.c

@@ -437,9 +437,18 @@ cnt_lookup(struct sess *sp)
 {
 	struct object *o;
 
-	VCL_hash_method(sp);		/* XXX: no-op for now */
+	assert(sp->http->f > sp->http->s);
+	assert(sp->http->f >= sp->http->t);
+	if (sp->obj == NULL) {
+		sp->hash_b = sp->http->f;
+		sp->hash_e = sp->hash_b;
+		VCL_hash_method(sp);		/* XXX: no-op for now */
+
+		/* XXX check error */
+	}
 
 	o = HSH_Lookup(sp);
+
 	if (o == NULL) {
 		/*
 		 * We hit a busy object, disembark worker thread and expect
@@ -451,6 +460,14 @@ cnt_lookup(struct sess *sp)
 		return (1);
 	}
 
+	xxxassert (sp->hash_e == sp->http->f);
+	if (sp->hash_e == sp->http->f) {
+		/* Nobody alloc'ed after us, free again */
+		sp->http->f = sp->hash_b;
+	}
+
+	sp->hash_b = sp->hash_e = NULL;
+
 	sp->obj = o;
 
 	/* If we inserted a new object it's a miss */

bin/varnishd/cache_hash.c

@@ -115,7 +115,6 @@ HSH_Lookup(struct sess *sp)
 	struct http *h;
 	struct objhead *oh;
 	struct object *o;
-	char *url, *host;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
@@ -125,9 +124,6 @@ HSH_Lookup(struct sess *sp)
 	h = sp->http;
 
 	HSH_Prealloc(sp);
-	url = h->hd[HTTP_HDR_URL].b;
-	if (!http_GetHdr(h, H_Host, &host))
-		host = url;
 	if (sp->obj != NULL) {
 		CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
 		o = sp->obj;
@@ -136,7 +132,9 @@ HSH_Lookup(struct sess *sp)
 		LOCK(&oh->mtx);
 		goto were_back;
 	}
-	oh = hash->lookup(url, host, w->nobjhead);
+VSLR(SLT_Debug, sp->fd, sp->hash_b, sp->hash_e);
+
+	oh = hash->lookup(sp->hash_b, sp->hash_e, w->nobjhead);
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 	if (oh == w->nobjhead)
 		w->nobjhead = NULL;
@@ -157,7 +155,7 @@ HSH_Lookup(struct sess *sp)
 			/* Object banned but not reaped yet */
 		} else if (o->ttl <= sp->t_req.tv_sec) {
 			/* Object expired */
-		} else if (BAN_CheckObject(o, url)) {
+		} else if (BAN_CheckObject(o, h->hd[HTTP_HDR_URL].b)) {
 			o->ttl = 0;
 			VSL(SLT_ExpBan, 0, "%u was banned", o->xid);
 			EXP_TTLchange(o);

bin/varnishd/cache_http.c

@@ -540,6 +540,7 @@ http_header_complete(struct http *hp)
 	hp->t = p;
 	assert(hp->t > hp->s);
 	assert(hp->t <= hp->v);
+	hp->f = hp->v;
 	return (1);
 }
 

bin/varnishd/cache_vrt.c

@@ -273,3 +273,21 @@ VRT_r_server_ip(struct sess *sp)
 
 	return ((struct sockaddr*)sp->mysockaddr);
 }
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_l_req_hash(struct sess *sp, const char *str)
+{
+	int l;
+
+	if (str == NULL)
+		str = "";
+	l = strlen(str);
+	xxxassert (sp->hash_e == sp->http->f);
+	xxxassert (sp->hash_e + l + 1 <= sp->http->e);
+	memcpy(sp->hash_e, str, l);
+	sp->hash_e[l] = '#';
+	sp->hash_e += l + 1;
+	sp->http->f += l + 1;
+}

bin/varnishd/hash_classic.c

@@ -121,22 +121,20 @@ hcl_start(void)
  */
 
 static struct objhead *
-hcl_lookup(const char *key1, const char *key2, struct objhead *noh)
+hcl_lookup(const char *b, const char *e, struct objhead *noh)
 {
 	struct hcl_entry *he, *he2;
 	struct hcl_hd *hp;
-	unsigned u1, digest, kl1, kl2, kl, r;
+	unsigned u1, digest, kl, r;
 	int i;
 
 	CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC);
 
-	digest = crc32_2s(key1, key2);
+	digest = crc32_l(b, e - b);
 
 	u1 = digest % hcl_nhash;
 	hp = &hcl_head[u1];
-	kl1 = strlen(key1) + 1;		/* Incl '/0' */
-	kl2 = strlen(key2);
-	kl = kl1 + kl2;
+	kl = e - b;
 	he2 = NULL;
 
 	for (r = 0; r < 2; r++ ) {
@@ -151,9 +149,7 @@ hcl_lookup(const char *key1, const char *key2, struct objhead *noh)
 				continue;
 			if (he->digest > digest)
 				break;
-			if (memcmp(he->key, key1, kl1))
-				continue;
-			if (memcmp(he->key + kl1, key2, kl2))
+			if (memcmp(he->key, b, kl))
 				continue;
 			he->refcnt++;
 			noh = he->oh;
@@ -190,8 +186,7 @@ hcl_lookup(const char *key1, const char *key2, struct objhead *noh)
 		noh->hashpriv = he2;
 
 		he2->key = (void*)(he2 + 1);
-		memcpy(he2->key, key1, kl1);
-		memcpy(he2->key + kl1, key2, kl2);
+		memcpy(he2->key, b, kl);
 	}
 	assert(he2 == NULL);		/* FlexeLint */
 	INCOMPL();

bin/varnishd/hash_simple_list.c

@@ -44,8 +44,8 @@
 
 struct hsl_entry {
 	TAILQ_ENTRY(hsl_entry)	list;
-	char			*key1;
-	char			*key2;
+	char			*key;
+	int			keylen;
 	struct objhead		*obj;
 	unsigned		refcnt;
 };
@@ -73,19 +73,19 @@ hsl_start(void)
  */
 
 static struct objhead *
-hsl_lookup(const char *key1, const char *key2, struct objhead *nobj)
+hsl_lookup(const char *b, const char *e, struct objhead *nobj)
 {
 	struct hsl_entry *he, *he2;
-	int i;
+	int i, l;
 
+	l = e - b;
 	LOCK(&hsl_mutex);
 	TAILQ_FOREACH(he, &hsl_head, list) {
-		i = strcmp(key1, he->key1);
-		if (i < 0)
+		if (l > he->keylen)
 			continue;
-		if (i > 0)
-			break;
-		i = strcmp(key2, he->key2);
+		if (l < he->keylen)
+			break;;
+		i = memcmp(b, he->key, l);
 		if (i < 0)
 			continue;
 		if (i > 0)
@@ -100,14 +100,13 @@ hsl_lookup(const char *key1, const char *key2, struct objhead *nobj)
 		UNLOCK(&hsl_mutex);
 		return (NULL);
 	}
-	he2 = calloc(sizeof *he2, 1);
+	he2 = calloc(sizeof *he2 + l, 1);
 	XXXAN(he2);
 	he2->obj = nobj;
 	he2->refcnt = 1;
-	he2->key1 = strdup(key1);
-	XXXAN(he2->key1);
-	he2->key2 = strdup(key2);
-	XXXAN(he2->key2);
+	he2->key = (void*)(he2 + 1);
+	he2->keylen = l;
+	memcpy(he2->key, b, l);
 	nobj->hashpriv = he2;
 	if (he != NULL)
 		TAILQ_INSERT_BEFORE(he, he2, list);
@@ -131,8 +130,6 @@ hsl_deref(struct objhead *obj)
 	he = obj->hashpriv;
 	LOCK(&hsl_mutex);
 	if (--he->refcnt == 0) {
-		free(he->key1);
-		free(he->key2);
 		TAILQ_REMOVE(&hsl_head, he, list);
 		free(he);
 		ret = 0;

bin/varnishd/mgt_vcc.c

@@ -88,6 +88,16 @@ static const char *default_vcl =
     "}\n"
     "\n"
     "sub vcl_hash {\n"
+    "    set req.hash += req.url;\n"
+#if 1
+    "    set req.hash += req.http.host;\n"
+#else
+    "    if (req.http.host) {\n"
+    "        set req.hash += req.http.host;\n"
+    "    } else {\n"
+    "        set req.hash += server.ip;\n"
+    "    }\n"
+#endif
     "    hash;\n"
     "}\n"
     "\n"