Add preliminary version of lock-less tree based lookup (see below)

Enable SHA256 digests by default, and put it in the objhead.  This
increases the size of the objhead by 32 bytes, but may drop
a bit again later, when other now unnecessary fields go away.

Test SHA256 for correct operation on startup.

About the "critbit" lookup:

To enable this, use "-hcritbit" argument.

"Crit Bit" trees, are also known under various other names, the original
version of the idea is probably the PATRICIA tree.

The basic concept is a tree structure which has nodes only where necessary
to tell the indices apart.

Our version of it, has some additional bells and whistles.

First lookups do not require any locks until we reach the objhead
we were looking for, or until we need to insert one which wasn't
there.

Second, the branch nodes are part of the objhead, as all but the
very first will need one, this saves malloc operations big time.

Now the down-sides:

There are still missing bits, amongst these the "cooling off" list,
for objheads that have been dereferenced, but where the branch-node
is not.  Currently we just leak that memory.

There is a race relating to node deref and unlocked lookup that is
not closed, weird things may happen until I fix it.

I'd be interested to hear how long it survives before it croaks,
but apart from that, would not advocate that you use it, until
I fix those remaining issues.



git-svn-id: http://www.varnish-cache.org/svn/trunk@3454 d4fa192b-c00b-0410-8231-f00ffab90ce4

bin/varnishd/Makefile.am

@@ -40,6 +40,7 @@ varnishd_SOURCES = \
 	cache_vrt_re.c \
 	cache_ws.c \
 	hash_classic.c \
+	hash_critbit.c \
 	hash_simple_list.c \
 	instance.c \
 	mgt_child.c \

bin/varnishd/cache_hash.c

@@ -251,7 +251,6 @@ HSH_Lookup(struct sess *sp)
 	struct http *h;
 	struct objhead *oh;
 	struct object *o, *busy_o, *grace_o;
-	unsigned char sha256[32];
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
@@ -259,12 +258,14 @@ HSH_Lookup(struct sess *sp)
 	AN(hash);
 	w = sp->wrk;
 	h = sp->http;
+
+	HSH_Prealloc(sp);
 	if (params->hash_sha256) {
-		SHA256_Final(sha256, sp->wrk->sha256ctx);
+		SHA256_Final(sp->wrk->nobjhead->digest, sp->wrk->sha256ctx);
+		sp->wrk->nobjhead->digest_len = 32;
 		/* WSP(sp, SLT_Debug, "SHA256: <%.32s>", sha256); */
 	}
 	
-	HSH_Prealloc(sp);
 	if (sp->objhead != NULL) {
 		CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC);
 		oh = sp->objhead;

bin/varnishd/hash_slinger.h

@@ -73,14 +73,24 @@ struct objhead {
 	VTAILQ_HEAD(,object)	objects;
 	char			*hash;
 	unsigned		hashlen;
+	unsigned char		digest[32];
+	unsigned char		digest_len;
 	VTAILQ_HEAD(, sess)	waitinglist;
 
-	/*------------------------------------------------------------
-	 * The fields below are for the sole private use of the hash 
-	 * implementation.
+	/*----------------------------------------------------
+	 * The fields below are for the sole private use of
+	 * the hash implementation(s).
 	 */
-	VTAILQ_ENTRY(objhead)	hoh_list;
-	void			*hoh_head;
-	unsigned		hoh_digest;
+	union {
+		void		*filler[3];
+		struct {
+			VTAILQ_ENTRY(objhead)	u_n_hoh_list;
+			void			*u_n_hoh_head;
+			unsigned		u_n_hoh_digest;
+		} n;
+	} u;
+#define hoh_list u.n.u_n_hoh_list
+#define hoh_head u.n.u_n_hoh_head
+#define hoh_digest u.n.u_n_hoh_digest
 };
 #endif /* VARNISH_CACHE_CHILD */

bin/varnishd/mgt_param.c

@@ -713,7 +713,7 @@ static const struct parspec input_parspec[] = {
 	{ "hash_sha256", tweak_bool, &master.hash_sha256, 0, 0,
 		"Use SHA256 compression of hash-strings",
 		0,
-		"off", "bool" },
+		"on", "bool" },
 	{ "log_hashstring", tweak_bool, &master.log_hash, 0, 0,
 		"Log the hash string to shared memory log.\n",
 		0,

bin/varnishd/varnishd.c

@@ -59,6 +59,7 @@
 
 #include "vsb.h"
 #include "vpf.h"
+#include "vsha256.h"
 
 #include "cli.h"
 #include "cli_priv.h"
@@ -157,11 +158,13 @@ setup_storage(const char *spec)
 
 extern struct hash_slinger hsl_slinger;
 extern struct hash_slinger hcl_slinger;
+extern struct hash_slinger hcb_slinger;
 
 static const struct choice hsh_choice[] = {
 	{ "classic",		&hcl_slinger },
 	{ "simple",		&hsl_slinger },
 	{ "simple_list",	&hsl_slinger },	/* backwards compat */
+	{ "critbit",		&hcb_slinger },
 	{ NULL,			NULL }
 };
 
@@ -450,6 +453,11 @@ main(int argc, char * const *argv)
 	assert(TIM_parse("Sunday, 06-Nov-94 08:49:37 GMT") == 784111777);
 	assert(TIM_parse("Sun Nov  6 08:49:37 1994") == 784111777);
 
+	/*
+	 * Check that our SHA256 works
+	 */
+	SHA256_Test();
+
 	memset(cli, 0, sizeof cli);
 	cli[0].sb = vsb_newauto();
 	XXXAN(cli[0].sb);

include/stat_field.h

@@ -126,3 +126,7 @@ MAC_STAT(n_purge_retire,	uint64_t, 'a', "N old purges deleted")
 MAC_STAT(n_purge_obj_test,	uint64_t, 'a', "N objects tested")
 MAC_STAT(n_purge_re_test,	uint64_t, 'a', "N regexps tested against")
 MAC_STAT(n_purge_dups,		uint64_t, 'a', "N duplicate purges removed")
+
+MAC_STAT(hcb_nolock,		uint64_t, 'a', "HCB Lookups without lock")
+MAC_STAT(hcb_lock,		uint64_t, 'a', "HCB Lookups with lock")
+MAC_STAT(hcb_insert,		uint64_t, 'a', "HCB Inserts")