Now that we do not need the new object while holding the highly trafficed

objhead lock, don't preallocate objects.

One unfortunate effect of this preallocation, only recently identified:

On systems with high number of worker threads and high hitrates, considerable
time could elapse between the preallocation of an object to a worker
thread and the use of that object.

This could make cache-misses use a much larger working set size than
really necessary.

I do not have access to any benchmarks that show a credible performance
difference, but reports are most welcome.




git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@3842 d4fa192b-c00b-0410-8231-f00ffab90ce4

bin/varnishd/cache.h

@@ -194,7 +194,6 @@ struct worker {
 	unsigned		magic;
 #define WORKER_MAGIC		0x6391adcf
 	struct objhead		*nobjhead;
-	struct object		*nobj;
 	struct objcore		*nobjcore;
 	struct dstat		*stats;
 

bin/varnishd/cache_center.c

@@ -314,10 +314,9 @@ cnt_error(struct sess *sp)
 	w = sp->wrk;
 	if (sp->obj == NULL) {
 		HSH_Prealloc(sp);
-		sp->obj = sp->wrk->nobj;
+		sp->obj = HSH_NewObject(sp);
 		sp->obj->xid = sp->xid;
 		sp->obj->entered = sp->t_req;
-		sp->wrk->nobj = NULL;
 	} else {
 		/* XXX: Null the headers ? */
 	}
@@ -683,8 +682,7 @@ cnt_lookup(struct sess *sp)
 		VSL_stats->cache_miss++;
 
 		AZ(oc->obj);
-		o = sp->wrk->nobj;
-		sp->wrk->nobj = NULL;
+		o = HSH_NewObject(sp);
 
 		o->objhead = oh;
 		o->objcore = oc;
@@ -821,8 +819,7 @@ cnt_pass(struct sess *sp)
 	assert(sp->handling == VCL_RET_PASS);
 	sp->acct_req.pass++;
 	HSH_Prealloc(sp);
-	sp->obj = sp->wrk->nobj;
-	sp->wrk->nobj = NULL;
+	sp->obj = HSH_NewObject(sp);
 	sp->sendbody = 1;
 	sp->step = STP_FETCH;
 	return (0);
@@ -1078,7 +1075,6 @@ CNT_Session(struct sess *sp)
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 		CHECK_OBJ_ORNULL(sp->obj, OBJECT_MAGIC);
 		CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-		CHECK_OBJ_ORNULL(w->nobj, OBJECT_MAGIC);
 		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
 
 		switch (sp->step) {
@@ -1093,7 +1089,6 @@ CNT_Session(struct sess *sp)
 		default:
 			WRONG("State engine misfire");
 		}
-		CHECK_OBJ_ORNULL(w->nobj, OBJECT_MAGIC);
 		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
 	}
 	WSL_Flush(w, 0);

bin/varnishd/cache_hash.c

@@ -79,15 +79,42 @@ HSH_Grace(double g)
 	return (g);
 }
 
+struct object *
+HSH_NewObject(struct sess *sp)
+{
+	struct object *o;
+	struct storage *st;
+
+	st = STV_alloc(sp, params->obj_workspace);
+	XXXAN(st);
+	assert(st->space > sizeof *o);
+	o = (void *)st->ptr; /* XXX: align ? */
+	st->len = sizeof *o;
+	memset(o, 0, sizeof *o);
+	o->objstore = st;
+	WS_Init(o->ws_o, "obj",
+	    st->ptr + st->len, st->space - st->len);
+	st->len = st->space;
+	WS_Assert(o->ws_o);
+	http_Setup(o->http, o->ws_o);
+	o->magic = OBJECT_MAGIC;
+	o->http->magic = HTTP_MAGIC;
+	o->refcnt = 1;
+	o->grace = NAN;
+	o->entered = NAN;
+	VTAILQ_INIT(&o->store);
+	VTAILQ_INIT(&o->esibits);
+	sp->wrk->stats->n_object++;
+	return (o);
+}
+
 /* Precreate an objhead and object for later use */
 void
-HSH_Prealloc(struct sess *sp)
+HSH_Prealloc(const struct sess *sp)
 {
 	struct worker *w;
 	struct objhead *oh;
 	struct objcore *oc;
-	struct object *o;
-	struct storage *st;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
@@ -113,31 +140,6 @@ HSH_Prealloc(struct sess *sp)
 	}
 	CHECK_OBJ_NOTNULL(w->nobjhead, OBJHEAD_MAGIC);
 
-	if (w->nobj == NULL) {
-		st = STV_alloc(sp, params->obj_workspace);
-		XXXAN(st);
-		assert(st->space > sizeof *w->nobj);
-		o = (void *)st->ptr; /* XXX: align ? */
-		st->len = sizeof *o;
-		memset(o, 0, sizeof *o);
-		o->objstore = st;
-		WS_Init(o->ws_o, "obj",
-		    st->ptr + st->len, st->space - st->len);
-		st->len = st->space;
-		WS_Assert(o->ws_o);
-		http_Setup(o->http, o->ws_o);
-		o->magic = OBJECT_MAGIC;
-		o->http->magic = HTTP_MAGIC;
-		o->refcnt = 1;
-		o->grace = NAN;
-		o->entered = NAN;
-		VTAILQ_INIT(&o->store);
-		VTAILQ_INIT(&o->esibits);
-		w->nobj = o;
-		w->stats->n_object++;
-
-	}
-	CHECK_OBJ_NOTNULL(w->nobj, OBJECT_MAGIC);
 }
 
 void
@@ -154,11 +156,6 @@ HSH_Cleanup(struct worker *w)
 		w->nobjhead = NULL;
 		w->stats->n_objecthead--;
 	}
-	if (w->nobj != NULL) {
-		STV_free(w->nobj->objstore);
-		w->nobj = NULL;
-		w->stats->n_object--;
-	}
 }
 
 void

bin/varnishd/cache_pool.c

@@ -431,10 +431,8 @@ wrk_do_cnt_sess(struct worker *w, void *priv)
 	AZ(sess->wrk);
 	THR_SetSession(sess);
 	sess->wrk = w;
-	CHECK_OBJ_ORNULL(w->nobj, OBJECT_MAGIC);
 	CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
 	CNT_Session(sess);
-	CHECK_OBJ_ORNULL(w->nobj, OBJECT_MAGIC);
 	CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
 	THR_SetSession(NULL);
 }

bin/varnishd/hash_slinger.h

@@ -50,7 +50,8 @@ struct hash_slinger {
 };
 
 /* cache_hash.c */
-void HSH_Prealloc(struct sess *sp);
+struct object *HSH_NewObject(struct sess *sp);
+void HSH_Prealloc(const struct sess *sp);
 void HSH_Cleanup(struct worker *w);
 void HSH_Freestore(struct object *o);
 void HSH_Copy(const struct sess *sp, struct objhead *o);