Give session memory pools a back pointer to their (thread) pool.

Schedule sessions from busy-lists and from the waiter via
SES_Schedule(), to Pool_Schedule() in the right pool.

bin/varnishd/cache.h

@@ -841,9 +841,9 @@ void PipeSession(struct sess *sp);
 
 /* cache_pool.c */
 void Pool_Init(void);
-int Pool_QueueSession(struct sess *sp);
 void Pool_Work_Thread(void *priv, struct worker *w);
 void Pool_Wait(struct sess *sp);
+int Pool_Schedule(struct pool *pp, struct sess *sp);
 
 #define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
 int WRW_Error(const struct worker *w);
@@ -864,8 +864,10 @@ struct sess *SES_Alloc(void);
 void SES_Close(struct sess *sp, const char *reason);
 void SES_Delete(struct sess *sp, const char *reason);
 void SES_Charge(struct sess *sp);
-struct sesspool *SES_NewPool(void);
+struct sesspool *SES_NewPool(struct pool *pp);
 void SES_DeletePool(struct sesspool *sp, struct worker *wrk);
+int SES_Schedule(struct sess *sp);
+
 
 /* cache_shmlog.c */
 void VSL_Init(void);

bin/varnishd/cache_hash.c

@@ -504,7 +504,7 @@ hsh_rush(struct objhead *oh)
 		AZ(sp->wrk);
 		VTAILQ_REMOVE(&wl->list, sp, list);
 		DSL(0x20, SLT_Debug, sp->id, "off waiting list");
-		if (Pool_QueueSession(sp)) {
+		if (SES_Schedule(sp)) {
 			/*
 			 * We could not schedule the session, leave the
 			 * rest on the busy list.

bin/varnishd/cache_pool.c

@@ -261,47 +261,33 @@ Pool_Work_Thread(void *priv, struct worker *w)
  */
 
 static int
-WRK_Queue(struct sess *sp)
+WRK_Queue(struct pool *pp, struct sess *sp)
 {
 	struct worker *w;
-	struct pool *qp;
-	static unsigned nq = 0;
-	unsigned onq;
 
-	/*
-	 * Select which pool we issue to
-	 * XXX: better alg ?
-	 * XXX: per CPU ?
-	 */
-	onq = nq + 1;
-	if (onq >= nwq)
-		onq = 0;
-	qp = wq[onq];
-	nq = onq;
-
-	Lck_Lock(&qp->mtx);
+	Lck_Lock(&pp->mtx);
 
 	/* If there are idle threads, we tickle the first one into action */
-	w = VTAILQ_FIRST(&qp->idle);
+	w = VTAILQ_FIRST(&pp->idle);
 	if (w != NULL) {
-		VTAILQ_REMOVE(&qp->idle, w, list);
-		Lck_Unlock(&qp->mtx);
+		VTAILQ_REMOVE(&pp->idle, w, list);
+		Lck_Unlock(&pp->mtx);
 		w->sp = sp;
 		AZ(pthread_cond_signal(&w->cond));
 		return (0);
 	}
 
 	/* If we have too much in the queue already, refuse. */
-	if (qp->lqueue > queue_max) {
-		qp->ndrop++;
-		Lck_Unlock(&qp->mtx);
+	if (pp->lqueue > queue_max) {
+		pp->ndrop++;
+		Lck_Unlock(&pp->mtx);
 		return (-1);
 	}
 
-	VTAILQ_INSERT_TAIL(&qp->queue, sp, poollist);
-	qp->nqueue++;
-	qp->lqueue++;
-	Lck_Unlock(&qp->mtx);
+	VTAILQ_INSERT_TAIL(&pp->queue, sp, poollist);
+	pp->nqueue++;
+	pp->lqueue++;
+	Lck_Unlock(&pp->mtx);
 	AZ(pthread_cond_signal(&herder_cond));
 	return (0);
 }
@@ -309,11 +295,11 @@ WRK_Queue(struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 int
-Pool_QueueSession(struct sess *sp)
+Pool_Schedule(struct pool *pp, struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk);
-	if (WRK_Queue(sp) == 0)
+	if (WRK_Queue(pp, sp) == 0)
 		return(0);
 
 	VSC_C_main->client_drop_late++;
@@ -332,7 +318,6 @@ Pool_QueueSession(struct sess *sp)
 		 */
 		VCL_Rel(&sp->vcl);
 	}
-	SES_Delete(sp, "dropped");
 	return (1);
 }
 
@@ -353,7 +338,7 @@ pool_mkpool(void)
 	VTAILQ_INIT(&pp->queue);
 	VTAILQ_INIT(&pp->idle);
 	VTAILQ_INIT(&pp->socks);
-	pp->sesspool = SES_NewPool();
+	pp->sesspool = SES_NewPool(pp);
 	AN(pp->sesspool);
 
 	VTAILQ_FOREACH(ls, &heritage.socks, list) {

bin/varnishd/cache_session.c

@@ -64,6 +64,7 @@ struct sessmem {
 struct sesspool {
 	unsigned		magic;
 #define SESSPOOL_MAGIC		0xd916e202
+	struct pool		*pool;
 	VTAILQ_HEAD(,sessmem)	freelist;
 	struct lock		mtx;
 	unsigned		nsess;
@@ -236,6 +237,35 @@ SES_Alloc(void)
 	return (sp);
 }
 
+/*--------------------------------------------------------------------
+ * Schedule a session back on a work-thread from its pool
+ */
+
+int
+SES_Schedule(struct sess *sp)
+{
+	struct sessmem *sm;
+	struct sesspool *pp;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+	AZ(sp->wrk);
+
+	sm = sp->mem;
+	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
+
+	pp = sm->pool;
+	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+
+	AN(pp->pool);
+
+	if (Pool_Schedule(pp->pool, sp)) {
+		SES_Delete(sp, "dropped");
+		return (1);
+	}
+	return (0);
+}
+
 /*--------------------------------------------------------------------
  * Handle a session (from waiter)
  *
@@ -255,7 +285,7 @@ SES_Handle(struct sess *sp, int status)
 		break;
 	case 1:
 		sp->step = STP_START;
-		(void)Pool_QueueSession(sp);
+		(void)SES_Schedule(sp);
 		break;
 	default:
 		WRONG("Unexpected return from HTC_Rx()");
@@ -355,12 +385,13 @@ SES_Delete(struct sess *sp, const char *reason)
  */
 
 struct sesspool *
-SES_NewPool(void)
+SES_NewPool(struct pool *pp)
 {
 	struct sesspool *sp;
 
 	ALLOC_OBJ(sp, SESSPOOL_MAGIC);
 	AN(sp);
+	sp->pool = pp;
 	VTAILQ_INIT(&sp->freelist);
 	Lck_New(&sp->mtx, lck_sessmem);
 	return (sp);