Don't let stream threads muck around in the stream list.

This should fix some H2 tickets.

bin/varnishd/http2/cache_http2_proto.c

@@ -171,6 +171,7 @@ h2_del_req(struct worker *wrk, struct h2_req *r2)
 	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
 	h2 = r2->h2sess;
 	CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC);
+	ASSERT_RXTHR(h2);
 	sp = h2->sess;
 	Lck_Lock(&sp->mtx);
 	assert(h2->refcnt > 0);
@@ -196,20 +197,19 @@ h2_kill_req(struct worker *wrk, const struct h2_sess *h2,
     struct h2_req *r2, h2_error h2e)
 {
 
+	ASSERT_RXTHR(h2);
+	AN(h2e);
 	VSLb(h2->vsl, SLT_Debug, "KILL st=%u state=%d", r2->stream, r2->state);
 	Lck_Lock(&h2->sess->mtx);
-	r2->error = h2e;
-	switch (r2->state) {
-	case H2_S_CLOS_REM:
+	if (r2->error == NULL)
+		r2->error = h2e;
+	if (r2->scheduled) {
 		if (r2->cond != NULL)
 			AZ(pthread_cond_signal(r2->cond));
 		r2 = NULL;
-		break;
-	case H2_S_OPEN:
-		(void)h2h_decode_fini(h2, r2->decode);
-		break;
-	default:
-		break;
+	} else {
+		if (r2->state == H2_S_OPEN)
+			(void)h2h_decode_fini(h2, r2->decode);
 	}
 	Lck_Unlock(&h2->sess->mtx);
 	if (r2 != NULL)
@@ -304,9 +304,9 @@ h2_rx_rst_stream(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 
 	if (h2->rxf_len != 4)			// rfc7540,l,2003,2004
 		return (H2CE_FRAME_SIZE_ERROR);
-	if (r2 != NULL)
-		h2_kill_req(wrk, h2, r2,
-		    h2_streamerror(vbe32dec(h2->rxf_data)));
+	if (r2 == NULL)
+		return (0);
+	h2_kill_req(wrk, h2, r2, h2_streamerror(vbe32dec(h2->rxf_data)));
 	return (0);
 }
 
@@ -473,7 +473,8 @@ h2_do_req(struct worker *wrk, void *priv)
 		AZ(req->ws->r);
 		r2->scheduled = 0;
 		r2->state = H2_S_CLOSED;
-		h2_del_req(wrk, r2);
+		if (r2->h2sess->error)
+			AZ(pthread_cond_signal(r2->h2sess->cond));
 	}
 	THR_SetRequest(NULL);
 }
@@ -744,7 +745,7 @@ static h2_error
 h2_procframe(struct worker *wrk, struct h2_sess *h2,
     h2_frame h2f)
 {
-	struct h2_req *r2 = NULL;
+	struct h2_req *r2 = NULL, *r22;
 	h2_error h2e;
 	char b[4];
 
@@ -767,9 +768,12 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2,
 		return (H2CE_PROTOCOL_ERROR);
 	}
 
-	VTAILQ_FOREACH(r2, &h2->streams, list)
+	VTAILQ_FOREACH_SAFE(r2, &h2->streams, list, r22) {
 		if (r2->stream == h2->rxf_stream)
 			break;
+		if (r2->state == H2_S_CLOSED && !r2->scheduled)
+			h2_del_req(wrk, r2);
+	}
 
 	if (r2 == NULL && h2f->act_sidle == 0) {
 		if (h2->rxf_stream <= h2->highest_stream)

bin/varnishd/http2/cache_http2_session.c

@@ -275,8 +275,9 @@ h2_new_session(struct worker *wrk, void *arg)
 	struct req *req;
 	struct sess *sp;
 	struct h2_sess *h2;
-	struct h2_req *r2;
+	struct h2_req *r2, *r22;
 	uintptr_t wsp;
+	int again;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CAST_OBJ_NOTNULL(req, arg, REQ_MAGIC);
@@ -316,32 +317,32 @@ h2_new_session(struct worker *wrk, void *arg)
 	}
 
 	AN(h2->error);
+
 	/* Delete all idle streams */
-	Lck_Lock(&sp->mtx);
 	VSLb(h2->vsl, SLT_Debug, "H2 CLEANUP %s", h2->error->name);
 	VTAILQ_FOREACH(r2, &h2->streams, list) {
 		if (r2->error == 0)
 			r2->error = h2->error;
-#if 0
-		if (r2->wrk != NULL)
-			AZ(pthread_cond_signal(&r2->wrk->cond)); // XXX Why?
-#endif
+		if (r2->cond != NULL)
+			AZ(pthread_cond_signal(r2->cond));
 	}
 	AZ(pthread_cond_signal(h2->cond));
-	while (1) {
-		VTAILQ_FOREACH(r2, &h2->streams, list)
-			if (r2->state != H2_S_CLOS_REM && r2 != h2->req0)
-				break;
-		if (r2 == NULL)
+	while(1) {
+		again = 0;
+		VTAILQ_FOREACH_SAFE(r2, &h2->streams, list, r22) {
+			if (r2 != h2->req0) {
+				h2_kill_req(wrk, h2, r2, h2->error);
+				again++;
+			}
+		}
+		if (!again)
 			break;
-		Lck_Unlock(&sp->mtx);
-		h2_kill_req(wrk, h2, r2, h2->error);
-		Lck_Lock(&sp->mtx);
+		VTAILQ_FOREACH(r2, &h2->streams, list)
+			VSLb(h2->vsl, SLT_Debug, "ST %u %d", r2->stream, r2->state);
+		Lck_Lock(&h2->sess->mtx);
+		(void)Lck_CondWait(h2->cond, &h2->sess->mtx, .1);
+		Lck_Unlock(&h2->sess->mtx);
 	}
-	VSLb(h2->vsl, SLT_Debug, "H2CLEAN done");
-	VTAILQ_FOREACH(r2, &h2->streams, list)
-		VSLb(h2->vsl, SLT_Debug, "ST %u %d", r2->stream, r2->state);
-	Lck_Unlock(&sp->mtx);
 	h2->cond = NULL;
 	h2_del_req(wrk, h2->req0);
 }