Let streaming delivery free the storage when done with it for PASS

requests.

This should make it possible to deliver arbitrarily sized objects
with pass mode.

bin/varnishd/cache.h

@@ -263,13 +263,18 @@ struct wrw {
 struct stream_ctx {
 	unsigned		magic;
 #define STREAM_CTX_MAGIC	0x8213728b
+
 #if 0
 	struct vgz		*vgz;
 	void			*obuf;
 	ssize_t			obuf_len;
 	ssize_t			obuf_ptr;
 #endif
+	/* Next byte we will take from storage */
 	ssize_t			stream_next;
+
+	/* First byte of storage if we free it as we go (pass) */
+	ssize_t			stream_front;
 };
 
 /*--------------------------------------------------------------------*/

bin/varnishd/cache_fetch.c

@@ -576,7 +576,11 @@ FetchBody(struct sess *sp)
 		uu = 0;
 		VTAILQ_FOREACH(st, &sp->obj->store, list)
 			uu += st->len;
-		assert(uu == sp->obj->len);
+		if (sp->objcore == NULL || (sp->objcore->flags & OC_F_PASS))
+			/* Streaming might have started freeing stuff */
+			assert (uu <= sp->obj->len);
+		else
+			assert(uu == sp->obj->len);
 	}
 
 	if (mklen > 0) {

bin/varnishd/cache_response.c

@@ -36,6 +36,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "stevedore.h"
 #include "vct.h"
 
 /*--------------------------------------------------------------------*/
@@ -429,7 +430,7 @@ RES_StreamPoll(const struct sess *sp)
 	if (sp->obj->len == sctx->stream_next)
 		return;
 	assert(sp->obj->len > sctx->stream_next);
-	l = 0;
+	l = sctx->stream_front;
 	VTAILQ_FOREACH(st, &sp->obj->store, list) {
 		if (st->len + l <= sctx->stream_next) {
 			l += st->len;
@@ -442,6 +443,22 @@ RES_StreamPoll(const struct sess *sp)
 		sctx->stream_next += l2;
 	}
 	(void)WRW_Flush(sp->wrk);
+
+	if (sp->objcore == NULL || (sp->objcore->flags & OC_F_PASS)) {
+		/*
+		 * This is a pass object, release storage as soon as we
+		 * have delivered it.
+		 */
+		while (1) {
+			st = VTAILQ_FIRST(&sp->obj->store);
+			if (st == NULL ||
+			    sctx->stream_front + st->len > sctx->stream_next)
+				break;
+			VTAILQ_REMOVE(&sp->obj->store, st, list);
+			sctx->stream_front += st->len;
+			STV_free(st);
+		}
+	}
 }
 
 void