Implement trimming for streamed objects in simple stevedores.

bin/varnishd/cache/cache.h

@@ -365,12 +365,10 @@ struct lru {
 };
 
 /* Stored object -----------------------------------------------------
- * Pointer to a stored object, and the methods it supports
+ * This is just to encapsulate the fields owned by the stevedore
  */
 
 struct storeobj {
-	unsigned		magic;
-#define STOREOBJ_MAGIC		0x6faed850
 	const struct stevedore	*stevedore;
 	void			*priv;
 	uintptr_t		priv2;
@@ -502,6 +500,8 @@ struct busyobj {
 
 	struct vsl_log		vsl[1];
 
+	void			*stevedore_priv;
+
 	uint8_t			digest[DIGEST_LEN];
 	struct vrt_privs	privs[1];
 };

bin/varnishd/storage/storage_persistent.c

@@ -585,22 +585,6 @@ smp_alloc(const struct stevedore *st, size_t size)
 	    size > 4096 ? 4096 : size, size, NULL, NULL, NULL));
 }
 
-/*--------------------------------------------------------------------
- * We don't track frees of storage, we track the objects which own the
- * storage and when there are no more objects in in the first segment,
- * it can be reclaimed.
- * XXX: We could free the last allocation, but does that happen ?
- */
-
-static void __match_proto__(storage_free_f)
-smp_free(struct storage *st)
-{
-
-	/* XXX */
-	(void)st;
-}
-
-
 /*--------------------------------------------------------------------*/
 
 const struct stevedore smp_stevedore = {
@@ -611,7 +595,7 @@ const struct stevedore smp_stevedore = {
 	.close	=	smp_close,
 	.alloc	=	smp_alloc,
 	.allocobj =	smp_allocobj,
-	.free	=	smp_free,
+	.free	=	NULL,
 	.signal_close = smp_signal_close,
 	.baninfo =	smp_baninfo,
 	.banexport =	smp_banexport,

bin/varnishd/storage/storage_persistent_silo.c

@@ -528,7 +528,7 @@ smp_oc_objgetlru(const struct objcore *oc)
 
 const struct obj_methods smp_oc_methods = {
 	.sml_getobj =		smp_oc_sml_getobj,
-	.objupdatemeta =		smp_oc_objupdatemeta,
+	.objupdatemeta =	smp_oc_objupdatemeta,
 	.objfree =		smp_oc_objfree,
 	.objgetlru =		smp_oc_objgetlru,
 };

bin/varnishd/storage/storage_simple.c

@@ -75,8 +75,8 @@ sml_stv_free(const struct stevedore *stv, struct storage *st)
 
 	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
 	CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
-	AN(stv->free);
-	stv->free(st);
+	if (stv->free != NULL)
+		stv->free(st);
 }
 
 /*--------------------------------------------------------------------
@@ -103,9 +103,9 @@ SML_MkObject(const struct stevedore *stv, struct objcore *oc, void *ptr)
 
 	VTAILQ_INIT(&o->list);
 
-	INIT_OBJ(oc->stobj, STOREOBJ_MAGIC);
 	oc->stobj->stevedore = stv;
 	oc->stobj->priv = o;
+	oc->stobj->priv2 = 0;
 	return (o);
 }
 
@@ -202,7 +202,6 @@ sml_objfree(struct worker *wrk, struct objcore *oc)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	CHECK_OBJ_NOTNULL(oc->stobj, STOREOBJ_MAGIC);
 	sml_slim(wrk, oc);
 	CAST_OBJ_NOTNULL(o, oc->stobj->priv, OBJECT_MAGIC);
 	o->magic = 0;
@@ -425,20 +424,16 @@ sml_trimstore(struct worker *wrk, struct objcore *oc)
 	const struct stevedore *stv;
 	struct storage *st, *st1;
 	struct object *o;
-	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
-	bo = oc->busyobj;
-	if (bo != NULL) {
-		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-		if (bo->do_stream)
-			return;
-	}
-
 	stv = oc->stobj->stevedore;
 	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
+
+	if (stv->free == NULL)
+		return;
+
 	o = sml_getobj(wrk, oc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	st = VTAILQ_LAST(&o->list, storagehead);
@@ -468,6 +463,32 @@ sml_trimstore(struct worker *wrk, struct objcore *oc)
 	st1->len = st->len;
 	VTAILQ_REMOVE(&o->list, st, list);
 	VTAILQ_INSERT_TAIL(&o->list, st1, list);
+	if (oc->busyobj == NULL) {
+		sml_stv_free(stv, st);
+	} else {
+		/* sml_stable frees this */
+		AZ(oc->busyobj->stevedore_priv);
+		oc->busyobj->stevedore_priv = st;
+	}
+}
+
+static void __match_proto__(objstable_f)
+sml_stable(struct worker *wrk, struct objcore *oc, struct busyobj *bo)
+{
+	const struct stevedore *stv;
+	struct storage *st;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+
+	if (bo->stevedore_priv == NULL)
+		return;
+	CAST_OBJ_NOTNULL(st, bo->stevedore_priv, STORAGE_MAGIC);
+	bo->stevedore_priv = 0;
+	CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+	stv = oc->stobj->stevedore;
+	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
 	sml_stv_free(stv, st);
 }
 
@@ -621,6 +642,7 @@ const struct obj_methods SML_methods = {
 	.objextend	= sml_extend,
 	.objgetlen	= sml_getlen,
 	.objtrimstore	= sml_trimstore,
+	.objstable	= sml_stable,
 	.objslim	= sml_slim,
 	.objgetattr	= sml_getattr,
 	.objsetattr	= sml_setattr,

bin/varnishtest/tests/c00073.vtc

+varnishtest "Test object trimming"
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Transfer-encoding: chunked"
+	delay .2
+	chunkedlen 4096
+	sema r1 sync 2
+	sema r2 sync 2
+	chunkedlen 0
+} -start
+
+varnish v1 \
+	-arg "-s malloc,1m" -vcl+backend { } -start
+
+client c1 {
+	txreq
+	rxresp
+} -start
+
+sema r1 sync 2
+varnish v1 -expect SMA.s0.g_bytes > 10000
+
+sema r2 sync 2
+
+client c1 -wait
+varnish v1 -expect SMA.s0.g_bytes < 6000