New 'h2_rxbuf_storage' param to set rxbuf stevedore

This parameter allows the user to choose which storage backend / stevedore
that the H/2 receive buffers are allocated from. By default it uses
Transient.

bin/varnishd/http2/cache_http2_proto.c

@@ -924,7 +924,8 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 			/* Cap the buffer size when we know this is the
 			 * single data frame. */
 			bufsize = len;
-		stvbuf = STV_AllocBuf(wrk, stv_transient,
+		CHECK_OBJ_NOTNULL(stv_h2_rxbuf, STEVEDORE_MAGIC);
+		stvbuf = STV_AllocBuf(wrk, stv_h2_rxbuf,
 		    bufsize + sizeof *rxbuf);
 		if (stvbuf == NULL) {
 			VSLb(h2->vsl, SLT_Debug,

bin/varnishd/mgt/mgt.h

@@ -218,6 +218,7 @@ char **MGT_NamedArg(const char *, const char **, const char *);
 
 
 /* stevedore_mgt.c */
+extern const char *mgt_stv_h2_rxbuf;
 void STV_Config(const char *spec);
 void STV_Config_Transient(void);
 

bin/varnishd/mgt/mgt_param.h

@@ -78,6 +78,7 @@ tweak_t tweak_timeout;
 tweak_t tweak_uint;
 tweak_t tweak_vsl_buffer;
 tweak_t tweak_vsl_reclen;
+tweak_t tweak_h2_rxbuf_storage;
 
 extern struct parspec mgt_parspec[]; /* mgt_param_tbl.c */
 extern struct parspec VSL_parspec[]; /* mgt_param_vsl.c */

bin/varnishd/mgt/mgt_param_tweak.c

@@ -42,6 +42,7 @@
 #include "mgt/mgt.h"
 
 #include "mgt/mgt_param.h"
+#include "storage/storage.h"
 #include "vav.h"
 #include "vnum.h"
 
@@ -488,3 +489,43 @@ tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par,
 	    "%u", mgt_param.wthread_max);
 	return (0);
 }
+
+/*--------------------------------------------------------------------
+ * Tweak 'h2_rxbuf_storage'
+ *
+ */
+
+int v_matchproto_(tweak_t)
+tweak_h2_rxbuf_storage(struct vsb *vsb, const struct parspec *par,
+    const char *arg)
+{
+	struct stevedore *stv;
+
+	/* XXX: If we want to remove the MUST_RESTART flag from the
+	 * h2_rxbuf_storage parameter, we could have a mechanism here
+	 * that when the child is running calls out through CLI to change
+	 * the stevedore being used. */
+
+	if (arg == NULL || arg == JSON_FMT)
+		return (tweak_string(vsb, par, arg));
+
+	if (!strcmp(arg, "Transient")) {
+		/* Always allow setting to the special name
+		 * "Transient". There will always be a stevedore with this
+		 * name, but it may not have been configured at the time
+		 * this is called. */
+	} else {
+		/* Only allow setting the value to a known configured
+		 * stevedore */
+		STV_Foreach(stv) {
+			CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
+			if (!strcmp(stv->ident, arg))
+				break;
+		}
+		if (stv == NULL) {
+			VSB_printf(vsb, "unknown storage backend '%s'", arg);
+			return (-1);
+		}
+	}
+	return (tweak_string(vsb, par, arg));
+}

bin/varnishd/storage/mgt_stevedore.c

@@ -54,6 +54,8 @@ static VTAILQ_HEAD(, stevedore) stevedores =
 
 struct stevedore *stv_transient;
 
+const char *mgt_stv_h2_rxbuf;
+
 /*--------------------------------------------------------------------*/
 
 int
@@ -247,7 +249,6 @@ STV_Config(const char *spec)
 void
 STV_Config_Transient(void)
 {
-
 	ASSERT_MGT();
 
 	VCLS_AddFunc(mgt_cls, MCF_AUTH, cli_stv);

bin/varnishd/storage/stevedore.c

@@ -43,6 +43,9 @@
 #include "storage/storage.h"
 #include "vrt_obj.h"
 
+extern const char *mgt_stv_h2_rxbuf;
+struct stevedore *stv_h2_rxbuf = NULL;
+
 static pthread_mutex_t stv_mtx;
 
 /*--------------------------------------------------------------------
@@ -172,13 +175,23 @@ STV_open(void)
 
 	ASSERT_CLI();
 	AZ(pthread_mutex_init(&stv_mtx, NULL));
+
+	/* This string was prepared for us before the fork, and should
+	 * point to a configured stevedore. */
+	AN(mgt_stv_h2_rxbuf);
+
+	stv_h2_rxbuf = NULL;
 	STV_Foreach(stv) {
+		CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
 		bprintf(buf, "storage.%s", stv->ident);
 		stv->vclname = strdup(buf);
 		AN(stv->vclname);
 		if (stv->open != NULL)
 			stv->open(stv);
+		if (!strcmp(stv->ident, mgt_stv_h2_rxbuf))
+			stv_h2_rxbuf = stv;
 	}
+	AN(stv_h2_rxbuf);
 }
 
 void

bin/varnishd/storage/storage.h

@@ -134,6 +134,7 @@ struct stevedore {
 };
 
 extern struct stevedore *stv_transient;
+extern struct stevedore *stv_h2_rxbuf;
 
 /*--------------------------------------------------------------------*/
 

bin/varnishtest/tests/t02022.vtc

+varnishtest "Test non-transient rxbuf stevedore with LRU nuking"
+
+barrier b1 sock 2 -cyclic
+
+server s1 {
+	rxreq
+	txresp -body asdf
+	rxreq
+	txresp -bodylen 1048000
+	rxreq
+	txresp -body ASDF
+} -start
+
+varnish v1 -arg "-srxbuf=malloc,1m -smain=malloc,1m" -vcl+backend {
+	import vtc;
+	sub vcl_recv {
+		if (req.url == "/1") {
+			vtc.barrier_sync("${b1_sock}");
+		}
+	}
+	sub vcl_backend_response {
+		if (bereq.url == "/2") {
+			set beresp.storage = storage.rxbuf;
+		} else {
+			set beresp.storage = storage.main;
+		}
+	}
+}
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.reset h2_initial_window_size"
+varnish v1 -cliok "param.reset h2_rx_window_low_water"
+varnish v1 -cliok "param.set h2_rxbuf_storage rxbuf"
+varnish v1 -cliok "param.set debug +syncvsl"
+
+varnish v1 -start
+
+client c1 {
+	stream 1 {
+		txreq -req POST -url /1 -hdr "content-length" "2048" -nostrend
+		txdata -datalen 2048
+		rxresp
+		expect resp.status == 200
+	} -start
+} -start
+
+varnish v1 -expect SMA.rxbuf.g_bytes >= 2048
+varnish v1 -expect SMA.Transient.g_bytes == 0
+varnish v1 -expect MAIN.n_lru_nuked == 0
+
+barrier b1 sync
+client c1 -wait
+
+varnish v1 -expect SMA.rxbuf.g_bytes == 0
+varnish v1 -expect SMA.Transient.g_bytes == 0
+varnish v1 -expect MAIN.n_lru_nuked == 0
+
+client c2 {
+	txreq -url /2
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 1048000
+} -run
+
+varnish v1 -expect SMA.rxbuf.g_bytes >= 1048000
+varnish v1 -expect MAIN.n_lru_nuked == 0
+
+client c3 {
+	stream 1 {
+		txreq -req POST -url /1 -hdr "content-length" "2048" -nostrend
+		txdata -datalen 2048
+		rxresp
+		expect resp.status == 200
+	} -start
+} -start
+
+varnish v1 -expect SMA.rxbuf.g_bytes >= 2048
+varnish v1 -expect SMA.rxbuf.g_bytes < 3000
+varnish v1 -expect SMA.Transient.g_bytes == 0
+varnish v1 -expect MAIN.n_lru_nuked == 1
+
+barrier b1 sync
+client c3 -wait
+
+varnish v1 -expect SMA.rxbuf.g_bytes == 0
+varnish v1 -expect SMA.Transient.g_bytes == 0

include/tbl/params.h

@@ -1657,10 +1657,28 @@ PARAM_PCRE2(
 	" messages."
 )
 
+/*--------------------------------------------------------------------
+ * Custom parameters with separate tweak function
+ */
+
+#  define PARAM_CUSTOM(nm, pv, def, ...) \
+	PARAM(, , nm, tweak_ ## nm, pv, NULL, NULL, def, NULL, __VA_ARGS__)
+
+PARAM_CUSTOM(
+	/* name */	h2_rxbuf_storage,
+	/* priv */	&mgt_stv_h2_rxbuf,
+	/* def */	"Transient",
+	/* descr */
+	"The name of the storage backend that HTTP/2 receive buffers"
+	" should be allocated from.",
+	/* flags */	MUST_RESTART
+)
+
 #  undef PARAM_ALL
 #  undef PARAM_PCRE2
 #  undef PARAM_STRING
 #  undef PARAM_VCC
+#  undef PARAM_CUSTOM
 #endif /* defined(PARAM_ALL) */
 
 #undef PARAM_MEMPOOL