Introduce a storage backend type (stevedore) 'default' which resolves to umem or malloc

On platforms where umem is available (specifically Solaris descendents), we want
to make umem the default allocator for performance reasons and also for vtc coverage
while still keeping the malloc type as fallback option.

bin/varnishd/mgt/mgt_main.c

@@ -127,7 +127,10 @@ usage(void)
 	printf(FMT, "-s [name=]kind[,options]", "Storage specification");
 	printf(FMT, "", "Can be specified multiple times.");
 #ifdef HAVE_LIBUMEM
+	printf(FMT, "", "  -s default (=umem)");
 	printf(FMT, "", "  -s umem");
+#else
+	printf(FMT, "", "  -s default (=malloc)");
 #endif
 	printf(FMT, "", "  -s malloc");
 	printf(FMT, "", "  -s file");
@@ -437,7 +440,7 @@ main(int argc, char * const *argv)
 	const char *n_arg = NULL;
 	const char *P_arg = NULL;
 	const char *S_arg = NULL;
-	const char *s_arg = "malloc,100m";
+	const char *s_arg = "default,100m";
 	const char *W_arg = NULL;
 	int s_arg_given = 0;
 	int novcl = 0;

bin/varnishd/storage/mgt_stevedore.c

@@ -124,6 +124,9 @@ static const struct choice STV_choice[] = {
 	{ "persistent",			&smp_fake_stevedore },
 #if defined(HAVE_LIBUMEM)
 	{ "umem",			&smu_stevedore },
+	{ "default",			&smu_stevedore },
+#else
+	{ "default",			&sma_stevedore },
 #endif
 	{ NULL,		NULL }
 };
@@ -217,7 +220,7 @@ STV_Config_Transient(void)
 
 	VCLS_AddFunc(mgt_cls, MCF_AUTH, cli_stv);
 	if (stv_transient == NULL)
-		STV_Config(TRANSIENT_STORAGE "=malloc");
+		STV_Config(TRANSIENT_STORAGE "=default");
 	AN(stv_transient);
 	VTAILQ_INSERT_TAIL(&stevedores, stv_transient, list);
 }

bin/varnishtest/tests/b00002.vtc

@@ -5,7 +5,7 @@ server s1 {
 	txresp -hdr "Cache-Control: max-age=120" -hdr "Connection: close" -body "012345\n"
 } -start
 
-varnish v1 -vcl+backend {
+varnish v1 -arg "-sTransient=default,1m" -vcl+backend {
 	sub vcl_recv {
 		return(pass);
 	}

bin/varnishtest/tests/b00006.vtc

-varnishtest "Check that -s malloc works"
+varnishtest "Check that -s default works"
 
 server s1 {
 	rxreq
 	txresp -hdr "Connection: close" -body "012345\n"
 } -start
 
-varnish v1 -arg "-s malloc" -vcl+backend {} -start
+varnish v1 -arg "-s default" -vcl+backend {} -start
 
 client c1 {
 	txreq -url "/"

bin/varnishtest/tests/c00018.vtc

@@ -91,7 +91,7 @@ logexpect l1 -v v1 -g raw {
 	expect 0 1007	VCL_call        {^PASS$}
 	expect 0 1007	VCL_return      {^fetch$}
 	expect 0 1007	Link            {^bereq 1008 pass$}
-	expect 0 1007	Storage         {^malloc Transient$}
+	expect 0 1007	Storage         {^(malloc|umem) Transient$}
 	expect 0 1007	RespProtocol    {^HTTP/1.1$}
 	expect 0 1007	RespStatus      {^100$}
 	expect 0 1007	RespReason      {^Continue$}
@@ -107,7 +107,7 @@ logexpect l1 -v v1 -g raw {
 	expect 0 1009	ReqHeader       {^Content-Length: 20$}
 	expect 0 1009	ReqHeader       {^X-Forwarded-For:}
 	expect 0 1009	VCL_call        {^RECV$}
-	expect 0 1009	Storage         {^malloc Transient$}
+	expect 0 1009	Storage         {^(malloc|umem) Transient$}
 	expect 0 1009	RespProtocol    {^HTTP/1.1$}
 	expect 0 1009	RespStatus      {^100$}
 	expect 0 1009	RespReason      {^Continue$}

bin/varnishtest/tests/c00027.vtc

@@ -18,7 +18,7 @@ server s1 {
 	txresp -body "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111"
 } -start
 
-varnish v1 -arg "-s malloc,1M" -vcl+backend {
+varnish v1 -arg "-s default,1M" -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.ttl = 10m;
 	}

bin/varnishtest/tests/c00044.vtc

@@ -18,9 +18,10 @@ server s1 {
 } -start
 
 varnish v1 \
-	-arg "-ss1=malloc,1m" \
-	-arg "-ss2=malloc,1m" \
-	-arg "-ss0=malloc,1m" \
+	-arg "-ss1=default,1m" \
+	-arg "-ss2=default,1m" \
+	-arg "-ss0=default,1m" \
+	-arg "-sTransient=default" \
 	-vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_stream = false;

bin/varnishtest/tests/c00045.vtc

@@ -10,9 +10,10 @@ server s1 {
 } -start
 
 varnish v1 \
-	-arg "-smalloc,1m" \
-	-arg "-smalloc,1m" \
-	-arg "-smalloc,1m" \
+	-arg "-sdefault,1m" \
+	-arg "-sdefault,1m" \
+	-arg "-sdefault,1m" \
+	-arg "-sTransient=default" \
 	-vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_stream = false;

bin/varnishtest/tests/c00046.vtc

@@ -6,9 +6,10 @@ server s1 {
 } -start
 
 varnish v1 \
-	-arg "-smalloc,1m" \
-	-arg "-smalloc,1m" \
-	-arg "-smalloc,1m" \
+	-arg "-sdefault,1m" \
+	-arg "-sdefault,1m" \
+	-arg "-sdefault,1m" \
+	-arg "-sTransient=default" \
 	-vcl+backend {
 	sub vcl_backend_response {
 		set beresp.storage = storage.s0;

bin/varnishtest/tests/c00073.vtc

@@ -14,7 +14,7 @@ server s1 {
 } -start
 
 varnish v1 \
-	-arg "-s malloc,1m" -vcl+backend { } -start
+	-arg "-s default,1m" -vcl+backend { } -start
 
 client c1 {
 	txreq

bin/varnishtest/tests/c00075.vtc

@@ -8,7 +8,7 @@ server s1 {
 } -start
 
 varnish v1 \
-	-arg "-s malloc,1m" -vcl+backend { } -start
+	-arg "-s default,1m" -vcl+backend { } -start
 
 varnish v1 -cliok "debug.fragfetch 1024"
 

bin/varnishtest/tests/c00078.vtc

@@ -6,9 +6,9 @@ server s1 -repeat 6 {
 } -start
 
 varnish v1 \
-    -arg "-ss1=malloc,1m" \
-    -arg "-ss2=malloc,1m" \
-    -arg "-ss0=malloc,1m" \
+    -arg "-ss1=default,1m" \
+    -arg "-ss2=default,1m" \
+    -arg "-ss0=default,1m" \
     -vcl+backend {
 	import vtc;
 	sub vcl_backend_response {

bin/varnishtest/tests/c00084.vtc

 varnishtest "legal symbol names"
 
-varnish v1 -arg "-s my-store=malloc" -vcl {
+varnish v1 -arg "-s my-store=default" -vcl {
 	import directors;
 
 	acl my-acl { "127.0.0.1"; }

bin/varnishtest/tests/e00007.vtc

@@ -31,7 +31,7 @@ server s1 {
 	}
 } -start
 
-varnish v1 -arg "-smalloc,2m" -vcl+backend {
+varnish v1 -arg "-sdefault,2m" -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_esi = true;
 	}

bin/varnishtest/tests/g00002.vtc

@@ -12,7 +12,7 @@ server s1 {
 } -start
 
 varnish v1 \
-	-arg "-smalloc,2m" \
+	-arg "-sdefault,2m" \
 	-cliok "param.set http_gzip_support true" \
 	-cliok "param.set gzip_memlevel 1" \
 	-vcl+backend {

bin/varnishtest/tests/r00776.vtc

@@ -9,7 +9,7 @@ server s1 {
 
 varnish v1 \
 	-arg "-p fetch_chunksize=4k" \
-	-arg "-s malloc,1m" -vcl+backend { } -start
+	-arg "-s default,1m" -vcl+backend { } -start
 
 client c1 {
 	txreq

bin/varnishtest/tests/r01036.vtc

@@ -7,7 +7,7 @@ server s1 {
 	txresp -bodylen 1572864
 } -start
 
-varnish v1 -arg "-smalloc,1M" -arg "-pgzip_level=0" -vcl+backend {
+varnish v1 -arg "-sdefault,1M" -arg "-pgzip_level=0" -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_stream = false;
 		set beresp.do_gzip = true;

bin/varnishtest/tests/r01037.vtc

@@ -14,7 +14,7 @@ server s1 {
 	chunkedlen 0
 } -start
 
-varnish v1 -arg "-smalloc,1M" -arg "-pgzip_level=0" -vcl+backend {
+varnish v1 -arg "-sdefault,1M" -arg "-pgzip_level=0" -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_esi = true;
 		set beresp.do_gzip = true;

bin/varnishtest/tests/r01140.vtc

@@ -19,7 +19,7 @@ server s1 {
 } -start
 
 varnish v1 -arg "-p nuke_limit=0 -p shortlived=0" \
-	-arg "-smalloc,1m" -vcl+backend {
+	-arg "-sdefault,1m" -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_stream = false;
 		# Unset Date header to not change the object sizes

bin/varnishtest/tests/r01175.vtc

-varnishtest "#1175 - -smalloc c_fail incremented by allocations, not bytes"
+varnishtest "#1175 - -sdefault c_fail incremented by allocations, not bytes"
 
 server s1 {
 	rxreq
 	txresp -nolen -hdr "Content-Length: 1048576"
 } -start
 
-varnish v1 -arg "-s test=malloc,1M" -vcl+backend {
+varnish v1 -arg "-s test=default,1M" -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.storage = storage.test;
 		set beresp.do_stream = false;

bin/varnishtest/tests/r01176.vtc

-varnishtest "-s Transient=malloc crash"
+varnishtest "-s Transient=default crash"
 
 server s1 {
 	rxreq
 	txresp
 } -start
 
-varnish v1 -arg "-s Transient=malloc" -vcl+backend {} -start
+varnish v1 -arg "-s Transient=default" -vcl+backend {} -start
 
 client c1 {
 	txreq

bin/varnishtest/tests/r01284.vtc

@@ -11,7 +11,7 @@ server s1 {
 
 varnish v1 \
 	-arg "-p nuke_limit=0" \
-	-arg "-sTransient=malloc,1m" \
+	-arg "-sTransient=default,1m" \
 	-vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_stream = false;

bin/varnishtest/tests/r01637.vtc

@@ -23,7 +23,7 @@ server s1 {
 	txresp -bodylen 9000
 } -start
 
-varnish v1  -arg "-smalloc,1M" -arg "-p nuke_limit=0 -p gzip_level=0" \
+varnish v1  -arg "-sdefault,1M" -arg "-p nuke_limit=0 -p gzip_level=0" \
     -vcl+backend {
 	sub vcl_backend_response {
 		if (bereq.url == "/url5") {

bin/varnishtest/tests/r01764.vtc

@@ -24,7 +24,7 @@ server s1 {
 	txresp -bodylen 1000000
 } -start
 
-varnish v1  -arg "-smalloc,1M" -arg "-p nuke_limit=1" -vcl+backend {
+varnish v1  -arg "-sdefault,1M" -arg "-p nuke_limit=1" -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_stream = false;
 	}

bin/varnishtest/tests/r01821.vtc

@@ -5,7 +5,7 @@ server s1 -repeat 2 {
 	txresp -bodylen 65535
 } -start
 
-varnish v1 -vcl+backend {
+varnish v1 -arg "-s Transient=default" -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.uncacheable = true;
 	}

bin/varnishtest/tests/r01914.vtc

@@ -9,7 +9,11 @@ server s1 {
 	txresp
 } -start
 
-varnish v1 -arg "-s malloc,1MB" -arg "-s malloc,1MB" -vcl+backend {
+varnish v1 \
+    -arg "-s default,1MB" \
+    -arg "-s default,1MB" \
+    -arg "-s Transient=default" \
+    -vcl+backend {
 	import std;
 	sub vcl_recv {
 		if (req.url == "/1") {

bin/varnishtest/tests/r01941.vtc

@@ -24,7 +24,7 @@ server s1 {
 } -start
 
 varnish v1 -arg "-pfetch_chunksize=4k" \
-	-arg "-smalloc,1m" -vcl+backend {
+	-arg "-sdefault,1m" -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_esi = true;
 	}

bin/varnishtest/tests/r02321.vtc

 varnishtest "Storage name collisions"
 
 # intentional collision
-shell -err -expect "Error: (-s main=malloc,100m) 'main' is already defined" {
+shell -err -expect "Error: (-s main=default,100m) 'main' is already defined" {
 	exec varnishd -a :0 -f '' -n ${tmpdir} \
-		-s main=malloc,10m -s main=malloc,100m
+		-s main=default,10m -s main=default,100m
 }
 
 # pseudo-accidental collision
-shell -err -expect "Error: (-s s0=malloc,100m) 's0' is already defined" {
+shell -err -expect "Error: (-s s0=default,100m) 's0' is already defined" {
 	exec varnishd -a :0 -f '' -n ${tmpdir} \
-		-s malloc,10m -s s0=malloc,100m
+		-s default,10m -s s0=default,100m
 }
 
 # Transient collision
-shell -err -expect "Error: (-s Transient=malloc,100m) 'Transient' is already defined" {
+shell -err -expect "Error: (-s Transient=default,100m) 'Transient' is already defined" {
 	exec varnishd -a :0 -f '' -n ${tmpdir} \
-		-s Transient=malloc,10m -s Transient=malloc,100m
+		-s Transient=default,10m -s Transient=default,100m
 }

bin/varnishtest/tests/r02325.vtc

 varnishtest "validate storage identifiers"
 
 shell -err -expect {Error: invalid -s name "///"=[...]} {
-	varnishd -a :0 -n ${tmpdir} -F -f '' -s ///=malloc
+	varnishd -a :0 -n ${tmpdir} -F -f '' -s ///=default
 }
 shell -err -expect {Error: Empty named -s argument "foo="} {
 	varnishd -a :0 -n ${tmpdir} -F -f '' -s foo=

bin/varnishtest/tests/r02406.vtc

 varnishtest "Long backend and storage names"
 
-varnish v1 -arg "-s acme_example_com_static_assets_malloc_storage=malloc"
+varnish v1 -arg "-s acme_example_com_static_assets_default_storage=default"
 
 varnish v1 -vcl {
 	backend be_0123456789_0123456789_0123456789_0123456789_0123456789_0123456789_0123456789_0123456789_0123456789_0123456789_0123456789_0123456789 {
@@ -8,5 +8,5 @@ varnish v1 -vcl {
 	}
 } -start
 
-varnish v1 -expect SM?.acme_example_com_static_assets_malloc_storage.c_req == 0
+varnish v1 -expect SM?.acme_example_com_static_assets_default_storage.c_req == 0
 varnish v1 -expect VBE.vcl1.be_0123456789_0123456789_0123456789_0123456789_0123456789_0123456789_0123456789_0123456789_0123456789_0123456789_0123456789_0123456789.req == 0

bin/varnishtest/tests/v00010.vtc

@@ -24,7 +24,7 @@ server s1 {
 	txresp -hdr "Foo: foo" -body "abcdef\n"
 } -start
 
-varnish v1 -arg "-smalloc,1m" -vcl+backend {
+varnish v1 -arg "-sdefault,1m" -vcl+backend {
 	import vtc;
 
 	sub vcl_backend_response {

doc/sphinx/reference/varnishd.rst

@@ -273,10 +273,20 @@ Storage Backend
 
 The following storage types are available:
 
+-s <default[,size]>
+
+  The default storage type resolves to umem where available and malloc
+  otherwise.
+
 -s <malloc[,size]>
 
   malloc is a memory based backend.
 
+-s <umem[,size]>
+
+  umem is a storage backend which is more efficient than malloc on
+  platforms where it is available.
+
 -s <file,path[,size[,granularity[,advice]]]>
 
   The file backend stores data in a file on disk. The file will be

doc/sphinx/tutorial/putting_varnish_on_port_80.rst

@@ -19,7 +19,7 @@ some text that looks like this::
                -T localhost:6082 \
                -f /etc/varnish/default.vcl \
                -S /etc/varnish/secret \
-               -s malloc,256m"
+               -s default,256m"
 
 Change it to::
 
@@ -27,7 +27,7 @@ Change it to::
                -T localhost:6082 \
                -f /etc/varnish/default.vcl \
                -S /etc/varnish/secret \
-               -s malloc,256m"
+               -s default,256m"
 
 Debian (v8+) / Ubuntu (v15.04+)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -40,7 +40,7 @@ Applying changes to the default service is best done by creating a new file
 
   [Service]
   ExecStart=
-  ExecStart=/usr/sbin/varnishd -a :80 -T localhost:6082 -f /etc/varnish/default.vcl -S /etc/varnish/secret -s malloc,256m
+  ExecStart=/usr/sbin/varnishd -a :80 -T localhost:6082 -f /etc/varnish/default.vcl -S /etc/varnish/secret -s default,256m
 
 This will override the ExecStart part of the default configuration shipped
 with Varnish Cache.

doc/sphinx/users-guide/command-line.rst

@@ -74,9 +74,9 @@ Other options
 
 Varnish comes with an abundance of useful command line arguments. We recommend that you study them but not necessary use them all, but to get started, the above will be sufficient.
 
-By default Varnish will use 100 megabytes of malloc(3) storage
-for caching objects, if you want to cache more than that, you
-should look at the '-s' argument.
+By default Varnish will use 100 megabytes of virtual memory (malloc(3)
+or libumem(3lib)) storage for caching objects, if you want to cache
+more than that, you should look at the '-s' argument.
 
 .. XXX: 3? benc
 

doc/sphinx/users-guide/storage-backends.rst

@@ -13,6 +13,14 @@ configuration is to use the malloc backend with a limited size. For a
 serious Varnish deployment you probably would want to adjust the storage
 settings.
 
+default
+~~~~~~~
+
+syntax: default[,size]
+
+The default storage backend is an alias to umem, where available, or
+malloc otherwise.
+
 malloc
 ~~~~~~
 
@@ -45,6 +53,24 @@ malloc's performance is bound to memory speed so it is very fast. If
 the dataset is bigger than available memory performance will
 depend on the operating systems ability to page effectively.
 
+umem
+~~~~
+
+syntax: umem[,size]
+
+Umem is a better alternative to the malloc backend where `libumem`_ is
+available. All other configuration aspects are considered equal to
+malloc.
+
+`libumem`_ implements a slab allocator similar to the kernel memory
+allocator used in virtually all modern operating systems and is
+considered more efficient and scalable than classical
+implementations. In particular, `libumem`_ is included in the family
+of OpenSolaris descendent operating systems where jemalloc(3) is not
+commonly available.
+
+.. _libumem: http://dtrace.org/blogs/ahl/2004/07/13/number-11-of-20-libumem/
+
 file
 ~~~~