support dynamic matching

configure.ac

 AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2013-2014 UPLEX Nils Goroll Systemoptimierung])
-AC_INIT([libvmod-re], [0.3])
+AC_INIT([libvmod-re], [0.4])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR(src/vmod_re.vcc)
 AM_CONFIG_HEADER(config.h)

src/tests/b06.vtc.disabled → src/tests/b06.vtc

@@ -5,27 +5,31 @@ server s1 {
 	txresp -hdr "Foo: baz" -hdr "Bar: baz" -body "foobar"
 	rxreq
 	expect req.url == "/foo"
-	txresp -hdr "Foo: baz" -hdr "Bar: baz" -body "foobar"
+	txresp -hdr "Foo: quux" -hdr "Bar: baz" -body "foobar"
 	rxreq
 	expect req.url == "/bar"
-	txresp -hdr "Foo: baz" -hdr "Bar: quux" -body "foobar"
+	txresp -hdr "Foo: quux" -hdr "Bar: quux" -body "foobar"
 	rxreq
 	expect req.url == "/baz"
-	txresp -hdr "Foo: baz" -hdr "Bar: baz" -body "foobar"
+	txresp -hdr "Foo: baz" -hdr "Bar: quux" -body "foobar"
 } -start
 
 varnish v1 -vcl+backend {
 	import re from "${vmod_topbuild}/src/.libs/libvmod_re.so";
 
-	sub vcl_fetch {
-		if (re.match(beresp.http.foo, beresp.http.bar)) {
+	sub vcl_init {
+	    	new baz = re.regex("baz");
+	}
+
+	sub vcl_backend_response {
+		if (baz.match(beresp.http.bar)) {
 			set beresp.http.static = "match";
 		}
 		else {
 			set beresp.http.static = "nomatch";
 		}
 
-		if (re.match_dyn(beresp.http.foo, beresp.http.bar)) {
+		if (baz.match_dyn(beresp.http.foo, beresp.http.bar)) {
 			set beresp.http.dynamic = "match";
 		}
 		else {
@@ -44,22 +48,22 @@ client c1 {
 
 	txreq -url "/foo"
 	rxresp
-	expect resp.http.foo == "baz"
+	expect resp.http.foo == "quux"
 	expect resp.http.bar == "baz"
 	expect resp.http.static  == "match"
-	expect resp.http.dynamic == "match"
+	expect resp.http.dynamic == "nomatch"
 
 	txreq -url "/bar"
 	rxresp 
-	expect resp.http.foo == "baz"
+	expect resp.http.foo == "quux"
 	expect resp.http.bar == "quux"
-	expect resp.http.static  == "match"
-	expect resp.http.dynamic == "nomatch"
+	expect resp.http.static  == "nomatch"
+	expect resp.http.dynamic == "match"
 
 	txreq -url "/baz"
 	rxresp 
 	expect resp.http.foo == "baz"
-	expect resp.http.bar == "baz"
-	expect resp.http.static  == "match"
-	expect resp.http.dynamic == "match"
+	expect resp.http.bar == "quux"
+	expect resp.http.static  == "nomatch"
+	expect resp.http.dynamic == "nomatch"
 } -run

src/tests/c01.vtc

@@ -13,7 +13,6 @@ varnish v1 -vcl+backend {
 		new barbaz = re.regex("(bar)(baz)");
 		new bazplus = re.regex("(baz)(.+)");
 		new fourdots = re.regex("(.)(.)(.)(.)");
-		new frobnitz = re.regex("(frob)(nitz)");
 	}
 
 	sub vcl_backend_response {
@@ -26,6 +25,15 @@ varnish v1 -vcl+backend {
 			set beresp.status = 999;
 		}
 
+		if (barbaz.match_dyn("(bar)(baz)", beresp.http.foo)) {
+			set beresp.http.foo0d = barbaz.backref(0, "error0");
+			set beresp.http.foo1d = barbaz.backref(1, "error1");
+			set beresp.http.foo2d = barbaz.backref(2, "error2");
+			set beresp.http.foo3d = barbaz.backref(3, "foofallback");
+		} else {
+			set beresp.status = 999;
+		}
+
 		if (bazplus.match(beresp.http.bar)) {
 			set beresp.http.bar0 = bazplus.backref(0, "error0");
 			set beresp.http.bar1 = bazplus.backref(1, "error1");
@@ -35,6 +43,14 @@ varnish v1 -vcl+backend {
 		        set beresp.status = 999;
 		}
 
+		if (bazplus.match_dyn("(baz)(.+)", beresp.http.bar)) {
+			set beresp.http.bar0d = bazplus.backref(0, "error0");
+			set beresp.http.bar1d = bazplus.backref(1, "error1");
+			set beresp.http.bar2d = bazplus.backref(2, "error2");
+			set beresp.http.bar3d = bazplus.backref(3, "barfallback");
+		} else {
+		        set beresp.status = 999;
+		}
 
 		if (barbaz.match(beresp.http.foo)
 		    && bazplus.match(beresp.http.bar)) {
@@ -50,6 +66,20 @@ varnish v1 -vcl+backend {
 			set beresp.status = 999;
 		}
 
+		if (barbaz.match_dyn("(bar)(baz)", beresp.http.foo)
+		    && bazplus.match_dyn("(baz)(.+)", beresp.http.bar)) {
+			set beresp.http.foo20d = barbaz.backref(0, "error0");
+			set beresp.http.foo21d = barbaz.backref(1, "error1");
+			set beresp.http.foo22d = barbaz.backref(2, "error2");
+			set beresp.http.foo23d = barbaz.backref(3, "foofallback");
+			set beresp.http.bar20d = bazplus.backref(0, "error0");
+			set beresp.http.bar21d = bazplus.backref(1, "error1");
+			set beresp.http.bar22d = bazplus.backref(2, "error2");
+			set beresp.http.bar23d = bazplus.backref(3, "barfallback");
+		} else {
+			set beresp.status = 999;
+		}
+
 		if (fourdots.match(beresp.http.barf)) {
 			set beresp.http.frap
 			    = "_" + fourdots.backref(0, "error0") + "_"
@@ -61,6 +91,16 @@ varnish v1 -vcl+backend {
 			set beresp.status = 999;
 		}
 
+		if (fourdots.match_dyn("(.)(.)(.)(.)", beresp.http.barf)) {
+			set beresp.http.frapd
+			    = "_" + fourdots.backref(0, "error0") + "_"
+			      + fourdots.backref(5, "")
+			      + fourdots.backref(4, "error4")
+			      + fourdots.backref(3, "error3")
+			      + fourdots.backref(2, "error2") + "p_";
+		} else {
+			set beresp.status = 999;
+		}
 }
 
 } -start
@@ -72,10 +112,18 @@ client c1 {
 	expect resp.http.foo1 == "bar"
 	expect resp.http.foo2 == "baz"
 	expect resp.http.foo3 == "foofallback"
+	expect resp.http.foo0d == "barbaz"
+	expect resp.http.foo1d == "bar"
+	expect resp.http.foo2d == "baz"
+	expect resp.http.foo3d == "foofallback"
 	expect resp.http.bar0 == "bazquux"
 	expect resp.http.bar1 == "baz"
 	expect resp.http.bar2 == "quux"
 	expect resp.http.bar3 == "barfallback"
+	expect resp.http.bar0d == "bazquux"
+	expect resp.http.bar1d == "baz"
+	expect resp.http.bar2d == "quux"
+	expect resp.http.bar3d == "barfallback"
 	expect resp.http.foo20 == "barbaz"
 	expect resp.http.foo21 == "bar"
 	expect resp.http.foo22 == "baz"
@@ -84,5 +132,14 @@ client c1 {
 	expect resp.http.bar21 == "baz"
 	expect resp.http.bar22 == "quux"
 	expect resp.http.bar23 == "barfallback"
+	expect resp.http.foo20d == "barbaz"
+	expect resp.http.foo21d == "bar"
+	expect resp.http.foo22d == "baz"
+	expect resp.http.foo23d == "foofallback"
+	expect resp.http.bar20d == "bazquux"
+	expect resp.http.bar21d == "baz"
+	expect resp.http.bar22d == "quux"
+	expect resp.http.bar23d == "barfallback"
 	expect resp.http.frap == "_barf_frap_"
+	expect resp.http.frapd == "_barf_frap_"
 } -run

src/tests/c03.vtc

-varnishtest "non-capturing parentheses"
+varnishtest "backref failure with dynamic matches"
+
+# same as c02.vtc, but with match_dyn()
 
 server s1 {
 	rxreq
-	txresp -hdr "Foo: barbaz" -body "1111\n"
+	txresp -hdr "Foo: barbaz" -hdr "Bar: bazquux" -hdr "Barf: barf" \
+	       -body "1111\n"
 } -start
 
 varnish v1 -vcl+backend {
 	import re from "${vmod_topbuild}/src/.libs/libvmod_re.so";
 
 	sub vcl_init {
-	       new barbaz = re.regex("(?:bar)(baz)");
+		new frobnitz = re.regex("");
+		new barbaz = re.regex("");
 	}
 
-	sub vcl_backend_response {
-		if (barbaz.match(beresp.http.foo)) {
-			set beresp.http.foo0 = barbaz.backref(0, "error0");
-			set beresp.http.foo1 = barbaz.backref(1, "error1");
-			set beresp.http.foo2 = barbaz.backref(2, "fallback");
-		} else {
-			set beresp.status = 999;
+	sub vcl_deliver {
+	    	set resp.http.nomatch = barbaz.backref(0, "fallback");
+
+		if (frobnitz.match_dyn("(frob)(nitz)", resp.http.foo)) {
+			set resp.http.frob = "nitz";
+		}
+		set resp.http.frob0 = frobnitz.backref(0, "fallback0");
+		set resp.http.frob1 = frobnitz.backref(1, "fallback1");
+		set resp.http.frob2 = frobnitz.backref(2, "fallback2");
+		set resp.http.frob3 = frobnitz.backref(3, "fallback3");
+		set resp.http.frob4 = frobnitz.backref(4, "fallback4");
+		set resp.http.frob5 = frobnitz.backref(5, "fallback5");
+		set resp.http.frob6 = frobnitz.backref(6, "fallback6");
+		set resp.http.frob7 = frobnitz.backref(7, "fallback7");
+		set resp.http.frob8 = frobnitz.backref(8, "fallback8");
+		set resp.http.frob9 = frobnitz.backref(9, "fallback9");
+		set resp.http.frob10 = frobnitz.backref(10, "fallback10");
+
+		if (barbaz.match_dyn("(bar)(baz)", resp.http.foo)) {
+			set resp.http.foo0 = barbaz.backref(0, "error0");
+			set resp.http.foo1 = barbaz.backref(1, "error1");
+			set resp.http.foo2 = barbaz.backref(2, "error2");
+			set resp.http.foo3 = barbaz.backref(3, "foofallback");
 		}
+		if (barbaz.match_dyn("(bar)(baz)", resp.http.barf)) {
+			set resp.http.puke = "match";
+		}
+		set resp.http.barf0 = barbaz.backref(0, "fallback0");
+		set resp.http.barf1 = barbaz.backref(1, "fallback1");
+		set resp.http.barf2 = barbaz.backref(2, "fallback2");
+		set resp.http.barf3 = barbaz.backref(3, "fallback3");
+		set resp.http.barf4 = barbaz.backref(4, "fallback4");
+		set resp.http.barf5 = barbaz.backref(5, "fallback5");
+		set resp.http.barf6 = barbaz.backref(6, "fallback6");
+		set resp.http.barf7 = barbaz.backref(7, "fallback7");
+		set resp.http.barf8 = barbaz.backref(8, "fallback8");
+		set resp.http.barf9 = barbaz.backref(9, "fallback9");
+		set resp.http.barf10 = barbaz.backref(10, "fallback10");
 	}
 
 } -start
@@ -27,7 +61,40 @@ varnish v1 -vcl+backend {
 client c1 {
 	txreq
 	rxresp
+	expect resp.status == 200
+	expect resp.http.nomatch == "fallback"
+	expect resp.http.frob == <undef>
+	expect resp.http.frob0 == "fallback0"
+	expect resp.http.frob1 == "fallback1"
+	expect resp.http.frob2 == "fallback2"
+	expect resp.http.frob3 == "fallback3"
+	expect resp.http.frob4 == "fallback4"
+	expect resp.http.frob5 == "fallback5"
+	expect resp.http.frob6 == "fallback6"
+	expect resp.http.frob7 == "fallback7"
+	expect resp.http.frob8 == "fallback8"
+	expect resp.http.frob9 == "fallback9"
+	expect resp.http.frob10 == "fallback10"
 	expect resp.http.foo0 == "barbaz"
-	expect resp.http.foo1 == "baz"
-	expect resp.http.foo2 == "fallback"
+	expect resp.http.foo1 == "bar"
+	expect resp.http.foo2 == "baz"
+	expect resp.http.foo3 == "foofallback"
+	expect resp.http.puke == <undef>
+	expect resp.http.barf0 == "fallback0"
+	expect resp.http.barf1 == "fallback1"
+	expect resp.http.barf2 == "fallback2"
+	expect resp.http.barf3 == "fallback3"
+	expect resp.http.barf4 == "fallback4"
+	expect resp.http.barf5 == "fallback5"
+	expect resp.http.barf6 == "fallback6"
+	expect resp.http.barf7 == "fallback7"
+	expect resp.http.barf8 == "fallback8"
+	expect resp.http.barf9 == "fallback9"
+	expect resp.http.barf10 == "fallback10"
+} -run
+
+logexpect l1 -v v1 -d 1 -g vxid -q "VCL_Error" {
+	expect 0 * Begin req
+	expect * = VCL_Error "vmod re: backref called without prior match"
+	expect * = End
 } -run

src/tests/c04.vtc

-varnishtest "limit to backrefs 0 to 10"
+varnishtest "non-capturing parentheses"
 
 server s1 {
 	rxreq
-	txresp -hdr "Foo: 12345678901" -hdr "Bar: 123456789012" -body "1111\n"
+	txresp -hdr "Foo: barbaz" -body "1111\n"
 } -start
 
 varnish v1 -vcl+backend {
 	import re from "${vmod_topbuild}/src/.libs/libvmod_re.so";
 
 	sub vcl_init {
-	       new tendots = re.regex("(.)(.)(.)(.)(.)(.)(.)(.)(.)(.)");
-	       new moredots = re.regex("(.)(.)(.)(.)(.)(.)(.)(.)(.)(.)(.)");
+	       new barbaz = re.regex("(?:bar)(baz)");
 	}
 
 	sub vcl_backend_response {
-		if (tendots.match(beresp.http.foo)) {
-			set beresp.http.foo0 = tendots.backref(0, "error0");
-			set beresp.http.foo1 = tendots.backref(1, "error1");
-			set beresp.http.foo2 = tendots.backref(2, "error2");
-			set beresp.http.foo3 = tendots.backref(3, "error3");
-			set beresp.http.foo4 = tendots.backref(4, "error4");
-			set beresp.http.foo5 = tendots.backref(5, "error5");
-			set beresp.http.foo6 = tendots.backref(6, "error6");
-			set beresp.http.foo7 = tendots.backref(7, "error7");
-			set beresp.http.foo8 = tendots.backref(8, "error8");
-			set beresp.http.foo9 = tendots.backref(9, "error9");
-			set beresp.http.foo10 = tendots.backref(10, "error10");
-			set beresp.http.foo11 = tendots.backref(11, "fallback");
+		if (barbaz.match(beresp.http.foo)) {
+			set beresp.http.foo0 = barbaz.backref(0, "error0");
+			set beresp.http.foo1 = barbaz.backref(1, "error1");
+			set beresp.http.foo2 = barbaz.backref(2, "fallback");
 		} else {
 			set beresp.status = 999;
 		}
-		if (moredots.match(beresp.http.bar)) {
-			set beresp.http.bar0 = moredots.backref(0, "error0");
-			set beresp.http.bar1 = moredots.backref(1, "error1");
-			set beresp.http.bar2 = moredots.backref(2, "error2");
-			set beresp.http.bar3 = moredots.backref(3, "error3");
-			set beresp.http.bar4 = moredots.backref(4, "error4");
-			set beresp.http.bar5 = moredots.backref(5, "error5");
-			set beresp.http.bar6 = moredots.backref(6, "error6");
-			set beresp.http.bar7 = moredots.backref(7, "error7");
-			set beresp.http.bar8 = moredots.backref(8, "error8");
-			set beresp.http.bar9 = moredots.backref(9, "error9");
-			set beresp.http.bar10 = moredots.backref(10, "error10");
-			set beresp.http.bar11 = moredots.backref(11, "fallback");
+
+		if (barbaz.match_dyn("(?:bar)(baz)", beresp.http.foo)) {
+			set beresp.http.foo0d = barbaz.backref(0, "error0");
+			set beresp.http.foo1d = barbaz.backref(1, "error1");
+			set beresp.http.foo2d = barbaz.backref(2, "fallback");
 		} else {
 			set beresp.status = 999;
 		}
@@ -53,37 +35,10 @@ varnish v1 -vcl+backend {
 client c1 {
 	txreq
 	rxresp
-	expect resp.status == 200
-	expect resp.http.foo0 == "1234567890"
-	expect resp.http.foo1 == "1"
-	expect resp.http.foo2 == "2"
-	expect resp.http.foo3 == "3"
-	expect resp.http.foo4 == "4"
-	expect resp.http.foo5 == "5"
-	expect resp.http.foo6 == "6"
-	expect resp.http.foo7 == "7"
-	expect resp.http.foo8 == "8"
-	expect resp.http.foo9 == "9"
-	expect resp.http.foo10 == "0"
-	expect resp.http.foo11 == "fallback"
-	expect resp.http.bar0 == "12345678901"
-	expect resp.http.bar1 == "1"
-	expect resp.http.bar2 == "2"
-	expect resp.http.bar3 == "3"
-	expect resp.http.bar4 == "4"
-	expect resp.http.bar5 == "5"
-	expect resp.http.bar6 == "6"
-	expect resp.http.bar7 == "7"
-	expect resp.http.bar8 == "8"
-	expect resp.http.bar9 == "9"
-	expect resp.http.bar10 == "0"
-	expect resp.http.bar11 == "fallback"
-} -run
-
-logexpect l1 -v v1 -d 1 -g vxid -q "VCL_Error" {
-	expect 0 * Begin req
-	expect * = VCL_Error "^vmod re: backref \d+ out of range$"
-	expect * = VCL_Error "^vmod re: capturing substrings exceed max \d+"
-	expect * = VCL_Error "^vmod re: backref \d+ out of range$"
-	expect * = End
+	expect resp.http.foo0 == "barbaz"
+	expect resp.http.foo1 == "baz"
+	expect resp.http.foo2 == "fallback"
+	expect resp.http.foo0d == "barbaz"
+	expect resp.http.foo1d == "baz"
+	expect resp.http.foo2d == "fallback"
 } -run

src/tests/c05.vtc

-varnishtest "re.backref not affected by standard VCL regex code"
+varnishtest "limit to backrefs 0 to 10"
 
 server s1 {
 	rxreq
-	txresp -hdr "Foo: barbaz" -body "1111\n"
+	txresp -hdr "Foo: 12345678901" -hdr "Bar: 123456789012" -body "1111\n"
 } -start
 
 varnish v1 -vcl+backend {
 	import re from "${vmod_topbuild}/src/.libs/libvmod_re.so";
 
 	sub vcl_init {
-	    	new barbaz = re.regex("(bar)baz");
+	       new tendots = re.regex("(.)(.)(.)(.)(.)(.)(.)(.)(.)(.)");
+	       new moredots = re.regex("(.)(.)(.)(.)(.)(.)(.)(.)(.)(.)(.)");
 	}
 
 	sub vcl_backend_response {
-	    	if (!barbaz.match(beresp.http.foo)) {
-		        set beresp.status = 999;
-		}
-
-		if (beresp.http.foo ~ "bar(baz)") {
-			set beresp.http.tilde0 = barbaz.backref(0, "tilde0");
-			set beresp.http.tilde1 = barbaz.backref(1, "tilde1");
+		if (tendots.match(beresp.http.foo)) {
+			set beresp.http.foo0 = tendots.backref(0, "error0");
+			set beresp.http.foo1 = tendots.backref(1, "error1");
+			set beresp.http.foo2 = tendots.backref(2, "error2");
+			set beresp.http.foo3 = tendots.backref(3, "error3");
+			set beresp.http.foo4 = tendots.backref(4, "error4");
+			set beresp.http.foo5 = tendots.backref(5, "error5");
+			set beresp.http.foo6 = tendots.backref(6, "error6");
+			set beresp.http.foo7 = tendots.backref(7, "error7");
+			set beresp.http.foo8 = tendots.backref(8, "error8");
+			set beresp.http.foo9 = tendots.backref(9, "error9");
+			set beresp.http.foo10 = tendots.backref(10, "error10");
+			set beresp.http.foo11 = tendots.backref(11, "fallback");
 		} else {
 			set beresp.status = 999;
 		}
-
-		if (beresp.http.foo !~ "bar(quux)") {
-			set beresp.http.neg0 = barbaz.backref(0, "neg0");
-			set beresp.http.neg1 = barbaz.backref(1, "neg1");
+		if (moredots.match(beresp.http.bar)) {
+			set beresp.http.bar0 = moredots.backref(0, "error0");
+			set beresp.http.bar1 = moredots.backref(1, "error1");
+			set beresp.http.bar2 = moredots.backref(2, "error2");
+			set beresp.http.bar3 = moredots.backref(3, "error3");
+			set beresp.http.bar4 = moredots.backref(4, "error4");
+			set beresp.http.bar5 = moredots.backref(5, "error5");
+			set beresp.http.bar6 = moredots.backref(6, "error6");
+			set beresp.http.bar7 = moredots.backref(7, "error7");
+			set beresp.http.bar8 = moredots.backref(8, "error8");
+			set beresp.http.bar9 = moredots.backref(9, "error9");
+			set beresp.http.bar10 = moredots.backref(10, "error10");
+			set beresp.http.bar11 = moredots.backref(11, "fallback");
 		} else {
 			set beresp.status = 999;
 		}
-
-		set beresp.http.bar = regsub(beresp.http.foo, "bar(baz)", "\1");
-		set beresp.http.regsub0 = barbaz.backref(0, "regsub0");
-		set beresp.http.regsub1 = barbaz.backref(1, "regsub1");
-
-		set beresp.http.bar = regsuball(beresp.http.foo, "(.)", "x");
-		set beresp.http.regsuball0 = barbaz.backref(0, "regsuball0");
-		set beresp.http.regsuball1 = barbaz.backref(1, "regsuball1");
 	}
 
 } -start
@@ -46,12 +54,36 @@ client c1 {
 	txreq
 	rxresp
 	expect resp.status == 200
-	expect resp.http.tilde0 == "barbaz"
-	expect resp.http.tilde1 == "bar"
-	expect resp.http.neg0 == "barbaz"
-	expect resp.http.neg1 == "bar"
-	expect resp.http.regsub0 == "barbaz"
-	expect resp.http.regsub1 == "bar"
-	expect resp.http.regsuball0 == "barbaz"
-	expect resp.http.regsuball1 == "bar"
+	expect resp.http.foo0 == "1234567890"
+	expect resp.http.foo1 == "1"
+	expect resp.http.foo2 == "2"
+	expect resp.http.foo3 == "3"
+	expect resp.http.foo4 == "4"
+	expect resp.http.foo5 == "5"
+	expect resp.http.foo6 == "6"
+	expect resp.http.foo7 == "7"
+	expect resp.http.foo8 == "8"
+	expect resp.http.foo9 == "9"
+	expect resp.http.foo10 == "0"
+	expect resp.http.foo11 == "fallback"
+	expect resp.http.bar0 == "12345678901"
+	expect resp.http.bar1 == "1"
+	expect resp.http.bar2 == "2"
+	expect resp.http.bar3 == "3"
+	expect resp.http.bar4 == "4"
+	expect resp.http.bar5 == "5"
+	expect resp.http.bar6 == "6"
+	expect resp.http.bar7 == "7"
+	expect resp.http.bar8 == "8"
+	expect resp.http.bar9 == "9"
+	expect resp.http.bar10 == "0"
+	expect resp.http.bar11 == "fallback"
+} -run
+
+logexpect l1 -v v1 -d 1 -g vxid -q "VCL_Error" {
+	expect 0 * Begin req
+	expect * = VCL_Error "^vmod re: backref \d+ out of range$"
+	expect * = VCL_Error "^vmod re: capturing substrings exceed max \d+"
+	expect * = VCL_Error "^vmod re: backref \d+ out of range$"
+	expect * = End
 } -run

src/tests/c06.vtc

+varnishtest "limit to backrefs 0 to 10 with dynamic matches"
+
+# same as c05.vtc with match_dyn()
+
+server s1 {
+	rxreq
+	txresp -hdr "Foo: 12345678901" -hdr "Bar: 123456789012" -body "1111\n"
+} -start
+
+varnish v1 -vcl+backend {
+	import re from "${vmod_topbuild}/src/.libs/libvmod_re.so";
+
+	sub vcl_init {
+	       new tendots = re.regex("");
+	       new moredots = re.regex("");
+	}
+
+	sub vcl_backend_response {
+		if (tendots.match_dyn("(.)(.)(.)(.)(.)(.)(.)(.)(.)(.)",
+				      beresp.http.foo)) {
+			set beresp.http.foo0 = tendots.backref(0, "error0");
+			set beresp.http.foo1 = tendots.backref(1, "error1");
+			set beresp.http.foo2 = tendots.backref(2, "error2");
+			set beresp.http.foo3 = tendots.backref(3, "error3");
+			set beresp.http.foo4 = tendots.backref(4, "error4");
+			set beresp.http.foo5 = tendots.backref(5, "error5");
+			set beresp.http.foo6 = tendots.backref(6, "error6");
+			set beresp.http.foo7 = tendots.backref(7, "error7");
+			set beresp.http.foo8 = tendots.backref(8, "error8");
+			set beresp.http.foo9 = tendots.backref(9, "error9");
+			set beresp.http.foo10 = tendots.backref(10, "error10");
+			set beresp.http.foo11 = tendots.backref(11, "fallback");
+		} else {
+			set beresp.status = 999;
+		}
+		if (moredots.match_dyn("(.)(.)(.)(.)(.)(.)(.)(.)(.)(.)(.)",
+				       beresp.http.bar)) {
+			set beresp.http.bar0 = moredots.backref(0, "error0");
+			set beresp.http.bar1 = moredots.backref(1, "error1");
+			set beresp.http.bar2 = moredots.backref(2, "error2");
+			set beresp.http.bar3 = moredots.backref(3, "error3");
+			set beresp.http.bar4 = moredots.backref(4, "error4");
+			set beresp.http.bar5 = moredots.backref(5, "error5");
+			set beresp.http.bar6 = moredots.backref(6, "error6");
+			set beresp.http.bar7 = moredots.backref(7, "error7");
+			set beresp.http.bar8 = moredots.backref(8, "error8");
+			set beresp.http.bar9 = moredots.backref(9, "error9");
+			set beresp.http.bar10 = moredots.backref(10, "error10");
+			set beresp.http.bar11 = moredots.backref(11, "fallback");
+		} else {
+			set beresp.status = 999;
+		}
+	}
+
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.http.foo0 == "1234567890"
+	expect resp.http.foo1 == "1"
+	expect resp.http.foo2 == "2"
+	expect resp.http.foo3 == "3"
+	expect resp.http.foo4 == "4"
+	expect resp.http.foo5 == "5"
+	expect resp.http.foo6 == "6"
+	expect resp.http.foo7 == "7"
+	expect resp.http.foo8 == "8"
+	expect resp.http.foo9 == "9"
+	expect resp.http.foo10 == "0"
+	expect resp.http.foo11 == "fallback"
+	expect resp.http.bar0 == "12345678901"
+	expect resp.http.bar1 == "1"
+	expect resp.http.bar2 == "2"
+	expect resp.http.bar3 == "3"
+	expect resp.http.bar4 == "4"
+	expect resp.http.bar5 == "5"
+	expect resp.http.bar6 == "6"
+	expect resp.http.bar7 == "7"
+	expect resp.http.bar8 == "8"
+	expect resp.http.bar9 == "9"
+	expect resp.http.bar10 == "0"
+	expect resp.http.bar11 == "fallback"
+} -run
+
+logexpect l1 -v v1 -d 1 -g vxid -q "VCL_Error" {
+	expect 0 * Begin req
+	expect * = VCL_Error "^vmod re: backref \d+ out of range$"
+	expect * = VCL_Error "^vmod re: capturing substrings exceed max \d+"
+	expect * = VCL_Error "^vmod re: backref \d+ out of range$"
+	expect * = End
+} -run

src/tests/c07.vtc

+varnishtest "re.backref not affected by standard VCL regex code"
+
+server s1 {
+	rxreq
+	txresp -hdr "Foo: barbaz" -body "1111\n"
+} -start
+
+varnish v1 -vcl+backend {
+	import re from "${vmod_topbuild}/src/.libs/libvmod_re.so";
+
+	sub vcl_init {
+	    	new barbaz = re.regex("(bar)baz");
+	}
+
+	sub vcl_backend_response {
+	    	if (!barbaz.match(beresp.http.foo)) {
+		        set beresp.status = 999;
+		}
+
+		if (beresp.http.foo ~ "bar(baz)") {
+			set beresp.http.tilde0 = barbaz.backref(0, "tilde0");
+			set beresp.http.tilde1 = barbaz.backref(1, "tilde1");
+		} else {
+			set beresp.status = 999;
+		}
+
+		if (beresp.http.foo !~ "bar(quux)") {
+			set beresp.http.neg0 = barbaz.backref(0, "neg0");
+			set beresp.http.neg1 = barbaz.backref(1, "neg1");
+		} else {
+			set beresp.status = 999;
+		}
+
+		set beresp.http.regsub
+		    = regsub(beresp.http.foo, "bar(baz)", "\1");
+		set beresp.http.regsub0 = barbaz.backref(0, "regsub0");
+		set beresp.http.regsub1 = barbaz.backref(1, "regsub1");
+
+		set beresp.http.regsuball
+		    = regsuball(beresp.http.foo, "(.)", "x");
+		set beresp.http.regsuball0 = barbaz.backref(0, "regsuball0");
+		set beresp.http.regsuball1 = barbaz.backref(1, "regsuball1");
+
+	    	if (!barbaz.match_dyn("(bar)baz", beresp.http.foo)) {
+		        set beresp.status = 999;
+		}
+
+		if (beresp.http.foo ~ "bar(baz)") {
+			set beresp.http.tilde0d = barbaz.backref(0, "tilde0d");
+			set beresp.http.tilde1d = barbaz.backref(1, "tilde1d");
+		} else {
+			set beresp.status = 999;
+		}
+
+		if (beresp.http.foo !~ "bar(quux)") {
+			set beresp.http.neg0d = barbaz.backref(0, "neg0d");
+			set beresp.http.neg1d = barbaz.backref(1, "neg1d");
+		} else {
+			set beresp.status = 999;
+		}
+
+		set beresp.http.regsubd
+		    = regsub(beresp.http.foo, "bar(baz)", "\1");
+		set beresp.http.regsub0d = barbaz.backref(0, "regsub0d");
+		set beresp.http.regsub1d = barbaz.backref(1, "regsub1d");
+
+		set beresp.http.regsuballd
+		    = regsuball(beresp.http.foo, "(.)", "x");
+		set beresp.http.regsuball0d = barbaz.backref(0, "regsuball0d");
+		set beresp.http.regsuball1d = barbaz.backref(1, "regsuball1d");
+	}
+
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.http.tilde0 == "barbaz"
+	expect resp.http.tilde1 == "bar"
+	expect resp.http.neg0 == "barbaz"
+	expect resp.http.neg1 == "bar"
+	expect resp.http.regsub == "baz"
+	expect resp.http.regsub0 == "barbaz"
+	expect resp.http.regsub1 == "bar"
+	expect resp.http.regsuball == "xxxxxx"
+	expect resp.http.regsuball0 == "barbaz"
+	expect resp.http.regsuball1 == "bar"
+	expect resp.http.tilde0d == "barbaz"
+	expect resp.http.tilde1d == "bar"
+	expect resp.http.neg0d == "barbaz"
+	expect resp.http.neg1d == "bar"
+	expect resp.http.regsubd == "baz"
+	expect resp.http.regsub0d == "barbaz"
+	expect resp.http.regsub1d == "bar"
+	expect resp.http.regsuballd == "xxxxxx"
+	expect resp.http.regsuball0d == "barbaz"
+	expect resp.http.regsuball1d == "bar"
+} -run

src/vmod_re.c

@@ -120,9 +120,9 @@ vmod_regex__fini(struct vmod_re_regex **rep)
 	FREE_OBJ(re);
 }
 
-VCL_BOOL __match_proto__()
-vmod_regex_match(const struct vrt_ctx *ctx, struct vmod_re_regex *re,
-		 VCL_STRING subject)
+static inline VCL_BOOL
+match(const struct vrt_ctx *ctx, struct vmod_re_regex *re, vre_t *vre,
+      VCL_STRING subject)
 {
 	ov_t *ov;
 	int s, nov[MAX_OV];
@@ -132,10 +132,13 @@ vmod_regex_match(const struct vrt_ctx *ctx, struct vmod_re_regex *re,
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(re, VMOD_RE_REGEX_MAGIC);
 
+	if (vre == NULL)
+		vre = re->vre;
+
 	AZ(pthread_setspecific(re->ovk, match_failed));
 
-	/* compilation error */
-	if (re->vre == NULL) {
+	/* compilation error at init time */
+	if (vre == NULL) {
 		AN(re->error);
 		VSLb(ctx->vsl, SLT_VCL_Error,
 		     "vmod re: error compiling regex: %s (position %d)",
@@ -147,7 +150,7 @@ vmod_regex_match(const struct vrt_ctx *ctx, struct vmod_re_regex *re,
 		subject = "";
 
 	/* XXX: cache_param->vre_limits incorrect?! */
-	s = VRE_exec(re->vre, subject, strlen(subject), 0, 0, nov, MAX_OV,
+	s = VRE_exec(vre, subject, strlen(subject), 0, 0, nov, MAX_OV,
 		     NULL);
 #if 0
 		     &cache_param->vre_limits);
@@ -188,6 +191,32 @@ vmod_regex_match(const struct vrt_ctx *ctx, struct vmod_re_regex *re,
 	return 1;
 }
 
+VCL_BOOL __match_proto__()
+vmod_regex_match(const struct vrt_ctx *ctx, struct vmod_re_regex *re,
+		 VCL_STRING subject)
+{
+	return match(ctx, re, NULL, subject);
+}
+
+VCL_BOOL __match_proto__()
+vmod_regex_match_dyn(const struct vrt_ctx *ctx, struct vmod_re_regex *re,
+		     VCL_STRING pattern, VCL_STRING subject)
+{
+	vre_t *vre;
+	int erroffset;
+	const char *error;
+
+	AN(pattern);
+	vre = VRE_compile(pattern, 0, &error, &erroffset);
+	if (vre == NULL) {
+		VSLb(ctx->vsl, SLT_VCL_Error,
+		     "vmod re: error compiling regex \"%s\": %s (position %d)",
+		     pattern, error, erroffset);
+		return 0;
+	}
+	return match(ctx, re, vre, subject);
+}
+
 VCL_STRING __match_proto__()
 vmod_regex_backref(const struct vrt_ctx *ctx, struct vmod_re_regex *re,
 		   VCL_INT refnum, VCL_STRING fallback)

src/vmod_re.vcc

@@ -52,7 +52,7 @@ substring ``"bar"`` from one request header into another::
 
 It requires two regex executions when a match is found, the second one
 less efficient than the first (since it must match the entire string
-while capturing a substring), and is just cumbersome.
+to be replaced while capturing a substring), and is just cumbersome.
 
 The equivalent solution with the VMOD looks like this::
 
@@ -73,7 +73,35 @@ the capture expression, only describing the substring to be
 matched. When a match with the ``match`` method succeeds, then a
 captured string can be obtained from the ``backref`` method.
 
-XXX: dynamic matches not yet implemented in this version
+The VMOD also supports dynamic regex matching with the ``match_dyn``
+method::
+
+	import re;
+
+	sub vcl_init {
+		new myregex = re.regex("");
+	}
+
+	sub vcl_backend_response {
+		if (myregex.match_dyn(beresp.http.Bar + "(\d+)",
+		                      req.http.Foo)) {
+		   set beresp.http.Baz = myregex.backref(1, "");
+		}
+	}
+
+In ``match_dyn``, the regex in the first argument is compiled when it
+is called, and matched against the string in the second argument; the
+regex provided in ``vcl_init`` is ignored. Subsequent calls to
+``backref`` extract substrings from the matched string.
+
+As with the constructor, the regex argument to ``match_dyn`` should
+contain any capturing expressions needed for calls to ``backref``.
+
+``match_dyn`` makes it possible to construct regexen whose contents
+are not fully known until runtime, but ``match`` is more efficient,
+since it re-uses the compiled expression obtained at VCL
+initialization. So if you are matching against a fixed pattern that
+never changes during the lifetime of VCL, use ``match``.
 
 $Object regex(STRING)
 
@@ -92,33 +120,45 @@ Example
 $Method BOOL .match(STRING)
 
 Description
-	Determines whether the given string matches the regex;
-	functionally equivalent to VCL's infix operator ``~``.
+	Determines whether the given string matches the regex compiled
+	by the constructor; functionally equivalent to VCL's infix
+	operator ``~``.
 
 Example
 	``if (myregex.match(beresp.http.Surrogate-Control)) { # ...``
 
+$Method BOOL .match_dyn(STRING, STRING)
+
+Description
+	Compiles the regular expression given in the first argument,
+	and determines whether it matches the string in the second
+	argument. The regex supplied in the constructor is ignored.
+
+Example
+	``if (myregex.match_dyn(req.http.Foo + "(\d+)",``
+				``beresp.http.Bar)) { # ...``
+
 $Method STRING .backref(INT, STRING)
 
 Description
 	Extracts the `nth` subexpression of the most recent successful
-	call of the ``match`` method for this object in the same VCL
-	subroutine call, or a fallback string in case the extraction
-	fails.  Backref 0 indicates the entire matched string. Thus
-	this function behaves like the ``\n`` symbols in ``regsub``
-	and ``regsuball``, and the ``$1``, ``$2`` ...  variables in
-	Perl.
+	call of the ``match`` or ``match_dyn`` method for this object
+	in the same VCL subroutine call, or a fallback string in case
+	the extraction fails.  Backref 0 indicates the entire matched
+	string. Thus this function behaves like the ``\n`` symbols in
+	``regsub`` and ``regsuball``, and the ``$1``, ``$2`` ...
+	variables in Perl.
 
 	After unsuccessful matches, the ``fallback`` string is returned
 	for any call to ``backref``.
 
 	The VCL infix operators ``~`` and ``!~`` do not affect this
-	function, nor do the functions ``regsub`` or ``regsuball``.
+	method, nor do the functions ``regsub`` or ``regsuball``.
 
 	If ``backref`` is called without any prior call to ``match``
-	for this object in the same VCL context, then an error message
-	is emitted to the Varnish log using the ``VCL_Error`` tag, and
-	the fallback string is returned.
+	or ``match_dyn`` for this object in the same VCL context, then
+	an error message is emitted to the Varnish log using the
+	``VCL_Error`` tag, and the fallback string is returned.
 
 Example
         ``set beresp.ttl = std.duration(myregex.backref(1, "120"), 120s);``
@@ -205,14 +245,17 @@ Version 0.2: various fixes, last version compatible with Varnish 3
 
 Version 0.3: compatible with Varnish 4
 
+Version 0.4: support dynamic matches
+
 
 LIMITATIONS
 ===========
 
-Regular expressions passed into the constructor are compiled at
-run-time, so there are no errors at VCL compile-time for invalid
-expressions. If an expression is invalid, then a ``VCL_error`` message
-is emitted to the Varnish log, and matches always fail.
+Regular expressions passed into the constructor and into ``match_dyn``
+are compiled at run-time, so there are no errors at VCL compile-time
+for invalid expressions. If an expression is invalid, then a
+``VCL_error`` message is emitted to the Varnish log, and matches
+always fail.
 
 The VMOD allocates memory for captured subexpressions from Varnish
 workspaces, whose sizes are determined by the runtime parameters
@@ -223,16 +266,17 @@ some overhead. For typical usage, the default workspace sizes are
 probably enough; but if you are matching against many, long strings in
 each client or backend context, you might need to increase the Varnish
 parameters for workspace sizes. If the VMOD cannot allocate enough
-workspace, then a ``VCL_error`` message is emitted, and both ``match``
-and ``backref`` will fail. (If you're just using the regexen for
-matching and not to capture backrefs, then you might as well just use
-the standard VCL operators ``~`` and ``!~``, and save the workspace.)
+workspace, then a ``VCL_error`` message is emitted, and the match
+methods as well as ``backref`` will fail. (If you're just using the
+regexen for matching and not to capture backrefs, then you might as
+well just use the standard VCL operators ``~`` and ``!~``, and save
+the workspace.)
 
 ``backref`` can extract up to 10 subexpressions, in addition to the
-full expression indicated by backref 0. If the ``match`` operation
-would have resulted in more than 11 captures (10 substrings and the
-full string), then a ``VCL_Error`` message is emitted to the Varnish
-log, and the captures are limited to 11.
+full expression indicated by backref 0. If a ``match`` or
+``match_dyn`` operation would have resulted in more than 11 captures
+(10 substrings and the full string), then a ``VCL_Error`` message is
+emitted to the Varnish log, and the captures are limited to 11.
 
 XXX: the following paragraph is currently not true, bug is under
 investigation