implement range checking on compile option parens_nest_limit, and test it

src/tests/compile_opts.vtc

@@ -19,6 +19,9 @@ varnishtest "compile options"
 # - how to test never_backslash_c? pcre2 testdata only mentions it in
 #   one test, and vtc can't parse the pattern, since it has more {'s
 #   than }'s.
+# - test for no_auto_capture requires backrefs
+# - test for no_auto_possess, no_dotstar_anchor and no_start_optimize
+#   require callouts and/or pattern_info
 
 # allow_empty_class
 varnish v1 -vcl {
@@ -737,3 +740,40 @@ client c1 {
 	expect resp.http.r7-2 == "false"
 	expect resp.http.r7-3 == "false"
 } -run
+
+# parens_nest_limit
+varnish v1 -vcl {
+	import pcre2 from "${vmod_topbuild}/src/.libs/libvmod_pcre2.so";
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_init {
+		new r = pcre2.regex("(((((a)))))", parens_nest_limit=5);
+	}
+}
+
+varnish v1 -errvcl {vmod pcre2 error: Cannot compile '(((((a)))))' in r constructor: parentheses are too deeply nested at offset 3} {
+	import pcre2 from "${vmod_topbuild}/src/.libs/libvmod_pcre2.so";
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_init {
+		new r = pcre2.regex("(((((a)))))", parens_nest_limit=2);
+	}
+}
+
+varnish v1 -errvcl {vmod pcre2 error: parens_nest_limit (-1) out of range in r constructor (must be >= 0 and <= 4294967295)} {
+	import pcre2 from "${vmod_topbuild}/src/.libs/libvmod_pcre2.so";
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_init {
+		new r = pcre2.regex("a", parens_nest_limit=-1);
+	}
+}
+
+varnish v1 -errvcl {vmod pcre2 error: parens_nest_limit (4294967296) out of range in r constructor (must be >= 0 and <= 4294967295)} {
+	import pcre2 from "${vmod_topbuild}/src/.libs/libvmod_pcre2.so";
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_init {
+		new r = pcre2.regex("a", parens_nest_limit=4294967296);
+	}
+}

src/vmod_pcre2.c

@@ -28,6 +28,7 @@
 #include "config.h"
 
 #include <stdint.h>
+#include <inttypes.h>
 #include <string.h>
 #include <locale.h>
 
@@ -194,8 +195,13 @@ vmod_regex__init(VRT_CTX, struct vmod_pcre2_regex **regexp,
 		     "constructor", (long long) max_pattern_len, vcl_name);
 		return;
 	}
+	if (parens_nest_limit < 0 || parens_nest_limit > UINT32_MAX) {
+		VERR(ctx, "parens_nest_limit (%lld) out of range in %s "
+		     "constructor (must be >= 0 and <= %" PRIu32 ")",
+		     (long long)parens_nest_limit, vcl_name, UINT32_MAX);
+		return;
+	}
 
-	/* XXX check that parens_nest_limit >= 0 and <= UINT32_MAX */
 	/* XXX check illegal combinations such as never_ucp && ucp ... ? */
 	if (anchored)
 		options |= PCRE2_ANCHORED;