Support for current varnish master / varnish 6

new now need the VARNISHSRC configure variable again pointing to the
varnish source tree to check the workspace_client parameter setting.

Regression tested with make check against:

varnish-5.2.1
varnish-5.1.0
varnish-4.1.9
varnish-3.0.7
varnish-2.1.5

README.rst

@@ -8,8 +8,8 @@ Varnish Device Classification Service Module
 
 :Manual section: 3
 :Authors: Nils Goroll
-:Date: 2015-06-16
-:Version: 0.5
+:Date: 2017-11-20
+:Version: 0.6
 
 SYNOPSIS
 ========
@@ -21,8 +21,8 @@ Command line
 
 	<prefix>/bin/dcs
 
-Varnish VMOD (Varnish 3 and Varnish 4)
---------------------------------------
+Varnish VMOD (Varnish 3 and newer)
+----------------------------------
 
 VCL:
 
@@ -135,8 +135,7 @@ INTERFACES
 
 The following use cases are supported
 
-* Use as a Varnish 4 module
-* Use as a Varnish 3 module
+* Use as a Varnish module (vmod) for varnish 3 and later
 * Use with Varnish 2 as inline-C
 * Command line tools
 
@@ -180,7 +179,13 @@ To build the documentation
 
 To build the varnish module (VMOD)
 
-* the source tree of the Varnish version the VMOD is to be used
+* For Varnish 4 and 5
+
+  A full installation of Varnish to be used with the VMOD.
+
+* For Varnish 3 as well as varnish 6 and later:
+
+  The source tree of the Varnish version the VMOD is to be used
   with. Varnish must have been configured and compiled in this source
   tree.
 
@@ -246,8 +251,8 @@ required:
    (normally `/usr/local`) or in the prefix specified by the `--prefix`
    argument to `configure`.
 
-Building the varnish module (vmod) for Varnish 4 and higher
------------------------------------------------------------
+Building the varnish module (vmod) for Varnish 4 and 5
+------------------------------------------------------
 
 If varnish is installed in a standard directory, generic :ref:BUILDING
 as explained above should also build the vmod.
@@ -270,6 +275,13 @@ When building the vmod, an additional
 
 step is recommended to run the bundled `varnishtest` tests.
 
+Building the varnish module (vmod) for Varnish 6
+------------------------------------------------
+
+In addition to the instructions given before for Varnish 4 and 5, a
+build against Varnish 6 needs access to the varnish source: the
+`VARNISHSRC` variable needs to be set as documented for Varnish 3 below.
+
 Building the varnish module (vmod) for Varnish 3
 ------------------------------------------------
 
@@ -555,6 +567,8 @@ HISTORY
 * Version 0.4: Also support Varnish 4. Take memory from Varnish workspace
   instead of stack for Varnish 4.
 
+* Version 0.5: Also uspport Varnish 6 (post 5.2)
+
 BUGS
 ====
 
@@ -569,7 +583,7 @@ SEE ALSO
 COPYRIGHT
 =========
 
-* Copyright 2014 UPLEX - Nils Goroll Systemoptimierung
+* Copyright 2014-2017 UPLEX - Nils Goroll Systemoptimierung
 
 LICENSE
 =======

check/tests6/dcs01.vtc

+# $Id$
+
+varnishtest "Test dcs vmod"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -arg "-p workspace_client=9k" -vcl+backend {
+	import dcs from "${vmod_topbuild}/src/.libs/libvmod_dcs.so";
+	import vtc;
+
+	sub vcl_recv {
+	    if (req.xid == "1001") {
+		# make sure we run into malloc for first request
+		if (vtc.workspace_free(client) <= 1024) {
+		    #
+		} else if (vtc.workspace_free(client) > 1024) {
+		    # has been adjusted
+		    vtc.workspace_alloc(client, -1024);
+		}
+	    }
+	    # this code will classify
+	    #
+	    # for best performance in production, only call classify
+	    # once or use inline-C to save the return value of
+	    # classify (int)
+	    # exotic use cases
+	    set req.http.xx-entry-key     = dcs.entry_key(dcs.classify());
+	    set req.http.xx-type-id       = dcs.type_id(dcs.classify());
+
+	    # common use cases
+	    set req.http.x-nb-classified  = dcs.type_name(dcs.classify());
+	    set req.http.X-DeviceClass    = dcs.type_class(dcs.classify());
+
+	    return (synth(200));
+	}
+
+	sub vcl_synth {
+	    set resp.http.xx-entry-key     = req.http.xx-entry-key;
+	    set resp.http.xx-type-id       = req.http.xx-type-id;
+
+	    set resp.http.x-nb-classified  = req.http.x-nb-classified;
+	    set resp.http.X-DeviceClass    = req.http.X-DeviceClass;
+	    synthetic ({"classified ad here
+"});
+	    return (deliver);
+	}
+
+} -start
+
+# set the workspace too small after the vmod has been initialized
+# so we run into the malloc case for the first request
+
+varnish v1 -cliok "param.show workspace_client"
+varnish v1 -cliok "param.set workspace_client 9k"
+varnish v1 -cliok "param.show workspace_client"
+
+logexpect l1 -v v1 -g raw -d 1 {
+	expect * 1001 VCL_call  {^RECV}
+	expect 0    = Error     {^notice: workspace_client is set too low}
+	expect 0    = Error     {^notice: malloc'ing}
+	expect 0    = ReqHeader {^xx-entry-key: unidentified}
+	expect 0    = Error     {^notice: malloc'ing}
+	expect 0    = ReqHeader {^xx-type-id: 0}
+	expect 0    = Error     {^notice: malloc'ing}
+	expect 0    = ReqHeader {^x-nb-classified: unidentified}
+	expect 0    = Error     {^notice: malloc'ing}
+	expect 0    = ReqHeader {^X-DeviceClass: desktop}
+
+	# for the next request, we must not see the notices because
+	# workspace should be adjusted now
+	expect * 1005 VCL_call  {^RECV}
+	expect 0    = ReqHeader {^xx-entry-key: android}
+	expect 0    = ReqHeader {^xx-type-id:}
+	expect 0    = ReqHeader {^x-nb-classified: Tablet}
+	expect 0    = ReqHeader {^X-DeviceClass: tablet}
+}
+
+logexpect l1 -start
+
+client c1 {
+	# no User-Agent
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.xx-entry-key == "unidentified"
+	expect resp.http.xx-type-id == "0"
+	expect resp.http.x-nb-classified == "unidentified"
+	expect resp.http.X-DeviceClass == "desktop"
+} -run
+
+client c1 {
+	txreq -hdr "user-agent: willgetignored" -hdr "x-wap-profile: anything"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.xx-entry-key == "generic wap"
+	# cant test - can change easily
+	# expect resp.http.xx-type-id == "11"
+	expect resp.http.x-nb-classified == "Mobile Phone"
+	expect resp.http.X-DeviceClass == "smartphone"
+} -run
+
+client c1 {
+	txreq -hdr "User-Agent: Opera/9.80 (Android; Opera Mini/19.0.1340/34.1309; U; de) Presto/2.8.119 Version/11.10" \
+	      -hdr "X-OperaMini-Phone-UA: Mozilla/5.0 (Linux; U; Android 4.3; de-de; ME302C Build/JSS15Q) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Safari/534.30"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.xx-entry-key == "android*me302c"
+	# cant test - can change easily
+	# expect resp.http.xx-type-id == "11"
+	expect resp.http.x-nb-classified == "Tablet"
+	expect resp.http.X-DeviceClass == "tablet"
+} -run
+
+# append additional headers
+client c1 {
+	txreq -hdr "User-Agent: motorokr z6/r60_g_80.xx.yyi mozilla/4.0" \
+	      -hdr "X-Device-User-Agent: (compatible; msie 6.0 linux;" \
+	      -hdr "X-Original-User-Agent: motorokr z6;nnn) profile/midp-2.0" \
+	      -hdr "X-Goog-Source: configuration/cldc-1.1 opera 8.50[yy] up.link/6.3.0.0.0 netbiscuits"
+
+	rxresp
+	expect resp.status == 200
+	expect resp.http.xx-entry-key == "motorokr z6/r60_g_80.xx.yyi mozilla/4.0 (compatible; msie 6.0 linux; motorokr z6;nnn) profile/midp-2.0 configuration/cldc-1.1 opera 8.50[yy] up.link/6.3.0.0.0 netbiscuits"
+	# cant test - can change easily
+	# expect resp.http.xx-type-id == "11"
+	expect resp.http.x-nb-classified == "Mobile Phone"
+	expect resp.http.X-DeviceClass == "smartphone"
+} -run
+
+logexpect l1 -wait

check/tests6/dcs02_excessive_ua_length.vtc

+# $Id$
+
+varnishtest "Test dcs vmod excessively long ua"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import dcs from "${vmod_topbuild}/src/.libs/libvmod_dcs.so";
+
+	sub vcl_recv {
+	    # this code will classify
+	    #
+	    # for best performance in production, only call classify
+	    # once or use inline-C to save the return value of
+	    # classify (int)
+
+	    # exotic use cases
+	    set req.http.xx-entry-key     = dcs.entry_key(dcs.classify());
+	    set req.http.xx-type-id       = dcs.type_id(dcs.classify());
+
+	    # common use cases
+	    set req.http.x-nb-classified  = dcs.type_name(dcs.classify());
+	    set req.http.X-DeviceClass    = dcs.type_class(dcs.classify());
+
+	    return (synth(200));
+	}
+
+	sub vcl_synth {
+	    set resp.http.xx-entry-key     = req.http.xx-entry-key;
+	    set resp.http.xx-type-id       = req.http.xx-type-id;
+
+	    set resp.http.x-nb-classified  = req.http.x-nb-classified;
+	    set resp.http.X-DeviceClass    = req.http.X-DeviceClass;
+	    synthetic ({"classified ad here
+"});
+	    return (deliver);
+	}
+
+} -start
+
+client c1 {
+	txreq -hdr "User-Agent: motorokr z6/r60_g_80.xx.yyi mozilla/4.0 (compatible; msie 6.0 linux; motorokr z6;nnn) profile/midp-2.0 configuration/cldc-1.1 opera 8.50[yy] up.link/6.3.0.0.0 netbiscuits motorokr z6/r60_g_80.xx.yyi mozilla/4.0 (compatible; msie 6.0 linux; motorokr z6;nnn) profile/midp-2.0 configuration/cldc-1.1 opera 8.50[yy] up.link/6.3.0.0.0 netbiscuits motorokr z6/r60_g_80.xx.yyi mozilla/4.0 (compatible; msie 6.0 linux; motorokr z6;nnn) profile/midp-2.0 configuration/cldc-1.1 opera 8.50[yy] up.link/6.3.0.0.0 netbiscuits motorokr z6/r60_g_80.xx.yyi mozilla/4.0 (compatible; msie 6.0 linux; motorokr z6;nnn) profile/midp-2.0 configuration/cldc-1.1 opera 8.50[yy] up.link/6.3.0.0.0 netbiscuits motorokr z6/r60_g_80.xx.yyi mozilla/4.0 (compatible; msie 6.0 linux; motorokr z6;nnn) profile/midp-2.0 configuration/cldc-1.1 opera 8.50[yy] up.link/6.3.0.0.0 netbiscuits motorokr z6/r60_g_80.xx.yyi mozilla/4.0 (compatible; msie 6.0 linux; motorokr z6;nnn) profile/midp-2.0 configuration/cldc-1.1 opera 8.50[yy] up.link/6.3.0.0.0 netbiscuits" \
+	      -hdr "X-Goog-Source: even longer"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.xx-entry-key == "motorokr z6/r60_g_80.xx.yyi mozilla/4.0 (compatible; msie 6.0 linux; motorokr z6;nnn) profile/midp-2.0 configuration/cldc-1.1 opera 8.50[yy] up.link/6.3.0.0.0 netbiscuits"
+	# cant test
+	# expect resp.http.xx-type-id == "11"
+	expect resp.http.x-nb-classified == "Mobile Phone"
+	expect resp.http.X-DeviceClass == "smartphone"
+} -run
+

configure.ac

@@ -130,11 +130,10 @@ if test "x$VMODDIR" != x && test "x$VMOD_DIR" = x ; then
 	AC_MSG_FAILURE([VMODDIR is now called VMOD_DIR])
 fi
 
-AC_ARG_VAR([VARNISHSRC], [path to Varnish source tree for vmod (varnish3)])
+AC_ARG_VAR([VARNISHSRC], [path to Varnish source (varnish3 / varnish6)])
 
 m4_ifdef([VARNISH_VMOD_INCLUDES],
 [
-if test "x$VARNISHSRC" = x; then
 	AM_CONDITIONAL([HAVE_VARNISH_M4], [true])
 	AC_MSG_NOTICE([Configuring from pkg-config (Varnish 4 or newer)])
 
@@ -158,7 +157,13 @@ if test "x$VARNISHSRC" = x; then
 
 	AC_SUBST([VARNISH_VERSION], [$($PKG_CONFIG --modversion varnishapi)])
 	case "x$VARNISH_VERSION" in
-	    xtrunk|x5*)
+	    xtrunk|x6*)
+		AC_SUBST([VCCFILE], [vmod_dcs5.vcc])
+		AC_DEFINE([VARNISH_MAJMIN], [60],
+		  [Define the Varnish major version we compile against])
+		AC_SUBST([TESTDIR], [tests6])
+		;;
+	    x5*)
 		AC_SUBST([VCCFILE], [vmod_dcs5.vcc])
 		AC_DEFINE([VARNISH_MAJMIN], [50],
 		  [Define the Varnish major version we compile against])
@@ -180,18 +185,25 @@ if test "x$VARNISHSRC" = x; then
 		AC_MSG_ERROR([unsupported varnish varsion])
 		;;
 	esac
-else
-	AM_CONDITIONAL([HAVE_VARNISH_M4], [false])
-	AC_MSG_NOTICE([VARNISHSRC overrides pkg-config])
-	AC_CHECK_FILE([$VARNISHSRC/lib/libvcc/vmodtool.py], [
-		AC_MSG_FAILURE([[VARNISHSRC seems to point to varnish4 sources.
-This is now desupported. Instead, point pkg-config to the right path
-and call autogen and configure with VARNISHSRC unset:
-
-	export PKG_CONFIG_PATH=.../pkgconfig
-	./autogen.sh && configure ...
-]])
-	])
+	if test "x$VARNISH_VERSION" = "xtrunk" ||
+	   test "x$VARNISH_VERSION" = "x6*" ; then
+	    if test "x$VARNISHSRC" = x; then
+		AC_MSG_FAILURE([Varnish trunk / varnish 6 needs VARNISHSRC])
+	    else
+		VARNISHSRC=`cd $VARNISHSRC && pwd`
+		AC_CHECK_FILE([$VARNISHSRC/bin/varnishd/common/common_param.h],[
+			VMOD_INCLUDES="$VMOD_INCLUDES -I$VARNISHSRC/bin/varnishd"
+			AC_SUBST([VMOD_INCLUDES])
+		],[
+			AC_MSG_FAILURE(
+[Could not find $VARNISHSRC/bin/varnishd/common/common_param.h])
+		])
+	    fi
+	elif test "x$VARNISHSRC" = x; then
+		: # OK
+	else
+		AC_MSG_FAILURE([VARNISHSRC must only be set for varnish < 4 and > 6])
+		# continue in common else case
 ],[
 AM_CONDITIONAL([HAVE_VARNISH_M4], [false])
 if test "x$VARNISHSRC" = x; then

src/dcs_varnish.c

@@ -78,10 +78,13 @@ const char *DCS_GetHdr(dcs_ctx *ctx, const struct gethdr_s *hdr) {
 #endif
 
 #ifdef DCS_USE_WS
-/* XXX better way? */
-#undef assert
-#include "cache/cache.h"
-#define DCS_USE_WS_MINIMUM PRNDUP(64 * 1024 + DCS_MATCH_MEM_SZ + UA_LIMIT)
+# if VARNISH_MAJMIN < 60
+#  undef assert
+#  include "cache/cache.h"
+# else
+#  include "vas.h"
+# endif /* VARNISH_MAJMIN < 60 */
+# define DCS_USE_WS_MINIMUM PRNDUP(64 * 1024 + DCS_MATCH_MEM_SZ + UA_LIMIT)
 #endif
 
 

src/vmod_dcs.c

@@ -30,10 +30,18 @@
 
 #include "dcs_config.h"
 
-#if VARNISH_MAJMIN >= 41
-#include "vcl.h"
+#if VARNISH_MAJMIN >= 60
+# include "cache/cache.h"
+# include "vcl.h"
+# include "vas.h"
+# include "common/common_param.h"
+#else
+# if VARNISH_MAJMIN >= 41
+#  include "vcl.h"
+# endif
+# include "vrt.h"
 #endif
-#include "vrt.h"
+
 #include "vcc_if.h"
 
 #define TWEAK_NOTICE(ctx, ...) do {					\