generate the class assignment code from a new config file - classes.conf

.gitignore

@@ -55,6 +55,7 @@ src/dcs
 src/dcs_test
 src/gen/dcs_classifier.c
 src/gen/dcs_classifier.h
+src/gen/dcs_type.c
 src/gen/dcs.db*
 src/gen/fixup.*
 src/vcc_if.c

README.rst

@@ -88,6 +88,24 @@ For demonstration purposes, we provice a simple database file with
 some minimal and incomplete classification information in
 `src/dcs_demo.db`. See :ref:using_the_demo_db for details.
 
+Meta Classes
+------------
+
+Classification types from the database file can be associated with
+meta-classes in the file `src/classes.conf`. Its format is
+
+::
+
+	[classname]
+	Typename from the database
+
+Note that the bundled tests need entries from the bundled
+classes.conf.
+
+During the build process, `gen_dcs_classifier.pl` emits warnings if
+entries are missing from the classes configuration or if entries
+remain unused. It may be advisable to update the configuration when
+these warnigs are seen.
 
 
 PERFORMANCE
@@ -355,9 +373,7 @@ might set `x-nb-classified` to "Mobile Phone"
 STRING type_class(INT)
 ----------------------
 
-Returns one of three meta type names "smartphone" (for mobile
-devices), "tablet" or "desktop" for the dcs db entry whose index is
-given as the integer argument.
+Returns one of the meta types defined in `src/classes.conf`
 
 Example:
 
@@ -499,6 +515,8 @@ HISTORY
         smartphone      mob
         tablet          tab
 
+* Version 0.3: Class assignments can now be defined in `src/classes.conf`
+
 BUGS
 ====
 

src/Makefile.am

@@ -18,12 +18,12 @@ lib_LTLIBRARIES = libdcs.la
 bin_PROGRAMS = dcs dcs_test
 
 libdcs_la_SOURCES = \
-	dcs_match.c \
-	dcs_type.c
+	dcs_match.c
 
 nodist_libdcs_la_SOURCES = \
 	gen/dcs_classifier.h \
-	gen/dcs_classifier.c
+	gen/dcs_classifier.c \
+	gen/dcs_type.c
 
 # -- dcs
 
@@ -41,7 +41,9 @@ dcs_test_SOURCES = \
 
 # -
 
-EXTRA_DIST = gen_dcs_classifier.pl
+EXTRA_DIST = \
+	gen_dcs_classifier.pl \
+	classes.conf
 
 # included c source
 if INSTALL_VARNISH2
@@ -56,12 +58,12 @@ dist_data_DATA = \
 	dcs_varnish.c \
 	dcs_match.h \
 	dcs_match.c \
-	dcs_type.h \
-	dcs_type.c
+	dcs_type.h
 
 nodist_data_DATA = \
 	gen/dcs_classifier.c \
-	gen/dcs_classifier.h
+	gen/dcs_classifier.h \
+	gen/dcs_type.c
 endif
 
 ## varnish 3 vmod
@@ -109,9 +111,9 @@ tests/*.vtc: libvmod_dcs.la
 	$(VARNISHSRC)/bin/varnishtest/varnishtest -Dvarnishd=$(VARNISHSRC)/bin/varnishd/varnishd -Dvmod_topbuild=$(abs_top_builddir) $@
 
 check: $(VMOD_TESTS)
+	./dcs_test
 
 EXTRA_DIST += \
 	vmod_dcs.vcc	\
 	$(VMOD_TESTS)
-
 endif

src/classes.conf

+# the first class is also the default class used for missing
+# entries
+
+[desktop]
+Bot
+Desktop-Browser
+DevTool
+Settop-Box, TV
+unidentified
+unknown
+
+[smartphone]
+Camera
+CE-Device
+MediaPlayer
+MobileConsole
+Mobile-Browser
+Mobile Phone
+wearable computer
+
+[tablet]
+eReader
+Tablet
+
+# obsolete in netbiscuits dcs db as of 2014-08-13
+#
+# [desktop]
+# Computer
+#
+# [smart-phone]
+# OperatingSystem

src/dcs_type.c

-/*
- * Copyright (c) 2014 UPLEX - Nils Goroll Systemoptimierung
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the
- *    distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS _AS IS_ AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
- * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
- * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "dcs_config.h"
-
-#include <errno.h>
-
-#include "dcs_classifier.h"
-#include "dcs_type.h"
-	    
-#define check_type_id(type_id, ret)					\
-	do {								\
-		if ((type_id < 0) || (type_id > (DCS_TYPE_COUNT - 1))) { \
-			errno = EINVAL;					\
-			return (ret);					\
-		}							\
-	} while(0)
-
-const char *
-dcs_type_name(int type_id /* enum dcs_type */) {
-	check_type_id(type_id, 0);
-	return dcs_type[type_id];
-}
-
-/*
- * useful meta-classification to just three string constants:
- *
- */
-enum dcs_type_class {
-	T_CLASS_MISSING = 0,
-	T_CLASS_MOB,
-	T_CLASS_TAB,
-	T_CLASS_DSK,
-	_T_CLASS_LIMIT
-};
-
-/*
- * the class string "smartphone" might not be optimal,
- * - read this as "mobile device".
- */
-const char * const dcs_type_class_str[_T_CLASS_LIMIT] = {
-	[T_CLASS_MISSING]	= "desktop",
-	[T_CLASS_MOB]		= "smartphone",
-	[T_CLASS_TAB]		= "tablet",
-	[T_CLASS_DSK]		= "desktop"
-};
-	
-enum dcs_type dcs_type2class[DCS_TYPE_COUNT] = {
-	[NB_T_UNIDENTIFIED]		= T_CLASS_DSK,
-	[NB_T_BOT]			= T_CLASS_DSK,
-	[NB_T_CAMERA]			= T_CLASS_MOB,
-	[NB_T_CE_DEVICE]		= T_CLASS_MOB,	// android 1.5; en-us; nimble
-	[NB_T_COMPUTER]			= T_CLASS_DSK,	// macintosh*intel*mac os
-	[NB_T_DESKTOP_BROWSER]		= T_CLASS_DSK,
-	[NB_T_DEVTOOL]			= T_CLASS_DSK,
-	[NB_T_EREADER]			= T_CLASS_TAB,
-	[NB_T_MEDIAPLAYER]		= T_CLASS_MOB,	// ipod, ms zune, yp-gb1
-	[NB_T_MOBILECONSOLE]		= T_CLASS_MOB,
-	[NB_T_MOBILE_BROWSER]		= T_CLASS_MOB,
-	[NB_T_MOBILE_PHONE]		= T_CLASS_MOB,
-	[NB_T_OPERATINGSYSTEM]		= T_CLASS_MOB,	// ios 4.2
-	[NB_T_SETTOP_BOX_TV]		= T_CLASS_DSK,
-	[NB_T_TABLET]			= T_CLASS_TAB,
-	[NB_T_WEARABLE_COMPUTER] 	= T_CLASS_MOB	// android 4.0*glass
-};
-
-const char *
-dcs_type_class(int type_id /* enum dcs_type */) {
-	check_type_id(type_id, 0);
-	return (dcs_type_class_str[dcs_type2class[type_id]]);
-}

src/gen/Makefile.am

 # gen/Makefile.am
 
-BUILT_SOURCES = dcs_classifier.c dcs_classifier.h
+BUILT_SOURCES = \
+	dcs_classifier.c \
+	dcs_classifier.h \
+	dcs_type.c
 
-dcs_classifier.c dcs_classifier.h: ../gen_dcs_classifier.pl $(DCS_DBFILE)
+dcs_classifier.c dcs_classifier.h dcs_type.c: ../gen_dcs_classifier.pl $(DCS_DBFILE) ../classes.conf
 	if [[ -f fixup.remove ]] && [[ -f fixup.reorder ]] ; then \
-		../gen_dcs_classifier.pl $(DCS_DBFILE) $(DCS_KEY) fixup.remove fixup.reorder ; \
+		../gen_dcs_classifier.pl $(DCS_DBFILE) $(DCS_KEY) ../classes.conf fixup.remove fixup.reorder ; \
 	else \
-		../gen_dcs_classifier.pl $(DCS_DBFILE) $(DCS_KEY) ; \
+		../gen_dcs_classifier.pl $(DCS_DBFILE) $(DCS_KEY) ../classes.conf ; \
 	fi
 
 # fixup db
 fixup.remove fixup.reorder fixup.out:
 	- rm fixup.remove fixup.reorder fixup.out
-	- rm dcs_classifier.c dcs_classifier.h
-	$(MAKE) $(AM_MAKEFLAGS) dcs_classifier.c dcs_classifier.h
+	- rm dcs_classifier.c dcs_classifier.h dcs_type.c
+	$(MAKE) $(AM_MAKEFLAGS) dcs_classifier.c dcs_classifier.h dcs_type.c
 	( cd .. ; $(MAKE) $(AM_MAKEFLAGS) dcs_test )
 	while ! ../dcs_test fixup.remove fixup.reorder >>fixup.out ; do \
 		( rm dcs_classifier.c dcs_classifier.h ; \
-		  if ! $(MAKE) $(AM_MAKEFLAGS) dcs_classifier.c dcs_classifier.h ; then break ; fi ; \
+		  if ! $(MAKE) $(AM_MAKEFLAGS) dcs_classifier.c dcs_classifier.h dcs_type.c ; then break ; fi ; \
 		  cd .. ; $(MAKE) $(AM_MAKEFLAGS) dcs_test ) \
 	done
 
@@ -41,4 +44,11 @@ dcs.db:	dcs.db.refresh
 		mv -f dcs.db.tmp dcs.db ;				\
 	fi
 
-CLEANFILES = dcs_classifier.c dcs_classifier.h fixup.remove fixup.reorder fixup.out dcs.db
\ No newline at end of file
+CLEANFILES = \
+	dcs_classifier.c \
+	dcs_classifier.h \
+	dcs_type.c \
+	fixup.remove \
+	fixup.reorder \
+	fixup.out \
+	dcs.db
\ No newline at end of file

src/gen_dcs_classifier.pl

@@ -67,13 +67,15 @@ use constant {
     F_MIN		=> 1,
     F_CLASSIFIER_C	=> 1,
     F_CLASSIFIER_H	=> 2,
-    F_CHECKSUM		=> 3,
-    F_MAX		=> 3
+    F_TYPE_C		=> 3,
+    F_CHECKSUM		=> 4,
+    F_MAX		=> 4
 };
 
 my @filenames;
 $filenames[F_CLASSIFIER_C] = 'dcs_classifier.c';
 $filenames[F_CLASSIFIER_H] = 'dcs_classifier.h';
+$filenames[F_TYPE_C] = 'dcs_type.c';
 $filenames[F_CHECKSUM] = 'dcs_classifier.checksum';
 
 {
@@ -464,6 +466,63 @@ sub type_enum($) {
     \$e;
 }
 
+# the format of the classes.conf is the same as that supported by
+# Config::IniFiles, but we cannot use this module because it does not
+# support empty values
+
+my %class2enum;
+my %enum2class;
+my %type2classenum;
+my $default_class;
+
+sub load_classes($) {
+    my ($fname) = @_;
+    my %classes;
+
+    die 'classes conf '.$fname.': '.$!
+	unless(-f $fname);
+
+    my $fh;
+    open($fh, '<', $fname) || die 'cant open '.$fname.' '.$!;
+
+    my $enum;
+    while (<$fh>) {
+	chomp;
+	next if (/^#|^\s*$/);
+	if (/^\s*\[([^\]]+)\]/) {
+	    my $class = $1;
+
+	    $default_class = $class
+		unless(defined($default_class));
+
+	    die $fname.': class '.$class.' used more than once'
+		if (exists($class2enum{$class}));
+
+	    my $e = enum_name('T_CLASS_', $class);
+	    $enum = \$e;
+
+	    die $fname.': class '.$class.' produces a C symbol name clash with '.$enum2class{$e}.
+		' - please rename either'
+		if (exists($enum2class{$e}));
+
+	    $enum2class{$e} = $class;
+	    $class2enum{$class} = $enum;
+	} else {
+	    s/\s+$//;
+
+	    if (exists($type2classenum{$_})) {
+		die $fname.': type '.$_.' already associated with class '.$enum2class{${$type2classenum{$_}}};
+	    }
+
+	    $type2classenum{$_} = $enum;
+	}
+    }
+
+    close($fh);
+
+    1;
+}
+
 sub load_remove($) {
     my ($fname) = @_;
     my @remove;
@@ -538,8 +597,9 @@ sub load_reorder($) {
 }
 
 my ($dbref, $dbchecksum) = load_classifier_db($ARGV[0], $ARGV[1]);
-my $removeref = load_remove($ARGV[2]);
-my $reorderref = load_reorder($ARGV[3]);
+load_classes($ARGV[2]);
+my $removeref = load_remove($ARGV[3]);
+my $reorderref = load_reorder($ARGV[4]);
 
 my @entries;
 use constant {
@@ -792,6 +852,7 @@ sub process_entry($) {
 }
 
 map { fixup_entry($_); } @entries;
+
 @entries = sort { $a->[ENTRY_ORDER] <=> $b->[ENTRY_ORDER] } 
            grep { $_->[ENTRY_ACTIVE] } @entries;
 
@@ -820,6 +881,7 @@ _VCL (F_CHECKSUM, VCL_TOP, $dbchecksum."\n");
 
 boilerplate_autogen_c(F_CLASSIFIER_C, $dbboilerplate);
 boilerplate_autogen_c(F_CLASSIFIER_H, undef);
+boilerplate_autogen_c(F_TYPE_C, undef);
 
 _VCL (F_CLASSIFIER_H, VCL_TOP, <<EOF);
 #ifndef DCS_CLASSIFIER_H
@@ -1275,4 +1337,71 @@ dcs_register_subkey_match(struct dcs_matchstate *state, dcs_subkey_id_t subkey_i
 
 EOF
 
+_VCL (F_TYPE_C, VCL_TOP, <<'EOF');
+#include "dcs_config.h"
+
+#include <errno.h>
+
+#include "dcs_classifier.h"
+#include "dcs_type.h"
+#define check_type_id(type_id, ret)						\
+	    do {								\
+		    if ((type_id < 0) || (type_id > (DCS_TYPE_COUNT - 1))) {	\
+			    errno = EINVAL;					\
+			    return (ret);					\
+		    }								\
+	    } while(0)
+
+const char *
+dcs_type_name(int type_id /* enum dcs_type */) {
+	check_type_id(type_id, 0);
+	return dcs_type[type_id];
+}
+
+EOF
+
+my %types_unused = %type2classenum;
+
+_VCL (F_TYPE_C, VCL_TOP,
+      "enum dcs_type_class {\n\t".
+      join(",\n\t", 
+	   '_T_CLASS_MISSING = 0',
+	   sort keys %enum2class,
+	   '_T_CLASS_LIMIT').
+      "\n};\n\n".
+      "const char * const dcs_type_class_str[_T_CLASS_LIMIT] = {\n\t".
+      join(",\n\t", 
+	   '[_T_CLASS_MISSING] = "'.$default_class.'"',
+	   map { '['.$_.'] = "'.$enum2class{$_}.'"' } sort keys %enum2class).
+      "\n};\n\n".
+      "enum dcs_type dcs_type2class[DCS_TYPE_COUNT] = {\n\t".
+      join(",\n\t", 
+	   map { 
+	       my $class_enum = $type2classenum{$_};
+
+	       if (defined($class_enum)) {
+		   delete $types_unused{$_};
+	       } else {
+		   warn 'No class assignment for type '.$_.' - mapping to default '.$default_class;
+		   $class_enum = \"_T_CLASS_MISSING";
+	       }
+	       # 4emacs "}
+
+	       '['.${type_enum($_)}.'] = '.$$class_enum
+	   } sort keys %nbtype2typeenum).
+      "\n};\n\n");
+
+_VCL (F_TYPE_C, VCL_TOP, <<EOF);
+const char *
+dcs_type_class(int type_id /* enum dcs_type */) {
+	check_type_id(type_id, 0);
+	return (dcs_type_class_str[dcs_type2class[type_id]]);
+}
+EOF
+
+if (scalar(%types_unused)) {
+    warn 'Unused type(s) in types.conf: '.join(", ", sort keys %types_unused);
+}
+
+
 VCL_close();