First babystep of the new selfdocumenting VSC counters.

The .VSC file steals .RST's directive syntax, but it is not a RST
file, and it is not processed as such.  Maybe later it will become
a .RST file, but that is TBD.

A C-struct for use in varnishd is built, (along with a JSON structure
which describes the counters.)

For now that structure is identical with what comes out of
include/tbl/vsc_f_main, so nothing special happens in the VSM yet.

bin/varnishd/Makefile.am

@@ -107,12 +107,14 @@ varnishd_SOURCES = \
 	waiter/cache_waiter_epoll.c \
 	waiter/cache_waiter_kqueue.c \
 	waiter/cache_waiter_poll.c \
-	waiter/cache_waiter_ports.c
+	waiter/cache_waiter_ports.c \
+	VSC_main.c
 
 nodist_varnishd_SOURCES = \
 	builtin_vcl.c
 
 noinst_HEADERS = \
+	VSC_main.h \
 	cache/cache_ban.h \
 	cache/cache_esi.h \
 	cache/cache_obj.h \
@@ -215,3 +217,12 @@ DISTCLEANFILES = builtin_vcl.c
 BUILT_SOURCES	=	vhp_hufdec.h
 DISTCLEANFILES	+=	vhp_hufdec.h
 
+#######################################################################
+VSC_main.c VSC_main.h: $(srcdir)/main.vsc $(top_builddir)/lib/libvcc/vsctool.py
+	$(PYTHON) $(top_builddir)/lib/libvcc/vsctool.py $(srcdir)/main.vsc
+
+$(varnishd_OBJECTS):	VSC_main.h
+
+EXTRA_DIST	+=	main.vsc
+DISTCLEANFILES	+=	VSC_main.c VSC_main.h
+

bin/varnishd/cache/cache.h

@@ -977,7 +977,6 @@ void SES_Set_String_Attr(struct sess *sp, enum sess_attr a, const char *src);
 const char *SES_Get_String_Attr(const struct sess *sp, enum sess_attr a);
 
 /* cache_shmlog.c */
-extern struct VSC_C_main *VSC_C_main;
 void *VSM_Alloc(unsigned size, const char *class, const char *type,
     const char *ident);
 void VSM_Free(void *ptr);

bin/varnishd/cache/cache_priv.h

@@ -29,6 +29,8 @@
  * Stuff that should *never* be exposed to a VMOD
  */
 
+#include "VSC_main.h"
+
 /* Prototypes etc ----------------------------------------------------*/
 
 /* cache_acceptor.c */
@@ -103,6 +105,7 @@ void SES_NewPool(struct pool *, unsigned pool_no);
 void SES_DestroyPool(struct pool *);
 
 /* cache_shmlog.c */
+extern struct VSC_main *VSC_C_main;
 void VSM_Init(void);
 void VSL_Setup(struct vsl_log *vsl, void *ptr, size_t len);
 void VSL_ChgId(struct vsl_log *vsl, const char *typ, const char *why,

bin/varnishd/cache/cache_shmlog.c

@@ -51,8 +51,7 @@ static uint32_t			*vsl_ptr;
 static unsigned			vsl_segment_n;
 static ssize_t			vsl_segsize;
 
-struct VSC_C_main       *VSC_C_main;
-
+struct VSC_main *VSC_C_main;
 
 static void
 vsl_sanity(const struct vsl_log *vsl)
@@ -471,6 +470,28 @@ VSL_End(struct vsl_log *vsl)
 
 /*--------------------------------------------------------------------*/
 
+void *
+VSC_Alloc(const char *nm, size_t sj, const unsigned char *zj, size_t szj,
+    const char *fmt, va_list va)
+{
+	(void)nm;
+	(void)zj;
+	(void)szj;
+	(void)sj;
+	(void)fmt;
+	(void)va;
+	return (0);
+}
+
+void
+VSC_Destroy(const char *nm, void *p)
+{
+	(void)nm;
+	(void)p;
+}
+
+/*--------------------------------------------------------------------*/
+
 static void *
 vsm_cleaner(void *priv)
 {
@@ -478,7 +499,7 @@ vsm_cleaner(void *priv)
 	THR_SetName("vsm_cleaner");
 	while (1) {
 		AZ(pthread_mutex_lock(&vsm_mtx));
-		VSM_common_cleaner(heritage.vsm, VSC_C_main);
+		VSM_common_cleaner(heritage.vsm, (void*)VSC_C_main);
 		AZ(pthread_mutex_unlock(&vsm_mtx));
 		VTIM_sleep(1.1);
 	}

bin/varnishd/common/common.h

@@ -33,6 +33,7 @@
 #endif
 #define COMMON_COMMON_H
 
+#include <stdarg.h>
 #include <stdint.h>
 
 #include <sys/types.h>
@@ -90,5 +91,9 @@ void VSM_common_copy(struct vsm_sc *to, const struct vsm_sc *from);
 void VSM_common_cleaner(struct vsm_sc *sc, struct VSC_C_main *stats);
 void VSM_common_ageupdate(const struct vsm_sc *sc);
 
+void *VSC_Alloc(const char *, size_t, const unsigned char *, size_t,
+    const char *, va_list);
+void VSC_Destroy(const char *, void *);
+
 /* mgt_cli.c */
 extern struct VCLS	*mgt_cls;

lib/libvcc/Makefile.am

@@ -36,7 +36,8 @@ EXTRA_DIST = \
 	generate.py
 
 dist_pkgdata_SCRIPTS = \
-	vmodtool.py
+	vmodtool.py \
+	vsctool.py
 
 vcc_obj.c vcc_fixed_token.c vcc_token_defs.h: \
 	$(top_builddir)/include/vcl.h

lib/libvcc/vsctool.py

+#!/usr/bin/env python
+#
+# Copyright (c) 2017 Varnish Software AS
+# All rights reserved.
+#
+# Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+"""
+This program compiles a .vsc file to C language constructs.
+"""
+
+from __future__ import print_function
+
+import getopt
+import json
+import sys
+import gzip
+import StringIO
+import collections
+
+def gzip_str(s):
+	out = StringIO.StringIO()
+	gzip.GzipFile(fileobj=out, mode="w").write(s)
+	return out.getvalue()
+
+def genhdr(fo, name):
+	fo.write('/*\n')
+	fo.write(' * NB:  This file is machine generated, DO NOT EDIT!\n')
+	fo.write(' *\n')
+	fo.write(' * Edit %s.vsc run lib/libvcc/vsctool.py instead.\n' % name)
+ 	fo.write(' */\n')
+ 	fo.write('\n')
+
+#######################################################################
+
+class vscset(object):
+	def __init__(self, name, m):
+		self.name = name
+		self.struct = "struct VSC_" + name
+		self.mbrs = []
+		self.head = m
+		self.completed = False
+
+	def addmbr(self, m):
+		assert not self.completed
+		self.mbrs.append(m)
+
+	def complete(self):
+		self.completed = True
+
+	def emit_json(self, fo):
+		dd = collections.OrderedDict()
+		dd["version"] = "1"
+		dd["name"] = self.name
+		dd["1line"] = self.head.param["oneliner"].strip()
+		dd["docs"] = self.head.getdoc()
+		dd["elements"] = len(self.mbrs)
+		el = collections.OrderedDict()
+		dd["elem"] = el
+		en = 0
+		for i in self.mbrs:
+			en += 1
+			ed = collections.OrderedDict()
+			el[i.arg] = ed
+			ed["index"] = en
+			ed["name"] = i.arg
+			ed["type"] = i.param["type"]
+			ed["level"] = i.param["level"]
+			ed["1line"] = i.param["oneliner"].strip()
+			ed["docs"] = i.getdoc()
+		s=json.dumps(dd, separators=(",",":"))
+		fo.write("\nstatic const size_t %s_jsonlen = %dL;\n" %
+		    (self.name, len(s)))
+		z = gzip_str(s)
+		fo.write("\nstatic const unsigned char");
+		fo.write(" %s_zjson[%d] = {\n" % (self.name, len(z)))
+		bz = bytearray(z)
+		t = "\t"
+		for i in bz:
+			t += "%d," % i
+			if len(t) >= 70:
+				fo.write(t + "\n")
+				t = "\t"
+		if len(t) > 1:
+			fo.write(t[:-1])
+		fo.write("\n};\n")
+		s = json.dumps(dd, indent=2, separators=(',', ': '))
+		fo.write("\n// ")
+		fo.write("\n// ".join(s.split("\n")))
+		fo.write("\n")
+
+
+	def emit_h(self):
+		fon="VSC_" + self.name + ".h"
+		fo = open(fon, "w")
+		genhdr(fo, self.name)
+		fo.write(self.struct + " {\n")
+		for i in self.mbrs:
+			fo.write("\tuint64_t\t%s;\n" % i.arg)
+		fo.write("};\n")
+		fo.write("\n");
+		fo.write(self.struct + " *VSC_" + self.name + "_New")
+		fo.write("(const char *fmt, ...);\n");
+		fo.write("void VSCL_" + self.name + "_Destroy")
+		fo.write("(" + self.struct + "**);\n")
+
+	def emit_c(self):
+		fon="VSC_" + self.name + ".c"
+		fo = open(fon, "w")
+		genhdr(fo, self.name)
+		fo.write('#include "config.h"\n')
+		fo.write('#include <stdarg.h>\n')
+		fo.write('#include <stdio.h>\n')
+		fo.write('#include <stdint.h>\n')
+		fo.write('#include "common/common.h"\n')
+		fo.write('#include "VSC_%s.h"\n' % self.name)
+		self.emit_json(fo)
+		fo.write("\n")
+		fo.write(self.struct + "*\n");
+		fo.write("VSC_" + self.name + "_New")
+		fo.write("(const char *fmt, ...)\n");
+		fo.write("{\n")
+		fo.write("\tva_list ap;\n")
+		fo.write("\t" + self.struct + " *retval;\n")
+		fo.write("\n")
+		fo.write("\tva_start(ap, fmt);\n")
+		fo.write("\tretval = VSC_Alloc")
+		fo.write('("' + self.name + '", ')
+		fo.write(self.name + "_jsonlen, ")
+		fo.write(self.name + "_zjson, ")
+		fo.write("sizeof " + self.name + "_zjson,\n")
+		fo.write("\t    fmt, ap);\n")
+		fo.write("\tva_end(ap);\n")
+		fo.write("\treturn(retval);\n")
+		fo.write("}\n")
+		fo.write("\n")
+		fo.write("void\n")
+		fo.write("VSCL_" + self.name + "_Destroy")
+		fo.write("(" + self.struct + "**pp)\n")
+		fo.write("{\n")
+		fo.write("\n")
+		fo.write("\tAN(pp);\n")
+		fo.write('\tVSC_Destroy("%s", *pp);\n' % self.name)
+		fo.write("\t*pp = NULL;\n")
+		fo.write("}\n")
+
+#######################################################################
+
+class directive(object):
+	def __init__(self, s):
+		ll = s.split("\n")
+		i = ll.pop(0).split("::", 2)
+		self.cmd = i[0]
+		self.arg = i[1].strip()
+		assert len(self.arg.split()) == 1
+
+		self.param = {}
+		while len(ll):
+			j = ll[0].split(":",2)
+			if len(j) != 3 or not j[0].isspace():
+				break
+			self.param[j[1]] = j[2].strip()
+			ll.pop(0)
+		self.ldoc = ll
+
+	def getdoc(self):
+		while len(self.ldoc) and self.ldoc[0].strip() == "":
+			self.ldoc.pop(0)
+		while len(self.ldoc) and self.ldoc[-1].strip() == "":
+			self.ldoc.pop(-1)
+		return self.ldoc
+
+	def moredoc(self, s):
+		self.getdoc()
+		self.ldoc += s.split("\n")
+
+	def emit_rst(self, fo):
+		fo.write("\n.. " + self.cmd + ":: " + self.arg + "\n")
+		self.emit_rst_doc(fo)
+
+	def emit_rst_doc(self, fo):
+		fo.write("\n".join(self.ldoc))
+
+	def emit_h(self, fo):
+		return
+
+class rst_vsc_begin(directive):
+	def __init__(self, s):
+		super(rst_vsc_begin, self).__init__(s)
+
+	def vscset(self, ss):
+		ss.append(vscset(self.arg, self))
+
+class rst_vsc(directive):
+	def __init__(self, s):
+		super(rst_vsc, self).__init__(s)
+		if "type" not in self.param:
+			self.param["type"] = "counter"
+		if "level" not in self.param:
+			self.param["level"] = "info"
+
+	def emit_rst(self, fo):
+		fo.write("\n``%s`` - " % self.arg)
+		fo.write("`%s` - " % self.param["type"])
+		fo.write("%s\n" % self.param["level"])
+		self.emit_rst_doc(fo)
+
+	def vscset(self, ss):
+		ss[-1].addmbr(self)
+
+
+class rst_vsc_end(directive):
+	def __init__(self, s):
+		super(rst_vsc_end, self).__init__(s)
+
+	def vscset(self, ss):
+		ss[-1].complete()
+
+class other(object):
+	def __init__(self, s):
+		self.s = s
+
+	def emit_rst(self, fo):
+		fo.write(self.s)
+
+	def emit_h(self, fo):
+		return
+
+	def vscset(self, ss):
+		return
+
+#######################################################################
+
+class vsc_file(object):
+	def __init__(self, fin):
+		self.c = []
+		scs = open(fin).read().split("\n.. ")
+		self.c.append(other(scs[0]))
+		ld = None
+		for i in scs[1:]:
+			j = i.split(None, 1)
+			f = {
+				"varnish_vsc_begin::":	rst_vsc_begin,
+				"varnish_vsc::":	rst_vsc,
+				"varnish_vsc_end::":	rst_vsc_end,
+			}.get(j[0])
+			if f is None:
+				s = "\n.. " + i
+				o = other(s)
+				if ld is not None:
+					ld.moredoc(s)
+			else:
+				o = f(i)
+				ld = o
+			self.c.append(o)
+
+		self.vscset = []
+		for i in self.c:
+			i.vscset(self.vscset)
+
+		print(self.vscset)
+
+	def emit_h(self):
+		for i in self.vscset:
+			i.emit_h()
+
+	def emit_c(self):
+		for i in self.vscset:
+			i.emit_c()
+
+	def emit_rst(self, fon):
+		fo = open(fon, "w")
+		for i in self.c:
+			i.emit_rst(fo)
+
+#######################################################################
+
+if __name__ == "__main__":
+
+	optlist, args = getopt.getopt(sys.argv[1:], "")
+
+	fo = sys.stdout
+
+	for f, v in optlist:
+		assert False
+	if len(args) != 1:
+		print("Need exactly one filename argument")
+		exit(2)
+
+	vf = vsc_file(args[0])
+	vf.emit_rst("_.rst")
+	vf.emit_h()
+	vf.emit_c()