Complete the VSC filtering, and make everything compile and pass tests.

Still not done, in particular: Do not roll any releases until
libvarnishapi symbo/version stuff has been polished.

Fixes	#829

bin/varnishadm/varnishadm.c

@@ -282,25 +282,27 @@ n_arg_sock(const char *n_arg)
 	struct VSM_data *vsd;
 	char *p;
 	int sock;
+	struct VSM_fantom vt;
 
 	vsd = VSM_New();
 	assert(VSL_Arg(vsd, 'n', n_arg));
-	if (VSM_Open(vsd, 1)) {
-		fprintf(stderr, "Could not open shared memory\n");
+	if (VSM_Open(vsd)) {
+		fprintf(stderr, "%s\n", VSM_Error(vsd));
 		return (-1);
 	}
 	if (T_arg == NULL) {
-		p = VSM_Find_Chunk(vsd, "Arg", "-T", "", NULL);
-		if (p == NULL)  {
+		if (VSM_Get(vsd, &vt, "Arg", "-T", "")) {
 			fprintf(stderr, "No -T arg in shared memory\n");
 			return (-1);
 		}
-		T_start = T_arg = strdup(p);
+		T_start = T_arg = strdup(vt.b);
 	}
 	if (S_arg == NULL) {
-		p = VSM_Find_Chunk(vsd, "Arg", "-S", "", NULL);
-		if (p != NULL)
-			S_arg = strdup(p);
+		if (VSM_Get(vsd, &vt, "Arg", "-S", "")) {
+			fprintf(stderr, "No -S arg in shared memory\n");
+			return (-1);
+		}
+		S_arg = strdup(vt.b);
 	}
 	sock = -1;
 	while (*T_arg) {

bin/varnishhist/varnishhist.c

@@ -351,7 +351,6 @@ main(int argc, char **argv)
 	match_tag = -1;
 
 	vd = VSM_New();
-	VSL_Setup(vd);
 
 	while ((o = getopt(argc, argv, VSL_ARGS "Vw:r:R:f:p:")) != -1) {
 		switch (o) {
@@ -424,8 +423,10 @@ main(int argc, char **argv)
 	}
 	strcpy(format + 4*(fnum-1), "%lf");
 
-	if (VSL_Open(vd, 1))
+	if (VSM_Open(vd)) {
+		fprintf(stderr, "%s\n", VSM_Error(vd));
 		exit(1);
+	}
 
 	log_ten = log(10.0);
 

bin/varnishlog/varnishlog.c

@@ -60,7 +60,7 @@ static uint64_t       bitmap[65536];
 #define F_INVCL		(1 << 0)
 
 static void
-h_order_finish(int fd, const struct VSM_data *vd)
+h_order_finish(int fd, struct VSM_data *vd)
 {
 
 	AZ(VSB_finish(ob[fd]));
@@ -72,7 +72,7 @@ h_order_finish(int fd, const struct VSM_data *vd)
 }
 
 static void
-clean_order(const struct VSM_data *vd)
+clean_order(struct VSM_data *vd)
 {
 	unsigned u;
 
@@ -234,7 +234,7 @@ open_log(const char *w_arg, int a_flag)
 }
 
 static void
-do_write(const struct VSM_data *vd, const char *w_arg, int a_flag)
+do_write(struct VSM_data *vd, const char *w_arg, int a_flag)
 {
 	int fd, i, l;
 	uint32_t *p;
@@ -243,7 +243,7 @@ do_write(const struct VSM_data *vd, const char *w_arg, int a_flag)
 	XXXAN(fd >= 0);
 	(void)signal(SIGHUP, sighup);
 	while (1) {
-		i = VSL_NextLog(vd, &p, NULL);
+		i = VSL_NextSLT(vd, &p, NULL);
 		if (i < 0)
 			break;
 		if (i > 0) {
@@ -285,7 +285,6 @@ main(int argc, char * const *argv)
 	struct VSM_data *vd;
 
 	vd = VSM_New();
-	VSL_Setup(vd);
 
 	while ((c = getopt(argc, argv, VSL_ARGS "aDP:uVw:oO")) != -1) {
 		switch (c) {
@@ -336,8 +335,10 @@ main(int argc, char * const *argv)
 	if ((argc - optind) > 0)
 		usage();
 
-	if (VSL_Open(vd, 1))
+	if (VSM_Open(vd)) {
+		fprintf(stderr, "%s\n", VSM_Error(vd));
 		exit(1);
+	}
 
 	if (P_arg && (pfh = VPF_Open(P_arg, 0644, NULL)) == NULL) {
 		perror(P_arg);

bin/varnishncsa/varnishncsa.c

@@ -846,7 +846,6 @@ main(int argc, char *argv[])
 	format = "%h %l %u %t \"%r\" %s %b \"%{Referer}i\" \"%{User-agent}i\"";
 
 	vd = VSM_New();
-	VSL_Setup(vd);
 
 	while ((c = getopt(argc, argv, VSL_ARGS "aDP:Vw:fF:")) != -1) {
 		switch (c) {
@@ -907,8 +906,10 @@ main(int argc, char *argv[])
 
 	VSL_Arg(vd, 'c', optarg);
 
-	if (VSL_Open(vd, 1))
-		exit(1);
+	if (VSM_Open(vd)) {
+		fprintf(stderr, "%s\n", VSM_Error(vd));
+		return (-1);
+	}
 
 	if (P_arg && (pfh = VPF_Open(P_arg, 0644, NULL)) == NULL) {
 		perror(P_arg);

bin/varnishreplay/varnishreplay.c

@@ -707,7 +707,6 @@ main(int argc, char *argv[])
 	const char *address = NULL;
 
 	vd = VSM_New();
-	VSL_Setup(vd);
 	debug = 0;
 
 	VSL_Arg(vd, 'c', NULL);
@@ -730,8 +729,10 @@ main(int argc, char *argv[])
 		usage();
 	}
 
-	if (VSL_Open(vd, 1))
+	if (VSM_Open(vd)) {
+		fprintf(stderr, "%s\n", VSM_Error(vd));
 		exit(1);
+	}
 
 	addr_info = init_connection(address);
 

bin/varnishtest/flint.lnt

@@ -25,3 +25,5 @@
 -e713	// Loss of precision (assignment) (unsigned long long to long long)
 -e574	// Signed-unsigned mix with relational
 -e835	//  A zero has been given as ___ argument to operator '___'  (<<)
+-e786	// String concatenation within initializer
+

bin/varnishtest/vtc_varnish.c

@@ -198,11 +198,7 @@ varnishlog_thread(void *priv)
 
 	CAST_OBJ_NOTNULL(v, priv, VARNISH_MAGIC);
 	vsl = VSM_New();
-	VSL_Setup(vsl);
 	(void)VSL_Arg(vsl, 'n', v->workdir);
-	while (v->pid  && VSL_Open(vsl, 0) != 0) {
-		assert(usleep(VSL_SLEEP_USEC) == 0 || errno == EINTR);
-	}
 	while (v->pid) {
 		if (VSL_Dispatch(vsl, h_addlog, v) <= 0)
 			break;
@@ -332,7 +328,6 @@ varnish_launch(struct varnish *v)
 	char *r;
 
 	v->vd = VSM_New();
-	VSC_Setup(v->vd);
 
 	/* Create listener socket */
 	nap = VSS_resolve("127.0.0.1", "0", &ap);
@@ -457,7 +452,7 @@ varnish_launch(struct varnish *v)
 	free(r);
 
 	(void)VSL_Arg(v->vd, 'n', v->workdir);
-	AZ(VSC_Open(v->vd, 1));
+	AZ(VSM_Open(v->vd));
 }
 
 /**********************************************************************
@@ -696,6 +691,8 @@ do_stat_cb(void *priv, const struct VSC_point * const pt)
 	const char *p = sp->target;
 	int i;
 
+	if (pt == NULL)
+		return(0);
 	if (strcmp(pt->class, "")) {
 		i = strlen(pt->class);
 		if (memcmp(pt->class, p, i))
@@ -714,10 +711,10 @@ do_stat_cb(void *priv, const struct VSC_point * const pt)
 			return (0);
 		p++;
 	}
-	if (strcmp(pt->name, p))
+	if (strcmp(pt->desc->name, p))
 		return (0);
 
-	assert(!strcmp(pt->fmt, "uint64_t"));
+	assert(!strcmp(pt->desc->fmt, "uint64_t"));
 	sp->val = *(const volatile uint64_t*)pt->ptr;
 	return (1);
 }
@@ -740,12 +737,12 @@ varnish_expect(const struct varnish *v, char * const *av) {
 		good = VSC_Iter(v->vd, do_stat_cb, &sp);
 		if (good < 0) {
 			VSM_Close(v->vd);
-			j = VSM_Open(v->vd, 0);
+			j = VSM_Open(v->vd);
 			if (j == 0)
 				continue;
 			do {
 				(void)usleep(100000);
-				j = VSM_Open(v->vd, 0);
+				j = VSM_Open(v->vd);
 				i++;
 			} while(i < 10 && j < 0);
 			if (j < 0)

bin/varnishtop/varnishtop.c

@@ -195,7 +195,7 @@ accumulate_thread(void *arg)
 
 	for (;;) {
 
-		i = VSL_NextLog(vd, &p, NULL);
+		i = VSL_NextSLT(vd, &p, NULL);
 		if (i < 0)
 			break;
 		if (i == 0) {
@@ -292,7 +292,7 @@ do_once(struct VSM_data *vd)
 {
 	uint32_t *p;
 
-	while (VSL_NextLog(vd, &p, NULL) > 0)
+	while (VSL_NextSLT(vd, &p, NULL) > 0)
 		accumulate(p);
 	dump();
 }
@@ -313,7 +313,6 @@ main(int argc, char **argv)
 	float period = 60; /* seconds */
 
 	vd = VSM_New();
-	VSL_Setup(vd);
 
 	while ((o = getopt(argc, argv, VSL_ARGS "1fVp:")) != -1) {
 		switch (o) {
@@ -345,11 +344,12 @@ main(int argc, char **argv)
 		}
 	}
 
-	if (VSL_Open(vd, 1))
+	if (VSM_Open(vd)) {
+		fprintf(stderr, "%s\n", VSM_Error(vd));
 		exit (1);
+	}
 
 	if (once) {
-		VSL_NonBlocking(vd, 1);
 		do_once(vd);
 	} else {
 		do_curses(vd, period);

include/vapi/vsc.h

@@ -88,13 +88,13 @@ int VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv);
 	 * Iterate over all statistics counters, calling "func" for
 	 * each counter not suppressed by any "-f" arguments.
 	 *
- 	 * Func is called with pt == NULL, whenever VSM allocations
+	 * Func is called with pt == NULL, whenever VSM allocations
 	 * change (child restart, allocations/deallocations)
 	 *
 	 * Returns:
-	 * 	!=0:	func returned non-zero
+	 *	!=0:	func returned non-zero
 	 *	-1:	No VSC's available
-	 *	0: 	Done
+	 *	0:	Done
 	 */
 
 /**********************************************************************

include/vapi/vsl.h

@@ -36,7 +36,7 @@
  *	VSL_Arg(vd, "r", "/some/file");
  * and once VSL_Dispatch()/VSL_NextSLT() will indicate EOF by returning -2.
  * Another file can then be opened with VSL_Arg() and processed.
- * 
+ *
  */
 
 #ifndef VAPI_VSL_H_INCLUDED

lib/libvarnishapi/libvarnishapi.map

@@ -75,3 +75,12 @@ LIBVARNISHAPI_1.1 {
 	VSL_Name2Tag;
 	# Variables:
 } LIBVARNISHAPI_1.0;
+
+LIBVARNISHAPI_1.2 {
+  global:
+	# Functions:
+	VSL_NextSLT;
+	VSM_Error;
+	VSM_Get;
+	# Variables:
+} LIBVARNISHAPI_1.0;

lib/libvarnishapi/vsc.c

@@ -50,8 +50,9 @@
 struct vsc_pt {
 	unsigned		magic;
 #define VSC_PT_MAGIC		0xa4ff159a
-	struct VSC_point	point;
 	VTAILQ_ENTRY(vsc_pt)	list;
+	struct VSM_fantom	vf;
+	struct VSC_point	point;
 };
 
 struct vsc_sf {
@@ -101,15 +102,10 @@ static void
 vsc_delete_pts(struct vsc *vsc)
 {
 	struct vsc_pt *pt;
-	struct VSM_fantom *vf = NULL;
 
 	while(!VTAILQ_EMPTY(&vsc->pt_list)) {
 		pt = VTAILQ_FIRST(&vsc->pt_list);
 		VTAILQ_REMOVE(&vsc->pt_list, pt, list);
-		if (pt->point.vf != vf) {
-			vf = pt->point.vf;
-			free(vf);
-		}
 		FREE_OBJ(pt);
 	}
 }
@@ -243,129 +239,13 @@ VSC_Main(struct VSM_data *vd)
 	return ((void*)vsc->main_fantom.b);
 }
 
-#if 0
-/*--------------------------------------------------------------------
- * -1 -> unknown stats encountered.
- */
-
-static inline int
-iter_test(const char *s1, const char *s2, int wc)
-{
-
-	if (s1 == NULL)
-		return (0);
-	if (!wc)
-		return (strcmp(s1, s2));
-	for (; *s1 != '\0' && *s1 == *s2; s1++, s2++)
-		continue;
-	return (*s1 != '\0');
-}
-
-static int
-iter_call(const struct vsc *vsc, VSC_iter_f *func, void *priv,
-    const struct VSC_point *const sp)
-{
-	struct vsc_sf *sf;
-	struct vsc_pt *pt;
-	int good;
-
-	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
-
-	ALLOC_OBJ(pt, VSC_PT_MAGIC);
-	AN(pt);
-
-	if (VTAILQ_EMPTY(&vsc->sf_list)) {
-		VTAILQ_INSERT_TAIL(&vsc->pt_list, pt, list);
-		return (func(priv, sp));
-	}
-
-	good = 0;
-	VTAILQ_FOREACH(sf, &vsc->sf_list, list) {
-		if (iter_test(sf->class, sp->class, sf->flags & VSC_SF_CL_WC))
-			continue;
-		if (iter_test(sf->ident, sp->ident, sf->flags & VSC_SF_ID_WC))
-			continue;
-		if (iter_test(sf->name, sp->desc->name, sf->flags & VSC_SF_NM_WC))
-			continue;
-		if (sf->flags & VSC_SF_EXCL)
-			good = 0;
-		else
-			good = 1;
-	}
-	if (!good)
-		return (0);
-	return (func(priv, sp));
-}
-
-#define VSC_DO(U,l,t)							\
-	static void							\
-	iter_##l(const struct vsc *vsc, struct VSM_fantom *vf,		\
-	    const struct VSC_desc *descs)				\
-	{								\
-		struct VSC_C_##l *st;					\
-		struct VSM_fantom *vf2;					\
-		int i;							\
-									\
-		CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);			\
-		st = vf->b;						\
-		sp.class = t;						\
-		sp.ident = vf->chunk->ident;				\
-		sp.desc = descs++;					\
-		vf2 = malloc(sizeof *vf2);				\
-		AN(vf2);						\
-		memcpy(vf2, vf, sizeof *vf2);
-
-#define VSC_F(nn,tt,ll,ff,dd,ee)					\
-		sp.ptr = &st->nn;					\
-		sp.vf = vf2;						\
-		i = iter_call(vsc, &sp);				\
-		if (i)							\
-			return(i);
-
-#define VSC_DONE(U,l,t)							\
-		return (0);						\
-	}
-
-#include "tbl/vsc_all.h"
-#undef VSC_DO
-#undef VSC_F
-#undef VSC_DONE
-
-static void
-vsc_build_pt_list(struct VSM_data *vd)
-{
-	struct vsc *vsc = vsc_setup(vd);
-	struct VSM_fantom vf;
-
-	vsc_delete_pts(vsc *vsc);
-
-	VSM_FOREACH_SAFE(&vf, vd) {
-		if (strcmp(vf.chunk->class, VSC_CLASS))
-			continue;
-		/*lint -save -e525 -e539 */
-#define VSC_F(n,t,l,f,d,e)
-#define VSC_DONE(a,b,c)
-#define VSC_DO(U,l,t)						\
-		if (!strcmp(vf.chunk->type, t))			\
-			iter_##l(vsc, &vf, VSC_desc_##l);
-#include "tbl/vsc_all.h"
-#undef VSC_F
-#undef VSC_DO
-#undef VSC_DONE
-		/*lint -restore */
-		break;
-	}
-	return (i);
-}
-#endif
-
 /*--------------------------------------------------------------------
  */
 
 static void
 vsc_add_pt(struct vsc *vsc, const char *class, const char *ident,
     const struct VSC_desc *desc, const volatile void *ptr,
-    struct VSM_fantom *vf)
+    const struct VSM_fantom *vf)
 {
 	struct vsc_pt *pt;
 
@@ -375,7 +255,8 @@ vsc_add_pt(struct vsc *vsc, const char *class, const char *ident,
 	pt->point.ident = ident;
 	pt->point.desc = desc;
 	pt->point.ptr = ptr;
-	pt->point.vf = vf;
+	pt->point.vf = &pt->vf;
+	pt->vf = *vf;
 	VTAILQ_INSERT_TAIL(&vsc->pt_list, pt, list);
 }
 
@@ -385,18 +266,14 @@ vsc_add_pt(struct vsc *vsc, const char *class, const char *ident,
 	    const struct VSC_desc *descs)				\
 	{								\
 		struct VSC_C_##l *st;					\
-		struct VSM_fantom *vf2;					\
 		const char *class = t;					\
 									\
 		CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);			\
 		st = vf->b;						\
-		vf2 = malloc(sizeof *vf2);				\
-		AN(vf2);						\
-		memcpy(vf2, vf, sizeof *vf2);
 
 #define VSC_F(nn,tt,ll,ff,dd,ee)					\
 		vsc_add_pt(vsc, class, vf->chunk->ident, descs++,	\
-		    &st->nn, vf2);
+		    &st->nn, vf);
 
 #define VSC_DONE(U,l,t)							\
 	}
@@ -409,6 +286,8 @@ vsc_add_pt(struct vsc *vsc, const char *class, const char *ident,
 /*--------------------------------------------------------------------
  */
 
+#include <stdio.h>
+
 static void
 vsc_build_pt_list(struct VSM_data *vd)
 {
@@ -432,8 +311,57 @@ vsc_build_pt_list(struct VSM_data *vd)
 #undef VSC_DONE
 		/*lint -restore */
 	}
+}
 
-	/* XXX: filter pt list */
+/*--------------------------------------------------------------------
+ */
+
+static inline int
+iter_test(const char *s1, const char *s2, int wc)
+{
+
+	if (s1 == NULL)
+		return (0);
+	if (!wc)
+		return (strcmp(s1, s2));
+	for (; *s1 != '\0' && *s1 == *s2; s1++, s2++)
+		continue;
+	return (*s1 != '\0');
+}
+
+static void
+vsc_filter_pt_list(struct VSM_data *vd)
+{
+	struct vsc *vsc = vsc_setup(vd);
+	struct vsc_sf *sf;
+	struct vsc_pt *pt, *pt2;
+	VTAILQ_HEAD(, vsc_pt)	pt_list;
+
+	if (VTAILQ_EMPTY(&vsc->sf_list))
+		return;
+
+	VTAILQ_INIT(&pt_list);
+	VTAILQ_FOREACH(sf, &vsc->sf_list, list) {
+		VTAILQ_FOREACH_SAFE(pt, &vsc->pt_list, list, pt2) {
+			if (iter_test(sf->class, pt->point.class,
+			    sf->flags & VSC_SF_CL_WC))
+				continue;
+			if (iter_test(sf->ident, pt->point.ident,
+			    sf->flags & VSC_SF_ID_WC))
+				continue;
+			if (iter_test(sf->name, pt->point.desc->name,
+			    sf->flags & VSC_SF_NM_WC))
+				continue;
+			VTAILQ_REMOVE(&vsc->pt_list, pt, list);
+			if (sf->flags & VSC_SF_EXCL) {
+				FREE_OBJ(pt);
+			} else {
+				VTAILQ_INSERT_TAIL(&pt_list, pt, list);
+			}
+		}
+	}
+	vsc_delete_pts(vsc);
+	VTAILQ_CONCAT(&vsc->pt_list, &pt_list, list);
 }
 
 /*--------------------------------------------------------------------
@@ -456,8 +384,10 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 			}
 		}
 		AN(vd->head);
-		func(priv, NULL);
+		/* Tell app that list will be nuked */
+		(void)func(priv, NULL);
 		vsc_build_pt_list(vd);
+		vsc_filter_pt_list(vd);
 	}
 	AN(vd->head);
 	VTAILQ_FOREACH(pt, &vsc->pt_list, list) {

lib/libvarnishapi/vsl.c

@@ -296,7 +296,7 @@ VSL_NextSLT(struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 				if (vrm->tag == t) {
 					i = VRE_exec(vrm->re, VSL_DATA(p),
 					    VSL_LEN(p), 0, 0, NULL, 0, NULL);
-					if (i >= 0) 	/* XXX ?? */
+					if (i >= 0)	/* XXX ?? */
 						*bits |= (uintmax_t)1 << j;
 				}
 				j++;

lib/libvarnishapi/vsl_arg.c

@@ -160,10 +160,10 @@ vsl_ix_arg(struct VSM_data *vd, const char *opt, int arg)
 			else
 				vbit_clr(vsl->vbm_supress, i);
 		} else if (i == -2) {
-			return (vsm_diag(vd, 
+			return (vsm_diag(vd,
 			    "\"%*.*s\" matches multiple tags\n", l, l, b));
 		} else {
-			return (vsm_diag(vd, 
+			return (vsm_diag(vd,
 			    "Could not match \"%*.*s\" to any tag\n", l, l, b));
 		}
 	}
@@ -243,10 +243,10 @@ vsl_k_arg(struct VSM_data *vd, const char *opt)
 	char *end;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	if (*opt == '\0') 
+	if (*opt == '\0')
 		return (vsm_diag(vd, "number required for -k\n"));
 	vsl->keep = strtoul(opt, &end, 10);
-	if (*end != '\0') 
+	if (*end != '\0')
 		return (vsm_diag(vd, "invalid number for -k\n"));
 	return (1);
 }

lib/libvarnishapi/vsm.c

@@ -86,6 +86,7 @@ vsm_diag(struct VSM_data *vd, const char *fmt, ...)
 	if (vd->diag == NULL)
 		vd->diag = VSB_new_auto();
 	AN(vd->diag);
+	VSB_clear(vd->diag);
 	va_start(ap, fmt);
 	VSB_vprintf(vd->diag, fmt, ap);
 	va_end(ap);
@@ -328,6 +329,8 @@ VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf)
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	AN(vf);
+	if (!vd->head)
+		return (0);
 	if (!vd->head->alloc_seq)
 		return (0);
 	if (vf->priv == vd->head->alloc_seq)