Make VCC produce a "proper" symbol table for the compiled VCL.

bin/varnishd/cache/cache_vcl.c

@@ -422,7 +422,7 @@ VCL_Close(struct vcl **vclp)
 }
 
 /*--------------------------------------------------------------------
- * NB: This function is called from the test-load subprocess.
+ * NB: This function is called in/from the test-load subprocess.
  */
 
 int

bin/varnishd/mgt/mgt.h

@@ -48,6 +48,7 @@
 
 #include "common/common_param.h"
 
+struct vjsn;
 struct vsc_seg;
 struct vsmw_cluster;
 #include "VSC_mgt.h"
@@ -216,8 +217,7 @@ void mgt_vcl_startup(struct cli *, const char *vclsrc, const char *origin,
 int mgt_push_vcls(struct cli *, unsigned *status, char **p);
 void mgt_vcl_export_labels(struct vcc *);
 int mgt_has_vcl(void);
-void mgt_vcl_depends(struct vclprog *vp1, const char *name);
-void mgt_vcl_vmod(struct vclprog *, const char *src, const char *dst);
+void mgt_vcl_symtab(struct vclprog *, struct vjsn *);
 extern char *mgt_cc_cmd;
 extern const char *mgt_vcl_path;
 extern const char *mgt_vmod_path;

bin/varnishd/mgt/mgt_vcc.c

@@ -45,8 +45,8 @@
 #include "libvcc.h"
 #include "vcli_serve.h"
 #include "vfil.h"
+#include "vjsn.h"
 #include "vsub.h"
-#include "vav.h"
 #include "vtim.h"
 
 struct vcc_priv {
@@ -57,6 +57,7 @@ struct vcc_priv {
 	const char	*vclsrcfile;
 	char		*csrcfile;
 	char		*libfile;
+	char		*symfile;
 };
 
 char *mgt_cc_cmd;
@@ -69,6 +70,7 @@ unsigned mgt_vcc_unsafe_path;
 
 #define VGC_SRC		"vgc.c"
 #define VGC_LIB		"vgc.so"
+#define VGC_SYM		"vgc.sym"
 
 /*--------------------------------------------------------------------*/
 
@@ -107,7 +109,8 @@ run_vcc(void *priv)
 	STV_Foreach(stv)
 		VCC_Predef(vcc, "VCL_STEVEDORE", stv->ident);
 	mgt_vcl_export_labels(vcc);
-	i = VCC_Compile(vcc, &sb, vp->vclsrc, vp->vclsrcfile, VGC_SRC);
+	i = VCC_Compile(vcc, &sb, vp->vclsrc, vp->vclsrcfile,
+	    VGC_SRC, VGC_SYM);
 	if (VSB_len(sb))
 		printf("%s", VSB_data(sb));
 	VSB_destroy(&sb);
@@ -212,7 +215,9 @@ static unsigned
 mgt_vcc_compile(struct vcc_priv *vp, struct vsb *sb, int C_flag)
 {
 	char *csrc;
+	const char *err;
 	unsigned subs;
+	struct vjsn *vj;
 
 	if (mgt_vcc_touchfile(vp->csrcfile, sb))
 		return (2);
@@ -228,6 +233,18 @@ mgt_vcc_compile(struct vcc_priv *vp, struct vsb *sb, int C_flag)
 		AN(csrc);
 		VSB_cat(sb, csrc);
 		free(csrc);
+
+		VSB_printf(sb, "/* EXTERNAL SYMBOL TABLE\n");
+		csrc = VFIL_readfile(NULL, vp->symfile, NULL);
+		AN(csrc);
+		VSB_cat(sb, csrc);
+		vj = vjsn_parse(csrc, &err);
+		if (err != NULL)
+			VSB_printf(sb, "# Parse error: %s\n", err);
+		if (vj != NULL)
+			vjsn_delete(&vj);
+		VSB_printf(sb, "*/\n");
+		free(csrc);
 	}
 
 	subs = VSUB_run(sb, run_cc, vp, "C-compiler", 10);
@@ -246,11 +263,10 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname,
 {
 	struct vcc_priv vp;
 	struct vsb *sb;
+	struct vjsn *vj;
 	unsigned status;
-	char buf[1024];
-	FILE *fcs;
-	char **av;
-	int ac;
+	const char *err;
+	char *p;
 
 	AN(cli);
 
@@ -319,6 +335,12 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname,
 	AN(vp.csrcfile);
 	VSB_clear(sb);
 
+	VSB_printf(sb, "%s/%s", vp.dir, VGC_SYM);
+	AZ(VSB_finish(sb));
+	vp.symfile = strdup(VSB_data(sb));
+	AN(vp.symfile);
+	VSB_clear(sb);
+
 	status = mgt_vcc_compile(&vp, sb, C_flag);
 
 	AZ(VSB_finish(sb));
@@ -330,6 +352,7 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname,
 		if (!MGT_DO_DEBUG(DBG_VCL_KEEP)) {
 			(void)unlink(vp.csrcfile);
 			(void)unlink(vp.libfile);
+			(void)unlink(vp.symfile);
 			(void)rmdir(vp.dir);
 		}
 		free(vp.csrcfile);
@@ -342,26 +365,17 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname,
 		return (NULL);
 	}
 
-	fcs = fopen(vp.csrcfile, "r");
-	AN(fcs);
-	while (1) {
-		AN(fgets(buf, sizeof buf, fcs));
-		if (memcmp(buf, VCC_INFO_PREFIX, strlen(VCC_INFO_PREFIX)))
-			break;
-		av = VAV_Parse(buf, &ac, 0);
-		AN(av);
-		AZ(av[0]);
-		AZ(strcmp(av[1], "/*"));
-		AZ(strcmp(av[ac-1], "*/"));
-		if (!strcmp(av[3], "VCL"))
-			mgt_vcl_depends(vcl, av[4]);
-		else if (!strcmp(av[3], "VMOD"))
-			mgt_vcl_vmod(vcl, av[4], av[5]);
-		else
-			WRONG("Wrong VCCINFO");
-		VAV_Free(av);
-	}
-	AZ(fclose(fcs));
+	p = VFIL_readfile(NULL, vp.symfile, NULL);
+	AN(p);
+	vj = vjsn_parse(p, &err);
+	if (err != NULL)
+		fprintf(stderr, "FATAL: Symtab parse error: %s\n%s\n",
+		    err, p);
+	AZ(err);
+	AN(vj);
+	free(p);
+	mgt_vcl_symtab(vcl, vj);
+	(void)unlink(vp.symfile);
 
 	if (!MGT_DO_DEBUG(DBG_VCL_KEEP))
 		(void)unlink(vp.csrcfile);

bin/varnishd/mgt/mgt_vcl.c

@@ -45,6 +45,7 @@
 #include "vcli_serve.h"
 #include "vct.h"
 #include "vev.h"
+#include "vjsn.h"
 #include "vtim.h"
 
 #define VCL_STATE(sym, str)					\
@@ -84,6 +85,7 @@ struct vclprog {
 	unsigned		warm;
 	const char *		state;
 	double			go_cold;
+	struct vjsn		*symtab;
 	VTAILQ_HEAD(, vcldep)	dfrom;
 	VTAILQ_HEAD(, vcldep)	dto;
 	int			nto;
@@ -309,17 +311,32 @@ mgt_vcl_del(struct vclprog *vp)
 		}
 	}
 	free(vp->name);
+	if (vp->symtab)
+		vjsn_delete(&vp->symtab);
 	FREE_OBJ(vp);
 }
 
-void
-mgt_vcl_depends(struct vclprog *vp1, const char *name)
+static const char *
+mgt_vcl_symtab_val(const struct vjsn_val *vv, const char *val)
+{
+	const struct vjsn_val *jv;
+
+	jv = vjsn_child(vv, val);
+	AN(jv);
+	assert(jv->type == VJSN_STRING);
+	AN(jv->value);
+	return (jv->value);
+}
+
+static void
+mgt_vcl_import_vcl(struct vclprog *vp1, const struct vjsn_val *vv)
 {
 	struct vclprog *vp2;
 
 	CHECK_OBJ_NOTNULL(vp1, VCLPROG_MAGIC);
+	AN(vv);
 
-	vp2 = mcf_vcl_byname(name);
+	vp2 = mcf_vcl_byname(mgt_vcl_symtab_val(vv, "name"));
 	CHECK_OBJ_NOTNULL(vp2, VCLPROG_MAGIC);
 	mgt_vcl_dep_add(vp1, vp2);
 }
@@ -336,8 +353,8 @@ mgt_vcl_cache_vmod(const char *nm, const char *fm, const char *to)
 	if (fo < 0 && errno == EEXIST)
 		return (0);
 	if (fo < 0) {
-		fprintf(stderr, "Creating copy of vmod %s: %s\n",
-		    nm, vstrerror(errno));
+		fprintf(stderr, "While creating copy of vmod %s:\n\t%s: %s\n",
+		    nm, to, vstrerror(errno));
 		return (1);
 	}
 	fi = open(fm, O_RDONLY);
@@ -366,27 +383,32 @@ mgt_vcl_cache_vmod(const char *nm, const char *fm, const char *to)
 	return (ret);
 }
 
-void
-mgt_vcl_vmod(struct vclprog *vp, const char *src, const char *dst)
+static void
+mgt_vcl_import_vmod(struct vclprog *vp, const struct vjsn_val *vv)
 {
 	struct vmodfile *vf;
 	struct vmoddep *vd;
+	const char *v_name;
+	const char *v_file;
+	const char *v_dst;
 
 	CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC);
-	AN(src);
-	AN(dst);
-	assert(!strncmp(dst, "./vmod_cache/", 13));
+	AN(vv);
+
+	v_name = mgt_vcl_symtab_val(vv, "name");
+	v_file = mgt_vcl_symtab_val(vv, "file");
+	v_dst = mgt_vcl_symtab_val(vv, "dst");
 
 	VTAILQ_FOREACH(vf, &vmodhead, list)
-		if (!strcmp(vf->fname, dst))
+		if (!strcmp(vf->fname, v_dst))
 			break;
 	if (vf == NULL) {
 		ALLOC_OBJ(vf, VMODFILE_MAGIC);
 		AN(vf);
-		REPLACE(vf->fname, dst);
+		REPLACE(vf->fname, v_dst);
 		AN(vf->fname);
 		VTAILQ_INIT(&vf->vcls);
-		AZ(mgt_vcl_cache_vmod(vp->name, src, dst));
+		AZ(mgt_vcl_cache_vmod(v_name, v_file, v_dst));
 		VTAILQ_INSERT_TAIL(&vmodhead, vf, list);
 	}
 	ALLOC_OBJ(vd, VMODDEP_MAGIC);
@@ -396,6 +418,33 @@ mgt_vcl_vmod(struct vclprog *vp, const char *src, const char *dst)
 	VTAILQ_INSERT_TAIL(&vf->vcls, vd, lto);
 }
 
+void
+mgt_vcl_symtab(struct vclprog *vp, struct vjsn *vj)
+{
+	struct vjsn_val *v1, *v2;
+	const char *typ;
+
+	CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC);
+	vp->symtab = vj;
+	assert(vj->value->type == VJSN_ARRAY);
+	VTAILQ_FOREACH(v1, &vj->value->children, list) {
+		assert(v1->type == VJSN_OBJECT);
+		v2 = vjsn_child(v1, "dir");
+		if (v2 == NULL)
+			continue;
+		assert(v2->type == VJSN_STRING);
+		if (strcmp(v2->value, "import"))
+			continue;
+		typ = mgt_vcl_symtab_val(v1, "type");
+		if (!strcmp(typ, "$VMOD"))
+			mgt_vcl_import_vmod(vp, v1);
+		else if (!strcmp(typ, "$VCL"))
+			mgt_vcl_import_vcl(vp, v1);
+		else
+			WRONG("Bad symtab import entry");
+	}
+}
+
 int
 mgt_has_vcl(void)
 {

include/libvcc.h

@@ -30,8 +30,6 @@
 
 struct vcc;
 
-#define VCC_INFO_PREFIX	"/* VCC_INFO"
-
 struct vcc *VCC_New(void);
 void VCC_Allow_InlineC(struct vcc *, unsigned);
 void VCC_Builtin_VCL(struct vcc *, const char *);
@@ -43,4 +41,4 @@ void VCC_Predef(struct vcc *, const char *type, const char *name);
 void VCC_VCL_Range(unsigned *, unsigned *);
 
 int VCC_Compile(struct vcc *, struct vsb **,
-    const char *, const char *, const char *);
+    const char *, const char *, const char *, const char *);

lib/libvcc/vcc_action.c

@@ -273,8 +273,11 @@ vcc_act_return_vcl(struct vcc *tl)
 	ERRCHK(tl);
 	AN(sym);
 	if (sym->eval_priv == NULL) {
-		VSB_printf(tl->fi, "%s VCL %s */\n", VCC_INFO_PREFIX,
-		    sym->name);
+		VSB_printf(tl->symtab, ",\n    {\n");
+		VSB_printf(tl->symtab, "\t\"dir\": \"import\",\n");
+		VSB_printf(tl->symtab, "\t\"type\": \"$VCL\",\n");
+		VSB_printf(tl->symtab, "\t\"name\": \"%s\"\n", sym->name);
+		VSB_printf(tl->symtab, "   }");
 
 		bprintf(buf, "vgc_vcl_%u", tl->unique++);
 		sym->eval_priv = strdup(buf);

lib/libvcc/vcc_compile.c

@@ -94,6 +94,30 @@ TlDup(struct vcc *tl, const char *s)
 	return (p);
 }
 
+static int
+TLWriteVSB(struct vcc *tl, const char *fn, const struct vsb *vsb,
+    const char *what)
+{
+	int fo;
+	int i;
+
+	fo = open(fn, O_WRONLY|O_TRUNC|O_CREAT, 0600);
+	if (fo < 0) {
+		VSB_printf(tl->sb,
+		    "Could not open %s file %s: %s\n",
+		    what, fn, strerror(errno));
+		return (-1);
+	}
+	i = VSB_tofile(fo, vsb);
+	if (i) {
+		VSB_printf(tl->sb,
+		    "Could not write %s to %s: %s\n",
+		    what, fn, strerror(errno));
+	}
+	closefd(&fo);
+	return (i);
+}
+
 /*--------------------------------------------------------------------*/
 
 struct proc *
@@ -590,7 +614,7 @@ vcc_resolve_includes(struct vcc *tl)
  */
 
 static struct vsb *
-vcc_CompileSource(struct vcc *tl, struct source *sp)
+vcc_CompileSource(struct vcc *tl, struct source *sp, const char *jfile)
 {
 	struct proc *p;
 	struct vsb *vsb;
@@ -691,14 +715,16 @@ vcc_CompileSource(struct vcc *tl, struct source *sp)
 
 	VCC_XrefTable(tl);
 
+	VSB_printf(tl->symtab, "\n]\n");
+	AZ(VSB_finish(tl->symtab));
+	if (TLWriteVSB(tl, jfile, tl->symtab, "Symbol table"))
+		return (NULL);
+
 	/* Combine it all */
 
 	vsb = VSB_new_auto();
 	AN(vsb);
 
-	AZ(VSB_finish(tl->fi));
-	VSB_cat(vsb, VSB_data(tl->fi));
-
 	vcl_output_lang_h(vsb);
 
 	EmitCoordinates(tl, vsb);
@@ -734,38 +760,27 @@ VCC_VCL_Range(unsigned *lo, unsigned *hi)
 int
 VCC_Compile(struct vcc *tl, struct vsb **sb,
     const char *vclsrc, const char *vclsrcfile,
-    const char *ofile)
+    const char *ofile, const char *jfile)
 {
 	struct source *sp;
 	struct vsb *r = NULL;
-	int fo, retval = 0;
+	int retval = 0;
 
 	CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
 	AN(sb);
 	AN(vclsrcfile);
 	AN(ofile);
+	AN(jfile);
 	if (vclsrc != NULL)
 		sp = vcc_new_source(vclsrc, NULL, vclsrcfile);
 	else
 		sp = vcc_file_source(tl, vclsrcfile);
 
 	if (sp != NULL)
-		r = vcc_CompileSource(tl, sp);
+		r = vcc_CompileSource(tl, sp, jfile);
 
 	if (r != NULL) {
-		fo = open(ofile, O_WRONLY|O_TRUNC|O_CREAT, 0600);
-		if (fo < 0) {
-			VSB_printf(tl->sb,
-			    "Could not open C-source file %s: %s\n",
-			    ofile, strerror(errno));
-		} else {
-			if (VSB_tofile(fo, r)) {
-				VSB_printf(tl->sb,
-				    "Could not write C-source to %s: %s\n",
-				    ofile, strerror(errno));
-			}
-			closefd(&fo);
-		}
+		retval = TLWriteVSB(tl, ofile, r, "C-source");
 		VSB_destroy(&r);
 	} else {
 		retval = -1;
@@ -798,8 +813,9 @@ VCC_New(void)
 
 	tl->nsources = 0;
 
-	tl->fi = VSB_new_auto();
-	assert(tl->fi != NULL);
+	tl->symtab = VSB_new_auto();
+	assert(tl->symtab != NULL);
+	VSB_printf(tl->symtab, "[\n    {\"version\": 0}");
 
 	tl->fc = VSB_new_auto();
 	assert(tl->fc != NULL);

lib/libvcc/vcc_compile.h

@@ -237,7 +237,7 @@ struct vcc {
 	int			hindent;
 	unsigned		cnt;
 
-	struct vsb		*fi;		/* VCC info to MGT */
+	struct vsb		*symtab;	/* VCC info to MGT */
 	struct vsb		*fc;		/* C-code */
 	struct vsb		*fh;		/* H-code (before C-code) */
 	struct vsb		*fb;		/* Body of current sub

lib/libvcc/vcc_vmod.c

@@ -357,8 +357,15 @@ vcc_ParseImport(struct vcc *tl)
 	VSB_printf(ifp->ini, "\t    ))\n");
 	VSB_printf(ifp->ini, "\t\treturn(1);");
 
-	VSB_printf(tl->fi, "%s VMOD %s ./vmod_cache/_vmod_%.*s.%s */\n",
-	    VCC_INFO_PREFIX, fnpx, PF(mod), vmd->file_id);
+	VSB_printf(tl->symtab, ",\n    {\n");
+	VSB_printf(tl->symtab, "\t\"dir\": \"import\",\n");
+	VSB_printf(tl->symtab, "\t\"type\": \"$VMOD\",\n");
+	VSB_printf(tl->symtab, "\t\"name\": \"%.*s\",\n", PF(mod));
+	VSB_printf(tl->symtab, "\t\"file\": \"%s\",\n", fnpx);
+	VSB_printf(tl->symtab,
+	    "\t\"dst\": \"./vmod_cache/_vmod_%.*s.%s\"\n",
+	    PF(mod), vmd->file_id);
+	VSB_printf(tl->symtab, "    }");
 
 	/* XXX: zero the function pointer structure ?*/
 	VSB_printf(ifp->fin, "\t\tVRT_priv_fini(&vmod_priv_%.*s);", PF(mod));