Redo our management of compiled VCL programs:

Take default_vcl out of heritage.

Keep track of all compiled VCL files and delete them at
exit.

After starting child, use CLI to load all vcl programs
and then issue "start" via the CLI.

In the cacher, don't start the acceptor until we get
a start command.



git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@637 d4fa192b-c00b-0410-8231-f00ffab90ce4

bin/varnishd/Makefile.am

@@ -12,6 +12,7 @@ varnishd_SOURCES = \
 	hash_slinger.h \
 	heritage.h \
 	mgt.h \
+	mgt_cli.h \
 	stevedore.h \
 	\
 	cache_acceptor.c \

bin/varnishd/cache_cli.c

@@ -19,10 +19,25 @@
 #include "sbuf.h"
 #include "heritage.h"
 
+/*--------------------------------------------------------------------*/
+
+static void
+cli_func_start(struct cli *cli, char **av, void *priv)
+{
+
+	(void)cli;
+	(void)av;
+	(void)priv;
+	VCA_Init();
+	return;
+}
+
+
 /*--------------------------------------------------------------------*/
 
 struct cli_proto CLI_cmds[] = {
 	{ CLI_PING,		cli_func_ping },
+	{ CLI_SERVER_START,	cli_func_start },
 #if 0
 	{ CLI_URL_QUERY,	cli_func_url_query },
 #endif

bin/varnishd/cache_main.c

@@ -27,7 +27,6 @@ child_main(void)
 	printf("Child starts\n");
 
 	VCL_Init();
-	VCL_Load(heritage.vcl_file, "boot", NULL);
 
 	HTTP_Init();
 	SES_Init();
@@ -36,7 +35,6 @@ child_main(void)
 	VSL_Init();
 	WRK_Init();
 
-	VCA_Init();
 	EXP_Init();
 	HSH_Init();
 	BAN_Init();

bin/varnishd/cache_vcl.c

@@ -96,12 +96,6 @@ VCL_Load(const char *fn, const char *name, struct cli *cli)
 
 	vcl->dlh = dlopen(fn, RTLD_NOW | RTLD_LOCAL);
 
-	/*
-	 * Delete the file, either we got hold of it, or we couldn't care
-	 * less about it anyway.
-	 */
-	(void)unlink(fn);
-
 	if (vcl->dlh == NULL) {
 		if (cli == NULL)
 			fprintf(stderr, "dlopen(%s): %s\n", fn, dlerror());

bin/varnishd/heritage.h

@@ -25,9 +25,6 @@ struct heritage {
 	int			vsl_fd;
 	unsigned		vsl_size;
 
-	/* Initial VCL file */
-	char			*vcl_file;
-
 	/* Storage method */
 	struct stevedore	*stevedore;
 

bin/varnishd/mgt.h

@@ -8,19 +8,20 @@
 void mgt_run(int dflag);
 void mgt_start_child(void);
 void mgt_stop_child(void);
+extern pid_t mgt_pid;
 
 /* mgt_cli.c */
 
 void mgt_cli_init(void);
 void mgt_cli_setup(int fdi, int fdo, int verbose);
+int mgt_cli_askchild(int *status, char **resp, const char *fmt, ...);
 void mgt_cli_start_child(int fdi, int fdo);
 void mgt_cli_stop_child(void);
-int mgt_cli_askchild(int *status, char **resp, const char *fmt, ...);
 
 /* mgt_vcc.c */
 void mgt_vcc_init(void);
-char *mgt_vcc_default(const char *bflag);
-char *mgt_vcc_file(const char *fflag);
+int mgt_vcc_default(const char *bflag, const char *fflag);
+int mgt_push_vcls_and_start(int *status, char **p);
 
 /* tcp.c */
 int open_tcp(const char *port);

bin/varnishd/mgt_child.c

@@ -21,9 +21,12 @@
 #include "libvarnish.h"
 #include "heritage.h"
 #include "mgt.h"
+#include "cli_priv.h"
+#include "mgt_cli.h"
+
+pid_t		mgt_pid;
 
 static pid_t		child_pid = -1;
-static pid_t		mgr_pid;
 static int		child_fds[2];
 static unsigned 	child_should_run;
 static pthread_t	child_listen_thread;
@@ -74,6 +77,7 @@ static void
 start_child(void)
 {
 	int i;
+	char *p;
 
 	AZ(pipe(&heritage.fds[0]));
 	AZ(pipe(&heritage.fds[2]));
@@ -115,6 +119,12 @@ start_child(void)
 	AZ(pthread_detach(child_listen_thread));
 	AZ(pthread_create(&child_poker_thread, NULL, child_poker, NULL));
 	AZ(pthread_detach(child_poker_thread));
+	if (mgt_push_vcls_and_start(&i, &p)) {
+		fprintf(stderr, "Pushing vcls failed:\n%s\n", p);
+		free(p);
+		exit (2);
+	}
+	child_ticker = 0;
 }
 
 /*--------------------------------------------------------------------*/
@@ -190,7 +200,7 @@ mgt_sigint(int arg)
 {
 
 	(void)arg;
-	if (getpid() != mgr_pid) {
+	if (getpid() != mgt_pid) {
 		printf("Got SIGINT\n");
 		exit (2);
 	}
@@ -212,7 +222,7 @@ mgt_run(int dflag)
 	struct sigaction sac;
 	int i;
 
-	mgr_pid = getpid();
+	mgt_pid = getpid();
 
 	if (dflag)
 		mgt_cli_setup(0, 1, 1);
@@ -275,18 +285,17 @@ mgt_run(int dflag)
 /*--------------------------------------------------------------------*/
 
 void
-mgt_start_child(void)
+mcf_server_startstop(struct cli *cli, char **av, void *priv)
 {
 
+	(void)cli;
+	(void)av;
+	if (priv != NULL) {
+		child_should_run = 0;
+		AZ(pthread_cond_signal(&child_cv));
+		return;
+	} 
 	dstarts = 0;
 	child_should_run = 1;
 	AZ(pthread_cond_signal(&child_cv));
 }
-
-void
-mgt_stop_child(void)
-{
-
-	child_should_run = 0;
-	AZ(pthread_cond_signal(&child_cv));
-}

bin/varnishd/mgt_cli.c

@@ -19,6 +19,7 @@
 #include "sbuf.h"
 #include "common_cli.h"
 #include "mgt.h"
+#include "mgt_cli.h"
 #include "shmlog.h"
 
 static int		cli_i = -1, cli_o = -1;
@@ -26,20 +27,6 @@ static pthread_mutex_t	cli_mtx;
 
 /*--------------------------------------------------------------------*/
 
-static void
-mcf_server_startstop(struct cli *cli, char **av, void *priv)
-{
-
-	(void)cli;
-	(void)av;
-	if (priv != NULL)
-		mgt_stop_child();
-	else
-		mgt_start_child();
-}
-
-/*--------------------------------------------------------------------*/
-
 static void
 mcf_stats(struct cli *cli, char **av, void *priv)
 {
@@ -122,10 +109,9 @@ static struct cli_proto mgt_cli_proto[] = {
 	{ CLI_SERVER_START,	mcf_server_startstop, NULL },
 	{ CLI_SERVER_STOP,	mcf_server_startstop, &cli_proto },
 	{ CLI_STATS,		mcf_stats, NULL },
-	{ CLI_CONFIG_LOAD },
+	{ CLI_CONFIG_LOAD,	mcf_config_load, NULL },
+	{ CLI_CONFIG_INLINE,	mcf_config_inline, NULL },
 #if 0
-	{ CLI_CONFIG_LOAD,	m_cli_func_config_load, NULL },
-	{ CLI_CONFIG_INLINE,	m_cli_func_config_inline, NULL },
 	{ CLI_SERVER_STOP,	m_cli_func_server_stop, NULL },
 	{ CLI_SERVER_RESTART },
 	{ CLI_PING,		m_cli_func_ping, NULL },

bin/varnishd/mgt_cli.h

+/*
+ * $Id$
+ */
+
+/* mgt_child.c */
+cli_func_t mcf_server_startstop;
+
+/* mgt_vcc.c */
+cli_func_t mcf_config_load;
+cli_func_t mcf_config_inline;

bin/varnishd/mgt_vcc.c

@@ -5,12 +5,15 @@
  */
 
 #include <string.h>
+#include <unistd.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <assert.h>
+#include <pthread.h>
 #include <sys/types.h>
 
 #include "sbuf.h"
+#include "queue.h"
 
 #include "libvarnish.h"
 #include "libvcl.h"
@@ -19,6 +22,19 @@
 #include "common_cli.h"
 
 #include "mgt.h"
+#include "mgt_cli.h"
+
+struct vcls {
+	TAILQ_ENTRY(vcls)	list;
+	char 			*name;
+	char			*fname;
+	int			active;
+};
+
+
+static TAILQ_HEAD(, vcls) vclhead = TAILQ_HEAD_INITIALIZER(vclhead);
+
+static pthread_mutex_t		vcc_mtx;
 
 /*--------------------------------------------------------------------*/
 
@@ -62,104 +78,163 @@ static const char *default_vcl =
 
 /*--------------------------------------------------------------------*/
 
-char *
-mgt_vcc_default(const char *bflag)
+static struct vcls *
+mgt_vcc_add(const char *name, char *file)
+{
+	struct vcls *vp;
+
+	vp = calloc(sizeof *vp, 1);
+	assert(vp != NULL);
+	vp->name = strdup(name);
+	vp->fname = file;
+	AZ(pthread_mutex_lock(&vcc_mtx));
+	TAILQ_INSERT_TAIL(&vclhead, vp, list);
+	AZ(pthread_mutex_unlock(&vcc_mtx));
+	return (vp);
+}
+
+static void
+mgt_vcc_del(struct vcls *vp)
+{
+	TAILQ_REMOVE(&vclhead, vp, list);
+	printf("unlink %s\n", vp->fname);
+	AZ(unlink(vp->fname));	/* XXX assert for now */
+	free(vp->fname);
+	free(vp->name);
+	free(vp);
+}
+
+static int
+mgt_vcc_delbyname(const char *name)
+{
+	struct vcls *vp;
+
+	TAILQ_FOREACH(vp, &vclhead, list) {
+		if (!strcmp(name, vp->name)) {
+			mgt_vcc_del(vp);
+			return (0);
+		}
+	}
+	return (1);
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+mgt_vcc_default(const char *bflag, const char *fflag)
 {
 	char *buf, *vf;
 	const char *p, *q;
 	struct sbuf *sb;
+	struct vcls *vp;
 
-	/*
-	 * XXX: should do a "HEAD /" on the -b argument to see that
-	 * XXX: it even works.  On the other hand, we should do that
-	 * XXX: for all backends in the cache process whenever we
-	 * XXX: change config, but for a complex VCL, it might not be
-	 * XXX: a bug for a backend to not reply at that time, so then
-	 * XXX: again: we should check it here in the "trivial" case.
-	 */
-	p = strchr(bflag, ' ');
-	if (p != NULL) {
-		q = p + 1;
-	} else {
-		p = strchr(bflag, '\0');
-		assert(p != NULL);
-		q = "http";
-	}
-	
-	buf = NULL;
-	asprintf(&buf,
-	    "backend default {\n"
-	    "    set backend.host = \"%*.*s\";\n"
-	    "    set backend.port = \"%s\";\n"
-	    "}\n", (int)(p - bflag), (int)(p - bflag), bflag, q);
-	assert(buf != NULL);
 	sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND);
 	assert(sb != NULL);
-	vf = VCC_Compile(sb, buf, NULL);
+	if (bflag != NULL) {
+		/*
+		 * XXX: should do a "HEAD /" on the -b argument to see that
+		 * XXX: it even works.  On the other hand, we should do that
+		 * XXX: for all backends in the cache process whenever we
+		 * XXX: change config, but for a complex VCL, it might not be
+		 * XXX: a bug for a backend to not reply at that time, so then
+		 * XXX: again: we should check it here in the "trivial" case.
+		 */
+		p = strchr(bflag, ' ');
+		if (p != NULL) {
+			q = p + 1;
+		} else {
+			p = strchr(bflag, '\0');
+			assert(p != NULL);
+			q = "http";
+		}
+		
+		buf = NULL;
+		asprintf(&buf,
+		    "backend default {\n"
+		    "    set backend.host = \"%*.*s\";\n"
+		    "    set backend.port = \"%s\";\n"
+		    "}\n", (int)(p - bflag), (int)(p - bflag), bflag, q);
+		assert(buf != NULL);
+		vf = VCC_Compile(sb, buf, NULL);
+		free(buf);
+	} else {
+		vf = VCC_CompileFile(sb, fflag);
+	}
 	sbuf_finish(sb);
 	if (sbuf_len(sb) > 0) {
 		fprintf(stderr, "%s", sbuf_data(sb));
-		free(buf);
 		sbuf_delete(sb);
-		return (NULL);
+		return (1);
 	}
 	sbuf_delete(sb);
-	free(buf);
-	return (vf);
+	vp = mgt_vcc_add("boot", vf);
+	vp->active = 1;
+	return (0);
 }
 
 /*--------------------------------------------------------------------*/
 
-char *
-mgt_vcc_file(const char *fflag)
+int
+mgt_push_vcls_and_start(int *status, char **p)
 {
-	char *vf;
-	struct sbuf *sb;
+	struct vcls *vp;
 
-	sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND);
-	assert(sb != NULL);
-	vf = VCC_CompileFile(sb, fflag);
-	sbuf_finish(sb);
-	if (sbuf_len(sb) > 0) {
-		fprintf(stderr, "%s", sbuf_data(sb));
-		sbuf_delete(sb);
-		return (NULL);
+	AZ(pthread_mutex_lock(&vcc_mtx));
+	TAILQ_FOREACH(vp, &vclhead, list) {
+		if (mgt_cli_askchild(status, p,
+		    "config.load %s %s\n", vp->name, vp->fname))
+			return (1);
+		if (vp->active)
+		if (mgt_cli_askchild(status, p,
+		    "config.use %s\n", vp->name, vp->fname))
+			return (1);
 	}
-	sbuf_delete(sb);
-	return (vf);
+	AZ(pthread_mutex_unlock(&vcc_mtx));
+	if (mgt_cli_askchild(status, p, "start\n"))
+		return (1);
+	return (0);
 }
 
 /*--------------------------------------------------------------------*/
 
+static
 void
-mgt_vcc_init(void)
+mgt_vcc_atexit(void)
 {
+	struct vcls *vp;
 
-	VCC_InitCompile(default_vcl);
+	if (getpid() != mgt_pid)
+		return;
+	AZ(pthread_mutex_lock(&vcc_mtx));
+	TAILQ_FOREACH(vp, &vclhead, list) {
+		printf("Has %s %s\n", vp->name, vp->fname);
+	}
+	while (1) {
+		vp = TAILQ_FIRST(&vclhead);
+		if (vp == NULL)
+			break;
+		mgt_vcc_del(vp);
+	}
+	AZ(pthread_mutex_unlock(&vcc_mtx));
 }
 
+void
+mgt_vcc_init(void)
+{
 
-#if 0
-#include <assert.h>
-#include <string.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <limits.h>
-#include <signal.h>
-#include <stdarg.h>
-#include <poll.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <time.h>
-#include <unistd.h>
+	AZ(pthread_mutex_init(&vcc_mtx, NULL));
+	VCC_InitCompile(default_vcl);
+	AZ(atexit(mgt_vcc_atexit));
+}
 
-#include "mgt.h"
+/*--------------------------------------------------------------------*/
 
-static void
-m_cli_func_config_inline(struct cli *cli, char **av, void *priv)
+void
+mcf_config_inline(struct cli *cli, char **av, void *priv)
 {
-	char *vf;
+	char *vf, *p;
 	struct sbuf *sb;
+	int status;
 
 	(void)priv;
 
@@ -170,20 +245,26 @@ m_cli_func_config_inline(struct cli *cli, char **av, void *priv)
 	if (sbuf_len(sb) > 0) {
 		cli_out(cli, "%s", sbuf_data(sb));
 		sbuf_delete(sb);
+		cli_result(cli, CLIS_PARAM);
 		return;
 	}
 	sbuf_delete(sb);
-	cli_suspend(cli);
-	mgt_child_request(cli_passthrough_cb, cli, NULL,
-	    "config.load %s %s", av[2], vf);
+	if (mgt_cli_askchild(&status, &p, "config.load %s %s\n", av[2], vf)) {
+		cli_result(cli, status);
+		cli_out(cli, "%s", p);
+		free(p);
+		return;
+	}
+	mgt_vcc_add(av[2], vf);
 }
 
-/* XXX: m prefix to avoid name clash */
-static void
-m_cli_func_config_load(struct cli *cli, char **av, void *priv)
+void
+mcf_config_load(struct cli *cli, char **av, void *priv)
 {
 	char *vf;
 	struct sbuf *sb;
+	int status;
+	char *p;
 
 	(void)priv;
 
@@ -194,12 +275,15 @@ m_cli_func_config_load(struct cli *cli, char **av, void *priv)
 	if (sbuf_len(sb) > 0) {
 		cli_out(cli, "%s", sbuf_data(sb));
 		sbuf_delete(sb);
+		cli_result(cli, CLIS_PARAM);
 		return;
 	}
 	sbuf_delete(sb);
-	cli_suspend(cli);
-	mgt_child_request(cli_passthrough_cb, cli, NULL,
-	    "config.load %s %s", av[2], vf);
+	if (mgt_cli_askchild(&status, &p, "config.load %s %s\n", av[2], vf)) {
+		cli_result(cli, status);
+		cli_out(cli, "%s", p);
+		free(p);
+		return;
+	}
+	mgt_vcc_add(av[2], vf);
 }
-
-#endif

bin/varnishd/varnishd.c

@@ -363,12 +363,8 @@ main(int argc, char *argv[])
 		usage();
 	}
 
-	if (bflag != NULL)
-		heritage.vcl_file = mgt_vcc_default(bflag);
-	else
-		heritage.vcl_file = mgt_vcc_file(fflag);
-	if (heritage.vcl_file == NULL)
-		exit (1);
+	if (mgt_vcc_default(bflag, fflag))
+		exit (2);
 
 	setup_storage(sflag);
 	setup_hash(hflag);