varnishevent - added configuration

bin/varnishevent/Makefile.am

@@ -14,6 +14,7 @@ varnishevent_SOURCES = \
 	data.c \
 	spscq.c \
 	writer.c \
+	config.c \
 	$(top_builddir)/lib/libvarnish/assert.c \
 	$(top_builddir)/lib/libvarnish/flopen.c \
 	$(top_builddir)/lib/libvarnish/version.c \

bin/varnishevent/config.c

+/*-
+ * Copyright (c) 2013 UPLEX Nils Goroll Systemoptimierung
+ * Copyright (c) 2013 Otto Gmbh & Co KG
+ * All rights reserved
+ * Use only with permission
+ *
+ * Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <strings.h>
+#include <ctype.h>
+#include <syslog.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <math.h>
+#include <unistd.h>
+#include <pwd.h>
+
+#include "varnishevent.h"
+#include "libvarnish.h"
+
+#define DEFAULT_USER "nobody"
+
+static const int facilitynum[8] =
+    { LOG_LOCAL0, LOG_LOCAL1, LOG_LOCAL2, LOG_LOCAL3, LOG_LOCAL4, LOG_LOCAL5,
+      LOG_LOCAL6, LOG_LOCAL7 };
+
+static int
+conf_getFacility(const char *facility) {
+    int localnum;
+    
+    if (strcasecmp(facility, "USER") == 0)
+        return LOG_USER;
+    if (strlen(facility) != 6
+        || strncasecmp(facility, "LOCAL", 5) != 0
+        || !isdigit(facility[5]))
+        return(-1);
+    localnum = atoi(&facility[5]);
+    if (localnum > 7)
+        return(-1);
+    return(facilitynum[localnum]);
+}
+
+static int
+conf_getUnsignedInt(const char *rval, unsigned *i)
+{
+    long n;
+    char *p;
+
+    errno = 0;
+    n = strtoul(rval, &p, 10);
+    if (errno)
+        return(errno);
+    if (strlen(p) != 0)
+        return(EINVAL);
+    if (n < 0 || n > UINT_MAX)
+        return(ERANGE);
+    *i = (unsigned int) n;
+    return(0);
+}
+
+#define confString(name,fld)         \
+    if (strcmp(lval, (name)) == 0) { \
+        strcpy((config.fld), rval);  \
+        return(0);                   \
+    }
+
+#define confUnsigned(name,fld)                   \
+    if (strcmp(lval, name) == 0) {               \
+        unsigned int i;                          \
+        int err = conf_getUnsignedInt(rval, &i); \
+        if (err != 0)                            \
+            return err;                          \
+        config.fld = i;                          \
+        return(0);                               \
+    }
+
+int
+CONF_Add(const char *lval, const char *rval)
+{
+    int ret;
+    
+    confString("pid.file", pid_file);
+    confString("varnish.name", varnish_name);
+    confString("log.file", log_file);
+    confString("varnish.bindump", varnish_bindump);
+
+    confUnsigned("max.reclen", max_reclen);
+    confUnsigned("max.headers", max_headers);
+    confUnsigned("max.fd", max_fd);
+    confUnsigned("max.data", max_data);
+    confUnsigned("restarts", restarts);
+    confUnsigned("monitor.interval", monitor_interval);
+
+    if (strcmp(lval, "syslog.facility") == 0) {
+        if ((ret = conf_getFacility(rval)) < 0)
+            return EINVAL;
+        config.syslog_facility = ret;
+        strcpy(config.syslog_facility_name, rval);
+        char *p = &config.syslog_facility_name[0];
+        do { *p = toupper(*p); } while (*++p);
+        return(0);
+    }
+
+    if (strcmp(lval, "user") == 0) {
+        struct passwd *pw;
+        
+        pw = getpwnam(rval);
+        if (pw == NULL)
+            return(EINVAL);
+        strcpy(config.user_name, pw->pw_name);
+        config.uid = pw->pw_uid;
+        config.gid = pw->pw_gid;
+        return(0);
+    }
+
+    return EINVAL;
+}
+
+static int
+conf_ParseLine(char *ptr, char **lval, char **rval)
+{
+    char *endlval;
+    
+    *lval = ptr;
+    while(*++ptr && !isspace(*ptr) && *ptr != '=')
+        ;
+    if (*ptr == '\0')
+        return(1);
+    endlval = ptr;
+    while(isspace(*ptr) && *++ptr)
+        ;
+    if (ptr == '\0' || *ptr != '=')
+        return(1);
+    while(*++ptr && isspace(*ptr))
+        ;
+    if (ptr == '\0')
+        return(1);
+    *endlval = '\0';
+    *rval = ptr;
+    return(0);
+}
+
+void
+CONF_Init(void)
+{
+    struct passwd *pw;
+
+    strcpy(config.pid_file, DEFAULT_PID_FILE);
+    config.varnish_name[0] = '\0';
+    config.log_file[0] = '\0';
+    config.varnish_bindump[0] = '\0';
+    config.syslog_facility = LOG_LOCAL0;
+    config.monitor_interval = 30;
+
+    config.max_reclen = DEFAULT_MAX_RECLEN;
+    config.max_headers = DEFAULT_MAX_HEADERS;
+    config.max_fd = DEFAULT_MAX_FD;
+    config.max_data = DEFAULT_MAX_DATA;
+
+    config.restarts = 1;
+    
+    pw = getpwnam(DEFAULT_USER);
+    if (pw == NULL)
+        pw = getpwuid(getuid());
+    AN(pw);
+    strcpy(config.user_name, pw->pw_name);
+    config.uid = pw->pw_uid;
+    config.gid = pw->pw_gid;
+}
+
+int
+CONF_ReadFile(const char *file) {
+    FILE *in;
+    char line[BUFSIZ];
+    int linenum = 0;
+
+    in = fopen(file, "r");
+    if (in == NULL) {
+        perror(file);
+        return(-1);
+    }
+    
+    while (fgets(line, BUFSIZ, in) != NULL) {
+        char orig[BUFSIZ];
+        
+        linenum++;
+        char *comment = strchr(line, '#');
+        if (comment != NULL)
+            *comment = '\0';
+        if (strlen(line) == 0)
+            continue;
+    
+        char *ptr = line + strlen(line) - 1;
+        while (ptr != line && isspace(*ptr))
+            --ptr;
+        ptr[isspace(*ptr) ? 0 : 1] = '\0';
+        if (strlen(line) == 0)
+            continue;
+
+        ptr = line;
+        while (isspace(*ptr) && *++ptr)
+            ;
+        strcpy(orig, ptr);
+        char *lval, *rval;
+        if (conf_ParseLine(ptr, &lval, &rval) != 0) {
+            fprintf(stderr, "Cannot parse %s line %d: '%s'\n", file, linenum,
+                    orig);
+            return(-1);
+        }
+
+        int ret;
+        if ((ret = CONF_Add((const char *) lval, (const char *) rval)) != 0) {
+            fprintf(stderr, "Error in %s line %d (%s): '%s'\n", file, linenum,
+                strerror(ret), orig);
+            return(-1);
+        }
+    }
+    return(0);
+}
+
+#define confdump(str,val) \
+    LOG_Log(LOG_DEBUG, "config: " str, (val))
+
+#if 0
+void
+CONF_Dump(void)
+{
+    confdump("pid.file = %s", config.pid_file);
+    confdump("varnish.name = %s", config.varnish_name);
+    confdump("log.file = %s",
+        strcmp(config.log_file,"-") == 0 ? "stdout" : config.log_file);
+    confdump("varnish.bindump = %s", config.varnish_bindump);
+    confdump("syslog.facility = %s", config.syslog_facility_name);
+    confdump("monitor.interval = %u", config.monitor_interval);
+    confdump("max.reclen = %u", config.max_reclen);
+    confdump("max.headers = %u", config.max_headers);
+    confdump("max.fd = %u", config.max_fd);
+    confdump("max.data = %u", config.max_data);
+    confdump("restarts = %u", config.restarts);
+    confdump("user = %s", config.user_name);
+}
+#endif

bin/varnishevent/data.c

@@ -30,6 +30,7 @@
  */
 
 #include <pthread.h>
+#include <stdlib.h>
 
 #include "varnishevent.h"
 #include "vas.h"
@@ -41,6 +42,7 @@ static const char *statename[3] = { "EMPTY", "OPEN", "DONE" };
 #endif
 
 static pthread_mutex_t freelist_lock;
+static char *bufptr;
 
 #if 0
 static void
@@ -67,27 +69,59 @@ DATA_Clear_Logline(logline_t *ll)
         ll->tag[i].len = 0;
 }
 
+#define INIT_LOG_RECORDS(fld) do {                                            \
+    (fld).record = (record_t *) calloc(config.max_headers, sizeof(record_t)); \
+    if ((fld).record == NULL)                                                 \
+        return errno;                                                         \
+    for (int j = 0; j < config.max_headers; j++) {                            \
+        (fld).record[j].magic = RECORD_MAGIC;                                 \
+        (fld).record[j].data = &bufptr[bufidx++ * config.max_reclen];         \
+    }                                                                         \
+} while (0)
+
 int
 DATA_Init(void)
 {
-    VSTAILQ_INIT(&freehead);
-    AZ(pthread_mutex_init(&freelist_lock, &attr_lock));
-    global_nfree = 0;
+    int bufidx = 0;
+    
+    int records = config.max_data * (MAX_VSL_TAG  + 3 * config.max_headers);
+    
+    bufptr = (char *) calloc(records, config.max_reclen);
+    if (bufptr == NULL)
+        return errno;
+
+    logline = (logline_t *) calloc(config.max_data, sizeof(logline_t));
+    if (logline == NULL)
+        return errno;
+
+    fd_tbl = (fd_t *) calloc(config.max_fd, sizeof(fd_t));
+    if (fd_tbl == NULL)
+        return errno;
     
-    /*
-     * XXX: allocate these tables from configured sizes
-     */
-    for (int i = 0; i < MAX_DATA; i++) {
+    VSTAILQ_INIT(&freehead);
+    for (int i = 0; i < config.max_data; i++) {
+        INIT_LOG_RECORDS(logline[i].rx_headers);
+        INIT_LOG_RECORDS(logline[i].tx_headers);
+        INIT_LOG_RECORDS(logline[i].vcl_log);
+        
+        for (int j = 0; j < MAX_VSL_TAG; j++) {
+            logline[i].tag[j].magic = RECORD_MAGIC;
+            logline[i].tag[j].data = &bufptr[bufidx++ * config.max_reclen];
+        }
+        
         logline[i].magic = LOGLINE_MAGIC;
         DATA_Clear_Logline(&logline[i]);
 	VSTAILQ_INSERT_TAIL(&freehead, &logline[i], freelist);
     }
 
-    for (int i = 0; i < MAX_FD; i++) {
-        fd_tbl[i].ll = NULL;
-        fd_tbl[i].state = FD_EMPTY;
+    for (int k = 0; k < config.max_fd; k++) {
+        fd_tbl[k].ll = NULL;
+        fd_tbl[k].state = FD_EMPTY;
     }
 
+    AZ(pthread_mutex_init(&freelist_lock, &attr_lock));
+    global_nfree = 0;
+    
     return(0);
 }
 

bin/varnishevent/spscq.c

@@ -45,9 +45,8 @@ unsigned SPSCQ_Len(void) {
 void
 SPSCQ_Enq(logline_t *ptr)
 {
-    /* XXX: compare to a configured max in the assert
-       XXX: is the assertion accurate if enqs & deqs are not synced? */
-    assert(enqs - deqs < MAX_DATA);
+    /* XXX: is the assertion accurate if enqs & deqs are not synced? */
+    assert(enqs - deqs < config.max_data);
     enqs++;
     VSTAILQ_INSERT_TAIL(&spscq_head, ptr, spscq);
 }

bin/varnishevent/varnishevent.c

@@ -160,9 +160,9 @@ collect(struct logline_t *lp, enum VSL_tag_e tag, unsigned spec,
             assert(lp->spec == spec);
     }
 
-    /* XXX: use a configured maximum, and issue a warning when too long */
-    if (len > MAX_RECLEN)
-        len = MAX_RECLEN;
+    /* XXX: issue a warning when too long */
+    if (len > config.max_reclen)
+        len = config.max_reclen;
 
     /* XXX: check against overflow for lp->n* */
     if (tag == SLT_RxHeader) {
@@ -307,7 +307,7 @@ usage(void)
 int
 main(int argc, char *argv[])
 {
-    int c;
+    int c, errnum;
     int a_flag = 0, D_flag = 0, format_flag = 0;
     const char *P_arg = NULL;
     const char *w_arg = NULL;
@@ -317,6 +317,7 @@ main(int argc, char *argv[])
 
     vd = VSM_New();
     VSL_Setup(vd);
+    CONF_Init();
 
     while ((c = getopt(argc, argv, VSL_ARGS "aDP:Vw:fF:")) != -1) {
         switch (c) {
@@ -407,9 +408,17 @@ main(int argc, char *argv[])
     AZ(pthread_mutexattr_setpshared(&attr_lock, PTHREAD_PROCESS_PRIVATE));
     AZ(pthread_condattr_setpshared(&attr_cond,  PTHREAD_PROCESS_PRIVATE)); 
 
-    WRT_Init(format, of);
-    DATA_Init();
+    /* XXX: log errors */
+    if ((errnum = DATA_Init()) != 0) {
+        fprintf(stderr, "Cannot init data structures: %s\n", strerror(errnum));
+        exit(1);
+    }
 
+    if ((errnum = WRT_Init(format, of)) != 0) {
+        fprintf(stderr, "Cannot init writer thread: %s\n", strerror(errnum));
+        exit(1);
+    }
+    
     WRT_Start();
     while (!WRT_Running())
         ;

bin/varnishevent/varnishevent.h

@@ -32,14 +32,17 @@
 #include <stdint.h>
 #include <stdio.h>
 #include <pthread.h>
+#include <sys/types.h>
 
 #include "vqueue.h"
 
-/* XXX: configure */
-#define MAX_RECLEN 255
-#define MAX_HEADERS 64
-#define MAX_FD 1024
-#define MAX_DATA 3072
+/* Defaults from Varnish 3.0.3 */
+#define DEFAULT_MAX_RECLEN 255	/* shm_reclen */
+#define DEFAULT_MAX_HEADERS 64	/* http_max_hdr */
+
+#define DEFAULT_MAX_FD 1024
+#define DEFAULT_MAX_DATA 3072
+#define DEFAULT_PID_FILE "/var/run/varnishevent.pid"
 
 #define MAX_VSL_TAG 256
 
@@ -50,14 +53,14 @@ typedef enum {
 } data_state_e;
 
 typedef struct {
-    /* XXX: allocate dynamically from configurable max_reclen */
-    char data[MAX_RECLEN];
+    unsigned magic;
+#define RECORD_MAGIC 0xdf4399b1
+    char *data;
     unsigned len;
 } record_t;
 
 typedef struct {
-    /* XXX: allocate dynamically from configurable max_headers */
-    record_t record[MAX_HEADERS];
+    record_t *record;
     unsigned nrec;
 } hdr_t;
 
@@ -75,17 +78,19 @@ typedef struct logline_t {
     VSTAILQ_ENTRY(logline_t) spscq;
 } logline_t;
 
-logline_t logline[MAX_DATA];
+logline_t *logline;
 
 typedef enum {
     FD_EMPTY = 0,
     FD_OPEN
 } fd_state_e;
 
-struct {
+typedef struct fd_t {
     logline_t *ll;
     fd_state_e state;
-} fd_tbl[MAX_FD];
+} fd_t;
+
+fd_t *fd_tbl;
 
 VSTAILQ_HEAD(freehead_s, logline_t);
 
@@ -106,9 +111,69 @@ pthread_mutex_t spscq_ready_lock;
 pthread_mutexattr_t attr_lock;
 pthread_condattr_t  attr_cond;
 
+struct config {
+    char        pid_file[BUFSIZ];
+    
+    /* VSL 'n' argument */
+    char        varnish_name[BUFSIZ];
+    
+    char        log_file[BUFSIZ];
+    
+    /* VSL 'r' argument */
+    char        varnish_bindump[BUFSIZ];
+    
+    int         syslog_facility;
+    char        syslog_facility_name[BUFSIZ];
+    unsigned    monitor_interval;
+    
+    /* varnishd param shm_reclen */
+    unsigned	max_reclen;
+
+    /* varnishd param http_max_hdr */
+    unsigned	max_headers;
+
+    unsigned	max_fd;
+    unsigned	max_data;    
+
+    unsigned    restarts;
+    char        user_name[BUFSIZ];
+    uid_t       uid;
+    gid_t       gid;
+} config;
+
 /* varnishevent.c */
 int RDR_Waiting(void);
 
+/* config.c */
+void CONF_Init(void);
+int CONF_Add(const char *lval, const char *rval);
+int CONF_ReadFile(const char *file);
+#if 0
+void CONF_Dump(void);
+#endif
+
+#if 0
+/* log.c */
+typedef void log_log_t(int level, const char *msg, ...);
+typedef void log_setlevel_t(int level);
+typedef void log_close_t(void);
+
+struct logconf {
+    log_log_t           *log;
+    log_setlevel_t      *setlevel;
+    log_close_t         *close;
+    FILE                *out;
+    int                 level;
+} logconf;
+
+int LOG_Open(const char *progname);
+/* XXX: __VA_ARGS__ can't be empty ... */
+#define LOG_Log0(level, msg) logconf.log(level, msg)
+#define LOG_Log(level, msg, ...) logconf.log(level, msg, __VA_ARGS__)
+#define LOG_SetLevel(level) logconf.setlevel(level)
+#define LOG_Close() logconf.close()
+#endif
+
 /* data.c */
 int DATA_Init(void);
 void DATA_Clear_Logline(logline_t *ll);

bin/varnishevent/writer.c

@@ -43,7 +43,6 @@
 #include "miniobj.h"
 #include "vsb.h"
 
-/* XXX: logging */
 #define LOG_Log0(l,s) fprintf(stderr, (s))
 #define LOG_Log(level, msg, ...) vfprintf(stderr, msg, __VA_ARGS__)
 
@@ -86,7 +85,6 @@ static const char *cformat;
 static const char *bformat
     = "%t dir=%d url=[%U] rc=[%s] len=[%b] xid=[%{X-Varnish}i] fetch_body=[%{tag:Fetch_Body}x]";
 
-
 static struct vsb *os;
 static FILE *fo;
 
@@ -109,6 +107,8 @@ static writer_data_t wrt_data;
     
 static unsigned run, cleaned = 0;
 
+static char *scratch;
+
 static inline void
 wrt_return_freelist(void)
 {
@@ -136,11 +136,12 @@ get_hdr(const char *hdr, hdr_t *tbl)
 static inline char *
 get_fld(record_t *rec, int n)
 {
-    char *ret = NULL, *s, str[MAX_RECLEN];
+    char *ret = NULL, *s;
     int i = 0;
 
-    strncpy(str, rec->data, rec->len);
-    s = str;
+    AN(scratch);
+    strncpy(scratch, rec->data, rec->len);
+    s = scratch;
     do {
         ret = strtok(s, " \t");
         s = NULL;
@@ -793,6 +794,10 @@ static void wrt_cleanup(void)
 int
 WRT_Init(const char *format, FILE *out)
 {
+    scratch = (char *) malloc(config.max_reclen);
+    if (scratch == NULL)
+        return errno;
+
     TIM_real_mono_diff = TIM_real() - TIM_mono();
     
     run = 1;