use -I regex filters to limit the amount of data copied from the log

src/format.c

@@ -39,6 +39,7 @@
 #include "vas.h"
 #include "miniobj.h"
 #include "base64.h"
+#include "vqueue.h"
 
 #include "varnishevent.h"
 #include "format.h"
@@ -53,7 +54,7 @@ typedef struct compiled_fmt_t {
 
 static struct vsb *payload;
 static struct vsb *bintag;
-static char *scratch;
+static char *scratch = NULL;
 
 static char empty[] = "";
 static char hit[] = "hit";
@@ -63,10 +64,17 @@ static char pipe[] = "pipe";
 static char error[] = "error";
 static char dash[] = "-";
 
-static struct vsb *i_arg;
+typedef struct include_t {
+    char *hdr;
+    VSTAILQ_ENTRY(include_t) inclist;
+} include_t;
+
+typedef VSTAILQ_HEAD(includehead_s, include_t) includehead_t;
 
 static compiled_fmt_t cformat, bformat, rformat;
-static char ctags[MAX_VSL_TAG], btags[MAX_VSL_TAG], rtags[MAX_VSL_TAG];
+static includehead_t cincl[MAX_VSL_TAG], bincl[MAX_VSL_TAG], rincl[MAX_VSL_TAG];
+static unsigned includes;
+static char **incl_arg = NULL;
 
 char *
 get_payload(const logline_t *rec)
@@ -721,44 +729,78 @@ add_fmt(const compiled_fmt_t *fmt, struct vsb *os, unsigned n,
     (C(type) ? format_ltr##_client : format_ltr##_backend)
 
 static void
-add_tag(enum VSL_transaction_e type, enum VSL_tag_e tag)
+add_tag(enum VSL_transaction_e type, enum VSL_tag_e tag, const char *hdr)
 {
+    includehead_t *inclhead;
+    include_t *incl;
+
     switch(type) {
     case VSL_t_req:
-        ctags[tag] = 1;
+        inclhead = &cincl[tag];
         break;
     case VSL_t_bereq:
-        btags[tag] = 1;
+        inclhead = &bincl[tag];
         break;
     case VSL_t_raw:
-        rtags[tag] = 1;
+        inclhead = &rincl[tag];
         break;
     default:
         WRONG("Illegal transaction type");
     }
+
+    /* Don't add the same include more than once */
+    VSTAILQ_FOREACH(incl, inclhead, inclist)
+        if ((hdr == NULL && incl->hdr == NULL)
+            || (hdr != NULL && incl->hdr != NULL
+                && (strcmp(incl->hdr, hdr) == 0)))
+            return;
+
+    incl = calloc(1, sizeof(include_t));
+    AN(incl);
+    if (hdr != NULL)
+        incl->hdr = strdup(hdr);
+    VSTAILQ_INSERT_TAIL(inclhead, incl, inclist);
+    includes++;
 }
 
 static void
 add_cb_tag(enum VSL_transaction_e type, enum VSL_tag_e ctag,
-           enum VSL_tag_e btag)
+           enum VSL_tag_e btag, const char *hdr)
 {
     enum VSL_tag_e tag;
-    char *tags;
 
     switch(type) {
     case VSL_t_req:
         tag = ctag;
-        tags = ctags;
         break;
     case VSL_t_bereq:
         tag = btag;
-        tags = btags;
         break;
     default:
         WRONG("Illegal transaction type");
     }
 
-    tags[tag] = 1;
+    add_tag(type, tag, hdr);
+}
+
+static void
+add_cb_tag_incl(enum VSL_transaction_e type, enum VSL_tag_e tag,
+                const char *chdr, const char *bhdr)
+{
+    const char *hdr;
+
+    switch(type) {
+    case VSL_t_req:
+        hdr = chdr;
+        break;
+    case VSL_t_bereq:
+        hdr = bhdr;
+        break;
+    default:
+        WRONG("Illegal transaction type");
+    }
+
+    add_tag(type, tag, hdr);
 }
 
 static int
@@ -826,7 +868,7 @@ compile_fmt(char * const format, compiled_fmt_t * const fmt,
 
         case 'b':
             add_fmt(fmt, os, n, FMT(type, format_b), NULL, SLT__Bogus);
-            add_cb_tag(type, SLT_ReqAcct, SLT_BereqAcct);
+            add_cb_tag(type, SLT_ReqAcct, SLT_BereqAcct, NULL);
             n++;
             break;
 
@@ -836,30 +878,30 @@ compile_fmt(char * const format, compiled_fmt_t * const fmt,
             
         case 'D':
             add_fmt(fmt, os, n, FMT(type, format_D), NULL, SLT__Bogus);
-            add_tag(type, SLT_Timestamp);
+            add_cb_tag_incl(type, SLT_Timestamp, "Resp", "BerespBody");
             n++;
             break;
             
         case 'H':
             add_fmt(fmt, os, n, FMT(type, format_H), NULL, SLT__Bogus);
-            add_cb_tag(type, SLT_ReqProtocol, SLT_BereqProtocol);
+            add_cb_tag(type, SLT_ReqProtocol, SLT_BereqProtocol, NULL);
             n++;
             break;
             
         case 'h':
             add_fmt(fmt, os, n, FMT(type, format_h), NULL, SLT__Bogus);
-            add_cb_tag(type, SLT_ReqStart, SLT_Backend);
+            add_cb_tag(type, SLT_ReqStart, SLT_Backend, NULL);
             n++;
             break;
             
         case 'I':
             add_fmt(fmt, os, n, FMT(type, format_I), NULL, SLT__Bogus);
             if (C(type)) {
-                ctags[SLT_ReqAcct] = 1;
-                ctags[SLT_PipeAcct] = 1;
+                add_tag(type, SLT_ReqAcct, NULL);
+                add_tag(type, SLT_PipeAcct, NULL);
             }
             else
-                btags[SLT_BereqAcct] = 1;
+                add_tag(type, SLT_BereqAcct, NULL);
             n++;
             break;
             
@@ -869,63 +911,64 @@ compile_fmt(char * const format, compiled_fmt_t * const fmt,
 
         case 'm':
             add_fmt(fmt, os, n, FMT(type, format_m), NULL, SLT__Bogus);
-            add_cb_tag(type, SLT_ReqMethod, SLT_BereqMethod);
+            add_cb_tag(type, SLT_ReqMethod, SLT_BereqMethod, NULL);
             n++;
             break;
 
         case 'O':
             add_fmt(fmt, os, n, FMT(type, format_O), NULL, SLT__Bogus);
             if (C(type)) {
-                ctags[SLT_ReqAcct] = 1;
-                ctags[SLT_PipeAcct] = 1;
+                add_tag(type, SLT_ReqAcct, NULL);
+                add_tag(type, SLT_PipeAcct, NULL);
             }
             else
-                btags[SLT_BereqAcct] = 1;
+                add_tag(type, SLT_BereqAcct, NULL);
             n++;
             break;
             
         case 'q':
             add_fmt(fmt, os, n, FMT(type, format_q), NULL, SLT__Bogus);
-            add_cb_tag(type, SLT_ReqURL, SLT_BereqURL);
+            add_cb_tag(type, SLT_ReqURL, SLT_BereqURL, NULL);
             n++;
             break;
 
         case 'r':
             add_fmt(fmt, os, n, FMT(type, format_r), NULL, SLT__Bogus);
-            add_cb_tag(type, SLT_ReqMethod, SLT_BereqMethod);
-            add_cb_tag(type, SLT_ReqHeader, SLT_BereqHeader);
-            add_cb_tag(type, SLT_ReqURL, SLT_BereqURL);
-            add_cb_tag(type, SLT_ReqProtocol, SLT_BereqProtocol);
+            add_cb_tag(type, SLT_ReqMethod, SLT_BereqMethod, NULL);
+            add_cb_tag(type, SLT_ReqHeader, SLT_BereqHeader, "Host");
+            add_cb_tag(type, SLT_ReqURL, SLT_BereqURL, NULL);
+            add_cb_tag(type, SLT_ReqProtocol, SLT_BereqProtocol, NULL);
             n++;
             break;
 
         case 's':
             add_fmt(fmt, os, n, FMT(type, format_s), NULL, SLT__Bogus);
-            add_cb_tag(type, SLT_RespStatus, SLT_BerespStatus);
+            add_cb_tag(type, SLT_RespStatus, SLT_BerespStatus, NULL);
             n++;
             break;
 
         case 't':
             add_fmt(fmt, os, n, format_t, NULL, SLT__Bogus);
-            add_tag(type, SLT_Timestamp);
+            if (type != VSL_t_raw)
+                add_tag(type, SLT_Timestamp, "Start");
             n++;
             break;
 
         case 'T':
             add_fmt(fmt, os, n, FMT(type, format_T), NULL, SLT__Bogus);
-            add_tag(type, SLT_Timestamp);
+            add_tag(type, SLT_Timestamp, "Start");
             n++;
             break;
             
         case 'U':
             add_fmt(fmt, os, n, FMT(type, format_U), NULL, SLT__Bogus);
-            add_cb_tag(type, SLT_ReqURL, SLT_BereqURL);
+            add_cb_tag(type, SLT_ReqURL, SLT_BereqURL, NULL);
             n++;
             break;
 
         case 'u':
             add_fmt(fmt, os, n, FMT(type, format_u), NULL, SLT__Bogus);
-            add_cb_tag(type, SLT_ReqHeader, SLT_BereqHeader);
+            add_cb_tag(type, SLT_ReqHeader, SLT_BereqHeader, "Authorization");
             n++;
             break;
 
@@ -946,19 +989,20 @@ compile_fmt(char * const format, compiled_fmt_t * const fmt,
             switch (ltr) {
             case 'i':
                 add_fmt(fmt, os, n, FMT(type, format_Xi), fname, SLT__Bogus);
-                add_cb_tag(type, SLT_ReqHeader, SLT_BereqHeader);
+                add_cb_tag(type, SLT_ReqHeader, SLT_BereqHeader, fname);
                 n++;
                 p = tmp;
                 break;
             case 'o':
                 add_fmt(fmt, os, n, FMT(type, format_Xo), fname, SLT__Bogus);
-                add_cb_tag(type, SLT_RespHeader, SLT_BerespHeader);
+                add_cb_tag(type, SLT_RespHeader, SLT_BerespHeader, fname);
                 n++;
                 p = tmp;
                 break;
             case 't':
                 add_fmt(fmt, os, n, format_Xt, fname, SLT__Bogus);
-                add_tag(type, SLT_Timestamp);
+                if (type != VSL_t_raw)
+                    add_tag(type, SLT_Timestamp, "Start");
                 n++;
                 p = tmp;
                 break;
@@ -966,13 +1010,13 @@ compile_fmt(char * const format, compiled_fmt_t * const fmt,
                 if (strcmp(fname, "Varnish:time_firstbyte") == 0) {
                     add_fmt(fmt, os, n, FMT(type, format_Xttfb), NULL,
                             SLT__Bogus);
-                    add_tag(type, SLT_Timestamp);
+                    add_cb_tag_incl(type, SLT_Timestamp, "Process", "Beresp");
                 }
                 else if (strcmp(fname, "Varnish:hitmiss") == 0) {
                     if (C(type)) {
                         add_fmt(fmt, os, n, format_VCL_disp, "m", SLT__Bogus);
-                        ctags[SLT_VCL_call] = 1;
-                        ctags[SLT_VCL_return] = 1;
+                        add_tag(type, SLT_VCL_call, NULL);
+                        add_tag(type, SLT_VCL_return, NULL);
                     }
                     else {
                         sprintf(err,
@@ -983,8 +1027,8 @@ compile_fmt(char * const format, compiled_fmt_t * const fmt,
                 else if (strcmp(fname, "Varnish:handling") == 0) {
                     if (C(type)) {
                         add_fmt(fmt, os, n, format_VCL_disp, "n", SLT__Bogus);
-                        ctags[SLT_VCL_call] = 1;
-                        ctags[SLT_VCL_return] = 1;
+                        add_tag(type, SLT_VCL_call, NULL);
+                        add_tag(type, SLT_VCL_return, NULL);
                     }
                     else {
                         sprintf(err,
@@ -998,7 +1042,7 @@ compile_fmt(char * const format, compiled_fmt_t * const fmt,
                     // Format: %{VCL_Log:keyname}x
                     // Logging: std.log("keyname:value")
                     add_fmt(fmt, os, n, format_VCL_Log, fname+8, SLT__Bogus);
-                    add_tag(type, SLT_VCL_Log);
+                    add_tag(type, SLT_VCL_Log, fname+8);
                 }
                 else if (strncmp(fname, "tag:", 4) == 0) {
                     int t = 0;
@@ -1009,7 +1053,7 @@ compile_fmt(char * const format, compiled_fmt_t * const fmt,
                         return 1;
                     }
                     add_fmt(fmt, os, n, format_SLT, NULL, t);
-                    add_tag(type, t);
+                    add_tag(type, t, NULL);
                 }
                 else {
                     sprintf(err, "Unknown format starting at: %s", fname);
@@ -1043,6 +1087,28 @@ compile_fmt(char * const format, compiled_fmt_t * const fmt,
     return 0;
 }
 
+static void
+fmt_build_I_arg(const includehead_t *inclhead, int *incl_idx)
+{
+    AN(incl_arg);
+    AN(includes);
+    AN(scratch);
+
+    for (int i = 0; i < MAX_VSL_TAG; i++) {
+        include_t *incl;
+
+        VSTAILQ_FOREACH(incl, &inclhead[i], inclist) {
+            assert(*incl_idx < includes);
+            if (incl->hdr == NULL)
+                sprintf(scratch, "%s:.", VSL_tags[i]);
+            else
+                sprintf(scratch, "%s:^\\s*%s\\s*:", VSL_tags[i], incl->hdr);
+            incl_arg[*incl_idx] = strdup(scratch);
+            *incl_idx += 1;
+        }
+    }
+}
+
 int
 FMT_Init(char *err)
 {
@@ -1058,13 +1124,12 @@ FMT_Init(char *err)
     if (bintag == NULL)
         return ENOMEM;
 
-    i_arg = VSB_new_auto();
-    if (i_arg == NULL)
-        return ENOMEM;
-
-    memset(ctags, 0, MAX_VSL_TAG);
-    memset(btags, 0, MAX_VSL_TAG);
-    memset(rtags, 0, MAX_VSL_TAG);
+    includes = 0;
+    for (int i = 0; i < MAX_VSL_TAG; i++) {
+        VSTAILQ_INIT(&cincl[i]);
+        VSTAILQ_INIT(&bincl[i]);
+        VSTAILQ_INIT(&rincl[i]);
+    }
 
     if (!EMPTY(config.cformat))
         if (compile_fmt(config.cformat, &cformat, VSL_t_req, err) != 0)
@@ -1078,44 +1143,46 @@ FMT_Init(char *err)
         if (compile_fmt(config.rformat, &rformat, VSL_t_raw, err) != 0)
             return EINVAL;
 
-    for (int i = 0; i < MAX_VSL_TAG; i++) {
-        char tag = ctags[i] | btags[i] | rtags[i];
-        if (tag) {
-            VSB_cat(i_arg, VSL_tags[i]);
-            VSB_cat(i_arg, ",");
-        }
+    if (includes > 0) {
+        incl_arg = calloc(includes + 1, sizeof(char *));
+        if (incl_arg == NULL)
+            return ENOMEM;
+        int incl_idx = 0;
+
+        fmt_build_I_arg(cincl, &incl_idx);
+        fmt_build_I_arg(bincl, &incl_idx);
+        fmt_build_I_arg(rincl, &incl_idx);
+        assert(incl_idx == includes);
     }
-    VSB_finish(i_arg);
 
     return 0;
 }
 
-char *
-FMT_Get_i_Arg(void)
+char **
+FMT_Get_I_Args(void)
 {
-    AN(i_arg);
-    return VSB_data(i_arg);
+    return incl_arg;
 }
 
 int
 FMT_Estimate_RecsPerTx(void)
 {
     int recs_per_tx = 0, recs_per_ctx = 0, recs_per_btx = 0;
+    include_t *incl;
+
+    if (includes == 0)
+        return 0;
 
     for (int i = 0; i < MAX_VSL_TAG; i++) {
-        if (rtags[i]) {
+        if (!VSTAILQ_EMPTY(&rincl[i])) {
             recs_per_tx = 1;
             break;
         }
     }
 
     for (int i = 0; i < MAX_VSL_TAG; i++) {
-        if (ctags[i]) {
+        VSTAILQ_FOREACH(incl, &cincl[i], inclist)
             switch(i) {
-            case SLT_ReqHeader:
-            case SLT_RespHeader:
-                recs_per_ctx += config.max_headers;
-                break;
             case SLT_VCL_call:
             case SLT_VCL_return:
                 recs_per_ctx += config.max_vcl_call;
@@ -1123,35 +1190,19 @@ FMT_Estimate_RecsPerTx(void)
             case SLT_VCL_Log:
                 recs_per_ctx += config.max_vcl_log;
                 break;
-            case SLT_Timestamp:
-                recs_per_ctx += config.max_timestamp;
-                break;
             default:
                 recs_per_ctx++;
             }
-        }
     }
     if (recs_per_ctx > recs_per_tx)
         recs_per_tx = recs_per_ctx;
 
-    for (int i = 0; i < MAX_VSL_TAG; i++) {
-        if (btags[i]) {
-            switch(i) {
-            case SLT_BereqHeader:
-            case SLT_BerespHeader:
-                recs_per_btx += config.max_headers;
-                break;
-            case SLT_VCL_Log:
+    for (int i = 0; i < MAX_VSL_TAG; i++)
+        VSTAILQ_FOREACH(incl, &bincl[i], inclist)
+            if (i == SLT_VCL_Log)
                 recs_per_btx += config.max_vcl_log;
-                break;
-            case SLT_Timestamp:
-                recs_per_btx += config.max_timestamp;
-                break;
-            default:
+            else
                 recs_per_btx++;
-            }
-        }
-    }
     if (recs_per_btx > recs_per_tx)
         recs_per_tx = recs_per_btx;
 
@@ -1207,12 +1258,36 @@ free_format(compiled_fmt_t *fmt)
     free(fmt->args);
 }
 
+static void
+free_incl(includehead_t inclhead[])
+{
+    include_t *incl;
+
+    for (int i = 0; i < MAX_VSL_TAG; i++)
+        VSTAILQ_FOREACH(incl, &inclhead[i], inclist) {
+            if (incl->hdr != NULL)
+                free(incl->hdr);
+            free(incl);
+        }
+}
+
 void
 FMT_Fini(void)
 {
     free(scratch);
     VSB_delete(payload);
-    VSB_delete(i_arg);
+    VSB_delete(bintag);
+
+    if (includes > 0) {
+        for (int i = 0; i <= includes; i++)
+            if (incl_arg[i] != NULL)
+                free((void *) incl_arg[i]);
+        free(incl_arg);
+
+        free_incl(cincl);
+        free_incl(bincl);
+        free_incl(rincl);
+    }
 
     if (!EMPTY(config.cformat))
         free_format(&cformat);

src/test/test_format.c

@@ -1294,7 +1294,7 @@ static const char
 *test_FMT_interface(void)
 {
 #define NRECS 20
-    char err[BUFSIZ], *i_arg, strftime_s[BUFSIZ];
+    char err[BUFSIZ], **i_args, strftime_s[BUFSIZ];
     int status, recs_per_tx;
     tx_t tx;
     logline_t *recs[NRECS];
@@ -1323,14 +1323,23 @@ static const char
     }
 
     /* Default client format */
-    i_arg = FMT_Get_i_Arg();
-#define DEFAULT_I_ARG "ReqMethod,ReqURL,ReqProtocol,ReqHeader,RespStatus," \
-        "ReqStart,Timestamp,ReqAcct,"
-    VMASSERT(strcmp(i_arg, DEFAULT_I_ARG) == 0, "'%s' != '%s'", i_arg,
-             DEFAULT_I_ARG);
+    i_args = FMT_Get_I_Args();
+    MAN(i_args);
+    const char *exp_default_I_args[] = {
+        "ReqMethod:.", "ReqURL:.", "ReqProtocol:.",
+        "ReqHeader:^\\s*Authorization\\s*:", "ReqHeader:^\\s*Host\\s*:",
+        "ReqHeader:^\\s*Referer\\s*:", "ReqHeader:^\\s*User-agent\\s*:",
+        "RespStatus:.", "ReqStart:.", "Timestamp:^\\s*Start\\s*:", "ReqAcct:.",
+        NULL
+    };
+    for (int i = 0; i_args[i] != NULL; i++) {
+        MAN(exp_default_I_args[i]);
+        VMASSERT(strcmp(i_args[i], exp_default_I_args[i]) == 0, "'%s' != '%s'",
+                 i_args[i], exp_default_I_args[i]);
+    }
 
     recs_per_tx = FMT_Estimate_RecsPerTx();
-    MASSERT(recs_per_tx == 78);
+    MASSERT(recs_per_tx == 11);
 
     tx.type = VSL_t_req;
     add_record_data(&tx, recs[0], c[0], T1, SLT_Timestamp);
@@ -1369,15 +1378,25 @@ static const char
     status = FMT_Init(err);
     VMASSERT(status == 0, "FMT_Init: %s", err);
 
-    i_arg = FMT_Get_i_Arg();
-#define FULL_CLIENT_I_ARG "Debug,ReqMethod,ReqURL,ReqProtocol,ReqHeader,"\
-        "RespStatus,RespHeader,VCL_acl,VCL_call,VCL_return,ReqStart,VCL_Log,"\
-        "Timestamp,ReqAcct,PipeAcct,"
-    VMASSERT(strcmp(i_arg, FULL_CLIENT_I_ARG) == 0, "'%s' != '%s'", i_arg,
-             FULL_CLIENT_I_ARG);
-
+    i_args = FMT_Get_I_Args();
+    MAN(i_args);
+    const char *exp_full_client_I_args[] = {
+        "Debug:.", "ReqMethod:.", "ReqURL:.", "ReqProtocol:.",
+        "ReqHeader:^\\s*Foo\\s*:", "ReqHeader:^\\s*Host\\s*:",
+        "ReqHeader:^\\s*Authorization\\s*:", "RespStatus:.",
+        "RespHeader:^\\s*Bar\\s*:", "VCL_acl:.", "VCL_call:.", "VCL_return:.", 
+        "ReqStart:.", "VCL_Log:^\\s*baz\\s*:", "Timestamp:^\\s*Resp\\s*:",
+        "Timestamp:^\\s*Start\\s*:", "Timestamp:^\\s*Process\\s*:", "ReqAcct:.",
+        "PipeAcct:.", NULL
+    };
+    for (int i = 0; i_args[i] != NULL; i++) {
+        MAN(exp_full_client_I_args[i]);
+        VMASSERT(strcmp(i_args[i], exp_full_client_I_args[i]) == 0,
+                 "'%s' != '%s'", i_args[i], exp_full_client_I_args[i]);
+    }
+    
     recs_per_tx = FMT_Estimate_RecsPerTx();
-    MASSERT(recs_per_tx == 175);
+    MASSERT(recs_per_tx == 46);
 
     set_record_data(recs[3], c[3], URL_QUERY_PAYLOAD, SLT_ReqURL);
     set_record_data(recs[6], c[6], "Host: foobar.com", SLT_ReqHeader);
@@ -1416,15 +1435,24 @@ static const char
     status = FMT_Init(err);
     VMASSERT(status == 0, "FMT_Init: %s", err);
 
-    i_arg = FMT_Get_i_Arg();
-#define FULL_BACKEND_I_ARG "Debug,Backend,BereqMethod,BereqURL,BereqProtocol,"\
-        "BereqHeader,BerespStatus,BerespHeader,Fetch_Body,VCL_Log,Timestamp,"\
-        "BereqAcct,"
-    VMASSERT(strcmp(i_arg, FULL_BACKEND_I_ARG) == 0, "'%s' != '%s'", i_arg,
-             FULL_BACKEND_I_ARG);
+    i_args = FMT_Get_I_Args();
+    const char *exp_full_backend_I_args[] = {
+        "Debug:.", "Backend:.", "BereqMethod:.", "BereqURL:.",
+        "BereqProtocol:.", "BereqHeader:^\\s*Foo\\s*:",
+        "BereqHeader:^\\s*Host\\s*:", "BereqHeader:^\\s*Authorization\\s*:",
+        "BerespStatus:.", "BerespHeader:^\\s*Bar\\s*:", "Fetch_Body:.",
+        "VCL_Log:^\\s*baz\\s*:", "Timestamp:^\\s*BerespBody\\s*:",
+        "Timestamp:^\\s*Start\\s*:", "Timestamp:^\\s*Beresp\\s*:",
+        "BereqAcct:.", NULL
+    };
+    for (int i = 0; i_args[i] != NULL; i++) {
+        MAN(exp_full_backend_I_args[i]);
+        VMASSERT(strcmp(i_args[i], exp_full_backend_I_args[i]) == 0,
+                 "'%s' != '%s'", i_args[i], exp_full_backend_I_args[i]);
+    }
 
     recs_per_tx = FMT_Estimate_RecsPerTx();
-    MASSERT(recs_per_tx == 154);
+    MASSERT(recs_per_tx == 25);
 
     tx.type = VSL_t_bereq;
     set_record_data(recs[1], c[1], BACKEND_PAYLOAD, SLT_Backend);
@@ -1467,10 +1495,13 @@ static const char
     status = FMT_Init(err);
     VMASSERT(status == 0, "FMT_Init: %s", err);
 
-    i_arg = FMT_Get_i_Arg();
-#define FULL_RAW_I_ARG "Backend_health,Timestamp,"
-    VMASSERT(strcmp(i_arg, FULL_RAW_I_ARG) == 0, "'%s' != '%s'", i_arg,
-             FULL_RAW_I_ARG);
+    i_args = FMT_Get_I_Args();
+    const char *exp_full_raw_I_args[] = { "Backend_health:.", NULL };
+    for (int i = 0; i_args[i] != NULL; i++) {
+        MAN(exp_full_raw_I_args[i]);
+        VMASSERT(strcmp(i_args[i], exp_full_raw_I_args[i]) == 0,
+                 "'%s' != '%s'", i_args[i], exp_full_raw_I_args[i]);
+    }
 
     recs_per_tx = FMT_Estimate_RecsPerTx();
     MASSERT(recs_per_tx == 1);

src/varnishevent.c

@@ -627,13 +627,16 @@ main(int argc, char *argv[])
             LOG_Log(LOG_INFO, "Reading varnish instance %s", scratch);
     }
 
-    strcpy(scratch, FMT_Get_i_Arg());
-    if (EMPTY(scratch)) {
-        LOG_Log0(LOG_ALERT, "Not configured to read any log data, exiting");
+    char **include_args = FMT_Get_I_Args();
+    if (include_args == 0) {
+        LOG_Log0(LOG_CRIT, "Not configured to read any data, exiting");
         exit(EXIT_FAILURE);
     }
-    assert(VSL_Arg(vsl, 'i', scratch) > 0);
-    LOG_Log(LOG_INFO, "Reading SHM tags: %s", scratch);
+    assert(VSL_Arg(vsl, 'C', NULL) > 0);
+    for (int i = 0; include_args[i] != NULL; i++) {
+        assert(VSL_Arg(vsl, 'I', include_args[i]) > 0);
+        LOG_Log(LOG_INFO, "Include filter: %s", include_args[i]);
+    }
 
     if (!EMPTY(config.cformat))
         assert(VSL_Arg(vsl, 'c', scratch) > 0);
@@ -667,17 +670,6 @@ main(int argc, char *argv[])
     rdr_chunk_free = DATA_Take_Freechunk(&rdr_chunk_freelist);
     assert(!VSTAILQ_EMPTY(&rdr_chunk_freelist));
 
-    WRT_Start();
-    /* XXX: configure wrt_waits and sleep interval? */
-    int wrt_waits = 0;
-    while (!WRT_Running()) {
-        if (wrt_waits++ > 10) {
-            LOG_Log0(LOG_ALERT, "Writer thread not running, giving up");
-            exit(EXIT_FAILURE);
-        }
-        VTIM_sleep(1);
-    }
-
     for (int i = 0; i < VSL_t__MAX; i++) {
         tx_type_log[i] = 0;
         tx_type_name[i] = 'X';
@@ -692,6 +684,17 @@ main(int argc, char *argv[])
     tx_type_name[VSL_t_bereq] = 'b';
     tx_type_name[VSL_t_raw] = '-';
 
+    WRT_Start();
+    /* XXX: configure wrt_waits and sleep interval? */
+    int wrt_waits = 0;
+    while (!WRT_Running()) {
+        if (wrt_waits++ > 10) {
+            LOG_Log0(LOG_ALERT, "Writer thread not running, giving up");
+            exit(EXIT_FAILURE);
+        }
+        VTIM_sleep(1);
+    }
+
     /* Main loop */
     term = 0;
     while (!term) {

src/varnishevent.h

@@ -288,7 +288,7 @@ void MON_Output(void);
 
 /* format.c */
 int FMT_Init(char *err);
-char *FMT_Get_i_Arg(void);
+char **FMT_Get_I_Args(void);
 int FMT_Estimate_RecsPerTx(void);
 void FMT_Format(tx_t *tx, struct vsb *os);
 void FMT_Fini(void);