reader thread uses -i filters for tags whose payloads do not need

to be matched against a regex

src/format.c

@@ -53,7 +53,7 @@ typedef struct compiled_fmt_t {
 } compiled_fmt_t;
 
 static struct vsb payload_storage, * const payload = &payload_storage,
-    bintag_storage, * const bintag = &bintag_storage;
+    bintag_storage, * const bintag = &bintag_storage, *i_arg;
 static char *scratch = NULL;
 
 static char empty[] = "";
@@ -73,7 +73,7 @@ typedef VSTAILQ_HEAD(includehead_s, include_t) includehead_t;
 
 static compiled_fmt_t cformat, bformat, rformat;
 static includehead_t cincl[MAX_VSL_TAG], bincl[MAX_VSL_TAG], rincl[MAX_VSL_TAG];
-static unsigned includes;
+static unsigned includes, include_rx;
 static char **incl_arg = NULL;
 
 char *
@@ -811,8 +811,10 @@ add_tag(enum VSL_transaction_e type, enum VSL_tag_e tag, const char *hdr)
 
     incl = calloc(1, sizeof(include_t));
     AN(incl);
-    if (hdr != NULL)
+    if (hdr != NULL) {
         incl->hdr = strdup(hdr);
+        include_rx++;
+    }
     VSTAILQ_INSERT_TAIL(inclhead, incl, inclist);
     includes++;
 }
@@ -1172,7 +1174,7 @@ compile_fmt(char * const format, compiled_fmt_t * const fmt,
 }
 
 static void
-fmt_build_I_arg(const includehead_t *inclhead, int *incl_idx)
+fmt_build_include_args(const includehead_t *inclhead, int *incl_idx)
 {
     AN(incl_arg);
     AN(includes);
@@ -1183,12 +1185,15 @@ fmt_build_I_arg(const includehead_t *inclhead, int *incl_idx)
 
         VSTAILQ_FOREACH(incl, &inclhead[i], inclist) {
             assert(*incl_idx < includes);
-            if (incl->hdr == NULL)
-                sprintf(scratch, "%s:.", VSL_tags[i]);
-            else
+            if (incl->hdr == NULL) {
+                VSB_cat(i_arg, VSL_tags[i]);
+                VSB_cat(i_arg, ",");
+            }
+            else {
                 sprintf(scratch, "%s:^\\s*%s\\s*:", VSL_tags[i], incl->hdr);
-            incl_arg[*incl_idx] = strdup(scratch);
-            *incl_idx += 1;
+                incl_arg[*incl_idx] = strdup(scratch);
+                *incl_idx += 1;
+            }
         }
     }
 }
@@ -1202,8 +1207,11 @@ FMT_Init(char *err)
 
     AN(VSB_new(payload, NULL, config.max_reclen + 1, VSB_FIXEDLEN));
     AN(VSB_new(bintag, NULL, config.max_reclen + 1, VSB_FIXEDLEN));
+    i_arg = VSB_new_auto();
+    AN(i_arg);
 
     includes = 0;
+    include_rx = 0;
     for (int i = 0; i < MAX_VSL_TAG; i++) {
         VSTAILQ_INIT(&cincl[i]);
         VSTAILQ_INIT(&bincl[i]);
@@ -1223,15 +1231,16 @@ FMT_Init(char *err)
             return EINVAL;
 
     if (includes > 0) {
-        incl_arg = calloc(includes + 1, sizeof(char *));
+        incl_arg = calloc(include_rx + 1, sizeof(char *));
         if (incl_arg == NULL)
             return ENOMEM;
         int incl_idx = 0;
 
-        fmt_build_I_arg(cincl, &incl_idx);
-        fmt_build_I_arg(bincl, &incl_idx);
-        fmt_build_I_arg(rincl, &incl_idx);
-        assert(incl_idx == includes);
+        fmt_build_include_args(cincl, &incl_idx);
+        fmt_build_include_args(bincl, &incl_idx);
+        fmt_build_include_args(rincl, &incl_idx);
+        assert(incl_idx == include_rx);
+        VSB_finish(i_arg);
     }
 
     return 0;
@@ -1243,6 +1252,13 @@ FMT_Get_I_Args(void)
     return incl_arg;
 }
 
+char *
+FMT_Get_i_Arg(void)
+{
+    assert(VSB_done(i_arg));
+    return VSB_data(i_arg);
+}
+
 int
 FMT_Estimate_RecsPerTx(void)
 {
@@ -1356,13 +1372,16 @@ FMT_Fini(void)
     free(scratch);
     VSB_delete(payload);
     VSB_delete(bintag);
+    VSB_delete(i_arg);
 
-    if (includes > 0) {
-        for (int i = 0; i <= includes; i++)
+    if (include_rx > 0) {
+        for (int i = 0; i <= include_rx; i++)
             if (incl_arg[i] != NULL)
                 free((void *) incl_arg[i]);
         free(incl_arg);
+    }
 
+    if (includes > 0) {
         free_incl(cincl);
         free_incl(bincl);
         free_incl(rincl);

src/test/regress.sh

@@ -21,9 +21,9 @@ fi
 # grep removes logs about table allocations and by the threads about
 # free lists, which are not relevant to the regression, and are not
 # predictable from one run to the next.
-CKSUM=$( sed -e 's/\(initializing\) \(.*\)/\1/' $LOG | egrep -v 'Writer: returned|Reader: took|Allocating table' | cksum )
+CKSUM=$( sed -e 's/\(initializing\) \(.*\)/\1/' $LOG | egrep -v 'Writer: returned|Reader: took|^DEBUG: Allocating' | cksum )
 
-if [ "$CKSUM" != '4104483911 69264089' ]; then
+if [ "$CKSUM" != '1684260647 69263259' ]; then
     echo "ERROR: Regression test varnishevent log incorrect cksum: $CKSUM"
     exit 1
 fi

src/test/test_format.c

@@ -1388,7 +1388,7 @@ static const char
 *test_FMT_interface(void)
 {
 #define NRECS 20
-    char err[BUFSIZ], **i_args, strftime_s[BUFSIZ];
+    char err[BUFSIZ], **i_args, *i_arg, strftime_s[BUFSIZ];
     int status, recs_per_tx;
     tx_t tx;
     logline_t *recs[NRECS];
@@ -1422,11 +1422,9 @@ static const char
     i_args = FMT_Get_I_Args();
     MAN(i_args);
     const char *exp_default_I_args[] = {
-        "ReqMethod:.", "ReqURL:.", "ReqProtocol:.",
         "ReqHeader:^\\s*Authorization\\s*:", "ReqHeader:^\\s*Host\\s*:",
         "ReqHeader:^\\s*Referer\\s*:", "ReqHeader:^\\s*User-agent\\s*:",
-        "RespStatus:.", "ReqStart:.", "Timestamp:^\\s*Start\\s*:", "ReqAcct:.",
-        NULL
+        "Timestamp:^\\s*Start\\s*:", NULL
     };
     for (int i = 0; i_args[i] != NULL; i++) {
         MAN(exp_default_I_args[i]);
@@ -1434,6 +1432,13 @@ static const char
                  i_args[i], exp_default_I_args[i]);
     }
 
+    i_arg = FMT_Get_i_Arg();
+    MAN(i_arg);
+#define DEFAULT_I_TAGS "ReqMethod,ReqURL,ReqProtocol,RespStatus,ReqStart,"\
+        "ReqAcct,"
+    VMASSERT(strcmp(i_arg, DEFAULT_I_TAGS) == 0,
+             "Default -i arg expected '%s' != '%s'", DEFAULT_I_TAGS, i_arg);
+
     recs_per_tx = FMT_Estimate_RecsPerTx();
     MASSERT(recs_per_tx == 11);
 
@@ -1478,13 +1483,11 @@ static const char
     i_args = FMT_Get_I_Args();
     MAN(i_args);
     const char *exp_full_client_I_args[] = {
-        "Debug:.", "ReqMethod:.", "ReqURL:.", "ReqProtocol:.",
         "ReqHeader:^\\s*Foo\\s*:", "ReqHeader:^\\s*Host\\s*:",
-        "ReqHeader:^\\s*Authorization\\s*:", "RespStatus:.",
-        "RespHeader:^\\s*Bar\\s*:", "VCL_acl:.", "VCL_call:.", "VCL_return:.", 
-        "ReqStart:.", "VCL_Log:^\\s*baz\\s*:", "Timestamp:^\\s*Resp\\s*:",
+        "ReqHeader:^\\s*Authorization\\s*:", "RespHeader:^\\s*Bar\\s*:",
+        "VCL_Log:^\\s*baz\\s*:", "Timestamp:^\\s*Resp\\s*:",
         "Timestamp:^\\s*Start\\s*:", "Timestamp:^\\s*Process\\s*:",
-        "Timestamp:^\\s*Req\\s*:", "ReqAcct:.", "PipeAcct:.", NULL
+        "Timestamp:^\\s*Req\\s*:", NULL
     };
     for (int i = 0; i_args[i] != NULL; i++) {
         MAN(exp_full_client_I_args[i]);
@@ -1492,6 +1495,14 @@ static const char
                  "'%s' != '%s'", i_args[i], exp_full_client_I_args[i]);
     }
     
+    i_arg = FMT_Get_i_Arg();
+    MAN(i_arg);
+#define FULL_CLIENT_I_TAGS "Debug,ReqMethod,ReqURL,ReqProtocol,RespStatus,"\
+        "VCL_acl,VCL_call,VCL_return,ReqStart,ReqAcct,PipeAcct,"
+    VMASSERT(strcmp(i_arg, FULL_CLIENT_I_TAGS) == 0,
+             "Full client -i arg expected '%s' != '%s'", FULL_CLIENT_I_TAGS,
+             i_arg);
+
     recs_per_tx = FMT_Estimate_RecsPerTx();
     VMASSERT(recs_per_tx == 47, "recs_per_tx(%d) != 47", recs_per_tx);
 
@@ -1538,13 +1549,11 @@ static const char
 
     i_args = FMT_Get_I_Args();
     const char *exp_full_backend_I_args[] = {
-        "Debug:.", "Backend:.", "BereqMethod:.", "BereqURL:.",
-        "BereqProtocol:.", "BereqHeader:^\\s*Foo\\s*:",
-        "BereqHeader:^\\s*Host\\s*:", "BereqHeader:^\\s*Authorization\\s*:",
-        "BerespStatus:.", "BerespHeader:^\\s*Bar\\s*:", "Fetch_Body:.",
+        "BereqHeader:^\\s*Foo\\s*:", "BereqHeader:^\\s*Host\\s*:",
+        "BereqHeader:^\\s*Authorization\\s*:", "BerespHeader:^\\s*Bar\\s*:",
         "VCL_Log:^\\s*baz\\s*:", "Timestamp:^\\s*BerespBody\\s*:",
         "Timestamp:^\\s*Start\\s*:", "Timestamp:^\\s*Beresp\\s*:",
-        "Timestamp:^\\s*Bereq\\s*:", "BereqAcct:.", NULL
+        "Timestamp:^\\s*Bereq\\s*:", NULL
     };
     for (int i = 0; i_args[i] != NULL; i++) {
         MAN(exp_full_backend_I_args[i]);
@@ -1552,6 +1561,14 @@ static const char
                  "'%s' != '%s'", i_args[i], exp_full_backend_I_args[i]);
     }
 
+    i_arg = FMT_Get_i_Arg();
+    MAN(i_arg);
+#define FULL_BACKEND_I_TAGS "Debug,Backend,BereqMethod,BereqURL,BereqProtocol,"\
+        "BerespStatus,Fetch_Body,BereqAcct,"
+    VMASSERT(strcmp(i_arg, FULL_BACKEND_I_TAGS) == 0,
+             "Full backend -i arg expected '%s' != '%s'", FULL_BACKEND_I_TAGS,
+             i_arg);
+
     recs_per_tx = FMT_Estimate_RecsPerTx();
     MASSERT(recs_per_tx == 26);
 
@@ -1600,12 +1617,14 @@ static const char
     VMASSERT(status == 0, "FMT_Init: %s", err);
 
     i_args = FMT_Get_I_Args();
-    const char *exp_full_raw_I_args[] = { "Backend_health:.", NULL };
-    for (int i = 0; i_args[i] != NULL; i++) {
-        MAN(exp_full_raw_I_args[i]);
-        VMASSERT(strcmp(i_args[i], exp_full_raw_I_args[i]) == 0,
-                 "'%s' != '%s'", i_args[i], exp_full_raw_I_args[i]);
-    }
+    MAZ(i_args[0]);
+
+    i_arg = FMT_Get_i_Arg();
+    MAN(i_arg);
+#define FULL_RAW_I_TAGS "Backend_health,"
+    VMASSERT(strcmp(i_arg, FULL_RAW_I_TAGS) == 0,
+             "Full raw -i arg expected '%s' != '%s'", FULL_RAW_I_TAGS,
+             i_arg);
 
     recs_per_tx = FMT_Estimate_RecsPerTx();
     MASSERT(recs_per_tx == 1);

src/varnishevent.c

@@ -702,14 +702,17 @@ main(int argc, char *argv[])
     }
 
     char **include_args = FMT_Get_I_Args();
-    if (include_args == 0) {
-        LOG_Log0(LOG_CRIT, "Not configured to read any data, exiting");
-        exit(EXIT_FAILURE);
+    if (include_args != NULL) {
+        assert(VSL_Arg(vsl, 'C', NULL) > 0);
+        for (int i = 0; include_args[i] != NULL; i++) {
+            LOG_Log(LOG_INFO, "Include filter: %s", include_args[i]);
+            assert(VSL_Arg(vsl, 'I', include_args[i]) > 0);
+        }
     }
-    assert(VSL_Arg(vsl, 'C', NULL) > 0);
-    for (int i = 0; include_args[i] != NULL; i++) {
-        assert(VSL_Arg(vsl, 'I', include_args[i]) > 0);
-        LOG_Log(LOG_INFO, "Include filter: %s", include_args[i]);
+    strcpy(scratch, FMT_Get_i_Arg());
+    if (!EMPTY(scratch)) {
+        LOG_Log(LOG_INFO, "Include tags: %s", scratch);
+        assert(VSL_Arg(vsl, 'i', scratch) > 0);
     }
 
     if (!EMPTY(config.cformat) && EMPTY(config.bformat))

src/varnishevent.h

@@ -256,6 +256,7 @@ void MON_Output(void);
 /* format.c */
 int FMT_Init(char *err);
 char **FMT_Get_I_Args(void);
+char *FMT_Get_i_Arg(void);
 int FMT_Estimate_RecsPerTx(void);
 void FMT_Format(tx_t *tx, struct vsb *os);
 void FMT_Fini(void);