begin re-working format.c, most of it currently #if'd out, but with

initial implementations of base functions for accessing data, and
first tests

.gitignore

@@ -27,6 +27,7 @@ src/test/.deps/
 *~
 src/test/test_data
 src/test/test_strfTIM
+src/test/test_format
 src/varnishevent
 *.1
 stamp-h1

src/Makefile.am

@@ -21,9 +21,9 @@ varnishevent_SOURCES = \
 	log.c \
 	monitor.c \
 	strfTIM.h \
-	strfTIM.c
+	strfTIM.c \
+	format.c
 
-#	format.c \
 #	handler.c
 
 varnishevent_LDADD = \

src/format.c

@@ -33,13 +33,13 @@
 #include <stdlib.h>
 #include <errno.h>
 
-#include "varnishevent.h"
-
 #include "vas.h"
 #include "miniobj.h"
 #include "base64.h"
+#include "vre.h"
 
-#define TAG(ll,t) ((ll)->tag[tag2idx[(t)]])
+#include "varnishevent.h"
+#include "format.h"
 
 typedef void formatter_f(logline_t *ll, char *name, enum VSL_tag_e tag,
     char **s, size_t *len);
@@ -57,6 +57,8 @@ typedef struct compiled_fmt_t {
     char tags[MAX_VSL_TAG];
 } compiled_fmt_t;
 
+#if 0
+
 static compiled_fmt_t cformat, bformat, zformat;
 
 static char i_arg[BUFSIZ] = "";
@@ -71,82 +73,124 @@ static char miss[5];
 static char pass[5];
 static char dash[2];
 
-static inline record_t *
-get_hdr(const char *hdr, hdr_t *tbl)
+#endif
+
+void
+get_payload(logline_t *rec)
 {
-    int l = strlen(hdr);
-    
-    for (int i = 0; i < tbl->nrec; i++)
-        if (strncasecmp(hdr, tbl->record[i].data, l) == 0
-            && tbl->record[i].data[l] == ':'
-            && tbl->record[i].data[l+1] == ' ')
-            return &tbl->record[i];
-    return NULL;
+    CHECK_OBJ_NOTNULL(rec, LOGLINE_MAGIC);
+
+    VSB_clear(payload);
+    if (rec->len) {
+        int n = rec->len;
+        chunk_t *chunk = VSTAILQ_FIRST(&rec->chunks);
+        while (n > 0 && chunk != NULL) {
+            CHECK_OBJ(chunk, CHUNK_MAGIC);
+            int cp = n;
+            if (cp > config.chunk_size)
+                cp = config.chunk_size;
+            VSB_bcat(payload, chunk->data, cp);
+            n -= cp;
+            chunk = VSTAILQ_NEXT(chunk, chunklist);
+        }
+    }
+    VSB_finish(payload);
 }
 
-#define GET_HDR(ll, dir, hdr) get_hdr((hdr), (ll)-> dir## _headers)
+/*
+ * Return the *last* record in tx that matches the tag
+ */
+logline_t *
+get_tag(tx_t *tx, enum VSL_tag_e tag)
+{
+    logline_t *rec, *tagrec = NULL;
 
-static inline char *
-get_fld(record_t *rec, int n)
+    CHECK_OBJ_NOTNULL(tx, TX_MAGIC);
+    VSTAILQ_FOREACH(rec, &tx->lines, linelist) {
+        CHECK_OBJ_NOTNULL(rec, LOGLINE_MAGIC);
+        if (rec->tag == tag)
+            tagrec = rec;
+    }
+    return tagrec;
+}
+
+/*
+ * hdr_re is a pre-compiled regex of the form "^\s*%s\*s:\s*(.+)$",
+ * formatted with the header name in place of %s.
+ * Return the captured substring (the header payload) of the *last* record
+ * in tx that matches the tag and the regex.
+ */
+char *
+get_hdr(tx_t *tx, enum VSL_tag_e tag, vre_t *hdr_re)
 {
-    char *ret = NULL, *s;
+    logline_t *rec;
+#define OV_SIZE (2 * 3)
+    int ov[OV_SIZE];
+    char *hdr_payload = NULL;
+
+    CHECK_OBJ_NOTNULL(tx, TX_MAGIC);
+    VSTAILQ_FOREACH(rec, &tx->lines, linelist) {
+        int s;
+
+        CHECK_OBJ_NOTNULL(rec, LOGLINE_MAGIC);
+        if (rec->tag != tag)
+            continue;
+        get_payload(rec);
+        s = VRE_exec(hdr_re, VSB_data(payload), rec->len, 0, VRE_CASELESS,
+                     ov, OV_SIZE, NULL);
+        assert(s >= VRE_ERROR_NOMATCH && s != 0);
+        if (s == VRE_ERROR_NOMATCH)
+            continue;
+        assert(ov[2] >= 0 && ov[3] >= ov[2]);
+        hdr_payload = VSB_data(payload) + ov[2];
+        hdr_payload[ov[3]] = '\0';
+    }
+
+    return hdr_payload;
+}
+
+char *
+get_fld(char *str, int n)
+{
+    char *fld = NULL, *s;
     int i = 0;
 
-    AN(scratch);
-    strncpy(scratch, rec->data, rec->len);
-    s = scratch;
+    s = str;
     do {
-        ret = strtok(s, " \t");
+        fld = strtok(s, " \t");
         s = NULL;
-    } while (++i < n && ret != NULL);
+    } while (++i < n && fld != NULL);
     
-    return ret;
+    return fld;
 }
 
-static inline int
-get_tm(tx_t *ll, struct tm * t) {
-    char *ts = NULL;
-    record_t *date_rec = NULL;
-    time_t tt = 0;
-            
-    if (C(ll->spec)) {
-        if (TAG(ll,SLT_ReqEnd).len
-            && (ts = get_fld(&TAG(ll,SLT_ReqEnd), 2)) != NULL)
-                tt = (time_t) atol(ts);
-        else
-            date_rec = GET_HDR(ll, rx, "Date");
-    }
-    else if (B(ll->spec)) {
-        if ((date_rec = GET_HDR(ll, rx, "Date")) == NULL)
-            tt = (time_t) ll->t;
-    }
-    else
-        tt = (time_t) ll->t;
-    
-    if (date_rec)
-        tt = TIM_parse(&date_rec->data[strlen("Date: ")]);
-    if (tt != 0) {
-        localtime_r(&tt, t);
-        return 1;
-    }
-    else
-        return 0;
+char *
+get_rec_fld(logline_t *rec, int n)
+{
+    get_payload(rec);
+    return get_fld(VSB_data(payload), n);
 }
 
-#define RETURN_REC(rec, s, len) do {            \
-    (*s) = (rec).data;                          \
-    (*(len)) = (rec).len;                       \
-} while (0)
+double
+get_tm(tx_t *tx)
+{
+    char *ts, *epochstr;
+    double epocht = 0;
 
-#define RETURN_HDR(rec, hdr, s, len) do {	\
-    (*s) = &((rec)->data[strlen(hdr)+2]);	\
-    (*(len)) = (rec)->len - (strlen(hdr)+2);	\
-} while (0)
+    CHECK_OBJ_NOTNULL(tx, TX_MAGIC);
 
-#define RETURN_FLD(rec, fld, s, len) do {	\
-    (*s) = get_fld(&(rec), fld);		\
-    (*(len)) = strlen(*s);			\
-} while (0)
+    ts = get_hdr(tx, SLT_Timestamp, time_start_re);
+    if (ts != NULL && (epochstr = get_fld(ts, 0)) != NULL) {
+        char *p;
+        epocht = strtod(epochstr, &p);
+    }
+    if (epocht == 0)
+        epocht = tx->t;
+
+    return epocht;
+}
+
+#if 0
 
 #define FORMAT(dir, ltr, slt) 						\
 static void								\
@@ -944,3 +988,5 @@ FMT_Shutdown(void)
     if (!EMPTY(config.zformat))
         free_format(&zformat);
 }
+
+#endif

src/format.h

+/*-
+ * Copyright (c) 2015 UPLEX Nils Goroll Systemoptimierung
+ * Copyright (c) 2015 Otto Gmbh & Co KG
+ * All rights reserved.
+ * Use only with permission
+ *
+ * Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/* including source must include varnishevent.h and vre.h */
+
+/* XXX: init as fixed size with length shm_reclen */
+struct vsb *payload;
+
+/* XXX: init time_start_re as VRE_compile(TS_START_REGEX) */
+#define TS_START_REGEX "^\\s*Start\\s*:\\s*(.+)$"
+vre_t *time_start_re;
+
+void get_payload(logline_t *rec);
+logline_t *get_tag(tx_t *tx, enum VSL_tag_e tag);
+char *get_hdr(tx_t *tx, enum VSL_tag_e tag, vre_t *hdr_re);
+char *get_fld(char *str, int n);
+char *get_rec_fld(logline_t *rec, int n);
+double get_tm(tx_t *tx);

src/test/Makefile.am

 AM_CPPFLAGS = @VARNISH_CFLAGS@ -I ${ac_varnish_pkgdataincludedir}
 
-TESTS = test_data test_strfTIM # test_writer regress.sh ncsa.sh vslarg.sh
+TESTS = test_data test_strfTIM test_format # test_writer regress.sh ncsa.sh vslarg.sh
 
-check_PROGRAMS = test_data test_strfTIM # test_writer
+check_PROGRAMS = test_data test_strfTIM test_format # test_writer
 
 test_data_SOURCES = \
 	minunit.h \
@@ -15,6 +15,15 @@ test_data_LDADD = \
 	../data.$(OBJEXT) \
 	@VARNISH_LIBS@ @VARNISH_LIBVARNISH_LIB@ -lm -lvarnish
 
+test_format_SOURCES = \
+	minunit.h \
+	test_format.c
+
+test_format_LDADD = \
+	../config.$(OBJEXT) \
+	../format.$(OBJEXT) \
+	@VARNISH_LIBS@ @VARNISH_LIBVARNISH_LIB@ -lm -lvarnish
+
 test_writer_SOURCES = \
 	minunit.h \
 	../writer.c

src/test/test_format.c

+/*-
+ * Copyright (c) 2015 UPLEX Nils Goroll Systemoptimierung
+ * Copyright (c) 2015 Otto Gmbh & Co KG
+ * All rights reserved
+ * Use only with permission
+ *
+ * Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <string.h>
+
+#include "vre.h"
+#include "minunit.h"
+
+#include "../varnishevent.h"
+#include "../format.h"
+
+int tests_run = 0;
+
+/* N.B.: Always run the tests in this order */
+static const char
+*test_format_init(void)
+{
+    const char *error;
+    int erroroffset;
+
+    printf("... initializing format tests\n");
+
+    CONF_Init();
+
+    payload = VSB_new(NULL, NULL, DEFAULT_MAX_RECLEN, VSB_FIXEDLEN);
+    MAN(payload);
+
+    time_start_re = VRE_compile(TS_START_REGEX, VRE_CASELESS, &error,
+                                &erroroffset);
+    VMASSERT(time_start_re != NULL,
+             "Error compiling " TS_START_REGEX ": %s (offset %d)",
+             error, erroroffset);
+
+    return NULL;
+}
+
+static const char
+*test_format_get_payload(void)
+{
+    logline_t rec;
+    chunk_t chunk;
+
+    printf("... testing get_payload()\n");
+
+    rec.magic = LOGLINE_MAGIC;
+    VSTAILQ_INIT(&rec.chunks);
+    chunk.magic = CHUNK_MAGIC;
+    chunk.data = (char *) calloc(1, DEFAULT_CHUNK_SIZE);
+    MAN(chunk.data);
+
+    /* Record with one chunk */
+#define SHORT_STRING "foo bar baz quux"
+    rec.len = strlen(SHORT_STRING);
+    sprintf(chunk.data, "%s", SHORT_STRING);
+    VSTAILQ_INSERT_TAIL(&rec.chunks, &chunk, chunklist);
+    get_payload(&rec);
+    MASSERT(strcmp(VSB_data(payload), SHORT_STRING) == 0);
+
+    return NULL;
+}
+
+static const char
+*all_tests(void)
+{
+    mu_run_test(test_format_init);
+    mu_run_test(test_format_get_payload);
+    return NULL;
+}
+
+TEST_RUNNER

src/varnishevent.h

@@ -121,6 +121,7 @@ typedef VSTAILQ_HEAD(linehead_s, logline_t) linehead_t;
 typedef struct tx_t {
     unsigned magic;
 #define TX_MAGIC 0xff463e42
+    double t;
     tx_state_e state;
     int32_t vxid;
     enum VSL_transaction_e type;