varnishevent - reduced memory footprint

bin/varnishevent/data.c

@@ -43,6 +43,7 @@ static const char *statename[3] = { "EMPTY", "OPEN", "DONE" };
 
 static pthread_mutex_t freelist_lock;
 static char *bufptr;
+static int ntags;
 
 #if 0
 static void
@@ -62,54 +63,84 @@ DATA_Clear_Logline(logline_t *ll)
     ll->state = DATA_EMPTY;
     ll->bitmap = 0;
     ll->spec = 0;
-    ll->rx_headers.nrec = 0;
-    ll->tx_headers.nrec = 0;
-    ll->vcl_log.nrec = 0;
-    for (int i = 0; i < MAX_VSL_TAG; i++)
+    if (ll->rx_headers)
+        ll->rx_headers->nrec = 0;
+    if (ll->tx_headers)
+        ll->tx_headers->nrec = 0;
+    if (ll->vcl_log)
+        ll->vcl_log->nrec = 0;
+    for (int i = 0; i < ntags; i++)
         ll->tag[i].len = 0;
 }
 
-#define INIT_LOG_RECORDS(fld) do {                                            \
-    (fld).record = (record_t *) calloc(config.max_headers, sizeof(record_t)); \
-    if ((fld).record == NULL)                                                 \
-        return errno;                                                         \
-    for (int j = 0; j < config.max_headers; j++) {                            \
-        (fld).record[j].magic = RECORD_MAGIC;                                 \
-        (fld).record[j].data = &bufptr[bufidx++ * config.max_reclen];         \
-    }                                                                         \
-} while (0)
+
+#define INIT_HDR_RECORDS(tag, hdr) do {                                 \
+    if (FMT_Read_Hdr(tag)) {                                            \
+        hdr = (hdr_t *) malloc(sizeof(hdr_t));                          \
+        if (hdr == NULL)                                                \
+            return errno;                                               \
+        hdr->record                                                     \
+            = (record_t *) calloc(config.max_headers, sizeof(record_t)); \
+        if (hdr->record == NULL)                                        \
+            return errno;                                               \
+        for (int j = 0; j < config.max_headers; j++) {                  \
+            hdr->record[j].magic = RECORD_MAGIC;                        \
+            hdr->record[j].data = &bufptr[bufidx++ * config.max_reclen]; \
+        }                                                               \
+    }                                                                   \
+    else {                                                              \
+        hdr = NULL;                                                     \
+    }                                                                   \
+} while(0)
 
 int
 DATA_Init(void)
 {
     int bufidx = 0;
+    int nrecords;
+
+    ntags = FMT_Get_nTags();
     
-    int records = config.max_data * (MAX_VSL_TAG  + 3 * config.max_headers);
-    
-    bufptr = (char *) calloc(records, config.max_reclen);
+    nrecords = config.max_data * ntags;
+    if (FMT_Read_Hdr(SLT_RxHeader))
+        nrecords += config.max_data * config.max_headers;
+    if (FMT_Read_Hdr(SLT_TxHeader))
+        nrecords += config.max_data * config.max_headers;
+    if (FMT_Read_Hdr(SLT_VCL_Log))
+        nrecords += config.max_data * config.max_headers;
+
+    LOG_Log(LOG_DEBUG, "Allocating space for %d records (%d bytes)", nrecords,
+            nrecords * config.max_reclen);
+    bufptr = (char *) calloc(nrecords, config.max_reclen);
     if (bufptr == NULL)
         return errno;
-
+    
     logline = (logline_t *) calloc(config.max_data, sizeof(logline_t));
     if (logline == NULL)
         return errno;
 
     VSTAILQ_INIT(&freehead);
     for (int i = 0; i < config.max_data; i++) {
-        INIT_LOG_RECORDS(logline[i].rx_headers);
-        INIT_LOG_RECORDS(logline[i].tx_headers);
-        INIT_LOG_RECORDS(logline[i].vcl_log);
-        
-        for (int j = 0; j < MAX_VSL_TAG; j++) {
+        logline[i].tag
+            = (record_t *) calloc(ntags, sizeof(record_t));
+        if (logline[i].tag == NULL)
+            return errno;
+        for (int j = 0; j < ntags; j++) {
             logline[i].tag[j].magic = RECORD_MAGIC;
             logline[i].tag[j].data = &bufptr[bufidx++ * config.max_reclen];
         }
+
+        INIT_HDR_RECORDS(SLT_RxHeader, logline[i].rx_headers);
+        INIT_HDR_RECORDS(SLT_TxHeader, logline[i].tx_headers);
+        INIT_HDR_RECORDS(SLT_VCL_Log,  logline[i].vcl_log);
         
         logline[i].magic = LOGLINE_MAGIC;
         DATA_Clear_Logline(&logline[i]);
 	VSTAILQ_INSERT_TAIL(&freehead, &logline[i], freelist);
     }
 
+    assert(bufidx == nrecords);
+
     AZ(pthread_mutex_init(&freelist_lock, &attr_lock));
     data_open = data_done = data_occ_hi = global_nfree = 0;
     
@@ -145,13 +176,15 @@ DATA_Return_Freelist(struct freehead_s *returned, unsigned nreturned)
     AZ(pthread_mutex_unlock(&freelist_lock));
 }
 
-#define DUMP_HDRS(vsb, ll, hdr) do {                                    \
-for (j = 0; j < ll->hdr.nrec; j++)                                      \
-    if (ll->hdr.record[j].len) {                                        \
-        VSB_putc(vsb, '[');                                             \
-        VSB_bcat(vsb, ll->hdr.record[j].data, ll->hdr.record[j].len);   \
-        VSB_cat(vsb, "] ");                                             \
-    }                                                                   \
+#define DUMP_HDRS(vsb, ll, hdr) do {                    \
+    if (ll->hdr)                                        \
+        for (j = 0; j < ll->hdr->nrec; j++)             \
+            if (ll->hdr->record[j].len) {               \
+                VSB_putc(vsb, '[');                     \
+                VSB_bcat(vsb, ll->hdr->record[j].data,  \
+                    ll->hdr->record[j].len);            \
+                VSB_cat(vsb, "] ");                     \
+            }                                           \
  } while (0)
 
 void
@@ -175,9 +208,9 @@ DATA_Dump(void)
             i, statename[ll->state],
             C(ll->spec) ? 'c' : B(ll->spec) ? 'b' : '-');
         
-        for (j = 0; j < MAX_VSL_TAG; j++)
+        for (j = 0; j < ntags; j++)
             if (ll->tag[j].len) {
-                VSB_cat(data, VSL_tags[j]);
+                VSB_cat(data, VSL_tags[idx2tag[j]]);
                 VSB_cat(data, "=[");
                 VSB_bcat(data, ll->tag[j].data, ll->tag[j].len);
                 VSB_cat(data, "] ");

bin/varnishevent/varnishevent.c

@@ -244,11 +244,11 @@ collect(struct logline_t *lp, enum VSL_tag_e tag, unsigned spec,
     }
 
     if (tag == SLT_RxHeader)
-        hdr = &lp->rx_headers;
+        hdr = lp->rx_headers;
     else if (tag == SLT_TxHeader)
-        hdr = &lp->tx_headers;
+        hdr = lp->tx_headers;
     else if (tag == SLT_VCL_Log)
-        hdr = &lp->vcl_log;
+        hdr = lp->vcl_log;
 
     if (hdr != NULL) {
         if (hdr->nrec < config.max_headers)
@@ -262,7 +262,7 @@ collect(struct logline_t *lp, enum VSL_tag_e tag, unsigned spec,
         }
     }
     else
-        rec = &lp->tag[tag];
+        rec = &lp->tag[tag2idx[tag]];
     
     memcpy(rec->data, ptr, len);
     rec->len = len;
@@ -296,10 +296,11 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
         assert(lp->state == DATA_EMPTY);
         AZ(lp->bitmap);
         AZ(lp->spec);
-        AZ(lp->rx_headers.nrec);
-        AZ(lp->tx_headers.nrec);
-        AZ(lp->vcl_log.nrec);
+        assert(lp->rx_headers == NULL || lp->rx_headers->nrec == 0);
+        assert(lp->tx_headers == NULL || lp->tx_headers->nrec == 0);
+        assert(lp->vcl_log == NULL || lp->vcl_log->nrec == 0);
         /* XXX: assert that all lp->tag[i].len == 0 ? */
+
         fd_tbl[fd].ll = lp;
         lp->state = DATA_OPEN;
         data_open++;
@@ -455,7 +456,7 @@ main(int argc, char *argv[])
             w_arg = optarg;
             break;
         case 'g':
-            D_flag = 1;
+            g_flag = 1;
             break;
         case 'G':
             strcpy(cli_config_filename, optarg);
@@ -582,8 +583,7 @@ main(int argc, char *argv[])
     else
         LOG_Log(LOG_INFO, "Reading varnish instance %s", scratch);
 
-    scratch[0] = '\0';
-    FMT_Get_i_Arg(scratch);
+    strcpy(scratch, FMT_Get_i_Arg());
     assert(VSL_Arg(vd, 'i', scratch) > 0);
     LOG_Log(LOG_INFO, "Reading SHM tags: %s", scratch);
     

bin/varnishevent/varnishevent.h

@@ -49,6 +49,8 @@
 #define DEFAULT_MAX_DATA 4096
 #define DEFAULT_PID_FILE "/var/run/varnishevent.pid"
 
+#define MAX_VSL_TAG 256
+
 #if 0
 #define DEFAULT_CFORMAT \
     "%h %l %u %t \"%r\" %s %b \"%{Referer}i\" \"%{User-agent}i\""
@@ -61,8 +63,6 @@
 #define ALT_CFORMAT \
   "%{X-Forwarded-For}i %l %t %U %s %b \"%{Referer}i\" \"%{User-agent}i\""
 
-#define MAX_VSL_TAG 256
-
 typedef enum {
     DATA_EMPTY = 0,
     DATA_OPEN,
@@ -84,13 +84,13 @@ typedef struct {
 typedef struct logline_t {
     unsigned magic;
 #define LOGLINE_MAGIC 0xf427a374
-    record_t tag[MAX_VSL_TAG];
+    record_t *tag;
     data_state_e state;
     uint64_t bitmap;	/* Bitmap for regex matches */
     unsigned spec;	/* 'b'/'c' */
-    hdr_t rx_headers; 	/* Received headers */
-    hdr_t tx_headers; 	/* Sent headers */
-    hdr_t vcl_log; 	/* VCL_Log entries */
+    hdr_t *rx_headers; 	/* Received headers */
+    hdr_t *tx_headers; 	/* Sent headers */
+    hdr_t *vcl_log; 	/* VCL_Log entries */
     VSTAILQ_ENTRY(logline_t) freelist;
     VSTAILQ_ENTRY(logline_t) spscq;
 } logline_t;
@@ -101,6 +101,9 @@ unsigned data_open;
 unsigned data_done;
 unsigned data_occ_hi;
 
+int tag2idx[MAX_VSL_TAG];
+enum VSL_tag_e idx2tag[MAX_VSL_TAG];
+
 VSTAILQ_HEAD(freehead_s, logline_t);
 
 struct freehead_s freehead;
@@ -231,6 +234,8 @@ void MON_StatsUpdate(stats_update_t update);
 
 /* format.c */
 int FMT_Init(char *err);
-void FMT_Get_i_Arg(char *i_arg);
+char *FMT_Get_i_Arg(void);
+int FMT_Get_nTags(void);
+int FMT_Read_Hdr(enum VSL_tag_e tag);
 void FMT_Format(logline_t *ll, struct vsb *os);
 void FMT_Shutdown(void);