Commit a69a4882 authored by Tollef Fog Heen's avatar Tollef Fog Heen

Implement support for multiple matches in varnish tools

Make -o tag:regex only act on transactions where the regex matches
tag, generalised from what varnishlog and varnishncsa already had
support for.  In addition, support multiple -o options which are then
and-ed together.
parent a15f37d3
......@@ -65,6 +65,7 @@ static unsigned next_hist;
static unsigned bucket_miss[HIST_BUCKETS];
static unsigned bucket_hit[HIST_BUCKETS];
static unsigned char hh[FD_SETSIZE];
static uint64_t bitmap[FD_SETSIZE];
static double log_ten;
......@@ -146,12 +147,12 @@ update(struct VSM_data *vd)
static int
h_hist(void *priv, enum vsl_tag tag, unsigned fd, unsigned len,
unsigned spec, const char *ptr)
unsigned spec, const char *ptr, uint64_t bm)
{
double b;
int i, j;
struct VSM_data *vd = priv;
(void)priv;
(void)len;
(void)spec;
......@@ -159,6 +160,8 @@ h_hist(void *priv, enum vsl_tag tag, unsigned fd, unsigned len,
/* oops */
return (0);
bitmap[fd] |= bm;
if (tag == SLT_Hit) {
hh[fd] = 1;
return (0);
......@@ -166,6 +169,12 @@ h_hist(void *priv, enum vsl_tag tag, unsigned fd, unsigned len,
if (tag != SLT_ReqEnd)
return (0);
if (!VSL_Matched(vd, bitmap[fd])) {
bitmap[fd] = 0;
hh[fd] = 0;
return (0);
}
/* determine processing time */
#if 1
i = sscanf(ptr, "%*d %*f %*f %*f %lf", &b);
......@@ -212,6 +221,7 @@ h_hist(void *priv, enum vsl_tag tag, unsigned fd, unsigned len,
next_hist = 0;
}
hh[fd] = 0;
bitmap[fd] = 0;
pthread_mutex_unlock(&mtx);
......@@ -225,7 +235,7 @@ accumulate_thread(void *arg)
int i;
for (;;) {
i = VSL_Dispatch(vd, h_hist, NULL);
i = VSL_Dispatch(vd, h_hist, vd);
if (i < 0)
break;
if (i == 0)
......
......@@ -51,47 +51,28 @@
static int b_flag, c_flag;
/* -------------------------------------------------------------------*/
static int
name2tag(const char *n)
{
int i;
for (i = 0; i < 256; i++) {
if (VSL_tags[i] == NULL)
continue;
if (!strcasecmp(n, VSL_tags[i]))
return (i);
}
return (-1);
}
/* Ordering-----------------------------------------------------------*/
static struct vsb *ob[65536];
static unsigned char flg[65536];
static enum vsl_tag last[65536];
static uint64_t bitmap[65536];
#define F_INVCL (1 << 0)
#define F_MATCH (1 << 1)
static int match_tag = -1;
static const vre_t *match_re;
static void
h_order_finish(int fd)
h_order_finish(int fd, struct VSM_data *vd)
{
AZ(vsb_finish(ob[fd]));
if (vsb_len(ob[fd]) > 1 &&
(match_tag == -1 || flg[fd] & F_MATCH))
if (vsb_len(ob[fd]) > 1 && VSL_Matched(vd, bitmap[fd])) {
printf("%s\n", vsb_data(ob[fd]));
flg[fd] &= ~F_MATCH;
}
bitmap[fd] = 0;
vsb_clear(ob[fd]);
}
static void
clean_order(void)
clean_order(struct VSM_data *vd)
{
unsigned u;
......@@ -99,38 +80,37 @@ clean_order(void)
if (ob[u] == NULL)
continue;
AZ(vsb_finish(ob[u]));
if (vsb_len(ob[u]) > 1 &&
(match_tag == -1 || flg[u] & F_MATCH))
if (vsb_len(ob[u]) > 1 && VSL_Matched(vd, bitmap[u])) {
printf("%s\n", vsb_data(ob[u]));
}
flg[u] = 0;
bitmap[u] = 0;
vsb_clear(ob[u]);
}
}
static int
h_order(void *priv, enum vsl_tag tag, unsigned fd, unsigned len,
unsigned spec, const char *ptr)
unsigned spec, const char *ptr, uint64_t bm)
{
char type;
(void)priv;
struct VSM_data *vd = priv;
bitmap[fd] |= bm;
type = (spec & VSL_S_CLIENT) ? 'c' :
(spec & VSL_S_BACKEND) ? 'b' : '-';
if (!(spec & (VSL_S_CLIENT|VSL_S_BACKEND))) {
if (!b_flag && !c_flag)
(void)VSL_H_Print(stdout, tag, fd, len, spec, ptr);
(void)VSL_H_Print(stdout, tag, fd, len, spec, ptr, bm);
return (0);
}
if (ob[fd] == NULL) {
ob[fd] = vsb_new_auto();
assert(ob[fd] != NULL);
}
if (tag == match_tag &&
VRE_exec(match_re, ptr, len, 0, 0, NULL, 0) > 0)
flg[fd] |= F_MATCH;
if ((tag == SLT_BackendOpen || tag == SLT_SessionOpen ||
(tag == SLT_ReqStart &&
last[fd] != SLT_SessionOpen &&
......@@ -146,7 +126,7 @@ h_order(void *priv, enum vsl_tag tag, unsigned fd, unsigned len,
if (last[fd] != SLT_SessionClose)
vsb_printf(ob[fd], "%5d %-12s %c %s\n",
fd, "Interrupted", type, VSL_tags[tag]);
h_order_finish(fd);
h_order_finish(fd, vd);
}
last[fd] = tag;
......@@ -182,7 +162,7 @@ h_order(void *priv, enum vsl_tag tag, unsigned fd, unsigned len,
case SLT_BackendClose:
case SLT_BackendReuse:
case SLT_StatSess:
h_order_finish(fd);
h_order_finish(fd, vd);
break;
default:
break;
......@@ -191,24 +171,10 @@ h_order(void *priv, enum vsl_tag tag, unsigned fd, unsigned len,
}
static void
do_order(struct VSM_data *vd, int argc, char * const *argv)
do_order(struct VSM_data *vd)
{
int i;
const char *error;
int erroroffset;
if (argc == 2) {
match_tag = name2tag(argv[0]);
if (match_tag < 0) {
fprintf(stderr, "Tag \"%s\" unknown\n", argv[0]);
exit(2);
}
match_re = VRE_compile(argv[1], 0, &error, &erroroffset);
if (match_re == NULL) {
fprintf(stderr, "Invalid regex: %s\n", error);
exit(2);
}
}
if (!b_flag) {
VSL_Select(vd, SLT_SessionOpen);
VSL_Select(vd, SLT_SessionClose);
......@@ -220,15 +186,15 @@ do_order(struct VSM_data *vd, int argc, char * const *argv)
VSL_Select(vd, SLT_BackendReuse);
}
while (1) {
i = VSL_Dispatch(vd, h_order, NULL);
i = VSL_Dispatch(vd, h_order, vd);
if (i == 0) {
clean_order();
clean_order(vd);
AZ(fflush(stdout));
}
else if (i < 0)
break;
}
clean_order();
clean_order(vd);
}
/*--------------------------------------------------------------------*/
......@@ -273,7 +239,7 @@ do_write(const struct VSM_data *vd, const char *w_arg, int a_flag)
XXXAN(fd >= 0);
(void)signal(SIGHUP, sighup);
while (1) {
i = VSL_NextLog(vd, &p);
i = VSL_NextLog(vd, &p, NULL);
if (i < 0)
break;
if (i > 0) {
......@@ -317,7 +283,7 @@ main(int argc, char * const *argv)
vd = VSM_New();
VSL_Setup(vd);
while ((c = getopt(argc, argv, VSL_ARGS "aDoP:uVw:")) != -1) {
while ((c = getopt(argc, argv, VSL_ARGS "aDP:uVw:")) != -1) {
switch (c) {
case 'a':
a_flag = 1;
......@@ -335,6 +301,7 @@ main(int argc, char * const *argv)
break;
case 'o':
o_flag = 1;
AN(VSL_Arg(vd, c, optarg));
break;
case 'P':
P_arg = optarg;
......@@ -386,7 +353,7 @@ main(int argc, char * const *argv)
setbuf(stdout, NULL);
if (o_flag)
do_order(vd, argc - optind, argv + optind);
do_order(vd);
while (VSL_Dispatch(vd, VSL_H_Print, stdout) >= 0) {
if (fflush(stdout) != 0) {
......
......@@ -100,31 +100,17 @@ static struct logline {
const char *df_handling; /* How the request was handled (hit/miss/pass/pipe) */
int active; /* Is log line in an active trans */
int complete; /* Is log line complete */
int matched; /* Did log line match */
uint64_t bitmap; /* Bitmap for regex matches */
} **ll;
struct VSM_data *vd;
static size_t nll;
static int o_flag = 0;
static int match_tag;
static const vre_t *match_tag_re;
static const char *format;
static int
name2tag(const char *n)
{
int i;
for (i = 0; i < 256; i++) {
if (VSL_tags[i] == NULL)
continue;
if (!strcasecmp(n, VSL_tags[i]))
return (i);
}
return (-1);
}
static int
isprefix(const char *str, const char *prefix, const char *end,
const char **next)
......@@ -346,11 +332,6 @@ collect_client(struct logline *lp, enum vsl_tag tag, unsigned spec,
assert(spec & VSL_S_CLIENT);
end = ptr + len;
/* Do -o matching if specified */
if (o_flag && match_tag == tag && lp->active &&
VRE_exec(match_tag_re, ptr, len, 0, 0, NULL, 0) > 0)
lp->matched = 1;
switch (tag) {
case SLT_ReqStart:
if (lp->active || lp->df_h != NULL) {
......@@ -496,7 +477,7 @@ collect_client(struct logline *lp, enum vsl_tag tag, unsigned spec,
static int
h_ncsa(void *priv, enum vsl_tag tag, unsigned fd,
unsigned len, unsigned spec, const char *ptr)
unsigned len, unsigned spec, const char *ptr, uint64_t bitmap)
{
struct logline *lp;
FILE *fo = priv;
......@@ -530,10 +511,12 @@ h_ncsa(void *priv, enum vsl_tag tag, unsigned fd,
return (reopen);
}
lp->bitmap |= bitmap;
if (!lp->complete)
return (reopen);
if (o_flag && !lp->matched)
if (o_flag && !VSL_Matched(vd, lp->bitmap))
/* -o is in effect matching rule failed. Don't display */
return (reopen);
......@@ -738,14 +721,13 @@ main(int argc, char *argv[])
const char *P_arg = NULL;
const char *w_arg = NULL;
struct pidfh *pfh = NULL;
struct VSM_data *vd;
FILE *of;
format = "%h %l %u %t \"%r\" %s %b \"%{Referer}i\" \"%{User-agent}i\"";
vd = VSM_New();
VSL_Setup(vd);
while ((c = getopt(argc, argv, VSL_ARGS "aDP:Vw:foF:")) != -1) {
while ((c = getopt(argc, argv, VSL_ARGS "aDP:Vw:fF:")) != -1) {
switch (c) {
case 'a':
a_flag = 1;
......@@ -795,7 +777,9 @@ main(int argc, char *argv[])
break;
case 'o':
o_flag = 1;
break;
if (VSL_Arg(vd, c, optarg) > 0)
break;
usage();
default:
if (VSL_Arg(vd, c, optarg) > 0)
break;
......@@ -805,26 +789,6 @@ main(int argc, char *argv[])
VSL_Arg(vd, 'c', optarg);
if (o_flag) {
const char *error;
int erroroffset;
if (argc-optind != 2) {
fprintf(stderr, "Wrong number of arguments when using -o\n");
exit(2);
}
match_tag = name2tag(argv[optind]);
if (match_tag < 0) {
fprintf(stderr, "Tag \"%s\" unknown\n", argv[optind]);
exit(2);
}
match_tag_re = VRE_compile(argv[optind + 1], 0, &error, &erroroffset);
if (match_tag_re==NULL) {
fprintf(stderr, "Invalid regex: %s\n", error);
exit(2);
}
}
if (VSL_Open(vd, 1))
exit(1);
......
......@@ -640,13 +640,14 @@ clear:
static int
gen_traffic(void *priv, enum vsl_tag tag, unsigned fd,
unsigned len, unsigned spec, const char *ptr)
unsigned len, unsigned spec, const char *ptr, uint64_t bitmap)
{
struct replay_thread *thr;
const char *end;
struct message *msg;
(void)priv;
(void)bitmap;
end = ptr + len;
......
......@@ -65,6 +65,7 @@ static unsigned next_hist;
static unsigned bucket_miss[HIST_BUCKETS];
static unsigned bucket_hit[HIST_BUCKETS];
static unsigned char hh[FD_SETSIZE];
static uint64_t bitmap[FD_SETSIZE];
static double log_ten;
......@@ -146,12 +147,12 @@ update(struct VSM_data *vd)
static int
h_hist(void *priv, enum vsl_tag tag, unsigned fd, unsigned len,
unsigned spec, const char *ptr)
unsigned spec, const char *ptr, uint64_t bm)
{
double b;
int i, j, tmp;
struct VSM_data *vd = priv;
(void)priv;
(void)len;
(void)spec;
......@@ -159,6 +160,8 @@ h_hist(void *priv, enum vsl_tag tag, unsigned fd, unsigned len,
/* oops */
return (0);
bitmap[fd] |= bm;
if (tag == SLT_Hit) {
hh[fd] = 1;
return (0);
......@@ -166,6 +169,12 @@ h_hist(void *priv, enum vsl_tag tag, unsigned fd, unsigned len,
if (tag != SLT_Length)
return (0);
if (!VSL_Matched(vd, bitmap[fd])) {
bitmap[fd] = 0;
hh[fd] = 0;
return (0);
}
/* determine processing time */
i = sscanf(ptr, "%d", &tmp);
assert(i == 1);
......@@ -213,6 +222,7 @@ h_hist(void *priv, enum vsl_tag tag, unsigned fd, unsigned len,
next_hist = 0;
}
hh[fd] = 0;
bitmap[fd] = 0;
pthread_mutex_unlock(&mtx);
......@@ -226,7 +236,7 @@ accumulate_thread(void *arg)
int i;
for (;;) {
i = VSL_Dispatch(vd, h_hist, NULL);
i = VSL_Dispatch(vd, h_hist, vd);
if (i < 0)
break;
if (i == 0)
......
......@@ -195,7 +195,7 @@ accumulate_thread(void *arg)
for (;;) {
i = VSL_NextLog(vd, &p);
i = VSL_NextLog(vd, &p, NULL);
if (i < 0)
break;
if (i == 0) {
......@@ -292,7 +292,7 @@ do_once(struct VSM_data *vd)
{
uint32_t *p;
while (VSL_NextLog(vd, &p) > 0)
while (VSL_NextLog(vd, &p, NULL) > 0)
accumulate(p);
dump();
}
......@@ -335,6 +335,9 @@ main(int argc, char **argv)
case 'V':
varnish_version("varnishtop");
exit(0);
case 'o':
fprintf(stderr, "-o is not supported\n");
exit(1);
default:
if (VSL_Arg(vd, o, optarg) > 0)
break;
......
......@@ -214,7 +214,7 @@ int VSL_Open(struct VSM_data *vd, int diag);
* != 0 on failure
*/
#define VSL_ARGS "bCcdI:i:k:n:r:s:X:x:"
#define VSL_ARGS "bCcdI:i:k:n:r:s:X:x:o:"
#define VSL_b_USAGE "[-b]"
#define VSL_c_USAGE "[-c]"
#define VSL_C_USAGE "[-C]"
......@@ -223,6 +223,7 @@ int VSL_Open(struct VSM_data *vd, int diag);
#define VSL_I_USAGE "[-I regexp]"
#define VSL_k_USAGE "[-k keep]"
#define VSL_n_USAGE VSM_n_USAGE
#define VSL_o_USAGE "[-o tag:regex]"
#define VSL_r_USAGE "[-r file]"
#define VSL_s_USAGE "[-s skip]"
#define VSL_x_USAGE "[-x tag]"
......@@ -232,6 +233,7 @@ int VSL_Open(struct VSM_data *vd, int diag);
VSL_I_USAGE " " \
VSL_k_USAGE " " \
VSL_n_USAGE " " \
VSL_o_USAGE " " \
VSL_r_USAGE " " \
VSL_s_USAGE " " \
VSL_X_USAGE " " \
......@@ -247,7 +249,8 @@ int VSL_Arg(struct VSM_data *vd, int arg, const char *opt);
*/
typedef int vsl_handler(void *priv, enum vsl_tag tag, unsigned fd,
unsigned len, unsigned spec, const char *ptr);
unsigned len, unsigned spec, const char *ptr, uint64_t bitmap);
#define VSL_S_CLIENT (1 << 0)
#define VSL_S_BACKEND (1 << 1)
vsl_handler VSL_H_Print;
......@@ -255,7 +258,8 @@ struct VSM_data;
void VSL_Select(const struct VSM_data *vd, unsigned tag);
void VSL_NonBlocking(const struct VSM_data *vd, int nb);
int VSL_Dispatch(struct VSM_data *vd, vsl_handler *func, void *priv);
int VSL_NextLog(const struct VSM_data *lh, uint32_t **pp);
int VSL_NextLog(const struct VSM_data *lh, uint32_t **pp, uint64_t *bitmap);
int VSL_Matched(const struct VSM_data *vd, uint64_t bitmap);
extern const char *VSL_tags[256];
......
......@@ -85,6 +85,9 @@ VSL_Setup(struct VSM_data *vd)
vsl->rbuflen = 256; /* XXX ?? */
vsl->rbuf = malloc(vsl->rbuflen * 4L);
assert(vsl->rbuf != NULL);
vsl->num_matchers = 0;
VTAILQ_INIT(&vsl->matchers);
}
/*--------------------------------------------------------------------*/
......@@ -193,7 +196,7 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
}
int
VSL_NextLog(const struct VSM_data *vd, uint32_t **pp)
VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *mb)
{
struct vsl *vsl;
uint32_t *p;
......@@ -255,6 +258,19 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp)
if (i != VRE_ERROR_NOMATCH)
continue;
}
if (mb != NULL) {
struct vsl_re_match *vrm;
int j = 0;
VTAILQ_FOREACH(vrm, &vsl->matchers, next) {
if (vrm->tag == t) {
i = VRE_exec(vrm->re, VSL_DATA(p),
VSL_LEN(p), 0, 0, NULL, 0);
if (i >= 0)
*mb |= 1 << j;
}
j++;
}
}
*pp = p;
return (1);
}
......@@ -269,13 +285,15 @@ VSL_Dispatch(struct VSM_data *vd, vsl_handler *func, void *priv)
int i;
unsigned u, l, s;
uint32_t *p;
uint64_t bitmap;
CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
vsl = vd->vsl;
CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
while (1) {
i = VSL_NextLog(vd, &p);
bitmap = 0;
i = VSL_NextLog(vd, &p, &bitmap);
if (i == 0 && VSM_ReOpen(vd, 0) == 1)
continue;
if (i != 1)
......@@ -287,7 +305,7 @@ VSL_Dispatch(struct VSM_data *vd, vsl_handler *func, void *priv)
s |= VSL_S_BACKEND;
if (vbit_test(vsl->vbm_client, u))
s |= VSL_S_CLIENT;
if (func(priv, VSL_TAG(p), u, l, s, VSL_DATA(p)))
if (func(priv, VSL_TAG(p), u, l, s, VSL_DATA(p), bitmap))
return (1);
}
}
......@@ -296,11 +314,12 @@ VSL_Dispatch(struct VSM_data *vd, vsl_handler *func, void *priv)
int
VSL_H_Print(void *priv, enum vsl_tag tag, unsigned fd, unsigned len,
unsigned spec, const char *ptr)
unsigned spec, const char *ptr, uint64_t bitmap)
{
FILE *fo = priv;
int type;
(void) bitmap;
assert(fo != NULL);
type = (spec & VSL_S_CLIENT) ? 'c' :
......@@ -366,3 +385,15 @@ VSL_Open(struct VSM_data *vd, int diag)
}
return (0);
}
/*--------------------------------------------------------------------*/
int VSL_Matched(const struct VSM_data *vd, uint64_t bitmap)
{
if (vd->vsl->num_matchers > 0) {
uint64_t t;
t = vd->vsl->num_matchers | (vd->vsl->num_matchers - 1);
return (bitmap == t);
}
return (1);
}
......@@ -28,6 +28,16 @@
*
*/
#include "vqueue.h"
struct vsl_re_match {
unsigned magic;
#define VSL_RE_MATCH_MAGIC 0x4013151e
int tag;
vre_t *re;
VTAILQ_ENTRY(vsl_re_match) next;
};
struct vsl {
unsigned magic;
#define VSL_MAGIC 0x7a31db38
......@@ -73,7 +83,10 @@ struct vsl {
int regflags;
vre_t *regincl;
vre_t *regexcl;
int num_matchers;
VTAILQ_HEAD(, vsl_re_match) matchers;
unsigned long skip;
unsigned long keep;
};
......@@ -148,6 +148,64 @@ vsl_ix_arg(const struct VSM_data *vd, const char *opt, int arg)
/*--------------------------------------------------------------------*/
static int
name2tag(const char *n)
{
int i;
for (i = 0; i < 256; i++) {
if (VSL_tags[i] == NULL)
continue;
if (!strcasecmp(n, VSL_tags[i]))
return (i);
}
return (-1);
}
static int
vsl_o_arg(const struct VSM_data *vd, const char *opt)
{
struct vsl_re_match *m;
const char *error;
char *o, *regex;
int erroroffset;
CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
ALLOC_OBJ(m, VSL_RE_MATCH_MAGIC);
AN(m);
if (! index(opt, ':')) {
fprintf(stderr, "No : found in -o option %s\n", opt);
return (-1);
}
o = strdup(opt);
AN(o);
regex = index(o, ':');
*regex = '\0';
regex++;
m->tag = name2tag(o);
if (m->tag == -1) {
fprintf(stderr, "Illegal tag %s specified\n", o);
free(o);
return (-1);
}
/* Get tag, regex */
m->re = VRE_compile(regex, vd->vsl->regflags, &error, &erroroffset);
if (m->re == NULL) {
fprintf(stderr, "Illegal regex: %s\n", error);
free(o);
return (-1);
}
vd->vsl->num_matchers++;
VTAILQ_INSERT_TAIL(&vd->vsl->matchers, m, next);
free(o);
return (1);
}
/*--------------------------------------------------------------------*/
static int
vsl_s_arg(const struct VSM_data *vd, const char *opt)
{
......@@ -206,6 +264,7 @@ VSL_Arg(struct VSM_data *vd, int arg, const char *opt)
case 'r': return (vsl_r_arg(vd, opt));
case 's': return (vsl_s_arg(vd, opt));
case 'I': case 'X': return (vsl_IX_arg(vd, opt, arg));
case 'o': return (vsl_o_arg(vd, opt));
case 'C': vd->vsl->regflags = VRE_CASELESS; return (1);
default:
return (0);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment