First implementation of query evaluation.

pkg/log/main_test.go

@@ -47,6 +47,7 @@ const (
 	reqLog   = "testdata/request.log"
 	sessLog  = "testdata/session.log"
 	rawLog   = "testdata/raw.log"
+	l1Log    = "testdata/l00001.log"
 	failPath = "ifAFileWithThisNameReallyExistsThenTestsFail"
 	target   = "http://localhost:8080"
 )
@@ -442,7 +443,7 @@ func checkReqResp(t *testing.T, tx Tx, req http.Request, resp http.Response) {
 }
 
 func TestMain(m *testing.M) {
-	files := []string{testFile, vxidLog, rawLog, reqLog, sessLog}
+	files := []string{testFile, vxidLog, rawLog, reqLog, sessLog, l1Log}
 	for _, file := range files {
 		if _, err := os.Stat(file); err != nil {
 			fmt.Fprintln(os.Stderr, "Cannot stat "+file+":", err)

pkg/log/query_eval.go

+/*-
+ * Copyright (c) 2018 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+package log
+
+import (
+	"bytes"
+	"strconv"
+)
+
+func bitTest(bits [32]byte, tag uint8) bool {
+	return (bits[tag >> 3] & (1 << (tag & 7))) != 0
+}
+
+func (expr qExpr) testVXID(tx Tx) bool {
+	n := uint32(expr.rhs.intVal)
+	switch expr.tok {
+	case eq:
+		return tx.VXID == n
+	case neq:
+		return tx.VXID != n
+	case lt:
+		return tx.VXID < n
+	case gt:
+		return tx.VXID > n
+	case leq:
+		return tx.VXID <= n
+	case geq:
+		return tx.VXID >= n
+	}
+	return false
+}
+
+func (expr qExpr) testRecord(rec Record) bool {
+	payload := rec.Payload
+	if len(expr.lhs.prefix) > 0 {
+		l := len(expr.lhs.prefix)
+		if l > len(payload) {
+			return false
+		}
+		if !bytes.EqualFold(expr.lhs.prefix, payload[:l]) {
+			return false
+		}
+		if payload[l] != byte(':') {
+			return false
+		}
+		payload = payload[l+1:]
+		payload = bytes.TrimLeft(payload, " \t\n\r")
+	}
+	if expr.lhs.field > 0 {
+		fld := expr.lhs.field
+		flds := bytes.Fields(payload)
+		if fld > len(flds) {
+			return false
+		}
+		payload = flds[fld-1]
+	}
+
+	if expr.rhs.rhsType == empty {
+		return true
+	}
+
+	if expr.tok > numericBegin && expr.tok < numericEnd {
+		if len(payload) == 0 {
+			return false
+		}
+		// XXX byte slice-only versions of the numeric conversions
+		switch expr.rhs.rhsType {
+		case integer:
+			val, err := strconv.ParseInt(string(payload), 10, 64)
+			if err != nil {
+				return false
+			}
+			n := expr.rhs.intVal
+			switch expr.tok {
+			case eq:
+				return val == n
+			case neq:
+				return val != n
+			case lt:
+				return val < n
+			case gt:
+				return val > n
+			case leq:
+				return val <= n
+			case geq:
+				return val >= n
+			}
+		case float:
+			val, err := strconv.ParseFloat(string(payload), 64)
+			if err != nil {
+				return false
+			}
+			n := expr.rhs.floatVal
+			switch expr.tok {
+			case eq:
+				return val == n
+			case neq:
+				return val != n
+			case lt:
+				return val < n
+			case gt:
+				return val > n
+			case leq:
+				return val <= n
+			case geq:
+				return val >= n
+			}
+		}
+	}
+
+	// XXX vsl_query.c has the CASELESS option
+	switch expr.tok {
+	case seq:
+		return bytes.Equal(payload, expr.rhs.strVal)
+	case sneq:
+		return !bytes.Equal(payload, expr.rhs.strVal)
+	case match:
+		return expr.rhs.regex.Match(payload)
+	case nomatch:
+		return !expr.rhs.regex.Match(payload)
+	}
+	return false
+}
+
+func (expr qExpr) testTxGrp(txGrp []Tx) bool {
+	if expr.lhs.vxid {
+		for _, tx := range txGrp {
+			if expr.testVXID(tx) {
+				return true
+			}
+		}
+		return false
+	}
+
+	for _, tx := range txGrp {
+		if expr.lhs.level >= 0 {
+			switch {
+			case expr.lhs.lvlCmp < 0:
+				if tx.Level > uint(expr.lhs.level) {
+					continue
+				}
+			case expr.lhs.lvlCmp > 0:
+				if tx.Level < uint(expr.lhs.level) {
+					continue
+				}
+			default:
+				if tx.Level != uint(expr.lhs.level) {
+					continue
+				}
+			}
+		}
+		for _, rec := range tx.Records {
+			if !bitTest(expr.lhs.tags, uint8(rec.Tag)) {
+				continue
+			}
+			if expr.testRecord(rec) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func (expr qExpr) eval(txGrp []Tx) bool {
+	switch expr.tok {
+	case or:
+		if expr.a.eval(txGrp) {
+			return true
+		}
+		return expr.b.eval(txGrp)
+	case and:
+		if !expr.a.eval(txGrp) {
+			return false
+		}
+		return expr.b.eval(txGrp)
+	case not:
+		return !expr.a.eval(txGrp)
+	default:
+		return expr.testTxGrp(txGrp)
+	}
+	return false
+}

pkg/log/query_eval_test.go

+/*-
+ * Copyright (c) 2018 UPLEX Nils Goroll Systemoptimierung
+ * All rights reserved
+ *
+ * Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+package log
+
+import "testing"
+
+//import "fmt"
+
+func TestTestVXID(t *testing.T) {
+	testTx := Tx{VXID: 1000}
+	vec := []struct {
+		query string
+		pass  bool
+	}{
+		{"vxid == 1000", true},
+		{"vxid != 1000", false},
+		{"vxid < 1000", false},
+		{"vxid <= 1000", true},
+		{"vxid > 1000", false},
+		{"vxid >= 1000", true},
+	}
+
+	for _, v := range vec {
+		expr, err := parseQuery(v.query)
+		if err != nil {
+			t.Fatal("parseQuery():", err)
+			continue
+		}
+		if expr.testVXID(testTx) != v.pass {
+			t.Errorf("testVXID() want=%v got=!want", v.pass)
+		}
+	}
+}
+
+// Tests from l00001.vtc
+func TestTestRecord(t *testing.T) {
+	vec := []struct {
+		query string
+		pass  string
+		fail  string
+	}{
+		{`Begin eq "req 1000 rxreq"`,
+			"req 1000 rxreq", "req 1001 rxreq"},
+		{`ReqProtocol ne "HTTP/1.0"`, "HTTP/1.1", "HTTP/1.0"},
+		{`RespStatus == 200`, "200", "404"},
+		{"RespStatus == 200.", "200", "503"},
+		{"RespStatus != 503", "200", "503"},
+		{"RespStatus != 503.", "200", "503"},
+		{"RespStatus < 201", "200", "503"},
+		{"RespStatus < 201.", "200", "503"},
+		{"RespStatus > 199", "200", "100"},
+		{"RespStatus > 199.", "200", "100"},
+		{"RespStatus <= 200", "200", "503"},
+		{"RespStatus <= 200.", "200", "503"},
+		{"RespStatus >= 200", "200", "100"},
+		{"RespStatus >= 200.", "200", "100"},
+		{`RespStatus ~ "^200$"`, "200", " 200 "},
+		{`RespStatus !~ "^404$"`, "200", "404"},
+		{"RespHeader[2] == 123", "x-test: 123 321", "x-test: 321 123"},
+		{"RespHeader[2] == 123.", "x-test: 123 321", "x-test: 321 123"},
+		{"RespHeader[2] eq 123", "x-test: 123 321", "x-test: 321 123"},
+		{"RespHeader[2] ne 123", "x-test: 321 123", "x-test: 123 321"},
+	}
+
+	for _, v := range vec {
+		expr, err := parseQuery(v.query)
+		if err != nil {
+			t.Fatal("parseQuery():", err)
+			continue
+		}
+
+		passRec := Record{Payload: []byte(v.pass)}
+		failRec := Record{Payload: []byte(v.fail)}
+		if !expr.testRecord(passRec) {
+			t.Errorf("query='%s' testRecord(%s) did not succeed "+
+				"as expected", v.query, v.pass)
+		}
+		if expr.testRecord(failRec) {
+			t.Errorf("query='%s' testRecord(%s) did not fail as "+
+				"expected", v.query, v.fail)
+		}
+	}
+}
+
+func BenchmarkTestRecord(b *testing.B) {
+	vec := []struct {
+		name    string
+		query   string
+		payload string
+	}{
+		{"seq", `Begin eq "req 1000 rxreq"`, "req 1000 rxreq"},
+		{"sneq", `Begin ne "req 1000 rxreq"`, "req 1000 rxreq"},
+		{"eqInt", `RespStatus == 200`, "200"},
+		{"eqFlt", "RespStatus == 200.", "200"},
+		{"neqInt", "RespStatus != 503", "200"},
+		{"neqFlt", "RespStatus != 503.", "200"},
+		{"ltInt", "RespStatus < 201", "200"},
+		{"ltFlt", "RespStatus < 201.", "200"},
+		{"gtInt", "RespStatus > 199", "200"},
+		{"gtFlt", "RespStatus > 199.", "200"},
+		{"leqInt", "RespStatus <= 200", "200"},
+		{"leqFlt", "RespStatus <= 200.", "200"},
+		{"geqInt", "RespStatus >= 200", "200"},
+		{"geqFlt", "RespStatus >= 200.", "200"},
+		{"match", `RespStatus ~ "^200$"`, "200"},
+		{"nomatch", `RespStatus !~ "^404$"`, "200"},
+		{"fldInt", "RespHeader[2] == 123", "x-test: 123 321"},
+		{"fldFlt", "RespHeader[2] == 123.", "x-test: 123 321"},
+		{"fldSeq", "RespHeader[2] eq 123", "x-test: 123 321"},
+		{"fldSneq", "RespHeader[2] ne 123", "x-test: 321 123"},
+	}
+
+	for _, v := range vec {
+		b.Run(v.name, func(b *testing.B) {
+			expr, err := parseQuery(v.query)
+			if err != nil {
+				b.Fatal("parseQuery():", err)
+				return
+			}
+
+			rec := Record{Payload: []byte(v.payload)}
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				expr.testRecord(rec)
+			}
+		})
+	}
+}
+
+func TestEvalQuery(t *testing.T) {
+	vec := []struct {
+		name  string
+		query string
+	}{
+		{"seq", `Begin eq "req 118200423 rxreq"`},
+		{"sneq", `Begin ne "req 1000 rxreq"`},
+		{"eqInt", `RespStatus == 200`},
+		{"eqFlt", "RespStatus == 200."},
+		{"neqInt", "RespStatus != 503"},
+		{"neqFlt", "RespStatus != 503."},
+		{"ltInt", "RespStatus < 201"},
+		{"ltFlt", "RespStatus < 201."},
+		{"gtInt", "RespStatus > 199"},
+		{"gtFlt", "RespStatus > 199."},
+		{"leqInt", "RespStatus <= 200"},
+		{"leqFlt", "RespStatus <= 200."},
+		{"geqInt", "RespStatus >= 200"},
+		{"geqFlt", "RespStatus >= 200."},
+		{"match", `RespStatus ~ "^200$"`},
+		{"nomatch", `RespStatus !~ "^404$"`},
+		{"fldInt", "RespHeader[2] == 123"},
+		{"fldFlt", "RespHeader[2] == 123."},
+		{"fldSeq", "RespHeader[2] eq 123"},
+		{"fldSneq", "RespHeader[2] ne 123"},
+
+		{"and", `RespStatus == 200 and RespStatus ~ "^200$"`},
+		{"or", `RespStatus == 404 or RespStatus ~ "^200$"`},
+		{"not", `RespStatus == 404 or not RespStatus ~ "^404"`},
+		{"exprGrp",
+			`(RespStatus == 200 or RespStatus == 404) and RespStatus == 200`},
+		{"andOr",
+			`RespStatus == 200 or RespStatus == 503 and RespStatus == 404`},
+		{"tagList", `Debug,Resp* == 200`},
+		{"prefix", `Resp*:x-test eq "123 321"`},
+		{"noRHS", `RespStatus`},
+		{"lvlEq", `{1}Begin ~ req`},
+		{"lvlEq", `{1}Begin ~ req`},
+		{"lvlLeq", `{2-}Begin ~ req`},
+		{"lvlGeq", `{0+}Begin ~ req`},
+
+		{"vxidEq", "vxid == 118200424"},
+		{"vxidNeq", "vxid != 1000"},
+		{"vxidLt", "vxid < 118200425"},
+		{"vxidLeq", "vxid <= 118200424"},
+		{"vxidGt", "vxid > 1000"},
+		{"vxidGeq", "vxid >= 1000"},
+	}
+
+	l := New()
+	defer l.Release()
+	c, err := l.NewCursorFile(l1Log)
+	if err != nil {
+		t.Fatal("NewCursorFile(l00001.log):", err)
+		return
+	}
+	defer c.Delete()
+	q, err := c.NewQuery(VXID, "")
+	if err != nil {
+		t.Fatal("NewQuery():", err)
+		return
+	}
+
+	txGrp, status := q.NextTxGroup()
+	if status != More {
+		t.Fatalf("NextTxGrp() status want=Mode got=%v", status)
+	}
+
+	for _, v := range vec {
+		t.Run(v.name, func(t *testing.T) {
+			expr, err := parseQuery(v.query)
+			if err != nil {
+				t.Fatal("parseQuery():", err)
+				return
+			}
+			if !expr.eval(txGrp) {
+				t.Error("eval(txGrp) want=true got=false")
+			}
+		})
+	}
+}
+
+func BenchmarkEvalQuery(b *testing.B) {
+	vec := []struct {
+		name  string
+		query string
+	}{
+		{"seq", `Begin eq "req 118200423 rxreq"`},
+		{"sneq", `Begin ne "req 1000 rxreq"`},
+		{"eqInt", `RespStatus == 200`},
+		{"eqFlt", "RespStatus == 200."},
+		{"neqInt", "RespStatus != 503"},
+		{"neqFlt", "RespStatus != 503."},
+		{"ltInt", "RespStatus < 201"},
+		{"ltFlt", "RespStatus < 201."},
+		{"gtInt", "RespStatus > 199"},
+		{"gtFlt", "RespStatus > 199."},
+		{"leqInt", "RespStatus <= 200"},
+		{"leqFlt", "RespStatus <= 200."},
+		{"geqInt", "RespStatus >= 200"},
+		{"geqFlt", "RespStatus >= 200."},
+		{"match", `RespStatus ~ "^200$"`},
+		{"nomatch", `RespStatus !~ "^404$"`},
+		{"fldInt", "RespHeader[2] == 123"},
+		{"fldFlt", "RespHeader[2] == 123."},
+		{"fldSeq", "RespHeader[2] eq 123"},
+		{"fldSneq", "RespHeader[2] ne 123"},
+
+		{"and", `RespStatus == 200 and RespStatus ~ "^200$"`},
+		{"or", `RespStatus == 404 or RespStatus ~ "^200$"`},
+		{"not", `RespStatus == 404 or not RespStatus ~ "^404"`},
+		{"exprGrp",
+			`(RespStatus == 200 or RespStatus == 404) and RespStatus == 200`},
+		{"andOr",
+			`RespStatus == 200 or RespStatus == 503 and RespStatus == 404`},
+		{"tagList", `Debug,Resp* == 200`},
+		{"prefix", `Resp*:x-test eq "123 321"`},
+		{"noRHS", `RespStatus`},
+		{"lvlEq", `{1}Begin ~ req`},
+		{"lvlEq", `{1}Begin ~ req`},
+		{"lvlLeq", `{2-}Begin ~ req`},
+		{"lvlGeq", `{0+}Begin ~ req`},
+
+		{"vxidEq", "vxid == 118200424"},
+		{"vxidNeq", "vxid != 1000"},
+		{"vxidLt", "vxid < 118200425"},
+		{"vxidLeq", "vxid <= 118200424"},
+		{"vxidGt", "vxid > 1000"},
+		{"vxidGeq", "vxid >= 1000"},
+	}
+
+	l := New()
+	defer l.Release()
+	c, err := l.NewCursorFile(l1Log)
+	if err != nil {
+		b.Fatal("NewCursorFile(l00001.log):", err)
+		return
+	}
+	defer c.Delete()
+	q, err := c.NewQuery(VXID, "")
+	if err != nil {
+		b.Fatal("NewQuery():", err)
+		return
+	}
+
+	txGrp, status := q.NextTxGroup()
+	if status != More {
+		b.Fatalf("NextTxGrp() status want=Mode got=%v", status)
+	}
+
+	for _, v := range vec {
+		b.Run(v.name, func(b *testing.B) {
+			expr, err := parseQuery(v.query)
+			if err != nil {
+				b.Fatal("parseQuery():", err)
+				return
+			}
+
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				expr.eval(txGrp)
+			}
+		})
+	}
+}