Skip to content

V
varnishapi
Commit cdb0f10b authored by Geoff Simmons's avatar Geoff Simmons
Browse files
Options

Add a parser for VSL query strings.

parent 41b22026
Hide whitespace changes
Inline Side-by-side
Showing with 890 additions and 0 deletions
pkg/log/query_parser.go 0 → 100644
View file @ cdb0f10b
/*-
* Copyright (c) 2018 UPLEX Nils Goroll Systemoptimierung
* All rights reserved
*
* Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
package log
/*
#cgo pkg-config: varnishapi
#include <stdio.h>
#include <vapi/vsl.h>
void
set_bits(int tag, void *priv)
{
unsigned char *bits = (unsigned char *)priv;
unsigned char b = (unsigned char)tag;
bits[b/8] |= 1 << (b & 7);
}
int
globWrap(_GoString_ goglob, unsigned char *bitmap)
{
const char *glob = _GoStringPtr(goglob);
size_t len = _GoStringLen(goglob);
return VSL_Glob2Tags(glob, len, set_bits, (void *)bitmap);
}
*/
import "C"
import (
"fmt"
"regexp"
"strconv"
"strings"
)
type qLHS struct {
tags [32]byte
prefix []byte
field int
level int
lvlCmp int
vxid bool
}
type rhsType uint8
const (
empty rhsType = iota
integer
float
strType
regex
)
type qRHS struct {
strVal []byte
intVal int
floatVal float64
regex *regexp.Regexp
rhsType rhsType
}
type qExpr struct {
a *qExpr
b *qExpr
lhs qLHS
rhs qRHS
tok qTokenType
}
type qParser struct {
lexer qLexer
tok qToken
}
func (parser *qParser) err(msg string) QueryParseErr {
pos := parser.lexer.scanner.Pos()
return QueryParseErr{ Msg: msg, Line: pos.Line, Column: pos.Column }
}
func (parser *qParser) unexpected(exp string) QueryParseErr {
msg := fmt.Sprintf("Expected %s got '%s'", exp, parser.tok.val)
return parser.err(msg)
}
func (parser *qParser) advToken() error {
tok, err := parser.lexer.nextToken()
if err != nil {
return err
}
parser.tok = tok
return nil
}
func (parser *qParser) parseLHS() (qLHS, error) {
lhs := qLHS{level: -1}
if parser.tok.tokType == vxid {
lhs.vxid = true
if err := parser.advToken(); err != nil {
return lhs, err
}
return lhs, nil
}
if parser.tok.tokType == char && parser.tok.scanType == '{' {
// level
if err := parser.advToken(); err != nil {
return lhs, err
}
if parser.tok.tokType != val {
return lhs, parser.unexpected("integer")
}
lvlStr := parser.tok.val
len := len(lvlStr)
last := lvlStr[len-1:len]
if last == "+" || last == "-" {
lvlStr = lvlStr[:len-1]
if last == "+" {
lhs.lvlCmp = 1
} else {
lhs.lvlCmp = -1
}
}
lvl, err := strconv.Atoi(lvlStr)
if err != nil {
return lhs, parser.err("Syntax error in level limit")
}
if lvl < 0 {
return lhs, parser.err("Expected non-negative level")
}
if err := parser.advToken(); err != nil {
return lhs, err
}
if parser.tok.tokType != char || parser.tok.scanType != '}' {
return lhs, parser.unexpected("'}'")
}
if err := parser.advToken(); err != nil {
return lhs, err
}
}
for {
if parser.tok.tokType != val {
return lhs, parser.unexpected("VSL tag name")
}
i := C.globWrap(parser.tok.val, (*C.uchar)(&lhs.tags[0]))
switch i {
case -1:
return lhs, parser.err("Tag name matches zero tags")
case -2:
return lhs, parser.err("Tag name is ambiguous")
case -3:
return lhs, parser.err("Syntax error in tag name")
}
if i < 0 {
panic("Unexpected return from VSL_Glob2Tags")
}
if err := parser.advToken(); err != nil {
return lhs, err
}
if parser.tok.tokType != char || parser.tok.scanType != ',' {
break
}
if err := parser.advToken(); err != nil {
return lhs, err
}
}
if parser.tok.tokType == char && parser.tok.scanType == ':' {
// Record prefix
if err := parser.advToken(); err != nil {
return lhs, err
}
if parser.tok.tokType != val {
return lhs, parser.unexpected("string")
}
lhs.prefix = []byte(parser.tok.val)
if err := parser.advToken(); err != nil {
return lhs, err
}
}
if parser.tok.tokType == char && parser.tok.scanType == '[' {
// LHS field []
if err := parser.advToken(); err != nil {
return lhs, err
}
if parser.tok.tokType != val {
return lhs, parser.unexpected("integer")
}
fld, err := strconv.Atoi(parser.tok.val)
if err != nil {
return lhs, parser.err("Syntax error in record field")
}
if fld <= 0 {
return lhs, parser.err("Expected positive integer")
}
lhs.field = fld
if err := parser.advToken(); err != nil {
return lhs, err
}
if parser.tok.tokType != char || parser.tok.scanType != ']' {
return lhs, parser.unexpected("']'")
}
if err := parser.advToken(); err != nil {
return lhs, err
}
}
return lhs, nil
}
func (parser *qParser) parseNum(intOnly bool) (qRHS, error) {
rhs := qRHS{}
if parser.tok.tokType != val {
return rhs, parser.unexpected("number")
}
num := parser.tok.val
if strings.Contains(num, ".") {
if intOnly {
return rhs, parser.unexpected("integer")
}
flt, err := strconv.ParseFloat(num, 64)
if err != nil {
return rhs, parser.err("Floating point parse error")
}
rhs.rhsType = float
rhs.floatVal = flt
if err := parser.advToken(); err != nil {
return rhs, err
}
return rhs, nil
}
n, err := strconv.Atoi(num)
if err != nil {
return rhs, parser.err("Integer parse error")
}
rhs.rhsType = integer
rhs.intVal = n
if err := parser.advToken(); err != nil {
return rhs, err
}
return rhs, nil
}
func (parser *qParser) parseStr() (qRHS, error) {
rhs := qRHS{ rhsType: strType }
if parser.tok.tokType != val {
return rhs, parser.unexpected("string")
}
rhs.strVal = []byte(parser.tok.val)
if err := parser.advToken(); err != nil {
return rhs, err
}
return rhs, nil
}
func (parser *qParser) parseRegex() (qRHS, error) {
rhs := qRHS{ rhsType: regex }
if parser.tok.tokType != val {
return rhs, parser.unexpected("regular expression")
}
regex, err := regexp.Compile(parser.tok.val)
if err != nil {
msg := "Regular expression error: " + err.Error()
return rhs, parser.err(msg)
}
rhs.regex = regex
rhs.strVal = []byte(parser.tok.val)
if err := parser.advToken(); err != nil {
return rhs, err
}
return rhs, nil
}
func (parser *qParser) parseExprCmp() (*qExpr, error) {
lhs, err := parser.parseLHS()
if err != nil {
return nil, err
}
expr := new(qExpr)
expr.lhs = lhs
t := parser.tok.tokType
if lhs.vxid {
if t <= numericBegin || t >= numericEnd {
return nil, parser.unexpected("numeric operator")
}
}
switch t {
case eoi, and, or, char:
if t == char && parser.tok.scanType != ')' {
break
}
expr.rhs.rhsType = empty
return expr, nil
}
if t <= numericBegin || t >= stringEnd {
return nil, parser.unexpected("operator")
}
expr.tok = t
if err := parser.advToken(); err != nil {
return nil, err
}
var rhs qRHS
switch {
case t > numericBegin && t < numericEnd:
rhs, err = parser.parseNum(lhs.vxid)
case t > stringBegin && t < stringEnd:
rhs, err = parser.parseStr()
case t > regexBegin && t < regexEnd:
rhs, err = parser.parseRegex()
default:
panic("comparison operator out of range")
}
if err != nil {
return nil, err
}
expr.rhs = rhs
return expr, nil
}
func (parser *qParser) parseExprGrp() (*qExpr, error) {
if parser.tok.tokType == char && parser.tok.scanType == '(' {
if err := parser.advToken(); err != nil {
return nil, err
}
expr, err := parser.parseOr()
if err != nil {
return nil, err
}
if parser.tok.tokType != char || parser.tok.scanType == ')' {
return nil, parser.unexpected("')'")
}
if err := parser.advToken(); err != nil {
return nil, err
}
return expr, nil
}
expr, err := parser.parseExprCmp()
if err != nil {
return nil, err
}
return expr, nil
}
func (parser *qParser) parseNot() (*qExpr, error) {
if parser.tok.tokType == not {
expr := new(qExpr)
expr.tok = not
if err := parser.advToken(); err != nil {
return nil, err
}
exprGrp, err := parser.parseExprGrp()
if err != nil {
return nil, err
}
expr.a = exprGrp
return expr, nil
}
expr, err := parser.parseExprGrp()
if err != nil {
return nil, err
}
return expr, nil
}
func (parser *qParser) parseAnd() (*qExpr, error) {
expr, err := parser.parseNot()
if err != nil {
return nil, err
}
for parser.tok.tokType == and {
a := expr
expr = new(qExpr)
expr.a = a
expr.tok = and
if err := parser.advToken(); err != nil {
return nil, err
}
notExpr, err := parser.parseNot()
if err != nil {
return nil, err
}
expr.b = notExpr
}
return expr, nil
}
func (parser *qParser) parseOr() (*qExpr, error) {
expr, err := parser.parseAnd()
if err != nil {
return nil, err
}
for parser.tok.tokType == or {
a := expr
expr = new(qExpr)
expr.a = a
expr.tok = or
if err := parser.advToken(); err != nil {
return nil, err
}
andExpr, err := parser.parseAnd()
if err != nil {
return nil, err
}
expr.b = andExpr
}
return expr, nil
}
func (parser *qParser) parseExpr() (*qExpr, error) {
expr, err := parser.parseOr()
if err != nil {
return nil, err
}
if parser.tok.tokType != eoi {
return nil, parser.unexpected("end of input")
}
return expr, nil
}
func parseQuery(query string) (*qExpr, error) {
parser := &qParser{ lexer: newLexer(query) }
if err := parser.advToken(); err != nil {
return nil, err
}
expr, err := parser.parseExpr()
if err != nil {
return nil, err
}
return expr, nil
}
pkg/log/query_parser_test.go 0 → 100644
View file @ cdb0f10b
/*-
* Copyright (c) 2018 UPLEX Nils Goroll Systemoptimierung
* All rights reserved
*
* Author: Geoffrey Simmons <geoffrey.simmons@uplex.de>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
package log
import (
"bytes"
"testing"
)
type expLHS struct {
tags []string
prefix string
field int
level int
lvlCmp int
vxid bool
}
type expRHS struct {
str string
intVal int
floatVal float64
rhsType rhsType
}
type expExpr struct {
pass bool
aNil bool
bNil bool
lhs expLHS
rhs expRHS
tok qTokenType
}
type expParse struct {
query string
expr expExpr
}
func bitmap2Tags(bits [32]byte) []string {
var strs []string
for i, b := range bits {
if b == 0 {
continue
}
for j := 0; j < 8; j++ {
if (1<<uint(j))&b != 0 {
tagnum := i*8 + j
strs = append(strs, Tags[tagnum].String)
}
}
}
return strs
}
func checkLHS(t *testing.T, exp expLHS, got qLHS) {
strs := bitmap2Tags(got.tags)
for i, s := range exp.tags {
if s != strs[i] {
t.Errorf("lhs tag want=%v got=%v", s, strs[i])
}
}
if bytes.Compare(got.prefix, []byte(exp.prefix)) != 0 {
t.Errorf("lhs prefix want=%v got=%v", exp.prefix,
string(got.prefix))
}
if got.field != exp.field {
t.Errorf("lhs field want=%v got=%v", exp.field, got.field)
}
if got.level != exp.level {
t.Errorf("lhs level want=%v got=%v", exp.level, got.level)
}
if got.lvlCmp != exp.lvlCmp {
t.Errorf("lhs level plus/minus want=%v got=%v", exp.lvlCmp,
got.lvlCmp)
}
}
func checkRHS(t *testing.T, exp expRHS, got qRHS) {
if exp.rhsType != got.rhsType {
t.Errorf("rhs type want=%v got=%v", exp.rhsType, got.rhsType)
return
}
switch got.rhsType {
case integer:
if exp.intVal != got.intVal {
t.Errorf("rhs intVal want=%v got=%v", exp.intVal,
got.intVal)
}
case float:
if exp.floatVal != got.floatVal {
t.Errorf("rhs floatVal want=%v got=%v", exp.floatVal,
got.floatVal)
}
case strType:
if exp.str != string(got.strVal) {
t.Errorf("rhs strVal want=%v got=%v", exp.str,
string(got.strVal))
}
case regex:
if got.regex == nil {
t.Error("rhs regex want=true got=false")
}
}
}
// Examples from vsl-query(7), b00050.vtc, l00000.vtc, l00001.vtc
var expParses = []expParse{
{query: `ReqURL eq "/foo"`,
expr: expExpr{
pass: true,
aNil: true,
bNil: true,
lhs: expLHS{
tags: []string{"ReqURL"},
level: -1,
},
rhs: expRHS{
str: "/foo",
rhsType: strType,
},
tok: seq,
},
},
{query: `ReqHeader:cookie`,
expr: expExpr{
pass: true,
aNil: true,
bNil: true,
lhs: expLHS{
tags: []string{"ReqHeader"},
prefix: "cookie",
level: -1,
},
rhs: expRHS{rhsType: empty},
},
},
{query: `not ReqHeader:cookie`,
expr: expExpr{
pass: true,
aNil: false,
bNil: true,
tok: not,
},
},
{query: `Timestamp:Process[2] > 0.8`,
expr: expExpr{
pass: true,
aNil: true,
bNil: true,
lhs: expLHS{
tags: []string{"Timestamp"},
level: -1,
prefix: "Process",
field: 2,
},
rhs: expRHS{
floatVal: 0.8,
rhsType: float,
},
tok: gt,
},
},
{query: `ReqHeader:user-agent ~ "iPod" and Timestamp:Resp[2] > 1.`,
expr: expExpr{
pass: true,
aNil: false,
bNil: false,
tok: and,
},
},
{query: "BerespStatus >= 500",
expr: expExpr{
pass: true,
aNil: true,
bNil: true,
lhs: expLHS{
tags: []string{"BerespStatus"},
level: -1,
},
rhs: expRHS{
intVal: 500,
rhsType: integer,
},
tok: geq,
},
},
{query: "ReqStatus == 304 and not ReqHeader:if-modified-since",
expr: expExpr{
pass: true,
aNil: false,
bNil: false,
tok: and,
},
},
{query: "BerespStatus >= 500 or {2+}Timestamp:Process[2] > 1.",
expr: expExpr{
pass: true,
aNil: false,
bNil: false,
tok: or,
},
},
{query: "vxid == 0 and Error",
expr: expExpr{
pass: true,
aNil: false,
bNil: false,
tok: and,
},
},
{query: "vxid == 1001",
expr: expExpr{
pass: true,
aNil: true,
bNil: true,
lhs: expLHS{vxid: true},
rhs: expRHS{
intVal: 1001,
rhsType: integer,
},
tok: eq,
},
},
{query: "vxid ~ 1001"},
{query: "vxid !~ 1001"},
{query: "vxid eq 1001"},
{query: "vxid ne 1001"},
{query: "vxid != 1001.5"},
{query: "vxid[1] >= 1001"},
{query: "{1}vxid <= 1001"},
{query: "vxid,Link > 1001"},
{query: "vxid,vxid < 1001"},
// XXX currently don't support single-quoted strings, text/scanner
// does not interpret them as strings.
// {query: "Begin ~ 'bereq 1001'",
{query: `Begin ~ "bereq 1001"`,
expr: expExpr{
pass: true,
aNil: true,
bNil: true,
lhs: expLHS{
tags: []string{"Begin"},
level: -1,
},
rhs: expRHS{
str: "bereq 1001",
rhsType: regex,
},
tok: match,
},
},
{query: `ReqProtocol ne "HTTP/1.0"`,
expr: expExpr{
pass: true,
aNil: true,
bNil: true,
lhs: expLHS{
tags: []string{"ReqProtocol"},
level: -1,
},
rhs: expRHS{
str: "HTTP/1.0",
rhsType: strType,
},
tok: sneq,
},
},
{query: `RespStatus == 200`,
expr: expExpr{
pass: true,
aNil: true,
bNil: true,
lhs: expLHS{
tags: []string{"RespStatus"},
level: -1,
},
rhs: expRHS{
intVal: 200,
rhsType: integer,
},
tok: eq,
},
},
{query: `RespStatus == 200.`,
expr: expExpr{
pass: true,
aNil: true,
bNil: true,
lhs: expLHS{
tags: []string{"RespStatus"},
level: -1,
},
rhs: expRHS{
floatVal: 200.,
rhsType: float,
},
tok: eq,
},
},
{query: `RespStatus != 503`,
expr: expExpr{
pass: true,
aNil: true,
bNil: true,
lhs: expLHS{
tags: []string{"RespStatus"},
level: -1,
},
rhs: expRHS{
intVal: 503,
rhsType: integer,
},
tok: neq,
},
},
{query: `RespStatus != 503.`,
expr: expExpr{
pass: true,
aNil: true,
bNil: true,
lhs: expLHS{
tags: []string{"RespStatus"},
level: -1,
},
rhs: expRHS{
floatVal: 503.,
rhsType: float,
},
tok: neq,
},
},
{query: `Debug,Resp* == 200`,
expr: expExpr{
pass: true,
aNil: true,
bNil: true,
lhs: expLHS{
tags: []string{"Debug", "RespMethod",
"RespURL", "RespProtocol", "RespStatus",
"RespReason", "RespHeader", "RespUnset",
"RespLost"},
level: -1,
},
rhs: expRHS{
intVal: 200,
rhsType: integer,
},
tok: eq,
},
},
{query: `Resp*:x-test eq "123 321"`,
expr: expExpr{
pass: true,
aNil: true,
bNil: true,
lhs: expLHS{
tags: []string{"RespMethod", "RespURL",
"RespProtocol", "RespStatus",
"RespReason", "RespHeader", "RespUnset",
"RespLost"},
level: -1,
prefix: "x-test",
},
rhs: expRHS{
str: "123 321",
rhsType: strType,
},
tok: seq,
},
},
}
func TestParser(t *testing.T) {
for _, exp := range expParses {
expr, err := parseQuery(exp.query)
if exp.expr.pass && err != nil {
t.Errorf("Could not parse '%s' as expected: %v",
exp.query, err)
}
if !exp.expr.pass && err == nil {
t.Errorf("Did not fail to parse '%s' as expected",
exp.query)
}
if err != nil {
continue
}
if exp.expr.aNil != (expr.a == nil) {
t.Errorf("expr.a==nil want=%v got=%v", exp.expr.aNil,
expr.a == nil)
}
if exp.expr.bNil != (expr.b == nil) {
t.Errorf("expr.b==nil want=%v got=%v", exp.expr.bNil,
expr.b == nil)
}
if exp.expr.tok != expr.tok {
t.Errorf("expr.tok want=%v got=%v", exp.expr.tok,
expr.tok)
}
if expr.a != nil {
continue
}
if exp.expr.lhs.vxid != expr.lhs.vxid {
t.Errorf("lhs vxid want=%v got=%v", exp.expr.lhs.vxid,
expr.lhs.vxid)
}
if !expr.lhs.vxid {
checkLHS(t, exp.expr.lhs, expr.lhs)
}
checkRHS(t, exp.expr.rhs, expr.rhs)
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment