johanels/akvorado
1
0
Fork 0
mirror of https://github.com/akvorado/akvorado.git synced 2025-12-12 06:24:10 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

console/filter: let parser tells us if we need the main table or not

Browse Source 
This is more robust this way. We also introduce the ability to reverse
the direction of a filter.
This commit is contained in:
Vincent Bernat
2022-08-04 18:32:44 +02:00
parent 2610bd48d1
commit 1aa260bae2
13 changed files with 322 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
console/clickhouse.go
View File

@@ -12,10 +12,7 @@ import (
"time"
)
var (
addressOrPortRegexp = regexp.MustCompile(`\b(?:Src|Dst)(?:Port|Addr)\b`)
resolutionRegexp = regexp.MustCompile(`{resolution->(\d+)}`)
)
var resolutionRegexp = regexp.MustCompile(`{resolution->(\d+)}`)
// flowsTable describe a consolidated or unconsolidated flows table.
type flowsTable struct {
@@ -29,14 +26,14 @@ type flowsTable struct {
// should contain `{table}` which will be replaced by the appropriate
// flows table and {timefilter} which will be replaced by the
// appropriate time filter.
func (c *Component) queryFlowsTable(query string, start, end time.Time, targetResolution time.Duration) string {
func (c *Component) queryFlowsTable(query string, mainTableRequired bool, start, end time.Time, targetResolution time.Duration) string {
c.flowsTablesLock.RLock()
defer c.flowsTablesLock.RUnlock()
// Select table
table := "flows"
resolution := time.Second
if !addressOrPortRegexp.MatchString(query) {
if !mainTableRequired {
// We can use the consolidated data. The first
// criteria is to find the tables matching the time
// criteria.

4
console/clickhouse_test.go
View File

@@ -75,6 +75,7 @@ func TestQueryFlowsTables(t *testing.T) {
Description string
Tables []flowsTable
Query string
MainTableRequired bool
Start time.Time
End time.Time
Resolution time.Duration
@@ -83,6 +84,7 @@ func TestQueryFlowsTables(t *testing.T) {
{
Description: "query with source port",
Query: "SELECT TimeReceived, SrcPort FROM {table} WHERE {timefilter}",
MainTableRequired: true,
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Expected: "SELECT TimeReceived, SrcPort FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')",
@@ -194,7 +196,7 @@ func TestQueryFlowsTables(t *testing.T) {
for _, tc := range cases {
t.Run(tc.Description, func(t *testing.T) {
c.flowsTables = tc.Tables
got := c.queryFlowsTable(tc.Query, tc.Start, tc.End, tc.Resolution)
got := c.queryFlowsTable(tc.Query, tc.MainTableRequired, tc.Start, tc.End, tc.Resolution)
if diff := helpers.Diff(got, tc.Expected); diff != "" {
t.Fatalf("queryFlowsTable(): (-got, +want):\n%s", diff)
}

8
console/filter.go
View File

@@ -42,7 +42,7 @@ func (c *Component) filterValidateHandlerFunc(gc *gin.Context) {
})
return
}
got, err := filter.Parse("", []byte(input.Filter))
got, err := filter.Parse("", []byte(input.Filter), filter.GlobalStore("meta", &filter.Meta{}))
if err == nil {
gc.JSON(http.StatusOK, filterValidateHandlerOutput{
Message: "ok",
@@ -84,7 +84,8 @@ func (c *Component) filterCompleteHandlerFunc(gc *gin.Context) {
completions := []filterCompletion{}
switch input.What {
case "column":
_, err := filter.Parse("", []byte{}, filter.Entrypoint("ConditionExpr"))
_, err := filter.Parse("", []byte{},
filter.Entrypoint("ConditionExpr"), filter.GlobalStore("meta", &filter.Meta{}))
if err != nil {
for _, candidate := range filter.Expected(err) {
if !strings.HasSuffix(candidate, `"i`) {
@@ -100,7 +101,8 @@ func (c *Component) filterCompleteHandlerFunc(gc *gin.Context) {
case "operator":
_, err := filter.Parse("",
[]byte(fmt.Sprintf("%s ", input.Column)),
filter.Entrypoint("ConditionExpr"))
filter.Entrypoint("ConditionExpr"),
filter.GlobalStore("meta", &filter.Meta{}))
if err != nil {
for _, candidate := range filter.Expected(err) {
if !strings.HasPrefix(candidate, `"`) {

6
console/filter/error_test.go
View File

@@ -13,7 +13,7 @@ func TestFilterHumanError(t *testing.T) {
_, err := Parse("", []byte(`
InIfDescription = "Gi0/0/0/0"
AND Proto = 1000
OR `))
OR `), GlobalStore("meta", &Meta{}))
expected := "at line 3, position 13: expecting an unsigned 8-bit integer"
if diff := helpers.Diff(HumanError(err), expected); diff != "" {
t.Errorf("HumanError() (-got, +want):\n%s", diff)
@@ -24,7 +24,7 @@ func TestAllErrors(t *testing.T) {
_, err := Parse("", []byte(`
InIfDescription = "Gi0/0/0/0"
AND Proto = 1000
OR`))
OR`), GlobalStore("meta", &Meta{}))
// Currently, the parser stops at the first error.
expected := Errors{
oneError{
@@ -40,7 +40,7 @@ OR`))
}
func TestExpected(t *testing.T) {
_, err := Parse("", []byte{}, Entrypoint("ConditionBoundaryExpr"))
_, err := Parse("", []byte{}, Entrypoint("ConditionBoundaryExpr"), GlobalStore("meta", &Meta{}))
expected := []string{`"InIfBoundary"i`, `"OutIfBoundary"i`}
if diff := helpers.Diff(Expected(err), expected); diff != "" {
t.Errorf("AllErrors() (-got, +want):\n%s", diff)

50
console/filter/helpers.go
View File

@@ -6,8 +6,36 @@ package filter
import (
"encoding/binary"
"net"
"strings"
)
// Meta is used to inject/retrieve state from the parser.
type Meta struct {
// ReverseDirection tells if we require the reverse direction for the provided filter (used as input)
ReverseDirection bool
// MainTableRequired tells if the main table is required to execute the expression (used as output)
MainTableRequired bool
}
func (c *current) reverseDirection(direct string) string {
if c.globalStore["meta"].(*Meta).ReverseDirection {
if strings.HasPrefix(direct, "Src") {
return "Dst" + direct[3:]
}
if strings.HasPrefix(direct, "Dst") {
return "Src" + direct[3:]
}
if strings.HasPrefix(direct, "In") {
return "Out" + direct[2:]
}
if strings.HasPrefix(direct, "Out") {
return "In" + direct[3:]
}
panic("no reverse?")
}
return direct
}
func lastIP(subnet *net.IPNet) net.IP {
if subnet.IP.To4() != nil {
// IPv4 case
@@ -24,3 +52,25 @@ func lastIP(subnet *net.IPNet) net.IP {
}
return ip
}
func quote(v interface{}) string {
return "'" + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(toString(v)) + "'"
}
func toSlice(v interface{}) []interface{} {
if v == nil {
return nil
}
return v.([]interface{})
}
func toString(v interface{}) string {
switch s := v.(type) {
case string:
return s
case []byte:
return string(s)
default:
panic("not a string")
}
}

89
console/filter/parser.peg
View File

@@ -4,39 +4,18 @@
package filter
// Convert SQL-like language for filters to SQL.
import (
"fmt"
"net"
"akvorado/common/helpers"
)
func quote(v interface{}) string {
return "'" + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(toString(v)) + "'"
}
func toSlice(v interface{}) []interface{} {
if v == nil {
return nil
}
return v.([]interface{})
}
func toString(v interface{}) string {
switch s := v.(type) {
case string:
return s
case []byte:
return string(s)
default:
panic("not a string")
}
}
}
Input ← _ expr:Expr _ EOF {
meta := c.globalStore["meta"].(*Meta)
_, ok := c.state["main-table-only"]
meta.MainTableRequired = ok
return expr, nil
}
@@ -69,8 +48,10 @@ ConditionExpr "conditional" ←
ColumnIP ←
"ExporterAddress"i { return "ExporterAddress", nil }
/ "SrcAddr"i { return "SrcAddr", nil }
/ "DstAddr"i { return "DstAddr", nil }
/ "SrcAddr"i #{ c.state["main-table-only"] = true ; return nil }
{ return c.reverseDirection("SrcAddr"), nil }
/ "DstAddr"i #{ c.state["main-table-only"] = true ; return nil }
{ return c.reverseDirection("DstAddr"), nil }
ConditionIPExpr "condition on IP" ←
column:ColumnIP _
operator:("=" / "!=") _ ip:IP {
@@ -92,26 +73,26 @@ ConditionStringExpr "condition on string" ←
/ "ExporterSite"i { return "ExporterSite", nil }
/ "ExporterRegion"i { return "ExporterRegion", nil }
/ "ExporterTenant"i { return "ExporterTenant", nil }
/ "SrcCountry"i { return "SrcCountry", nil }
/ "DstCountry"i { return "DstCountry", nil }
/ "SrcNetName"i { return "SrcNetName", nil }
/ "DstNetName"i { return "DstNetName", nil }
/ "SrcNetRole"i { return "SrcNetRole", nil }
/ "DstNetRole"i { return "DstNetRole", nil }
/ "SrcNetSite"i { return "SrcNetSite", nil }
/ "DstNetSite"i { return "DstNetSite", nil }
/ "SrcNetRegion"i { return "SrcNetRegion", nil }
/ "DstNetRegion"i { return "DstNetRegion", nil }
/ "SrcNetTenant"i { return "SrcNetTenant", nil }
/ "DstNetTenant"i { return "DstNetTenant", nil }
/ "InIfName"i { return "InIfName", nil }
/ "OutIfName"i { return "OutIfName", nil }
/ "InIfDescription"i { return "InIfDescription", nil }
/ "OutIfDescription"i { return "OutIfDescription", nil }
/ "InIfConnectivity"i { return "InIfConnectivity", nil }
/ "OutIfConnectivity"i { return "OutIfConnectivity", nil }
/ "InIfProvider"i { return "InIfProvider", nil }
/ "OutIfProvider"i { return "OutIfProvider", nil }) _
/ "SrcCountry"i { return c.reverseDirection("SrcCountry"), nil }
/ "DstCountry"i { return c.reverseDirection("DstCountry"), nil }
/ "SrcNetName"i { return c.reverseDirection("SrcNetName"), nil }
/ "DstNetName"i { return c.reverseDirection("DstNetName"), nil }
/ "SrcNetRole"i { return c.reverseDirection("SrcNetRole"), nil }
/ "DstNetRole"i { return c.reverseDirection("DstNetRole"), nil }
/ "SrcNetSite"i { return c.reverseDirection("SrcNetSite"), nil }
/ "DstNetSite"i { return c.reverseDirection("DstNetSite"), nil }
/ "SrcNetRegion"i { return c.reverseDirection("SrcNetRegion"), nil }
/ "DstNetRegion"i { return c.reverseDirection("DstNetRegion"), nil }
/ "SrcNetTenant"i { return c.reverseDirection("SrcNetTenant"), nil }
/ "DstNetTenant"i { return c.reverseDirection("DstNetTenant"), nil }
/ "InIfName"i { return c.reverseDirection("InIfName"), nil }
/ "OutIfName"i { return c.reverseDirection("OutIfName"), nil }
/ "InIfDescription"i { return c.reverseDirection("InIfDescription"), nil }
/ "OutIfDescription"i { return c.reverseDirection("OutIfDescription"), nil }
/ "InIfConnectivity"i { return c.reverseDirection("InIfConnectivity"), nil }
/ "OutIfConnectivity"i { return c.reverseDirection("OutIfConnectivity"), nil }
/ "InIfProvider"i { return c.reverseDirection("InIfProvider"), nil }
/ "OutIfProvider"i { return c.reverseDirection("OutIfProvider"), nil }) _
rcond:RConditionStringExpr {
return fmt.Sprintf("%s %s", toString(column), toString(rcond)), nil
}
@@ -124,16 +105,16 @@ RConditionStringExpr "condition on string" ←
}
ConditionBoundaryExpr "condition on boundary" ←
column:("InIfBoundary"i { return "InIfBoundary", nil }
/ "OutIfBoundary"i { return "OutIfBoundary", nil }) _
column:("InIfBoundary"i { return c.reverseDirection("InIfBoundary"), nil }
/ "OutIfBoundary"i { return c.reverseDirection("OutIfBoundary"), nil }) _
operator:("=" / "!=") _
boundary:("external"i / "internal"i / "undefined"i) {
return fmt.Sprintf("%s %s %s", toString(column), toString(operator),
quote(strings.ToLower(toString(boundary)))), nil
}
ConditionSpeedExpr "condition on speed" ←
column:("InIfSpeed"i { return "InIfSpeed", nil }
/ "OutIfSpeed"i { return "OutIfSpeed", nil }) _
column:("InIfSpeed"i { return c.reverseDirection("InIfSpeed"), nil }
/ "OutIfSpeed"i { return c.reverseDirection("OutIfSpeed"), nil }) _
operator:("=" / ">=" / "<=" / "<" / ">" / "!=") _
value:Unsigned64 {
return fmt.Sprintf("%s %s %s", toString(column), toString(operator), toString(value)), nil
@@ -145,15 +126,15 @@ ConditionForwardingStatusExpr "condition on forwarding status" ←
return fmt.Sprintf("%s %s %s", toString(column), toString(operator), toString(value)), nil
}
ConditionPortExpr "condition on port" ←
column:("SrcPort"i { return "SrcPort", nil }
/ "DstPort"i { return "DstPort", nil }) _
column:("SrcPort"i #{ c.state["main-table-only"] = true ; return nil } { return c.reverseDirection("SrcPort"), nil }
/ "DstPort"i #{ c.state["main-table-only"] = true ; return nil } { return c.reverseDirection("DstPort"), nil }) _
operator:("=" / ">=" / "<=" / "<" / ">" / "!=") _ value:Unsigned16 {
return fmt.Sprintf("%s %s %s", toString(column), toString(operator), toString(value)), nil
}
ConditionASExpr "condition on AS number" ←
column:("SrcAS"i { return "SrcAS", nil }
/ "DstAS"i { return "DstAS", nil }) _
column:("SrcAS"i { return c.reverseDirection("SrcAS"), nil }
/ "DstAS"i { return c.reverseDirection("DstAS"), nil }) _
rcond:RConditionASExpr {
return fmt.Sprintf("%s %s", toString(column), toString(rcond)), nil
}

281
console/filter/parser_test.go
View File

@@ -13,110 +13,192 @@ func TestValidFilter(t *testing.T) {
cases := []struct {
Input string
Output string
MetaIn Meta
MetaOut Meta
}{
{`ExporterName = 'something'`, `ExporterName = 'something'`},
{`exportername = 'something'`, `ExporterName = 'something'`},
{`ExporterName='something'`, `ExporterName = 'something'`},
{`ExporterName="something"`, `ExporterName = 'something'`},
{`ExporterName="something'"`, `ExporterName = 'something\''`},
{`ExporterName="something\"`, `ExporterName = 'something\\'`},
{`ExporterName!="something"`, `ExporterName != 'something'`},
{`ExporterName IN ("something")`, `ExporterName IN ('something')`},
{`ExporterName IN ("something","something else")`, `ExporterName IN ('something', 'something else')`},
{`ExporterName LIKE "something%"`, `ExporterName LIKE 'something%'`},
{`ExporterName UNLIKE "something%"`, `ExporterName NOT LIKE 'something%'`},
{`ExporterName IUNLIKE "something%"`, `ExporterName NOT ILIKE 'something%'`},
{`ExporterName="something with spaces"`, `ExporterName = 'something with spaces'`},
{`ExporterName="something with 'quotes'"`, `ExporterName = 'something with \'quotes\''`},
{`ExporterAddress=203.0.113.1`, `ExporterAddress = toIPv6('203.0.113.1')`},
{`ExporterAddress=2001:db8::1`, `ExporterAddress = toIPv6('2001:db8::1')`},
{`ExporterAddress=2001:db8:0::1`, `ExporterAddress = toIPv6('2001:db8::1')`},
{`ExporterAddress << 2001:db8:0::/64`,
`ExporterAddress BETWEEN toIPv6('2001:db8::') AND toIPv6('2001:db8::ffff:ffff:ffff:ffff')`},
{`ExporterAddress << 2001:db8::c000/115`,
`ExporterAddress BETWEEN toIPv6('2001:db8::c000') AND toIPv6('2001:db8::dfff')`},
{`ExporterAddress << 192.168.0.0/24`,
`ExporterAddress BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`},
{`DstAddr << 192.168.0.0/24`,
`DstAddr BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`},
{`SrcAddr << 192.168.0.1/24`,
`SrcAddr BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`},
{`DstAddr !<< 192.168.0.0/24`,
`DstAddr NOT BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`},
{`DstAddr !<< 192.168.0.128/27`,
`DstAddr NOT BETWEEN toIPv6('::ffff:192.168.0.128') AND toIPv6('::ffff:192.168.0.159')`},
{`ExporterGroup= "group"`, `ExporterGroup = 'group'`},
{`SrcAddr=203.0.113.1`, `SrcAddr = toIPv6('203.0.113.1')`},
{`DstAddr=203.0.113.2`, `DstAddr = toIPv6('203.0.113.2')`},
{`SrcNetName="alpha"`, `SrcNetName = 'alpha'`},
{`DstNetName="alpha"`, `DstNetName = 'alpha'`},
{`DstNetRole="stuff"`, `DstNetRole = 'stuff'`},
{`SrcNetTenant="mobile"`, `SrcNetTenant = 'mobile'`},
{`SrcAS=12322`, `SrcAS = 12322`},
{`SrcAS=AS12322`, `SrcAS = 12322`},
{`SrcAS=as12322`, `SrcAS = 12322`},
{`SrcAS IN(12322, 29447)`, `SrcAS IN (12322, 29447)`},
{`SrcAS IN( 12322 , 29447 )`, `SrcAS IN (12322, 29447)`},
{`SrcAS NOTIN(12322, 29447)`, `SrcAS NOT IN (12322, 29447)`},
{`SrcAS NOTIN (AS12322, 29447)`, `SrcAS NOT IN (12322, 29447)`},
{`DstAS=12322`, `DstAS = 12322`},
{`SrcCountry='FR'`, `SrcCountry = 'FR'`},
{`DstCountry='FR'`, `DstCountry = 'FR'`},
{`InIfName='Gi0/0/0/1'`, `InIfName = 'Gi0/0/0/1'`},
{`OutIfName = 'Gi0/0/0/1'`, `OutIfName = 'Gi0/0/0/1'`},
{`InIfDescription='Some description'`, `InIfDescription = 'Some description'`},
{`OutIfDescription='Some other description'`, `OutIfDescription = 'Some other description'`},
{`InIfSpeed>=1000`, `InIfSpeed >= 1000`},
{`InIfSpeed!=1000`, `InIfSpeed != 1000`},
{`InIfSpeed<1000`, `InIfSpeed < 1000`},
{`OutIfSpeed!=1000`, `OutIfSpeed != 1000`},
{`InIfConnectivity = 'pni'`, `InIfConnectivity = 'pni'`},
{`OutIfConnectivity = 'ix'`, `OutIfConnectivity = 'ix'`},
{`InIfProvider = 'cogent'`, `InIfProvider = 'cogent'`},
{`OutIfProvider = 'telia'`, `OutIfProvider = 'telia'`},
{`InIfBoundary = external`, `InIfBoundary = 'external'`},
{`InIfBoundary = EXTERNAL`, `InIfBoundary = 'external'`},
{`OutIfBoundary != internal`, `OutIfBoundary != 'internal'`},
{`EType = ipv4`, `EType = 2048`},
{`EType != ipv6`, `EType != 34525`},
{`Proto = 1`, `Proto = 1`},
{`Proto = 'gre'`, `dictGetOrDefault('protocols', 'name', Proto, '???') = 'gre'`},
{`SrcPort = 80`, `SrcPort = 80`},
{`DstPort > 1024`, `DstPort > 1024`},
{`ForwardingStatus >= 128`, `ForwardingStatus >= 128`},
{`PacketSize > 1500`, `Bytes/Packets > 1500`},
{`DstPort > 1024 AND SrcPort < 1024`, `DstPort > 1024 AND SrcPort < 1024`},
{`DstPort > 1024 OR SrcPort < 1024`, `DstPort > 1024 OR SrcPort < 1024`},
{`NOT DstPort > 1024 AND SrcPort < 1024`, `NOT DstPort > 1024 AND SrcPort < 1024`},
{`not DstPort > 1024 and SrcPort < 1024`, `NOT DstPort > 1024 AND SrcPort < 1024`},
{`DstPort > 1024 AND SrcPort < 1024 OR InIfSpeed >= 1000`,
`DstPort > 1024 AND SrcPort < 1024 OR InIfSpeed >= 1000`},
{`DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`,
`DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`},
{` DstPort > 1024 AND ( SrcPort < 1024 OR InIfSpeed >= 1000 ) `,
`DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`},
{`DstPort > 1024 AND(SrcPort < 1024 OR InIfSpeed >= 1000)`,
`DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`},
{`DstPort > 1024
{Input: `ExporterName = 'something'`, Output: `ExporterName = 'something'`},
{Input: `ExporterName = 'something'`, Output: `ExporterName = 'something'`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `exportername = 'something'`, Output: `ExporterName = 'something'`},
{Input: `ExporterName='something'`, Output: `ExporterName = 'something'`},
{Input: `ExporterName="something"`, Output: `ExporterName = 'something'`},
{Input: `ExporterName="something'"`, Output: `ExporterName = 'something\''`},
{Input: `ExporterName="something\"`, Output: `ExporterName = 'something\\'`},
{Input: `ExporterName!="something"`, Output: `ExporterName != 'something'`},
{Input: `ExporterName IN ("something")`, Output: `ExporterName IN ('something')`},
{Input: `ExporterName IN ("something","something else")`, Output: `ExporterName IN ('something', 'something else')`},
{Input: `ExporterName LIKE "something%"`, Output: `ExporterName LIKE 'something%'`},
{Input: `ExporterName UNLIKE "something%"`, Output: `ExporterName NOT LIKE 'something%'`},
{Input: `ExporterName IUNLIKE "something%"`, Output: `ExporterName NOT ILIKE 'something%'`},
{Input: `ExporterName="something with spaces"`, Output: `ExporterName = 'something with spaces'`},
{Input: `ExporterName="something with 'quotes'"`, Output: `ExporterName = 'something with \'quotes\''`},
{Input: `ExporterAddress=203.0.113.1`, Output: `ExporterAddress = toIPv6('203.0.113.1')`},
{Input: `ExporterAddress=2001:db8::1`, Output: `ExporterAddress = toIPv6('2001:db8::1')`},
{Input: `ExporterAddress=2001:db8:0::1`, Output: `ExporterAddress = toIPv6('2001:db8::1')`},
{
Input: `ExporterAddress << 2001:db8:0::/64`,
Output: `ExporterAddress BETWEEN toIPv6('2001:db8::') AND toIPv6('2001:db8::ffff:ffff:ffff:ffff')`,
}, {
Input: `ExporterAddress << 2001:db8::c000/115`,
Output: `ExporterAddress BETWEEN toIPv6('2001:db8::c000') AND toIPv6('2001:db8::dfff')`,
}, {
Input: `ExporterAddress << 192.168.0.0/24`,
Output: `ExporterAddress BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`,
}, {
Input: `DstAddr << 192.168.0.0/24`,
Output: `DstAddr BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`,
MetaOut: Meta{MainTableRequired: true},
}, {
Input: `DstAddr << 192.168.0.0/24`,
Output: `SrcAddr BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`,
MetaIn: Meta{ReverseDirection: true},
MetaOut: Meta{ReverseDirection: true, MainTableRequired: true},
}, {
Input: `SrcAddr << 192.168.0.1/24`,
Output: `SrcAddr BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`,
MetaOut: Meta{MainTableRequired: true},
}, {
Input: `DstAddr !<< 192.168.0.0/24`,
Output: `DstAddr NOT BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`,
MetaOut: Meta{MainTableRequired: true},
}, {
Input: `DstAddr !<< 192.168.0.128/27`,
Output: `DstAddr NOT BETWEEN toIPv6('::ffff:192.168.0.128') AND toIPv6('::ffff:192.168.0.159')`,
MetaOut: Meta{MainTableRequired: true},
},
{Input: `ExporterGroup= "group"`, Output: `ExporterGroup = 'group'`},
{Input: `SrcAddr=203.0.113.1`, Output: `SrcAddr = toIPv6('203.0.113.1')`,
MetaOut: Meta{MainTableRequired: true}},
{Input: `DstAddr=203.0.113.2`, Output: `DstAddr = toIPv6('203.0.113.2')`,
MetaOut: Meta{MainTableRequired: true}},
{Input: `SrcNetName="alpha"`, Output: `SrcNetName = 'alpha'`},
{Input: `DstNetName="alpha"`, Output: `DstNetName = 'alpha'`},
{Input: `DstNetRole="stuff"`, Output: `DstNetRole = 'stuff'`},
{Input: `SrcNetTenant="mobile"`, Output: `SrcNetTenant = 'mobile'`},
{Input: `SrcAS=12322`, Output: `SrcAS = 12322`},
{Input: `SrcAS=AS12322`, Output: `SrcAS = 12322`},
{Input: `SrcAS=AS12322`, Output: `DstAS = 12322`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `SrcAS=as12322`, Output: `SrcAS = 12322`},
{Input: `SrcAS IN(12322, 29447)`, Output: `SrcAS IN (12322, 29447)`},
{Input: `SrcAS IN( 12322 , 29447 )`, Output: `SrcAS IN (12322, 29447)`},
{Input: `SrcAS NOTIN(12322, 29447)`, Output: `SrcAS NOT IN (12322, 29447)`},
{Input: `SrcAS NOTIN (AS12322, 29447)`, Output: `SrcAS NOT IN (12322, 29447)`},
{Input: `DstAS=12322`, Output: `DstAS = 12322`},
{Input: `SrcCountry='FR'`, Output: `SrcCountry = 'FR'`},
{Input: `SrcCountry='FR'`, Output: `DstCountry = 'FR'`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `DstCountry='FR'`, Output: `DstCountry = 'FR'`},
{Input: `DstCountry='FR'`, Output: `SrcCountry = 'FR'`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `InIfName='Gi0/0/0/1'`, Output: `InIfName = 'Gi0/0/0/1'`},
{Input: `InIfName='Gi0/0/0/1'`, Output: `OutIfName = 'Gi0/0/0/1'`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `OutIfName = 'Gi0/0/0/1'`, Output: `OutIfName = 'Gi0/0/0/1'`},
{Input: `OutIfName = 'Gi0/0/0/1'`, Output: `InIfName = 'Gi0/0/0/1'`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `InIfDescription='Some description'`, Output: `InIfDescription = 'Some description'`},
{Input: `InIfDescription='Some description'`, Output: `OutIfDescription = 'Some description'`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `OutIfDescription='Some other description'`, Output: `OutIfDescription = 'Some other description'`},
{Input: `OutIfDescription='Some other description'`, Output: `InIfDescription = 'Some other description'`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `InIfSpeed>=1000`, Output: `InIfSpeed >= 1000`},
{Input: `InIfSpeed>=1000`, Output: `OutIfSpeed >= 1000`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `InIfSpeed!=1000`, Output: `InIfSpeed != 1000`},
{Input: `InIfSpeed<1000`, Output: `InIfSpeed < 1000`},
{Input: `OutIfSpeed!=1000`, Output: `OutIfSpeed != 1000`},
{Input: `OutIfSpeed!=1000`, Output: `InIfSpeed != 1000`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `InIfConnectivity = 'pni'`, Output: `InIfConnectivity = 'pni'`},
{Input: `InIfConnectivity = 'pni'`, Output: `OutIfConnectivity = 'pni'`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `OutIfConnectivity = 'ix'`, Output: `OutIfConnectivity = 'ix'`},
{Input: `OutIfConnectivity = 'ix'`, Output: `InIfConnectivity = 'ix'`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `InIfProvider = 'cogent'`, Output: `InIfProvider = 'cogent'`},
{Input: `InIfProvider = 'cogent'`, Output: `OutIfProvider = 'cogent'`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `OutIfProvider = 'telia'`, Output: `OutIfProvider = 'telia'`},
{Input: `OutIfProvider = 'telia'`, Output: `InIfProvider = 'telia'`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `InIfBoundary = external`, Output: `InIfBoundary = 'external'`},
{Input: `InIfBoundary = external`, Output: `OutIfBoundary = 'external'`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `InIfBoundary = EXTERNAL`, Output: `InIfBoundary = 'external'`},
{Input: `InIfBoundary = EXTERNAL`, Output: `OutIfBoundary = 'external'`,
MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}},
{Input: `OutIfBoundary != internal`, Output: `OutIfBoundary != 'internal'`},
{Input: `EType = ipv4`, Output: `EType = 2048`},
{Input: `EType != ipv6`, Output: `EType != 34525`},
{Input: `Proto = 1`, Output: `Proto = 1`},
{Input: `Proto = 'gre'`, Output: `dictGetOrDefault('protocols', 'name', Proto, '???') = 'gre'`},
{Input: `SrcPort = 80`, Output: `SrcPort = 80`,
MetaOut: Meta{MainTableRequired: true}},
{Input: `SrcPort = 80`, Output: `DstPort = 80`,
MetaIn: Meta{ReverseDirection: true},
MetaOut: Meta{ReverseDirection: true, MainTableRequired: true}},
{Input: `DstPort > 1024`, Output: `DstPort > 1024`,
MetaOut: Meta{MainTableRequired: true}},
{Input: `ForwardingStatus >= 128`, Output: `ForwardingStatus >= 128`},
{Input: `PacketSize > 1500`, Output: `Bytes/Packets > 1500`},
{Input: `DstPort > 1024 AND SrcPort < 1024`, Output: `DstPort > 1024 AND SrcPort < 1024`,
MetaOut: Meta{MainTableRequired: true}},
{Input: `DstPort > 1024 OR SrcPort < 1024`, Output: `DstPort > 1024 OR SrcPort < 1024`,
MetaOut: Meta{MainTableRequired: true}},
{Input: `NOT DstPort > 1024 AND SrcPort < 1024`, Output: `NOT DstPort > 1024 AND SrcPort < 1024`,
MetaOut: Meta{MainTableRequired: true}},
{Input: `not DstPort > 1024 and SrcPort < 1024`, Output: `NOT DstPort > 1024 AND SrcPort < 1024`,
MetaOut: Meta{MainTableRequired: true}},
{
Input: `DstPort > 1024 AND SrcPort < 1024 OR InIfSpeed >= 1000`,
Output: `DstPort > 1024 AND SrcPort < 1024 OR InIfSpeed >= 1000`,
MetaOut: Meta{MainTableRequired: true},
}, {
Input: `DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`,
Output: `DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`,
MetaOut: Meta{MainTableRequired: true},
}, {
Input: ` DstPort > 1024 AND ( SrcPort < 1024 OR InIfSpeed >= 1000 ) `,
Output: `DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`,
MetaOut: Meta{MainTableRequired: true},
}, {
Input: `DstPort > 1024 AND(SrcPort < 1024 OR InIfSpeed >= 1000)`,
Output: `DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`,
MetaOut: Meta{MainTableRequired: true},
}, {
Input: `DstPort > 1024
AND (SrcPort < 1024 OR InIfSpeed >= 1000)`,
`DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`},
{`(ExporterAddress=203.0.113.1)`, `(ExporterAddress = toIPv6('203.0.113.1'))`},
{`ForwardingStatus >= 128 -- Nothing`, `ForwardingStatus >= 128`},
{`
Output: `DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`,
MetaOut: Meta{MainTableRequired: true},
},
{Input: `(ExporterAddress=203.0.113.1)`, Output: `(ExporterAddress = toIPv6('203.0.113.1'))`},
{Input: `ForwardingStatus >= 128 -- Nothing`, Output: `ForwardingStatus >= 128`},
{
Input: `
-- Example of commented request
-- Here we go
DstPort > 1024 -- Non-privileged port
AND SrcAS = AS12322 -- Proxad ASN`, `DstPort > 1024 AND SrcAS = 12322`},
{`InIfDescription = "This contains a -- comment" -- nope`,
`InIfDescription = 'This contains a -- comment'`},
{`InIfDescription = "This contains a /* comment"`,
`InIfDescription = 'This contains a /* comment'`},
{`OutIfProvider /* That's the output provider */ = 'telia'`, `OutIfProvider = 'telia'`},
{`OutIfProvider /* That's the
output provider */ = 'telia'`, `OutIfProvider = 'telia'`},
AND SrcAS = AS12322 -- Proxad ASN`,
Output: `DstPort > 1024 AND SrcAS = 12322`,
MetaOut: Meta{MainTableRequired: true},
}, {
Input: `InIfDescription = "This contains a -- comment" -- nope`,
Output: `InIfDescription = 'This contains a -- comment'`,
}, {
Input: `InIfDescription = "This contains a /* comment"`,
Output: `InIfDescription = 'This contains a /* comment'`,
},
{Input: `OutIfProvider /* That's the output provider */ = 'telia'`, Output: `OutIfProvider = 'telia'`},
{
Input: `OutIfProvider /* That's the
output provider */ = 'telia'`,
Output: `OutIfProvider = 'telia'`,
},
}
for _, tc := range cases {
got, err := Parse("", []byte(tc.Input))
got, err := Parse("", []byte(tc.Input), GlobalStore("meta", &tc.MetaIn))
if err != nil {
t.Errorf("Parse(%q) error:\n%+v", tc.Input, err)
continue
@@ -124,6 +206,9 @@ output provider */ = 'telia'`, `OutIfProvider = 'telia'`},
if diff := helpers.Diff(got.(string), tc.Output); diff != "" {
t.Errorf("Parse(%q) (-got, +want):\n%s", tc.Input, diff)
}
if diff := helpers.Diff(tc.MetaIn, tc.MetaOut); diff != "" {
t.Errorf("Parse(%q) meta (-got, +want):\n%s", tc.Input, diff)
}
}
}
@@ -157,7 +242,7 @@ func TestInvalidFilter(t *testing.T) {
{`SrcAS IN (AS12322,`},
}
for _, tc := range cases {
out, err := Parse("", []byte(tc.Input))
out, err := Parse("", []byte(tc.Input), GlobalStore("meta", &Meta{}))
t.Logf("out: %v", out)
if err == nil {
t.Errorf("Parse(%q) didn't throw an error", tc.Input)

2
console/graph.go
View File

@@ -126,7 +126,7 @@ func (c *Component) graphHandlerFunc(gc *gin.Context) {
if resolution < time.Second {
resolution = time.Second
}
sqlQuery = c.queryFlowsTable(sqlQuery,
sqlQuery = c.queryFlowsTable(sqlQuery, input.Filter.mainTableRequired,
input.Start, input.End, resolution)
gc.Header("X-SQL-Query", strings.ReplaceAll(sqlQuery, "\n", " "))

2
console/graph_test.go
View File

@@ -93,7 +93,7 @@ ORDER BY time WITH FILL
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Dimensions: []queryColumn{},
Filter: queryFilter{"DstCountry = 'FR' AND SrcCountry = 'US'"},
Filter: queryFilter{filter: "DstCountry = 'FR' AND SrcCountry = 'US'"},
Units: "l3bps",
},
Expected: `

15
console/query.go
View File

@@ -124,6 +124,8 @@ func (gc *queryColumn) UnmarshalText(input []byte) error {
type queryFilter struct {
filter string
reverseFilter string
mainTableRequired bool
}
func (gf queryFilter) MarshalText() ([]byte, error) {
@@ -131,14 +133,21 @@ func (gf queryFilter) MarshalText() ([]byte, error) {
}
func (gf *queryFilter) UnmarshalText(input []byte) error {
if strings.TrimSpace(string(input)) == "" {
*gf = queryFilter{""}
*gf = queryFilter{}
return nil
}
got, err := filter.Parse("", input)
meta := &filter.Meta{}
direct, err := filter.Parse("", input, filter.GlobalStore("meta", meta))
if err != nil {
return fmt.Errorf("cannot parse filter: %s", filter.HumanError(err))
}
*gf = queryFilter{got.(string)}
meta = &filter.Meta{ReverseDirection: true}
reverse, err := filter.Parse("", input, filter.GlobalStore("meta", meta))
*gf = queryFilter{
filter: direct.(string),
reverseFilter: reverse.(string),
mainTableRequired: meta.MainTableRequired,
}
return nil
}

2
console/sankey.go
View File

@@ -115,7 +115,7 @@ func (c *Component) sankeyHandlerFunc(gc *gin.Context) {
}
// Prepare and execute query
sqlQuery = c.queryFlowsTable(sqlQuery,
sqlQuery = c.queryFlowsTable(sqlQuery, input.Filter.mainTableRequired,
input.Start, input.End, resolution)
gc.Header("X-SQL-Query", strings.ReplaceAll(sqlQuery, "\n", " "))
results := []struct {

2
console/sankey_test.go
View File

@@ -93,7 +93,7 @@ ORDER BY xps DESC`,
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Dimensions: []queryColumn{queryColumnSrcAS, queryColumnExporterName},
Limit: 10,
Filter: queryFilter{"DstCountry = 'FR'"},
Filter: queryFilter{filter: "DstCountry = 'FR'"},
Units: "l3bps",
},
Expected: `

7
console/widgets.go
View File

@@ -108,6 +108,7 @@ func (c *Component) widgetTopHandlerFunc(gc *gin.Context) {
selector string
groupby string
filter string
mainTableRequired bool
)
switch gc.Param("name") {
@@ -139,9 +140,11 @@ func (c *Component) widgetTopHandlerFunc(gc *gin.Context) {
case "src-port":
selector = `concat(dictGetOrDefault('protocols', 'name', Proto, '???'), '/', toString(SrcPort))`
groupby = `Proto, SrcPort`
mainTableRequired = true
case "dst-port":
selector = `concat(dictGetOrDefault('protocols', 'name', Proto, '???'), '/', toString(DstPort))`
groupby = `Proto, DstPort`
mainTableRequired = true
}
if groupby == "" {
groupby = selector
@@ -160,7 +163,7 @@ WHERE {timefilter}
GROUP BY %s
ORDER BY Percent DESC
LIMIT 5
`, filter, selector, selector, filter, groupby), now.Add(-5*time.Minute), now, time.Minute)
`, filter, selector, selector, filter, groupby), mainTableRequired, now.Add(-5*time.Minute), now, time.Minute)
gc.Header("X-SQL-Query", query)
results := []topResult{}
@@ -202,7 +205,7 @@ GROUP BY Time
ORDER BY Time WITH FILL
FROM toStartOfInterval({timefilter.Start}, INTERVAL %s second)
TO {timefilter.Stop}
STEP %s`, slot, slot, slot, slot), now.Add(-24*time.Hour), now, time.Duration(interval)*time.Second)
STEP %s`, slot, slot, slot, slot), false, now.Add(-24*time.Hour), now, time.Duration(interval)*time.Second)
gc.Header("X-SQL-Query", query)
results := []struct {