From 1aa260bae20b8c51dbc74c0b44e25dd05b595377 Mon Sep 17 00:00:00 2001 From: Vincent Bernat Date: Thu, 4 Aug 2022 18:32:44 +0200 Subject: [PATCH] console/filter: let parser tells us if we need the main table or not This is more robust this way. We also introduce the ability to reverse the direction of a filter. --- console/clickhouse.go | 9 +- console/clickhouse_test.go | 28 ++-- console/filter.go | 8 +- console/filter/error_test.go | 6 +- console/filter/helpers.go | 50 ++++++ console/filter/parser.peg | 91 +++++------ console/filter/parser_test.go | 285 ++++++++++++++++++++++------------ console/graph.go | 2 +- console/graph_test.go | 2 +- console/query.go | 17 +- console/sankey.go | 2 +- console/sankey_test.go | 2 +- console/widgets.go | 13 +- 13 files changed, 322 insertions(+), 193 deletions(-) diff --git a/console/clickhouse.go b/console/clickhouse.go index 444d1a62..ee42e8ce 100644 --- a/console/clickhouse.go +++ b/console/clickhouse.go @@ -12,10 +12,7 @@ import ( "time" ) -var ( - addressOrPortRegexp = regexp.MustCompile(`\b(?:Src|Dst)(?:Port|Addr)\b`) - resolutionRegexp = regexp.MustCompile(`{resolution->(\d+)}`) -) +var resolutionRegexp = regexp.MustCompile(`{resolution->(\d+)}`) // flowsTable describe a consolidated or unconsolidated flows table. type flowsTable struct { @@ -29,14 +26,14 @@ type flowsTable struct { // should contain `{table}` which will be replaced by the appropriate // flows table and {timefilter} which will be replaced by the // appropriate time filter. -func (c *Component) queryFlowsTable(query string, start, end time.Time, targetResolution time.Duration) string { +func (c *Component) queryFlowsTable(query string, mainTableRequired bool, start, end time.Time, targetResolution time.Duration) string { c.flowsTablesLock.RLock() defer c.flowsTablesLock.RUnlock() // Select table table := "flows" resolution := time.Second - if !addressOrPortRegexp.MatchString(query) { + if !mainTableRequired { // We can use the consolidated data. The first // criteria is to find the tables matching the time // criteria. diff --git a/console/clickhouse_test.go b/console/clickhouse_test.go index 404dd9b4..2095fd83 100644 --- a/console/clickhouse_test.go +++ b/console/clickhouse_test.go @@ -72,20 +72,22 @@ AND engine LIKE '%MergeTree' func TestQueryFlowsTables(t *testing.T) { cases := []struct { - Description string - Tables []flowsTable - Query string - Start time.Time - End time.Time - Resolution time.Duration - Expected string + Description string + Tables []flowsTable + Query string + MainTableRequired bool + Start time.Time + End time.Time + Resolution time.Duration + Expected string }{ { - Description: "query with source port", - Query: "SELECT TimeReceived, SrcPort FROM {table} WHERE {timefilter}", - Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC), - End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC), - Expected: "SELECT TimeReceived, SrcPort FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')", + Description: "query with source port", + Query: "SELECT TimeReceived, SrcPort FROM {table} WHERE {timefilter}", + MainTableRequired: true, + Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC), + Expected: "SELECT TimeReceived, SrcPort FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')", }, { Description: "only flows table available", Tables: []flowsTable{{"flows", 0, time.Date(2022, 03, 10, 15, 45, 10, 0, time.UTC)}}, @@ -194,7 +196,7 @@ func TestQueryFlowsTables(t *testing.T) { for _, tc := range cases { t.Run(tc.Description, func(t *testing.T) { c.flowsTables = tc.Tables - got := c.queryFlowsTable(tc.Query, tc.Start, tc.End, tc.Resolution) + got := c.queryFlowsTable(tc.Query, tc.MainTableRequired, tc.Start, tc.End, tc.Resolution) if diff := helpers.Diff(got, tc.Expected); diff != "" { t.Fatalf("queryFlowsTable(): (-got, +want):\n%s", diff) } diff --git a/console/filter.go b/console/filter.go index 8eca1f72..4f465f6b 100644 --- a/console/filter.go +++ b/console/filter.go @@ -42,7 +42,7 @@ func (c *Component) filterValidateHandlerFunc(gc *gin.Context) { }) return } - got, err := filter.Parse("", []byte(input.Filter)) + got, err := filter.Parse("", []byte(input.Filter), filter.GlobalStore("meta", &filter.Meta{})) if err == nil { gc.JSON(http.StatusOK, filterValidateHandlerOutput{ Message: "ok", @@ -84,7 +84,8 @@ func (c *Component) filterCompleteHandlerFunc(gc *gin.Context) { completions := []filterCompletion{} switch input.What { case "column": - _, err := filter.Parse("", []byte{}, filter.Entrypoint("ConditionExpr")) + _, err := filter.Parse("", []byte{}, + filter.Entrypoint("ConditionExpr"), filter.GlobalStore("meta", &filter.Meta{})) if err != nil { for _, candidate := range filter.Expected(err) { if !strings.HasSuffix(candidate, `"i`) { @@ -100,7 +101,8 @@ func (c *Component) filterCompleteHandlerFunc(gc *gin.Context) { case "operator": _, err := filter.Parse("", []byte(fmt.Sprintf("%s ", input.Column)), - filter.Entrypoint("ConditionExpr")) + filter.Entrypoint("ConditionExpr"), + filter.GlobalStore("meta", &filter.Meta{})) if err != nil { for _, candidate := range filter.Expected(err) { if !strings.HasPrefix(candidate, `"`) { diff --git a/console/filter/error_test.go b/console/filter/error_test.go index 3a1c085a..0bff06a9 100644 --- a/console/filter/error_test.go +++ b/console/filter/error_test.go @@ -13,7 +13,7 @@ func TestFilterHumanError(t *testing.T) { _, err := Parse("", []byte(` InIfDescription = "Gi0/0/0/0" AND Proto = 1000 -OR `)) +OR `), GlobalStore("meta", &Meta{})) expected := "at line 3, position 13: expecting an unsigned 8-bit integer" if diff := helpers.Diff(HumanError(err), expected); diff != "" { t.Errorf("HumanError() (-got, +want):\n%s", diff) @@ -24,7 +24,7 @@ func TestAllErrors(t *testing.T) { _, err := Parse("", []byte(` InIfDescription = "Gi0/0/0/0" AND Proto = 1000 -OR`)) +OR`), GlobalStore("meta", &Meta{})) // Currently, the parser stops at the first error. expected := Errors{ oneError{ @@ -40,7 +40,7 @@ OR`)) } func TestExpected(t *testing.T) { - _, err := Parse("", []byte{}, Entrypoint("ConditionBoundaryExpr")) + _, err := Parse("", []byte{}, Entrypoint("ConditionBoundaryExpr"), GlobalStore("meta", &Meta{})) expected := []string{`"InIfBoundary"i`, `"OutIfBoundary"i`} if diff := helpers.Diff(Expected(err), expected); diff != "" { t.Errorf("AllErrors() (-got, +want):\n%s", diff) diff --git a/console/filter/helpers.go b/console/filter/helpers.go index 5de080ea..108321cd 100644 --- a/console/filter/helpers.go +++ b/console/filter/helpers.go @@ -6,8 +6,36 @@ package filter import ( "encoding/binary" "net" + "strings" ) +// Meta is used to inject/retrieve state from the parser. +type Meta struct { + // ReverseDirection tells if we require the reverse direction for the provided filter (used as input) + ReverseDirection bool + // MainTableRequired tells if the main table is required to execute the expression (used as output) + MainTableRequired bool +} + +func (c *current) reverseDirection(direct string) string { + if c.globalStore["meta"].(*Meta).ReverseDirection { + if strings.HasPrefix(direct, "Src") { + return "Dst" + direct[3:] + } + if strings.HasPrefix(direct, "Dst") { + return "Src" + direct[3:] + } + if strings.HasPrefix(direct, "In") { + return "Out" + direct[2:] + } + if strings.HasPrefix(direct, "Out") { + return "In" + direct[3:] + } + panic("no reverse?") + } + return direct +} + func lastIP(subnet *net.IPNet) net.IP { if subnet.IP.To4() != nil { // IPv4 case @@ -24,3 +52,25 @@ func lastIP(subnet *net.IPNet) net.IP { } return ip } + +func quote(v interface{}) string { + return "'" + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(toString(v)) + "'" +} + +func toSlice(v interface{}) []interface{} { + if v == nil { + return nil + } + return v.([]interface{}) +} + +func toString(v interface{}) string { + switch s := v.(type) { + case string: + return s + case []byte: + return string(s) + default: + panic("not a string") + } +} diff --git a/console/filter/parser.peg b/console/filter/parser.peg index bf380434..f48419c5 100644 --- a/console/filter/parser.peg +++ b/console/filter/parser.peg @@ -3,40 +3,19 @@ // SPDX-License-Identifier: AGPL-3.0-only package filter - - // Convert SQL-like language for filters to SQL. - + import ( "fmt" "net" "akvorado/common/helpers" ) - - func quote(v interface{}) string { - return "'" + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(toString(v)) + "'" - } - - func toSlice(v interface{}) []interface{} { - if v == nil { - return nil - } - return v.([]interface{}) - } - - func toString(v interface{}) string { - switch s := v.(type) { - case string: - return s - case []byte: - return string(s) - default: - panic("not a string") - } - } } Input ← _ expr:Expr _ EOF { + meta := c.globalStore["meta"].(*Meta) + _, ok := c.state["main-table-only"] + meta.MainTableRequired = ok return expr, nil } @@ -69,8 +48,10 @@ ConditionExpr "conditional" ← ColumnIP ← "ExporterAddress"i { return "ExporterAddress", nil } - / "SrcAddr"i { return "SrcAddr", nil } - / "DstAddr"i { return "DstAddr", nil } + / "SrcAddr"i #{ c.state["main-table-only"] = true ; return nil } + { return c.reverseDirection("SrcAddr"), nil } + / "DstAddr"i #{ c.state["main-table-only"] = true ; return nil } + { return c.reverseDirection("DstAddr"), nil } ConditionIPExpr "condition on IP" ← column:ColumnIP _ operator:("=" / "!=") _ ip:IP { @@ -92,26 +73,26 @@ ConditionStringExpr "condition on string" ← / "ExporterSite"i { return "ExporterSite", nil } / "ExporterRegion"i { return "ExporterRegion", nil } / "ExporterTenant"i { return "ExporterTenant", nil } - / "SrcCountry"i { return "SrcCountry", nil } - / "DstCountry"i { return "DstCountry", nil } - / "SrcNetName"i { return "SrcNetName", nil } - / "DstNetName"i { return "DstNetName", nil } - / "SrcNetRole"i { return "SrcNetRole", nil } - / "DstNetRole"i { return "DstNetRole", nil } - / "SrcNetSite"i { return "SrcNetSite", nil } - / "DstNetSite"i { return "DstNetSite", nil } - / "SrcNetRegion"i { return "SrcNetRegion", nil } - / "DstNetRegion"i { return "DstNetRegion", nil } - / "SrcNetTenant"i { return "SrcNetTenant", nil } - / "DstNetTenant"i { return "DstNetTenant", nil } - / "InIfName"i { return "InIfName", nil } - / "OutIfName"i { return "OutIfName", nil } - / "InIfDescription"i { return "InIfDescription", nil } - / "OutIfDescription"i { return "OutIfDescription", nil } - / "InIfConnectivity"i { return "InIfConnectivity", nil } - / "OutIfConnectivity"i { return "OutIfConnectivity", nil } - / "InIfProvider"i { return "InIfProvider", nil } - / "OutIfProvider"i { return "OutIfProvider", nil }) _ + / "SrcCountry"i { return c.reverseDirection("SrcCountry"), nil } + / "DstCountry"i { return c.reverseDirection("DstCountry"), nil } + / "SrcNetName"i { return c.reverseDirection("SrcNetName"), nil } + / "DstNetName"i { return c.reverseDirection("DstNetName"), nil } + / "SrcNetRole"i { return c.reverseDirection("SrcNetRole"), nil } + / "DstNetRole"i { return c.reverseDirection("DstNetRole"), nil } + / "SrcNetSite"i { return c.reverseDirection("SrcNetSite"), nil } + / "DstNetSite"i { return c.reverseDirection("DstNetSite"), nil } + / "SrcNetRegion"i { return c.reverseDirection("SrcNetRegion"), nil } + / "DstNetRegion"i { return c.reverseDirection("DstNetRegion"), nil } + / "SrcNetTenant"i { return c.reverseDirection("SrcNetTenant"), nil } + / "DstNetTenant"i { return c.reverseDirection("DstNetTenant"), nil } + / "InIfName"i { return c.reverseDirection("InIfName"), nil } + / "OutIfName"i { return c.reverseDirection("OutIfName"), nil } + / "InIfDescription"i { return c.reverseDirection("InIfDescription"), nil } + / "OutIfDescription"i { return c.reverseDirection("OutIfDescription"), nil } + / "InIfConnectivity"i { return c.reverseDirection("InIfConnectivity"), nil } + / "OutIfConnectivity"i { return c.reverseDirection("OutIfConnectivity"), nil } + / "InIfProvider"i { return c.reverseDirection("InIfProvider"), nil } + / "OutIfProvider"i { return c.reverseDirection("OutIfProvider"), nil }) _ rcond:RConditionStringExpr { return fmt.Sprintf("%s %s", toString(column), toString(rcond)), nil } @@ -124,16 +105,16 @@ RConditionStringExpr "condition on string" ← } ConditionBoundaryExpr "condition on boundary" ← - column:("InIfBoundary"i { return "InIfBoundary", nil } - / "OutIfBoundary"i { return "OutIfBoundary", nil }) _ + column:("InIfBoundary"i { return c.reverseDirection("InIfBoundary"), nil } + / "OutIfBoundary"i { return c.reverseDirection("OutIfBoundary"), nil }) _ operator:("=" / "!=") _ boundary:("external"i / "internal"i / "undefined"i) { return fmt.Sprintf("%s %s %s", toString(column), toString(operator), quote(strings.ToLower(toString(boundary)))), nil } ConditionSpeedExpr "condition on speed" ← - column:("InIfSpeed"i { return "InIfSpeed", nil } - / "OutIfSpeed"i { return "OutIfSpeed", nil }) _ + column:("InIfSpeed"i { return c.reverseDirection("InIfSpeed"), nil } + / "OutIfSpeed"i { return c.reverseDirection("OutIfSpeed"), nil }) _ operator:("=" / ">=" / "<=" / "<" / ">" / "!=") _ value:Unsigned64 { return fmt.Sprintf("%s %s %s", toString(column), toString(operator), toString(value)), nil @@ -145,15 +126,15 @@ ConditionForwardingStatusExpr "condition on forwarding status" ← return fmt.Sprintf("%s %s %s", toString(column), toString(operator), toString(value)), nil } ConditionPortExpr "condition on port" ← - column:("SrcPort"i { return "SrcPort", nil } - / "DstPort"i { return "DstPort", nil }) _ + column:("SrcPort"i #{ c.state["main-table-only"] = true ; return nil } { return c.reverseDirection("SrcPort"), nil } + / "DstPort"i #{ c.state["main-table-only"] = true ; return nil } { return c.reverseDirection("DstPort"), nil }) _ operator:("=" / ">=" / "<=" / "<" / ">" / "!=") _ value:Unsigned16 { return fmt.Sprintf("%s %s %s", toString(column), toString(operator), toString(value)), nil } ConditionASExpr "condition on AS number" ← - column:("SrcAS"i { return "SrcAS", nil } - / "DstAS"i { return "DstAS", nil }) _ + column:("SrcAS"i { return c.reverseDirection("SrcAS"), nil } + / "DstAS"i { return c.reverseDirection("DstAS"), nil }) _ rcond:RConditionASExpr { return fmt.Sprintf("%s %s", toString(column), toString(rcond)), nil } diff --git a/console/filter/parser_test.go b/console/filter/parser_test.go index d933a3b5..3f05efc7 100644 --- a/console/filter/parser_test.go +++ b/console/filter/parser_test.go @@ -11,112 +11,194 @@ import ( func TestValidFilter(t *testing.T) { cases := []struct { - Input string - Output string + Input string + Output string + MetaIn Meta + MetaOut Meta }{ - {`ExporterName = 'something'`, `ExporterName = 'something'`}, - {`exportername = 'something'`, `ExporterName = 'something'`}, - {`ExporterName='something'`, `ExporterName = 'something'`}, - {`ExporterName="something"`, `ExporterName = 'something'`}, - {`ExporterName="something'"`, `ExporterName = 'something\''`}, - {`ExporterName="something\"`, `ExporterName = 'something\\'`}, - {`ExporterName!="something"`, `ExporterName != 'something'`}, - {`ExporterName IN ("something")`, `ExporterName IN ('something')`}, - {`ExporterName IN ("something","something else")`, `ExporterName IN ('something', 'something else')`}, - {`ExporterName LIKE "something%"`, `ExporterName LIKE 'something%'`}, - {`ExporterName UNLIKE "something%"`, `ExporterName NOT LIKE 'something%'`}, - {`ExporterName IUNLIKE "something%"`, `ExporterName NOT ILIKE 'something%'`}, - {`ExporterName="something with spaces"`, `ExporterName = 'something with spaces'`}, - {`ExporterName="something with 'quotes'"`, `ExporterName = 'something with \'quotes\''`}, - {`ExporterAddress=203.0.113.1`, `ExporterAddress = toIPv6('203.0.113.1')`}, - {`ExporterAddress=2001:db8::1`, `ExporterAddress = toIPv6('2001:db8::1')`}, - {`ExporterAddress=2001:db8:0::1`, `ExporterAddress = toIPv6('2001:db8::1')`}, - {`ExporterAddress << 2001:db8:0::/64`, - `ExporterAddress BETWEEN toIPv6('2001:db8::') AND toIPv6('2001:db8::ffff:ffff:ffff:ffff')`}, - {`ExporterAddress << 2001:db8::c000/115`, - `ExporterAddress BETWEEN toIPv6('2001:db8::c000') AND toIPv6('2001:db8::dfff')`}, - {`ExporterAddress << 192.168.0.0/24`, - `ExporterAddress BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`}, - {`DstAddr << 192.168.0.0/24`, - `DstAddr BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`}, - {`SrcAddr << 192.168.0.1/24`, - `SrcAddr BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`}, - {`DstAddr !<< 192.168.0.0/24`, - `DstAddr NOT BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`}, - {`DstAddr !<< 192.168.0.128/27`, - `DstAddr NOT BETWEEN toIPv6('::ffff:192.168.0.128') AND toIPv6('::ffff:192.168.0.159')`}, - {`ExporterGroup= "group"`, `ExporterGroup = 'group'`}, - {`SrcAddr=203.0.113.1`, `SrcAddr = toIPv6('203.0.113.1')`}, - {`DstAddr=203.0.113.2`, `DstAddr = toIPv6('203.0.113.2')`}, - {`SrcNetName="alpha"`, `SrcNetName = 'alpha'`}, - {`DstNetName="alpha"`, `DstNetName = 'alpha'`}, - {`DstNetRole="stuff"`, `DstNetRole = 'stuff'`}, - {`SrcNetTenant="mobile"`, `SrcNetTenant = 'mobile'`}, - {`SrcAS=12322`, `SrcAS = 12322`}, - {`SrcAS=AS12322`, `SrcAS = 12322`}, - {`SrcAS=as12322`, `SrcAS = 12322`}, - {`SrcAS IN(12322, 29447)`, `SrcAS IN (12322, 29447)`}, - {`SrcAS IN( 12322 , 29447 )`, `SrcAS IN (12322, 29447)`}, - {`SrcAS NOTIN(12322, 29447)`, `SrcAS NOT IN (12322, 29447)`}, - {`SrcAS NOTIN (AS12322, 29447)`, `SrcAS NOT IN (12322, 29447)`}, - {`DstAS=12322`, `DstAS = 12322`}, - {`SrcCountry='FR'`, `SrcCountry = 'FR'`}, - {`DstCountry='FR'`, `DstCountry = 'FR'`}, - {`InIfName='Gi0/0/0/1'`, `InIfName = 'Gi0/0/0/1'`}, - {`OutIfName = 'Gi0/0/0/1'`, `OutIfName = 'Gi0/0/0/1'`}, - {`InIfDescription='Some description'`, `InIfDescription = 'Some description'`}, - {`OutIfDescription='Some other description'`, `OutIfDescription = 'Some other description'`}, - {`InIfSpeed>=1000`, `InIfSpeed >= 1000`}, - {`InIfSpeed!=1000`, `InIfSpeed != 1000`}, - {`InIfSpeed<1000`, `InIfSpeed < 1000`}, - {`OutIfSpeed!=1000`, `OutIfSpeed != 1000`}, - {`InIfConnectivity = 'pni'`, `InIfConnectivity = 'pni'`}, - {`OutIfConnectivity = 'ix'`, `OutIfConnectivity = 'ix'`}, - {`InIfProvider = 'cogent'`, `InIfProvider = 'cogent'`}, - {`OutIfProvider = 'telia'`, `OutIfProvider = 'telia'`}, - {`InIfBoundary = external`, `InIfBoundary = 'external'`}, - {`InIfBoundary = EXTERNAL`, `InIfBoundary = 'external'`}, - {`OutIfBoundary != internal`, `OutIfBoundary != 'internal'`}, - {`EType = ipv4`, `EType = 2048`}, - {`EType != ipv6`, `EType != 34525`}, - {`Proto = 1`, `Proto = 1`}, - {`Proto = 'gre'`, `dictGetOrDefault('protocols', 'name', Proto, '???') = 'gre'`}, - {`SrcPort = 80`, `SrcPort = 80`}, - {`DstPort > 1024`, `DstPort > 1024`}, - {`ForwardingStatus >= 128`, `ForwardingStatus >= 128`}, - {`PacketSize > 1500`, `Bytes/Packets > 1500`}, - {`DstPort > 1024 AND SrcPort < 1024`, `DstPort > 1024 AND SrcPort < 1024`}, - {`DstPort > 1024 OR SrcPort < 1024`, `DstPort > 1024 OR SrcPort < 1024`}, - {`NOT DstPort > 1024 AND SrcPort < 1024`, `NOT DstPort > 1024 AND SrcPort < 1024`}, - {`not DstPort > 1024 and SrcPort < 1024`, `NOT DstPort > 1024 AND SrcPort < 1024`}, - {`DstPort > 1024 AND SrcPort < 1024 OR InIfSpeed >= 1000`, - `DstPort > 1024 AND SrcPort < 1024 OR InIfSpeed >= 1000`}, - {`DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`, - `DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`}, - {` DstPort > 1024 AND ( SrcPort < 1024 OR InIfSpeed >= 1000 ) `, - `DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`}, - {`DstPort > 1024 AND(SrcPort < 1024 OR InIfSpeed >= 1000)`, - `DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`}, - {`DstPort > 1024 + {Input: `ExporterName = 'something'`, Output: `ExporterName = 'something'`}, + {Input: `ExporterName = 'something'`, Output: `ExporterName = 'something'`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `exportername = 'something'`, Output: `ExporterName = 'something'`}, + {Input: `ExporterName='something'`, Output: `ExporterName = 'something'`}, + {Input: `ExporterName="something"`, Output: `ExporterName = 'something'`}, + {Input: `ExporterName="something'"`, Output: `ExporterName = 'something\''`}, + {Input: `ExporterName="something\"`, Output: `ExporterName = 'something\\'`}, + {Input: `ExporterName!="something"`, Output: `ExporterName != 'something'`}, + {Input: `ExporterName IN ("something")`, Output: `ExporterName IN ('something')`}, + {Input: `ExporterName IN ("something","something else")`, Output: `ExporterName IN ('something', 'something else')`}, + {Input: `ExporterName LIKE "something%"`, Output: `ExporterName LIKE 'something%'`}, + {Input: `ExporterName UNLIKE "something%"`, Output: `ExporterName NOT LIKE 'something%'`}, + {Input: `ExporterName IUNLIKE "something%"`, Output: `ExporterName NOT ILIKE 'something%'`}, + {Input: `ExporterName="something with spaces"`, Output: `ExporterName = 'something with spaces'`}, + {Input: `ExporterName="something with 'quotes'"`, Output: `ExporterName = 'something with \'quotes\''`}, + {Input: `ExporterAddress=203.0.113.1`, Output: `ExporterAddress = toIPv6('203.0.113.1')`}, + {Input: `ExporterAddress=2001:db8::1`, Output: `ExporterAddress = toIPv6('2001:db8::1')`}, + {Input: `ExporterAddress=2001:db8:0::1`, Output: `ExporterAddress = toIPv6('2001:db8::1')`}, + { + Input: `ExporterAddress << 2001:db8:0::/64`, + Output: `ExporterAddress BETWEEN toIPv6('2001:db8::') AND toIPv6('2001:db8::ffff:ffff:ffff:ffff')`, + }, { + Input: `ExporterAddress << 2001:db8::c000/115`, + Output: `ExporterAddress BETWEEN toIPv6('2001:db8::c000') AND toIPv6('2001:db8::dfff')`, + }, { + Input: `ExporterAddress << 192.168.0.0/24`, + Output: `ExporterAddress BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`, + }, { + Input: `DstAddr << 192.168.0.0/24`, + Output: `DstAddr BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`, + MetaOut: Meta{MainTableRequired: true}, + }, { + Input: `DstAddr << 192.168.0.0/24`, + Output: `SrcAddr BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`, + MetaIn: Meta{ReverseDirection: true}, + MetaOut: Meta{ReverseDirection: true, MainTableRequired: true}, + }, { + Input: `SrcAddr << 192.168.0.1/24`, + Output: `SrcAddr BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`, + MetaOut: Meta{MainTableRequired: true}, + }, { + Input: `DstAddr !<< 192.168.0.0/24`, + Output: `DstAddr NOT BETWEEN toIPv6('::ffff:192.168.0.0') AND toIPv6('::ffff:192.168.0.255')`, + MetaOut: Meta{MainTableRequired: true}, + }, { + Input: `DstAddr !<< 192.168.0.128/27`, + Output: `DstAddr NOT BETWEEN toIPv6('::ffff:192.168.0.128') AND toIPv6('::ffff:192.168.0.159')`, + MetaOut: Meta{MainTableRequired: true}, + }, + {Input: `ExporterGroup= "group"`, Output: `ExporterGroup = 'group'`}, + {Input: `SrcAddr=203.0.113.1`, Output: `SrcAddr = toIPv6('203.0.113.1')`, + MetaOut: Meta{MainTableRequired: true}}, + {Input: `DstAddr=203.0.113.2`, Output: `DstAddr = toIPv6('203.0.113.2')`, + MetaOut: Meta{MainTableRequired: true}}, + {Input: `SrcNetName="alpha"`, Output: `SrcNetName = 'alpha'`}, + {Input: `DstNetName="alpha"`, Output: `DstNetName = 'alpha'`}, + {Input: `DstNetRole="stuff"`, Output: `DstNetRole = 'stuff'`}, + {Input: `SrcNetTenant="mobile"`, Output: `SrcNetTenant = 'mobile'`}, + {Input: `SrcAS=12322`, Output: `SrcAS = 12322`}, + {Input: `SrcAS=AS12322`, Output: `SrcAS = 12322`}, + {Input: `SrcAS=AS12322`, Output: `DstAS = 12322`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `SrcAS=as12322`, Output: `SrcAS = 12322`}, + {Input: `SrcAS IN(12322, 29447)`, Output: `SrcAS IN (12322, 29447)`}, + {Input: `SrcAS IN( 12322 , 29447 )`, Output: `SrcAS IN (12322, 29447)`}, + {Input: `SrcAS NOTIN(12322, 29447)`, Output: `SrcAS NOT IN (12322, 29447)`}, + {Input: `SrcAS NOTIN (AS12322, 29447)`, Output: `SrcAS NOT IN (12322, 29447)`}, + {Input: `DstAS=12322`, Output: `DstAS = 12322`}, + {Input: `SrcCountry='FR'`, Output: `SrcCountry = 'FR'`}, + {Input: `SrcCountry='FR'`, Output: `DstCountry = 'FR'`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `DstCountry='FR'`, Output: `DstCountry = 'FR'`}, + {Input: `DstCountry='FR'`, Output: `SrcCountry = 'FR'`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `InIfName='Gi0/0/0/1'`, Output: `InIfName = 'Gi0/0/0/1'`}, + {Input: `InIfName='Gi0/0/0/1'`, Output: `OutIfName = 'Gi0/0/0/1'`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `OutIfName = 'Gi0/0/0/1'`, Output: `OutIfName = 'Gi0/0/0/1'`}, + {Input: `OutIfName = 'Gi0/0/0/1'`, Output: `InIfName = 'Gi0/0/0/1'`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `InIfDescription='Some description'`, Output: `InIfDescription = 'Some description'`}, + {Input: `InIfDescription='Some description'`, Output: `OutIfDescription = 'Some description'`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `OutIfDescription='Some other description'`, Output: `OutIfDescription = 'Some other description'`}, + {Input: `OutIfDescription='Some other description'`, Output: `InIfDescription = 'Some other description'`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `InIfSpeed>=1000`, Output: `InIfSpeed >= 1000`}, + {Input: `InIfSpeed>=1000`, Output: `OutIfSpeed >= 1000`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `InIfSpeed!=1000`, Output: `InIfSpeed != 1000`}, + {Input: `InIfSpeed<1000`, Output: `InIfSpeed < 1000`}, + {Input: `OutIfSpeed!=1000`, Output: `OutIfSpeed != 1000`}, + {Input: `OutIfSpeed!=1000`, Output: `InIfSpeed != 1000`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `InIfConnectivity = 'pni'`, Output: `InIfConnectivity = 'pni'`}, + {Input: `InIfConnectivity = 'pni'`, Output: `OutIfConnectivity = 'pni'`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `OutIfConnectivity = 'ix'`, Output: `OutIfConnectivity = 'ix'`}, + {Input: `OutIfConnectivity = 'ix'`, Output: `InIfConnectivity = 'ix'`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `InIfProvider = 'cogent'`, Output: `InIfProvider = 'cogent'`}, + {Input: `InIfProvider = 'cogent'`, Output: `OutIfProvider = 'cogent'`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `OutIfProvider = 'telia'`, Output: `OutIfProvider = 'telia'`}, + {Input: `OutIfProvider = 'telia'`, Output: `InIfProvider = 'telia'`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `InIfBoundary = external`, Output: `InIfBoundary = 'external'`}, + {Input: `InIfBoundary = external`, Output: `OutIfBoundary = 'external'`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `InIfBoundary = EXTERNAL`, Output: `InIfBoundary = 'external'`}, + {Input: `InIfBoundary = EXTERNAL`, Output: `OutIfBoundary = 'external'`, + MetaIn: Meta{ReverseDirection: true}, MetaOut: Meta{ReverseDirection: true}}, + {Input: `OutIfBoundary != internal`, Output: `OutIfBoundary != 'internal'`}, + {Input: `EType = ipv4`, Output: `EType = 2048`}, + {Input: `EType != ipv6`, Output: `EType != 34525`}, + {Input: `Proto = 1`, Output: `Proto = 1`}, + {Input: `Proto = 'gre'`, Output: `dictGetOrDefault('protocols', 'name', Proto, '???') = 'gre'`}, + {Input: `SrcPort = 80`, Output: `SrcPort = 80`, + MetaOut: Meta{MainTableRequired: true}}, + {Input: `SrcPort = 80`, Output: `DstPort = 80`, + MetaIn: Meta{ReverseDirection: true}, + MetaOut: Meta{ReverseDirection: true, MainTableRequired: true}}, + {Input: `DstPort > 1024`, Output: `DstPort > 1024`, + MetaOut: Meta{MainTableRequired: true}}, + {Input: `ForwardingStatus >= 128`, Output: `ForwardingStatus >= 128`}, + {Input: `PacketSize > 1500`, Output: `Bytes/Packets > 1500`}, + {Input: `DstPort > 1024 AND SrcPort < 1024`, Output: `DstPort > 1024 AND SrcPort < 1024`, + MetaOut: Meta{MainTableRequired: true}}, + {Input: `DstPort > 1024 OR SrcPort < 1024`, Output: `DstPort > 1024 OR SrcPort < 1024`, + MetaOut: Meta{MainTableRequired: true}}, + {Input: `NOT DstPort > 1024 AND SrcPort < 1024`, Output: `NOT DstPort > 1024 AND SrcPort < 1024`, + MetaOut: Meta{MainTableRequired: true}}, + {Input: `not DstPort > 1024 and SrcPort < 1024`, Output: `NOT DstPort > 1024 AND SrcPort < 1024`, + MetaOut: Meta{MainTableRequired: true}}, + { + Input: `DstPort > 1024 AND SrcPort < 1024 OR InIfSpeed >= 1000`, + Output: `DstPort > 1024 AND SrcPort < 1024 OR InIfSpeed >= 1000`, + MetaOut: Meta{MainTableRequired: true}, + }, { + Input: `DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`, + Output: `DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`, + MetaOut: Meta{MainTableRequired: true}, + }, { + Input: ` DstPort > 1024 AND ( SrcPort < 1024 OR InIfSpeed >= 1000 ) `, + Output: `DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`, + MetaOut: Meta{MainTableRequired: true}, + }, { + Input: `DstPort > 1024 AND(SrcPort < 1024 OR InIfSpeed >= 1000)`, + Output: `DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`, + MetaOut: Meta{MainTableRequired: true}, + }, { + Input: `DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`, - `DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`}, - {`(ExporterAddress=203.0.113.1)`, `(ExporterAddress = toIPv6('203.0.113.1'))`}, - {`ForwardingStatus >= 128 -- Nothing`, `ForwardingStatus >= 128`}, - {` + Output: `DstPort > 1024 AND (SrcPort < 1024 OR InIfSpeed >= 1000)`, + MetaOut: Meta{MainTableRequired: true}, + }, + {Input: `(ExporterAddress=203.0.113.1)`, Output: `(ExporterAddress = toIPv6('203.0.113.1'))`}, + {Input: `ForwardingStatus >= 128 -- Nothing`, Output: `ForwardingStatus >= 128`}, + { + Input: ` -- Example of commented request -- Here we go DstPort > 1024 -- Non-privileged port -AND SrcAS = AS12322 -- Proxad ASN`, `DstPort > 1024 AND SrcAS = 12322`}, - {`InIfDescription = "This contains a -- comment" -- nope`, - `InIfDescription = 'This contains a -- comment'`}, - {`InIfDescription = "This contains a /* comment"`, - `InIfDescription = 'This contains a /* comment'`}, - {`OutIfProvider /* That's the output provider */ = 'telia'`, `OutIfProvider = 'telia'`}, - {`OutIfProvider /* That's the -output provider */ = 'telia'`, `OutIfProvider = 'telia'`}, +AND SrcAS = AS12322 -- Proxad ASN`, + Output: `DstPort > 1024 AND SrcAS = 12322`, + MetaOut: Meta{MainTableRequired: true}, + }, { + Input: `InIfDescription = "This contains a -- comment" -- nope`, + Output: `InIfDescription = 'This contains a -- comment'`, + }, { + Input: `InIfDescription = "This contains a /* comment"`, + Output: `InIfDescription = 'This contains a /* comment'`, + }, + {Input: `OutIfProvider /* That's the output provider */ = 'telia'`, Output: `OutIfProvider = 'telia'`}, + { + Input: `OutIfProvider /* That's the +output provider */ = 'telia'`, + Output: `OutIfProvider = 'telia'`, + }, } for _, tc := range cases { - got, err := Parse("", []byte(tc.Input)) + got, err := Parse("", []byte(tc.Input), GlobalStore("meta", &tc.MetaIn)) if err != nil { t.Errorf("Parse(%q) error:\n%+v", tc.Input, err) continue @@ -124,6 +206,9 @@ output provider */ = 'telia'`, `OutIfProvider = 'telia'`}, if diff := helpers.Diff(got.(string), tc.Output); diff != "" { t.Errorf("Parse(%q) (-got, +want):\n%s", tc.Input, diff) } + if diff := helpers.Diff(tc.MetaIn, tc.MetaOut); diff != "" { + t.Errorf("Parse(%q) meta (-got, +want):\n%s", tc.Input, diff) + } } } @@ -157,7 +242,7 @@ func TestInvalidFilter(t *testing.T) { {`SrcAS IN (AS12322,`}, } for _, tc := range cases { - out, err := Parse("", []byte(tc.Input)) + out, err := Parse("", []byte(tc.Input), GlobalStore("meta", &Meta{})) t.Logf("out: %v", out) if err == nil { t.Errorf("Parse(%q) didn't throw an error", tc.Input) diff --git a/console/graph.go b/console/graph.go index 6c18266c..1328727d 100644 --- a/console/graph.go +++ b/console/graph.go @@ -126,7 +126,7 @@ func (c *Component) graphHandlerFunc(gc *gin.Context) { if resolution < time.Second { resolution = time.Second } - sqlQuery = c.queryFlowsTable(sqlQuery, + sqlQuery = c.queryFlowsTable(sqlQuery, input.Filter.mainTableRequired, input.Start, input.End, resolution) gc.Header("X-SQL-Query", strings.ReplaceAll(sqlQuery, "\n", " ")) diff --git a/console/graph_test.go b/console/graph_test.go index 39f1e760..554b4b6f 100644 --- a/console/graph_test.go +++ b/console/graph_test.go @@ -93,7 +93,7 @@ ORDER BY time WITH FILL End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC), Points: 100, Dimensions: []queryColumn{}, - Filter: queryFilter{"DstCountry = 'FR' AND SrcCountry = 'US'"}, + Filter: queryFilter{filter: "DstCountry = 'FR' AND SrcCountry = 'US'"}, Units: "l3bps", }, Expected: ` diff --git a/console/query.go b/console/query.go index 6c0c7487..1d6a4916 100644 --- a/console/query.go +++ b/console/query.go @@ -123,7 +123,9 @@ func (gc *queryColumn) UnmarshalText(input []byte) error { } type queryFilter struct { - filter string + filter string + reverseFilter string + mainTableRequired bool } func (gf queryFilter) MarshalText() ([]byte, error) { @@ -131,14 +133,21 @@ func (gf queryFilter) MarshalText() ([]byte, error) { } func (gf *queryFilter) UnmarshalText(input []byte) error { if strings.TrimSpace(string(input)) == "" { - *gf = queryFilter{""} + *gf = queryFilter{} return nil } - got, err := filter.Parse("", input) + meta := &filter.Meta{} + direct, err := filter.Parse("", input, filter.GlobalStore("meta", meta)) if err != nil { return fmt.Errorf("cannot parse filter: %s", filter.HumanError(err)) } - *gf = queryFilter{got.(string)} + meta = &filter.Meta{ReverseDirection: true} + reverse, err := filter.Parse("", input, filter.GlobalStore("meta", meta)) + *gf = queryFilter{ + filter: direct.(string), + reverseFilter: reverse.(string), + mainTableRequired: meta.MainTableRequired, + } return nil } diff --git a/console/sankey.go b/console/sankey.go index d821984f..b36eee6b 100644 --- a/console/sankey.go +++ b/console/sankey.go @@ -115,7 +115,7 @@ func (c *Component) sankeyHandlerFunc(gc *gin.Context) { } // Prepare and execute query - sqlQuery = c.queryFlowsTable(sqlQuery, + sqlQuery = c.queryFlowsTable(sqlQuery, input.Filter.mainTableRequired, input.Start, input.End, resolution) gc.Header("X-SQL-Query", strings.ReplaceAll(sqlQuery, "\n", " ")) results := []struct { diff --git a/console/sankey_test.go b/console/sankey_test.go index 7b11a3c2..0e10d7e1 100644 --- a/console/sankey_test.go +++ b/console/sankey_test.go @@ -93,7 +93,7 @@ ORDER BY xps DESC`, End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC), Dimensions: []queryColumn{queryColumnSrcAS, queryColumnExporterName}, Limit: 10, - Filter: queryFilter{"DstCountry = 'FR'"}, + Filter: queryFilter{filter: "DstCountry = 'FR'"}, Units: "l3bps", }, Expected: ` diff --git a/console/widgets.go b/console/widgets.go index 8c582f29..8b03a2ee 100644 --- a/console/widgets.go +++ b/console/widgets.go @@ -105,9 +105,10 @@ type topResult struct { func (c *Component) widgetTopHandlerFunc(gc *gin.Context) { ctx := c.t.Context(gc.Request.Context()) var ( - selector string - groupby string - filter string + selector string + groupby string + filter string + mainTableRequired bool ) switch gc.Param("name") { @@ -139,9 +140,11 @@ func (c *Component) widgetTopHandlerFunc(gc *gin.Context) { case "src-port": selector = `concat(dictGetOrDefault('protocols', 'name', Proto, '???'), '/', toString(SrcPort))` groupby = `Proto, SrcPort` + mainTableRequired = true case "dst-port": selector = `concat(dictGetOrDefault('protocols', 'name', Proto, '???'), '/', toString(DstPort))` groupby = `Proto, DstPort` + mainTableRequired = true } if groupby == "" { groupby = selector @@ -160,7 +163,7 @@ WHERE {timefilter} GROUP BY %s ORDER BY Percent DESC LIMIT 5 -`, filter, selector, selector, filter, groupby), now.Add(-5*time.Minute), now, time.Minute) +`, filter, selector, selector, filter, groupby), mainTableRequired, now.Add(-5*time.Minute), now, time.Minute) gc.Header("X-SQL-Query", query) results := []topResult{} @@ -202,7 +205,7 @@ GROUP BY Time ORDER BY Time WITH FILL FROM toStartOfInterval({timefilter.Start}, INTERVAL %s second) TO {timefilter.Stop} - STEP %s`, slot, slot, slot, slot), now.Add(-24*time.Hour), now, time.Duration(interval)*time.Second) + STEP %s`, slot, slot, slot, slot), false, now.Add(-24*time.Hour), now, time.Duration(interval)*time.Second) gc.Header("X-SQL-Query", query) results := []struct {