diff --git a/common/schema/definition.go b/common/schema/definition.go index 2313bee2..1c75e083 100644 --- a/common/schema/definition.go +++ b/common/schema/definition.go @@ -138,15 +138,16 @@ func flows() Schema { ProtobufType: protoreflect.Uint64Kind, }, {Key: ColumnSamplingRate, NoDisable: true, ClickHouseType: "UInt64", ConsoleNotDimension: true}, - {Key: ColumnExporterAddress, ClickHouseType: "LowCardinality(IPv6)"}, - {Key: ColumnExporterName, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, - {Key: ColumnExporterGroup, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, - {Key: ColumnExporterRole, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, - {Key: ColumnExporterSite, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, - {Key: ColumnExporterRegion, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, - {Key: ColumnExporterTenant, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, + {Key: ColumnExporterAddress, ParserType: "ip", ClickHouseType: "LowCardinality(IPv6)"}, + {Key: ColumnExporterName, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, + {Key: ColumnExporterGroup, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, + {Key: ColumnExporterRole, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, + {Key: ColumnExporterSite, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, + {Key: ColumnExporterRegion, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, + {Key: ColumnExporterTenant, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, { Key: ColumnSrcAddr, + ParserType: "ip", ClickHouseMainOnly: true, ClickHouseType: "IPv6", ClickHouseCodec: "ZSTD(1)", @@ -172,56 +173,66 @@ END`, {Key: ColumnSrcAS, ClickHouseType: "UInt32"}, { Key: ColumnSrcNetName, + ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'name', SrcAddr, '')", }, { Key: ColumnDstNetName, + ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'name', DstAddr, '')", }, { Key: ColumnSrcNetRole, + ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'role', SrcAddr, '')", }, { Key: ColumnDstNetRole, + ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'role', DstAddr, '')", }, { Key: ColumnSrcNetSite, + ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'site', SrcAddr, '')", }, { Key: ColumnDstNetSite, + ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'site', DstAddr, '')", }, { Key: ColumnSrcNetRegion, + ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'region', SrcAddr, '')", }, { Key: ColumnDstNetRegion, + ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'region', DstAddr, '')", }, { Key: ColumnSrcNetTenant, + ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'tenant', SrcAddr, '')", }, { Key: ColumnDstNetTenant, + ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'tenant', DstAddr, '')", }, - {Key: ColumnSrcVlan, ClickHouseType: "UInt16", Disabled: true, Group: ColumnGroupL2}, - {Key: ColumnSrcCountry, ClickHouseType: "FixedString(2)"}, + {Key: ColumnSrcVlan, ParserType: "uint", ClickHouseType: "UInt16", Disabled: true, Group: ColumnGroupL2}, + {Key: ColumnSrcCountry, ParserType: "string", ClickHouseType: "FixedString(2)"}, { Key: ColumnDstASPath, ClickHouseMainOnly: true, @@ -271,11 +282,11 @@ END`, ClickHouseTransformTo: "arrayMap((asn, l1, l2) -> ((bitShiftLeft(CAST(asn, 'UInt128'), 64) + bitShiftLeft(CAST(l1, 'UInt128'), 32)) + CAST(l2, 'UInt128')), DstLargeCommunitiesASN, DstLargeCommunitiesLocalData1, DstLargeCommunitiesLocalData2)", ConsoleNotDimension: true, }, - {Key: ColumnInIfName, ClickHouseType: "LowCardinality(String)"}, - {Key: ColumnInIfDescription, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, - {Key: ColumnInIfSpeed, ClickHouseType: "UInt32", ClickHouseNotSortingKey: true}, - {Key: ColumnInIfConnectivity, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, - {Key: ColumnInIfProvider, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, + {Key: ColumnInIfName, ParserType: "string", ClickHouseType: "LowCardinality(String)"}, + {Key: ColumnInIfDescription, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, + {Key: ColumnInIfSpeed, ParserType: "uint", ClickHouseType: "UInt32", ClickHouseNotSortingKey: true}, + {Key: ColumnInIfConnectivity, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, + {Key: ColumnInIfProvider, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, { Key: ColumnInIfBoundary, ClickHouseType: "Enum8('undefined' = 0, 'external' = 1, 'internal' = 2)", @@ -290,7 +301,7 @@ END`, }, {Key: ColumnEType, ClickHouseType: "UInt32"}, // TODO: UInt16 but hard to change, primary key {Key: ColumnProto, ClickHouseType: "UInt32"}, // TODO: UInt8 but hard to change, primary key - {Key: ColumnSrcPort, ClickHouseType: "UInt16", ClickHouseMainOnly: true}, + {Key: ColumnSrcPort, ParserType: "uint", ClickHouseType: "UInt16", ClickHouseMainOnly: true}, { Key: ColumnBytes, NoDisable: true, @@ -310,6 +321,7 @@ END`, { Key: ColumnPacketSize, Depends: []ColumnKey{ColumnBytes, ColumnPackets}, + ParserType: "uint", ClickHouseType: "UInt64", ClickHouseAlias: "intDiv(Bytes, Packets)", ConsoleNotDimension: true, @@ -334,11 +346,12 @@ END`, return fmt.Sprintf("multiIf(%s)", strings.Join(conditions, ", ")) }(), }, - {Key: ColumnForwardingStatus, ClickHouseType: "UInt32"}, // TODO: UInt8 but hard to change, primary key + {Key: ColumnForwardingStatus, ParserType: "uint", ClickHouseType: "UInt32"}, // TODO: UInt8 but hard to change, primary key { Key: ColumnSrcAddrNAT, Disabled: true, Group: ColumnGroupNAT, + ParserType: "ip", ClickHouseType: "IPv6", ClickHouseMainOnly: true, ConsoleTruncateIP: true, @@ -347,25 +360,27 @@ END`, Key: ColumnSrcPortNAT, Disabled: true, Group: ColumnGroupNAT, + ParserType: "uint", ClickHouseType: "UInt16", ClickHouseMainOnly: true, }, {Key: ColumnSrcMAC, Disabled: true, Group: ColumnGroupL2, ClickHouseType: "UInt64"}, - {Key: ColumnIPTTL, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt8"}, - {Key: ColumnIPTos, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt8"}, - {Key: ColumnIPFragmentID, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt32"}, - {Key: ColumnIPFragmentOffset, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt16"}, - {Key: ColumnIPv6FlowLabel, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt32"}, - {Key: ColumnTCPFlags, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt16"}, - {Key: ColumnICMPv4Type, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt8"}, - {Key: ColumnICMPv4Code, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt8"}, - {Key: ColumnICMPv6Type, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt8"}, - {Key: ColumnICMPv6Code, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt8"}, + {Key: ColumnIPTTL, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt8"}, + {Key: ColumnIPTos, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt8"}, + {Key: ColumnIPFragmentID, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt32"}, + {Key: ColumnIPFragmentOffset, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt16"}, + {Key: ColumnIPv6FlowLabel, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt32"}, + {Key: ColumnTCPFlags, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt16"}, + {Key: ColumnICMPv4Type, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt8"}, + {Key: ColumnICMPv4Code, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt8"}, + {Key: ColumnICMPv6Type, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt8"}, + {Key: ColumnICMPv6Code, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt8"}, { Key: ColumnICMPv4, Depends: []ColumnKey{ColumnProto, ColumnICMPv4Type, ColumnICMPv4Code}, Disabled: true, Group: ColumnGroupL3L4, + ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseAlias: `if(Proto = 1, ` + `dictGetOrDefault('icmp', 'name', tuple(Proto, ICMPv4Type, ICMPv4Code), ` + @@ -376,6 +391,7 @@ END`, Depends: []ColumnKey{ColumnProto, ColumnICMPv6Type, ColumnICMPv6Code}, Disabled: true, Group: ColumnGroupL3L4, + ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseAlias: `if(Proto = 58, ` + `dictGetOrDefault('icmp', 'name', tuple(Proto, ICMPv6Type, ICMPv6Code), ` + @@ -384,6 +400,7 @@ END`, { Key: ColumnNextHop, Disabled: true, + ParserType: "ip", ClickHouseType: "LowCardinality(IPv6)", ClickHouseCodec: "ZSTD(1)", }, diff --git a/common/schema/root.go b/common/schema/root.go index 4f551a1e..ab21a26b 100644 --- a/common/schema/root.go +++ b/common/schema/root.go @@ -84,10 +84,10 @@ func New(config Configuration) (*Component, error) { } } + // Add new columns from custom dictionaries after the static ones as we dont + // reference the dicts in the code and they are created during runtime from + // the config, this is enough for us. customDictColumns := []Column{} - // add new columns from custom dictionaries after the static ones - // as we dont reference the dicts in the code and they are created during runtime from the config, this is enough for us. - for dname, v := range config.CustomDictionaries { for _, d := range v.Dimensions { // check if we can actually create the dictionary (we need to know what to match on) @@ -130,12 +130,21 @@ func New(config Configuration) (*Component, error) { l = cases.Title(language.Und).String(a.Name) } name := fmt.Sprintf("%s%s", d, l) - // compute the key for this new dynamic column, added after the last dynamic column key := ColumnLast + schema.dynamicColumns + parserType := "" + switch a.Type { + case "IPv6": + parserType = "ip" + case "String": + parserType = "string" + case "UInt8", "UInt16", "UInt32", "UInt64": + parserType = "uint" + } customDictColumns = append(customDictColumns, Column{ Key: key, Name: name, + ParserType: parserType, ClickHouseType: fmt.Sprintf("LowCardinality(%s)", a.Type), ClickHouseGenerateFrom: fmt.Sprintf("dictGet('custom_dict_%s', '%s', %s)", dname, a.Name, matchingString), diff --git a/common/schema/types.go b/common/schema/types.go index affe246e..6b3a67e0 100644 --- a/common/schema/types.go +++ b/common/schema/types.go @@ -33,6 +33,9 @@ type Column struct { Group ColumnGroup Depends []ColumnKey + // For parser. + ParserType string + // For ClickHouse. `NotSortingKey' is for columns generated from other // columns. It is only useful if not ClickHouseMainOnly and not Alias. // `GenerateFrom' is for a column that's generated from an SQL expression diff --git a/console/data/docs/02-configuration.md b/console/data/docs/02-configuration.md index 10771056..31e6742c 100644 --- a/console/data/docs/02-configuration.md +++ b/console/data/docs/02-configuration.md @@ -559,9 +559,6 @@ to enrich your flow with additional informations not possible to get in the classifier. This works by providing the database with a CSV files containing the values. -**Note:** filtering by dictionaries is not possible with the current state of -development. - ```yaml schema: custom-dictionaries: diff --git a/console/data/docs/99-changelog.md b/console/data/docs/99-changelog.md index ee901ebf..33d583fd 100644 --- a/console/data/docs/99-changelog.md +++ b/console/data/docs/99-changelog.md @@ -13,6 +13,7 @@ identified with a specific icon: ## Unreleased +- 🌱 *console*: add filtering support for custom columns - 🌱 *inlet*: update [Expr](https://expr.medv.io/), the language behind the classifiers: support for variables - 🌱 *orchestrator*: improve performance when looking up for `SrcNetPrefix` and diff --git a/console/filter.go b/console/filter.go index 67de04dd..bfa4377b 100644 --- a/console/filter.go +++ b/console/filter.go @@ -6,6 +6,7 @@ package console import ( "fmt" "net/http" + "sort" "strconv" "strings" @@ -84,21 +85,22 @@ func (c *Component) filterCompleteHandlerFunc(gc *gin.Context) { completions := []filterCompletion{} switch input.What { case "column": - _, err := filter.Parse("", []byte{}, - filter.Entrypoint("ConditionExpr"), filter.GlobalStore("meta", &filter.Meta{Schema: c.d.Schema})) - if err != nil { - for _, candidate := range filter.Expected(err) { - if !strings.HasSuffix(candidate, `"i`) { - continue - } - candidate = candidate[1 : len(candidate)-2] - if column, ok := c.d.Schema.LookupColumnByName(candidate); ok && !column.Disabled { - completions = append(completions, filterCompletion{ - Label: candidate, - Detail: "column name", - }) - } + // We use the schema directly. + columns := []string{} + for _, column := range c.d.Schema.Columns() { + if column.Disabled { + continue } + if strings.HasPrefix(strings.ToLower(column.Name), strings.ToLower(input.Prefix)) { + columns = append(columns, column.Name) + } + } + sort.Strings(columns) + for _, column := range columns { + completions = append(completions, filterCompletion{ + Label: column, + Detail: "column name", + }) } case "operator": _, err := filter.Parse("", diff --git a/console/filter/helpers.go b/console/filter/helpers.go index 17ce3464..ea457a3c 100644 --- a/console/filter/helpers.go +++ b/console/filter/helpers.go @@ -100,7 +100,7 @@ func (c *current) compileExpr(expr []any, meta *Meta) string { } // acceptColumn normalizes and returns the matched column name. It should be -// used in predicate code blocks. +// used in action code blocks. func (c *current) acceptColumn() (schema.Column, error) { name := string(c.text) sch := c.globalStore["meta"].(*Meta).Schema @@ -112,6 +112,28 @@ func (c *current) acceptColumn() (schema.Column, error) { return schema.Column{}, fmt.Errorf("unknown column %q", name) } +// columnIsOfType returns true if the column is of one of the provided type. It +// should be used in predicate code blocks. +func (c *current) columnIsOfType(name any, types ...string) (bool, error) { + nameSl := name.([]any) + var columnName string + for _, s := range nameSl { + columnName += string(s.([]byte)) + } + sch := c.globalStore["meta"].(*Meta).Schema + for _, column := range sch.Columns() { + if strings.EqualFold(columnName, column.Name) { + for _, t := range types { + if column.ParserType == t { + return true, nil + } + } + return false, nil + } + } + return false, nil +} + // getColumn gets a column by its name. func (c *current) getColumn(name string) schema.Column { sch := c.globalStore["meta"].(*Meta).Schema diff --git a/console/filter/parser.peg b/console/filter/parser.peg index eae255af..7c65415f 100644 --- a/console/filter/parser.peg +++ b/console/filter/parser.peg @@ -48,12 +48,9 @@ ConditionExpr "conditional" ← / ConditionProtoExpr ColumnIP ← - "ExporterAddress"i !IdentStart { return c.acceptColumn() } - / "SrcAddr"i !IdentStart { return c.acceptColumn() } - / "DstAddr"i !IdentStart { return c.acceptColumn() } - / "SrcAddrNAT"i !IdentStart { return c.acceptColumn() } - / "DstAddrNAT"i !IdentStart { return c.acceptColumn() } - / "NextHop"i !IdentStart { return c.acceptColumn() } + column:[A-Za-z0-9]+ !IdentStart + &{ return c.columnIsOfType(column, "ip") } + { return c.acceptColumn() } ConditionIPExpr "condition on IP" ← column:ColumnIP _ operator:("=" / "!=") _ ip:IP { @@ -101,34 +98,9 @@ ConditionMACExpr "condition on MAC" ← } ConditionStringExpr "condition on string" ← - column:("ExporterName"i !IdentStart { return c.acceptColumn() } - / "ExporterGroup"i !IdentStart { return c.acceptColumn() } - / "ExporterRole"i !IdentStart { return c.acceptColumn() } - / "ExporterSite"i !IdentStart { return c.acceptColumn() } - / "ExporterRegion"i !IdentStart { return c.acceptColumn() } - / "ExporterTenant"i !IdentStart { return c.acceptColumn() } - / "SrcCountry"i !IdentStart { return c.acceptColumn() } - / "DstCountry"i !IdentStart { return c.acceptColumn() } - / "SrcNetName"i !IdentStart { return c.acceptColumn() } - / "DstNetName"i !IdentStart { return c.acceptColumn() } - / "SrcNetRole"i !IdentStart { return c.acceptColumn() } - / "DstNetRole"i !IdentStart { return c.acceptColumn() } - / "SrcNetSite"i !IdentStart { return c.acceptColumn() } - / "DstNetSite"i !IdentStart { return c.acceptColumn() } - / "SrcNetRegion"i !IdentStart { return c.acceptColumn() } - / "DstNetRegion"i !IdentStart { return c.acceptColumn() } - / "SrcNetTenant"i !IdentStart { return c.acceptColumn() } - / "DstNetTenant"i !IdentStart { return c.acceptColumn() } - / "InIfName"i !IdentStart { return c.acceptColumn() } - / "OutIfName"i !IdentStart { return c.acceptColumn() } - / "InIfDescription"i !IdentStart { return c.acceptColumn() } - / "OutIfDescription"i !IdentStart { return c.acceptColumn() } - / "InIfConnectivity"i !IdentStart { return c.acceptColumn() } - / "OutIfConnectivity"i !IdentStart { return c.acceptColumn() } - / "InIfProvider"i !IdentStart { return c.acceptColumn() } - / "OutIfProvider"i !IdentStart { return c.acceptColumn() } - / "ICMPv4"i !IdentStart { return c.acceptColumn() } - / "ICMPv6"i !IdentStart { return c.acceptColumn() }) _ + column:(value:[A-Za-z0-9]+ !IdentStart + &{ return c.columnIsOfType(value, "string") } + { return c.acceptColumn() }) _ rcond:RConditionStringExpr { return []any{column, rcond}, nil } @@ -149,26 +121,9 @@ ConditionBoundaryExpr "condition on boundary" ← } ConditionUintExpr "condition on integer" ← - column:("InIfSpeed"i !IdentStart { return c.acceptColumn() } - / "OutIfSpeed"i !IdentStart { return c.acceptColumn() } - / "SrcPort"i !IdentStart { return c.acceptColumn() } - / "DstPort"i !IdentStart { return c.acceptColumn() } - / "SrcPortNAT"i !IdentStart { return c.acceptColumn() } - / "DstPortNAT"i !IdentStart { return c.acceptColumn() } - / "SrcVlan"i !IdentStart { return c.acceptColumn() } - / "DstVlan"i !IdentStart { return c.acceptColumn() } - / "PacketSize"i !IdentStart { return c.acceptColumn() } - / "ForwardingStatus"i !IdentStart { return c.acceptColumn() } - / "IPTTL"i !IdentStart { return c.acceptColumn() } - / "IPTos"i !IdentStart { return c.acceptColumn() } - / "IPFragmentID"i !IdentStart { return c.acceptColumn() } - / "IPFragmentOffset"i !IdentStart { return c.acceptColumn() } - / "IPv6FlowLabel"i !IdentStart { return c.acceptColumn() } - / "TCPFlags"i !IdentStart { return c.acceptColumn() } - / "ICMPv4Type"i !IdentStart { return c.acceptColumn() } - / "ICMPv4Code"i !IdentStart { return c.acceptColumn() } - / "ICMPv6Type"i !IdentStart { return c.acceptColumn() } - / "ICMPv6Code"i !IdentStart { return c.acceptColumn() }) _ + column:(value:[A-Za-z0-9]+ !IdentStart + &{ return c.columnIsOfType(value, "uint") } + { return c.acceptColumn() }) _ operator:("=" / ">=" / "<=" / "<" / ">" / "!=") _ value:Unsigned64 { return []any{column, operator, value}, nil diff --git a/console/filter/parser_test.go b/console/filter/parser_test.go index 7610f807..41c58372 100644 --- a/console/filter/parser_test.go +++ b/console/filter/parser_test.go @@ -320,9 +320,32 @@ output provider */ = 'telia'`, {Input: `icmpv4type = 8 AND icmpv4code = 0`, Output: `ICMPv4Type = 8 AND ICMPv4Code = 0`}, {Input: `icmpv6type = 8 or icmpv6code = 0`, Output: `ICMPv6Type = 8 OR ICMPv6Code = 0`}, {Input: `icmpv6 = "echo-reply"`, Output: `ICMPv6 = 'echo-reply'`}, + {Input: `SrcAddrDimensionAttribute = "Test"`, Output: `SrcAddrDimensionAttribute = 'Test'`}, + {Input: `DstAddrDimensionAttribute = "Test"`, Output: `DstAddrDimensionAttribute = 'Test'`}, + {Input: `DstAddrRole = "Test"`, Output: `DstAddrRole = 'Test'`}, + {Input: `DstAddrPriority = 200`, Output: `DstAddrPriority = 200`}, + {Input: `DstAddrSibling = 2001:db8::1`, Output: `DstAddrSibling = toIPv6('2001:db8::1')`}, + {Input: `SrcAddrDimensionAttribute IN ("Test", "None")`, Output: `SrcAddrDimensionAttribute IN ('Test', 'None')`}, } + config := schema.DefaultConfiguration() + config.CustomDictionaries = make(map[string]schema.CustomDict) + config.CustomDictionaries["test"] = schema.CustomDict{ + Keys: []schema.CustomDictKey{ + {Name: "SrcAddr", Type: "String"}, + }, + Attributes: []schema.CustomDictAttribute{ + {Name: "csv_col_name", Type: "String", Label: "DimensionAttribute"}, + {Name: "role", Type: "String"}, + {Name: "priority", Type: "UInt16"}, + {Name: "sibling", Type: "IPv6"}, + }, + Source: "test.csv", + Dimensions: []string{"SrcAddr", "DstAddr"}, + } + s, _ := schema.New(config) + s = s.EnableAllColumns() for _, tc := range cases { - tc.MetaIn.Schema = schema.NewMock(t).EnableAllColumns() + tc.MetaIn.Schema = s tc.MetaOut.Schema = tc.MetaIn.Schema got, err := Parse("", []byte(tc.Input), GlobalStore("meta", &tc.MetaIn)) if err != nil { @@ -413,11 +436,29 @@ func TestInvalidFilter(t *testing.T) { {Input: `SrcVlan = 1000`}, {Input: `DstVlan = 1000`}, {Input: `SrcMAC = 00:11:22:33:44:55:66`, EnableAll: true}, + {Input: `SrcAddrDimensionAttribute = 8`}, + {Input: `InvalidDimensionAttribute = "Test"`}, } + config := schema.DefaultConfiguration() + config.CustomDictionaries = make(map[string]schema.CustomDict) + config.CustomDictionaries["test"] = schema.CustomDict{ + Keys: []schema.CustomDictKey{ + {Name: "SrcAddr", Type: "string"}, + }, + Attributes: []schema.CustomDictAttribute{ + {Name: "csv_col_name", Type: "string", Label: "DimensionAttribute"}, + {Name: "role", Type: "string"}, + }, + Source: "test.csv", + Dimensions: []string{"SrcAddr", "DstAddr"}, + } + + s, _ := schema.New(config) + for _, tc := range cases { sch := schema.NewMock(t) if tc.EnableAll { - sch.EnableAllColumns() + s = s.EnableAllColumns() } out, err := Parse("", []byte(tc.Input), GlobalStore("meta", &Meta{Schema: sch})) if err == nil { diff --git a/console/filter_test.go b/console/filter_test.go index ec8afc78..502505ce 100644 --- a/console/filter_test.go +++ b/console/filter_test.go @@ -158,23 +158,11 @@ LIMIT 20`, "6540"). { URL: "/api/v0/console/filter/complete", StatusCode: 200, - JSONInput: gin.H{"what": "column", "prefix": "dSt"}, + JSONInput: gin.H{"what": "column", "prefix": "dSta"}, JSONOutput: gin.H{"completions": []gin.H{ - {"label": "Dst1stAS", "detail": "column name", "quoted": false}, - {"label": "Dst2ndAS", "detail": "column name", "quoted": false}, - {"label": "Dst3rdAS", "detail": "column name", "quoted": false}, {"label": "DstAS", "detail": "column name", "quoted": false}, {"label": "DstASPath", "detail": "column name", "quoted": false}, {"label": "DstAddr", "detail": "column name", "quoted": false}, - {"label": "DstCommunities", "detail": "column name", "quoted": false}, - {"label": "DstCountry", "detail": "column name", "quoted": false}, - {"label": "DstNetName", "detail": "column name", "quoted": false}, - {"label": "DstNetPrefix", "detail": "column name", "quoted": false}, - {"label": "DstNetRegion", "detail": "column name", "quoted": false}, - {"label": "DstNetRole", "detail": "column name", "quoted": false}, - {"label": "DstNetSite", "detail": "column name", "quoted": false}, - {"label": "DstNetTenant", "detail": "column name", "quoted": false}, - {"label": "DstPort", "detail": "column name", "quoted": false}, }}, }, {