johanels/akvorado
1
0
Fork 0
mirror of https://github.com/akvorado/akvorado.git synced 2025-12-11 22:14:02 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

console/filter: add filtering support for custom columns

Browse Source 
Some of the code is based on #870.
This commit is contained in:
Vincent Bernat
2023-09-16 17:18:57 +02:00
parent e3fdd6bcc8
commit 6dc0b512c6
10 changed files with 152 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
common/schema/definition.go
View File

@@ -138,15 +138,16 @@ func flows() Schema {
ProtobufType: protoreflect.Uint64Kind,
},
{Key: ColumnSamplingRate, NoDisable: true, ClickHouseType: "UInt64", ConsoleNotDimension: true},
{Key: ColumnExporterAddress, ClickHouseType: "LowCardinality(IPv6)"},
{Key: ColumnExporterName, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnExporterGroup, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnExporterRole, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnExporterSite, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnExporterRegion, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnExporterTenant, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnExporterAddress, ParserType: "ip", ClickHouseType: "LowCardinality(IPv6)"},
{Key: ColumnExporterName, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnExporterGroup, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnExporterRole, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnExporterSite, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnExporterRegion, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnExporterTenant, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{
Key: ColumnSrcAddr,
ParserType: "ip",
ClickHouseMainOnly: true,
ClickHouseType: "IPv6",
ClickHouseCodec: "ZSTD(1)",
@@ -172,56 +173,66 @@ END`,
{Key: ColumnSrcAS, ClickHouseType: "UInt32"},
{
Key: ColumnSrcNetName,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'name', SrcAddr, '')",
},
{
Key: ColumnDstNetName,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'name', DstAddr, '')",
},
{
Key: ColumnSrcNetRole,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'role', SrcAddr, '')",
},
{
Key: ColumnDstNetRole,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'role', DstAddr, '')",
},
{
Key: ColumnSrcNetSite,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'site', SrcAddr, '')",
},
{
Key: ColumnDstNetSite,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'site', DstAddr, '')",
},
{
Key: ColumnSrcNetRegion,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'region', SrcAddr, '')",
},
{
Key: ColumnDstNetRegion,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'region', DstAddr, '')",
},
{
Key: ColumnSrcNetTenant,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'tenant', SrcAddr, '')",
},
{
Key: ColumnDstNetTenant,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: "dictGetOrDefault('networks', 'tenant', DstAddr, '')",
},
{Key: ColumnSrcVlan, ClickHouseType: "UInt16", Disabled: true, Group: ColumnGroupL2},
{Key: ColumnSrcCountry, ClickHouseType: "FixedString(2)"},
{Key: ColumnSrcVlan, ParserType: "uint", ClickHouseType: "UInt16", Disabled: true, Group: ColumnGroupL2},
{Key: ColumnSrcCountry, ParserType: "string", ClickHouseType: "FixedString(2)"},
{
Key: ColumnDstASPath,
ClickHouseMainOnly: true,
@@ -271,11 +282,11 @@ END`,
ClickHouseTransformTo: "arrayMap((asn, l1, l2) -> ((bitShiftLeft(CAST(asn, 'UInt128'), 64) + bitShiftLeft(CAST(l1, 'UInt128'), 32)) + CAST(l2, 'UInt128')), DstLargeCommunitiesASN, DstLargeCommunitiesLocalData1, DstLargeCommunitiesLocalData2)",
ConsoleNotDimension: true,
},
{Key: ColumnInIfName, ClickHouseType: "LowCardinality(String)"},
{Key: ColumnInIfDescription, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnInIfSpeed, ClickHouseType: "UInt32", ClickHouseNotSortingKey: true},
{Key: ColumnInIfConnectivity, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnInIfProvider, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnInIfName, ParserType: "string", ClickHouseType: "LowCardinality(String)"},
{Key: ColumnInIfDescription, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnInIfSpeed, ParserType: "uint", ClickHouseType: "UInt32", ClickHouseNotSortingKey: true},
{Key: ColumnInIfConnectivity, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnInIfProvider, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{
Key: ColumnInIfBoundary,
ClickHouseType: "Enum8('undefined' = 0, 'external' = 1, 'internal' = 2)",
@@ -290,7 +301,7 @@ END`,
},
{Key: ColumnEType, ClickHouseType: "UInt32"}, // TODO: UInt16 but hard to change, primary key
{Key: ColumnProto, ClickHouseType: "UInt32"}, // TODO: UInt8 but hard to change, primary key
{Key: ColumnSrcPort, ClickHouseType: "UInt16", ClickHouseMainOnly: true},
{Key: ColumnSrcPort, ParserType: "uint", ClickHouseType: "UInt16", ClickHouseMainOnly: true},
{
Key: ColumnBytes,
NoDisable: true,
@@ -310,6 +321,7 @@ END`,
{
Key: ColumnPacketSize,
Depends: []ColumnKey{ColumnBytes, ColumnPackets},
ParserType: "uint",
ClickHouseType: "UInt64",
ClickHouseAlias: "intDiv(Bytes, Packets)",
ConsoleNotDimension: true,
@@ -334,11 +346,12 @@ END`,
return fmt.Sprintf("multiIf(%s)", strings.Join(conditions, ", "))
}(),
},
{Key: ColumnForwardingStatus, ClickHouseType: "UInt32"}, // TODO: UInt8 but hard to change, primary key
{Key: ColumnForwardingStatus, ParserType: "uint", ClickHouseType: "UInt32"}, // TODO: UInt8 but hard to change, primary key
{
Key: ColumnSrcAddrNAT,
Disabled: true,
Group: ColumnGroupNAT,
ParserType: "ip",
ClickHouseType: "IPv6",
ClickHouseMainOnly: true,
ConsoleTruncateIP: true,
@@ -347,25 +360,27 @@ END`,
Key: ColumnSrcPortNAT,
Disabled: true,
Group: ColumnGroupNAT,
ParserType: "uint",
ClickHouseType: "UInt16",
ClickHouseMainOnly: true,
},
{Key: ColumnSrcMAC, Disabled: true, Group: ColumnGroupL2, ClickHouseType: "UInt64"},
{Key: ColumnIPTTL, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt8"},
{Key: ColumnIPTos, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt8"},
{Key: ColumnIPFragmentID, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt32"},
{Key: ColumnIPFragmentOffset, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt16"},
{Key: ColumnIPv6FlowLabel, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt32"},
{Key: ColumnTCPFlags, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt16"},
{Key: ColumnICMPv4Type, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt8"},
{Key: ColumnICMPv4Code, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt8"},
{Key: ColumnICMPv6Type, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt8"},
{Key: ColumnICMPv6Code, Disabled: true, Group: ColumnGroupL3L4, ClickHouseType: "UInt8"},
{Key: ColumnIPTTL, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt8"},
{Key: ColumnIPTos, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt8"},
{Key: ColumnIPFragmentID, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt32"},
{Key: ColumnIPFragmentOffset, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt16"},
{Key: ColumnIPv6FlowLabel, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt32"},
{Key: ColumnTCPFlags, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt16"},
{Key: ColumnICMPv4Type, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt8"},
{Key: ColumnICMPv4Code, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt8"},
{Key: ColumnICMPv6Type, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt8"},
{Key: ColumnICMPv6Code, Disabled: true, Group: ColumnGroupL3L4, ParserType: "uint", ClickHouseType: "UInt8"},
{
Key: ColumnICMPv4,
Depends: []ColumnKey{ColumnProto, ColumnICMPv4Type, ColumnICMPv4Code},
Disabled: true,
Group: ColumnGroupL3L4,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseAlias: `if(Proto = 1, ` +
`dictGetOrDefault('icmp', 'name', tuple(Proto, ICMPv4Type, ICMPv4Code), ` +
@@ -376,6 +391,7 @@ END`,
Depends: []ColumnKey{ColumnProto, ColumnICMPv6Type, ColumnICMPv6Code},
Disabled: true,
Group: ColumnGroupL3L4,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseAlias: `if(Proto = 58, ` +
`dictGetOrDefault('icmp', 'name', tuple(Proto, ICMPv6Type, ICMPv6Code), ` +
@@ -384,6 +400,7 @@ END`,
{
Key: ColumnNextHop,
Disabled: true,
ParserType: "ip",
ClickHouseType: "LowCardinality(IPv6)",
ClickHouseCodec: "ZSTD(1)",
},

17
common/schema/root.go
View File

@@ -84,10 +84,10 @@ func New(config Configuration) (*Component, error) {
}
}
// Add new columns from custom dictionaries after the static ones as we dont
// reference the dicts in the code and they are created during runtime from
// the config, this is enough for us.
customDictColumns := []Column{}
// add new columns from custom dictionaries after the static ones
// as we dont reference the dicts in the code and they are created during runtime from the config, this is enough for us.
for dname, v := range config.CustomDictionaries {
for _, d := range v.Dimensions {
// check if we can actually create the dictionary (we need to know what to match on)
@@ -130,12 +130,21 @@ func New(config Configuration) (*Component, error) {
l = cases.Title(language.Und).String(a.Name)
}
name := fmt.Sprintf("%s%s", d, l)
// compute the key for this new dynamic column, added after the last dynamic column
key := ColumnLast + schema.dynamicColumns
parserType := ""
switch a.Type {
case "IPv6":
parserType = "ip"
case "String":
parserType = "string"
case "UInt8", "UInt16", "UInt32", "UInt64":
parserType = "uint"
}
customDictColumns = append(customDictColumns,
Column{
Key: key,
Name: name,
ParserType: parserType,
ClickHouseType: fmt.Sprintf("LowCardinality(%s)", a.Type),
ClickHouseGenerateFrom: fmt.Sprintf("dictGet('custom_dict_%s', '%s', %s)", dname, a.Name,
matchingString),

3
common/schema/types.go
View File

@@ -33,6 +33,9 @@ type Column struct {
Group ColumnGroup
Depends []ColumnKey
// For parser.
ParserType string
// For ClickHouse. `NotSortingKey' is for columns generated from other
// columns. It is only useful if not ClickHouseMainOnly and not Alias.
// `GenerateFrom' is for a column that's generated from an SQL expression

3
console/data/docs/02-configuration.md
View File

@@ -559,9 +559,6 @@ to enrich your flow with additional informations not possible to get in the
classifier. This works by providing the database with a CSV files containing the
values.
**Note:** filtering by dictionaries is not possible with the current state of
development.
```yaml
schema:
custom-dictionaries:

1
console/data/docs/99-changelog.md
View File

@@ -13,6 +13,7 @@ identified with a specific icon:
## Unreleased
- 🌱 *console*: add filtering support for custom columns
- 🌱 *inlet*: update [Expr](https://expr.medv.io/), the language behind the
classifiers: support for variables
- 🌱 *orchestrator*: improve performance when looking up for `SrcNetPrefix` and

30
console/filter.go
View File

@@ -6,6 +6,7 @@ package console
import (
"fmt"
"net/http"
"sort"
"strconv"
"strings"
@@ -84,21 +85,22 @@ func (c *Component) filterCompleteHandlerFunc(gc *gin.Context) {
completions := []filterCompletion{}
switch input.What {
case "column":
_, err := filter.Parse("", []byte{},
filter.Entrypoint("ConditionExpr"), filter.GlobalStore("meta", &filter.Meta{Schema: c.d.Schema}))
if err != nil {
for _, candidate := range filter.Expected(err) {
if !strings.HasSuffix(candidate, `"i`) {
continue
}
candidate = candidate[1 : len(candidate)-2]
if column, ok := c.d.Schema.LookupColumnByName(candidate); ok && !column.Disabled {
completions = append(completions, filterCompletion{
Label: candidate,
Detail: "column name",
})
}
// We use the schema directly.
columns := []string{}
for _, column := range c.d.Schema.Columns() {
if column.Disabled {
continue
}
if strings.HasPrefix(strings.ToLower(column.Name), strings.ToLower(input.Prefix)) {
columns = append(columns, column.Name)
}
}
sort.Strings(columns)
for _, column := range columns {
completions = append(completions, filterCompletion{
Label: column,
Detail: "column name",
})
}
case "operator":
_, err := filter.Parse("",

24
console/filter/helpers.go
View File

@@ -100,7 +100,7 @@ func (c *current) compileExpr(expr []any, meta *Meta) string {
}
// acceptColumn normalizes and returns the matched column name. It should be
// used in predicate code blocks.
// used in action code blocks.
func (c *current) acceptColumn() (schema.Column, error) {
name := string(c.text)
sch := c.globalStore["meta"].(*Meta).Schema
@@ -112,6 +112,28 @@ func (c *current) acceptColumn() (schema.Column, error) {
return schema.Column{}, fmt.Errorf("unknown column %q", name)
}
// columnIsOfType returns true if the column is of one of the provided type. It
// should be used in predicate code blocks.
func (c *current) columnIsOfType(name any, types ...string) (bool, error) {
nameSl := name.([]any)
var columnName string
for _, s := range nameSl {
columnName += string(s.([]byte))
}
sch := c.globalStore["meta"].(*Meta).Schema
for _, column := range sch.Columns() {
if strings.EqualFold(columnName, column.Name) {
for _, t := range types {
if column.ParserType == t {
return true, nil
}
}
return false, nil
}
}
return false, nil
}
// getColumn gets a column by its name.
func (c *current) getColumn(name string) schema.Column {
sch := c.globalStore["meta"].(*Meta).Schema

63
console/filter/parser.peg
View File

@@ -48,12 +48,9 @@ ConditionExpr "conditional" ←
/ ConditionProtoExpr
ColumnIP ←
"ExporterAddress"i !IdentStart { return c.acceptColumn() }
/ "SrcAddr"i !IdentStart { return c.acceptColumn() }
/ "DstAddr"i !IdentStart { return c.acceptColumn() }
/ "SrcAddrNAT"i !IdentStart { return c.acceptColumn() }
/ "DstAddrNAT"i !IdentStart { return c.acceptColumn() }
/ "NextHop"i !IdentStart { return c.acceptColumn() }
column:[A-Za-z0-9]+ !IdentStart
&{ return c.columnIsOfType(column, "ip") }
{ return c.acceptColumn() }
ConditionIPExpr "condition on IP" ←
column:ColumnIP _
operator:("=" / "!=") _ ip:IP {
@@ -101,34 +98,9 @@ ConditionMACExpr "condition on MAC" ←
}
ConditionStringExpr "condition on string" ←
column:("ExporterName"i !IdentStart { return c.acceptColumn() }
/ "ExporterGroup"i !IdentStart { return c.acceptColumn() }
/ "ExporterRole"i !IdentStart { return c.acceptColumn() }
/ "ExporterSite"i !IdentStart { return c.acceptColumn() }
/ "ExporterRegion"i !IdentStart { return c.acceptColumn() }
/ "ExporterTenant"i !IdentStart { return c.acceptColumn() }
/ "SrcCountry"i !IdentStart { return c.acceptColumn() }
/ "DstCountry"i !IdentStart { return c.acceptColumn() }
/ "SrcNetName"i !IdentStart { return c.acceptColumn() }
/ "DstNetName"i !IdentStart { return c.acceptColumn() }
/ "SrcNetRole"i !IdentStart { return c.acceptColumn() }
/ "DstNetRole"i !IdentStart { return c.acceptColumn() }
/ "SrcNetSite"i !IdentStart { return c.acceptColumn() }
/ "DstNetSite"i !IdentStart { return c.acceptColumn() }
/ "SrcNetRegion"i !IdentStart { return c.acceptColumn() }
/ "DstNetRegion"i !IdentStart { return c.acceptColumn() }
/ "SrcNetTenant"i !IdentStart { return c.acceptColumn() }
/ "DstNetTenant"i !IdentStart { return c.acceptColumn() }
/ "InIfName"i !IdentStart { return c.acceptColumn() }
/ "OutIfName"i !IdentStart { return c.acceptColumn() }
/ "InIfDescription"i !IdentStart { return c.acceptColumn() }
/ "OutIfDescription"i !IdentStart { return c.acceptColumn() }
/ "InIfConnectivity"i !IdentStart { return c.acceptColumn() }
/ "OutIfConnectivity"i !IdentStart { return c.acceptColumn() }
/ "InIfProvider"i !IdentStart { return c.acceptColumn() }
/ "OutIfProvider"i !IdentStart { return c.acceptColumn() }
/ "ICMPv4"i !IdentStart { return c.acceptColumn() }
/ "ICMPv6"i !IdentStart { return c.acceptColumn() }) _
column:(value:[A-Za-z0-9]+ !IdentStart
&{ return c.columnIsOfType(value, "string") }
{ return c.acceptColumn() }) _
rcond:RConditionStringExpr {
return []any{column, rcond}, nil
}
@@ -149,26 +121,9 @@ ConditionBoundaryExpr "condition on boundary" ←
}
ConditionUintExpr "condition on integer" ←
column:("InIfSpeed"i !IdentStart { return c.acceptColumn() }
/ "OutIfSpeed"i !IdentStart { return c.acceptColumn() }
/ "SrcPort"i !IdentStart { return c.acceptColumn() }
/ "DstPort"i !IdentStart { return c.acceptColumn() }
/ "SrcPortNAT"i !IdentStart { return c.acceptColumn() }
/ "DstPortNAT"i !IdentStart { return c.acceptColumn() }
/ "SrcVlan"i !IdentStart { return c.acceptColumn() }
/ "DstVlan"i !IdentStart { return c.acceptColumn() }
/ "PacketSize"i !IdentStart { return c.acceptColumn() }
/ "ForwardingStatus"i !IdentStart { return c.acceptColumn() }
/ "IPTTL"i !IdentStart { return c.acceptColumn() }
/ "IPTos"i !IdentStart { return c.acceptColumn() }
/ "IPFragmentID"i !IdentStart { return c.acceptColumn() }
/ "IPFragmentOffset"i !IdentStart { return c.acceptColumn() }
/ "IPv6FlowLabel"i !IdentStart { return c.acceptColumn() }
/ "TCPFlags"i !IdentStart { return c.acceptColumn() }
/ "ICMPv4Type"i !IdentStart { return c.acceptColumn() }
/ "ICMPv4Code"i !IdentStart { return c.acceptColumn() }
/ "ICMPv6Type"i !IdentStart { return c.acceptColumn() }
/ "ICMPv6Code"i !IdentStart { return c.acceptColumn() }) _
column:(value:[A-Za-z0-9]+ !IdentStart
&{ return c.columnIsOfType(value, "uint") }
{ return c.acceptColumn() }) _
operator:("=" / ">=" / "<=" / "<" / ">" / "!=") _
value:Unsigned64 {
return []any{column, operator, value}, nil

45
console/filter/parser_test.go
View File

@@ -320,9 +320,32 @@ output provider */ = 'telia'`,
{Input: `icmpv4type = 8 AND icmpv4code = 0`, Output: `ICMPv4Type = 8 AND ICMPv4Code = 0`},
{Input: `icmpv6type = 8 or icmpv6code = 0`, Output: `ICMPv6Type = 8 OR ICMPv6Code = 0`},
{Input: `icmpv6 = "echo-reply"`, Output: `ICMPv6 = 'echo-reply'`},
{Input: `SrcAddrDimensionAttribute = "Test"`, Output: `SrcAddrDimensionAttribute = 'Test'`},
{Input: `DstAddrDimensionAttribute = "Test"`, Output: `DstAddrDimensionAttribute = 'Test'`},
{Input: `DstAddrRole = "Test"`, Output: `DstAddrRole = 'Test'`},
{Input: `DstAddrPriority = 200`, Output: `DstAddrPriority = 200`},
{Input: `DstAddrSibling = 2001:db8::1`, Output: `DstAddrSibling = toIPv6('2001:db8::1')`},
{Input: `SrcAddrDimensionAttribute IN ("Test", "None")`, Output: `SrcAddrDimensionAttribute IN ('Test', 'None')`},
}
config := schema.DefaultConfiguration()
config.CustomDictionaries = make(map[string]schema.CustomDict)
config.CustomDictionaries["test"] = schema.CustomDict{
Keys: []schema.CustomDictKey{
{Name: "SrcAddr", Type: "String"},
},
Attributes: []schema.CustomDictAttribute{
{Name: "csv_col_name", Type: "String", Label: "DimensionAttribute"},
{Name: "role", Type: "String"},
{Name: "priority", Type: "UInt16"},
{Name: "sibling", Type: "IPv6"},
},
Source: "test.csv",
Dimensions: []string{"SrcAddr", "DstAddr"},
}
s, _ := schema.New(config)
s = s.EnableAllColumns()
for _, tc := range cases {
tc.MetaIn.Schema = schema.NewMock(t).EnableAllColumns()
tc.MetaIn.Schema = s
tc.MetaOut.Schema = tc.MetaIn.Schema
got, err := Parse("", []byte(tc.Input), GlobalStore("meta", &tc.MetaIn))
if err != nil {
@@ -413,11 +436,29 @@ func TestInvalidFilter(t *testing.T) {
{Input: `SrcVlan = 1000`},
{Input: `DstVlan = 1000`},
{Input: `SrcMAC = 00:11:22:33:44:55:66`, EnableAll: true},
{Input: `SrcAddrDimensionAttribute = 8`},
{Input: `InvalidDimensionAttribute = "Test"`},
}
config := schema.DefaultConfiguration()
config.CustomDictionaries = make(map[string]schema.CustomDict)
config.CustomDictionaries["test"] = schema.CustomDict{
Keys: []schema.CustomDictKey{
{Name: "SrcAddr", Type: "string"},
},
Attributes: []schema.CustomDictAttribute{
{Name: "csv_col_name", Type: "string", Label: "DimensionAttribute"},
{Name: "role", Type: "string"},
},
Source: "test.csv",
Dimensions: []string{"SrcAddr", "DstAddr"},
}
s, _ := schema.New(config)
for _, tc := range cases {
sch := schema.NewMock(t)
if tc.EnableAll {
sch.EnableAllColumns()
s = s.EnableAllColumns()
}
out, err := Parse("", []byte(tc.Input), GlobalStore("meta", &Meta{Schema: sch}))
if err == nil {

14
console/filter_test.go
View File

@@ -158,23 +158,11 @@ LIMIT 20`, "6540").
{
URL: "/api/v0/console/filter/complete",
StatusCode: 200,
JSONInput: gin.H{"what": "column", "prefix": "dSt"},
JSONInput: gin.H{"what": "column", "prefix": "dSta"},
JSONOutput: gin.H{"completions": []gin.H{
{"label": "Dst1stAS", "detail": "column name", "quoted": false},
{"label": "Dst2ndAS", "detail": "column name", "quoted": false},
{"label": "Dst3rdAS", "detail": "column name", "quoted": false},
{"label": "DstAS", "detail": "column name", "quoted": false},
{"label": "DstASPath", "detail": "column name", "quoted": false},
{"label": "DstAddr", "detail": "column name", "quoted": false},
{"label": "DstCommunities", "detail": "column name", "quoted": false},
{"label": "DstCountry", "detail": "column name", "quoted": false},
{"label": "DstNetName", "detail": "column name", "quoted": false},
{"label": "DstNetPrefix", "detail": "column name", "quoted": false},
{"label": "DstNetRegion", "detail": "column name", "quoted": false},
{"label": "DstNetRole", "detail": "column name", "quoted": false},
{"label": "DstNetSite", "detail": "column name", "quoted": false},
{"label": "DstNetTenant", "detail": "column name", "quoted": false},
{"label": "DstPort", "detail": "column name", "quoted": false},
}},
},
{