diff --git a/common/schema/definition.go b/common/schema/definition.go index 1896683c..2313bee2 100644 --- a/common/schema/definition.go +++ b/common/schema/definition.go @@ -159,9 +159,10 @@ func flows() Schema { ConsoleNotDimension: true, }, { - Key: ColumnSrcNetPrefix, - ClickHouseMainOnly: true, - ClickHouseType: "String", + Key: ColumnSrcNetPrefix, + ClickHouseMainOnly: true, + ClickHouseType: "String", + ClickHouseMaterializedType: "LowCardinality(String)", ClickHouseAlias: `CASE WHEN EType = 0x800 THEN concat(replaceRegexpOne(IPv6CIDRToRange(SrcAddr, (96 + SrcNetMask)::UInt8).1::String, '^::ffff:', ''), '/', SrcNetMask::String) WHEN EType = 0x86dd THEN concat(IPv6CIDRToRange(SrcAddr, SrcNetMask).1::String, '/', SrcNetMask::String) diff --git a/common/schema/root.go b/common/schema/root.go index dcafd86d..4f551a1e 100644 --- a/common/schema/root.go +++ b/common/schema/root.go @@ -30,9 +30,15 @@ func New(config Configuration) (*Component, error) { if column.ClickHouseAlias != "" { column.ClickHouseGenerateFrom = column.ClickHouseAlias column.ClickHouseAlias = "" + column.ClickHouseMaterialized = true } else { return nil, fmt.Errorf("no alias configured for %s that can be converted to generate", k) } + + // in case we have another data type for materialized columns, set it + if column.ClickHouseMaterializedType != "" { + column.ClickHouseType = column.ClickHouseMaterializedType + } } } for _, k := range config.Enabled { diff --git a/common/schema/types.go b/common/schema/types.go index 06aa10ac..affe246e 100644 --- a/common/schema/types.go +++ b/common/schema/types.go @@ -39,14 +39,18 @@ type Column struct { // instead of being retrieved from the protobuf. `TransformFrom' and // `TransformTo' work in pairs. The first one is the set of column in the // raw table while the second one is how to transform it for the main table. - ClickHouseType string - ClickHouseCodec string - ClickHouseAlias string - ClickHouseNotSortingKey bool - ClickHouseGenerateFrom string - ClickHouseTransformFrom []Column - ClickHouseTransformTo string - ClickHouseMainOnly bool + ClickHouseType string + ClickHouseMaterializedType string + ClickHouseCodec string + ClickHouseAlias string + ClickHouseNotSortingKey bool + ClickHouseGenerateFrom string + ClickHouseTransformFrom []Column + ClickHouseTransformTo string + ClickHouseMainOnly bool + + // ClickHouseMaterialized indicates that the column was materialized (and is not by default) + ClickHouseMaterialized bool // For the console. `ClickHouseTruncateIP' makes the specified column // truncatable when used as a dimension. diff --git a/console/filter/helpers.go b/console/filter/helpers.go index 6e10782c..17ce3464 100644 --- a/console/filter/helpers.go +++ b/console/filter/helpers.go @@ -127,11 +127,21 @@ func (c *current) parsePrefix(direction string) ([]any, error) { if err != nil { return []any{}, errors.New("expecting a prefix") } + // if the prefix was materialized, we can directly access it + col := c.getColumn(fmt.Sprintf("%sNetPrefix", direction)) + if col.ClickHouseMaterialized { + return []any{ + fmt.Sprintf("%sNetPrefix", direction), "=", + fmt.Sprintf("'%s'", net.String())}, nil + } + // it the prefix is not materialized, we use the "between" operator + c.globalStore["meta"].(*Meta).MainTableRequired = true prefix := "::ffff:" if net.Addr().Is6() { prefix = "" } return []any{ + fmt.Sprintf("%sAddr", direction), fmt.Sprintf("BETWEEN toIPv6('%s%s') AND toIPv6('%s%s') AND", prefix, net.Masked().Addr().String(), prefix, lastIP(net).String()), c.getColumn(fmt.Sprintf("%sNetMask", direction)), "=", net.Bits(), diff --git a/console/filter/parser.peg b/console/filter/parser.peg index 6edf795e..eae255af 100644 --- a/console/filter/parser.peg +++ b/console/filter/parser.peg @@ -78,8 +78,8 @@ ConditionPrefixExpr "condition on prefix" ← operator:("=" / "!=") _ prefix:SourcePrefix { switch toString(operator) { - case "=": return []any{c.getColumn("SrcAddr"), prefix}, nil - case "!=": return []any{"NOT (", c.getColumn("SrcAddr"), prefix, ")"}, nil + case "=": return []any{prefix}, nil + case "!=": return []any{"NOT (", prefix, ")"}, nil } return "", nil } @@ -87,8 +87,8 @@ ConditionPrefixExpr "condition on prefix" ← operator:("=" / "!=") _ prefix:DestinationPrefix { switch toString(operator) { - case "=": return []any{c.getColumn("DstAddr"), prefix}, nil - case "!=": return []any{"NOT (", c.getColumn("DstAddr"), prefix, ")"}, nil + case "=": return []any{prefix}, nil + case "!=": return []any{"NOT (", prefix, ")"}, nil } return "", nil } diff --git a/console/filter/parser_test.go b/console/filter/parser_test.go index eb3b99c4..7610f807 100644 --- a/console/filter/parser_test.go +++ b/console/filter/parser_test.go @@ -338,6 +338,52 @@ output provider */ = 'telia'`, } } +func TestValidMaterializedFilter(t *testing.T) { + cases := []struct { + Input string + Output string + MetaIn Meta + MetaOut Meta + }{ + { + Input: `DstNetPrefix = 192.168.0.128/27`, + Output: `DstNetPrefix = '192.168.0.128/27'`, + MetaOut: Meta{MainTableRequired: false}, + }, + { + Input: `SrcNetPrefix = 192.168.0.128/27`, + Output: `SrcNetPrefix = '192.168.0.128/27'`, + MetaOut: Meta{MainTableRequired: false}, + }, + { + Input: `SrcNetPrefix = 2001:db8::/48`, + Output: `SrcNetPrefix = '2001:db8::/48'`, + MetaOut: Meta{MainTableRequired: false}, + }, + } + for _, tc := range cases { + s := schema.NewMock(t).EnableAllColumns() + cd, _ := s.Schema.LookupColumnByKey(schema.ColumnDstNetPrefix) + cd.ClickHouseMaterialized = true + cs, _ := s.Schema.LookupColumnByKey(schema.ColumnSrcNetPrefix) + cs.ClickHouseMaterialized = true + + tc.MetaIn.Schema = s + tc.MetaOut.Schema = tc.MetaIn.Schema + got, err := Parse("", []byte(tc.Input), GlobalStore("meta", &tc.MetaIn)) + if err != nil { + t.Errorf("Parse(%q) error:\n%+v", tc.Input, err) + continue + } + if diff := helpers.Diff(got.(string), tc.Output); diff != "" { + t.Errorf("Parse(%q) (-got, +want):\n%s", tc.Input, diff) + } + if diff := helpers.Diff(tc.MetaIn, tc.MetaOut); diff != "" { + t.Errorf("Parse(%q) meta (-got, +want):\n%s", tc.Input, diff) + } + } +} + func TestInvalidFilter(t *testing.T) { cases := []struct { Input string