feat: add option for materialized types & improve filter performance for materialized Prefixes

common/schema/definition.go

@@ -162,6 +162,7 @@ func flows() Schema {
 				Key:                        ColumnSrcNetPrefix,
 				ClickHouseMainOnly:         true,
 				ClickHouseType:             "String",
+				ClickHouseMaterializedType: "LowCardinality(String)",
 				ClickHouseAlias: `CASE
  WHEN EType = 0x800 THEN concat(replaceRegexpOne(IPv6CIDRToRange(SrcAddr, (96 + SrcNetMask)::UInt8).1::String, '^::ffff:', ''), '/', SrcNetMask::String)
  WHEN EType = 0x86dd THEN concat(IPv6CIDRToRange(SrcAddr, SrcNetMask).1::String, '/', SrcNetMask::String)

common/schema/root.go

@@ -30,9 +30,15 @@ func New(config Configuration) (*Component, error) {
 			if column.ClickHouseAlias != "" {
 				column.ClickHouseGenerateFrom = column.ClickHouseAlias
 				column.ClickHouseAlias = ""
+				column.ClickHouseMaterialized = true
 			} else {
 				return nil, fmt.Errorf("no alias configured for %s that can be converted to generate", k)
 			}
+
+			// in case we have another data type for materialized columns, set it
+			if column.ClickHouseMaterializedType != "" {
+				column.ClickHouseType = column.ClickHouseMaterializedType
+			}
 		}
 	}
 	for _, k := range config.Enabled {

common/schema/types.go

@@ -40,6 +40,7 @@ type Column struct {
 	// `TransformTo' work in pairs. The first one is the set of column in the
 	// raw table while the second one is how to transform it for the main table.
 	ClickHouseType             string
+	ClickHouseMaterializedType string
 	ClickHouseCodec            string
 	ClickHouseAlias            string
 	ClickHouseNotSortingKey    bool
@@ -48,6 +49,9 @@ type Column struct {
 	ClickHouseTransformTo      string
 	ClickHouseMainOnly         bool
 
+	// ClickHouseMaterialized indicates that the column was materialized (and is not by default)
+	ClickHouseMaterialized bool
+
 	// For the console. `ClickHouseTruncateIP' makes the specified column
 	// truncatable when used as a dimension.
 	ConsoleNotDimension bool

console/filter/helpers.go

@@ -127,11 +127,21 @@ func (c *current) parsePrefix(direction string) ([]any, error) {
 	if err != nil {
 		return []any{}, errors.New("expecting a prefix")
 	}
+	// if the prefix was materialized, we can directly access it
+	col := c.getColumn(fmt.Sprintf("%sNetPrefix", direction))
+	if col.ClickHouseMaterialized {
+		return []any{
+			fmt.Sprintf("%sNetPrefix", direction), "=",
+			fmt.Sprintf("'%s'", net.String())}, nil
+	}
+	// it the prefix is not materialized, we use the "between" operator
+	c.globalStore["meta"].(*Meta).MainTableRequired = true
 	prefix := "::ffff:"
 	if net.Addr().Is6() {
 		prefix = ""
 	}
 	return []any{
+		fmt.Sprintf("%sAddr", direction),
 		fmt.Sprintf("BETWEEN toIPv6('%s%s') AND toIPv6('%s%s') AND",
 			prefix, net.Masked().Addr().String(), prefix, lastIP(net).String()),
 		c.getColumn(fmt.Sprintf("%sNetMask", direction)), "=", net.Bits(),

console/filter/parser.peg

@@ -78,8 +78,8 @@ ConditionPrefixExpr "condition on prefix" ←
    operator:("=" / "!=") _
    prefix:SourcePrefix {
      switch toString(operator) {
-       case "=": return []any{c.getColumn("SrcAddr"), prefix}, nil
-       case "!=": return []any{"NOT (", c.getColumn("SrcAddr"), prefix, ")"}, nil
+       case "=": return []any{prefix}, nil
+       case "!=": return []any{"NOT (", prefix, ")"}, nil
      }
      return "", nil
    }
@@ -87,8 +87,8 @@ ConditionPrefixExpr "condition on prefix" ←
    operator:("=" / "!=") _
    prefix:DestinationPrefix {
      switch toString(operator) {
-       case "=": return []any{c.getColumn("DstAddr"), prefix}, nil
-       case "!=": return []any{"NOT (", c.getColumn("DstAddr"), prefix, ")"}, nil
+       case "=": return []any{prefix}, nil
+       case "!=": return []any{"NOT (", prefix, ")"}, nil
      }
      return "", nil
    }

console/filter/parser_test.go

@@ -338,6 +338,52 @@ output provider */ = 'telia'`,
 	}
 }
 
+func TestValidMaterializedFilter(t *testing.T) {
+	cases := []struct {
+		Input   string
+		Output  string
+		MetaIn  Meta
+		MetaOut Meta
+	}{
+		{
+			Input:   `DstNetPrefix = 192.168.0.128/27`,
+			Output:  `DstNetPrefix = '192.168.0.128/27'`,
+			MetaOut: Meta{MainTableRequired: false},
+		},
+		{
+			Input:   `SrcNetPrefix = 192.168.0.128/27`,
+			Output:  `SrcNetPrefix = '192.168.0.128/27'`,
+			MetaOut: Meta{MainTableRequired: false},
+		},
+		{
+			Input:   `SrcNetPrefix = 2001:db8::/48`,
+			Output:  `SrcNetPrefix = '2001:db8::/48'`,
+			MetaOut: Meta{MainTableRequired: false},
+		},
+	}
+	for _, tc := range cases {
+		s := schema.NewMock(t).EnableAllColumns()
+		cd, _ := s.Schema.LookupColumnByKey(schema.ColumnDstNetPrefix)
+		cd.ClickHouseMaterialized = true
+		cs, _ := s.Schema.LookupColumnByKey(schema.ColumnSrcNetPrefix)
+		cs.ClickHouseMaterialized = true
+
+		tc.MetaIn.Schema = s
+		tc.MetaOut.Schema = tc.MetaIn.Schema
+		got, err := Parse("", []byte(tc.Input), GlobalStore("meta", &tc.MetaIn))
+		if err != nil {
+			t.Errorf("Parse(%q) error:\n%+v", tc.Input, err)
+			continue
+		}
+		if diff := helpers.Diff(got.(string), tc.Output); diff != "" {
+			t.Errorf("Parse(%q) (-got, +want):\n%s", tc.Input, diff)
+		}
+		if diff := helpers.Diff(tc.MetaIn, tc.MetaOut); diff != "" {
+			t.Errorf("Parse(%q) meta (-got, +want):\n%s", tc.Input, diff)
+		}
+	}
+}
+
 func TestInvalidFilter(t *testing.T) {
 	cases := []struct {
 		Input     string