common/schema: use an ordered map for columns

For ClickHouse, we can iterate, but for other aspects, we need a direct
access.

common/schema/clickhouse.go

@@ -63,7 +63,8 @@ func (schema Schema) SelectColumns(options ...TableOption) []string {
 }
 
 func (schema Schema) iterate(fn func(column Column), options ...TableOption) {
-	for _, column := range schema.Columns {
+	for pair := schema.Columns.Front(); pair != nil; pair = pair.Next() {
+		column := pair.Value
 		if slices.Contains(options, SkipTimeReceived) && column.Name == "TimeReceived" {
 			continue
 		}
@@ -101,7 +102,8 @@ func (schema Schema) iterate(fn func(column Column), options ...TableOption) {
 // SortingKeys returns the list of sorting keys, prefixed by the primary keys.
 func (schema Schema) SortingKeys() []string {
 	cols := append([]string{}, schema.PrimaryKeys...)
-	for _, column := range schema.Columns {
+	for pair := schema.Columns.Front(); pair != nil; pair = pair.Next() {
+		column := pair.Value
 		if column.NotSortingKey || column.MainOnly {
 			continue
 		}

common/schema/flows.go

@@ -6,6 +6,8 @@ package schema
 import (
 	"fmt"
 	"strings"
+
+	orderedmap "github.com/elliotchance/orderedmap/v2"
 )
 
 // Flows is the data schema for flows tables. Any column starting with Src/InIf
@@ -24,7 +26,7 @@ var Flows = Schema{
 		"DstAS",
 		"SamplingRate",
 	},
-	Columns: []Column{
+	Columns: buildMapFromColumns([]Column{
 		{
 			Name:  "TimeReceived",
 			Type:  "DateTime",
@@ -143,44 +145,39 @@ END`,
 			}(),
 		},
 		{Name: "ForwardingStatus", Type: "UInt32"},
-	},
+	}),
 }
 
-func init() {
-	// Expand the schema Src → Dst and InIf → OutIf
-	newSchema := []Column{}
-	for _, column := range Flows.Columns {
-		newSchema = append(newSchema, column)
+func buildMapFromColumns(columns []Column) *orderedmap.OrderedMap[string, Column] {
+	omap := orderedmap.NewOrderedMap[string, Column]()
+	for _, column := range columns {
+		// Add non-main columns with an alias to NotSortingKey
+		if !column.MainOnly && column.Alias != "" {
+			column.NotSortingKey = true
+		}
+		omap.Set(column.Name, column)
+		// Expand the schema Src → Dst and InIf → OutIf
 		if strings.HasPrefix(column.Name, "Src") {
 			column.Name = fmt.Sprintf("Dst%s", column.Name[3:])
 			column.Alias = strings.ReplaceAll(column.Alias, "Src", "Dst")
-			newSchema = append(newSchema, column)
+			omap.Set(column.Name, column)
 		} else if strings.HasPrefix(column.Name, "InIf") {
 			column.Name = fmt.Sprintf("OutIf%s", column.Name[4:])
 			column.Alias = strings.ReplaceAll(column.Alias, "InIf", "OutIf")
-			newSchema = append(newSchema, column)
+			omap.Set(column.Name, column)
 		}
 	}
-	Flows.Columns = newSchema
+	return omap
+}
 
-	// Add non-main columns with an alias to NotSortingKey
-	for idx, column := range Flows.Columns {
-		if !column.MainOnly && column.Alias != "" {
-			Flows.Columns[idx].NotSortingKey = true
-		}
-	}
-
-	// Also do some checks.
-outer:
+func init() {
 	for _, key := range Flows.PrimaryKeys {
-		for _, column := range Flows.Columns {
-			if column.Name == key {
-				if column.NotSortingKey {
-					panic(fmt.Sprintf("primary key %q is marked as a non-sorting key", key))
-				}
-				continue outer
+		if column, ok := Flows.Columns.Get(key); !ok {
+			panic(fmt.Sprintf("primary key %q not a column", key))
+		} else {
+			if column.NotSortingKey {
+				panic(fmt.Sprintf("primary key %q is marked as a non-sorting key", key))
 			}
 		}
-		panic(fmt.Sprintf("primary key %q not a column", key))
 	}
 }

common/schema/root.go

@@ -6,9 +6,12 @@
 // will use it.
 package schema
 
+import orderedmap "github.com/elliotchance/orderedmap/v2"
+
 // Schema is the data schema.
 type Schema struct {
-	Columns []Column
+	// We use an ordered map for direct access to columns.
+	Columns *orderedmap.OrderedMap[string, Column]
 
 	// For ClickHouse. This is the set of primary keys (order is important and
 	// may not follow column order).

console/filter/helpers.go

@@ -35,10 +35,8 @@ func ReverseColumnDirection(name string) string {
 	if strings.HasPrefix(name, "Out") {
 		candidate = "In" + name[3:]
 	}
-	for _, column := range schema.Flows.Columns {
-		if candidate == column.Name {
-			return candidate
-		}
+	if column, ok := schema.Flows.Columns.Get(candidate); ok {
+		return column.Name
 	}
 	return name
 }
@@ -47,7 +45,8 @@ func ReverseColumnDirection(name string) string {
 // in predicate code blocks.
 func (c *current) acceptColumn() (string, error) {
 	name := string(c.text)
-	for _, column := range schema.Flows.Columns {
+	for pair := schema.Flows.Columns.Front(); pair != nil; pair = pair.Next() {
+		column := pair.Value
 		if strings.EqualFold(name, column.Name) {
 			if c.globalStore["meta"].(*Meta).ReverseDirection {
 				return ReverseColumnDirection(column.Name), nil
@@ -62,12 +61,9 @@ func (c *current) acceptColumn() (string, error) {
 // in state change blocks. Unfortunately, it cannot extract matched text, so it
 // should be provided.
 func (c *current) metaColumn(name string) error {
-	for _, column := range schema.Flows.Columns {
-		if strings.EqualFold(name, column.Name) {
-			if column.MainOnly {
-				c.state["main-table-only"] = true
-			}
-			return nil
+	if column, ok := schema.Flows.Columns.Get(name); ok {
+		if column.MainOnly {
+			c.state["main-table-only"] = true
 		}
 	}
 	return nil

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/docker/docker v20.10.22+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/eapache/go-resiliency v1.3.0
+	github.com/elliotchance/orderedmap/v2 v2.2.0
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/gin-gonic/gin v1.8.2
 	github.com/glebarez/sqlite v1.6.0
@@ -28,6 +29,7 @@ require (
 	github.com/mattn/go-isatty v0.0.17
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/netsampler/goflow2 v1.1.1-0.20221008154147-57fad2e0c837
+	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799
 	github.com/oschwald/maxminddb-golang v1.10.0
 	github.com/osrg/gobgp/v3 v3.10.0
 	github.com/prometheus/client_golang v1.14.0
@@ -43,6 +45,7 @@ require (
 	golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15
 	golang.org/x/sys v0.3.0
 	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9
+	google.golang.org/protobuf v1.28.1
 	gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
 	gopkg.in/yaml.v2 v2.4.0
 	gorm.io/gorm v1.24.3
@@ -95,7 +98,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/paulmach/orb v0.7.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/pierrec/lz4/v4 v4.1.17 // indirect
@@ -123,7 +125,6 @@ require (
 	golang.org/x/net v0.4.0 // indirect
 	golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 // indirect
 	golang.org/x/text v0.5.0 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gotest.tools/v3 v3.3.0 // indirect
 	modernc.org/libc v1.21.5 // indirect

go.sum

@@ -100,6 +100,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
+github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk=
+github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=

orchestrator/clickhouse/migrations_helpers.go

@@ -125,7 +125,8 @@ LAYOUT({{ .Layout }}())
 func (c *Component) createExportersView(ctx context.Context) error {
 	// Select the columns we need
 	cols := []string{}
-	for _, column := range schema.Flows.Columns {
+	for pair := schema.Flows.Columns.Front(); pair != nil; pair = pair.Next() {
+		column := pair.Value
 		if column.Name == "TimeReceived" || strings.HasPrefix(column.Name, "Exporter") {
 			cols = append(cols, column.Name)
 		}
@@ -398,7 +399,8 @@ ORDER BY position ASC
 	modifications := []string{}
 	previousColumn := ""
 outer:
-	for _, wantedColumn := range schema.Flows.Columns {
+	for pair := schema.Flows.Columns.Front(); pair != nil; pair = pair.Next() {
+		wantedColumn := pair.Value
 		if resolution.Interval > 0 && wantedColumn.MainOnly {
 			continue
 		}