From aae92e7fe001b442c00d94ebdfd297fc3aa49b0c Mon Sep 17 00:00:00 2001 From: Vincent Bernat Date: Tue, 3 Jan 2023 17:45:50 +0100 Subject: [PATCH] common/schema: use an ordered map for columns For ClickHouse, we can iterate, but for other aspects, we need a direct access. --- common/schema/clickhouse.go | 6 ++- common/schema/flows.go | 49 +++++++++---------- common/schema/root.go | 5 +- console/filter/helpers.go | 18 +++---- go.mod | 5 +- go.sum | 2 + orchestrator/clickhouse/migrations_helpers.go | 6 ++- 7 files changed, 47 insertions(+), 44 deletions(-) diff --git a/common/schema/clickhouse.go b/common/schema/clickhouse.go index 3e0735ed..e015fbe2 100644 --- a/common/schema/clickhouse.go +++ b/common/schema/clickhouse.go @@ -63,7 +63,8 @@ func (schema Schema) SelectColumns(options ...TableOption) []string { } func (schema Schema) iterate(fn func(column Column), options ...TableOption) { - for _, column := range schema.Columns { + for pair := schema.Columns.Front(); pair != nil; pair = pair.Next() { + column := pair.Value if slices.Contains(options, SkipTimeReceived) && column.Name == "TimeReceived" { continue } @@ -101,7 +102,8 @@ func (schema Schema) iterate(fn func(column Column), options ...TableOption) { // SortingKeys returns the list of sorting keys, prefixed by the primary keys. func (schema Schema) SortingKeys() []string { cols := append([]string{}, schema.PrimaryKeys...) - for _, column := range schema.Columns { + for pair := schema.Columns.Front(); pair != nil; pair = pair.Next() { + column := pair.Value if column.NotSortingKey || column.MainOnly { continue } diff --git a/common/schema/flows.go b/common/schema/flows.go index ecc4ce96..c500420d 100644 --- a/common/schema/flows.go +++ b/common/schema/flows.go @@ -6,6 +6,8 @@ package schema import ( "fmt" "strings" + + orderedmap "github.com/elliotchance/orderedmap/v2" ) // Flows is the data schema for flows tables. Any column starting with Src/InIf @@ -24,7 +26,7 @@ var Flows = Schema{ "DstAS", "SamplingRate", }, - Columns: []Column{ + Columns: buildMapFromColumns([]Column{ { Name: "TimeReceived", Type: "DateTime", @@ -143,44 +145,39 @@ END`, }(), }, {Name: "ForwardingStatus", Type: "UInt32"}, - }, + }), } -func init() { - // Expand the schema Src → Dst and InIf → OutIf - newSchema := []Column{} - for _, column := range Flows.Columns { - newSchema = append(newSchema, column) +func buildMapFromColumns(columns []Column) *orderedmap.OrderedMap[string, Column] { + omap := orderedmap.NewOrderedMap[string, Column]() + for _, column := range columns { + // Add non-main columns with an alias to NotSortingKey + if !column.MainOnly && column.Alias != "" { + column.NotSortingKey = true + } + omap.Set(column.Name, column) + // Expand the schema Src → Dst and InIf → OutIf if strings.HasPrefix(column.Name, "Src") { column.Name = fmt.Sprintf("Dst%s", column.Name[3:]) column.Alias = strings.ReplaceAll(column.Alias, "Src", "Dst") - newSchema = append(newSchema, column) + omap.Set(column.Name, column) } else if strings.HasPrefix(column.Name, "InIf") { column.Name = fmt.Sprintf("OutIf%s", column.Name[4:]) column.Alias = strings.ReplaceAll(column.Alias, "InIf", "OutIf") - newSchema = append(newSchema, column) + omap.Set(column.Name, column) } } - Flows.Columns = newSchema + return omap +} - // Add non-main columns with an alias to NotSortingKey - for idx, column := range Flows.Columns { - if !column.MainOnly && column.Alias != "" { - Flows.Columns[idx].NotSortingKey = true - } - } - - // Also do some checks. -outer: +func init() { for _, key := range Flows.PrimaryKeys { - for _, column := range Flows.Columns { - if column.Name == key { - if column.NotSortingKey { - panic(fmt.Sprintf("primary key %q is marked as a non-sorting key", key)) - } - continue outer + if column, ok := Flows.Columns.Get(key); !ok { + panic(fmt.Sprintf("primary key %q not a column", key)) + } else { + if column.NotSortingKey { + panic(fmt.Sprintf("primary key %q is marked as a non-sorting key", key)) } } - panic(fmt.Sprintf("primary key %q not a column", key)) } } diff --git a/common/schema/root.go b/common/schema/root.go index 771e8989..6d505e73 100644 --- a/common/schema/root.go +++ b/common/schema/root.go @@ -6,9 +6,12 @@ // will use it. package schema +import orderedmap "github.com/elliotchance/orderedmap/v2" + // Schema is the data schema. type Schema struct { - Columns []Column + // We use an ordered map for direct access to columns. + Columns *orderedmap.OrderedMap[string, Column] // For ClickHouse. This is the set of primary keys (order is important and // may not follow column order). diff --git a/console/filter/helpers.go b/console/filter/helpers.go index 592af3e2..4d933d6c 100644 --- a/console/filter/helpers.go +++ b/console/filter/helpers.go @@ -35,10 +35,8 @@ func ReverseColumnDirection(name string) string { if strings.HasPrefix(name, "Out") { candidate = "In" + name[3:] } - for _, column := range schema.Flows.Columns { - if candidate == column.Name { - return candidate - } + if column, ok := schema.Flows.Columns.Get(candidate); ok { + return column.Name } return name } @@ -47,7 +45,8 @@ func ReverseColumnDirection(name string) string { // in predicate code blocks. func (c *current) acceptColumn() (string, error) { name := string(c.text) - for _, column := range schema.Flows.Columns { + for pair := schema.Flows.Columns.Front(); pair != nil; pair = pair.Next() { + column := pair.Value if strings.EqualFold(name, column.Name) { if c.globalStore["meta"].(*Meta).ReverseDirection { return ReverseColumnDirection(column.Name), nil @@ -62,12 +61,9 @@ func (c *current) acceptColumn() (string, error) { // in state change blocks. Unfortunately, it cannot extract matched text, so it // should be provided. func (c *current) metaColumn(name string) error { - for _, column := range schema.Flows.Columns { - if strings.EqualFold(name, column.Name) { - if column.MainOnly { - c.state["main-table-only"] = true - } - return nil + if column, ok := schema.Flows.Columns.Get(name); ok { + if column.MainOnly { + c.state["main-table-only"] = true } } return nil diff --git a/go.mod b/go.mod index e4f32395..39d94fee 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/docker/docker v20.10.22+incompatible github.com/docker/go-connections v0.4.0 github.com/eapache/go-resiliency v1.3.0 + github.com/elliotchance/orderedmap/v2 v2.2.0 github.com/fsnotify/fsnotify v1.6.0 github.com/gin-gonic/gin v1.8.2 github.com/glebarez/sqlite v1.6.0 @@ -28,6 +29,7 @@ require ( github.com/mattn/go-isatty v0.0.17 github.com/mitchellh/mapstructure v1.5.0 github.com/netsampler/goflow2 v1.1.1-0.20221008154147-57fad2e0c837 + github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 github.com/oschwald/maxminddb-golang v1.10.0 github.com/osrg/gobgp/v3 v3.10.0 github.com/prometheus/client_golang v1.14.0 @@ -43,6 +45,7 @@ require ( golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 golang.org/x/sys v0.3.0 golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 + google.golang.org/protobuf v1.28.1 gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 gopkg.in/yaml.v2 v2.4.0 gorm.io/gorm v1.24.3 @@ -95,7 +98,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect github.com/paulmach/orb v0.7.1 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect @@ -123,7 +125,6 @@ require ( golang.org/x/net v0.4.0 // indirect golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 // indirect golang.org/x/text v0.5.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.3.0 // indirect modernc.org/libc v1.21.5 // indirect diff --git a/go.sum b/go.sum index 92ce83ae..d3566ede 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk= +github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= diff --git a/orchestrator/clickhouse/migrations_helpers.go b/orchestrator/clickhouse/migrations_helpers.go index b2a028dd..12fee1d1 100644 --- a/orchestrator/clickhouse/migrations_helpers.go +++ b/orchestrator/clickhouse/migrations_helpers.go @@ -125,7 +125,8 @@ LAYOUT({{ .Layout }}()) func (c *Component) createExportersView(ctx context.Context) error { // Select the columns we need cols := []string{} - for _, column := range schema.Flows.Columns { + for pair := schema.Flows.Columns.Front(); pair != nil; pair = pair.Next() { + column := pair.Value if column.Name == "TimeReceived" || strings.HasPrefix(column.Name, "Exporter") { cols = append(cols, column.Name) } @@ -398,7 +399,8 @@ ORDER BY position ASC modifications := []string{} previousColumn := "" outer: - for _, wantedColumn := range schema.Flows.Columns { + for pair := schema.Flows.Columns.Front(); pair != nil; pair = pair.Next() { + wantedColumn := pair.Value if resolution.Interval > 0 && wantedColumn.MainOnly { continue }