johanels/akvorado
1
0
Fork 0
mirror of https://github.com/akvorado/akvorado.git synced 2025-12-11 22:14:02 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

schema/common: rename MainOnly to ClickHouseMainOnly

Browse Source
This commit is contained in:
Vincent Bernat
2023-01-18 18:09:45 +01:00
parent c5aa1e7bfa
commit eba3af5183
6 changed files with 27 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
common/schema/clickhouse.go
View File

@@ -67,7 +67,7 @@ func (schema Schema) clickhouseIterate(fn func(Column), options ...ClickHouseTab
if slices.Contains(options, ClickHouseSkipTimeReceived) && column.Key == ColumnTimeReceived {
continue
}
if slices.Contains(options, ClickHouseSkipMainOnlyColumns) && column.MainOnly {
if slices.Contains(options, ClickHouseSkipMainOnlyColumns) && column.ClickHouseMainOnly {
continue
}
if slices.Contains(options, ClickHouseSkipGeneratedColumns) && column.ClickHouseGenerateFrom != "" {
@@ -102,7 +102,7 @@ func (schema Schema) clickhouseIterate(fn func(Column), options ...ClickHouseTab
func (schema Schema) ClickHouseSortingKeys() []string {
cols := schema.ClickHousePrimaryKeys()
for _, column := range schema.columns {
if column.ClickHouseNotSortingKey || column.MainOnly {
if column.ClickHouseNotSortingKey || column.ClickHouseMainOnly {
continue
}
if !slices.Contains(cols, column.Name) {

36
common/schema/definition.go
View File

@@ -118,18 +118,18 @@ func flows() Schema {
{Key: ColumnExporterRegion, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{Key: ColumnExporterTenant, ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true},
{
Key: ColumnSrcAddr,
MainOnly: true,
ClickHouseType: "IPv6",
Key: ColumnSrcAddr,
ClickHouseMainOnly: true,
ClickHouseType: "IPv6",
}, {
Key: ColumnSrcNetMask,
MainOnly: true,
ClickHouseMainOnly: true,
ClickHouseType: "UInt8",
ConsoleNotDimension: true,
}, {
Key: ColumnSrcNetPrefix,
MainOnly: true,
ClickHouseType: "String",
Key: ColumnSrcNetPrefix,
ClickHouseMainOnly: true,
ClickHouseType: "String",
ClickHouseAlias: `CASE
WHEN EType = 0x800 THEN concat(replaceRegexpOne(IPv6CIDRToRange(SrcAddr, (96 + SrcNetMask)::UInt8).1::String, '^::ffff:', ''), '/', SrcNetMask::String)
WHEN EType = 0x86dd THEN concat(IPv6CIDRToRange(SrcAddr, SrcNetMask).1::String, '/', SrcNetMask::String)
@@ -160,9 +160,9 @@ END`,
},
{Key: ColumnSrcCountry, ClickHouseType: "FixedString(2)"},
{
Key: ColumnDstASPath,
MainOnly: true,
ClickHouseType: "Array(UInt32)",
Key: ColumnDstASPath,
ClickHouseMainOnly: true,
ClickHouseType: "Array(UInt32)",
}, {
Key: ColumnDst1stAS,
ClickHouseType: "UInt32",
@@ -176,13 +176,13 @@ END`,
ClickHouseType: "UInt32",
ClickHouseGenerateFrom: "c_DstASPath[3]",
}, {
Key: ColumnDstCommunities,
MainOnly: true,
ClickHouseType: "Array(UInt32)",
Key: ColumnDstCommunities,
ClickHouseMainOnly: true,
ClickHouseType: "Array(UInt32)",
}, {
Key: ColumnDstLargeCommunities,
MainOnly: true,
ClickHouseType: "Array(UInt128)",
Key: ColumnDstLargeCommunities,
ClickHouseMainOnly: true,
ClickHouseType: "Array(UInt128)",
ClickHouseTransformFrom: []Column{
{
Key: ColumnDstLargeCommunitiesASN,
@@ -219,7 +219,7 @@ END`,
},
{Key: ColumnEType, ClickHouseType: "UInt32"}, // TODO: UInt16 but hard to change, primary key
{Key: ColumnProto, ClickHouseType: "UInt32"}, // TODO: UInt8 but hard to change, primary key
{Key: ColumnSrcPort, ClickHouseType: "UInt16", MainOnly: true},
{Key: ColumnSrcPort, ClickHouseType: "UInt16", ClickHouseMainOnly: true},
{Key: ColumnBytes, ClickHouseType: "UInt64", ClickHouseNotSortingKey: true, ConsoleNotDimension: true},
{Key: ColumnPackets, ClickHouseType: "UInt64", ClickHouseNotSortingKey: true, ConsoleNotDimension: true},
{
@@ -273,7 +273,7 @@ func (schema Schema) finalize() Schema {
}
// Add non-main columns with an alias to NotSortingKey
if !column.MainOnly && column.ClickHouseAlias != "" {
if !column.ClickHouseMainOnly && column.ClickHouseAlias != "" {
column.ClickHouseNotSortingKey = true
}

8
common/schema/types.go
View File

@@ -23,12 +23,11 @@ type Schema struct {
// Column represents a column of data.
type Column struct {
Key ColumnKey
Name string
MainOnly bool
Key ColumnKey
Name string
// For ClickHouse. `NotSortingKey' is for columns generated from other
// columns. It is only useful if not MainOnly and not Alias. `GenerateFrom'
// columns. It is only useful if not ClickHouseMainOnly and not Alias. `GenerateFrom'
// is for a column that's generated from an SQL expression instead of being
// retrieved from the protobuf. `TransformFrom' and `TransformTo' work in
// pairs. The first one is the set of column in the raw table while the
@@ -40,6 +39,7 @@ type Column struct {
ClickHouseGenerateFrom string
ClickHouseTransformFrom []Column
ClickHouseTransformTo string
ClickHouseMainOnly bool
// For the console.
ConsoleNotDimension bool

2
console/filter/helpers.go
View File

@@ -64,7 +64,7 @@ func (c *current) acceptColumn() (string, error) {
// should be provided.
func (c *current) metaColumn(name string) error {
if column, ok := c.globalStore["meta"].(*Meta).Schema.LookupColumnByName(name); ok {
if column.MainOnly {
if column.ClickHouseMainOnly {
c.state["main-table-only"] = true
}
}

2
console/query.go
View File

@@ -15,7 +15,7 @@ func requireMainTable(sch *schema.Component, qcs []query.Column, qf query.Filter
return true
}
for _, qc := range qcs {
if column, ok := sch.LookupColumnByKey(qc.Key()); ok && column.MainOnly {
if column, ok := sch.LookupColumnByKey(qc.Key()); ok && column.ClickHouseMainOnly {
return true
}
}

2
orchestrator/clickhouse/migrations_helpers.go
View File

@@ -403,7 +403,7 @@ ORDER BY position ASC
previousColumn := ""
outer:
for _, wantedColumn := range c.d.Schema.Columns() {
if resolution.Interval > 0 && wantedColumn.MainOnly {
if resolution.Interval > 0 && wantedColumn.ClickHouseMainOnly {
continue
}
// Check if the column already exists