johanels/akvorado
1
0
Fork 0
mirror of https://github.com/akvorado/akvorado.git synced 2025-12-12 06:24:10 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

chore: capitalize comments

Browse Source
This commit is contained in:
Vincent Bernat
2024-01-22 20:34:08 +01:00
parent d9e245e416
commit cec8661387
6 changed files with 29 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
common/schema/root.go
View File

@@ -35,7 +35,7 @@ func New(config Configuration) (*Component, error) {
return nil, fmt.Errorf("no alias configured for %s that can be converted to generate", k) return nil, fmt.Errorf("no alias configured for %s that can be converted to generate", k)
} }
// in case we have another data type for materialized columns, set it // In case we have another data type for materialized columns, set it
if column.ClickHouseMaterializedType != "" { if column.ClickHouseMaterializedType != "" {
column.ClickHouseType = column.ClickHouseMaterializedType column.ClickHouseType = column.ClickHouseMaterializedType
} }
@@ -90,27 +90,28 @@ func New(config Configuration) (*Component, error) {
customDictColumns := []Column{} customDictColumns := []Column{}
for dname, v := range config.CustomDictionaries { for dname, v := range config.CustomDictionaries {
for _, d := range v.Dimensions { for _, d := range v.Dimensions {
// check if we can actually create the dictionary (we need to know what to match on) // Check if we can actually create the dictionary (we need to know what to match on)
if len(v.Keys) == 0 { if len(v.Keys) == 0 {
return nil, fmt.Errorf("custom dictionary %s has no keys, this is not supported", dname) return nil, fmt.Errorf("custom dictionary %s has no keys, this is not supported", dname)
} }
if len(v.Keys) > 1 { if len(v.Keys) > 1 {
// if more than one key is present, every key needs either a MatchDimension or a MatchDimensionSuffix // If more than one key is present, every key needs either a MatchDimension or a MatchDimensionSuffix
for _, kv := range v.Keys { for _, kv := range v.Keys {
if kv.MatchDimension == "" && kv.MatchDimensionSuffix == "" { if kv.MatchDimension == "" && kv.MatchDimensionSuffix == "" {
return nil, fmt.Errorf("custom dictionary %s has more than one key, but key %s has neither MatchDimension nor MatchDimensionSuffix set", dname, kv.Name) return nil, fmt.Errorf("custom dictionary %s has more than one key, but key %s has neither MatchDimension nor MatchDimensionSuffix set", dname, kv.Name)
} }
} }
} }
// first, we need to build the matching string for this // First, we need to build the matching string for this
matchingList := []string{} matchingList := []string{}
// prefer match dimension or match dimension suffix if available // Prefer match dimension or match dimension suffix if available
for _, kv := range v.Keys { for _, kv := range v.Keys {
if kv.MatchDimension != "" { if kv.MatchDimension != "" {
matchingList = append(matchingList, kv.MatchDimension) matchingList = append(matchingList, kv.MatchDimension)
continue continue
} }
// match post is appended after the dimension name, and useful if we wanna match a subkey e.g. both in Src/Dst // Match post is appended after the dimension name, and useful
// if we wanna match a subkey e.g. both in Src/Dst
if kv.MatchDimensionSuffix != "" { if kv.MatchDimensionSuffix != "" {
matchingList = append(matchingList, fmt.Sprintf("%s%s", d, kv.MatchDimensionSuffix)) matchingList = append(matchingList, fmt.Sprintf("%s%s", d, kv.MatchDimensionSuffix))
} }
@@ -119,12 +120,13 @@ func New(config Configuration) (*Component, error) {
if len(matchingList) > 0 { if len(matchingList) > 0 {
matchingString = fmt.Sprintf("(%s)", strings.Join(matchingList, ",")) matchingString = fmt.Sprintf("(%s)", strings.Join(matchingList, ","))
} else { } else {
// if match dimension and match dimension suffix are both not available, we use the dimension name (e.g. SrcAddr) // If match dimension and match dimension suffix are both not
// available, we use the dimension name (e.g. SrcAddr)
matchingString = d matchingString = d
} }
for _, a := range v.Attributes { for _, a := range v.Attributes {
// add the dimension combined with capitalizing the name of the dimension field // Add the dimension combined with capitalizing the name of the dimension field
l := a.Label l := a.Label
if l == "" { if l == "" {
l = cases.Title(language.Und).String(a.Name) l = cases.Title(language.Und).String(a.Name)

10
common/schema/root_test.go
View File

@@ -89,7 +89,7 @@ func TestCustomDictionaries(t *testing.T) {
srcRoleFound := false srcRoleFound := false
dstRoleFound := false dstRoleFound := false
// check if srcAddrAttribute and dstAddrAttribute are in s.columns, and have the correct type/generatefrom // Check if srcAddrAttribute and dstAddrAttribute are in s.columns, and have the correct type/generatefrom
for _, column := range s.Columns() { for _, column := range s.Columns() {
if column.Name == "SrcAddrDimensionAttribute" { if column.Name == "SrcAddrDimensionAttribute" {
srcFound = true srcFound = true
@@ -109,7 +109,7 @@ func TestCustomDictionaries(t *testing.T) {
t.Fatalf("DstAddrDimensionAttribute should be generated from `dictGet('custom_dict_test', 'csv_col_name', DstAddr)`, is %s", column.ClickHouseGenerateFrom) t.Fatalf("DstAddrDimensionAttribute should be generated from `dictGet('custom_dict_test', 'csv_col_name', DstAddr)`, is %s", column.ClickHouseGenerateFrom)
} }
} }
// this part only tests default dimension name generation // This part only tests default dimension name generation
if column.Name == "SrcAddrRole" { if column.Name == "SrcAddrRole" {
srcRoleFound = true srcRoleFound = true
} }
@@ -159,7 +159,7 @@ func TestCustomDictionariesMatcher(t *testing.T) {
outFound := false outFound := false
inFound := false inFound := false
// check if srcAddrAttribute and dstAddrAttribute are in s.columns, and have the correct type/generatefrom // Check if srcAddrAttribute and dstAddrAttribute are in s.columns, and have the correct type/generatefrom
for _, column := range s.Columns() { for _, column := range s.Columns() {
if column.Name == "OutIfDimensionAttribute" { if column.Name == "OutIfDimensionAttribute" {
outFound = true outFound = true
@@ -189,7 +189,7 @@ func TestCustomDictionariesMatcher(t *testing.T) {
} }
} }
// we need MatchDimension or MatchDimensionSuffix for multiple keys // We need MatchDimension or MatchDimensionSuffix for multiple keys
func TestCustomDictMultiKeyErr(t *testing.T) { func TestCustomDictMultiKeyErr(t *testing.T) {
config := schema.DefaultConfiguration() config := schema.DefaultConfiguration()
config.CustomDictionaries = make(map[string]schema.CustomDict) config.CustomDictionaries = make(map[string]schema.CustomDict)
@@ -216,7 +216,7 @@ func TestCustomDictMultiKeyErr(t *testing.T) {
} }
} }
// a dict without key makes no sense, catch this // A dict without key makes no sense, catch this
func TestCustomDictNoKeyErr(t *testing.T) { func TestCustomDictNoKeyErr(t *testing.T) {
config := schema.DefaultConfiguration() config := schema.DefaultConfiguration()
config.CustomDictionaries = make(map[string]schema.CustomDict) config.CustomDictionaries = make(map[string]schema.CustomDict)

4
console/filter/helpers.go
View File

@@ -149,14 +149,14 @@ func (c *current) parsePrefix(direction string) ([]any, error) {
if err != nil { if err != nil {
return []any{}, errors.New("expecting a prefix") return []any{}, errors.New("expecting a prefix")
} }
// if the prefix was materialized, we can directly access it // If the prefix was materialized, we can directly access it
col := c.getColumn(fmt.Sprintf("%sNetPrefix", direction)) col := c.getColumn(fmt.Sprintf("%sNetPrefix", direction))
if col.ClickHouseMaterialized { if col.ClickHouseMaterialized {
return []any{ return []any{
fmt.Sprintf("%sNetPrefix", direction), "=", fmt.Sprintf("%sNetPrefix", direction), "=",
fmt.Sprintf("'%s'", net.String())}, nil fmt.Sprintf("'%s'", net.String())}, nil
} }
// it the prefix is not materialized, we use the "between" operator // If the prefix is not materialized, we use the "between" operator
c.globalStore["meta"].(*Meta).MainTableRequired = true c.globalStore["meta"].(*Meta).MainTableRequired = true
prefix := "::ffff:" prefix := "::ffff:"
if net.Addr().Is6() { if net.Addr().Is6() {

2
orchestrator/clickhouse/http.go
View File

@@ -105,7 +105,7 @@ func (c *Component) registerHTTPHandlers() error {
w.Write(result.Bytes()) w.Write(result.Bytes())
})) }))
// add handler for custom dicts // Add handler for custom dicts
for name, dict := range c.d.Schema.GetCustomDictConfig() { for name, dict := range c.d.Schema.GetCustomDictConfig() {
// we need to call this a func to avoid issues with the for loop // we need to call this a func to avoid issues with the for loop
k := name k := name

6
orchestrator/clickhouse/migrations.go
View File

@@ -84,7 +84,7 @@ func (c *Component) migrateDatabase() error {
return err return err
} }
// prepare custom dictionary migrations // Prepare custom dictionary migrations
var dictMigrations []func() error var dictMigrations []func() error
for k, v := range c.d.Schema.GetCustomDictConfig() { for k, v := range c.d.Schema.GetCustomDictConfig() {
var schemaStr []string var schemaStr []string
@@ -100,7 +100,7 @@ func (c *Component) migrateDatabase() error {
if a.Default != "" { if a.Default != "" {
defaultValue = a.Default defaultValue = a.Default
} }
// this is only an attribute. We only need it in the schema // This is only an attribute. We only need it in the schema
schemaStr = append(schemaStr, fmt.Sprintf("`%s` %s DEFAULT '%s'", a.Name, a.Type, defaultValue)) schemaStr = append(schemaStr, fmt.Sprintf("`%s` %s DEFAULT '%s'", a.Name, a.Type, defaultValue))
} }
// we need to do this as function, otherwise we get problems with the for. // we need to do this as function, otherwise we get problems with the for.
@@ -111,7 +111,7 @@ func (c *Component) migrateDatabase() error {
}(k, v, schemaStr) }(k, v, schemaStr)
dictMigrations = append(dictMigrations, m) dictMigrations = append(dictMigrations, m)
} }
// create custom dictionaries // Create custom dictionaries
err = c.wrapMigrations(dictMigrations...) err = c.wrapMigrations(dictMigrations...)
if err != nil { if err != nil {
return err return err

16
orchestrator/clickhouse/migrations_test.go
View File

@@ -526,7 +526,7 @@ func TestCustomDictMigration(t *testing.T) {
t.Fatal("No migration applied when enabling a custom dictionary") t.Fatal("No migration applied when enabling a custom dictionary")
} }
// check if the rows were created in the main flows table // Check if the rows were created in the main flows table
row := ch.d.ClickHouse.QueryRow(context.Background(), ` row := ch.d.ClickHouse.QueryRow(context.Background(), `
SELECT toString(groupArray(tuple(name, type, default_expression))) SELECT toString(groupArray(tuple(name, type, default_expression)))
FROM system.columns FROM system.columns
@@ -542,14 +542,14 @@ func TestCustomDictMigration(t *testing.T) {
t.Fatalf("Unexpected state:\n%s", diff) t.Fatalf("Unexpected state:\n%s", diff)
} }
// check if the rows were created in the consumer flows table // Check if the rows were created in the consumer flows table
rowConsumer := ch.d.ClickHouse.QueryRow(context.Background(), ` rowConsumer := ch.d.ClickHouse.QueryRow(context.Background(), `
SHOW CREATE flows_ZUYGDTE3EBIXX352XPM3YEEFV4_raw_consumer`) SHOW CREATE flows_ZUYGDTE3EBIXX352XPM3YEEFV4_raw_consumer`)
var existingConsumer string var existingConsumer string
if err := rowConsumer.Scan(&existingConsumer); err != nil { if err := rowConsumer.Scan(&existingConsumer); err != nil {
t.Fatalf("Scan() error:\n%+v", err) t.Fatalf("Scan() error:\n%+v", err)
} }
// check if the definitions are part of the consumer // Check if the definitions are part of the consumer
expectedStatements := []string{ expectedStatements := []string{
"dictGet('default.custom_dict_test', 'csv_col_name', DstAddr) AS DstAddrDimensionAttribute", "dictGet('default.custom_dict_test', 'csv_col_name', DstAddr) AS DstAddrDimensionAttribute",
"dictGet('default.custom_dict_test', 'csv_col_name', SrcAddr) AS SrcAddrDimensionAttribute", "dictGet('default.custom_dict_test', 'csv_col_name', SrcAddr) AS SrcAddrDimensionAttribute",
@@ -562,7 +562,7 @@ func TestCustomDictMigration(t *testing.T) {
} }
} }
// check if the dictionary was created // Check if the dictionary was created
dictCreate := ch.d.ClickHouse.QueryRow(context.Background(), ` dictCreate := ch.d.ClickHouse.QueryRow(context.Background(), `
SHOW CREATE custom_dict_test`) SHOW CREATE custom_dict_test`)
var dictCreateString string var dictCreateString string
@@ -575,7 +575,7 @@ func TestCustomDictMigration(t *testing.T) {
} }
}) })
} }
// next test: with the custom dict removed again, the cols should still exist, but the consumer should be gone // Next test: with the custom dict removed again, the cols should still exist, but the consumer should be gone
if !t.Failed() { if !t.Failed() {
t.Run("remove custom dictionary", func(t *testing.T) { t.Run("remove custom dictionary", func(t *testing.T) {
r := reporter.NewMock(t) r := reporter.NewMock(t)
@@ -605,7 +605,7 @@ func TestCustomDictMigration(t *testing.T) {
t.Fatal("No migration applied when disabling the custom dict") t.Fatal("No migration applied when disabling the custom dict")
} }
// check if the rows were created in the main flows table // Check if the rows were created in the main flows table
row := ch.d.ClickHouse.QueryRow(context.Background(), ` row := ch.d.ClickHouse.QueryRow(context.Background(), `
SELECT toString(groupArray(tuple(name, type, default_expression))) SELECT toString(groupArray(tuple(name, type, default_expression)))
FROM system.columns FROM system.columns
@@ -621,14 +621,14 @@ func TestCustomDictMigration(t *testing.T) {
t.Fatalf("Unexpected state:\n%s", diff) t.Fatalf("Unexpected state:\n%s", diff)
} }
// check if the rows were removed in the consumer flows table // Check if the rows were removed in the consumer flows table
rowConsumer := ch.d.ClickHouse.QueryRow(context.Background(), ` rowConsumer := ch.d.ClickHouse.QueryRow(context.Background(), `
SHOW CREATE flows_ZUYGDTE3EBIXX352XPM3YEEFV4_raw_consumer`) SHOW CREATE flows_ZUYGDTE3EBIXX352XPM3YEEFV4_raw_consumer`)
var existingConsumer string var existingConsumer string
if err := rowConsumer.Scan(&existingConsumer); err != nil { if err := rowConsumer.Scan(&existingConsumer); err != nil {
t.Fatalf("Scan() error:\n%+v", err) t.Fatalf("Scan() error:\n%+v", err)
} }
// check if the definitions are missing in the consumer // Check if the definitions are missing in the consumer
expectedStatements := []string{ expectedStatements := []string{
"dictGet('default.custom_dict_test', 'csv_col_name', DstAddr) AS DstAddrDimensionAttribute", "dictGet('default.custom_dict_test', 'csv_col_name', DstAddr) AS DstAddrDimensionAttribute",
"dictGet('default.custom_dict_test', 'csv_col_name', SrcAddr) AS SrcAddrDimensionAttribute", "dictGet('default.custom_dict_test', 'csv_col_name', SrcAddr) AS SrcAddrDimensionAttribute",