johanels/akvorado
1
0
Fork 0
mirror of https://github.com/akvorado/akvorado.git synced 2025-12-12 06:24:10 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

orchestrator/clickhouse: optimize dictionary lookup for networks

Browse Source
This commit is contained in:
Vincent Bernat
2024-03-13 20:32:20 +01:00
parent 345aa6c09b
commit dcdbf208d1
3 changed files with 81 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
common/schema/definition.go
View File

@@ -246,111 +246,111 @@ END`,
{
Key: ColumnSrcAS,
ClickHouseType: "UInt32",
ClickHouseGenerateFrom: fmt.Sprintf("if(SrcAS = 0, dictGetOrDefault('%s', 'asn', SrcAddr, 0), SrcAS)", DictionaryNetworks),
ClickHouseGenerateFrom: "if(SrcAS = 0, c_SrcNetworks[asn], SrcAS)",
ClickHouseSelfGenerated: true,
},
{
Key: ColumnDstAS,
ClickHouseType: "UInt32",
ClickHouseGenerateFrom: fmt.Sprintf("if(DstAS = 0, dictGetOrDefault('%s', 'asn', DstAddr, 0), DstAS)", DictionaryNetworks),
ClickHouseGenerateFrom: "if(DstAS = 0, c_DstNetworks[asn], DstAS)",
ClickHouseSelfGenerated: true,
},
{
Key: ColumnSrcNetName,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'name', SrcAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_SrcNetworks[name]",
},
{
Key: ColumnDstNetName,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'name', DstAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_DstNetworks[name]",
},
{
Key: ColumnSrcNetRole,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'role', SrcAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_SrcNetworks[role]",
},
{
Key: ColumnDstNetRole,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'role', DstAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_DstNetworks[role]",
},
{
Key: ColumnSrcNetSite,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'site', SrcAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_SrcNetworks[site]",
},
{
Key: ColumnDstNetSite,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'site', DstAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_DstNetworks[site]",
},
{
Key: ColumnSrcNetRegion,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'region', SrcAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_SrcNetworks[region]",
},
{
Key: ColumnDstNetRegion,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'region', DstAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_DstNetworks[region]",
},
{
Key: ColumnSrcNetTenant,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'tenant', SrcAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_SrcNetworks[tenant]",
},
{
Key: ColumnDstNetTenant,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'tenant', DstAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_DstNetworks[tenant]",
},
{Key: ColumnSrcVlan, ParserType: "uint", ClickHouseType: "UInt16", Disabled: true, Group: ColumnGroupL2},
{
Key: ColumnSrcCountry,
ParserType: "string",
ClickHouseType: "FixedString(2)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'country', SrcAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_SrcNetworks[country]",
},
{
Key: ColumnDstCountry,
ParserType: "string",
ClickHouseType: "FixedString(2)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'country', DstAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_DstNetworks[country]",
},
{
Key: ColumnSrcGeoCity,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'city', SrcAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_SrcNetworks[city]",
},
{
Key: ColumnDstGeoCity,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'city', DstAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_DstNetworks[city]",
},
{
Key: ColumnSrcGeoState,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'state', SrcAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_SrcNetworks[state]",
},
{
Key: ColumnDstGeoState,
ParserType: "string",
ClickHouseType: "LowCardinality(String)",
ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'state', DstAddr, '')", DictionaryNetworks),
ClickHouseGenerateFrom: "c_DstNetworks[state]",
},
{
Key: ColumnDstASPath,