diff --git a/.gitignore b/.gitignore index 0be8f329..12189f13 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ *.pb.go mock_*.go -/console/frontend/data/fields.json /console/data/frontend/ *~ diff --git a/Makefile b/Makefile index a53bfc9a..59d0759e 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,7 @@ export CGO_ENABLED=0 FLOW_VERSION := $(shell sed -n 's/^const CurrentSchemaVersion = //p' inlet/flow/schemas.go) GENERATED_JS = \ - console/frontend/node_modules \ - console/frontend/data/fields.json + console/frontend/node_modules GENERATED_GO = \ inlet/flow/decoder/flow-ANY.pb.go \ common/clickhousedb/mocks/mock_driver.go \ @@ -99,10 +98,6 @@ console/filter/parser.go: console/filter/parser.peg | $(PIGEON) ; $(info $(M) ge console/frontend/node_modules: console/frontend/package.json console/frontend/package-lock.json console/frontend/node_modules: ; $(info $(M) fetching node modules…) $Q (cd console/frontend ; npm ci --silent --no-audit --no-fund) && touch $@ -console/frontend/data/fields.json: console/query_consts.go ; $(info $(M) generate list of selectable fields…) - $Q sed -En -e 's/^\tqueryColumn([a-zA-Z0-9]+)( .*|$$)/ "\1"/p' $< \ - | sed -E -e '$$ ! s/$$/,/' -e '1s/^ */[/' -e '$$s/$$/]/' > $@ - $Q test -s $@ console/data/frontend: $(GENERATED_JS) console/data/frontend: $(shell $(LSFILES) console/frontend 2> /dev/null) console/data/frontend: ; $(info $(M) building console frontend…) diff --git a/common/schema/clickhouse.go b/common/schema/clickhouse.go index e015fbe2..2087e51d 100644 --- a/common/schema/clickhouse.go +++ b/common/schema/clickhouse.go @@ -36,6 +36,8 @@ const ( SkipAliasedColumns // SkipTimeReceived skips the time received column SkipTimeReceived + // SkipNotDimension skips columns that cannot be used as a dimension + SkipNotDimension // UseTransformFromType uses the type from TransformFrom if any UseTransformFromType // SubstituteGenerates changes the column name to use the default generated value @@ -80,6 +82,9 @@ func (schema Schema) iterate(fn func(column Column), options ...TableOption) { if slices.Contains(options, SkipAliasedColumns) && column.Alias != "" { continue } + if slices.Contains(options, SkipNotDimension) && column.NotSelectable { + continue + } if slices.Contains(options, UseTransformFromType) && column.TransformFrom != nil { for _, ocol := range column.TransformFrom { // We assume we only need to use name/type diff --git a/common/schema/flows.go b/common/schema/flows.go index c500420d..c142e991 100644 --- a/common/schema/flows.go +++ b/common/schema/flows.go @@ -28,11 +28,12 @@ var Flows = Schema{ }, Columns: buildMapFromColumns([]Column{ { - Name: "TimeReceived", - Type: "DateTime", - Codec: "DoubleDelta, LZ4", + Name: "TimeReceived", + Type: "DateTime", + Codec: "DoubleDelta, LZ4", + NotSelectable: true, }, - {Name: "SamplingRate", Type: "UInt64"}, + {Name: "SamplingRate", Type: "UInt64", NotSelectable: true}, {Name: "ExporterAddress", Type: "LowCardinality(IPv6)"}, {Name: "ExporterName", Type: "LowCardinality(String)", NotSortingKey: true}, {Name: "ExporterGroup", Type: "LowCardinality(String)", NotSortingKey: true}, @@ -45,9 +46,10 @@ var Flows = Schema{ Type: "IPv6", MainOnly: true, }, { - Name: "SrcNetMask", - Type: "UInt8", - MainOnly: true, + Name: "SrcNetMask", + Type: "UInt8", + MainOnly: true, + NotSelectable: true, }, { Name: "SrcNetPrefix", Type: "String", @@ -110,7 +112,8 @@ END`, {Name: "DstLargeCommunities.LocalData1", Type: "Array(UInt32)"}, {Name: "DstLargeCommunities.LocalData2", Type: "Array(UInt32)"}, }, - TransformTo: "arrayMap((asn, l1, l2) -> ((bitShiftLeft(CAST(asn, 'UInt128'), 64) + bitShiftLeft(CAST(l1, 'UInt128'), 32)) + CAST(l2, 'UInt128')), `DstLargeCommunities.ASN`, `DstLargeCommunities.LocalData1`, `DstLargeCommunities.LocalData2`)", + TransformTo: "arrayMap((asn, l1, l2) -> ((bitShiftLeft(CAST(asn, 'UInt128'), 64) + bitShiftLeft(CAST(l1, 'UInt128'), 32)) + CAST(l2, 'UInt128')), `DstLargeCommunities.ASN`, `DstLargeCommunities.LocalData1`, `DstLargeCommunities.LocalData2`)", + NotSelectable: true, }, {Name: "InIfName", Type: "LowCardinality(String)"}, {Name: "InIfDescription", Type: "String", NotSortingKey: true}, @@ -121,12 +124,13 @@ END`, {Name: "EType", Type: "UInt32"}, {Name: "Proto", Type: "UInt32"}, {Name: "SrcPort", Type: "UInt32", MainOnly: true}, - {Name: "Bytes", Type: "UInt64", NotSortingKey: true}, - {Name: "Packets", Type: "UInt64", NotSortingKey: true}, + {Name: "Bytes", Type: "UInt64", NotSortingKey: true, NotSelectable: true}, + {Name: "Packets", Type: "UInt64", NotSortingKey: true, NotSelectable: true}, { - Name: "PacketSize", - Type: "UInt64", - Alias: "intDiv(Bytes, Packets)", + Name: "PacketSize", + Type: "UInt64", + Alias: "intDiv(Bytes, Packets)", + NotSelectable: true, }, { Name: "PacketSizeBucket", Type: "LowCardinality(String)", diff --git a/common/schema/root.go b/common/schema/root.go index 6d505e73..d96f5610 100644 --- a/common/schema/root.go +++ b/common/schema/root.go @@ -36,4 +36,7 @@ type Column struct { GenerateFrom string TransformFrom []Column TransformTo string + + // For the console. + NotSelectable bool } diff --git a/console/config.go b/console/config.go index ae632c1d..d347bc83 100644 --- a/console/config.go +++ b/console/config.go @@ -7,6 +7,8 @@ import ( "net/http" "time" + "akvorado/common/schema" + "github.com/gin-gonic/gin" ) @@ -50,7 +52,7 @@ func DefaultConfiguration() Configuration { Start: "6 hours ago", End: "now", Filter: "InIfBoundary = external", - Dimensions: []queryColumn{queryColumnSrcAS}, + Dimensions: []queryColumn{"SrcAS"}, Limit: 10, }, HomepageTopWidgets: []string{"src-as", "src-port", "protocol", "src-country", "etype"}, @@ -65,5 +67,6 @@ func (c *Component) configHandlerFunc(gc *gin.Context) { "defaultVisualizeOptions": c.config.DefaultVisualizeOptions, "dimensionsLimit": c.config.DimensionsLimit, "homepageTopWidgets": c.config.HomepageTopWidgets, + "dimensions": schema.Flows.SelectColumns(schema.SkipNotDimension), }) } diff --git a/console/config_test.go b/console/config_test.go index 6c364677..6a48104e 100644 --- a/console/config_test.go +++ b/console/config_test.go @@ -28,8 +28,57 @@ func TestConfigHandler(t *testing.T) { "dimensions": []string{"SrcAS"}, "limit": 10, }, - "dimensionsLimit": 50, "homepageTopWidgets": []string{"src-as", "src-port", "protocol", "src-country", "etype"}, + "dimensionsLimit": 50, + "dimensions": []string{"ExporterAddress", + "ExporterName", + "ExporterGroup", + "ExporterRole", + "ExporterSite", + "ExporterRegion", + "ExporterTenant", + "SrcAddr", + "DstAddr", + "SrcNetPrefix", + "DstNetPrefix", + "SrcAS", + "DstAS", + "SrcNetName", + "DstNetName", + "SrcNetRole", + "DstNetRole", + "SrcNetSite", + "DstNetSite", + "SrcNetRegion", + "DstNetRegion", + "SrcNetTenant", + "DstNetTenant", + "SrcCountry", + "DstCountry", + "DstASPath", + "Dst1stAS", + "Dst2ndAS", + "Dst3rdAS", + "DstCommunities", + "InIfName", + "OutIfName", + "InIfDescription", + "OutIfDescription", + "InIfSpeed", + "OutIfSpeed", + "InIfConnectivity", + "OutIfConnectivity", + "InIfProvider", + "OutIfProvider", + "InIfBoundary", + "OutIfBoundary", + "EType", + "Proto", + "SrcPort", + "DstPort", + "PacketSizeBucket", + "ForwardingStatus", + }, }, }, }) diff --git a/console/filter/helpers.go b/console/filter/helpers.go index 4d933d6c..2d050b48 100644 --- a/console/filter/helpers.go +++ b/console/filter/helpers.go @@ -45,13 +45,12 @@ func ReverseColumnDirection(name string) string { // in predicate code blocks. func (c *current) acceptColumn() (string, error) { name := string(c.text) - for pair := schema.Flows.Columns.Front(); pair != nil; pair = pair.Next() { - column := pair.Value - if strings.EqualFold(name, column.Name) { + for _, columnName := range schema.Flows.Columns.Keys() { + if strings.EqualFold(name, columnName) { if c.globalStore["meta"].(*Meta).ReverseDirection { - return ReverseColumnDirection(column.Name), nil + return ReverseColumnDirection(columnName), nil } - return column.Name, nil + return columnName, nil } } return "", fmt.Errorf("unknown column %q", name) diff --git a/console/frontend/data/.placeholder b/console/frontend/data/.placeholder deleted file mode 100644 index e69de29b..00000000 diff --git a/console/frontend/src/components/InputDimensions.vue b/console/frontend/src/components/InputDimensions.vue index cc469759..5e13b920 100644 --- a/console/frontend/src/components/InputDimensions.vue +++ b/console/frontend/src/components/InputDimensions.vue @@ -58,7 +58,6 @@ import { dataColor } from "@/utils"; import InputString from "@/components/InputString.vue"; import InputListBox from "@/components/InputListBox.vue"; import { ServerConfigKey } from "@/components/ServerConfigProvider.vue"; -import fields from "@data/fields.json"; import { isEqual } from "lodash-es"; const props = withDefaults( @@ -75,7 +74,7 @@ const emit = defineEmits<{ }>(); const serverConfiguration = inject(ServerConfigKey)!; -const selectedDimensions = ref>([]); +const selectedDimensions = ref>([]); const dimensionsError = computed(() => { if (selectedDimensions.value.length < props.minDimensions) { return "At least two dimensions are required"; @@ -99,25 +98,27 @@ const limitError = computed(() => { }); const hasErrors = computed(() => !!limitError.value || !!dimensionsError.value); -const dimensions = fields.map((v, idx) => ({ - id: idx + 1, - name: v, - color: dataColor( - ["Exporter", "Src", "Dst", "In", "Out", ""] - .map((p) => v.startsWith(p)) - .indexOf(true) - ), -})); +const dimensions = computed(() => + serverConfiguration.value?.dimensions.map((v, idx) => ({ + id: idx + 1, + name: v, + color: dataColor( + ["Exporter", "Src", "Dst", "In", "Out", ""] + .map((p) => v.startsWith(p)) + .indexOf(true) + ), + })) +); -const removeDimension = (dimension: typeof dimensions[0]) => { +const removeDimension = (dimension: typeof dimensions.value[0]) => { selectedDimensions.value = selectedDimensions.value.filter( (d) => d !== dimension ); }; watch( - () => props.modelValue, - (value) => { + () => [props.modelValue, dimensions.value] as const, + ([value, dimensions]) => { if (value) { limit.value = value.limit.toString(); } diff --git a/console/frontend/src/components/ServerConfigProvider.vue b/console/frontend/src/components/ServerConfigProvider.vue index 0ed70363..e9425dee 100644 --- a/console/frontend/src/components/ServerConfigProvider.vue +++ b/console/frontend/src/components/ServerConfigProvider.vue @@ -30,6 +30,7 @@ type ServerConfig = { dimensions: string[]; limit: number; }; + dimensions: string[]; dimensionsLimit: number; homepageTopWidgets: string[]; }; diff --git a/console/frontend/tsconfig.app.json b/console/frontend/tsconfig.app.json index a0f18812..cdbea1d7 100644 --- a/console/frontend/tsconfig.app.json +++ b/console/frontend/tsconfig.app.json @@ -1,13 +1,12 @@ { "extends": "@vue/tsconfig/tsconfig.web.json", - "include": ["env.d.ts", "src/**/*", "src/**/*.vue", "data/*.json"], + "include": ["env.d.ts", "src/**/*", "src/**/*.vue"], "exclude": ["src/**/__tests__/*"], "compilerOptions": { "composite": true, "baseUrl": ".", "paths": { - "@/*": ["./src/*"], - "@data/*": ["./data/*"] + "@/*": ["./src/*"] } } } diff --git a/console/frontend/vite.config.ts b/console/frontend/vite.config.ts index 6fbcc9cb..ffc7ded3 100644 --- a/console/frontend/vite.config.ts +++ b/console/frontend/vite.config.ts @@ -12,7 +12,6 @@ export default defineConfig({ resolve: { alias: { "@": fileURLToPath(new URL("./src", import.meta.url)), - "@data": fileURLToPath(new URL("./data", import.meta.url)), }, }, build: { diff --git a/console/graph_test.go b/console/graph_test.go index df1bec55..756f1d14 100644 --- a/console/graph_test.go +++ b/console/graph_test.go @@ -21,8 +21,8 @@ func TestGraphInputReverseDirection(t *testing.T) { End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC), Points: 100, Dimensions: []queryColumn{ - queryColumnExporterName, - queryColumnInIfProvider, + "ExporterName", + "InIfProvider", }, Filter: queryFilter{ Filter: "DstCountry = 'FR' AND SrcCountry = 'US'", @@ -36,8 +36,8 @@ func TestGraphInputReverseDirection(t *testing.T) { End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC), Points: 100, Dimensions: []queryColumn{ - queryColumnExporterName, - queryColumnOutIfProvider, + "ExporterName", + "OutIfProvider", }, Filter: queryFilter{ Filter: "SrcCountry = 'FR' AND DstCountry = 'US'", @@ -120,8 +120,8 @@ func TestGraphPreviousPeriod(t *testing.T) { Start: start, End: end, Dimensions: []queryColumn{ - queryColumnExporterAddress, - queryColumnExporterName, + "ExporterAddress", + "ExporterName", }, } got := input.previousPeriod() @@ -328,8 +328,8 @@ ORDER BY time WITH FILL Points: 100, Limit: 20, Dimensions: []queryColumn{ - queryColumnExporterName, - queryColumnInIfProvider, + "ExporterName", + "InIfProvider", }, Filter: queryFilter{}, Units: "l3bps", @@ -360,8 +360,8 @@ ORDER BY time WITH FILL Points: 100, Limit: 20, Dimensions: []queryColumn{ - queryColumnExporterName, - queryColumnInIfProvider, + "ExporterName", + "InIfProvider", }, Filter: queryFilter{}, Units: "l3bps", @@ -409,8 +409,8 @@ ORDER BY time WITH FILL Points: 100, Limit: 20, Dimensions: []queryColumn{ - queryColumnExporterName, - queryColumnInIfProvider, + "ExporterName", + "InIfProvider", }, Filter: queryFilter{}, Units: "l3bps", diff --git a/console/query.go b/console/query.go index 2254f64c..0fe61353 100644 --- a/console/query.go +++ b/console/query.go @@ -9,50 +9,33 @@ import ( "strings" "akvorado/common/helpers" + "akvorado/common/schema" "akvorado/console/filter" ) -type queryColumn int +type queryColumn string func (qc queryColumn) MarshalText() ([]byte, error) { - got, ok := queryColumnMap.LoadValue(qc) - if ok { - return []byte(got), nil - } - return nil, errors.New("unknown field") + return []byte(qc), nil } func (qc queryColumn) String() string { - got, _ := queryColumnMap.LoadValue(qc) - return got + return string(qc) } func (qc *queryColumn) UnmarshalText(input []byte) error { - got, ok := queryColumnMap.LoadKey(string(input)) - if ok { - *qc = got + name := string(input) + if column, ok := schema.Flows.Columns.Get(name); ok && !column.NotSelectable { + *qc = queryColumn(name) return nil } return errors.New("unknown field") } -// queryColumnsRequiringMainTable lists query columns only present in -// the main table. Also check filter/parser.peg. -var queryColumnsRequiringMainTable = map[queryColumn]struct{}{ - queryColumnSrcAddr: {}, - queryColumnDstAddr: {}, - queryColumnSrcNetPrefix: {}, - queryColumnDstNetPrefix: {}, - queryColumnSrcPort: {}, - queryColumnDstPort: {}, - queryColumnDstASPath: {}, - queryColumnDstCommunities: {}, -} - func requireMainTable(qcs []queryColumn, qf queryFilter) bool { if qf.MainTableRequired { return true } for _, qc := range qcs { - if _, ok := queryColumnsRequiringMainTable[qc]; ok { + if column, ok := schema.Flows.Columns.Get(string(qc)); ok && column.MainOnly { return true } } @@ -98,21 +81,21 @@ func (qf *queryFilter) UnmarshalText(input []byte) error { func (qc queryColumn) toSQLSelect() string { var strValue string switch qc { - case queryColumnExporterAddress, queryColumnSrcAddr, queryColumnDstAddr: + case "ExporterAddress", "SrcAddr", "DstAddr": strValue = fmt.Sprintf("replaceRegexpOne(IPv6NumToString(%s), '^::ffff:', '')", qc) - case queryColumnSrcAS, queryColumnDstAS, queryColumnDst1stAS, queryColumnDst2ndAS, queryColumnDst3rdAS: + case "SrcAS", "DstAS", "Dst1stAS", "Dst2ndAS", "Dst3rdAS": strValue = fmt.Sprintf(`concat(toString(%s), ': ', dictGetOrDefault('asns', 'name', %s, '???'))`, qc, qc) - case queryColumnEType: + case "EType": strValue = fmt.Sprintf(`if(EType = %d, 'IPv4', if(EType = %d, 'IPv6', '???'))`, helpers.ETypeIPv4, helpers.ETypeIPv6) - case queryColumnProto: + case "Proto": strValue = `dictGetOrDefault('protocols', 'name', Proto, '???')` - case queryColumnInIfSpeed, queryColumnOutIfSpeed, queryColumnSrcPort, queryColumnDstPort, queryColumnForwardingStatus, queryColumnInIfBoundary, queryColumnOutIfBoundary: + case "InIfSpeed", "OutIfSpeed", "SrcPort", "DstPort", "ForwardingStatus", "InIfBoundary", "OutIfBoundary": strValue = fmt.Sprintf("toString(%s)", qc) - case queryColumnDstASPath: + case "DstASPath": strValue = `arrayStringConcat(DstASPath, ' ')` - case queryColumnDstCommunities: + case "DstCommunities": strValue = `arrayStringConcat(arrayConcat(arrayMap(c -> concat(toString(bitShiftRight(c, 16)), ':', toString(bitAnd(c, 0xffff))), DstCommunities), arrayMap(c -> concat(toString(bitAnd(bitShiftRight(c, 64), 0xffffffff)), ':', toString(bitAnd(bitShiftRight(c, 32), 0xffffffff)), ':', toString(bitAnd(c, 0xffffffff))), DstLargeCommunities)), ' ')` default: strValue = qc.String() @@ -122,19 +105,15 @@ func (qc queryColumn) toSQLSelect() string { // reverseDirection reverse the direction of a column (src/dst, in/out) func (qc queryColumn) reverseDirection() queryColumn { - value, ok := queryColumnMap.LoadKey(filter.ReverseColumnDirection(qc.String())) - if !ok { - panic("unknown reverse column") - } - return value + return queryColumn(filter.ReverseColumnDirection(string(qc))) } // fixQueryColumnName fix capitalization of the provided column name func fixQueryColumnName(name string) string { name = strings.ToLower(name) - for _, target := range queryColumnMap.Values() { - if strings.ToLower(target) == name { - return target + for _, k := range schema.Flows.Columns.Keys() { + if strings.ToLower(k) == name { + return k } } return "" diff --git a/console/query_consts.go b/console/query_consts.go deleted file mode 100644 index 662a80cd..00000000 --- a/console/query_consts.go +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -package console - -import "akvorado/common/helpers/bimap" - -const ( - queryColumnExporterAddress queryColumn = iota + 1 - queryColumnExporterName - queryColumnExporterGroup - queryColumnExporterRole - queryColumnExporterSite - queryColumnExporterRegion - queryColumnExporterTenant - queryColumnSrcAS - queryColumnSrcNetName - queryColumnSrcNetRole - queryColumnSrcNetSite - queryColumnSrcNetRegion - queryColumnSrcNetTenant - queryColumnSrcCountry - queryColumnInIfName - queryColumnInIfDescription - queryColumnInIfSpeed - queryColumnInIfConnectivity - queryColumnInIfProvider - queryColumnInIfBoundary - queryColumnEType - queryColumnProto - queryColumnSrcPort - queryColumnSrcAddr - queryColumnSrcNetPrefix - queryColumnDstAS - queryColumnDstASPath - queryColumnDst1stAS - queryColumnDst2ndAS - queryColumnDst3rdAS - queryColumnDstCommunities - queryColumnDstNetName - queryColumnDstNetRole - queryColumnDstNetSite - queryColumnDstNetRegion - queryColumnDstNetTenant - queryColumnDstCountry - queryColumnOutIfName - queryColumnOutIfDescription - queryColumnOutIfSpeed - queryColumnOutIfConnectivity - queryColumnOutIfProvider - queryColumnOutIfBoundary - queryColumnDstAddr - queryColumnDstNetPrefix - queryColumnDstPort - queryColumnForwardingStatus - queryColumnPacketSizeBucket -) - -var queryColumnMap = bimap.New(map[queryColumn]string{ - queryColumnExporterAddress: "ExporterAddress", - queryColumnExporterName: "ExporterName", - queryColumnExporterGroup: "ExporterGroup", - queryColumnExporterRole: "ExporterRole", - queryColumnExporterSite: "ExporterSite", - queryColumnExporterRegion: "ExporterRegion", - queryColumnExporterTenant: "ExporterTenant", - queryColumnSrcAddr: "SrcAddr", - queryColumnDstAddr: "DstAddr", - queryColumnSrcNetPrefix: "SrcNetPrefix", - queryColumnDstNetPrefix: "DstNetPrefix", - queryColumnSrcAS: "SrcAS", - queryColumnDstAS: "DstAS", - queryColumnDstASPath: "DstASPath", - queryColumnDst1stAS: "Dst1stAS", - queryColumnDst2ndAS: "Dst2ndAS", - queryColumnDst3rdAS: "Dst3rdAS", - queryColumnDstCommunities: "DstCommunities", - queryColumnSrcNetName: "SrcNetName", - queryColumnDstNetName: "DstNetName", - queryColumnSrcNetRole: "SrcNetRole", - queryColumnDstNetRole: "DstNetRole", - queryColumnSrcNetSite: "SrcNetSite", - queryColumnDstNetSite: "DstNetSite", - queryColumnSrcNetRegion: "SrcNetRegion", - queryColumnDstNetRegion: "DstNetRegion", - queryColumnSrcNetTenant: "SrcNetTenant", - queryColumnDstNetTenant: "DstNetTenant", - queryColumnSrcCountry: "SrcCountry", - queryColumnDstCountry: "DstCountry", - queryColumnInIfName: "InIfName", - queryColumnOutIfName: "OutIfName", - queryColumnInIfDescription: "InIfDescription", - queryColumnOutIfDescription: "OutIfDescription", - queryColumnInIfSpeed: "InIfSpeed", - queryColumnOutIfSpeed: "OutIfSpeed", - queryColumnInIfConnectivity: "InIfConnectivity", - queryColumnOutIfConnectivity: "OutIfConnectivity", - queryColumnInIfProvider: "InIfProvider", - queryColumnOutIfProvider: "OutIfProvider", - queryColumnInIfBoundary: "InIfBoundary", - queryColumnOutIfBoundary: "OutIfBoundary", - queryColumnEType: "EType", - queryColumnProto: "Proto", - queryColumnSrcPort: "SrcPort", - queryColumnDstPort: "DstPort", - queryColumnForwardingStatus: "ForwardingStatus", - queryColumnPacketSizeBucket: "PacketSizeBucket", -}) diff --git a/console/query_test.go b/console/query_test.go index efc8f7f1..4ecb2570 100644 --- a/console/query_test.go +++ b/console/query_test.go @@ -16,14 +16,14 @@ func TestRequireMainTable(t *testing.T) { Expected bool }{ {[]queryColumn{}, queryFilter{}, false}, - {[]queryColumn{queryColumnSrcAS}, queryFilter{}, false}, - {[]queryColumn{queryColumnExporterAddress}, queryFilter{}, false}, - {[]queryColumn{queryColumnSrcPort}, queryFilter{}, true}, - {[]queryColumn{queryColumnSrcAddr}, queryFilter{}, true}, - {[]queryColumn{queryColumnDstPort}, queryFilter{}, true}, - {[]queryColumn{queryColumnDstAddr}, queryFilter{}, true}, - {[]queryColumn{queryColumnSrcAS, queryColumnDstAddr}, queryFilter{}, true}, - {[]queryColumn{queryColumnDstAddr, queryColumnSrcAS}, queryFilter{}, true}, + {[]queryColumn{"SrcAS"}, queryFilter{}, false}, + {[]queryColumn{"ExporterAddress"}, queryFilter{}, false}, + {[]queryColumn{"SrcPort"}, queryFilter{}, true}, + {[]queryColumn{"SrcAddr"}, queryFilter{}, true}, + {[]queryColumn{"DstPort"}, queryFilter{}, true}, + {[]queryColumn{"DstAddr"}, queryFilter{}, true}, + {[]queryColumn{"SrcAS", "DstAddr"}, queryFilter{}, true}, + {[]queryColumn{"DstAddr", "SrcAS"}, queryFilter{}, true}, {[]queryColumn{}, queryFilter{MainTableRequired: true}, true}, } for idx, tc := range cases { @@ -34,40 +34,65 @@ func TestRequireMainTable(t *testing.T) { } } +func TestUnmarshalQueryColumn(t *testing.T) { + cases := []struct { + Input string + Expected string + Error bool + }{ + {"DstAddr", "DstAddr", false}, + {"TimeReceived", "", true}, + {"Nothing", "", true}, + } + for _, tc := range cases { + var qc queryColumn + err := qc.UnmarshalText([]byte(tc.Input)) + if err != nil && !tc.Error { + t.Fatalf("UnmarshalText(%q) error:\n%+v", tc.Input, err) + } + if err == nil && tc.Error { + t.Fatalf("UnmarshalText(%q) did not error", tc.Input) + } + if diff := helpers.Diff(qc, tc.Expected); diff != "" { + t.Fatalf("UnmarshalText(%q) (-got, +want):\n%s", tc.Input, diff) + } + } +} + func TestQueryColumnSQLSelect(t *testing.T) { cases := []struct { Input queryColumn Expected string }{ { - Input: queryColumnSrcAddr, + Input: "SrcAddr", Expected: `replaceRegexpOne(IPv6NumToString(SrcAddr), '^::ffff:', '')`, }, { - Input: queryColumnDstAS, + Input: "DstAS", Expected: `concat(toString(DstAS), ': ', dictGetOrDefault('asns', 'name', DstAS, '???'))`, }, { - Input: queryColumnDst2ndAS, + Input: "Dst2ndAS", Expected: `concat(toString(Dst2ndAS), ': ', dictGetOrDefault('asns', 'name', Dst2ndAS, '???'))`, }, { - Input: queryColumnProto, + Input: "Proto", Expected: `dictGetOrDefault('protocols', 'name', Proto, '???')`, }, { - Input: queryColumnEType, + Input: "EType", Expected: `if(EType = 2048, 'IPv4', if(EType = 34525, 'IPv6', '???'))`, }, { - Input: queryColumnOutIfSpeed, + Input: "OutIfSpeed", Expected: `toString(OutIfSpeed)`, }, { - Input: queryColumnExporterName, + Input: "ExporterName", Expected: `ExporterName`, }, { - Input: queryColumnPacketSizeBucket, + Input: "PacketSizeBucket", Expected: `PacketSizeBucket`, }, { - Input: queryColumnDstASPath, + Input: "DstASPath", Expected: `arrayStringConcat(DstASPath, ' ')`, }, { - Input: queryColumnDstCommunities, + Input: "DstCommunities", Expected: `arrayStringConcat(arrayConcat(arrayMap(c -> concat(toString(bitShiftRight(c, 16)), ':', toString(bitAnd(c, 0xffff))), DstCommunities), arrayMap(c -> concat(toString(bitAnd(bitShiftRight(c, 64), 0xffffffff)), ':', toString(bitAnd(bitShiftRight(c, 32), 0xffffffff)), ':', toString(bitAnd(c, 0xffffffff))), DstLargeCommunities)), ' ')`, }, } diff --git a/console/sankey_test.go b/console/sankey_test.go index 40b62475..2929de42 100644 --- a/console/sankey_test.go +++ b/console/sankey_test.go @@ -25,7 +25,7 @@ func TestSankeyQuerySQL(t *testing.T) { Input: sankeyHandlerInput{ Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC), End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC), - Dimensions: []queryColumn{queryColumnSrcAS, queryColumnExporterName}, + Dimensions: []queryColumn{"SrcAS", "ExporterName"}, Limit: 5, Filter: queryFilter{}, Units: "l3bps", @@ -49,7 +49,7 @@ ORDER BY xps DESC Input: sankeyHandlerInput{ Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC), End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC), - Dimensions: []queryColumn{queryColumnSrcAS, queryColumnExporterName}, + Dimensions: []queryColumn{"SrcAS", "ExporterName"}, Limit: 5, Filter: queryFilter{}, Units: "l2bps", @@ -74,7 +74,7 @@ ORDER BY xps DESC Input: sankeyHandlerInput{ Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC), End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC), - Dimensions: []queryColumn{queryColumnSrcAS, queryColumnExporterName}, + Dimensions: []queryColumn{"SrcAS", "ExporterName"}, Limit: 5, Filter: queryFilter{}, Units: "pps", @@ -98,7 +98,7 @@ ORDER BY xps DESC Input: sankeyHandlerInput{ Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC), End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC), - Dimensions: []queryColumn{queryColumnSrcAS, queryColumnExporterName}, + Dimensions: []queryColumn{"SrcAS", "ExporterName"}, Limit: 10, Filter: queryFilter{Filter: "DstCountry = 'FR'"}, Units: "l3bps",