Files
akvorado/console/graph_test.go
Vincent Bernat 67703cc61e console: use templates to build SQL query
This is needed if we want to be able to mix use of several tables
inside a single query (for example, flows_1m0s for a part of the query
and flows_5m0s for another part to overlay historical data).

Also, the way we handle time buckets is now cleaner. The previous way
had two stages of rounding and was incorrect. We were discarding the
first and last value for this reason. The new way only has one stage
of rounding and is correct. It tries hard to align the buckets at the
specified start time. We don't need to discard these values anymore.
We still discard the last one because it could be incomplete (when end
is "now").
2022-08-09 11:45:40 +02:00

594 lines
17 KiB
Go

// SPDX-FileCopyrightText: 2022 Free Mobile
// SPDX-License-Identifier: AGPL-3.0-only
package console
import (
"fmt"
"strings"
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/golang/mock/gomock"
"akvorado/common/helpers"
)
func TestGraphInputReverseDirection(t *testing.T) {
input := graphHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Dimensions: []queryColumn{
queryColumnExporterName,
queryColumnInIfProvider,
},
Filter: queryFilter{
Filter: "DstCountry = 'FR' AND SrcCountry = 'US'",
ReverseFilter: "SrcCountry = 'FR' AND DstCountry = 'US'",
},
Units: "l3bps",
}
original1 := fmt.Sprintf("%+v", input)
expected := graphHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Dimensions: []queryColumn{
queryColumnExporterName,
queryColumnOutIfProvider,
},
Filter: queryFilter{
Filter: "SrcCountry = 'FR' AND DstCountry = 'US'",
ReverseFilter: "DstCountry = 'FR' AND SrcCountry = 'US'",
},
Units: "l3bps",
}
got := input.reverseDirection()
original2 := fmt.Sprintf("%+v", input)
if diff := helpers.Diff(got, expected); diff != "" {
t.Fatalf("reverseDirection() (-got, +want):\n%s", diff)
}
if original1 != original2 {
t.Fatalf("reverseDirection() modified original to:\n-%s\n+%s", original1, original2)
}
}
func TestGraphQuerySQL(t *testing.T) {
cases := []struct {
Description string
Input graphHandlerInput
Expected string
}{
{
Description: "no dimensions, no filters, bps",
Input: graphHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Dimensions: []queryColumn{},
Filter: queryFilter{},
Units: "l3bps",
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
SELECT 1 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
{{ .Units }}/{{ .Interval }} AS xps,
emptyArrayString() AS dimensions
FROM {{ .Table }}
WHERE {{ .Timefilter }}
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }}
STEP {{ .Interval }})
{{ end }}`,
}, {
Description: "no dimensions, no filters, l2 bps",
Input: graphHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Dimensions: []queryColumn{},
Filter: queryFilter{},
Units: "l2bps",
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l2bps"}@@ }}
SELECT 1 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
{{ .Units }}/{{ .Interval }} AS xps,
emptyArrayString() AS dimensions
FROM {{ .Table }}
WHERE {{ .Timefilter }}
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }}
STEP {{ .Interval }})
{{ end }}
`,
}, {
Description: "no dimensions, no filters, pps",
Input: graphHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Dimensions: []queryColumn{},
Filter: queryFilter{},
Units: "pps",
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"pps"}@@ }}
SELECT 1 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
{{ .Units }}/{{ .Interval }} AS xps,
emptyArrayString() AS dimensions
FROM {{ .Table }}
WHERE {{ .Timefilter }}
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }}
STEP {{ .Interval }})
{{ end }}`,
}, {
Description: "no dimensions",
Input: graphHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Dimensions: []queryColumn{},
Filter: queryFilter{Filter: "DstCountry = 'FR' AND SrcCountry = 'US'"},
Units: "l3bps",
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
SELECT 1 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
{{ .Units }}/{{ .Interval }} AS xps,
emptyArrayString() AS dimensions
FROM {{ .Table }}
WHERE {{ .Timefilter }} AND (DstCountry = 'FR' AND SrcCountry = 'US')
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }}
STEP {{ .Interval }})
{{ end }}`,
}, {
Description: "no dimensions, escaped filter",
Input: graphHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Dimensions: []queryColumn{},
Filter: queryFilter{Filter: "InIfDescription = '{{ hello }}' AND SrcCountry = 'US'"},
Units: "l3bps",
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
SELECT 1 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
{{ .Units }}/{{ .Interval }} AS xps,
emptyArrayString() AS dimensions
FROM {{ .Table }}
WHERE {{ .Timefilter }} AND (InIfDescription = '{{"{{"}} hello }}' AND SrcCountry = 'US')
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }}
STEP {{ .Interval }})
{{ end }}`,
}, {
Description: "no dimensions, reverse direction",
Input: graphHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Dimensions: []queryColumn{},
Filter: queryFilter{
Filter: "DstCountry = 'FR' AND SrcCountry = 'US'",
ReverseFilter: "SrcCountry = 'FR' AND DstCountry = 'US'",
},
Units: "l3bps",
Bidirectional: true,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
SELECT 1 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
{{ .Units }}/{{ .Interval }} AS xps,
emptyArrayString() AS dimensions
FROM {{ .Table }}
WHERE {{ .Timefilter }} AND (DstCountry = 'FR' AND SrcCountry = 'US')
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }}
STEP {{ .Interval }})
{{ end }}
UNION ALL
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
SELECT 2 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
{{ .Units }}/{{ .Interval }} AS xps,
emptyArrayString() AS dimensions
FROM {{ .Table }}
WHERE {{ .Timefilter }} AND (SrcCountry = 'FR' AND DstCountry = 'US')
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }}
STEP {{ .Interval }})
{{ end }}`,
}, {
Description: "no filters",
Input: graphHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Limit: 20,
Dimensions: []queryColumn{
queryColumnExporterName,
queryColumnInIfProvider,
},
Filter: queryFilter{},
Units: "l3bps",
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
WITH
rows AS (SELECT ExporterName, InIfProvider FROM {{ .Table }} WHERE {{ .Timefilter }} GROUP BY ExporterName, InIfProvider ORDER BY SUM(Bytes) DESC LIMIT 20)
SELECT 1 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
{{ .Units }}/{{ .Interval }} AS xps,
if((ExporterName, InIfProvider) IN rows, [ExporterName, InIfProvider], ['Other', 'Other']) AS dimensions
FROM {{ .Table }}
WHERE {{ .Timefilter }}
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }}
STEP {{ .Interval }})
{{ end }}`,
}, {
Description: "no filters, reverse",
Input: graphHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Limit: 20,
Dimensions: []queryColumn{
queryColumnExporterName,
queryColumnInIfProvider,
},
Filter: queryFilter{},
Units: "l3bps",
Bidirectional: true,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
WITH
rows AS (SELECT ExporterName, InIfProvider FROM {{ .Table }} WHERE {{ .Timefilter }} GROUP BY ExporterName, InIfProvider ORDER BY SUM(Bytes) DESC LIMIT 20)
SELECT 1 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
{{ .Units }}/{{ .Interval }} AS xps,
if((ExporterName, InIfProvider) IN rows, [ExporterName, InIfProvider], ['Other', 'Other']) AS dimensions
FROM {{ .Table }}
WHERE {{ .Timefilter }}
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }}
STEP {{ .Interval }})
{{ end }}
UNION ALL
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
SELECT 2 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
{{ .Units }}/{{ .Interval }} AS xps,
if((ExporterName, OutIfProvider) IN rows, [ExporterName, OutIfProvider], ['Other', 'Other']) AS dimensions
FROM {{ .Table }}
WHERE {{ .Timefilter }}
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }}
STEP {{ .Interval }})
{{ end }}`,
},
}
for _, tc := range cases {
tc.Expected = strings.ReplaceAll(tc.Expected, "@@", "`")
t.Run(tc.Description, func(t *testing.T) {
got := tc.Input.toSQL()
if diff := helpers.Diff(strings.Split(strings.TrimSpace(got), "\n"),
strings.Split(strings.TrimSpace(tc.Expected), "\n")); diff != "" {
t.Errorf("toSQL (-got, +want):\n%s", diff)
}
})
}
}
func TestGraphHandler(t *testing.T) {
_, h, mockConn, _ := NewMock(t, DefaultConfiguration())
base := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
// Single direction
expectedSQL := []struct {
Axis uint8 `ch:"axis"`
Time time.Time `ch:"time"`
Xps float64 `ch:"xps"`
Dimensions []string `ch:"dimensions"`
}{
{1, base, 1000, []string{"router1", "provider1"}},
{1, base, 2000, []string{"router1", "provider2"}},
{1, base, 1200, []string{"router2", "provider2"}},
{1, base, 1100, []string{"router2", "provider3"}},
{1, base, 1900, []string{"Other", "Other"}},
{1, base.Add(time.Minute), 500, []string{"router1", "provider1"}},
{1, base.Add(time.Minute), 5000, []string{"router1", "provider2"}},
{1, base.Add(time.Minute), 900, []string{"router2", "provider4"}},
{1, base.Add(time.Minute), 100, []string{"Other", "Other"}},
{1, base.Add(2 * time.Minute), 100, []string{"router1", "provider1"}},
{1, base.Add(2 * time.Minute), 3000, []string{"router1", "provider2"}},
{1, base.Add(2 * time.Minute), 100, []string{"router2", "provider4"}},
{1, base.Add(2 * time.Minute), 100, []string{"Other", "Other"}},
}
mockConn.EXPECT().
Select(gomock.Any(), gomock.Any(), gomock.Any()).
SetArg(1, expectedSQL).
Return(nil)
// Bidirectional
expectedSQL = []struct {
Axis uint8 `ch:"axis"`
Time time.Time `ch:"time"`
Xps float64 `ch:"xps"`
Dimensions []string `ch:"dimensions"`
}{
{1, base, 1000, []string{"router1", "provider1"}},
{1, base, 2000, []string{"router1", "provider2"}},
{1, base, 1200, []string{"router2", "provider2"}},
{1, base, 1100, []string{"router2", "provider3"}},
{1, base, 1900, []string{"Other", "Other"}},
{1, base.Add(time.Minute), 500, []string{"router1", "provider1"}},
{1, base.Add(time.Minute), 5000, []string{"router1", "provider2"}},
{1, base.Add(time.Minute), 900, []string{"router2", "provider4"}},
// Axes can be mixed. In reality, it seems they cannot
// be interleaved, but ClickHouse documentation does
// not say it is not possible.
{2, base, 100, []string{"router1", "provider1"}},
{2, base, 200, []string{"router1", "provider2"}},
{2, base, 120, []string{"router2", "provider2"}},
{1, base.Add(time.Minute), 100, []string{"Other", "Other"}},
{1, base.Add(2 * time.Minute), 100, []string{"router1", "provider1"}},
{2, base, 110, []string{"router2", "provider3"}},
{2, base, 190, []string{"Other", "Other"}},
{2, base.Add(time.Minute), 50, []string{"router1", "provider1"}},
{2, base.Add(time.Minute), 500, []string{"router1", "provider2"}},
{1, base.Add(2 * time.Minute), 3000, []string{"router1", "provider2"}},
{1, base.Add(2 * time.Minute), 100, []string{"router2", "provider4"}},
{1, base.Add(2 * time.Minute), 100, []string{"Other", "Other"}},
{2, base.Add(time.Minute), 90, []string{"router2", "provider4"}},
{2, base.Add(time.Minute), 10, []string{"Other", "Other"}},
{2, base.Add(2 * time.Minute), 10, []string{"router1", "provider1"}},
{2, base.Add(2 * time.Minute), 300, []string{"router1", "provider2"}},
{2, base.Add(2 * time.Minute), 10, []string{"router2", "provider4"}},
{2, base.Add(2 * time.Minute), 10, []string{"Other", "Other"}},
}
mockConn.EXPECT().
Select(gomock.Any(), gomock.Any(), gomock.Any()).
SetArg(1, expectedSQL).
Return(nil)
helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
{
Description: "single direction",
URL: "/api/v0/console/graph",
JSONInput: gin.H{
"start": time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
"end": time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
"points": 100,
"limit": 20,
"dimensions": []string{"ExporterName", "InIfProvider"},
"filter": "DstCountry = 'FR' AND SrcCountry = 'US'",
"units": "l3bps",
"bidirectional": false,
},
JSONOutput: gin.H{
// Sorted by sum of bps
"rows": [][]string{
{"router1", "provider2"}, // 10000
{"router1", "provider1"}, // 1600
{"router2", "provider2"}, // 1200
{"router2", "provider3"}, // 1100
{"router2", "provider4"}, // 1000
{"Other", "Other"}, // 2100
},
"t": []string{
"2009-11-10T23:00:00Z",
"2009-11-10T23:01:00Z",
"2009-11-10T23:02:00Z",
},
"points": [][]int{
{2000, 5000, 3000},
{1000, 500, 100},
{1200, 0, 0},
{1100, 0, 0},
{0, 900, 100},
{1900, 100, 100},
},
"min": []int{
2000,
100,
1200,
1100,
100,
100,
},
"max": []int{
5000,
1000,
1200,
1100,
900,
1900,
},
"average": []int{
3333,
533,
400,
366,
333,
700,
},
"95th": []int{
4000,
750,
600,
550,
500,
1000,
},
"axis": []int{
1, 1, 1, 1, 1, 1,
},
},
}, {
Description: "bidirectional",
URL: "/api/v0/console/graph",
JSONInput: gin.H{
"start": time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
"end": time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
"points": 100,
"limit": 20,
"dimensions": []string{"ExporterName", "InIfProvider"},
"filter": "DstCountry = 'FR' AND SrcCountry = 'US'",
"units": "l3bps",
"bidirectional": true,
},
JSONOutput: gin.H{
// Sorted by sum of bps
"rows": [][]string{
{"router1", "provider2"}, // 10000
{"router1", "provider1"}, // 1600
{"router2", "provider2"}, // 1200
{"router2", "provider3"}, // 1100
{"router2", "provider4"}, // 1000
{"Other", "Other"}, // 2100
{"router1", "provider2"}, // 1000
{"router1", "provider1"}, // 160
{"router2", "provider2"}, // 120
{"router2", "provider3"}, // 110
{"router2", "provider4"}, // 100
{"Other", "Other"}, // 210
},
"t": []string{
"2009-11-10T23:00:00Z",
"2009-11-10T23:01:00Z",
"2009-11-10T23:02:00Z",
},
"points": [][]int{
{2000, 5000, 3000},
{1000, 500, 100},
{1200, 0, 0},
{1100, 0, 0},
{0, 900, 100},
{1900, 100, 100},
{200, 500, 300},
{100, 50, 10},
{120, 0, 0},
{110, 0, 0},
{0, 90, 10},
{190, 10, 10},
},
"min": []int{
2000,
100,
1200,
1100,
100,
100,
200,
10,
120,
110,
10,
10,
},
"max": []int{
5000,
1000,
1200,
1100,
900,
1900,
500,
100,
120,
110,
90,
190,
},
"average": []int{
3333,
533,
400,
366,
333,
700,
333,
53,
40,
36,
33,
70,
},
"95th": []int{
4000,
750,
600,
550,
500,
1000,
400,
75,
60,
55,
50,
100,
},
"axis": []int{
1, 1, 1, 1, 1, 1,
2, 2, 2, 2, 2, 2,
},
},
},
})
}