johanels/akvorado
1
0
Fork 0
mirror of https://github.com/akvorado/akvorado.git synced 2025-12-12 06:24:10 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

console: simplify SQL request by computing target resolution in Go

Browse Source
This commit is contained in:
Vincent Bernat
2022-07-26 23:51:32 +02:00
parent 0fb5107bf9
commit fad0c5db16
6 changed files with 48 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
console/clickhouse.go
View File

@@ -12,7 +12,10 @@ import (
"time" "time"
) )
var addressOrPortRegexp = regexp.MustCompile(`\b(?:Src|Dst)(?:Port|Addr)\b`) var (
addressOrPortRegexp = regexp.MustCompile(`\b(?:Src|Dst)(?:Port|Addr)\b`)
resolutionRegexp = regexp.MustCompile(`{resolution->(\d+)}`)
)
// flowsTable describe a consolidated or unconsolidated flows table. // flowsTable describe a consolidated or unconsolidated flows table.
type flowsTable struct { type flowsTable struct {
@@ -95,6 +98,18 @@ func (c *Component) queryFlowsTable(query string, start, end time.Time, targetRe
query = strings.ReplaceAll(query, "{timefilter.Stop}", timeFilterStop) query = strings.ReplaceAll(query, "{timefilter.Stop}", timeFilterStop)
query = strings.ReplaceAll(query, "{table}", table) query = strings.ReplaceAll(query, "{table}", table)
query = strings.ReplaceAll(query, "{resolution}", strconv.Itoa(int(resolution.Seconds()))) query = strings.ReplaceAll(query, "{resolution}", strconv.Itoa(int(resolution.Seconds())))
query = resolutionRegexp.ReplaceAllStringFunc(query, func(in string) string {
matches := resolutionRegexp.FindStringSubmatch(in)
target, err := strconv.Atoi(matches[1])
if err != nil {
panic(err)
}
target = target / int(resolution.Seconds()) * int(resolution.Seconds())
if target < 1 {
target = 1
}
return strconv.Itoa(target)
})
return query return query
} }

8
console/clickhouse_test.go
View File

@@ -146,11 +146,11 @@ func TestQueryFlowsTables(t *testing.T) {
{"flows", 0, time.Date(2022, 03, 10, 16, 45, 10, 0, time.UTC)}, {"flows", 0, time.Date(2022, 03, 10, 16, 45, 10, 0, time.UTC)},
{"flows_1m0s", time.Minute, time.Date(2022, 03, 10, 17, 45, 10, 0, time.UTC)}, {"flows_1m0s", time.Minute, time.Date(2022, 03, 10, 17, 45, 10, 0, time.UTC)},
}, },
Query: "SELECT 1 FROM {table} WHERE {timefilter} // {resolution}", Query: "SELECT 1 FROM {table} WHERE {timefilter} // {resolution} // {resolution->864}",
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC), Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC), End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Resolution: 30 * time.Second, Resolution: 30 * time.Second,
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC') // 1", Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC') // 1 // 864",
}, { }, {
Description: "select consolidated table better resolution", Description: "select consolidated table better resolution",
Tables: []flowsTable{ Tables: []flowsTable{
@@ -158,11 +158,11 @@ func TestQueryFlowsTables(t *testing.T) {
{"flows_5m0s", 5 * time.Minute, time.Date(2022, 04, 2, 22, 45, 10, 0, time.UTC)}, {"flows_5m0s", 5 * time.Minute, time.Date(2022, 04, 2, 22, 45, 10, 0, time.UTC)},
{"flows_1m0s", time.Minute, time.Date(2022, 04, 2, 22, 45, 10, 0, time.UTC)}, {"flows_1m0s", time.Minute, time.Date(2022, 04, 2, 22, 45, 10, 0, time.UTC)},
}, },
Query: "SELECT 1 FROM {table} WHERE {timefilter} // {resolution}", Query: "SELECT 1 FROM {table} WHERE {timefilter} // {resolution} // {resolution->864}",
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC), Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC), End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Resolution: 2 * time.Minute, Resolution: 2 * time.Minute,
Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:00', 'UTC') AND toDateTime('2022-04-11 15:45:00', 'UTC') // 60", Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:00', 'UTC') AND toDateTime('2022-04-11 15:45:00', 'UTC') // 60 // 840",
}, { }, {
Description: "select consolidated table better range", Description: "select consolidated table better range",
Tables: []flowsTable{ Tables: []flowsTable{

4
console/graph.go
View File

@@ -40,7 +40,7 @@ type graphHandlerOutput struct {
// graphHandlerInputToSQL converts a graph input to an SQL request // graphHandlerInputToSQL converts a graph input to an SQL request
func (input graphHandlerInput) toSQL() (string, error) { func (input graphHandlerInput) toSQL() (string, error) {
interval := int64((input.End.Sub(input.Start).Seconds())) / int64(input.Points) interval := int64((input.End.Sub(input.Start).Seconds())) / int64(input.Points)
slot := fmt.Sprintf(`max2(intDiv(%d, {resolution})*{resolution}, 1)`, interval) slot := fmt.Sprintf(`{resolution->%d}`, interval)
// Filter // Filter
where := input.Filter.filter where := input.Filter.filter
@@ -105,7 +105,7 @@ GROUP BY time, dimensions
ORDER BY time WITH FILL ORDER BY time WITH FILL
FROM toStartOfInterval({timefilter.Start}, INTERVAL %s second) FROM toStartOfInterval({timefilter.Start}, INTERVAL %s second)
TO {timefilter.Stop} TO {timefilter.Stop}
STEP toUInt32(%s)`, withStr, strings.Join(fields, ",\n "), where, slot, slot) STEP %s`, withStr, strings.Join(fields, ",\n "), where, slot, slot)
return sqlQuery, nil return sqlQuery, nil
} }

40
console/graph_test.go
View File

@@ -32,16 +32,16 @@ func TestGraphQuerySQL(t *testing.T) {
}, },
Expected: ` Expected: `
SELECT SELECT
toStartOfInterval(TimeReceived, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second) AS time, toStartOfInterval(TimeReceived, INTERVAL {resolution->864} second) AS time,
SUM(Bytes*SamplingRate*8/max2(intDiv(864, {resolution})*{resolution}, 1)) AS xps, SUM(Bytes*SamplingRate*8/{resolution->864}) AS xps,
emptyArrayString() AS dimensions emptyArrayString() AS dimensions
FROM {table} FROM {table}
WHERE {timefilter} WHERE {timefilter}
GROUP BY time, dimensions GROUP BY time, dimensions
ORDER BY time WITH FILL ORDER BY time WITH FILL
FROM toStartOfInterval({timefilter.Start}, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second) FROM toStartOfInterval({timefilter.Start}, INTERVAL {resolution->864} second)
TO {timefilter.Stop} TO {timefilter.Stop}
STEP toUInt32(max2(intDiv(864, {resolution})*{resolution}, 1))`, STEP {resolution->864}`,
}, { }, {
Description: "no dimensions, no filters, l2 bps", Description: "no dimensions, no filters, l2 bps",
Input: graphHandlerInput{ Input: graphHandlerInput{
@@ -54,16 +54,16 @@ ORDER BY time WITH FILL
}, },
Expected: ` Expected: `
SELECT SELECT
toStartOfInterval(TimeReceived, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second) AS time, toStartOfInterval(TimeReceived, INTERVAL {resolution->864} second) AS time,
SUM((Bytes+18*Packets)*SamplingRate*8/max2(intDiv(864, {resolution})*{resolution}, 1)) AS xps, SUM((Bytes+18*Packets)*SamplingRate*8/{resolution->864}) AS xps,
emptyArrayString() AS dimensions emptyArrayString() AS dimensions
FROM {table} FROM {table}
WHERE {timefilter} WHERE {timefilter}
GROUP BY time, dimensions GROUP BY time, dimensions
ORDER BY time WITH FILL ORDER BY time WITH FILL
FROM toStartOfInterval({timefilter.Start}, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second) FROM toStartOfInterval({timefilter.Start}, INTERVAL {resolution->864} second)
TO {timefilter.Stop} TO {timefilter.Stop}
STEP toUInt32(max2(intDiv(864, {resolution})*{resolution}, 1))`, STEP {resolution->864}`,
}, { }, {
Description: "no dimensions, no filters, pps", Description: "no dimensions, no filters, pps",
Input: graphHandlerInput{ Input: graphHandlerInput{
@@ -76,16 +76,16 @@ ORDER BY time WITH FILL
}, },
Expected: ` Expected: `
SELECT SELECT
toStartOfInterval(TimeReceived, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second) AS time, toStartOfInterval(TimeReceived, INTERVAL {resolution->864} second) AS time,
SUM(Packets*SamplingRate/max2(intDiv(864, {resolution})*{resolution}, 1)) AS xps, SUM(Packets*SamplingRate/{resolution->864}) AS xps,
emptyArrayString() AS dimensions emptyArrayString() AS dimensions
FROM {table} FROM {table}
WHERE {timefilter} WHERE {timefilter}
GROUP BY time, dimensions GROUP BY time, dimensions
ORDER BY time WITH FILL ORDER BY time WITH FILL
FROM toStartOfInterval({timefilter.Start}, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second) FROM toStartOfInterval({timefilter.Start}, INTERVAL {resolution->864} second)
TO {timefilter.Stop} TO {timefilter.Stop}
STEP toUInt32(max2(intDiv(864, {resolution})*{resolution}, 1))`, STEP {resolution->864}`,
}, { }, {
Description: "no dimensions", Description: "no dimensions",
Input: graphHandlerInput{ Input: graphHandlerInput{
@@ -98,16 +98,16 @@ ORDER BY time WITH FILL
}, },
Expected: ` Expected: `
SELECT SELECT
toStartOfInterval(TimeReceived, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second) AS time, toStartOfInterval(TimeReceived, INTERVAL {resolution->864} second) AS time,
SUM(Bytes*SamplingRate*8/max2(intDiv(864, {resolution})*{resolution}, 1)) AS xps, SUM(Bytes*SamplingRate*8/{resolution->864}) AS xps,
emptyArrayString() AS dimensions emptyArrayString() AS dimensions
FROM {table} FROM {table}
WHERE {timefilter} AND (DstCountry = 'FR' AND SrcCountry = 'US') WHERE {timefilter} AND (DstCountry = 'FR' AND SrcCountry = 'US')
GROUP BY time, dimensions GROUP BY time, dimensions
ORDER BY time WITH FILL ORDER BY time WITH FILL
FROM toStartOfInterval({timefilter.Start}, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second) FROM toStartOfInterval({timefilter.Start}, INTERVAL {resolution->864} second)
TO {timefilter.Stop} TO {timefilter.Stop}
STEP toUInt32(max2(intDiv(864, {resolution})*{resolution}, 1))`, STEP {resolution->864}`,
}, { }, {
Description: "no filters", Description: "no filters",
Input: graphHandlerInput{ Input: graphHandlerInput{
@@ -126,16 +126,16 @@ ORDER BY time WITH FILL
WITH WITH
rows AS (SELECT ExporterName, InIfProvider FROM {table} WHERE {timefilter} GROUP BY ExporterName, InIfProvider ORDER BY SUM(Bytes) DESC LIMIT 20) rows AS (SELECT ExporterName, InIfProvider FROM {table} WHERE {timefilter} GROUP BY ExporterName, InIfProvider ORDER BY SUM(Bytes) DESC LIMIT 20)
SELECT SELECT
toStartOfInterval(TimeReceived, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second) AS time, toStartOfInterval(TimeReceived, INTERVAL {resolution->864} second) AS time,
SUM(Bytes*SamplingRate*8/max2(intDiv(864, {resolution})*{resolution}, 1)) AS xps, SUM(Bytes*SamplingRate*8/{resolution->864}) AS xps,
if((ExporterName, InIfProvider) IN rows, [ExporterName, InIfProvider], ['Other', 'Other']) AS dimensions if((ExporterName, InIfProvider) IN rows, [ExporterName, InIfProvider], ['Other', 'Other']) AS dimensions
FROM {table} FROM {table}
WHERE {timefilter} WHERE {timefilter}
GROUP BY time, dimensions GROUP BY time, dimensions
ORDER BY time WITH FILL ORDER BY time WITH FILL
FROM toStartOfInterval({timefilter.Start}, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second) FROM toStartOfInterval({timefilter.Start}, INTERVAL {resolution->864} second)
TO {timefilter.Stop} TO {timefilter.Stop}
STEP toUInt32(max2(intDiv(864, {resolution})*{resolution}, 1))`, STEP {resolution->864}`,
}, },
} }
for _, tc := range cases { for _, tc := range cases {

4
console/widgets.go
View File

@@ -189,7 +189,7 @@ func (c *Component) widgetGraphHandlerFunc(gc *gin.Context) {
params.Points = 200 params.Points = 200
} }
interval := int64((24 * time.Hour).Seconds()) / int64(params.Points) interval := int64((24 * time.Hour).Seconds()) / int64(params.Points)
slot := fmt.Sprintf(`max2(intDiv(%d, {resolution})*{resolution}, 1)`, interval) slot := fmt.Sprintf(`{resolution->%d}`, interval)
now := c.d.Clock.Now() now := c.d.Clock.Now()
query := c.queryFlowsTable(fmt.Sprintf(` query := c.queryFlowsTable(fmt.Sprintf(`
SELECT SELECT
@@ -202,7 +202,7 @@ GROUP BY Time
ORDER BY Time WITH FILL ORDER BY Time WITH FILL
FROM toStartOfInterval({timefilter.Start}, INTERVAL %s second) FROM toStartOfInterval({timefilter.Start}, INTERVAL %s second)
TO {timefilter.Stop} TO {timefilter.Stop}
STEP toUInt32(%s)`, slot, slot, slot, slot), now.Add(-24*time.Hour), now, time.Duration(interval)*time.Second) STEP %s`, slot, slot, slot, slot), now.Add(-24*time.Hour), now, time.Duration(interval)*time.Second)
gc.Header("X-SQL-Query", query) gc.Header("X-SQL-Query", query)
results := []struct { results := []struct {

8
console/widgets_test.go
View File

@@ -231,16 +231,16 @@ func TestWidgetGraph(t *testing.T) {
mockConn.EXPECT(). mockConn.EXPECT().
Select(gomock.Any(), gomock.Any(), ` Select(gomock.Any(), gomock.Any(), `
SELECT SELECT
toStartOfInterval(TimeReceived, INTERVAL max2(intDiv(864, 1)*1, 1) second) AS Time, toStartOfInterval(TimeReceived, INTERVAL 864 second) AS Time,
SUM(Bytes*SamplingRate*8/max2(intDiv(864, 1)*1, 1))/1000/1000/1000 AS Gbps SUM(Bytes*SamplingRate*8/864)/1000/1000/1000 AS Gbps
FROM flows FROM flows
WHERE TimeReceived BETWEEN toDateTime('2009-11-10 23:00:00', 'UTC') AND toDateTime('2009-11-11 23:00:00', 'UTC') WHERE TimeReceived BETWEEN toDateTime('2009-11-10 23:00:00', 'UTC') AND toDateTime('2009-11-11 23:00:00', 'UTC')
AND InIfBoundary = 'external' AND InIfBoundary = 'external'
GROUP BY Time GROUP BY Time
ORDER BY Time WITH FILL ORDER BY Time WITH FILL
FROM toStartOfInterval(toDateTime('2009-11-10 23:00:00', 'UTC'), INTERVAL max2(intDiv(864, 1)*1, 1) second) FROM toStartOfInterval(toDateTime('2009-11-10 23:00:00', 'UTC'), INTERVAL 864 second)
TO toDateTime('2009-11-11 23:00:00', 'UTC') TO toDateTime('2009-11-11 23:00:00', 'UTC')
STEP toUInt32(max2(intDiv(864, 1)*1, 1))`). STEP 864`).
SetArg(1, expected). SetArg(1, expected).
Return(nil) Return(nil)