johanels/akvorado
1
0
Fork 0
mirror of https://github.com/akvorado/akvorado.git synced 2025-12-11 22:14:02 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

console: simplify SQL request by computing target resolution in Go

Browse Source
This commit is contained in:
Vincent Bernat
2022-07-26 23:51:32 +02:00
parent 0fb5107bf9
commit fad0c5db16
6 changed files with 48 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
console/clickhouse.go
View File

@@ -12,7 +12,10 @@ import (
"time"
)
var addressOrPortRegexp = regexp.MustCompile(`\b(?:Src|Dst)(?:Port|Addr)\b`)
var (
addressOrPortRegexp = regexp.MustCompile(`\b(?:Src|Dst)(?:Port|Addr)\b`)
resolutionRegexp = regexp.MustCompile(`{resolution->(\d+)}`)
)
// flowsTable describe a consolidated or unconsolidated flows table.
type flowsTable struct {
@@ -95,6 +98,18 @@ func (c *Component) queryFlowsTable(query string, start, end time.Time, targetRe
query = strings.ReplaceAll(query, "{timefilter.Stop}", timeFilterStop)
query = strings.ReplaceAll(query, "{table}", table)
query = strings.ReplaceAll(query, "{resolution}", strconv.Itoa(int(resolution.Seconds())))
query = resolutionRegexp.ReplaceAllStringFunc(query, func(in string) string {
matches := resolutionRegexp.FindStringSubmatch(in)
target, err := strconv.Atoi(matches[1])
if err != nil {
panic(err)
}
target = target / int(resolution.Seconds()) * int(resolution.Seconds())
if target < 1 {
target = 1
}
return strconv.Itoa(target)
})
return query
}

8
console/clickhouse_test.go
View File

@@ -146,11 +146,11 @@ func TestQueryFlowsTables(t *testing.T) {
{"flows", 0, time.Date(2022, 03, 10, 16, 45, 10, 0, time.UTC)},
{"flows_1m0s", time.Minute, time.Date(2022, 03, 10, 17, 45, 10, 0, time.UTC)},
},
Query: "SELECT 1 FROM {table} WHERE {timefilter} // {resolution}",
Query: "SELECT 1 FROM {table} WHERE {timefilter} // {resolution} // {resolution->864}",
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Resolution: 30 * time.Second,
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC') // 1",
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC') // 1 // 864",
}, {
Description: "select consolidated table better resolution",
Tables: []flowsTable{
@@ -158,11 +158,11 @@ func TestQueryFlowsTables(t *testing.T) {
{"flows_5m0s", 5 * time.Minute, time.Date(2022, 04, 2, 22, 45, 10, 0, time.UTC)},
{"flows_1m0s", time.Minute, time.Date(2022, 04, 2, 22, 45, 10, 0, time.UTC)},
},
Query: "SELECT 1 FROM {table} WHERE {timefilter} // {resolution}",
Query: "SELECT 1 FROM {table} WHERE {timefilter} // {resolution} // {resolution->864}",
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Resolution: 2 * time.Minute,
Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:00', 'UTC') AND toDateTime('2022-04-11 15:45:00', 'UTC') // 60",
Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:00', 'UTC') AND toDateTime('2022-04-11 15:45:00', 'UTC') // 60 // 840",
}, {
Description: "select consolidated table better range",
Tables: []flowsTable{

4
console/graph.go
View File

@@ -40,7 +40,7 @@ type graphHandlerOutput struct {
// graphHandlerInputToSQL converts a graph input to an SQL request
func (input graphHandlerInput) toSQL() (string, error) {
interval := int64((input.End.Sub(input.Start).Seconds())) / int64(input.Points)
slot := fmt.Sprintf(`max2(intDiv(%d, {resolution})*{resolution}, 1)`, interval)
slot := fmt.Sprintf(`{resolution->%d}`, interval)
// Filter
where := input.Filter.filter
@@ -105,7 +105,7 @@ GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM toStartOfInterval({timefilter.Start}, INTERVAL %s second)
TO {timefilter.Stop}
STEP toUInt32(%s)`, withStr, strings.Join(fields, ",\n "), where, slot, slot)
STEP %s`, withStr, strings.Join(fields, ",\n "), where, slot, slot)
return sqlQuery, nil
}

40
console/graph_test.go
View File

@@ -32,16 +32,16 @@ func TestGraphQuerySQL(t *testing.T) {
},
Expected: `
SELECT
toStartOfInterval(TimeReceived, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second) AS time,
SUM(Bytes*SamplingRate*8/max2(intDiv(864, {resolution})*{resolution}, 1)) AS xps,
toStartOfInterval(TimeReceived, INTERVAL {resolution->864} second) AS time,
SUM(Bytes*SamplingRate*8/{resolution->864}) AS xps,
emptyArrayString() AS dimensions
FROM {table}
WHERE {timefilter}
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM toStartOfInterval({timefilter.Start}, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second)
FROM toStartOfInterval({timefilter.Start}, INTERVAL {resolution->864} second)
TO {timefilter.Stop}
STEP toUInt32(max2(intDiv(864, {resolution})*{resolution}, 1))`,
STEP {resolution->864}`,
}, {
Description: "no dimensions, no filters, l2 bps",
Input: graphHandlerInput{
@@ -54,16 +54,16 @@ ORDER BY time WITH FILL
},
Expected: `
SELECT
toStartOfInterval(TimeReceived, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second) AS time,
SUM((Bytes+18*Packets)*SamplingRate*8/max2(intDiv(864, {resolution})*{resolution}, 1)) AS xps,
toStartOfInterval(TimeReceived, INTERVAL {resolution->864} second) AS time,
SUM((Bytes+18*Packets)*SamplingRate*8/{resolution->864}) AS xps,
emptyArrayString() AS dimensions
FROM {table}
WHERE {timefilter}
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM toStartOfInterval({timefilter.Start}, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second)
FROM toStartOfInterval({timefilter.Start}, INTERVAL {resolution->864} second)
TO {timefilter.Stop}
STEP toUInt32(max2(intDiv(864, {resolution})*{resolution}, 1))`,
STEP {resolution->864}`,
}, {
Description: "no dimensions, no filters, pps",
Input: graphHandlerInput{
@@ -76,16 +76,16 @@ ORDER BY time WITH FILL
},
Expected: `
SELECT
toStartOfInterval(TimeReceived, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second) AS time,
SUM(Packets*SamplingRate/max2(intDiv(864, {resolution})*{resolution}, 1)) AS xps,
toStartOfInterval(TimeReceived, INTERVAL {resolution->864} second) AS time,
SUM(Packets*SamplingRate/{resolution->864}) AS xps,
emptyArrayString() AS dimensions
FROM {table}
WHERE {timefilter}
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM toStartOfInterval({timefilter.Start}, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second)
FROM toStartOfInterval({timefilter.Start}, INTERVAL {resolution->864} second)
TO {timefilter.Stop}
STEP toUInt32(max2(intDiv(864, {resolution})*{resolution}, 1))`,
STEP {resolution->864}`,
}, {
Description: "no dimensions",
Input: graphHandlerInput{
@@ -98,16 +98,16 @@ ORDER BY time WITH FILL
},
Expected: `
SELECT
toStartOfInterval(TimeReceived, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second) AS time,
SUM(Bytes*SamplingRate*8/max2(intDiv(864, {resolution})*{resolution}, 1)) AS xps,
toStartOfInterval(TimeReceived, INTERVAL {resolution->864} second) AS time,
SUM(Bytes*SamplingRate*8/{resolution->864}) AS xps,
emptyArrayString() AS dimensions
FROM {table}
WHERE {timefilter} AND (DstCountry = 'FR' AND SrcCountry = 'US')
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM toStartOfInterval({timefilter.Start}, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second)
FROM toStartOfInterval({timefilter.Start}, INTERVAL {resolution->864} second)
TO {timefilter.Stop}
STEP toUInt32(max2(intDiv(864, {resolution})*{resolution}, 1))`,
STEP {resolution->864}`,
}, {
Description: "no filters",
Input: graphHandlerInput{
@@ -126,16 +126,16 @@ ORDER BY time WITH FILL
WITH
rows AS (SELECT ExporterName, InIfProvider FROM {table} WHERE {timefilter} GROUP BY ExporterName, InIfProvider ORDER BY SUM(Bytes) DESC LIMIT 20)
SELECT
toStartOfInterval(TimeReceived, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second) AS time,
SUM(Bytes*SamplingRate*8/max2(intDiv(864, {resolution})*{resolution}, 1)) AS xps,
toStartOfInterval(TimeReceived, INTERVAL {resolution->864} second) AS time,
SUM(Bytes*SamplingRate*8/{resolution->864}) AS xps,
if((ExporterName, InIfProvider) IN rows, [ExporterName, InIfProvider], ['Other', 'Other']) AS dimensions
FROM {table}
WHERE {timefilter}
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM toStartOfInterval({timefilter.Start}, INTERVAL max2(intDiv(864, {resolution})*{resolution}, 1) second)
FROM toStartOfInterval({timefilter.Start}, INTERVAL {resolution->864} second)
TO {timefilter.Stop}
STEP toUInt32(max2(intDiv(864, {resolution})*{resolution}, 1))`,
STEP {resolution->864}`,
},
}
for _, tc := range cases {

4
console/widgets.go
View File

@@ -189,7 +189,7 @@ func (c *Component) widgetGraphHandlerFunc(gc *gin.Context) {
params.Points = 200
}
interval := int64((24 * time.Hour).Seconds()) / int64(params.Points)
slot := fmt.Sprintf(`max2(intDiv(%d, {resolution})*{resolution}, 1)`, interval)
slot := fmt.Sprintf(`{resolution->%d}`, interval)
now := c.d.Clock.Now()
query := c.queryFlowsTable(fmt.Sprintf(`
SELECT
@@ -202,7 +202,7 @@ GROUP BY Time
ORDER BY Time WITH FILL
FROM toStartOfInterval({timefilter.Start}, INTERVAL %s second)
TO {timefilter.Stop}
STEP toUInt32(%s)`, slot, slot, slot, slot), now.Add(-24*time.Hour), now, time.Duration(interval)*time.Second)
STEP %s`, slot, slot, slot, slot), now.Add(-24*time.Hour), now, time.Duration(interval)*time.Second)
gc.Header("X-SQL-Query", query)
results := []struct {

8
console/widgets_test.go
View File

@@ -231,16 +231,16 @@ func TestWidgetGraph(t *testing.T) {
mockConn.EXPECT().
Select(gomock.Any(), gomock.Any(), `
SELECT
toStartOfInterval(TimeReceived, INTERVAL max2(intDiv(864, 1)*1, 1) second) AS Time,
SUM(Bytes*SamplingRate*8/max2(intDiv(864, 1)*1, 1))/1000/1000/1000 AS Gbps
toStartOfInterval(TimeReceived, INTERVAL 864 second) AS Time,
SUM(Bytes*SamplingRate*8/864)/1000/1000/1000 AS Gbps
FROM flows
WHERE TimeReceived BETWEEN toDateTime('2009-11-10 23:00:00', 'UTC') AND toDateTime('2009-11-11 23:00:00', 'UTC')
AND InIfBoundary = 'external'
GROUP BY Time
ORDER BY Time WITH FILL
FROM toStartOfInterval(toDateTime('2009-11-10 23:00:00', 'UTC'), INTERVAL max2(intDiv(864, 1)*1, 1) second)
FROM toStartOfInterval(toDateTime('2009-11-10 23:00:00', 'UTC'), INTERVAL 864 second)
TO toDateTime('2009-11-11 23:00:00', 'UTC')
STEP toUInt32(max2(intDiv(864, 1)*1, 1))`).
STEP 864`).
SetArg(1, expected).
Return(nil)