johanels/akvorado
1
0
Fork 0
mirror of https://github.com/akvorado/akvorado.git synced 2025-12-12 06:24:10 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

console: use templates to build SQL query

Browse Source 
This is needed if we want to be able to mix use of several tables
inside a single query (for example, flows_1m0s for a part of the query
and flows_5m0s for another part to overlay historical data).

Also, the way we handle time buckets is now cleaner. The previous way
had two stages of rounding and was incorrect. We were discarding the
first and last value for this reason. The new way only has one stage
of rounding and is correct. It tries hard to align the buckets at the
specified start time. We don't need to discard these values anymore.
We still discard the last one because it could be incomplete (when end
is "now").
This commit is contained in:
Vincent Bernat
2022-08-09 11:40:51 +02:00
parent 61b3baa406
commit 67703cc61e
11 changed files with 535 additions and 344 deletions
Download Patch File Download Diff File Expand all files Collapse all files

186
console/clickhouse_test.go
View File

@@ -4,6 +4,7 @@
package console
import (
"fmt"
"testing"
"time"
@@ -70,113 +71,143 @@ AND engine LIKE '%MergeTree'
}
}
func TestQueryFlowsTables(t *testing.T) {
func TestFinalizeQuery(t *testing.T) {
cases := []struct {
Description string
Tables []flowsTable
Query string
MainTableRequired bool
Start time.Time
End time.Time
Resolution time.Duration
Expected string
Description string
Tables []flowsTable
Query string
Context inputContext
Expected string
}{
{
Description: "query with source port",
Query: "SELECT TimeReceived, SrcPort FROM {table} WHERE {timefilter}",
MainTableRequired: true,
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Expected: "SELECT TimeReceived, SrcPort FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')",
Description: "simple query without additional tables",
Query: "SELECT 1 FROM {{ .Table }} WHERE {{ .Timefilter }}",
Context: inputContext{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 86400,
},
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')",
}, {
Description: "query with source port",
Query: "SELECT TimeReceived, SrcPort FROM {{ .Table }} WHERE {{ .Timefilter }}",
Context: inputContext{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
MainTableRequired: true,
Points: 86400,
},
Expected: "SELECT TimeReceived, SrcPort FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')",
}, {
Description: "only flows table available",
Tables: []flowsTable{{"flows", 0, time.Date(2022, 03, 10, 15, 45, 10, 0, time.UTC)}},
Query: "SELECT 1 FROM {table} WHERE {timefilter}",
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')",
Query: "SELECT 1 FROM {{ .Table }} WHERE {{ .Timefilter }}",
Context: inputContext{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 86400,
},
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')",
}, {
Description: "timefilter.Start and timefilter.Stop",
Tables: []flowsTable{{"flows", 0, time.Date(2022, 03, 10, 15, 45, 10, 0, time.UTC)}},
Query: "SELECT {timefilter.Start}, {timefilter.Stop}",
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Expected: "SELECT toDateTime('2022-04-10 15:45:10', 'UTC'), toDateTime('2022-04-11 15:45:10', 'UTC')",
Query: "SELECT {{ .TimefilterStart }}, {{ .TimefilterEnd }}",
Context: inputContext{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 86400,
},
Expected: "SELECT toDateTime('2022-04-10 15:45:10', 'UTC'), toDateTime('2022-04-11 15:45:10', 'UTC')",
}, {
Description: "only flows table and out of range request",
Tables: []flowsTable{{"flows", 0, time.Date(2022, 04, 10, 22, 45, 10, 0, time.UTC)}},
Query: "SELECT 1 FROM {table} WHERE {timefilter}",
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')",
Query: "SELECT 1 FROM {{ .Table }} WHERE {{ .Timefilter }}",
Context: inputContext{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 86400,
},
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')",
}, {
Description: "select consolidated table",
Tables: []flowsTable{
{"flows", 0, time.Date(2022, 03, 10, 22, 45, 10, 0, time.UTC)},
{"flows_1m0s", time.Minute, time.Date(2022, 04, 2, 22, 45, 10, 0, time.UTC)},
},
Query: "SELECT 1 FROM {table} WHERE {timefilter}",
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Resolution: 2 * time.Minute,
Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:00', 'UTC') AND toDateTime('2022-04-11 15:45:00', 'UTC')",
Query: "SELECT 1 FROM {{ .Table }} WHERE {{ .Timefilter }} // {{ .Interval }}",
Context: inputContext{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 720, // 2-minute resolution
},
Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:00', 'UTC') AND toDateTime('2022-04-11 15:45:00', 'UTC') // 120",
}, {
Description: "select consolidated table out of range",
Tables: []flowsTable{
{"flows", 0, time.Date(2022, 04, 10, 22, 45, 10, 0, time.UTC)},
{"flows_1m0s", time.Minute, time.Date(2022, 04, 10, 17, 45, 10, 0, time.UTC)},
},
Query: "SELECT 1 FROM {table} WHERE {timefilter}",
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Resolution: 2 * time.Minute,
Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:00', 'UTC') AND toDateTime('2022-04-11 15:45:00', 'UTC')",
Query: "SELECT 1 FROM {{ .Table }} WHERE {{ .Timefilter }}",
Context: inputContext{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 720, // 2-minute resolution,
},
Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:00', 'UTC') AND toDateTime('2022-04-11 15:45:00', 'UTC')",
}, {
Description: "select flows table out of range",
Tables: []flowsTable{
{"flows", 0, time.Date(2022, 04, 10, 16, 45, 10, 0, time.UTC)},
{"flows_1m0s", time.Minute, time.Date(2022, 04, 10, 17, 45, 10, 0, time.UTC)},
},
Query: "SELECT 1 FROM {table} WHERE {timefilter}",
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Resolution: 2 * time.Minute,
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')",
Query: "SELECT 1 FROM {{ .Table }} WHERE {{ .Timefilter }}",
Context: inputContext{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 720, // 2-minute resolution,
},
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')",
}, {
Description: "select flows table better resolution",
Description: "select flows table with better resolution",
Tables: []flowsTable{
{"flows", 0, time.Date(2022, 03, 10, 16, 45, 10, 0, time.UTC)},
{"flows_1m0s", time.Minute, time.Date(2022, 03, 10, 17, 45, 10, 0, time.UTC)},
},
Query: "SELECT 1 FROM {table} WHERE {timefilter} // {resolution} // {resolution->864}",
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Resolution: 30 * time.Second,
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC') // 1 // 864",
Query: "SELECT 1 FROM {{ .Table }} WHERE {{ .Timefilter }} // {{ .Interval }}",
Context: inputContext{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 2880,
},
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC') // 30",
}, {
Description: "select consolidated table better resolution",
Description: "select consolidated table with better resolution",
Tables: []flowsTable{
{"flows", 0, time.Date(2022, 03, 10, 22, 45, 10, 0, time.UTC)},
{"flows_5m0s", 5 * time.Minute, time.Date(2022, 04, 2, 22, 45, 10, 0, time.UTC)},
{"flows_1m0s", time.Minute, time.Date(2022, 04, 2, 22, 45, 10, 0, time.UTC)},
},
Query: "SELECT 1 FROM {table} WHERE {timefilter} // {resolution} // {resolution->864}",
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Resolution: 2 * time.Minute,
Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:00', 'UTC') AND toDateTime('2022-04-11 15:45:00', 'UTC') // 60 // 840",
Query: "SELECT 1 FROM {{ .Table }} WHERE {{ .Timefilter }} // {{ .Interval }}",
Context: inputContext{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 720, // 2-minute resolution,
},
Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:00', 'UTC') AND toDateTime('2022-04-11 15:45:00', 'UTC') // 120",
}, {
Description: "select consolidated table better range",
Description: "select consolidated table with better range",
Tables: []flowsTable{
{"flows", 0, time.Date(2022, 04, 10, 22, 45, 10, 0, time.UTC)},
{"flows_5m0s", 5 * time.Minute, time.Date(2022, 04, 2, 22, 45, 10, 0, time.UTC)},
{"flows_1m0s", time.Minute, time.Date(2022, 04, 10, 22, 45, 10, 0, time.UTC)},
},
Query: "SELECT 1 FROM {table} WHERE {timefilter}",
Start: time.Date(2022, 04, 10, 15, 46, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 46, 10, 0, time.UTC),
Resolution: 2 * time.Minute,
Expected: "SELECT 1 FROM flows_5m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:00', 'UTC') AND toDateTime('2022-04-11 15:45:00', 'UTC')",
Query: "SELECT 1 FROM {{ .Table }} WHERE {{ .Timefilter }}",
Context: inputContext{
Start: time.Date(2022, 04, 10, 15, 46, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 46, 10, 0, time.UTC),
Points: 720, // 2-minute resolution,
},
Expected: "SELECT 1 FROM flows_5m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:00', 'UTC') AND toDateTime('2022-04-11 15:45:00', 'UTC')",
}, {
Description: "select best resolution when equality for oldest data",
Tables: []flowsTable{
@@ -184,11 +215,31 @@ func TestQueryFlowsTables(t *testing.T) {
{"flows_1m0s", time.Minute, time.Date(2022, 04, 10, 22, 40, 00, 0, time.UTC)},
{"flows_1h0m0s", time.Hour, time.Date(2022, 04, 10, 22, 00, 10, 0, time.UTC)},
},
Query: "SELECT 1 FROM {table} WHERE {timefilter}",
Start: time.Date(2022, 04, 10, 15, 46, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 46, 10, 0, time.UTC),
Resolution: 2 * time.Minute,
Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:46:00', 'UTC') AND toDateTime('2022-04-11 15:46:00', 'UTC')",
Query: "SELECT 1 FROM {{ .Table }} WHERE {{ .Timefilter }}",
Context: inputContext{
Start: time.Date(2022, 04, 10, 15, 46, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 46, 10, 0, time.UTC),
Points: 720, // 2-minute resolution,
},
Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:46:00', 'UTC') AND toDateTime('2022-04-11 15:46:00', 'UTC')",
}, {
Description: "query with escaped template",
Query: `SELECT TimeReceived, SrcPort WHERE InIfDescription = '{{"{{"}} hello }}'`,
Context: inputContext{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 86400,
},
Expected: `SELECT TimeReceived, SrcPort WHERE InIfDescription = '{{ hello }}'`,
}, {
Description: "use of ToStartOfInterval",
Query: `{{ call .ToStartOfInterval "TimeReceived" }}`,
Context: inputContext{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 720,
},
Expected: `toStartOfInterval(TimeReceived + INTERVAL 50 second, INTERVAL 120 second) - INTERVAL 50 second`,
},
}
@@ -196,9 +247,10 @@ func TestQueryFlowsTables(t *testing.T) {
for _, tc := range cases {
t.Run(tc.Description, func(t *testing.T) {
c.flowsTables = tc.Tables
got := c.queryFlowsTable(tc.Query, tc.MainTableRequired, tc.Start, tc.End, tc.Resolution)
got := c.finalizeQuery(
fmt.Sprintf(`{{ with %s }}%s{{ end }}`, templateContext(tc.Context), tc.Query))
if diff := helpers.Diff(got, tc.Expected); diff != "" {
t.Fatalf("queryFlowsTable(): (-got, +want):\n%s", diff)
t.Fatalf("finalizeQuery(): (-got, +want):\n%s", diff)
}
})
}