johanels/akvorado
1
0
Fork 0
mirror of https://github.com/akvorado/akvorado.git synced 2025-12-11 22:14:02 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

console: use a less hacky way to pass context to build SQL templates

Browse Source 
Instead of trying to embed that in the template, provide a list of
templates with their associated input contexts and join them with UNION
ALL.
This commit is contained in:
Vincent Bernat
2025-09-04 07:26:16 +02:00
parent d3448222fc
commit 74146e428d
7 changed files with 382 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
console/clickhouse.go
View File

@@ -5,7 +5,6 @@ package console
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"sort"
@@ -82,33 +81,17 @@ AND (engine LIKE '%MergeTree' OR engine = 'Distributed')
return nil
}
// finalizeQuery builds the finalized query. A single "context"
// function is provided to return a `Context` struct with all the
// information needed.
func (c *Component) finalizeQuery(query string) string {
t := template.Must(template.New("query").
Funcs(template.FuncMap{
"context": c.contextFunc,
}).
Option("missingkey=error").
Parse(strings.TrimSpace(query)))
buf := bytes.NewBufferString("")
if err := t.Execute(buf, nil); err != nil {
c.r.Err(err).Str("query", query).Msg("invalid query")
panic(err)
}
return buf.String()
}
// inputContext is the intermeidate context provided by the input handler.
type inputContext struct {
Start time.Time `json:"start"`
End time.Time `json:"end"`
StartForInterval *time.Time `json:"start-for-interval,omitempty"`
MainTableRequired bool `json:"main-table-required,omitempty"`
Points uint `json:"points"`
Units string `json:"units,omitempty"`
Start time.Time
End time.Time
StartForTableSelection *time.Time
MainTableRequired bool
Points uint
Units string
}
// context is the context to finalize the template.
type context struct {
Table string
Timefilter string
@@ -119,6 +102,12 @@ type context struct {
ToStartOfInterval func(string) string
}
// templateQuery holds a template string and its associated input context.
type templateQuery struct {
Template string
Context inputContext
}
// templateEscape escapes `{{` and `}}` from a string. In fact, only
// the opening tag needs to be escaped.
func templateEscape(input string) string {
@@ -133,22 +122,20 @@ func templateWhere(qf query.Filter) string {
return fmt.Sprintf(`{{ .Timefilter }} AND (%s)`, templateEscape(qf.Direct()))
}
// templateTable builds a template directive to select the right table
func templateContext(context inputContext) string {
encoded, err := json.Marshal(context)
if err != nil {
panic(err)
// finalizeTemplateQueries builds the finalized queries from a list of templateQuery.
// Each template is processed with its associated context and combined with UNION ALL.
func (c *Component) finalizeTemplateQueries(queries []templateQuery) string {
parts := make([]string, len(queries))
for i, q := range queries {
parts[i] = c.finalizeTemplateQuery(q)
}
return fmt.Sprintf("context `%s`", string(encoded))
return strings.Join(parts, "\nUNION ALL\n")
}
func (c *Component) contextFunc(inputStr string) context {
var input inputContext
if err := json.Unmarshal([]byte(inputStr), &input); err != nil {
panic(err)
}
table, computedInterval, targetInterval := c.computeTableAndInterval(input)
// finalizeTemplateQuery builds the finalized query for a single templateQuery
func (c *Component) finalizeTemplateQuery(query templateQuery) string {
input := query.Context
table, computedInterval, targetInterval := c.computeTableAndInterval(query.Context)
// Make start/end match the computed interval (currently equal to the table resolution)
start := input.Start.Truncate(computedInterval)
@@ -195,7 +182,8 @@ func (c *Component) contextFunc(inputStr string) context {
}
c.metrics.clickhouseQueries.WithLabelValues(table).Inc()
return context{
context := context{
Table: table,
Timefilter: timefilter,
TimefilterStart: timefilterStart,
@@ -211,6 +199,16 @@ func (c *Component) contextFunc(inputStr string) context {
diffOffset)
},
}
t := template.Must(template.New("query").
Option("missingkey=error").
Parse(strings.TrimSpace(query.Template)))
buf := bytes.NewBufferString("")
if err := t.Execute(buf, context); err != nil {
c.r.Err(err).Str("query", query.Template).Msg("invalid query")
panic(err)
}
return buf.String()
}
func (c *Component) computeTableAndInterval(input inputContext) (string, time.Duration, time.Duration) {
@@ -223,8 +221,8 @@ func (c *Component) computeTableAndInterval(input inputContext) (string, time.Du
targetIntervalForTableSelection = time.Second
}
startForTableSelection := input.Start
if input.StartForInterval != nil {
startForTableSelection = *input.StartForInterval
if input.StartForTableSelection != nil {
startForTableSelection = *input.StartForTableSelection
}
table, computedInterval := c.getBestTable(startForTableSelection, targetIntervalForTableSelection)
return table, computedInterval, targetInterval

11
console/clickhouse_test.go
View File

@@ -4,7 +4,6 @@
package console
import (
"fmt"
"testing"
"time"
@@ -192,7 +191,7 @@ func TestFinalizeQuery(t *testing.T) {
Context: inputContext{
Start: time.Date(2022, 3, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 3, 11, 15, 45, 10, 0, time.UTC),
StartForInterval: func() *time.Time {
StartForTableSelection: func() *time.Time {
t := time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC)
return &t
}(),
@@ -294,10 +293,12 @@ func TestFinalizeQuery(t *testing.T) {
for _, tc := range cases {
t.Run(tc.Description, func(t *testing.T) {
c.flowsTables = tc.Tables
got := c.finalizeQuery(
fmt.Sprintf(`{{ with %s }}%s{{ end }}`, templateContext(tc.Context), tc.Query))
got := c.finalizeTemplateQuery(templateQuery{
Template: tc.Query,
Context: tc.Context,
})
if diff := helpers.Diff(got, tc.Expected); diff != "" {
t.Fatalf("finalizeQuery(): (-got, +want):\n%s", diff)
t.Fatalf("finalizeTemplateQuery(): (-got, +want):\n%s", diff)
}
})
}

47
console/line.go
View File

@@ -100,7 +100,7 @@ type toSQL1Options struct {
mainTableRequired bool
}
func (input graphLineHandlerInput) toSQL1(axis int, options toSQL1Options) string {
func (input graphLineHandlerInput) toSQL1(axis int, options toSQL1Options) templateQuery {
var startForInterval *time.Time
var offsetShift string
if !options.offsetedStart.IsZero() {
@@ -159,8 +159,7 @@ func (input graphLineHandlerInput) toSQL1(axis int, options toSQL1Options) strin
}
}
sqlQuery := fmt.Sprintf(`
{{ with %s }}%s
template := fmt.Sprintf(`%s
SELECT %d AS axis, * FROM (
SELECT
%s
@@ -171,54 +170,58 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}%s
TO {{ .TimefilterEnd }} + INTERVAL 1 second%s
STEP {{ .Interval }}
INTERPOLATE (dimensions AS %s))
{{ end }}`,
templateContext(inputContext{
Start: input.Start,
End: input.End,
StartForInterval: startForInterval,
MainTableRequired: options.mainTableRequired,
Points: input.Points,
Units: units,
}),
INTERPOLATE (dimensions AS %s))`,
withStr, axis, strings.Join(fields, ",\n "), where, offsetShift, offsetShift,
dimensionsInterpolate,
)
return strings.TrimSpace(sqlQuery)
context := inputContext{
Start: input.Start,
End: input.End,
StartForTableSelection: startForInterval,
MainTableRequired: options.mainTableRequired,
Points: input.Points,
Units: units,
}
return templateQuery{
Template: strings.TrimSpace(template),
Context: context,
}
}
// toSQL converts a graph input to an SQL request
func (input graphLineHandlerInput) toSQL() string {
func (input graphLineHandlerInput) toSQL() []templateQuery {
// Calculate mainTableRequired once and use it for all axes to ensure
// consistency. This is useful as previous period will remove the
// dimensions.
mainTableRequired := requireMainTable(input.schema, input.Dimensions, input.Filter)
parts := []string{input.toSQL1(1, toSQL1Options{
queries := []templateQuery{input.toSQL1(1, toSQL1Options{
mainTableRequired: mainTableRequired,
})}
if input.Bidirectional {
parts = append(parts, input.reverseDirection().toSQL1(2, toSQL1Options{
queries = append(queries, input.reverseDirection().toSQL1(2, toSQL1Options{
skipWithClause: true,
reverseDirection: true,
mainTableRequired: mainTableRequired,
}))
}
if input.PreviousPeriod {
parts = append(parts, input.previousPeriod().toSQL1(3, toSQL1Options{
queries = append(queries, input.previousPeriod().toSQL1(3, toSQL1Options{
skipWithClause: true,
offsetedStart: input.Start,
mainTableRequired: mainTableRequired,
}))
}
if input.Bidirectional && input.PreviousPeriod {
parts = append(parts, input.reverseDirection().previousPeriod().toSQL1(4, toSQL1Options{
queries = append(queries, input.reverseDirection().previousPeriod().toSQL1(4, toSQL1Options{
skipWithClause: true,
reverseDirection: true,
offsetedStart: input.Start,
mainTableRequired: mainTableRequired,
}))
}
return strings.Join(parts, "\nUNION ALL\n")
return queries
}
func (c *Component) graphLineHandlerFunc(gc *gin.Context) {
@@ -243,8 +246,8 @@ func (c *Component) graphLineHandlerFunc(gc *gin.Context) {
return
}
sqlQuery := input.toSQL()
sqlQuery = c.finalizeQuery(sqlQuery)
queries := input.toSQL()
sqlQuery := c.finalizeTemplateQueries(queries)
gc.Header("X-SQL-Query", strings.ReplaceAll(sqlQuery, "\n", " "))
results := []struct {

310
console/line_test.go
View File

@@ -5,7 +5,6 @@ package console
import (
"fmt"
"strings"
"testing"
"time"
@@ -165,7 +164,7 @@ func TestGraphQuerySQL(t *testing.T) {
Description string
Pos helpers.Pos
Input graphLineHandlerInput
Expected string
Expected []templateQuery
}{
{
Description: "no dimensions, no filters, bps",
@@ -180,9 +179,15 @@ func TestGraphQuerySQL(t *testing.T) {
},
Points: 100,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Units: "l3bps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1)
SELECT 1 AS axis, * FROM (
SELECT
@@ -196,8 +201,9 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS emptyArrayString()))
{{ end }}`,
INTERPOLATE (dimensions AS emptyArrayString()))`,
},
},
}, {
Description: "no dimensions, no filters, l2 bps",
Pos: helpers.Mark(),
@@ -211,9 +217,15 @@ ORDER BY time WITH FILL
},
Points: 100,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l2bps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Units: "l2bps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1)
SELECT 1 AS axis, * FROM (
SELECT
@@ -227,9 +239,9 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS emptyArrayString()))
{{ end }}
`,
INTERPOLATE (dimensions AS emptyArrayString()))`,
},
},
}, {
Description: "no dimensions, no filters, pps",
Pos: helpers.Mark(),
@@ -243,9 +255,15 @@ ORDER BY time WITH FILL
},
Points: 100,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"pps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Units: "pps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1)
SELECT 1 AS axis, * FROM (
SELECT
@@ -259,8 +277,9 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS emptyArrayString()))
{{ end }}`,
INTERPOLATE (dimensions AS emptyArrayString()))`,
},
},
}, {
Description: "truncated source address",
Pos: helpers.Mark(),
@@ -276,9 +295,16 @@ ORDER BY time WITH FILL
},
Points: 100,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","main-table-required":true,"points":100,"units":"l3bps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Units: "l3bps",
MainTableRequired: true,
},
Template: `WITH
source AS (SELECT * REPLACE (tupleElement(IPv6CIDRToRange(SrcAddr, if(tupleElement(IPv6CIDRToRange(SrcAddr, 96), 1) = toIPv6('::ffff:0.0.0.0'), 120, 48)), 1) AS SrcAddr) FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1),
rows AS (SELECT SrcAddr FROM source WHERE {{ .Timefilter }} AND (SrcAddr BETWEEN toIPv6('::ffff:1.0.0.0') AND toIPv6('::ffff:1.255.255.255')) GROUP BY SrcAddr ORDER BY {{ .Units }} DESC LIMIT 0)
SELECT 1 AS axis, * FROM (
@@ -293,8 +319,9 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS ['Other']))
{{ end }}`,
INTERPOLATE (dimensions AS ['Other']))`,
},
},
}, {
Description: "no dimensions",
Pos: helpers.Mark(),
@@ -308,9 +335,15 @@ ORDER BY time WITH FILL
},
Points: 100,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Units: "l3bps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1)
SELECT 1 AS axis, * FROM (
SELECT
@@ -324,8 +357,9 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS emptyArrayString()))
{{ end }}`,
INTERPOLATE (dimensions AS emptyArrayString()))`,
},
},
}, {
Description: "no dimensions, escaped filter",
Pos: helpers.Mark(),
@@ -339,9 +373,15 @@ ORDER BY time WITH FILL
},
Points: 100,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Units: "l3bps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1)
SELECT 1 AS axis, * FROM (
SELECT
@@ -355,8 +395,9 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS emptyArrayString()))
{{ end }}`,
INTERPOLATE (dimensions AS emptyArrayString()))`,
},
},
}, {
Description: "no dimensions, reverse direction",
Pos: helpers.Mark(),
@@ -371,9 +412,15 @@ ORDER BY time WITH FILL
Points: 100,
Bidirectional: true,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Units: "l3bps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1)
SELECT 1 AS axis, * FROM (
SELECT
@@ -387,11 +434,15 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS emptyArrayString()))
{{ end }}
UNION ALL
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
SELECT 2 AS axis, * FROM (
INTERPOLATE (dimensions AS emptyArrayString()))`,
}, {
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Units: "l3bps",
},
Template: `SELECT 2 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
{{ .Units }}/{{ .Interval }} AS xps,
@@ -403,8 +454,9 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS emptyArrayString()))
{{ end }}`,
INTERPOLATE (dimensions AS emptyArrayString()))`,
},
},
}, {
Description: "no dimensions, reverse direction, inl2%",
Pos: helpers.Mark(),
@@ -419,9 +471,15 @@ ORDER BY time WITH FILL
Points: 100,
Bidirectional: true,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"inl2%"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Units: "inl2%",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1)
SELECT 1 AS axis, * FROM (
SELECT
@@ -435,11 +493,15 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS emptyArrayString()))
{{ end }}
UNION ALL
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"outl2%"}@@ }}
SELECT 2 AS axis, * FROM (
INTERPOLATE (dimensions AS emptyArrayString()))`,
}, {
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Units: "outl2%",
},
Template: `SELECT 2 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
{{ .Units }}/{{ .Interval }} AS xps,
@@ -451,8 +513,9 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS emptyArrayString()))
{{ end }}`,
INTERPOLATE (dimensions AS emptyArrayString()))`,
},
},
}, {
Description: "no filters",
Pos: helpers.Mark(),
@@ -470,9 +533,15 @@ ORDER BY time WITH FILL
},
Points: 100,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Units: "l3bps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1),
rows AS (SELECT ExporterName, InIfProvider FROM source WHERE {{ .Timefilter }} GROUP BY ExporterName, InIfProvider ORDER BY {{ .Units }} DESC LIMIT 20)
SELECT 1 AS axis, * FROM (
@@ -487,8 +556,9 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS ['Other', 'Other']))
{{ end }}`,
INTERPOLATE (dimensions AS ['Other', 'Other']))`,
},
},
}, {
Description: "no filters, limitType by max",
Pos: helpers.Mark(),
@@ -507,9 +577,15 @@ ORDER BY time WITH FILL
},
Points: 100,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Units: "l3bps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1),
rows AS (SELECT ExporterName, InIfProvider FROM ( SELECT ExporterName, InIfProvider, {{ .Units }} AS sum_at_time FROM source WHERE {{ .Timefilter }} GROUP BY ExporterName, InIfProvider ) GROUP BY ExporterName, InIfProvider ORDER BY MAX(sum_at_time) DESC LIMIT 20)
SELECT 1 AS axis, * FROM (
@@ -524,8 +600,9 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS ['Other', 'Other']))
{{ end }}`,
INTERPOLATE (dimensions AS ['Other', 'Other']))`,
},
},
}, {
Description: "no filters, reverse",
Pos: helpers.Mark(),
@@ -544,9 +621,15 @@ ORDER BY time WITH FILL
Points: 100,
Bidirectional: true,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Units: "l3bps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1),
rows AS (SELECT ExporterName, InIfProvider FROM source WHERE {{ .Timefilter }} GROUP BY ExporterName, InIfProvider ORDER BY {{ .Units }} DESC LIMIT 20)
SELECT 1 AS axis, * FROM (
@@ -561,11 +644,15 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS ['Other', 'Other']))
{{ end }}
UNION ALL
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
SELECT 2 AS axis, * FROM (
INTERPOLATE (dimensions AS ['Other', 'Other']))`,
}, {
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Units: "l3bps",
},
Template: `SELECT 2 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
{{ .Units }}/{{ .Interval }} AS xps,
@@ -577,8 +664,9 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS ['Other', 'Other']))
{{ end }}`,
INTERPOLATE (dimensions AS ['Other', 'Other']))`,
},
},
}, {
Description: "no filters, previous period",
Pos: helpers.Mark(),
@@ -597,9 +685,15 @@ ORDER BY time WITH FILL
Points: 100,
PreviousPeriod: true,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Units: "l3bps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1),
rows AS (SELECT ExporterName, InIfProvider FROM source WHERE {{ .Timefilter }} GROUP BY ExporterName, InIfProvider ORDER BY {{ .Units }} DESC LIMIT 20)
SELECT 1 AS axis, * FROM (
@@ -614,11 +708,19 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS ['Other', 'Other']))
{{ end }}
UNION ALL
{{ with context @@{"start":"2022-04-09T15:45:10Z","end":"2022-04-10T15:45:10Z","start-for-interval":"2022-04-10T15:45:10Z","points":100,"units":"l3bps"}@@ }}
SELECT 3 AS axis, * FROM (
INTERPOLATE (dimensions AS ['Other', 'Other']))`,
}, {
Context: inputContext{
Start: time.Date(2022, 4, 9, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
StartForTableSelection: func() *time.Time {
t := time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC)
return &t
}(),
Points: 100,
Units: "l3bps",
},
Template: `SELECT 3 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} + INTERVAL 86400 second AS time,
{{ .Units }}/{{ .Interval }} AS xps,
@@ -630,8 +732,9 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }} + INTERVAL 86400 second
TO {{ .TimefilterEnd }} + INTERVAL 1 second + INTERVAL 86400 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS emptyArrayString()))
{{ end }}`,
INTERPOLATE (dimensions AS emptyArrayString()))`,
},
},
}, {
Description: "previous period while main table is required",
Pos: helpers.Mark(),
@@ -649,9 +752,16 @@ ORDER BY time WITH FILL
Points: 100,
PreviousPeriod: true,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","main-table-required":true,"points":100,"units":"l3bps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
MainTableRequired: true,
Points: 100,
Units: "l3bps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1),
rows AS (SELECT SrcAddr, DstAddr FROM source WHERE {{ .Timefilter }} AND (InIfBoundary = 'external') GROUP BY SrcAddr, DstAddr ORDER BY {{ .Units }} DESC LIMIT 0)
SELECT 1 AS axis, * FROM (
@@ -666,11 +776,20 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS ['Other', 'Other']))
{{ end }}
UNION ALL
{{ with context @@{"start":"2022-04-09T15:45:10Z","end":"2022-04-10T15:45:10Z","start-for-interval":"2022-04-10T15:45:10Z","main-table-required":true,"points":100,"units":"l3bps"}@@ }}
SELECT 3 AS axis, * FROM (
INTERPOLATE (dimensions AS ['Other', 'Other']))`,
}, {
Context: inputContext{
Start: time.Date(2022, 4, 9, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
StartForTableSelection: func() *time.Time {
t := time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC)
return &t
}(),
MainTableRequired: true,
Points: 100,
Units: "l3bps",
},
Template: `SELECT 3 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} + INTERVAL 86400 second AS time,
{{ .Units }}/{{ .Interval }} AS xps,
@@ -682,8 +801,9 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }} + INTERVAL 86400 second
TO {{ .TimefilterEnd }} + INTERVAL 1 second + INTERVAL 86400 second
STEP {{ .Interval }}
INTERPOLATE (dimensions AS emptyArrayString()))
{{ end }}`,
INTERPOLATE (dimensions AS emptyArrayString()))`,
},
},
},
}
for _, tc := range cases {
@@ -694,11 +814,9 @@ ORDER BY time WITH FILL
if err := tc.Input.Filter.Validate(tc.Input.schema); err != nil {
t.Fatalf("%sValidate() error:\n%+v", tc.Pos, err)
}
tc.Expected = strings.ReplaceAll(tc.Expected, "@@", "`")
t.Run(tc.Description, func(t *testing.T) {
got := tc.Input.toSQL()
if diff := helpers.Diff(strings.Split(strings.TrimSpace(got), "\n"),
strings.Split(strings.TrimSpace(tc.Expected), "\n")); diff != "" {
if diff := helpers.Diff(got, tc.Expected); diff != "" {
t.Errorf("%stoSQL (-got, +want):\n%s", tc.Pos, diff)
}
})

25
console/sankey.go
View File

@@ -36,7 +36,7 @@ type sankeyLink struct {
}
// sankeyHandlerInputToSQL converts a sankey query to an SQL request
func (input graphSankeyHandlerInput) toSQL() (string, error) {
func (input graphSankeyHandlerInput) toSQL() ([]templateQuery, error) {
where := templateWhere(input.Filter)
// Select
@@ -61,8 +61,7 @@ func (input graphSankeyHandlerInput) toSQL() (string, error) {
}
with = append(with, selectSankeyRowsByLimitType(input, dimensions, where))
sqlQuery := fmt.Sprintf(`
{{ with %s }}
template := fmt.Sprintf(`
WITH
%s
SELECT
@@ -70,17 +69,21 @@ SELECT
FROM source
WHERE %s
GROUP BY dimensions
ORDER BY xps DESC
{{ end }}`,
templateContext(inputContext{
ORDER BY xps DESC`,
strings.Join(with, ",\n "), strings.Join(fields, ",\n "), where)
context := inputContext{
Start: input.Start,
End: input.End,
MainTableRequired: requireMainTable(input.schema, input.Dimensions, input.Filter),
Points: 20,
Units: input.Units,
}),
strings.Join(with, ",\n "), strings.Join(fields, ",\n "), where)
return strings.TrimSpace(sqlQuery), nil
}
return []templateQuery{{
Template: strings.TrimSpace(template),
Context: context,
}}, nil
}
func (c *Component) graphSankeyHandlerFunc(gc *gin.Context) {
@@ -105,14 +108,14 @@ func (c *Component) graphSankeyHandlerFunc(gc *gin.Context) {
return
}
sqlQuery, err := input.toSQL()
queries, err := input.toSQL()
if err != nil {
gc.JSON(http.StatusBadRequest, gin.H{"message": helpers.Capitalize(err.Error())})
return
}
// Prepare and execute query
sqlQuery = c.finalizeQuery(sqlQuery)
sqlQuery := c.finalizeTemplateQueries(queries)
gc.Header("X-SQL-Query", strings.ReplaceAll(sqlQuery, "\n", " "))
results := []struct {
Xps float64 `ch:"xps"`

93
console/sankey_test.go
View File

@@ -4,7 +4,6 @@
package console
import (
"strings"
"testing"
"time"
@@ -21,7 +20,7 @@ func TestSankeyQuerySQL(t *testing.T) {
Description string
Pos helpers.Pos
Input graphSankeyHandlerInput
Expected string
Expected []templateQuery
}{
{
Description: "two dimensions, no filters, l3 bps",
@@ -39,9 +38,15 @@ func TestSankeyQuerySQL(t *testing.T) {
Units: "l3bps",
},
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":20,"units":"l3bps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 20,
Units: "l3bps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1),
(SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE {{ .Timefilter }}) AS range,
rows AS (SELECT SrcAS, ExporterName FROM source WHERE {{ .Timefilter }} GROUP BY SrcAS, ExporterName ORDER BY {{ .Units }} DESC LIMIT 5)
@@ -52,8 +57,9 @@ SELECT
FROM source
WHERE {{ .Timefilter }}
GROUP BY dimensions
ORDER BY xps DESC
{{ end }}`,
ORDER BY xps DESC`,
},
},
}, {
Description: "two dimensions, no filters, l3 bps, limitType by max",
Pos: helpers.Mark(),
@@ -71,9 +77,15 @@ ORDER BY xps DESC
Units: "l3bps",
},
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":20,"units":"l3bps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 20,
Units: "l3bps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1),
(SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE {{ .Timefilter }}) AS range,
rows AS (SELECT SrcAS, ExporterName FROM ( SELECT SrcAS, ExporterName, {{ .Units }} AS sum_at_time FROM source WHERE {{ .Timefilter }} GROUP BY SrcAS, ExporterName ) GROUP BY SrcAS, ExporterName ORDER BY MAX(sum_at_time) DESC LIMIT 5)
@@ -84,8 +96,9 @@ SELECT
FROM source
WHERE {{ .Timefilter }}
GROUP BY dimensions
ORDER BY xps DESC
{{ end }}`,
ORDER BY xps DESC`,
},
},
}, {
Description: "two dimensions, no filters, l2 bps",
Pos: helpers.Mark(),
@@ -102,9 +115,15 @@ ORDER BY xps DESC
Units: "l2bps",
},
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":20,"units":"l2bps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 20,
Units: "l2bps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1),
(SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE {{ .Timefilter }}) AS range,
rows AS (SELECT SrcAS, ExporterName FROM source WHERE {{ .Timefilter }} GROUP BY SrcAS, ExporterName ORDER BY {{ .Units }} DESC LIMIT 5)
@@ -115,9 +134,9 @@ SELECT
FROM source
WHERE {{ .Timefilter }}
GROUP BY dimensions
ORDER BY xps DESC
{{ end }}
`,
ORDER BY xps DESC`,
},
},
}, {
Description: "two dimensions, no filters, pps",
Pos: helpers.Mark(),
@@ -134,9 +153,15 @@ ORDER BY xps DESC
Units: "pps",
},
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":20,"units":"pps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 20,
Units: "pps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1),
(SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE {{ .Timefilter }}) AS range,
rows AS (SELECT SrcAS, ExporterName FROM source WHERE {{ .Timefilter }} GROUP BY SrcAS, ExporterName ORDER BY {{ .Units }} DESC LIMIT 5)
@@ -147,8 +172,9 @@ SELECT
FROM source
WHERE {{ .Timefilter }}
GROUP BY dimensions
ORDER BY xps DESC
{{ end }}`,
ORDER BY xps DESC`,
},
},
}, {
Description: "two dimensions, with filter",
Pos: helpers.Mark(),
@@ -165,9 +191,15 @@ ORDER BY xps DESC
Units: "l3bps",
},
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":20,"units":"l3bps"}@@ }}
WITH
Expected: []templateQuery{
{
Context: inputContext{
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
Points: 20,
Units: "l3bps",
},
Template: `WITH
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1),
(SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE {{ .Timefilter }} AND (DstCountry = 'FR')) AS range,
rows AS (SELECT SrcAS, ExporterName FROM source WHERE {{ .Timefilter }} AND (DstCountry = 'FR') GROUP BY SrcAS, ExporterName ORDER BY {{ .Units }} DESC LIMIT 10)
@@ -178,8 +210,9 @@ SELECT
FROM source
WHERE {{ .Timefilter }} AND (DstCountry = 'FR')
GROUP BY dimensions
ORDER BY xps DESC
{{ end }}`,
ORDER BY xps DESC`,
},
},
},
}
for _, tc := range cases {
@@ -190,11 +223,9 @@ ORDER BY xps DESC
if err := tc.Input.Filter.Validate(tc.Input.schema); err != nil {
t.Fatalf("%sValidate() error:\n%+v", tc.Pos, err)
}
tc.Expected = strings.ReplaceAll(tc.Expected, "@@", "`")
t.Run(tc.Description, func(t *testing.T) {
got, _ := tc.Input.toSQL()
if diff := helpers.Diff(strings.Split(strings.TrimSpace(got), "\n"),
strings.Split(strings.TrimSpace(tc.Expected), "\n")); diff != "" {
if diff := helpers.Diff(got, tc.Expected); diff != "" {
t.Errorf("%stoSQL (-got, +want):\n%s", tc.Pos, diff)
}
})

32
console/widgets.go
View File

@@ -192,8 +192,7 @@ func (c *Component) widgetTopHandlerFunc(gc *gin.Context) {
}
now := c.d.Clock.Now()
query := c.finalizeQuery(fmt.Sprintf(`
{{ with %s }}
template := fmt.Sprintf(`
WITH
(SELECT SUM(Bytes*SamplingRate) FROM {{ .Table }} WHERE {{ .Timefilter }} %s) AS Total
SELECT
@@ -204,15 +203,18 @@ WHERE {{ .Timefilter }}
%s
GROUP BY %s
ORDER BY Percent DESC
LIMIT 5
{{ end }}`,
templateContext(inputContext{
LIMIT 5`,
filter, selector, selector, filter, groupby)
query := c.finalizeTemplateQuery(templateQuery{
Template: template,
Context: inputContext{
Start: now.Add(-5 * time.Minute),
End: now,
MainTableRequired: mainTableRequired,
Points: 5,
}),
filter, selector, selector, filter, groupby))
},
})
gc.Header("X-SQL-Query", query)
results := []topResult{}
@@ -232,8 +234,7 @@ func (c *Component) widgetGraphHandlerFunc(gc *gin.Context) {
}
ctx := c.t.Context(gc.Request.Context())
now := c.d.Clock.Now()
query := c.finalizeQuery(fmt.Sprintf(`
{{ with %s }}
template := fmt.Sprintf(`
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} AS Time,
SUM(Bytes*SamplingRate*8/{{ .Interval }})/1000/1000/1000 AS Gbps
@@ -244,15 +245,18 @@ GROUP BY Time
ORDER BY Time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }}
{{ end }}`,
templateContext(inputContext{
STEP {{ .Interval }}`,
filter)
query := c.finalizeTemplateQuery(templateQuery{
Template: template,
Context: inputContext{
Start: now.Add(-c.config.HomepageGraphTimeRange),
End: now,
MainTableRequired: false,
Points: 200,
}),
filter))
},
})
gc.Header("X-SQL-Query", query)
results := []struct {