diff --git a/console/clickhouse.go b/console/clickhouse.go index 45c6ab62..594b3839 100644 --- a/console/clickhouse.go +++ b/console/clickhouse.go @@ -5,7 +5,6 @@ package console import ( "bytes" - "encoding/json" "errors" "fmt" "sort" @@ -82,33 +81,17 @@ AND (engine LIKE '%MergeTree' OR engine = 'Distributed') return nil } -// finalizeQuery builds the finalized query. A single "context" -// function is provided to return a `Context` struct with all the -// information needed. -func (c *Component) finalizeQuery(query string) string { - t := template.Must(template.New("query"). - Funcs(template.FuncMap{ - "context": c.contextFunc, - }). - Option("missingkey=error"). - Parse(strings.TrimSpace(query))) - buf := bytes.NewBufferString("") - if err := t.Execute(buf, nil); err != nil { - c.r.Err(err).Str("query", query).Msg("invalid query") - panic(err) - } - return buf.String() -} - +// inputContext is the intermeidate context provided by the input handler. type inputContext struct { - Start time.Time `json:"start"` - End time.Time `json:"end"` - StartForInterval *time.Time `json:"start-for-interval,omitempty"` - MainTableRequired bool `json:"main-table-required,omitempty"` - Points uint `json:"points"` - Units string `json:"units,omitempty"` + Start time.Time + End time.Time + StartForTableSelection *time.Time + MainTableRequired bool + Points uint + Units string } +// context is the context to finalize the template. type context struct { Table string Timefilter string @@ -119,6 +102,12 @@ type context struct { ToStartOfInterval func(string) string } +// templateQuery holds a template string and its associated input context. +type templateQuery struct { + Template string + Context inputContext +} + // templateEscape escapes `{{` and `}}` from a string. In fact, only // the opening tag needs to be escaped. func templateEscape(input string) string { @@ -133,22 +122,20 @@ func templateWhere(qf query.Filter) string { return fmt.Sprintf(`{{ .Timefilter }} AND (%s)`, templateEscape(qf.Direct())) } -// templateTable builds a template directive to select the right table -func templateContext(context inputContext) string { - encoded, err := json.Marshal(context) - if err != nil { - panic(err) +// finalizeTemplateQueries builds the finalized queries from a list of templateQuery. +// Each template is processed with its associated context and combined with UNION ALL. +func (c *Component) finalizeTemplateQueries(queries []templateQuery) string { + parts := make([]string, len(queries)) + for i, q := range queries { + parts[i] = c.finalizeTemplateQuery(q) } - return fmt.Sprintf("context `%s`", string(encoded)) + return strings.Join(parts, "\nUNION ALL\n") } -func (c *Component) contextFunc(inputStr string) context { - var input inputContext - if err := json.Unmarshal([]byte(inputStr), &input); err != nil { - panic(err) - } - - table, computedInterval, targetInterval := c.computeTableAndInterval(input) +// finalizeTemplateQuery builds the finalized query for a single templateQuery +func (c *Component) finalizeTemplateQuery(query templateQuery) string { + input := query.Context + table, computedInterval, targetInterval := c.computeTableAndInterval(query.Context) // Make start/end match the computed interval (currently equal to the table resolution) start := input.Start.Truncate(computedInterval) @@ -195,7 +182,8 @@ func (c *Component) contextFunc(inputStr string) context { } c.metrics.clickhouseQueries.WithLabelValues(table).Inc() - return context{ + + context := context{ Table: table, Timefilter: timefilter, TimefilterStart: timefilterStart, @@ -211,6 +199,16 @@ func (c *Component) contextFunc(inputStr string) context { diffOffset) }, } + + t := template.Must(template.New("query"). + Option("missingkey=error"). + Parse(strings.TrimSpace(query.Template))) + buf := bytes.NewBufferString("") + if err := t.Execute(buf, context); err != nil { + c.r.Err(err).Str("query", query.Template).Msg("invalid query") + panic(err) + } + return buf.String() } func (c *Component) computeTableAndInterval(input inputContext) (string, time.Duration, time.Duration) { @@ -223,8 +221,8 @@ func (c *Component) computeTableAndInterval(input inputContext) (string, time.Du targetIntervalForTableSelection = time.Second } startForTableSelection := input.Start - if input.StartForInterval != nil { - startForTableSelection = *input.StartForInterval + if input.StartForTableSelection != nil { + startForTableSelection = *input.StartForTableSelection } table, computedInterval := c.getBestTable(startForTableSelection, targetIntervalForTableSelection) return table, computedInterval, targetInterval diff --git a/console/clickhouse_test.go b/console/clickhouse_test.go index 5b41b7c3..83ee91d3 100644 --- a/console/clickhouse_test.go +++ b/console/clickhouse_test.go @@ -4,7 +4,6 @@ package console import ( - "fmt" "testing" "time" @@ -192,7 +191,7 @@ func TestFinalizeQuery(t *testing.T) { Context: inputContext{ Start: time.Date(2022, 3, 10, 15, 45, 10, 0, time.UTC), End: time.Date(2022, 3, 11, 15, 45, 10, 0, time.UTC), - StartForInterval: func() *time.Time { + StartForTableSelection: func() *time.Time { t := time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC) return &t }(), @@ -294,10 +293,12 @@ func TestFinalizeQuery(t *testing.T) { for _, tc := range cases { t.Run(tc.Description, func(t *testing.T) { c.flowsTables = tc.Tables - got := c.finalizeQuery( - fmt.Sprintf(`{{ with %s }}%s{{ end }}`, templateContext(tc.Context), tc.Query)) + got := c.finalizeTemplateQuery(templateQuery{ + Template: tc.Query, + Context: tc.Context, + }) if diff := helpers.Diff(got, tc.Expected); diff != "" { - t.Fatalf("finalizeQuery(): (-got, +want):\n%s", diff) + t.Fatalf("finalizeTemplateQuery(): (-got, +want):\n%s", diff) } }) } diff --git a/console/line.go b/console/line.go index dd25f287..3e9dc796 100644 --- a/console/line.go +++ b/console/line.go @@ -100,7 +100,7 @@ type toSQL1Options struct { mainTableRequired bool } -func (input graphLineHandlerInput) toSQL1(axis int, options toSQL1Options) string { +func (input graphLineHandlerInput) toSQL1(axis int, options toSQL1Options) templateQuery { var startForInterval *time.Time var offsetShift string if !options.offsetedStart.IsZero() { @@ -159,8 +159,7 @@ func (input graphLineHandlerInput) toSQL1(axis int, options toSQL1Options) strin } } - sqlQuery := fmt.Sprintf(` -{{ with %s }}%s + template := fmt.Sprintf(`%s SELECT %d AS axis, * FROM ( SELECT %s @@ -171,54 +170,58 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }}%s TO {{ .TimefilterEnd }} + INTERVAL 1 second%s STEP {{ .Interval }} - INTERPOLATE (dimensions AS %s)) -{{ end }}`, - templateContext(inputContext{ - Start: input.Start, - End: input.End, - StartForInterval: startForInterval, - MainTableRequired: options.mainTableRequired, - Points: input.Points, - Units: units, - }), + INTERPOLATE (dimensions AS %s))`, withStr, axis, strings.Join(fields, ",\n "), where, offsetShift, offsetShift, dimensionsInterpolate, ) - return strings.TrimSpace(sqlQuery) + + context := inputContext{ + Start: input.Start, + End: input.End, + StartForTableSelection: startForInterval, + MainTableRequired: options.mainTableRequired, + Points: input.Points, + Units: units, + } + + return templateQuery{ + Template: strings.TrimSpace(template), + Context: context, + } } // toSQL converts a graph input to an SQL request -func (input graphLineHandlerInput) toSQL() string { +func (input graphLineHandlerInput) toSQL() []templateQuery { // Calculate mainTableRequired once and use it for all axes to ensure // consistency. This is useful as previous period will remove the // dimensions. mainTableRequired := requireMainTable(input.schema, input.Dimensions, input.Filter) - parts := []string{input.toSQL1(1, toSQL1Options{ + queries := []templateQuery{input.toSQL1(1, toSQL1Options{ mainTableRequired: mainTableRequired, })} if input.Bidirectional { - parts = append(parts, input.reverseDirection().toSQL1(2, toSQL1Options{ + queries = append(queries, input.reverseDirection().toSQL1(2, toSQL1Options{ skipWithClause: true, reverseDirection: true, mainTableRequired: mainTableRequired, })) } if input.PreviousPeriod { - parts = append(parts, input.previousPeriod().toSQL1(3, toSQL1Options{ + queries = append(queries, input.previousPeriod().toSQL1(3, toSQL1Options{ skipWithClause: true, offsetedStart: input.Start, mainTableRequired: mainTableRequired, })) } if input.Bidirectional && input.PreviousPeriod { - parts = append(parts, input.reverseDirection().previousPeriod().toSQL1(4, toSQL1Options{ + queries = append(queries, input.reverseDirection().previousPeriod().toSQL1(4, toSQL1Options{ skipWithClause: true, reverseDirection: true, offsetedStart: input.Start, mainTableRequired: mainTableRequired, })) } - return strings.Join(parts, "\nUNION ALL\n") + return queries } func (c *Component) graphLineHandlerFunc(gc *gin.Context) { @@ -243,8 +246,8 @@ func (c *Component) graphLineHandlerFunc(gc *gin.Context) { return } - sqlQuery := input.toSQL() - sqlQuery = c.finalizeQuery(sqlQuery) + queries := input.toSQL() + sqlQuery := c.finalizeTemplateQueries(queries) gc.Header("X-SQL-Query", strings.ReplaceAll(sqlQuery, "\n", " ")) results := []struct { diff --git a/console/line_test.go b/console/line_test.go index 33936fa4..3875122b 100644 --- a/console/line_test.go +++ b/console/line_test.go @@ -5,7 +5,6 @@ package console import ( "fmt" - "strings" "testing" "time" @@ -165,7 +164,7 @@ func TestGraphQuerySQL(t *testing.T) { Description string Pos helpers.Pos Input graphLineHandlerInput - Expected string + Expected []templateQuery }{ { Description: "no dimensions, no filters, bps", @@ -180,9 +179,15 @@ func TestGraphQuerySQL(t *testing.T) { }, Points: 100, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 100, + Units: "l3bps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1) SELECT 1 AS axis, * FROM ( SELECT @@ -196,8 +201,9 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS emptyArrayString())) -{{ end }}`, + INTERPOLATE (dimensions AS emptyArrayString()))`, + }, + }, }, { Description: "no dimensions, no filters, l2 bps", Pos: helpers.Mark(), @@ -211,9 +217,15 @@ ORDER BY time WITH FILL }, Points: 100, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l2bps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 100, + Units: "l2bps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1) SELECT 1 AS axis, * FROM ( SELECT @@ -227,9 +239,9 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS emptyArrayString())) -{{ end }} -`, + INTERPOLATE (dimensions AS emptyArrayString()))`, + }, + }, }, { Description: "no dimensions, no filters, pps", Pos: helpers.Mark(), @@ -243,9 +255,15 @@ ORDER BY time WITH FILL }, Points: 100, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"pps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 100, + Units: "pps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1) SELECT 1 AS axis, * FROM ( SELECT @@ -259,8 +277,9 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS emptyArrayString())) -{{ end }}`, + INTERPOLATE (dimensions AS emptyArrayString()))`, + }, + }, }, { Description: "truncated source address", Pos: helpers.Mark(), @@ -276,9 +295,16 @@ ORDER BY time WITH FILL }, Points: 100, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","main-table-required":true,"points":100,"units":"l3bps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 100, + Units: "l3bps", + MainTableRequired: true, + }, + Template: `WITH source AS (SELECT * REPLACE (tupleElement(IPv6CIDRToRange(SrcAddr, if(tupleElement(IPv6CIDRToRange(SrcAddr, 96), 1) = toIPv6('::ffff:0.0.0.0'), 120, 48)), 1) AS SrcAddr) FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1), rows AS (SELECT SrcAddr FROM source WHERE {{ .Timefilter }} AND (SrcAddr BETWEEN toIPv6('::ffff:1.0.0.0') AND toIPv6('::ffff:1.255.255.255')) GROUP BY SrcAddr ORDER BY {{ .Units }} DESC LIMIT 0) SELECT 1 AS axis, * FROM ( @@ -293,8 +319,9 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS ['Other'])) -{{ end }}`, + INTERPOLATE (dimensions AS ['Other']))`, + }, + }, }, { Description: "no dimensions", Pos: helpers.Mark(), @@ -308,9 +335,15 @@ ORDER BY time WITH FILL }, Points: 100, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 100, + Units: "l3bps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1) SELECT 1 AS axis, * FROM ( SELECT @@ -324,8 +357,9 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS emptyArrayString())) -{{ end }}`, + INTERPOLATE (dimensions AS emptyArrayString()))`, + }, + }, }, { Description: "no dimensions, escaped filter", Pos: helpers.Mark(), @@ -339,9 +373,15 @@ ORDER BY time WITH FILL }, Points: 100, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 100, + Units: "l3bps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1) SELECT 1 AS axis, * FROM ( SELECT @@ -355,8 +395,9 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS emptyArrayString())) -{{ end }}`, + INTERPOLATE (dimensions AS emptyArrayString()))`, + }, + }, }, { Description: "no dimensions, reverse direction", Pos: helpers.Mark(), @@ -371,9 +412,15 @@ ORDER BY time WITH FILL Points: 100, Bidirectional: true, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 100, + Units: "l3bps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1) SELECT 1 AS axis, * FROM ( SELECT @@ -387,11 +434,15 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS emptyArrayString())) -{{ end }} -UNION ALL -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }} -SELECT 2 AS axis, * FROM ( + INTERPOLATE (dimensions AS emptyArrayString()))`, + }, { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 100, + Units: "l3bps", + }, + Template: `SELECT 2 AS axis, * FROM ( SELECT {{ call .ToStartOfInterval "TimeReceived" }} AS time, {{ .Units }}/{{ .Interval }} AS xps, @@ -403,8 +454,9 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS emptyArrayString())) -{{ end }}`, + INTERPOLATE (dimensions AS emptyArrayString()))`, + }, + }, }, { Description: "no dimensions, reverse direction, inl2%", Pos: helpers.Mark(), @@ -419,9 +471,15 @@ ORDER BY time WITH FILL Points: 100, Bidirectional: true, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"inl2%"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 100, + Units: "inl2%", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1) SELECT 1 AS axis, * FROM ( SELECT @@ -435,11 +493,15 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS emptyArrayString())) -{{ end }} -UNION ALL -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"outl2%"}@@ }} -SELECT 2 AS axis, * FROM ( + INTERPOLATE (dimensions AS emptyArrayString()))`, + }, { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 100, + Units: "outl2%", + }, + Template: `SELECT 2 AS axis, * FROM ( SELECT {{ call .ToStartOfInterval "TimeReceived" }} AS time, {{ .Units }}/{{ .Interval }} AS xps, @@ -451,8 +513,9 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS emptyArrayString())) -{{ end }}`, + INTERPOLATE (dimensions AS emptyArrayString()))`, + }, + }, }, { Description: "no filters", Pos: helpers.Mark(), @@ -470,9 +533,15 @@ ORDER BY time WITH FILL }, Points: 100, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 100, + Units: "l3bps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1), rows AS (SELECT ExporterName, InIfProvider FROM source WHERE {{ .Timefilter }} GROUP BY ExporterName, InIfProvider ORDER BY {{ .Units }} DESC LIMIT 20) SELECT 1 AS axis, * FROM ( @@ -487,8 +556,9 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS ['Other', 'Other'])) -{{ end }}`, + INTERPOLATE (dimensions AS ['Other', 'Other']))`, + }, + }, }, { Description: "no filters, limitType by max", Pos: helpers.Mark(), @@ -507,9 +577,15 @@ ORDER BY time WITH FILL }, Points: 100, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 100, + Units: "l3bps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1), rows AS (SELECT ExporterName, InIfProvider FROM ( SELECT ExporterName, InIfProvider, {{ .Units }} AS sum_at_time FROM source WHERE {{ .Timefilter }} GROUP BY ExporterName, InIfProvider ) GROUP BY ExporterName, InIfProvider ORDER BY MAX(sum_at_time) DESC LIMIT 20) SELECT 1 AS axis, * FROM ( @@ -524,8 +600,9 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS ['Other', 'Other'])) -{{ end }}`, + INTERPOLATE (dimensions AS ['Other', 'Other']))`, + }, + }, }, { Description: "no filters, reverse", Pos: helpers.Mark(), @@ -544,9 +621,15 @@ ORDER BY time WITH FILL Points: 100, Bidirectional: true, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 100, + Units: "l3bps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1), rows AS (SELECT ExporterName, InIfProvider FROM source WHERE {{ .Timefilter }} GROUP BY ExporterName, InIfProvider ORDER BY {{ .Units }} DESC LIMIT 20) SELECT 1 AS axis, * FROM ( @@ -561,11 +644,15 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS ['Other', 'Other'])) -{{ end }} -UNION ALL -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }} -SELECT 2 AS axis, * FROM ( + INTERPOLATE (dimensions AS ['Other', 'Other']))`, + }, { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 100, + Units: "l3bps", + }, + Template: `SELECT 2 AS axis, * FROM ( SELECT {{ call .ToStartOfInterval "TimeReceived" }} AS time, {{ .Units }}/{{ .Interval }} AS xps, @@ -577,8 +664,9 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS ['Other', 'Other'])) -{{ end }}`, + INTERPOLATE (dimensions AS ['Other', 'Other']))`, + }, + }, }, { Description: "no filters, previous period", Pos: helpers.Mark(), @@ -597,9 +685,15 @@ ORDER BY time WITH FILL Points: 100, PreviousPeriod: true, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 100, + Units: "l3bps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1), rows AS (SELECT ExporterName, InIfProvider FROM source WHERE {{ .Timefilter }} GROUP BY ExporterName, InIfProvider ORDER BY {{ .Units }} DESC LIMIT 20) SELECT 1 AS axis, * FROM ( @@ -614,11 +708,19 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS ['Other', 'Other'])) -{{ end }} -UNION ALL -{{ with context @@{"start":"2022-04-09T15:45:10Z","end":"2022-04-10T15:45:10Z","start-for-interval":"2022-04-10T15:45:10Z","points":100,"units":"l3bps"}@@ }} -SELECT 3 AS axis, * FROM ( + INTERPOLATE (dimensions AS ['Other', 'Other']))`, + }, { + Context: inputContext{ + Start: time.Date(2022, 4, 9, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + StartForTableSelection: func() *time.Time { + t := time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC) + return &t + }(), + Points: 100, + Units: "l3bps", + }, + Template: `SELECT 3 AS axis, * FROM ( SELECT {{ call .ToStartOfInterval "TimeReceived" }} + INTERVAL 86400 second AS time, {{ .Units }}/{{ .Interval }} AS xps, @@ -630,8 +732,9 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} + INTERVAL 86400 second TO {{ .TimefilterEnd }} + INTERVAL 1 second + INTERVAL 86400 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS emptyArrayString())) -{{ end }}`, + INTERPOLATE (dimensions AS emptyArrayString()))`, + }, + }, }, { Description: "previous period while main table is required", Pos: helpers.Mark(), @@ -649,9 +752,16 @@ ORDER BY time WITH FILL Points: 100, PreviousPeriod: true, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","main-table-required":true,"points":100,"units":"l3bps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + MainTableRequired: true, + Points: 100, + Units: "l3bps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1), rows AS (SELECT SrcAddr, DstAddr FROM source WHERE {{ .Timefilter }} AND (InIfBoundary = 'external') GROUP BY SrcAddr, DstAddr ORDER BY {{ .Units }} DESC LIMIT 0) SELECT 1 AS axis, * FROM ( @@ -666,11 +776,20 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS ['Other', 'Other'])) -{{ end }} -UNION ALL -{{ with context @@{"start":"2022-04-09T15:45:10Z","end":"2022-04-10T15:45:10Z","start-for-interval":"2022-04-10T15:45:10Z","main-table-required":true,"points":100,"units":"l3bps"}@@ }} -SELECT 3 AS axis, * FROM ( + INTERPOLATE (dimensions AS ['Other', 'Other']))`, + }, { + Context: inputContext{ + Start: time.Date(2022, 4, 9, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + StartForTableSelection: func() *time.Time { + t := time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC) + return &t + }(), + MainTableRequired: true, + Points: 100, + Units: "l3bps", + }, + Template: `SELECT 3 AS axis, * FROM ( SELECT {{ call .ToStartOfInterval "TimeReceived" }} + INTERVAL 86400 second AS time, {{ .Units }}/{{ .Interval }} AS xps, @@ -682,8 +801,9 @@ ORDER BY time WITH FILL FROM {{ .TimefilterStart }} + INTERVAL 86400 second TO {{ .TimefilterEnd }} + INTERVAL 1 second + INTERVAL 86400 second STEP {{ .Interval }} - INTERPOLATE (dimensions AS emptyArrayString())) -{{ end }}`, + INTERPOLATE (dimensions AS emptyArrayString()))`, + }, + }, }, } for _, tc := range cases { @@ -694,11 +814,9 @@ ORDER BY time WITH FILL if err := tc.Input.Filter.Validate(tc.Input.schema); err != nil { t.Fatalf("%sValidate() error:\n%+v", tc.Pos, err) } - tc.Expected = strings.ReplaceAll(tc.Expected, "@@", "`") t.Run(tc.Description, func(t *testing.T) { got := tc.Input.toSQL() - if diff := helpers.Diff(strings.Split(strings.TrimSpace(got), "\n"), - strings.Split(strings.TrimSpace(tc.Expected), "\n")); diff != "" { + if diff := helpers.Diff(got, tc.Expected); diff != "" { t.Errorf("%stoSQL (-got, +want):\n%s", tc.Pos, diff) } }) diff --git a/console/sankey.go b/console/sankey.go index 967f9369..bf1df70e 100644 --- a/console/sankey.go +++ b/console/sankey.go @@ -36,7 +36,7 @@ type sankeyLink struct { } // sankeyHandlerInputToSQL converts a sankey query to an SQL request -func (input graphSankeyHandlerInput) toSQL() (string, error) { +func (input graphSankeyHandlerInput) toSQL() ([]templateQuery, error) { where := templateWhere(input.Filter) // Select @@ -61,8 +61,7 @@ func (input graphSankeyHandlerInput) toSQL() (string, error) { } with = append(with, selectSankeyRowsByLimitType(input, dimensions, where)) - sqlQuery := fmt.Sprintf(` -{{ with %s }} + template := fmt.Sprintf(` WITH %s SELECT @@ -70,17 +69,21 @@ SELECT FROM source WHERE %s GROUP BY dimensions -ORDER BY xps DESC -{{ end }}`, - templateContext(inputContext{ - Start: input.Start, - End: input.End, - MainTableRequired: requireMainTable(input.schema, input.Dimensions, input.Filter), - Points: 20, - Units: input.Units, - }), +ORDER BY xps DESC`, strings.Join(with, ",\n "), strings.Join(fields, ",\n "), where) - return strings.TrimSpace(sqlQuery), nil + + context := inputContext{ + Start: input.Start, + End: input.End, + MainTableRequired: requireMainTable(input.schema, input.Dimensions, input.Filter), + Points: 20, + Units: input.Units, + } + + return []templateQuery{{ + Template: strings.TrimSpace(template), + Context: context, + }}, nil } func (c *Component) graphSankeyHandlerFunc(gc *gin.Context) { @@ -105,14 +108,14 @@ func (c *Component) graphSankeyHandlerFunc(gc *gin.Context) { return } - sqlQuery, err := input.toSQL() + queries, err := input.toSQL() if err != nil { gc.JSON(http.StatusBadRequest, gin.H{"message": helpers.Capitalize(err.Error())}) return } // Prepare and execute query - sqlQuery = c.finalizeQuery(sqlQuery) + sqlQuery := c.finalizeTemplateQueries(queries) gc.Header("X-SQL-Query", strings.ReplaceAll(sqlQuery, "\n", " ")) results := []struct { Xps float64 `ch:"xps"` diff --git a/console/sankey_test.go b/console/sankey_test.go index 89ec0338..af686e67 100644 --- a/console/sankey_test.go +++ b/console/sankey_test.go @@ -4,7 +4,6 @@ package console import ( - "strings" "testing" "time" @@ -21,7 +20,7 @@ func TestSankeyQuerySQL(t *testing.T) { Description string Pos helpers.Pos Input graphSankeyHandlerInput - Expected string + Expected []templateQuery }{ { Description: "two dimensions, no filters, l3 bps", @@ -39,9 +38,15 @@ func TestSankeyQuerySQL(t *testing.T) { Units: "l3bps", }, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":20,"units":"l3bps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 20, + Units: "l3bps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1), (SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE {{ .Timefilter }}) AS range, rows AS (SELECT SrcAS, ExporterName FROM source WHERE {{ .Timefilter }} GROUP BY SrcAS, ExporterName ORDER BY {{ .Units }} DESC LIMIT 5) @@ -52,8 +57,9 @@ SELECT FROM source WHERE {{ .Timefilter }} GROUP BY dimensions -ORDER BY xps DESC -{{ end }}`, +ORDER BY xps DESC`, + }, + }, }, { Description: "two dimensions, no filters, l3 bps, limitType by max", Pos: helpers.Mark(), @@ -71,9 +77,15 @@ ORDER BY xps DESC Units: "l3bps", }, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":20,"units":"l3bps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 20, + Units: "l3bps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1), (SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE {{ .Timefilter }}) AS range, rows AS (SELECT SrcAS, ExporterName FROM ( SELECT SrcAS, ExporterName, {{ .Units }} AS sum_at_time FROM source WHERE {{ .Timefilter }} GROUP BY SrcAS, ExporterName ) GROUP BY SrcAS, ExporterName ORDER BY MAX(sum_at_time) DESC LIMIT 5) @@ -84,8 +96,9 @@ SELECT FROM source WHERE {{ .Timefilter }} GROUP BY dimensions -ORDER BY xps DESC -{{ end }}`, +ORDER BY xps DESC`, + }, + }, }, { Description: "two dimensions, no filters, l2 bps", Pos: helpers.Mark(), @@ -102,9 +115,15 @@ ORDER BY xps DESC Units: "l2bps", }, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":20,"units":"l2bps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 20, + Units: "l2bps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1), (SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE {{ .Timefilter }}) AS range, rows AS (SELECT SrcAS, ExporterName FROM source WHERE {{ .Timefilter }} GROUP BY SrcAS, ExporterName ORDER BY {{ .Units }} DESC LIMIT 5) @@ -115,9 +134,9 @@ SELECT FROM source WHERE {{ .Timefilter }} GROUP BY dimensions -ORDER BY xps DESC -{{ end }} -`, +ORDER BY xps DESC`, + }, + }, }, { Description: "two dimensions, no filters, pps", Pos: helpers.Mark(), @@ -134,9 +153,15 @@ ORDER BY xps DESC Units: "pps", }, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":20,"units":"pps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 20, + Units: "pps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1), (SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE {{ .Timefilter }}) AS range, rows AS (SELECT SrcAS, ExporterName FROM source WHERE {{ .Timefilter }} GROUP BY SrcAS, ExporterName ORDER BY {{ .Units }} DESC LIMIT 5) @@ -147,8 +172,9 @@ SELECT FROM source WHERE {{ .Timefilter }} GROUP BY dimensions -ORDER BY xps DESC -{{ end }}`, +ORDER BY xps DESC`, + }, + }, }, { Description: "two dimensions, with filter", Pos: helpers.Mark(), @@ -165,9 +191,15 @@ ORDER BY xps DESC Units: "l3bps", }, }, - Expected: ` -{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":20,"units":"l3bps"}@@ }} -WITH + Expected: []templateQuery{ + { + Context: inputContext{ + Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC), + Points: 20, + Units: "l3bps", + }, + Template: `WITH source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1), (SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE {{ .Timefilter }} AND (DstCountry = 'FR')) AS range, rows AS (SELECT SrcAS, ExporterName FROM source WHERE {{ .Timefilter }} AND (DstCountry = 'FR') GROUP BY SrcAS, ExporterName ORDER BY {{ .Units }} DESC LIMIT 10) @@ -178,8 +210,9 @@ SELECT FROM source WHERE {{ .Timefilter }} AND (DstCountry = 'FR') GROUP BY dimensions -ORDER BY xps DESC -{{ end }}`, +ORDER BY xps DESC`, + }, + }, }, } for _, tc := range cases { @@ -190,11 +223,9 @@ ORDER BY xps DESC if err := tc.Input.Filter.Validate(tc.Input.schema); err != nil { t.Fatalf("%sValidate() error:\n%+v", tc.Pos, err) } - tc.Expected = strings.ReplaceAll(tc.Expected, "@@", "`") t.Run(tc.Description, func(t *testing.T) { got, _ := tc.Input.toSQL() - if diff := helpers.Diff(strings.Split(strings.TrimSpace(got), "\n"), - strings.Split(strings.TrimSpace(tc.Expected), "\n")); diff != "" { + if diff := helpers.Diff(got, tc.Expected); diff != "" { t.Errorf("%stoSQL (-got, +want):\n%s", tc.Pos, diff) } }) diff --git a/console/widgets.go b/console/widgets.go index 2370b83a..8d6646a5 100644 --- a/console/widgets.go +++ b/console/widgets.go @@ -192,8 +192,7 @@ func (c *Component) widgetTopHandlerFunc(gc *gin.Context) { } now := c.d.Clock.Now() - query := c.finalizeQuery(fmt.Sprintf(` -{{ with %s }} + template := fmt.Sprintf(` WITH (SELECT SUM(Bytes*SamplingRate) FROM {{ .Table }} WHERE {{ .Timefilter }} %s) AS Total SELECT @@ -204,15 +203,18 @@ WHERE {{ .Timefilter }} %s GROUP BY %s ORDER BY Percent DESC -LIMIT 5 -{{ end }}`, - templateContext(inputContext{ +LIMIT 5`, + filter, selector, selector, filter, groupby) + + query := c.finalizeTemplateQuery(templateQuery{ + Template: template, + Context: inputContext{ Start: now.Add(-5 * time.Minute), End: now, MainTableRequired: mainTableRequired, Points: 5, - }), - filter, selector, selector, filter, groupby)) + }, + }) gc.Header("X-SQL-Query", query) results := []topResult{} @@ -232,8 +234,7 @@ func (c *Component) widgetGraphHandlerFunc(gc *gin.Context) { } ctx := c.t.Context(gc.Request.Context()) now := c.d.Clock.Now() - query := c.finalizeQuery(fmt.Sprintf(` -{{ with %s }} + template := fmt.Sprintf(` SELECT {{ call .ToStartOfInterval "TimeReceived" }} AS Time, SUM(Bytes*SamplingRate*8/{{ .Interval }})/1000/1000/1000 AS Gbps @@ -244,15 +245,18 @@ GROUP BY Time ORDER BY Time WITH FILL FROM {{ .TimefilterStart }} TO {{ .TimefilterEnd }} + INTERVAL 1 second - STEP {{ .Interval }} -{{ end }}`, - templateContext(inputContext{ + STEP {{ .Interval }}`, + filter) + + query := c.finalizeTemplateQuery(templateQuery{ + Template: template, + Context: inputContext{ Start: now.Add(-c.config.HomepageGraphTimeRange), End: now, MainTableRequired: false, Points: 200, - }), - filter)) + }, + }) gc.Header("X-SQL-Query", query) results := []struct {