console: fix intermittent failure when requesting previous period

Notably when the main table is required, but also on rare conditions
when another table would be selected because of the interval selection.

This is not perfect as sometimes, we won't have the data.

console/clickhouse.go

@@ -215,19 +215,18 @@ func (c *Component) contextFunc(inputStr string) context {
 
 func (c *Component) computeTableAndInterval(input inputContext) (string, time.Duration, time.Duration) {
 	targetInterval := time.Duration(uint64(input.End.Sub(input.Start)) / uint64(input.Points))
-	if targetInterval < time.Second {
+	targetInterval = max(targetInterval, time.Second)
-		targetInterval = time.Second
-	}
 
 	// Select table
 	targetIntervalForTableSelection := targetInterval
 	if input.MainTableRequired {
 		targetIntervalForTableSelection = time.Second
 	}
-	table, computedInterval := c.getBestTable(input.Start, targetIntervalForTableSelection)
+	startForTableSelection := input.Start
 	if input.StartForInterval != nil {
-		_, computedInterval = c.getBestTable(*input.StartForInterval, targetIntervalForTableSelection)
+		startForTableSelection = *input.StartForInterval
 	}
+	table, computedInterval := c.getBestTable(startForTableSelection, targetIntervalForTableSelection)
 	return table, computedInterval, targetInterval
 }
 

console/clickhouse_test.go

@@ -183,7 +183,7 @@ func TestFinalizeQuery(t *testing.T) {
 			},
 			Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC') // 30",
 		}, {
-			Description: "use flows table for resolution (but flows_1m0s for data)",
+			Description: "use flows table for resolution and for data",
 			Tables: []flowsTable{
 				{"flows", 0, time.Date(2022, 4, 10, 10, 45, 10, 0, time.UTC)},
 				{"flows_1m0s", time.Minute, time.Date(2022, 3, 10, 10, 45, 10, 0, time.UTC)},
@@ -198,7 +198,7 @@ func TestFinalizeQuery(t *testing.T) {
 				}(),
 				Points: 2880, // 30-second resolution
 			},
-			Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-03-10 15:45:10', 'UTC') AND toDateTime('2022-03-11 15:45:10', 'UTC') // 30",
+			Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-03-10 15:45:10', 'UTC') AND toDateTime('2022-03-11 15:45:10', 'UTC') // 30",
 		}, {
 			Description: "select flows table with better resolution",
 			Tables: []flowsTable{

console/data/docs/99-changelog.md

@@ -14,6 +14,7 @@ identified with a specific icon:
 ## Unreleased
 
 - 🩹 *console*: fix deletion of saved filters
+- 🩹 *console*: fix intermittent failure when requesting previous period
 - 🩹 *docker*: move healthcheck for IPinfo updater into Dockerfile to avoid
   "unhealthy" state on non-updated installation
 - 🌱 *docker*: enable access log for Traefik

console/line.go

@@ -94,9 +94,10 @@ func (input graphLineHandlerInput) previousPeriod() graphLineHandlerInput {
 }
 
 type toSQL1Options struct {
-	skipWithClause   bool
+	skipWithClause    bool
-	reverseDirection bool
+	reverseDirection  bool
-	offsetedStart    time.Time
+	offsetedStart     time.Time
+	mainTableRequired bool
 }
 
 func (input graphLineHandlerInput) toSQL1(axis int, options toSQL1Options) string {
@@ -176,7 +177,7 @@ ORDER BY time WITH FILL
 			Start:             input.Start,
 			End:               input.End,
 			StartForInterval:  startForInterval,
-			MainTableRequired: requireMainTable(input.schema, input.Dimensions, input.Filter),
+			MainTableRequired: options.mainTableRequired,
 			Points:            input.Points,
 			Units:             units,
 		}),
@@ -188,26 +189,33 @@ ORDER BY time WITH FILL
 
 // toSQL converts a graph input to an SQL request
 func (input graphLineHandlerInput) toSQL() string {
-	parts := []string{input.toSQL1(1, toSQL1Options{})}
+	// Calculate mainTableRequired once and use it for all axes to ensure
-	// Handle specific options. We have to align time periods in
+	// consistency. This is useful as previous period will remove the
-	// case the previous period does not use the same offsets.
+	// dimensions.
+	mainTableRequired := requireMainTable(input.schema, input.Dimensions, input.Filter)
+	parts := []string{input.toSQL1(1, toSQL1Options{
+		mainTableRequired: mainTableRequired,
+	})}
 	if input.Bidirectional {
 		parts = append(parts, input.reverseDirection().toSQL1(2, toSQL1Options{
-			skipWithClause:   true,
+			skipWithClause:    true,
-			reverseDirection: true,
+			reverseDirection:  true,
+			mainTableRequired: mainTableRequired,
 		}))
 	}
 	if input.PreviousPeriod {
 		parts = append(parts, input.previousPeriod().toSQL1(3, toSQL1Options{
-			skipWithClause: true,
+			skipWithClause:    true,
-			offsetedStart:  input.Start,
+			offsetedStart:     input.Start,
+			mainTableRequired: mainTableRequired,
 		}))
 	}
 	if input.Bidirectional && input.PreviousPeriod {
 		parts = append(parts, input.reverseDirection().previousPeriod().toSQL1(4, toSQL1Options{
-			skipWithClause:   true,
+			skipWithClause:    true,
-			reverseDirection: true,
+			reverseDirection:  true,
-			offsetedStart:    input.Start,
+			offsetedStart:     input.Start,
+			mainTableRequired: mainTableRequired,
 		}))
 	}
 	return strings.Join(parts, "\nUNION ALL\n")

console/line_test.go

@@ -624,6 +624,58 @@ SELECT
 FROM source
 WHERE {{ .Timefilter }}
 GROUP BY time, dimensions
+ORDER BY time WITH FILL
+ FROM {{ .TimefilterStart }} + INTERVAL 86400 second
+ TO {{ .TimefilterEnd }} + INTERVAL 1 second + INTERVAL 86400 second
+ STEP {{ .Interval }}
+ INTERPOLATE (dimensions AS emptyArrayString()))
+{{ end }}`,
+		}, {
+			Description: "previous period while main table is required",
+			Pos:         helpers.Mark(),
+			Input: graphLineHandlerInput{
+				graphCommonHandlerInput: graphCommonHandlerInput{
+					Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
+					End:   time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
+					Dimensions: []query.Column{
+						query.NewColumn("SrcAddr"),
+						query.NewColumn("DstAddr"),
+					},
+					Filter: query.NewFilter("InIfBoundary = external"),
+					Units:  "l3bps",
+				},
+				Points:         100,
+				PreviousPeriod: true,
+			},
+			Expected: `
+{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","main-table-required":true,"points":100,"units":"l3bps"}@@ }}
+WITH
+ source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1),
+ rows AS (SELECT SrcAddr, DstAddr FROM source WHERE {{ .Timefilter }} AND (InIfBoundary = 'external') GROUP BY SrcAddr, DstAddr ORDER BY {{ .Units }} DESC LIMIT 0)
+SELECT 1 AS axis, * FROM (
+SELECT
+ {{ call .ToStartOfInterval "TimeReceived" }} AS time,
+ {{ .Units }}/{{ .Interval }} AS xps,
+ if((SrcAddr, DstAddr) IN rows, [replaceRegexpOne(IPv6NumToString(SrcAddr), '^::ffff:', ''), replaceRegexpOne(IPv6NumToString(DstAddr), '^::ffff:', '')], ['Other', 'Other']) AS dimensions
+FROM source
+WHERE {{ .Timefilter }} AND (InIfBoundary = 'external')
+GROUP BY time, dimensions
+ORDER BY time WITH FILL
+ FROM {{ .TimefilterStart }}
+ TO {{ .TimefilterEnd }} + INTERVAL 1 second
+ STEP {{ .Interval }}
+ INTERPOLATE (dimensions AS ['Other', 'Other']))
+{{ end }}
+UNION ALL
+{{ with context @@{"start":"2022-04-09T15:45:10Z","end":"2022-04-10T15:45:10Z","start-for-interval":"2022-04-10T15:45:10Z","main-table-required":true,"points":100,"units":"l3bps"}@@ }}
+SELECT 3 AS axis, * FROM (
+SELECT
+ {{ call .ToStartOfInterval "TimeReceived" }} + INTERVAL 86400 second AS time,
+ {{ .Units }}/{{ .Interval }} AS xps,
+ emptyArrayString() AS dimensions
+FROM source
+WHERE {{ .Timefilter }} AND (InIfBoundary = 'external')
+GROUP BY time, dimensions
 ORDER BY time WITH FILL
  FROM {{ .TimefilterStart }} + INTERVAL 86400 second
  TO {{ .TimefilterEnd }} + INTERVAL 1 second + INTERVAL 86400 second