feat: configure limit type (#1482)

* Add limit type field selection

* Take into account LimitType to generate SQL request for Line graph

Also, add LimitType in default configuration for console

* Take into account LimitType to generate SQL request for Sankey graph

* Refactor on SQL query used by line and sankey

* Add limitType description in doc

* Order by max in graphLine when limitType max is used

* Fix query when using top max

Revert some modifications, as they were no longer relevant with the query fixed.

* Rework way to sort by max in line graph type

* Add configuration validation on LimitType

---------

Co-authored-by: Dimitri Baudrier <github.52grm@simplelogin.com>

console/config.go

@@ -45,6 +45,8 @@ type VisualizeOptionsConfiguration struct {
 	Dimensions []query.Column `json:"dimensions"`
 	// Limit is the default limit to use
 	Limit int `json:"limit" validate:"min=5"`
+	// LimitType is the default limitType to use
+	LimitType string `json:"limitType" validate:"oneof=Avg Max"`
 	// Bidirectional tells if a graph should be bidirectional (all except sankey)
 	Bidirectional bool `json:"bidirectional"`
 	// PreviousPeriod tells if a graph should display the previous period (for stacked)
@@ -61,6 +63,7 @@ func DefaultConfiguration() Configuration {
 			Filter:     "InIfBoundary = external",
 			Dimensions: []query.Column{query.NewColumn("SrcAS")},
 			Limit:      10,
+			LimitType:  "Avg",
 		},
 		HomepageTopWidgets:     []string{"src-as", "src-port", "protocol", "src-country", "etype"},
 		DimensionsLimit:        50,

console/config_test.go

@@ -26,6 +26,7 @@ func TestConfigHandler(t *testing.T) {
 					"filter":         "InIfBoundary = external",
 					"dimensions":     []string{"SrcAS"},
 					"limit":          10,
+					"limitType":      "Avg",
 					"bidirectional":  false,
 					"previousPeriod": false,
 				},

console/data/docs/02-configuration.md

@@ -913,7 +913,7 @@ The console itself accepts the following keys:
  - `default-visualize-options` to define default options for the "visualize"
    tab. It takes the following keys: `graph-type` (one of `stacked`,
    `stacked100`, `lines`, `grid`, or `sankey`), `start`, `end`, `filter`,
-   `dimensions` (a list), `limit`, `bidirectional` (a bool), `previous-period`
+   `dimensions` (a list), `limit`, `limitType`, `bidirectional` (a bool), `previous-period`
    (a bool)
  - `homepage-top-widgets` to define the widgets to display on the home page
    (among `src-as`, `dst-as`, `src-country`, `dst-country`, `exporter`,

console/data/docs/03-usage.md

@@ -151,6 +151,10 @@ aspect of the graph.
   "limit" parameter tells how many. The remaining values are
   categorized as "Other".
 
+- Associated with the `limit` parameter, the `limitType` parameter help find traffic surges according to 2 modes:
+  - Avg: default mode, the query focuses on getting the highest cumulative traffics over the time selection.
+  - Max: the query focuses on getting the traffic bursts over the time selection.
+
 - The filter box contains an SQL-like expression to limit the data to be
   graphed. It features an auto-completion system that can be triggered manually
   with `Ctrl-Space`. `Ctrl-Enter` executes the request. Filters can be saved by

console/frontend/src/components/InputDimensions.vue

@@ -61,12 +61,27 @@
         label="IPv6 /x"
         :error="truncate6Error"
       />
+    </div>
+    <div class="flex flex-row flex-nowrap gap-2">
       <InputString
         v-model="limit"
         class="grow"
         label="Limit"
         :error="limitError"
       />
+      <InputListBox
+        v-model="limitType"
+        :items="computationModeList"
+        class="order-3 grow basis-full sm:max-lg:order-3 sm:max-lg:basis-0"
+        label="Top by"
+      >
+        <template #selected>{{ limitType.name }}</template>
+        <template #item="{ name }">
+          <div class="flex w-full items-center justify-between">
+            <span>{{ name }}</span>
+          </div>
+        </template>
+      </InputListBox>
     </div>
   </div>
 </template>
@@ -117,6 +132,21 @@ const limitError = computed(() => {
   }
   return "";
 });
+
+const computationModes = {
+  avg: "Avg",
+  max: "Max",
+} as const;
+
+const computationModeList = Object.entries(computationModes).map(
+  ([k, v], idx) => ({
+    id: idx + 1,
+    type: k as keyof typeof computationModes, // why isn't it infered?
+    name: v,
+  }),
+);
+const limitType = ref(computationModeList[0]);
+
 const canAggregate = computed(
   () =>
     intersection(
@@ -147,7 +177,6 @@ const hasErrors = computed(
     !!truncate4Error.value ||
     !!truncate6Error.value,
 );
-
 const dimensions = computed(
   () =>
     serverConfiguration.value?.dimensions.map((v, idx) => ({
@@ -166,12 +195,14 @@ const removeDimension = (dimension: (typeof dimensions.value)[0]) => {
     (d) => d !== dimension,
   );
 };
-
 watch(
   () => [props.modelValue, dimensions.value] as const,
   ([value, dimensions]) => {
     if (value) {
       limit.value = value.limit.toString();
+      limitType.value =
+        computationModeList.find((mode) => mode.name === value.limitType) ||
+        computationModeList[0];
       truncate4.value = value.truncate4.toString();
       truncate6.value = value.truncate6.toString();
     }
@@ -183,11 +214,19 @@ watch(
   { immediate: true, deep: true },
 );
 watch(
-  [selectedDimensions, limit, truncate4, truncate6, hasErrors] as const,
+  [
-  ([selected, limit, truncate4, truncate6, hasErrors]) => {
+    selectedDimensions,
+    limit,
+    limitType,
+    truncate4,
+    truncate6,
+    hasErrors,
+  ] as const,
+  ([selected, limit, limitType, truncate4, truncate6, hasErrors]) => {
     const updated = {
       selected: selected.map((d) => d.name),
       limit: parseInt(limit),
+      limitType: limitType.name,
       truncate4: parseInt(truncate4),
       truncate6: parseInt(truncate6),
       errors: hasErrors,
@@ -208,6 +247,7 @@ watch(
 export type ModelType = {
   selected: string[];
   limit: number;
+  limitType: string;
   truncate4: number;
   truncate6: number;
   errors?: boolean;

console/frontend/src/components/ServerConfigProvider.vue

@@ -29,6 +29,7 @@ type ServerConfig = {
     filter: string;
     dimensions: string[];
     limit: number;
+    limitType: string;
     bidirectional: boolean;
     previousPeriod: boolean;
   };

console/frontend/src/views/VisualizePage/OptionsPanel.vue

@@ -192,6 +192,7 @@ const options = computed((): InternalModelType => {
     humanEnd: timeRange.value?.end,
     dimensions: dimensions.value?.selected,
     limit: dimensions.value?.limit,
+    limitType: dimensions.value?.limitType,
     "truncate-v4": dimensions.value?.truncate4,
     "truncate-v6": dimensions.value?.truncate6,
     filter: filter.value?.expression,
@@ -243,6 +244,7 @@ watch(
       humanEnd: defaultOptions.end,
       dimensions: toRaw(defaultOptions.dimensions),
       limit: defaultOptions.limit,
+      limitType: defaultOptions.limitType,
       "truncate-v4": 32,
       "truncate-v6": 128,
       filter: defaultOptions.filter,
@@ -262,6 +264,7 @@ watch(
     dimensions.value = {
       selected: [...currentValue.dimensions],
       limit: currentValue.limit,
+      limitType: currentValue.limitType,
       truncate4: currentValue["truncate-v4"] || 32,
       truncate6: currentValue["truncate-v6"] || 128,
     };
@@ -296,6 +299,7 @@ export type ModelType = {
   humanEnd: string;
   dimensions: string[];
   limit: number;
+  limitType: string;
   "truncate-v4": number;
   "truncate-v6": number;
   filter: string;

console/graph.go

@@ -18,8 +18,9 @@ type graphCommonHandlerInput struct {
 	schema         *schema.Component
 	Start          time.Time      `json:"start" binding:"required"`
 	End            time.Time      `json:"end" binding:"required,gtfield=Start"`
-	Dimensions     []query.Column `json:"dimensions"`                          // group by ...
+	Dimensions     []query.Column `json:"dimensions"`            // group by ...
-	Limit          int            `json:"limit" binding:"min=1"`               // limit product of dimensions
+	Limit          int            `json:"limit" binding:"min=1"` // limit product of dimensions
+	LimitType      string         `json:"limitType" validate:"oneof=Avg Max"`
 	Filter         query.Filter   `json:"filter"`                              // where ...
 	TruncateAddrV4 int            `json:"truncate-v4" binding:"min=0,max=32"`  // 0 or 32 = no truncation
 	TruncateAddrV6 int            `json:"truncate-v6" binding:"min=0,max=128"` // 0 or 128 = no truncation

console/line.go

@@ -139,12 +139,7 @@ func (input graphLineHandlerInput) toSQL1(axis int, options toSQL1Options) strin
 	if !options.skipWithClause {
 		with := []string{fmt.Sprintf("source AS (%s)", input.sourceSelect())}
 		if len(dimensions) > 0 {
-			with = append(with, fmt.Sprintf(
+			with = append(with, selectLineRowsByLimitType(input, dimensions, where))
-				"rows AS (SELECT %s FROM source WHERE %s GROUP BY %s ORDER BY {{ .Units }} DESC LIMIT %d)",
-				strings.Join(dimensions, ", "),
-				where,
-				strings.Join(dimensions, ", "),
-				input.Limit))
 		}
 		if len(with) > 0 {
 			withStr = fmt.Sprintf("\nWITH\n %s", strings.Join(with, ",\n "))
@@ -289,6 +284,7 @@ func (c *Component) graphLineHandlerFunc(gc *gin.Context) {
 	rows := map[int]map[string][]string{} // for each axis, a map from row to list of dimensions
 	points := map[int]map[string][]int{}  // for each axis, a map from row to list of points (one point per ts)
 	sums := map[int]map[string]uint64{}   // for each axis, a map from row to sum (for sorting purpose)
+	maxes := map[int]map[string]uint64{}  // for each axis, a map from row to max (for sorting purpose)
 	lastTimeForAxis := map[int]time.Time{}
 	timeIndexForAxis := map[int]int{}
 	for _, result := range results {
@@ -303,6 +299,7 @@ func (c *Component) graphLineHandlerFunc(gc *gin.Context) {
 			rows[axis] = map[string][]string{}
 			points[axis] = map[string][]int{}
 			sums[axis] = map[string]uint64{}
+			maxes[axis] = map[string]uint64{}
 		}
 		if result.Time != lastTime {
 			// New timestamp, increment time index
@@ -317,9 +314,13 @@ func (c *Component) graphLineHandlerFunc(gc *gin.Context) {
 			row := make([]int, len(output.Time))
 			points[axis][rowKey] = row
 			sums[axis][rowKey] = 0
+			maxes[axis][rowKey] = 0
 		}
 		points[axis][rowKey][timeIndexForAxis[axis]] = int(result.Xps)
 		sums[axis][rowKey] += uint64(result.Xps)
+		if uint64(result.Xps) > maxes[axis][rowKey] {
+			maxes[axis][rowKey] = uint64(result.Xps)
+		}
 	}
 	// Sort axes
 	sort.Ints(axes)
@@ -339,6 +340,10 @@ func (c *Component) graphLineHandlerFunc(gc *gin.Context) {
 			if rows[axis][jKey][0] == "Other" {
 				return true
 			}
+			if input.LimitType == "Max" {
+				return maxes[axis][iKey] > maxes[axis][jKey]
+			}
+
 			return sums[axis][iKey] > sums[axis][jKey]
 		})
 	}

console/query.go

@@ -4,6 +4,7 @@
 package console
 
 import (
+	"fmt"
 	"strings"
 
 	"akvorado/common/schema"
@@ -32,3 +33,36 @@ func (c *Component) fixQueryColumnName(name string) string {
 	}
 	return ""
 }
+
+func selectSankeyRowsByLimitType(input graphSankeyHandlerInput, dimensions []string, where string) string {
+	return selectRowsByLimitType(input.graphCommonHandlerInput, dimensions, where)
+}
+
+func selectLineRowsByLimitType(input graphLineHandlerInput, dimensions []string, where string) string {
+	return selectRowsByLimitType(input.graphCommonHandlerInput, dimensions, where)
+}
+
+func selectRowsByLimitType(input graphCommonHandlerInput, dimensions []string, where string) string {
+	var rowsType string
+	var source string
+	var orderBy string
+	if input.LimitType == "Max" {
+		source = fmt.Sprintf("( SELECT %s AS sum_at_time FROM source WHERE %s GROUP BY %s )",
+			strings.Join(append(dimensions, "{{ .Units }}"), ", "),
+			where,
+			strings.Join(dimensions, ", "),
+		)
+		orderBy = "MAX(sum_at_time)"
+	} else {
+		source = fmt.Sprintf("source WHERE %s", where)
+		orderBy = "{{ .Units }}"
+	}
+	rowsType = fmt.Sprintf(
+		"rows AS (SELECT %s FROM %s GROUP BY %s ORDER BY %s DESC LIMIT %d)",
+		strings.Join(dimensions, ", "),
+		source,
+		strings.Join(dimensions, ", "),
+		orderBy,
+		input.Limit)
+	return rowsType
+}

console/sankey.go

@@ -57,14 +57,8 @@ func (input graphSankeyHandlerInput) toSQL() (string, error) {
 	// With
 	with := []string{
 		fmt.Sprintf("source AS (%s)", input.sourceSelect()),
-		fmt.Sprintf(`(SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE %s) AS range`, where),
+		fmt.Sprintf(`(SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE %s) AS range`, where)}
-		fmt.Sprintf(
+	with = append(with, selectSankeyRowsByLimitType(input, dimensions, where))
-			"rows AS (SELECT %s FROM source WHERE %s GROUP BY %s ORDER BY {{ .Units }} DESC LIMIT %d)",
-			strings.Join(dimensions, ", "),
-			where,
-			strings.Join(dimensions, ", "),
-			input.Limit),
-	}
 
 	sqlQuery := fmt.Sprintf(`
 {{ with %s }}

console/sankey_test.go

@@ -53,6 +53,38 @@ FROM source
 WHERE {{ .Timefilter }}
 GROUP BY dimensions
 ORDER BY xps DESC
+{{ end }}`,
+		}, {
+			Description: "two dimensions, no filters, l3 bps, limitType by max",
+			Pos:         helpers.Mark(),
+			Input: graphSankeyHandlerInput{
+				graphCommonHandlerInput{
+					Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
+					End:   time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
+					Dimensions: []query.Column{
+						query.NewColumn("SrcAS"),
+						query.NewColumn("ExporterName"),
+					},
+					Limit:     5,
+					LimitType: "Max",
+					Filter:    query.Filter{},
+					Units:     "l3bps",
+				},
+			},
+			Expected: `
+{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":20,"units":"l3bps"}@@ }}
+WITH
+ source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1),
+ (SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE {{ .Timefilter }}) AS range,
+ rows AS (SELECT SrcAS, ExporterName FROM ( SELECT SrcAS, ExporterName, {{ .Units }} AS sum_at_time FROM source WHERE {{ .Timefilter }} GROUP BY SrcAS, ExporterName ) GROUP BY SrcAS, ExporterName ORDER BY MAX(sum_at_time) DESC LIMIT 5)
+SELECT
+ {{ .Units }}/range AS xps,
+ [if(SrcAS IN (SELECT SrcAS FROM rows), concat(toString(SrcAS), ': ', dictGetOrDefault('asns', 'name', SrcAS, '???')), 'Other'),
+  if(ExporterName IN (SELECT ExporterName FROM rows), ExporterName, 'Other')] AS dimensions
+FROM source
+WHERE {{ .Timefilter }}
+GROUP BY dimensions
+ORDER BY xps DESC
 {{ end }}`,
 		}, {
 			Description: "two dimensions, no filters, l2 bps",