mirror of
https://github.com/akvorado/akvorado.git
synced 2025-12-12 06:24:10 +01:00
feat: configure limit type (#1482)
* Add limit type field selection * Take into account LimitType to generate SQL request for Line graph Also, add LimitType in default configuration for console * Take into account LimitType to generate SQL request for Sankey graph * Refactor on SQL query used by line and sankey * Add limitType description in doc * Order by max in graphLine when limitType max is used * Fix query when using top max Revert some modifications, as they were no longer relevant with the query fixed. * Rework way to sort by max in line graph type * Add configuration validation on LimitType --------- Co-authored-by: Dimitri Baudrier <github.52grm@simplelogin.com>
This commit is contained in:
dimbdr
GitHub
@@ -45,6 +45,8 @@ type VisualizeOptionsConfiguration struct {
|
||||
Dimensions []query.Column `json:"dimensions"`
|
||||
// Limit is the default limit to use
|
||||
Limit int `json:"limit" validate:"min=5"`
|
||||
// LimitType is the default limitType to use
|
||||
LimitType string `json:"limitType" validate:"oneof=Avg Max"`
|
||||
// Bidirectional tells if a graph should be bidirectional (all except sankey)
|
||||
Bidirectional bool `json:"bidirectional"`
|
||||
// PreviousPeriod tells if a graph should display the previous period (for stacked)
|
||||
@@ -61,6 +63,7 @@ func DefaultConfiguration() Configuration {
|
||||
Filter: "InIfBoundary = external",
|
||||
Dimensions: []query.Column{query.NewColumn("SrcAS")},
|
||||
Limit: 10,
|
||||
LimitType: "Avg",
|
||||
},
|
||||
HomepageTopWidgets: []string{"src-as", "src-port", "protocol", "src-country", "etype"},
|
||||
DimensionsLimit: 50,
|
||||
|
||||
@@ -26,6 +26,7 @@ func TestConfigHandler(t *testing.T) {
|
||||
"filter": "InIfBoundary = external",
|
||||
"dimensions": []string{"SrcAS"},
|
||||
"limit": 10,
|
||||
"limitType": "Avg",
|
||||
"bidirectional": false,
|
||||
"previousPeriod": false,
|
||||
},
|
||||
|
||||
@@ -913,7 +913,7 @@ The console itself accepts the following keys:
|
||||
- `default-visualize-options` to define default options for the "visualize"
|
||||
tab. It takes the following keys: `graph-type` (one of `stacked`,
|
||||
`stacked100`, `lines`, `grid`, or `sankey`), `start`, `end`, `filter`,
|
||||
`dimensions` (a list), `limit`, `bidirectional` (a bool), `previous-period`
|
||||
`dimensions` (a list), `limit`, `limitType`, `bidirectional` (a bool), `previous-period`
|
||||
(a bool)
|
||||
- `homepage-top-widgets` to define the widgets to display on the home page
|
||||
(among `src-as`, `dst-as`, `src-country`, `dst-country`, `exporter`,
|
||||
|
||||
@@ -151,6 +151,10 @@ aspect of the graph.
|
||||
"limit" parameter tells how many. The remaining values are
|
||||
categorized as "Other".
|
||||
|
||||
- Associated with the `limit` parameter, the `limitType` parameter help find traffic surges according to 2 modes:
|
||||
- Avg: default mode, the query focuses on getting the highest cumulative traffics over the time selection.
|
||||
- Max: the query focuses on getting the traffic bursts over the time selection.
|
||||
|
||||
- The filter box contains an SQL-like expression to limit the data to be
|
||||
graphed. It features an auto-completion system that can be triggered manually
|
||||
with `Ctrl-Space`. `Ctrl-Enter` executes the request. Filters can be saved by
|
||||
|
||||
@@ -61,12 +61,27 @@
|
||||
label="IPv6 /x"
|
||||
:error="truncate6Error"
|
||||
/>
|
||||
</div>
|
||||
<div class="flex flex-row flex-nowrap gap-2">
|
||||
<InputString
|
||||
v-model="limit"
|
||||
class="grow"
|
||||
label="Limit"
|
||||
:error="limitError"
|
||||
/>
|
||||
<InputListBox
|
||||
v-model="limitType"
|
||||
:items="computationModeList"
|
||||
class="order-3 grow basis-full sm:max-lg:order-3 sm:max-lg:basis-0"
|
||||
label="Top by"
|
||||
>
|
||||
<template #selected>{{ limitType.name }}</template>
|
||||
<template #item="{ name }">
|
||||
<div class="flex w-full items-center justify-between">
|
||||
<span>{{ name }}</span>
|
||||
</div>
|
||||
</template>
|
||||
</InputListBox>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -117,6 +132,21 @@ const limitError = computed(() => {
|
||||
}
|
||||
return "";
|
||||
});
|
||||
|
||||
const computationModes = {
|
||||
avg: "Avg",
|
||||
max: "Max",
|
||||
} as const;
|
||||
|
||||
const computationModeList = Object.entries(computationModes).map(
|
||||
([k, v], idx) => ({
|
||||
id: idx + 1,
|
||||
type: k as keyof typeof computationModes, // why isn't it infered?
|
||||
name: v,
|
||||
}),
|
||||
);
|
||||
const limitType = ref(computationModeList[0]);
|
||||
|
||||
const canAggregate = computed(
|
||||
() =>
|
||||
intersection(
|
||||
@@ -147,7 +177,6 @@ const hasErrors = computed(
|
||||
!!truncate4Error.value ||
|
||||
!!truncate6Error.value,
|
||||
);
|
||||
|
||||
const dimensions = computed(
|
||||
() =>
|
||||
serverConfiguration.value?.dimensions.map((v, idx) => ({
|
||||
@@ -166,12 +195,14 @@ const removeDimension = (dimension: (typeof dimensions.value)[0]) => {
|
||||
(d) => d !== dimension,
|
||||
);
|
||||
};
|
||||
|
||||
watch(
|
||||
() => [props.modelValue, dimensions.value] as const,
|
||||
([value, dimensions]) => {
|
||||
if (value) {
|
||||
limit.value = value.limit.toString();
|
||||
limitType.value =
|
||||
computationModeList.find((mode) => mode.name === value.limitType) ||
|
||||
computationModeList[0];
|
||||
truncate4.value = value.truncate4.toString();
|
||||
truncate6.value = value.truncate6.toString();
|
||||
}
|
||||
@@ -183,11 +214,19 @@ watch(
|
||||
{ immediate: true, deep: true },
|
||||
);
|
||||
watch(
|
||||
[selectedDimensions, limit, truncate4, truncate6, hasErrors] as const,
|
||||
([selected, limit, truncate4, truncate6, hasErrors]) => {
|
||||
[
|
||||
selectedDimensions,
|
||||
limit,
|
||||
limitType,
|
||||
truncate4,
|
||||
truncate6,
|
||||
hasErrors,
|
||||
] as const,
|
||||
([selected, limit, limitType, truncate4, truncate6, hasErrors]) => {
|
||||
const updated = {
|
||||
selected: selected.map((d) => d.name),
|
||||
limit: parseInt(limit),
|
||||
limitType: limitType.name,
|
||||
truncate4: parseInt(truncate4),
|
||||
truncate6: parseInt(truncate6),
|
||||
errors: hasErrors,
|
||||
@@ -208,6 +247,7 @@ watch(
|
||||
export type ModelType = {
|
||||
selected: string[];
|
||||
limit: number;
|
||||
limitType: string;
|
||||
truncate4: number;
|
||||
truncate6: number;
|
||||
errors?: boolean;
|
||||
|
||||
@@ -29,6 +29,7 @@ type ServerConfig = {
|
||||
filter: string;
|
||||
dimensions: string[];
|
||||
limit: number;
|
||||
limitType: string;
|
||||
bidirectional: boolean;
|
||||
previousPeriod: boolean;
|
||||
};
|
||||
|
||||
@@ -192,6 +192,7 @@ const options = computed((): InternalModelType => {
|
||||
humanEnd: timeRange.value?.end,
|
||||
dimensions: dimensions.value?.selected,
|
||||
limit: dimensions.value?.limit,
|
||||
limitType: dimensions.value?.limitType,
|
||||
"truncate-v4": dimensions.value?.truncate4,
|
||||
"truncate-v6": dimensions.value?.truncate6,
|
||||
filter: filter.value?.expression,
|
||||
@@ -243,6 +244,7 @@ watch(
|
||||
humanEnd: defaultOptions.end,
|
||||
dimensions: toRaw(defaultOptions.dimensions),
|
||||
limit: defaultOptions.limit,
|
||||
limitType: defaultOptions.limitType,
|
||||
"truncate-v4": 32,
|
||||
"truncate-v6": 128,
|
||||
filter: defaultOptions.filter,
|
||||
@@ -262,6 +264,7 @@ watch(
|
||||
dimensions.value = {
|
||||
selected: [...currentValue.dimensions],
|
||||
limit: currentValue.limit,
|
||||
limitType: currentValue.limitType,
|
||||
truncate4: currentValue["truncate-v4"] || 32,
|
||||
truncate6: currentValue["truncate-v6"] || 128,
|
||||
};
|
||||
@@ -296,6 +299,7 @@ export type ModelType = {
|
||||
humanEnd: string;
|
||||
dimensions: string[];
|
||||
limit: number;
|
||||
limitType: string;
|
||||
"truncate-v4": number;
|
||||
"truncate-v6": number;
|
||||
filter: string;
|
||||
|
||||
@@ -20,6 +20,7 @@ type graphCommonHandlerInput struct {
|
||||
End time.Time `json:"end" binding:"required,gtfield=Start"`
|
||||
Dimensions []query.Column `json:"dimensions"` // group by ...
|
||||
Limit int `json:"limit" binding:"min=1"` // limit product of dimensions
|
||||
LimitType string `json:"limitType" validate:"oneof=Avg Max"`
|
||||
Filter query.Filter `json:"filter"` // where ...
|
||||
TruncateAddrV4 int `json:"truncate-v4" binding:"min=0,max=32"` // 0 or 32 = no truncation
|
||||
TruncateAddrV6 int `json:"truncate-v6" binding:"min=0,max=128"` // 0 or 128 = no truncation
|
||||
|
||||
@@ -139,12 +139,7 @@ func (input graphLineHandlerInput) toSQL1(axis int, options toSQL1Options) strin
|
||||
if !options.skipWithClause {
|
||||
with := []string{fmt.Sprintf("source AS (%s)", input.sourceSelect())}
|
||||
if len(dimensions) > 0 {
|
||||
with = append(with, fmt.Sprintf(
|
||||
"rows AS (SELECT %s FROM source WHERE %s GROUP BY %s ORDER BY {{ .Units }} DESC LIMIT %d)",
|
||||
strings.Join(dimensions, ", "),
|
||||
where,
|
||||
strings.Join(dimensions, ", "),
|
||||
input.Limit))
|
||||
with = append(with, selectLineRowsByLimitType(input, dimensions, where))
|
||||
}
|
||||
if len(with) > 0 {
|
||||
withStr = fmt.Sprintf("\nWITH\n %s", strings.Join(with, ",\n "))
|
||||
@@ -289,6 +284,7 @@ func (c *Component) graphLineHandlerFunc(gc *gin.Context) {
|
||||
rows := map[int]map[string][]string{} // for each axis, a map from row to list of dimensions
|
||||
points := map[int]map[string][]int{} // for each axis, a map from row to list of points (one point per ts)
|
||||
sums := map[int]map[string]uint64{} // for each axis, a map from row to sum (for sorting purpose)
|
||||
maxes := map[int]map[string]uint64{} // for each axis, a map from row to max (for sorting purpose)
|
||||
lastTimeForAxis := map[int]time.Time{}
|
||||
timeIndexForAxis := map[int]int{}
|
||||
for _, result := range results {
|
||||
@@ -303,6 +299,7 @@ func (c *Component) graphLineHandlerFunc(gc *gin.Context) {
|
||||
rows[axis] = map[string][]string{}
|
||||
points[axis] = map[string][]int{}
|
||||
sums[axis] = map[string]uint64{}
|
||||
maxes[axis] = map[string]uint64{}
|
||||
}
|
||||
if result.Time != lastTime {
|
||||
// New timestamp, increment time index
|
||||
@@ -317,9 +314,13 @@ func (c *Component) graphLineHandlerFunc(gc *gin.Context) {
|
||||
row := make([]int, len(output.Time))
|
||||
points[axis][rowKey] = row
|
||||
sums[axis][rowKey] = 0
|
||||
maxes[axis][rowKey] = 0
|
||||
}
|
||||
points[axis][rowKey][timeIndexForAxis[axis]] = int(result.Xps)
|
||||
sums[axis][rowKey] += uint64(result.Xps)
|
||||
if uint64(result.Xps) > maxes[axis][rowKey] {
|
||||
maxes[axis][rowKey] = uint64(result.Xps)
|
||||
}
|
||||
}
|
||||
// Sort axes
|
||||
sort.Ints(axes)
|
||||
@@ -339,6 +340,10 @@ func (c *Component) graphLineHandlerFunc(gc *gin.Context) {
|
||||
if rows[axis][jKey][0] == "Other" {
|
||||
return true
|
||||
}
|
||||
if input.LimitType == "Max" {
|
||||
return maxes[axis][iKey] > maxes[axis][jKey]
|
||||
}
|
||||
|
||||
return sums[axis][iKey] > sums[axis][jKey]
|
||||
})
|
||||
}
|
||||
|
||||
@@ -481,6 +481,43 @@ SELECT
|
||||
FROM source
|
||||
WHERE {{ .Timefilter }}
|
||||
GROUP BY time, dimensions
|
||||
ORDER BY time WITH FILL
|
||||
FROM {{ .TimefilterStart }}
|
||||
TO {{ .TimefilterEnd }} + INTERVAL 1 second
|
||||
STEP {{ .Interval }}
|
||||
INTERPOLATE (dimensions AS ['Other', 'Other']))
|
||||
{{ end }}`,
|
||||
}, {
|
||||
Description: "no filters, limitType by max",
|
||||
Pos: helpers.Mark(),
|
||||
Input: graphLineHandlerInput{
|
||||
graphCommonHandlerInput: graphCommonHandlerInput{
|
||||
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
|
||||
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
|
||||
Limit: 20,
|
||||
LimitType: "Max",
|
||||
Dimensions: []query.Column{
|
||||
query.NewColumn("ExporterName"),
|
||||
query.NewColumn("InIfProvider"),
|
||||
},
|
||||
Filter: query.Filter{},
|
||||
Units: "l3bps",
|
||||
},
|
||||
Points: 100,
|
||||
},
|
||||
Expected: `
|
||||
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
|
||||
WITH
|
||||
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1),
|
||||
rows AS (SELECT ExporterName, InIfProvider FROM ( SELECT ExporterName, InIfProvider, {{ .Units }} AS sum_at_time FROM source WHERE {{ .Timefilter }} GROUP BY ExporterName, InIfProvider ) GROUP BY ExporterName, InIfProvider ORDER BY MAX(sum_at_time) DESC LIMIT 20)
|
||||
SELECT 1 AS axis, * FROM (
|
||||
SELECT
|
||||
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
|
||||
{{ .Units }}/{{ .Interval }} AS xps,
|
||||
if((ExporterName, InIfProvider) IN rows, [ExporterName, InIfProvider], ['Other', 'Other']) AS dimensions
|
||||
FROM source
|
||||
WHERE {{ .Timefilter }}
|
||||
GROUP BY time, dimensions
|
||||
ORDER BY time WITH FILL
|
||||
FROM {{ .TimefilterStart }}
|
||||
TO {{ .TimefilterEnd }} + INTERVAL 1 second
|
||||
@@ -618,6 +655,7 @@ func TestGraphLineHandler(t *testing.T) {
|
||||
_, h, mockConn, _ := NewMock(t, DefaultConfiguration())
|
||||
base := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
|
||||
|
||||
t.Run("sort by avg", func(t *testing.T) {
|
||||
// Single direction
|
||||
expectedSQL := []struct {
|
||||
Axis uint8 `ch:"axis"`
|
||||
@@ -730,6 +768,7 @@ func TestGraphLineHandler(t *testing.T) {
|
||||
"end": time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
|
||||
"points": 100,
|
||||
"limit": 20,
|
||||
"limitType": "Avg",
|
||||
"dimensions": []string{"ExporterName", "InIfProvider"},
|
||||
"filter": "DstCountry = 'FR' AND SrcCountry = 'US'",
|
||||
"units": "l3bps",
|
||||
@@ -805,6 +844,7 @@ func TestGraphLineHandler(t *testing.T) {
|
||||
"end": time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
|
||||
"points": 100,
|
||||
"limit": 20,
|
||||
"limitType": "Avg",
|
||||
"dimensions": []string{"ExporterName", "InIfProvider"},
|
||||
"filter": "DstCountry = 'FR' AND SrcCountry = 'US'",
|
||||
"units": "l3bps",
|
||||
@@ -924,6 +964,7 @@ func TestGraphLineHandler(t *testing.T) {
|
||||
"end": time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
|
||||
"points": 100,
|
||||
"limit": 20,
|
||||
"limitType": "Avg",
|
||||
"dimensions": []string{"ExporterName", "InIfProvider"},
|
||||
"filter": "DstCountry = 'FR' AND SrcCountry = 'US'",
|
||||
"units": "l3bps",
|
||||
@@ -1002,6 +1043,397 @@ func TestGraphLineHandler(t *testing.T) {
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("sort by max", func(t *testing.T) {
|
||||
// Single direction
|
||||
expectedSQL := []struct {
|
||||
Axis uint8 `ch:"axis"`
|
||||
Time time.Time `ch:"time"`
|
||||
Xps float64 `ch:"xps"`
|
||||
Dimensions []string `ch:"dimensions"`
|
||||
}{
|
||||
{1, base, 1000, []string{"router1", "provider1"}},
|
||||
{1, base, 2000, []string{"router1", "provider2"}},
|
||||
{1, base, 1200, []string{"router2", "provider2"}},
|
||||
{1, base, 1100, []string{"router2", "provider3"}},
|
||||
{1, base, 1900, []string{"Other", "Other"}},
|
||||
{1, base.Add(time.Minute), 500, []string{"router1", "provider1"}},
|
||||
{1, base.Add(time.Minute), 5000, []string{"router1", "provider2"}},
|
||||
{1, base.Add(time.Minute), 900, []string{"router2", "provider4"}},
|
||||
{1, base.Add(time.Minute), 100, []string{"Other", "Other"}},
|
||||
{1, base.Add(2 * time.Minute), 100, []string{"router1", "provider1"}},
|
||||
{1, base.Add(2 * time.Minute), 3000, []string{"router1", "provider2"}},
|
||||
{1, base.Add(2 * time.Minute), 100, []string{"router2", "provider4"}},
|
||||
{1, base.Add(2 * time.Minute), 100, []string{"Other", "Other"}},
|
||||
}
|
||||
mockConn.EXPECT().
|
||||
Select(gomock.Any(), gomock.Any(), gomock.Any()).
|
||||
SetArg(1, expectedSQL).
|
||||
Return(nil)
|
||||
|
||||
// Bidirectional
|
||||
expectedSQL = []struct {
|
||||
Axis uint8 `ch:"axis"`
|
||||
Time time.Time `ch:"time"`
|
||||
Xps float64 `ch:"xps"`
|
||||
Dimensions []string `ch:"dimensions"`
|
||||
}{
|
||||
{1, base, 1000, []string{"router1", "provider1"}},
|
||||
{1, base, 2000, []string{"router1", "provider2"}},
|
||||
{1, base, 1200, []string{"router2", "provider2"}},
|
||||
{1, base, 1100, []string{"router2", "provider3"}},
|
||||
{1, base, 1900, []string{"Other", "Other"}},
|
||||
{1, base.Add(time.Minute), 500, []string{"router1", "provider1"}},
|
||||
{1, base.Add(time.Minute), 5000, []string{"router1", "provider2"}},
|
||||
{1, base.Add(time.Minute), 900, []string{"router2", "provider4"}},
|
||||
|
||||
// Axes can be mixed. In reality, it seems they cannot
|
||||
// be interleaved, but ClickHouse documentation does
|
||||
// not say it is not possible.
|
||||
{2, base, 100, []string{"router1", "provider1"}},
|
||||
{2, base, 200, []string{"router1", "provider2"}},
|
||||
{2, base, 120, []string{"router2", "provider2"}},
|
||||
|
||||
{1, base.Add(time.Minute), 100, []string{"Other", "Other"}},
|
||||
{1, base.Add(2 * time.Minute), 100, []string{"router1", "provider1"}},
|
||||
|
||||
{2, base, 110, []string{"router2", "provider3"}},
|
||||
{2, base, 190, []string{"Other", "Other"}},
|
||||
{2, base.Add(time.Minute), 50, []string{"router1", "provider1"}},
|
||||
{2, base.Add(time.Minute), 500, []string{"router1", "provider2"}},
|
||||
|
||||
{1, base.Add(2 * time.Minute), 3000, []string{"router1", "provider2"}},
|
||||
{1, base.Add(2 * time.Minute), 100, []string{"router2", "provider4"}},
|
||||
{1, base.Add(2 * time.Minute), 100, []string{"Other", "Other"}},
|
||||
|
||||
{2, base.Add(time.Minute), 90, []string{"router2", "provider4"}},
|
||||
{2, base.Add(time.Minute), 10, []string{"Other", "Other"}},
|
||||
{2, base.Add(2 * time.Minute), 10, []string{"router1", "provider1"}},
|
||||
{2, base.Add(2 * time.Minute), 300, []string{"router1", "provider2"}},
|
||||
{2, base.Add(2 * time.Minute), 10, []string{"router2", "provider4"}},
|
||||
{2, base.Add(2 * time.Minute), 10, []string{"Other", "Other"}},
|
||||
}
|
||||
mockConn.EXPECT().
|
||||
Select(gomock.Any(), gomock.Any(), gomock.Any()).
|
||||
SetArg(1, expectedSQL).
|
||||
Return(nil)
|
||||
|
||||
// Previous period
|
||||
expectedSQL = []struct {
|
||||
Axis uint8 `ch:"axis"`
|
||||
Time time.Time `ch:"time"`
|
||||
Xps float64 `ch:"xps"`
|
||||
Dimensions []string `ch:"dimensions"`
|
||||
}{
|
||||
{1, base, 1000, []string{"router1", "provider1"}},
|
||||
{1, base, 2000, []string{"router1", "provider2"}},
|
||||
{1, base, 1200, []string{"router2", "provider2"}},
|
||||
{1, base, 1100, []string{"router2", "provider3"}},
|
||||
{1, base, 1900, []string{"Other", "Other"}},
|
||||
{1, base.Add(time.Minute), 500, []string{"router1", "provider1"}},
|
||||
{1, base.Add(time.Minute), 5000, []string{"router1", "provider2"}},
|
||||
{1, base.Add(time.Minute), 900, []string{"router2", "provider4"}},
|
||||
{1, base.Add(time.Minute), 100, []string{"Other", "Other"}},
|
||||
{1, base.Add(2 * time.Minute), 100, []string{"router1", "provider1"}},
|
||||
{1, base.Add(2 * time.Minute), 3000, []string{"router1", "provider2"}},
|
||||
{1, base.Add(2 * time.Minute), 100, []string{"router2", "provider4"}},
|
||||
{1, base.Add(2 * time.Minute), 100, []string{"Other", "Other"}},
|
||||
|
||||
{3, base, 8000, []string{}},
|
||||
{3, base.Add(time.Minute), 6000, []string{}},
|
||||
{3, base.Add(2 * time.Minute), 4500, []string{}},
|
||||
}
|
||||
mockConn.EXPECT().
|
||||
Select(gomock.Any(), gomock.Any(), gomock.Any()).
|
||||
SetArg(1, expectedSQL).
|
||||
Return(nil)
|
||||
|
||||
helpers.TestHTTPEndpoints(t, h.LocalAddr(), helpers.HTTPEndpointCases{
|
||||
{
|
||||
Description: "single direction",
|
||||
URL: "/api/v0/console/graph/line",
|
||||
JSONInput: gin.H{
|
||||
"start": time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
|
||||
"end": time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
|
||||
"points": 100,
|
||||
"limit": 20,
|
||||
"limitType": "Max",
|
||||
"dimensions": []string{"ExporterName", "InIfProvider"},
|
||||
"filter": "DstCountry = 'FR' AND SrcCountry = 'US'",
|
||||
"units": "l3bps",
|
||||
"bidirectional": false,
|
||||
},
|
||||
JSONOutput: gin.H{
|
||||
// Sorted by max of bps
|
||||
"rows": [][]string{
|
||||
{"router1", "provider2"},
|
||||
{"router2", "provider2"},
|
||||
{"router2", "provider3"},
|
||||
{"router1", "provider1"},
|
||||
{"router2", "provider4"},
|
||||
{"Other", "Other"},
|
||||
},
|
||||
"t": []string{
|
||||
"2009-11-10T23:00:00Z",
|
||||
"2009-11-10T23:01:00Z",
|
||||
"2009-11-10T23:02:00Z",
|
||||
},
|
||||
"points": [][]int{
|
||||
{2000, 5000, 3000},
|
||||
{1200, 0, 0},
|
||||
{1100, 0, 0},
|
||||
{1000, 500, 100},
|
||||
{0, 900, 100},
|
||||
{1900, 100, 100},
|
||||
},
|
||||
"min": []int{
|
||||
2000,
|
||||
1200,
|
||||
1100,
|
||||
100,
|
||||
100,
|
||||
100,
|
||||
},
|
||||
"max": []int{
|
||||
5000,
|
||||
1200,
|
||||
1100,
|
||||
1000,
|
||||
900,
|
||||
1900,
|
||||
},
|
||||
"average": []int{
|
||||
3333,
|
||||
400,
|
||||
366,
|
||||
533,
|
||||
333,
|
||||
700,
|
||||
},
|
||||
"95th": []int{
|
||||
4000,
|
||||
600,
|
||||
550,
|
||||
750,
|
||||
500,
|
||||
1000,
|
||||
},
|
||||
"axis": []int{
|
||||
1, 1, 1, 1, 1, 1,
|
||||
},
|
||||
"axis-names": map[int]string{
|
||||
1: "Direct",
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Description: "bidirectional",
|
||||
URL: "/api/v0/console/graph/line",
|
||||
JSONInput: gin.H{
|
||||
"start": time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
|
||||
"end": time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
|
||||
"points": 100,
|
||||
"limit": 20,
|
||||
"limitType": "Max",
|
||||
"dimensions": []string{"ExporterName", "InIfProvider"},
|
||||
"filter": "DstCountry = 'FR' AND SrcCountry = 'US'",
|
||||
"units": "l3bps",
|
||||
"bidirectional": true,
|
||||
},
|
||||
JSONOutput: gin.H{
|
||||
// Sorted by sum of bps
|
||||
"rows": [][]string{
|
||||
{"router1", "provider2"}, // 10000
|
||||
{"router2", "provider2"}, // 1200
|
||||
{"router2", "provider3"}, // 1100
|
||||
{"router1", "provider1"}, // 1600
|
||||
{"router2", "provider4"}, // 1000
|
||||
{"Other", "Other"}, // 2100
|
||||
|
||||
{"router1", "provider2"}, // 1000
|
||||
{"router2", "provider2"}, // 120
|
||||
{"router2", "provider3"}, // 110
|
||||
{"router1", "provider1"}, // 160
|
||||
{"router2", "provider4"}, // 100
|
||||
{"Other", "Other"}, // 210
|
||||
},
|
||||
"t": []string{
|
||||
"2009-11-10T23:00:00Z",
|
||||
"2009-11-10T23:01:00Z",
|
||||
"2009-11-10T23:02:00Z",
|
||||
},
|
||||
"points": [][]int{
|
||||
{2000, 5000, 3000},
|
||||
{1200, 0, 0},
|
||||
{1100, 0, 0},
|
||||
{1000, 500, 100},
|
||||
{0, 900, 100},
|
||||
{1900, 100, 100},
|
||||
|
||||
{200, 500, 300},
|
||||
{120, 0, 0},
|
||||
{110, 0, 0},
|
||||
{100, 50, 10},
|
||||
{0, 90, 10},
|
||||
{190, 10, 10},
|
||||
},
|
||||
"min": []int{
|
||||
2000,
|
||||
1200,
|
||||
1100,
|
||||
100,
|
||||
100,
|
||||
100,
|
||||
|
||||
200,
|
||||
120,
|
||||
110,
|
||||
10,
|
||||
10,
|
||||
10,
|
||||
},
|
||||
"max": []int{
|
||||
5000,
|
||||
1200,
|
||||
1100,
|
||||
1000,
|
||||
900,
|
||||
1900,
|
||||
|
||||
500,
|
||||
120,
|
||||
110,
|
||||
100,
|
||||
90,
|
||||
190,
|
||||
},
|
||||
"average": []int{
|
||||
3333,
|
||||
400,
|
||||
366,
|
||||
533,
|
||||
333,
|
||||
700,
|
||||
|
||||
333,
|
||||
40,
|
||||
36,
|
||||
53,
|
||||
33,
|
||||
70,
|
||||
},
|
||||
"95th": []int{
|
||||
4000,
|
||||
600,
|
||||
550,
|
||||
750,
|
||||
500,
|
||||
1000,
|
||||
|
||||
400,
|
||||
60,
|
||||
55,
|
||||
75,
|
||||
50,
|
||||
100,
|
||||
},
|
||||
"axis": []int{
|
||||
1, 1, 1, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2,
|
||||
},
|
||||
"axis-names": map[int]string{
|
||||
1: "Direct",
|
||||
2: "Reverse",
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Description: "previous period",
|
||||
URL: "/api/v0/console/graph/line",
|
||||
JSONInput: gin.H{
|
||||
"start": time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
|
||||
"end": time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
|
||||
"points": 100,
|
||||
"limit": 20,
|
||||
"limitType": "Max",
|
||||
"dimensions": []string{"ExporterName", "InIfProvider"},
|
||||
"filter": "DstCountry = 'FR' AND SrcCountry = 'US'",
|
||||
"units": "l3bps",
|
||||
"bidirectional": false,
|
||||
"previous-period": true,
|
||||
},
|
||||
JSONOutput: gin.H{
|
||||
// Sorted by sum of bps
|
||||
"rows": [][]string{
|
||||
{"router1", "provider2"}, // 10000
|
||||
{"router2", "provider2"}, // 1200
|
||||
{"router2", "provider3"}, // 1100
|
||||
{"router1", "provider1"}, // 1600
|
||||
{"router2", "provider4"}, // 1000
|
||||
{"Other", "Other"}, // 2100
|
||||
{"Other", "Other"}, // Previous day
|
||||
},
|
||||
"t": []string{
|
||||
"2009-11-10T23:00:00Z",
|
||||
"2009-11-10T23:01:00Z",
|
||||
"2009-11-10T23:02:00Z",
|
||||
},
|
||||
"points": [][]int{
|
||||
{2000, 5000, 3000},
|
||||
{1200, 0, 0},
|
||||
{1100, 0, 0},
|
||||
{1000, 500, 100},
|
||||
{0, 900, 100},
|
||||
{1900, 100, 100},
|
||||
{8000, 6000, 4500},
|
||||
},
|
||||
"min": []int{
|
||||
2000,
|
||||
1200,
|
||||
1100,
|
||||
100,
|
||||
100,
|
||||
100,
|
||||
4500,
|
||||
},
|
||||
"max": []int{
|
||||
5000,
|
||||
1200,
|
||||
1100,
|
||||
1000,
|
||||
900,
|
||||
1900,
|
||||
8000,
|
||||
},
|
||||
"average": []int{
|
||||
3333,
|
||||
400,
|
||||
366,
|
||||
533,
|
||||
333,
|
||||
700,
|
||||
6166,
|
||||
},
|
||||
"95th": []int{
|
||||
4000,
|
||||
600,
|
||||
550,
|
||||
750,
|
||||
500,
|
||||
1000,
|
||||
7000,
|
||||
},
|
||||
"axis": []int{
|
||||
1, 1, 1, 1, 1, 1,
|
||||
3,
|
||||
},
|
||||
"axis-names": map[int]string{
|
||||
1: "Direct",
|
||||
3: "Previous day",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetTableInterval(t *testing.T) {
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"akvorado/common/schema"
|
||||
@@ -32,3 +33,36 @@ func (c *Component) fixQueryColumnName(name string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func selectSankeyRowsByLimitType(input graphSankeyHandlerInput, dimensions []string, where string) string {
|
||||
return selectRowsByLimitType(input.graphCommonHandlerInput, dimensions, where)
|
||||
}
|
||||
|
||||
func selectLineRowsByLimitType(input graphLineHandlerInput, dimensions []string, where string) string {
|
||||
return selectRowsByLimitType(input.graphCommonHandlerInput, dimensions, where)
|
||||
}
|
||||
|
||||
func selectRowsByLimitType(input graphCommonHandlerInput, dimensions []string, where string) string {
|
||||
var rowsType string
|
||||
var source string
|
||||
var orderBy string
|
||||
if input.LimitType == "Max" {
|
||||
source = fmt.Sprintf("( SELECT %s AS sum_at_time FROM source WHERE %s GROUP BY %s )",
|
||||
strings.Join(append(dimensions, "{{ .Units }}"), ", "),
|
||||
where,
|
||||
strings.Join(dimensions, ", "),
|
||||
)
|
||||
orderBy = "MAX(sum_at_time)"
|
||||
} else {
|
||||
source = fmt.Sprintf("source WHERE %s", where)
|
||||
orderBy = "{{ .Units }}"
|
||||
}
|
||||
rowsType = fmt.Sprintf(
|
||||
"rows AS (SELECT %s FROM %s GROUP BY %s ORDER BY %s DESC LIMIT %d)",
|
||||
strings.Join(dimensions, ", "),
|
||||
source,
|
||||
strings.Join(dimensions, ", "),
|
||||
orderBy,
|
||||
input.Limit)
|
||||
return rowsType
|
||||
}
|
||||
|
||||
@@ -57,14 +57,8 @@ func (input graphSankeyHandlerInput) toSQL() (string, error) {
|
||||
// With
|
||||
with := []string{
|
||||
fmt.Sprintf("source AS (%s)", input.sourceSelect()),
|
||||
fmt.Sprintf(`(SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE %s) AS range`, where),
|
||||
fmt.Sprintf(
|
||||
"rows AS (SELECT %s FROM source WHERE %s GROUP BY %s ORDER BY {{ .Units }} DESC LIMIT %d)",
|
||||
strings.Join(dimensions, ", "),
|
||||
where,
|
||||
strings.Join(dimensions, ", "),
|
||||
input.Limit),
|
||||
}
|
||||
fmt.Sprintf(`(SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE %s) AS range`, where)}
|
||||
with = append(with, selectSankeyRowsByLimitType(input, dimensions, where))
|
||||
|
||||
sqlQuery := fmt.Sprintf(`
|
||||
{{ with %s }}
|
||||
|
||||
@@ -53,6 +53,38 @@ FROM source
|
||||
WHERE {{ .Timefilter }}
|
||||
GROUP BY dimensions
|
||||
ORDER BY xps DESC
|
||||
{{ end }}`,
|
||||
}, {
|
||||
Description: "two dimensions, no filters, l3 bps, limitType by max",
|
||||
Pos: helpers.Mark(),
|
||||
Input: graphSankeyHandlerInput{
|
||||
graphCommonHandlerInput{
|
||||
Start: time.Date(2022, 4, 10, 15, 45, 10, 0, time.UTC),
|
||||
End: time.Date(2022, 4, 11, 15, 45, 10, 0, time.UTC),
|
||||
Dimensions: []query.Column{
|
||||
query.NewColumn("SrcAS"),
|
||||
query.NewColumn("ExporterName"),
|
||||
},
|
||||
Limit: 5,
|
||||
LimitType: "Max",
|
||||
Filter: query.Filter{},
|
||||
Units: "l3bps",
|
||||
},
|
||||
},
|
||||
Expected: `
|
||||
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":20,"units":"l3bps"}@@ }}
|
||||
WITH
|
||||
source AS (SELECT * FROM {{ .Table }} SETTINGS asterisk_include_alias_columns = 1),
|
||||
(SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM source WHERE {{ .Timefilter }}) AS range,
|
||||
rows AS (SELECT SrcAS, ExporterName FROM ( SELECT SrcAS, ExporterName, {{ .Units }} AS sum_at_time FROM source WHERE {{ .Timefilter }} GROUP BY SrcAS, ExporterName ) GROUP BY SrcAS, ExporterName ORDER BY MAX(sum_at_time) DESC LIMIT 5)
|
||||
SELECT
|
||||
{{ .Units }}/range AS xps,
|
||||
[if(SrcAS IN (SELECT SrcAS FROM rows), concat(toString(SrcAS), ': ', dictGetOrDefault('asns', 'name', SrcAS, '???')), 'Other'),
|
||||
if(ExporterName IN (SELECT ExporterName FROM rows), ExporterName, 'Other')] AS dimensions
|
||||
FROM source
|
||||
WHERE {{ .Timefilter }}
|
||||
GROUP BY dimensions
|
||||
ORDER BY xps DESC
|
||||
{{ end }}`,
|
||||
}, {
|
||||
Description: "two dimensions, no filters, l2 bps",
|
||||
|
||||
Reference in New Issue
Block a user