console: add "previous period" mode

This displays a line for the previous period on stacked graphs.
Previous period depends on the current period. It could be hour, day,
week, month, or year.
This commit is contained in:
Vincent Bernat
2022-08-09 11:59:53 +02:00
parent f95a398c4c
commit 454a3eb9d3
10 changed files with 578 additions and 152 deletions

View File

@@ -98,6 +98,7 @@ func (c *Component) finalizeQuery(query string) string {
type inputContext struct {
Start time.Time `json:"start"`
End time.Time `json:"end"`
StartForInterval *time.Time `json:"start-for-interval,omitempty"`
MainTableRequired bool `json:"main-table-required,omitempty"`
Points uint `json:"points"`
Units string `json:"units,omitempty"`
@@ -152,59 +153,14 @@ func (c *Component) contextFunc(inputStr string) context {
targetInterval = time.Second
}
c.flowsTablesLock.RLock()
defer c.flowsTablesLock.RUnlock()
// Select table
table := "flows"
computedInterval := time.Second
if !input.MainTableRequired && len(c.flowsTables) > 0 {
// We can use the consolidated data. The first
// criteria is to find the tables matching the time
// criteria.
candidates := []int{}
for idx, table := range c.flowsTables {
if input.Start.After(table.Oldest.Add(table.Resolution)) {
candidates = append(candidates, idx)
targetIntervalForTableSelection := targetInterval
if input.MainTableRequired {
targetIntervalForTableSelection = time.Second
}
}
if len(candidates) == 0 {
// No candidate, fallback to the one with oldest data
best := 0
for idx, table := range c.flowsTables {
if c.flowsTables[best].Oldest.After(table.Oldest.Add(table.Resolution)) {
best = idx
}
}
candidates = []int{best}
// Add other candidates that are not far off in term of oldest data
for idx, table := range c.flowsTables {
if idx == best {
continue
}
if c.flowsTables[best].Oldest.After(table.Oldest) {
candidates = append(candidates, idx)
}
}
}
if len(candidates) > 1 {
// Use interval to find the best one
best := 0
for _, idx := range candidates {
if c.flowsTables[idx].Resolution > targetInterval {
continue
}
if c.flowsTables[idx].Resolution > c.flowsTables[best].Resolution {
best = idx
}
}
candidates = []int{best}
}
table = c.flowsTables[candidates[0]].Name
computedInterval = c.flowsTables[candidates[0]].Resolution
}
if computedInterval < time.Second {
computedInterval = time.Second
table, computedInterval := c.getBestTable(input.Start, targetIntervalForTableSelection)
if input.StartForInterval != nil {
_, computedInterval = c.getBestTable(*input.StartForInterval, targetIntervalForTableSelection)
}
// Make start/end match the computed interval (currently equal to the table resolution)
@@ -257,3 +213,61 @@ func (c *Component) contextFunc(inputStr string) context {
},
}
}
// Get the best table starting at the specified time.
func (c *Component) getBestTable(start time.Time, targetInterval time.Duration) (string, time.Duration) {
c.flowsTablesLock.RLock()
defer c.flowsTablesLock.RUnlock()
table := "flows"
computedInterval := time.Second
if len(c.flowsTables) > 0 {
// We can use the consolidated data. The first
// criteria is to find the tables matching the time
// criteria.
candidates := []int{}
for idx, table := range c.flowsTables {
if start.After(table.Oldest.Add(table.Resolution)) {
candidates = append(candidates, idx)
}
}
if len(candidates) == 0 {
// No candidate, fallback to the one with oldest data
best := 0
for idx, table := range c.flowsTables {
if c.flowsTables[best].Oldest.After(table.Oldest.Add(table.Resolution)) {
best = idx
}
}
candidates = []int{best}
// Add other candidates that are not far off in term of oldest data
for idx, table := range c.flowsTables {
if idx == best {
continue
}
if c.flowsTables[best].Oldest.After(table.Oldest) {
candidates = append(candidates, idx)
}
}
}
if len(candidates) > 1 {
// Use interval to find the best one
best := 0
for _, idx := range candidates {
if c.flowsTables[idx].Resolution > targetInterval {
continue
}
if c.flowsTables[idx].Resolution > c.flowsTables[best].Resolution {
best = idx
}
}
candidates = []int{best}
}
table = c.flowsTables[candidates[0]].Name
computedInterval = c.flowsTables[candidates[0]].Resolution
}
if computedInterval < time.Second {
computedInterval = time.Second
}
return table, computedInterval
}

View File

@@ -167,6 +167,36 @@ func TestFinalizeQuery(t *testing.T) {
Points: 720, // 2-minute resolution,
},
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')",
}, {
Description: "use flows table for resolution (control for next case)",
Tables: []flowsTable{
{"flows", 0, time.Date(2022, 04, 10, 10, 45, 10, 0, time.UTC)},
{"flows_1m0s", time.Minute, time.Date(2022, 03, 10, 10, 45, 10, 0, time.UTC)},
},
Query: "SELECT 1 FROM {{ .Table }} WHERE {{ .Timefilter }} // {{ .Interval }}",
Context: inputContext{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 2880, // 30-second resolution
},
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC') // 30",
}, {
Description: "use flows table for resolution (but flows_1m0s for data)",
Tables: []flowsTable{
{"flows", 0, time.Date(2022, 04, 10, 10, 45, 10, 0, time.UTC)},
{"flows_1m0s", time.Minute, time.Date(2022, 03, 10, 10, 45, 10, 0, time.UTC)},
},
Query: "SELECT 1 FROM {{ .Table }} WHERE {{ .Timefilter }} // {{ .Interval }}",
Context: inputContext{
Start: time.Date(2022, 03, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 03, 11, 15, 45, 10, 0, time.UTC),
StartForInterval: func() *time.Time {
t := time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC)
return &t
}(),
Points: 2880, // 30-second resolution
},
Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-03-10 15:45:10', 'UTC') AND toDateTime('2022-03-11 15:45:10', 'UTC') // 30",
}, {
Description: "select flows table with better resolution",
Tables: []flowsTable{

View File

@@ -115,10 +115,19 @@ aspect of the graph.
- The unit to use on the Y-axis: layer-3 bits per second, layer-2 bits
per second (should match interface counters), or packets par second.
- Four graph types are provided: “stacked”, “lines” and “grid” to
- Four graph types are provided: “stacked”, “lines”, and “grid” to
display time series and “sankey” to show flow distributions between
various dimensions.
- For “stacked”, “lines”, and “grid” graphs, the *bidirectional*
option adds the flows in the opposite direction to the graph. They
are displayed as a negative value on the graph.
- For “stacked” graphs, the *previous period* option adds a line for
the traffic levels as they were on the previous period. Depending on
the current period, the previous period can be the previous hour,
day, week, month, or year.
- The time range can be set from a list of preset or directly using
natural language. The parsing is done by
[SugarJS](https://sugarjs.com/dates/#/Parsing) which provides

View File

@@ -13,7 +13,8 @@ identified with a specific icon:
## Unreleased
-*console*: add a bidirectional mode for graphs to also display flows in the opposite direction
-*console*: add an option to also display flows in the opposite direction on time series graph
-*console*: add an option to also display the previous period (day, week, month, year) on stacked graphs
- 🌱 *inlet*: Kafka key is now a 4-byte random value making scaling less dependent on the number of exporters
- 🌱 *demo-exporter*: add a setting to automatically generate a reverse flow
- 🌱 *docker-compose*: loosen required privileges for `conntrack-fixer`

View File

@@ -62,7 +62,7 @@ import DataGraph from "./VisualizePage/DataGraph.vue";
import OptionsPanel from "./VisualizePage/OptionsPanel.vue";
import RequestSummary from "./VisualizePage/RequestSummary.vue";
import { graphTypes } from "./VisualizePage/constants";
import { isEqual } from "lodash-es";
import { isEqual, omit } from "lodash-es";
const graphHeight = ref(500);
const highlightedSerie = ref(null);
@@ -109,12 +109,16 @@ const encodedState = computed(() => encodeState(state.value));
// Fetch data
const fetchedData = ref({});
const payload = computed(() => ({
const finalState = computed(() => ({
...state.value,
start: SugarDate.create(state.value.start),
end: SugarDate.create(state.value.end),
}));
const request = ref({}); // Same as payload, but once request is successful
const jsonPayload = computed(() => ({
...omit(finalState.value, ["previousPeriod", "graphType"]),
"previous-period": finalState.value.previousPeriod,
}));
const request = ref({}); // Same as finalState, but once request is successful
const { data, isFetching, aborted, abort, canAbort, error } = useFetch("", {
beforeFetch(ctx) {
// Add the URL. Not a computed value as if we change both payload
@@ -146,12 +150,7 @@ const { data, isFetching, aborted, abort, canAbort, error } = useFetch("", {
console.groupEnd();
fetchedData.value = {
...data,
dimensions: payload.value.dimensions,
start: payload.value.start,
end: payload.value.end,
graphType: payload.value.graphType,
units: payload.value.units,
bidirectional: payload.value.bidirectional,
...omit(finalState.value, ["limit", "filter", "points"]),
};
// Also update URL.
@@ -166,13 +165,13 @@ const { data, isFetching, aborted, abort, canAbort, error } = useFetch("", {
}
// Keep current payload for state
request.value = payload.value;
request.value = finalState.value;
return ctx;
},
refetch: true,
})
.post(payload)
.post(jsonPayload)
.json();
const errorMessage = computed(
() =>

View File

@@ -63,49 +63,6 @@ const commonGraph = {
brush: {
xAxisIndex: "all",
},
tooltip: {
confine: true,
trigger: "axis",
axisPointer: {
type: "cross",
label: { backgroundColor: "#6a7985" },
},
formatter: (params) => {
// We will use a custom formatter, notably to handle bidirectional tooltips.
if (params.length === 0) return;
let table = [],
bidirectional = false;
params.forEach((param) => {
let idx = findIndex(table, (r) => r.seriesName === param.seriesName);
if (idx === -1) {
table.push({
marker: param.marker,
seriesName: param.seriesName,
});
idx = table.length - 1;
}
const val = param.value[param.seriesIndex + 1];
if (table[idx].col1 !== undefined || val < 0) {
table[idx].col2 = val;
bidirectional = true;
} else table[idx].col1 = val;
});
const rows = table
.map(
(row) => `<tr>
<td>${row.marker} ${row.seriesName}</td>
<td class="pl-2">${bidirectional ? "↑" : ""}<b>${formatXps(
row.col1 || 0
)}</b></td>
<td class="pl-2">${bidirectional ? "↓" : ""}<b>${
bidirectional ? formatXps(row.col2 || 0) : ""
}</b></td>
</tr>`
)
.join("");
return `${params[0].axisValueLabel}<table>${rows}</table>`;
},
},
};
const graph = computed(() => {
const theme = isDark.value ? "dark" : "light";
@@ -123,7 +80,7 @@ const graph = computed(() => {
// Unfortunately, eCharts does not seem to make it easy
// to inverse an axis and put the result below. Therefore,
// we use negative values for the second axis.
(row, rowIdx) => row[timeIdx] * (data.axis[rowIdx] == 1 ? 1 : -1)
(row, rowIdx) => row[timeIdx] * (data.axis[rowIdx] % 2 ? 1 : -1)
),
])
.slice(0, -1),
@@ -141,6 +98,57 @@ const graph = computed(() => {
axisPointer: {
label: { formatter: ({ value }) => formatXps(value) },
},
},
tooltip = {
confine: true,
trigger: "axis",
axisPointer: {
type: "cross",
label: { backgroundColor: "#6a7985" },
},
formatter: (params) => {
// We will use a custom formatter, notably to handle bidirectional tooltips.
if (params.length === 0) return;
let table = [];
params.forEach((param) => {
const axis = data.axis[param.seriesIndex];
const seriesName = [1, 2].includes(axis)
? param.seriesName
: data["axis-names"][axis];
const key = `${Math.floor((axis - 1) / 2)}-${seriesName}`;
let idx = findIndex(table, (r) => r.key === key);
if (idx === -1) {
table.push({
key,
seriesName,
marker: param.marker,
up: 0,
down: 0,
});
idx = table.length - 1;
}
const val = param.value[param.seriesIndex + 1];
if (axis % 2 == 1) table[idx].up = val;
else table[idx].down = val;
});
const rows = table
.map((row) =>
[
`<tr>`,
`<td>${row.marker} ${row.seriesName}</td>`,
`<td class="pl-2">${data.bidirectional ? "↑" : ""}<b>${formatXps(
row.up || 0
)}</b></td>`,
data.bidirectional
? `<td class="pl-2">↓<b>${formatXps(row.down || 0)}</b></td>`
: "",
`</tr>`,
].join("")
)
.join("");
return `${params[0].axisValueLabel}<table>${rows}</table>`;
},
};
// Lines and stacked areas
@@ -158,6 +166,7 @@ const graph = computed(() => {
xAxis,
yAxis,
dataset,
tooltip,
series: data.rows
.map((row, idx) => {
const isOther = row.some((name) => name === "Other"),
@@ -185,7 +194,23 @@ const graph = computed(() => {
seriesId: idx + 1,
},
};
if (data.graphType === graphTypes.stacked) {
if ([3, 4].includes(data.axis[idx])) {
serie = {
...serie,
itemStyle: {
color: dataColorGrey(1, false, theme),
},
lineStyle: {
color: dataColorGrey(1, false, theme),
width: 1,
type: "dashed",
},
};
}
if (
data.graphType === graphTypes.stacked &&
[1, 2].includes(data.axis[idx])
) {
serie = {
...serie,
stack: data.axis[idx],

View File

@@ -8,7 +8,7 @@
v-if="axes.length > 1"
class="border-b border-gray-200 text-center text-sm font-medium text-gray-500 dark:border-gray-700 dark:text-gray-400"
>
<ul class="-mb-px flex flex-wrap">
<ul class="flex flex-wrap">
<li v-for="{ id: axis, name } in axes" :key="axis" class="mr-2">
<button
class="pointer-cursor inline-block rounded-t-lg border-b-2 border-transparent p-4 hover:border-gray-300 hover:text-gray-600 dark:hover:text-gray-300"
@@ -99,7 +99,7 @@ import { graphTypes } from "./constants";
const { isDark } = inject("theme");
const { stacked, lines, grid, sankey } = graphTypes;
import { uniq, uniqWith, isEqual, findIndex, takeWhile } from "lodash-es";
import { uniqWith, isEqual, findIndex, takeWhile, toPairs } from "lodash-es";
const highlight = (index) => {
if (index === null) {
@@ -119,10 +119,10 @@ const highlight = (index) => {
emit("highlighted", originalIndex);
};
const axes = computed(() =>
uniq(props.data.axis ?? []).map((axis) => ({
id: axis,
name: { 1: "Direct", 2: "Reverse" }[axis] ?? "Unknown",
}))
toPairs(props.data["axis-names"])
.map(([k, v]) => ({ id: Number(k), name: v }))
.filter(({ id }) => [1, 2].includes(id))
.sort(({ id: id1 }, { id: id2 }) => id1 - id2)
);
const selectedAxis = ref(1);
const displayedAxis = computed(() =>

View File

@@ -71,12 +71,20 @@
</div>
</template>
</InputListBox>
<div
class="order-4 flex grow flex-row justify-between gap-x-3 sm:order-2 sm:grow-0 sm:flex-col lg:order-4 lg:grow lg:flex-row"
>
<InputCheckbox
v-if="[stacked, lines, grid].includes(graphType.name)"
v-model="bidirectional"
class="order-4 sm:order-2 lg:order-4"
label="Bidirectional"
/>
<InputCheckbox
v-if="[stacked].includes(graphType.name)"
v-model="previousPeriod"
label="Previous period"
/>
</div>
</div>
<SectionLabel>Time range</SectionLabel>
<InputTimeRange v-model="timeRange" />
@@ -141,6 +149,7 @@ const dimensions = ref([]);
const filter = ref({});
const units = ref("l3bps");
const bidirectional = ref(false);
const previousPeriod = ref(false);
const options = computed(() => ({
// Common to all graph types
@@ -151,13 +160,16 @@ const options = computed(() => ({
limit: dimensions.value.limit,
filter: filter.value.expression,
units: units.value,
// Only for time series
// Depending on the graph type...
...([stacked, lines].includes(graphType.value.name) && {
bidirectional: bidirectional.value,
previousPeriod:
graphType.value.name === stacked ? previousPeriod.value : false,
points: 200,
}),
...(graphType.value.name === grid && {
bidirectional: bidirectional.value,
previousPeriod: false,
points: 50,
}),
}));
@@ -184,6 +196,7 @@ watch(
filter: _filter = defaultOptions?.filter,
units: _units = "l3bps",
bidirectional: _bidirectional = false,
previousPeriod: _previousPeriod = false,
} = modelValue;
// Dispatch values in refs
@@ -197,6 +210,7 @@ watch(
filter.value = { expression: _filter };
units.value = _units;
bidirectional.value = _bidirectional;
previousPeriod.value = _previousPeriod;
// A bit risky, but it seems to work.
if (!isEqual(modelValue, options.value)) {

View File

@@ -25,6 +25,7 @@ type graphHandlerInput struct {
Filter queryFilter `json:"filter"` // where ...
Units string `json:"units" binding:"required,oneof=pps l2bps l3bps"`
Bidirectional bool `json:"bidirectional"`
PreviousPeriod bool `json:"previous-period"`
}
// graphHandlerOutput describes the output for the /graph endpoint. A
@@ -36,6 +37,7 @@ type graphHandlerOutput struct {
Rows [][]string `json:"rows"` // List of rows
Points [][]int `json:"points"` // t → row → xps
Axis []int `json:"axis"` // row → axis
AxisNames map[int]string `json:"axis-names"`
Average []int `json:"average"` // row → average xps
Min []int `json:"min"` // row → min xps
Max []int `json:"max"` // row → max xps
@@ -53,12 +55,66 @@ func (input graphHandlerInput) reverseDirection() graphHandlerInput {
return input
}
func (input graphHandlerInput) toSQL1(axis int, skipWith bool) string {
// nearestPeriod returns the name and period matching the provided
// period length. The year is a special case as we don't know its
// exact length.
func nearestPeriod(period time.Duration) (time.Duration, string) {
switch {
case period < 2*time.Hour:
return time.Hour, "hour"
case period < 2*24*time.Hour:
return 24 * time.Hour, "day"
case period < 2*7*24*time.Hour:
return 7 * 24 * time.Hour, "week"
case period < 2*4*7*24*time.Hour:
// We use 4 weeks, not 1 month
return 4 * 7 * 24 * time.Hour, "month"
default:
return 0, "year"
}
}
// previousPeriod shifts the provided input to the previous period.
// The chosen period depend on the current period. For less than
// 2-hour period, the previous period is the hour. For less than 2-day
// period, this is the day. For less than 2-weeks, this is the week,
// for less than 2-months, this is the month, otherwise, this is the
// year. Also, dimensions are stripped.
func (input graphHandlerInput) previousPeriod() graphHandlerInput {
input.Dimensions = []queryColumn{}
diff := input.End.Sub(input.Start)
period, _ := nearestPeriod(diff)
if period == 0 {
// We use a full year this time (think for example we
// want to see how was New Year Eve compared to last
// year)
input.Start = input.Start.AddDate(-1, 0, 0)
input.End = input.End.AddDate(-1, 0, 0)
return input
}
input.Start = input.Start.Add(-period)
input.End = input.End.Add(-period)
return input
}
type toSQL1Options struct {
skipWithClause bool
offsetedStart time.Time
}
func (input graphHandlerInput) toSQL1(axis int, options toSQL1Options) string {
var startForInterval *time.Time
var offsetShift string
if !options.offsetedStart.IsZero() {
startForInterval = &options.offsetedStart
offsetShift = fmt.Sprintf(" + INTERVAL %d second",
int64(options.offsetedStart.Sub(input.Start).Seconds()))
}
where := templateWhere(input.Filter)
// Select
fields := []string{
`{{ call .ToStartOfInterval "TimeReceived" }} AS time`,
fmt.Sprintf(`{{ call .ToStartOfInterval "TimeReceived" }}%s AS time`, offsetShift),
`{{ .Units }}/{{ .Interval }} AS xps`,
}
selectFields := []string{}
@@ -81,7 +137,7 @@ func (input graphHandlerInput) toSQL1(axis int, skipWith bool) string {
// With
with := []string{}
if len(dimensions) > 0 && !skipWith {
if len(dimensions) > 0 && !options.skipWithClause {
with = append(with, fmt.Sprintf(
"rows AS (SELECT %s FROM {{ .Table }} WHERE %s GROUP BY %s ORDER BY SUM(Bytes) DESC LIMIT %d)",
strings.Join(dimensions, ", "),
@@ -103,28 +159,43 @@ FROM {{ .Table }}
WHERE %s
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
FROM {{ .TimefilterStart }}%s
TO {{ .TimefilterEnd }} + INTERVAL 1 second%s
STEP {{ .Interval }})
{{ end }}`,
templateContext(inputContext{
Start: input.Start,
End: input.End,
StartForInterval: startForInterval,
MainTableRequired: requireMainTable(input.Dimensions, input.Filter),
Points: input.Points,
Units: input.Units,
}),
withStr, axis, strings.Join(fields, ",\n "), where)
withStr, axis, strings.Join(fields, ",\n "), where, offsetShift, offsetShift)
return strings.TrimSpace(sqlQuery)
}
// graphHandlerInputToSQL converts a graph input to an SQL request
func (input graphHandlerInput) toSQL() string {
parts := []string{input.toSQL1(1, false)}
parts := []string{input.toSQL1(1, toSQL1Options{})}
// Handle specific options. We have to align time periods in
// case the previous period does not use the same offsets.
if input.Bidirectional {
parts = append(parts, input.reverseDirection().toSQL1(2, true))
parts = append(parts, input.reverseDirection().toSQL1(2, toSQL1Options{skipWithClause: true}))
}
return strings.TrimSpace(strings.Join(parts, "\nUNION ALL\n"))
if input.PreviousPeriod {
parts = append(parts, input.previousPeriod().toSQL1(3, toSQL1Options{
skipWithClause: true,
offsetedStart: input.Start,
}))
}
if input.Bidirectional && input.PreviousPeriod {
parts = append(parts, input.reverseDirection().previousPeriod().toSQL1(4, toSQL1Options{
skipWithClause: true,
offsetedStart: input.Start,
}))
}
return strings.Join(parts, "\nUNION ALL\n")
}
func (c *Component) graphHandlerFunc(gc *gin.Context) {
@@ -246,6 +317,7 @@ func (c *Component) graphHandlerFunc(gc *gin.Context) {
}
output.Rows = make([][]string, totalRows)
output.Axis = make([]int, totalRows)
output.AxisNames = make(map[int]string)
output.Points = make([][]int, totalRows)
output.Average = make([]int, totalRows)
output.Min = make([]int, totalRows)
@@ -303,5 +375,17 @@ func (c *Component) graphHandlerFunc(gc *gin.Context) {
}
}
for _, axis := range output.Axis {
switch axis {
case 1:
output.AxisNames[axis] = "Direct"
case 2:
output.AxisNames[axis] = "Reverse"
case 3, 4:
diff := input.End.Sub(input.Start)
_, name := nearestPeriod(diff)
output.AxisNames[axis] = fmt.Sprintf("Previous %s", name)
}
}
gc.JSON(http.StatusOK, output)
}

View File

@@ -55,6 +55,88 @@ func TestGraphInputReverseDirection(t *testing.T) {
}
}
func TestGraphPreviousPeriod(t *testing.T) {
const longForm = "Jan 2, 2006 at 15:04"
cases := []struct {
Start string
End string
ExpectedStart string
ExpectedEnd string
}{
{
"Jan 2, 2020 at 15:04", "Jan 2, 2020 at 16:04",
"Jan 2, 2020 at 14:04", "Jan 2, 2020 at 15:04",
}, {
"Jan 2, 2020 at 15:04", "Jan 2, 2020 at 16:34",
"Jan 2, 2020 at 14:04", "Jan 2, 2020 at 15:34",
}, {
"Jan 2, 2020 at 15:04", "Jan 2, 2020 at 17:34",
"Jan 1, 2020 at 15:04", "Jan 1, 2020 at 17:34",
}, {
"Jan 2, 2020 at 15:04", "Jan 3, 2020 at 17:34",
"Jan 1, 2020 at 15:04", "Jan 2, 2020 at 17:34",
}, {
"Jan 10, 2020 at 15:04", "Jan 13, 2020 at 17:34",
"Jan 3, 2020 at 15:04", "Jan 6, 2020 at 17:34",
}, {
"Jan 10, 2020 at 15:04", "Jan 15, 2020 at 17:34",
"Jan 3, 2020 at 15:04", "Jan 8, 2020 at 17:34",
}, {
"Jan 10, 2020 at 15:04", "Jan 20, 2020 at 17:34",
"Jan 3, 2020 at 15:04", "Jan 13, 2020 at 17:34",
}, {
"Feb 10, 2020 at 15:04", "Feb 25, 2020 at 17:34",
"Jan 13, 2020 at 15:04", "Jan 28, 2020 at 17:34",
}, {
"Feb 10, 2020 at 15:04", "Mar 25, 2020 at 17:34",
"Jan 13, 2020 at 15:04", "Feb 26, 2020 at 17:34",
}, {
"Feb 10, 2020 at 15:04", "Jul 25, 2020 at 17:34",
"Feb 10, 2019 at 15:04", "Jul 25, 2019 at 17:34",
}, {
"Feb 10, 2019 at 15:04", "Jul 25, 2020 at 17:34",
"Feb 10, 2018 at 15:04", "Jul 25, 2019 at 17:34",
},
}
for _, tc := range cases {
t.Run(fmt.Sprintf("%s to %s", tc.Start, tc.End), func(t *testing.T) {
start, err := time.Parse(longForm, tc.Start)
if err != nil {
t.Fatalf("time.Parse(%q) error:\n%+v", tc.Start, err)
}
end, err := time.Parse(longForm, tc.End)
if err != nil {
t.Fatalf("time.Parse(%q) error:\n%+v", tc.End, err)
}
expectedStart, err := time.Parse(longForm, tc.ExpectedStart)
if err != nil {
t.Fatalf("time.Parse(%q) error:\n%+v", tc.ExpectedStart, err)
}
expectedEnd, err := time.Parse(longForm, tc.ExpectedEnd)
if err != nil {
t.Fatalf("time.Parse(%q) error:\n%+v", tc.ExpectedEnd, err)
}
input := graphHandlerInput{
Start: start,
End: end,
Dimensions: []queryColumn{
queryColumnExporterAddress,
queryColumnExporterName,
},
}
got := input.previousPeriod()
expected := graphHandlerInput{
Start: expectedStart,
End: expectedEnd,
Dimensions: []queryColumn{},
}
if diff := helpers.Diff(got, expected); diff != "" {
t.Fatalf("previousPeriod() (-got, +want):\n%s", diff)
}
})
}
}
func TestGraphQuerySQL(t *testing.T) {
cases := []struct {
Description string
@@ -308,6 +390,53 @@ ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }})
{{ end }}`,
}, {
Description: "no filters, previous period",
Input: graphHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Limit: 20,
Dimensions: []queryColumn{
queryColumnExporterName,
queryColumnInIfProvider,
},
Filter: queryFilter{},
Units: "l3bps",
PreviousPeriod: true,
},
Expected: `
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
WITH
rows AS (SELECT ExporterName, InIfProvider FROM {{ .Table }} WHERE {{ .Timefilter }} GROUP BY ExporterName, InIfProvider ORDER BY SUM(Bytes) DESC LIMIT 20)
SELECT 1 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
{{ .Units }}/{{ .Interval }} AS xps,
if((ExporterName, InIfProvider) IN rows, [ExporterName, InIfProvider], ['Other', 'Other']) AS dimensions
FROM {{ .Table }}
WHERE {{ .Timefilter }}
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM {{ .TimefilterStart }}
TO {{ .TimefilterEnd }} + INTERVAL 1 second
STEP {{ .Interval }})
{{ end }}
UNION ALL
{{ with context @@{"start":"2022-04-09T15:45:10Z","end":"2022-04-10T15:45:10Z","start-for-interval":"2022-04-10T15:45:10Z","points":100,"units":"l3bps"}@@ }}
SELECT 3 AS axis, * FROM (
SELECT
{{ call .ToStartOfInterval "TimeReceived" }} + INTERVAL 86400 second AS time,
{{ .Units }}/{{ .Interval }} AS xps,
emptyArrayString() AS dimensions
FROM {{ .Table }}
WHERE {{ .Timefilter }}
GROUP BY time, dimensions
ORDER BY time WITH FILL
FROM {{ .TimefilterStart }} + INTERVAL 86400 second
TO {{ .TimefilterEnd }} + INTERVAL 1 second + INTERVAL 86400 second
STEP {{ .Interval }})
{{ end }}`,
},
}
@@ -400,6 +529,36 @@ func TestGraphHandler(t *testing.T) {
SetArg(1, expectedSQL).
Return(nil)
// Previous period
expectedSQL = []struct {
Axis uint8 `ch:"axis"`
Time time.Time `ch:"time"`
Xps float64 `ch:"xps"`
Dimensions []string `ch:"dimensions"`
}{
{1, base, 1000, []string{"router1", "provider1"}},
{1, base, 2000, []string{"router1", "provider2"}},
{1, base, 1200, []string{"router2", "provider2"}},
{1, base, 1100, []string{"router2", "provider3"}},
{1, base, 1900, []string{"Other", "Other"}},
{1, base.Add(time.Minute), 500, []string{"router1", "provider1"}},
{1, base.Add(time.Minute), 5000, []string{"router1", "provider2"}},
{1, base.Add(time.Minute), 900, []string{"router2", "provider4"}},
{1, base.Add(time.Minute), 100, []string{"Other", "Other"}},
{1, base.Add(2 * time.Minute), 100, []string{"router1", "provider1"}},
{1, base.Add(2 * time.Minute), 3000, []string{"router1", "provider2"}},
{1, base.Add(2 * time.Minute), 100, []string{"router2", "provider4"}},
{1, base.Add(2 * time.Minute), 100, []string{"Other", "Other"}},
{3, base, 8000, []string{}},
{3, base.Add(time.Minute), 6000, []string{}},
{3, base.Add(2 * time.Minute), 4500, []string{}},
}
mockConn.EXPECT().
Select(gomock.Any(), gomock.Any(), gomock.Any()).
SetArg(1, expectedSQL).
Return(nil)
helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
{
Description: "single direction",
@@ -472,6 +631,9 @@ func TestGraphHandler(t *testing.T) {
"axis": []int{
1, 1, 1, 1, 1, 1,
},
"axis-names": map[int]string{
1: "Direct",
},
},
}, {
Description: "bidirectional",
@@ -587,6 +749,94 @@ func TestGraphHandler(t *testing.T) {
1, 1, 1, 1, 1, 1,
2, 2, 2, 2, 2, 2,
},
"axis-names": map[int]string{
1: "Direct",
2: "Reverse",
},
},
}, {
Description: "previous period",
URL: "/api/v0/console/graph",
JSONInput: gin.H{
"start": time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
"end": time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
"points": 100,
"limit": 20,
"dimensions": []string{"ExporterName", "InIfProvider"},
"filter": "DstCountry = 'FR' AND SrcCountry = 'US'",
"units": "l3bps",
"bidirectional": false,
"previous-period": true,
},
JSONOutput: gin.H{
// Sorted by sum of bps
"rows": [][]string{
{"router1", "provider2"}, // 10000
{"router1", "provider1"}, // 1600
{"router2", "provider2"}, // 1200
{"router2", "provider3"}, // 1100
{"router2", "provider4"}, // 1000
{"Other", "Other"}, // 2100
{"Other", "Other"}, // Previous day
},
"t": []string{
"2009-11-10T23:00:00Z",
"2009-11-10T23:01:00Z",
"2009-11-10T23:02:00Z",
},
"points": [][]int{
{2000, 5000, 3000},
{1000, 500, 100},
{1200, 0, 0},
{1100, 0, 0},
{0, 900, 100},
{1900, 100, 100},
{8000, 6000, 4500},
},
"min": []int{
2000,
100,
1200,
1100,
100,
100,
4500,
},
"max": []int{
5000,
1000,
1200,
1100,
900,
1900,
8000,
},
"average": []int{
3333,
533,
400,
366,
333,
700,
6166,
},
"95th": []int{
4000,
750,
600,
550,
500,
1000,
7000,
},
"axis": []int{
1, 1, 1, 1, 1, 1,
3,
},
"axis-names": map[int]string{
1: "Direct",
3: "Previous day",
},
},
},
})