mirror of
https://github.com/akvorado/akvorado.git
synced 2025-12-12 06:24:10 +01:00
console: add "previous period" mode
This displays a line for the previous period on stacked graphs. Previous period depends on the current period. It could be hour, day, week, month, or year.
This commit is contained in:
@@ -98,6 +98,7 @@ func (c *Component) finalizeQuery(query string) string {
|
||||
type inputContext struct {
|
||||
Start time.Time `json:"start"`
|
||||
End time.Time `json:"end"`
|
||||
StartForInterval *time.Time `json:"start-for-interval,omitempty"`
|
||||
MainTableRequired bool `json:"main-table-required,omitempty"`
|
||||
Points uint `json:"points"`
|
||||
Units string `json:"units,omitempty"`
|
||||
@@ -152,59 +153,14 @@ func (c *Component) contextFunc(inputStr string) context {
|
||||
targetInterval = time.Second
|
||||
}
|
||||
|
||||
c.flowsTablesLock.RLock()
|
||||
defer c.flowsTablesLock.RUnlock()
|
||||
|
||||
// Select table
|
||||
table := "flows"
|
||||
computedInterval := time.Second
|
||||
if !input.MainTableRequired && len(c.flowsTables) > 0 {
|
||||
// We can use the consolidated data. The first
|
||||
// criteria is to find the tables matching the time
|
||||
// criteria.
|
||||
candidates := []int{}
|
||||
for idx, table := range c.flowsTables {
|
||||
if input.Start.After(table.Oldest.Add(table.Resolution)) {
|
||||
candidates = append(candidates, idx)
|
||||
targetIntervalForTableSelection := targetInterval
|
||||
if input.MainTableRequired {
|
||||
targetIntervalForTableSelection = time.Second
|
||||
}
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
// No candidate, fallback to the one with oldest data
|
||||
best := 0
|
||||
for idx, table := range c.flowsTables {
|
||||
if c.flowsTables[best].Oldest.After(table.Oldest.Add(table.Resolution)) {
|
||||
best = idx
|
||||
}
|
||||
}
|
||||
candidates = []int{best}
|
||||
// Add other candidates that are not far off in term of oldest data
|
||||
for idx, table := range c.flowsTables {
|
||||
if idx == best {
|
||||
continue
|
||||
}
|
||||
if c.flowsTables[best].Oldest.After(table.Oldest) {
|
||||
candidates = append(candidates, idx)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(candidates) > 1 {
|
||||
// Use interval to find the best one
|
||||
best := 0
|
||||
for _, idx := range candidates {
|
||||
if c.flowsTables[idx].Resolution > targetInterval {
|
||||
continue
|
||||
}
|
||||
if c.flowsTables[idx].Resolution > c.flowsTables[best].Resolution {
|
||||
best = idx
|
||||
}
|
||||
}
|
||||
candidates = []int{best}
|
||||
}
|
||||
table = c.flowsTables[candidates[0]].Name
|
||||
computedInterval = c.flowsTables[candidates[0]].Resolution
|
||||
}
|
||||
if computedInterval < time.Second {
|
||||
computedInterval = time.Second
|
||||
table, computedInterval := c.getBestTable(input.Start, targetIntervalForTableSelection)
|
||||
if input.StartForInterval != nil {
|
||||
_, computedInterval = c.getBestTable(*input.StartForInterval, targetIntervalForTableSelection)
|
||||
}
|
||||
|
||||
// Make start/end match the computed interval (currently equal to the table resolution)
|
||||
@@ -257,3 +213,61 @@ func (c *Component) contextFunc(inputStr string) context {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Get the best table starting at the specified time.
|
||||
func (c *Component) getBestTable(start time.Time, targetInterval time.Duration) (string, time.Duration) {
|
||||
c.flowsTablesLock.RLock()
|
||||
defer c.flowsTablesLock.RUnlock()
|
||||
|
||||
table := "flows"
|
||||
computedInterval := time.Second
|
||||
if len(c.flowsTables) > 0 {
|
||||
// We can use the consolidated data. The first
|
||||
// criteria is to find the tables matching the time
|
||||
// criteria.
|
||||
candidates := []int{}
|
||||
for idx, table := range c.flowsTables {
|
||||
if start.After(table.Oldest.Add(table.Resolution)) {
|
||||
candidates = append(candidates, idx)
|
||||
}
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
// No candidate, fallback to the one with oldest data
|
||||
best := 0
|
||||
for idx, table := range c.flowsTables {
|
||||
if c.flowsTables[best].Oldest.After(table.Oldest.Add(table.Resolution)) {
|
||||
best = idx
|
||||
}
|
||||
}
|
||||
candidates = []int{best}
|
||||
// Add other candidates that are not far off in term of oldest data
|
||||
for idx, table := range c.flowsTables {
|
||||
if idx == best {
|
||||
continue
|
||||
}
|
||||
if c.flowsTables[best].Oldest.After(table.Oldest) {
|
||||
candidates = append(candidates, idx)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(candidates) > 1 {
|
||||
// Use interval to find the best one
|
||||
best := 0
|
||||
for _, idx := range candidates {
|
||||
if c.flowsTables[idx].Resolution > targetInterval {
|
||||
continue
|
||||
}
|
||||
if c.flowsTables[idx].Resolution > c.flowsTables[best].Resolution {
|
||||
best = idx
|
||||
}
|
||||
}
|
||||
candidates = []int{best}
|
||||
}
|
||||
table = c.flowsTables[candidates[0]].Name
|
||||
computedInterval = c.flowsTables[candidates[0]].Resolution
|
||||
}
|
||||
if computedInterval < time.Second {
|
||||
computedInterval = time.Second
|
||||
}
|
||||
return table, computedInterval
|
||||
}
|
||||
|
||||
@@ -167,6 +167,36 @@ func TestFinalizeQuery(t *testing.T) {
|
||||
Points: 720, // 2-minute resolution,
|
||||
},
|
||||
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')",
|
||||
}, {
|
||||
Description: "use flows table for resolution (control for next case)",
|
||||
Tables: []flowsTable{
|
||||
{"flows", 0, time.Date(2022, 04, 10, 10, 45, 10, 0, time.UTC)},
|
||||
{"flows_1m0s", time.Minute, time.Date(2022, 03, 10, 10, 45, 10, 0, time.UTC)},
|
||||
},
|
||||
Query: "SELECT 1 FROM {{ .Table }} WHERE {{ .Timefilter }} // {{ .Interval }}",
|
||||
Context: inputContext{
|
||||
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
|
||||
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
|
||||
Points: 2880, // 30-second resolution
|
||||
},
|
||||
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC') // 30",
|
||||
}, {
|
||||
Description: "use flows table for resolution (but flows_1m0s for data)",
|
||||
Tables: []flowsTable{
|
||||
{"flows", 0, time.Date(2022, 04, 10, 10, 45, 10, 0, time.UTC)},
|
||||
{"flows_1m0s", time.Minute, time.Date(2022, 03, 10, 10, 45, 10, 0, time.UTC)},
|
||||
},
|
||||
Query: "SELECT 1 FROM {{ .Table }} WHERE {{ .Timefilter }} // {{ .Interval }}",
|
||||
Context: inputContext{
|
||||
Start: time.Date(2022, 03, 10, 15, 45, 10, 0, time.UTC),
|
||||
End: time.Date(2022, 03, 11, 15, 45, 10, 0, time.UTC),
|
||||
StartForInterval: func() *time.Time {
|
||||
t := time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC)
|
||||
return &t
|
||||
}(),
|
||||
Points: 2880, // 30-second resolution
|
||||
},
|
||||
Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-03-10 15:45:10', 'UTC') AND toDateTime('2022-03-11 15:45:10', 'UTC') // 30",
|
||||
}, {
|
||||
Description: "select flows table with better resolution",
|
||||
Tables: []flowsTable{
|
||||
|
||||
@@ -115,10 +115,19 @@ aspect of the graph.
|
||||
- The unit to use on the Y-axis: layer-3 bits per second, layer-2 bits
|
||||
per second (should match interface counters), or packets par second.
|
||||
|
||||
- Four graph types are provided: “stacked”, “lines” and “grid” to
|
||||
- Four graph types are provided: “stacked”, “lines”, and “grid” to
|
||||
display time series and “sankey” to show flow distributions between
|
||||
various dimensions.
|
||||
|
||||
- For “stacked”, “lines”, and “grid” graphs, the *bidirectional*
|
||||
option adds the flows in the opposite direction to the graph. They
|
||||
are displayed as a negative value on the graph.
|
||||
|
||||
- For “stacked” graphs, the *previous period* option adds a line for
|
||||
the traffic levels as they were on the previous period. Depending on
|
||||
the current period, the previous period can be the previous hour,
|
||||
day, week, month, or year.
|
||||
|
||||
- The time range can be set from a list of preset or directly using
|
||||
natural language. The parsing is done by
|
||||
[SugarJS](https://sugarjs.com/dates/#/Parsing) which provides
|
||||
|
||||
@@ -13,7 +13,8 @@ identified with a specific icon:
|
||||
|
||||
## Unreleased
|
||||
|
||||
- ✨ *console*: add a bidirectional mode for graphs to also display flows in the opposite direction
|
||||
- ✨ *console*: add an option to also display flows in the opposite direction on time series graph
|
||||
- ✨ *console*: add an option to also display the previous period (day, week, month, year) on stacked graphs
|
||||
- 🌱 *inlet*: Kafka key is now a 4-byte random value making scaling less dependent on the number of exporters
|
||||
- 🌱 *demo-exporter*: add a setting to automatically generate a reverse flow
|
||||
- 🌱 *docker-compose*: loosen required privileges for `conntrack-fixer`
|
||||
|
||||
@@ -62,7 +62,7 @@ import DataGraph from "./VisualizePage/DataGraph.vue";
|
||||
import OptionsPanel from "./VisualizePage/OptionsPanel.vue";
|
||||
import RequestSummary from "./VisualizePage/RequestSummary.vue";
|
||||
import { graphTypes } from "./VisualizePage/constants";
|
||||
import { isEqual } from "lodash-es";
|
||||
import { isEqual, omit } from "lodash-es";
|
||||
|
||||
const graphHeight = ref(500);
|
||||
const highlightedSerie = ref(null);
|
||||
@@ -109,12 +109,16 @@ const encodedState = computed(() => encodeState(state.value));
|
||||
|
||||
// Fetch data
|
||||
const fetchedData = ref({});
|
||||
const payload = computed(() => ({
|
||||
const finalState = computed(() => ({
|
||||
...state.value,
|
||||
start: SugarDate.create(state.value.start),
|
||||
end: SugarDate.create(state.value.end),
|
||||
}));
|
||||
const request = ref({}); // Same as payload, but once request is successful
|
||||
const jsonPayload = computed(() => ({
|
||||
...omit(finalState.value, ["previousPeriod", "graphType"]),
|
||||
"previous-period": finalState.value.previousPeriod,
|
||||
}));
|
||||
const request = ref({}); // Same as finalState, but once request is successful
|
||||
const { data, isFetching, aborted, abort, canAbort, error } = useFetch("", {
|
||||
beforeFetch(ctx) {
|
||||
// Add the URL. Not a computed value as if we change both payload
|
||||
@@ -146,12 +150,7 @@ const { data, isFetching, aborted, abort, canAbort, error } = useFetch("", {
|
||||
console.groupEnd();
|
||||
fetchedData.value = {
|
||||
...data,
|
||||
dimensions: payload.value.dimensions,
|
||||
start: payload.value.start,
|
||||
end: payload.value.end,
|
||||
graphType: payload.value.graphType,
|
||||
units: payload.value.units,
|
||||
bidirectional: payload.value.bidirectional,
|
||||
...omit(finalState.value, ["limit", "filter", "points"]),
|
||||
};
|
||||
|
||||
// Also update URL.
|
||||
@@ -166,13 +165,13 @@ const { data, isFetching, aborted, abort, canAbort, error } = useFetch("", {
|
||||
}
|
||||
|
||||
// Keep current payload for state
|
||||
request.value = payload.value;
|
||||
request.value = finalState.value;
|
||||
|
||||
return ctx;
|
||||
},
|
||||
refetch: true,
|
||||
})
|
||||
.post(payload)
|
||||
.post(jsonPayload)
|
||||
.json();
|
||||
const errorMessage = computed(
|
||||
() =>
|
||||
|
||||
@@ -63,49 +63,6 @@ const commonGraph = {
|
||||
brush: {
|
||||
xAxisIndex: "all",
|
||||
},
|
||||
tooltip: {
|
||||
confine: true,
|
||||
trigger: "axis",
|
||||
axisPointer: {
|
||||
type: "cross",
|
||||
label: { backgroundColor: "#6a7985" },
|
||||
},
|
||||
formatter: (params) => {
|
||||
// We will use a custom formatter, notably to handle bidirectional tooltips.
|
||||
if (params.length === 0) return;
|
||||
let table = [],
|
||||
bidirectional = false;
|
||||
params.forEach((param) => {
|
||||
let idx = findIndex(table, (r) => r.seriesName === param.seriesName);
|
||||
if (idx === -1) {
|
||||
table.push({
|
||||
marker: param.marker,
|
||||
seriesName: param.seriesName,
|
||||
});
|
||||
idx = table.length - 1;
|
||||
}
|
||||
const val = param.value[param.seriesIndex + 1];
|
||||
if (table[idx].col1 !== undefined || val < 0) {
|
||||
table[idx].col2 = val;
|
||||
bidirectional = true;
|
||||
} else table[idx].col1 = val;
|
||||
});
|
||||
const rows = table
|
||||
.map(
|
||||
(row) => `<tr>
|
||||
<td>${row.marker} ${row.seriesName}</td>
|
||||
<td class="pl-2">${bidirectional ? "↑" : ""}<b>${formatXps(
|
||||
row.col1 || 0
|
||||
)}</b></td>
|
||||
<td class="pl-2">${bidirectional ? "↓" : ""}<b>${
|
||||
bidirectional ? formatXps(row.col2 || 0) : ""
|
||||
}</b></td>
|
||||
</tr>`
|
||||
)
|
||||
.join("");
|
||||
return `${params[0].axisValueLabel}<table>${rows}</table>`;
|
||||
},
|
||||
},
|
||||
};
|
||||
const graph = computed(() => {
|
||||
const theme = isDark.value ? "dark" : "light";
|
||||
@@ -123,7 +80,7 @@ const graph = computed(() => {
|
||||
// Unfortunately, eCharts does not seem to make it easy
|
||||
// to inverse an axis and put the result below. Therefore,
|
||||
// we use negative values for the second axis.
|
||||
(row, rowIdx) => row[timeIdx] * (data.axis[rowIdx] == 1 ? 1 : -1)
|
||||
(row, rowIdx) => row[timeIdx] * (data.axis[rowIdx] % 2 ? 1 : -1)
|
||||
),
|
||||
])
|
||||
.slice(0, -1),
|
||||
@@ -141,6 +98,57 @@ const graph = computed(() => {
|
||||
axisPointer: {
|
||||
label: { formatter: ({ value }) => formatXps(value) },
|
||||
},
|
||||
},
|
||||
tooltip = {
|
||||
confine: true,
|
||||
trigger: "axis",
|
||||
axisPointer: {
|
||||
type: "cross",
|
||||
label: { backgroundColor: "#6a7985" },
|
||||
},
|
||||
formatter: (params) => {
|
||||
// We will use a custom formatter, notably to handle bidirectional tooltips.
|
||||
if (params.length === 0) return;
|
||||
|
||||
let table = [];
|
||||
params.forEach((param) => {
|
||||
const axis = data.axis[param.seriesIndex];
|
||||
const seriesName = [1, 2].includes(axis)
|
||||
? param.seriesName
|
||||
: data["axis-names"][axis];
|
||||
const key = `${Math.floor((axis - 1) / 2)}-${seriesName}`;
|
||||
let idx = findIndex(table, (r) => r.key === key);
|
||||
if (idx === -1) {
|
||||
table.push({
|
||||
key,
|
||||
seriesName,
|
||||
marker: param.marker,
|
||||
up: 0,
|
||||
down: 0,
|
||||
});
|
||||
idx = table.length - 1;
|
||||
}
|
||||
const val = param.value[param.seriesIndex + 1];
|
||||
if (axis % 2 == 1) table[idx].up = val;
|
||||
else table[idx].down = val;
|
||||
});
|
||||
const rows = table
|
||||
.map((row) =>
|
||||
[
|
||||
`<tr>`,
|
||||
`<td>${row.marker} ${row.seriesName}</td>`,
|
||||
`<td class="pl-2">${data.bidirectional ? "↑" : ""}<b>${formatXps(
|
||||
row.up || 0
|
||||
)}</b></td>`,
|
||||
data.bidirectional
|
||||
? `<td class="pl-2">↓<b>${formatXps(row.down || 0)}</b></td>`
|
||||
: "",
|
||||
`</tr>`,
|
||||
].join("")
|
||||
)
|
||||
.join("");
|
||||
return `${params[0].axisValueLabel}<table>${rows}</table>`;
|
||||
},
|
||||
};
|
||||
|
||||
// Lines and stacked areas
|
||||
@@ -158,6 +166,7 @@ const graph = computed(() => {
|
||||
xAxis,
|
||||
yAxis,
|
||||
dataset,
|
||||
tooltip,
|
||||
series: data.rows
|
||||
.map((row, idx) => {
|
||||
const isOther = row.some((name) => name === "Other"),
|
||||
@@ -185,7 +194,23 @@ const graph = computed(() => {
|
||||
seriesId: idx + 1,
|
||||
},
|
||||
};
|
||||
if (data.graphType === graphTypes.stacked) {
|
||||
if ([3, 4].includes(data.axis[idx])) {
|
||||
serie = {
|
||||
...serie,
|
||||
itemStyle: {
|
||||
color: dataColorGrey(1, false, theme),
|
||||
},
|
||||
lineStyle: {
|
||||
color: dataColorGrey(1, false, theme),
|
||||
width: 1,
|
||||
type: "dashed",
|
||||
},
|
||||
};
|
||||
}
|
||||
if (
|
||||
data.graphType === graphTypes.stacked &&
|
||||
[1, 2].includes(data.axis[idx])
|
||||
) {
|
||||
serie = {
|
||||
...serie,
|
||||
stack: data.axis[idx],
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
v-if="axes.length > 1"
|
||||
class="border-b border-gray-200 text-center text-sm font-medium text-gray-500 dark:border-gray-700 dark:text-gray-400"
|
||||
>
|
||||
<ul class="-mb-px flex flex-wrap">
|
||||
<ul class="flex flex-wrap">
|
||||
<li v-for="{ id: axis, name } in axes" :key="axis" class="mr-2">
|
||||
<button
|
||||
class="pointer-cursor inline-block rounded-t-lg border-b-2 border-transparent p-4 hover:border-gray-300 hover:text-gray-600 dark:hover:text-gray-300"
|
||||
@@ -99,7 +99,7 @@ import { graphTypes } from "./constants";
|
||||
const { isDark } = inject("theme");
|
||||
const { stacked, lines, grid, sankey } = graphTypes;
|
||||
|
||||
import { uniq, uniqWith, isEqual, findIndex, takeWhile } from "lodash-es";
|
||||
import { uniqWith, isEqual, findIndex, takeWhile, toPairs } from "lodash-es";
|
||||
|
||||
const highlight = (index) => {
|
||||
if (index === null) {
|
||||
@@ -119,10 +119,10 @@ const highlight = (index) => {
|
||||
emit("highlighted", originalIndex);
|
||||
};
|
||||
const axes = computed(() =>
|
||||
uniq(props.data.axis ?? []).map((axis) => ({
|
||||
id: axis,
|
||||
name: { 1: "Direct", 2: "Reverse" }[axis] ?? "Unknown",
|
||||
}))
|
||||
toPairs(props.data["axis-names"])
|
||||
.map(([k, v]) => ({ id: Number(k), name: v }))
|
||||
.filter(({ id }) => [1, 2].includes(id))
|
||||
.sort(({ id: id1 }, { id: id2 }) => id1 - id2)
|
||||
);
|
||||
const selectedAxis = ref(1);
|
||||
const displayedAxis = computed(() =>
|
||||
|
||||
@@ -71,12 +71,20 @@
|
||||
</div>
|
||||
</template>
|
||||
</InputListBox>
|
||||
<div
|
||||
class="order-4 flex grow flex-row justify-between gap-x-3 sm:order-2 sm:grow-0 sm:flex-col lg:order-4 lg:grow lg:flex-row"
|
||||
>
|
||||
<InputCheckbox
|
||||
v-if="[stacked, lines, grid].includes(graphType.name)"
|
||||
v-model="bidirectional"
|
||||
class="order-4 sm:order-2 lg:order-4"
|
||||
label="Bidirectional"
|
||||
/>
|
||||
<InputCheckbox
|
||||
v-if="[stacked].includes(graphType.name)"
|
||||
v-model="previousPeriod"
|
||||
label="Previous period"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<SectionLabel>Time range</SectionLabel>
|
||||
<InputTimeRange v-model="timeRange" />
|
||||
@@ -141,6 +149,7 @@ const dimensions = ref([]);
|
||||
const filter = ref({});
|
||||
const units = ref("l3bps");
|
||||
const bidirectional = ref(false);
|
||||
const previousPeriod = ref(false);
|
||||
|
||||
const options = computed(() => ({
|
||||
// Common to all graph types
|
||||
@@ -151,13 +160,16 @@ const options = computed(() => ({
|
||||
limit: dimensions.value.limit,
|
||||
filter: filter.value.expression,
|
||||
units: units.value,
|
||||
// Only for time series
|
||||
// Depending on the graph type...
|
||||
...([stacked, lines].includes(graphType.value.name) && {
|
||||
bidirectional: bidirectional.value,
|
||||
previousPeriod:
|
||||
graphType.value.name === stacked ? previousPeriod.value : false,
|
||||
points: 200,
|
||||
}),
|
||||
...(graphType.value.name === grid && {
|
||||
bidirectional: bidirectional.value,
|
||||
previousPeriod: false,
|
||||
points: 50,
|
||||
}),
|
||||
}));
|
||||
@@ -184,6 +196,7 @@ watch(
|
||||
filter: _filter = defaultOptions?.filter,
|
||||
units: _units = "l3bps",
|
||||
bidirectional: _bidirectional = false,
|
||||
previousPeriod: _previousPeriod = false,
|
||||
} = modelValue;
|
||||
|
||||
// Dispatch values in refs
|
||||
@@ -197,6 +210,7 @@ watch(
|
||||
filter.value = { expression: _filter };
|
||||
units.value = _units;
|
||||
bidirectional.value = _bidirectional;
|
||||
previousPeriod.value = _previousPeriod;
|
||||
|
||||
// A bit risky, but it seems to work.
|
||||
if (!isEqual(modelValue, options.value)) {
|
||||
|
||||
102
console/graph.go
102
console/graph.go
@@ -25,6 +25,7 @@ type graphHandlerInput struct {
|
||||
Filter queryFilter `json:"filter"` // where ...
|
||||
Units string `json:"units" binding:"required,oneof=pps l2bps l3bps"`
|
||||
Bidirectional bool `json:"bidirectional"`
|
||||
PreviousPeriod bool `json:"previous-period"`
|
||||
}
|
||||
|
||||
// graphHandlerOutput describes the output for the /graph endpoint. A
|
||||
@@ -36,6 +37,7 @@ type graphHandlerOutput struct {
|
||||
Rows [][]string `json:"rows"` // List of rows
|
||||
Points [][]int `json:"points"` // t → row → xps
|
||||
Axis []int `json:"axis"` // row → axis
|
||||
AxisNames map[int]string `json:"axis-names"`
|
||||
Average []int `json:"average"` // row → average xps
|
||||
Min []int `json:"min"` // row → min xps
|
||||
Max []int `json:"max"` // row → max xps
|
||||
@@ -53,12 +55,66 @@ func (input graphHandlerInput) reverseDirection() graphHandlerInput {
|
||||
return input
|
||||
}
|
||||
|
||||
func (input graphHandlerInput) toSQL1(axis int, skipWith bool) string {
|
||||
// nearestPeriod returns the name and period matching the provided
|
||||
// period length. The year is a special case as we don't know its
|
||||
// exact length.
|
||||
func nearestPeriod(period time.Duration) (time.Duration, string) {
|
||||
switch {
|
||||
case period < 2*time.Hour:
|
||||
return time.Hour, "hour"
|
||||
case period < 2*24*time.Hour:
|
||||
return 24 * time.Hour, "day"
|
||||
case period < 2*7*24*time.Hour:
|
||||
return 7 * 24 * time.Hour, "week"
|
||||
case period < 2*4*7*24*time.Hour:
|
||||
// We use 4 weeks, not 1 month
|
||||
return 4 * 7 * 24 * time.Hour, "month"
|
||||
default:
|
||||
return 0, "year"
|
||||
}
|
||||
}
|
||||
|
||||
// previousPeriod shifts the provided input to the previous period.
|
||||
// The chosen period depend on the current period. For less than
|
||||
// 2-hour period, the previous period is the hour. For less than 2-day
|
||||
// period, this is the day. For less than 2-weeks, this is the week,
|
||||
// for less than 2-months, this is the month, otherwise, this is the
|
||||
// year. Also, dimensions are stripped.
|
||||
func (input graphHandlerInput) previousPeriod() graphHandlerInput {
|
||||
input.Dimensions = []queryColumn{}
|
||||
diff := input.End.Sub(input.Start)
|
||||
period, _ := nearestPeriod(diff)
|
||||
if period == 0 {
|
||||
// We use a full year this time (think for example we
|
||||
// want to see how was New Year Eve compared to last
|
||||
// year)
|
||||
input.Start = input.Start.AddDate(-1, 0, 0)
|
||||
input.End = input.End.AddDate(-1, 0, 0)
|
||||
return input
|
||||
}
|
||||
input.Start = input.Start.Add(-period)
|
||||
input.End = input.End.Add(-period)
|
||||
return input
|
||||
}
|
||||
|
||||
type toSQL1Options struct {
|
||||
skipWithClause bool
|
||||
offsetedStart time.Time
|
||||
}
|
||||
|
||||
func (input graphHandlerInput) toSQL1(axis int, options toSQL1Options) string {
|
||||
var startForInterval *time.Time
|
||||
var offsetShift string
|
||||
if !options.offsetedStart.IsZero() {
|
||||
startForInterval = &options.offsetedStart
|
||||
offsetShift = fmt.Sprintf(" + INTERVAL %d second",
|
||||
int64(options.offsetedStart.Sub(input.Start).Seconds()))
|
||||
}
|
||||
where := templateWhere(input.Filter)
|
||||
|
||||
// Select
|
||||
fields := []string{
|
||||
`{{ call .ToStartOfInterval "TimeReceived" }} AS time`,
|
||||
fmt.Sprintf(`{{ call .ToStartOfInterval "TimeReceived" }}%s AS time`, offsetShift),
|
||||
`{{ .Units }}/{{ .Interval }} AS xps`,
|
||||
}
|
||||
selectFields := []string{}
|
||||
@@ -81,7 +137,7 @@ func (input graphHandlerInput) toSQL1(axis int, skipWith bool) string {
|
||||
|
||||
// With
|
||||
with := []string{}
|
||||
if len(dimensions) > 0 && !skipWith {
|
||||
if len(dimensions) > 0 && !options.skipWithClause {
|
||||
with = append(with, fmt.Sprintf(
|
||||
"rows AS (SELECT %s FROM {{ .Table }} WHERE %s GROUP BY %s ORDER BY SUM(Bytes) DESC LIMIT %d)",
|
||||
strings.Join(dimensions, ", "),
|
||||
@@ -103,28 +159,43 @@ FROM {{ .Table }}
|
||||
WHERE %s
|
||||
GROUP BY time, dimensions
|
||||
ORDER BY time WITH FILL
|
||||
FROM {{ .TimefilterStart }}
|
||||
TO {{ .TimefilterEnd }} + INTERVAL 1 second
|
||||
FROM {{ .TimefilterStart }}%s
|
||||
TO {{ .TimefilterEnd }} + INTERVAL 1 second%s
|
||||
STEP {{ .Interval }})
|
||||
{{ end }}`,
|
||||
templateContext(inputContext{
|
||||
Start: input.Start,
|
||||
End: input.End,
|
||||
StartForInterval: startForInterval,
|
||||
MainTableRequired: requireMainTable(input.Dimensions, input.Filter),
|
||||
Points: input.Points,
|
||||
Units: input.Units,
|
||||
}),
|
||||
withStr, axis, strings.Join(fields, ",\n "), where)
|
||||
withStr, axis, strings.Join(fields, ",\n "), where, offsetShift, offsetShift)
|
||||
return strings.TrimSpace(sqlQuery)
|
||||
}
|
||||
|
||||
// graphHandlerInputToSQL converts a graph input to an SQL request
|
||||
func (input graphHandlerInput) toSQL() string {
|
||||
parts := []string{input.toSQL1(1, false)}
|
||||
parts := []string{input.toSQL1(1, toSQL1Options{})}
|
||||
// Handle specific options. We have to align time periods in
|
||||
// case the previous period does not use the same offsets.
|
||||
if input.Bidirectional {
|
||||
parts = append(parts, input.reverseDirection().toSQL1(2, true))
|
||||
parts = append(parts, input.reverseDirection().toSQL1(2, toSQL1Options{skipWithClause: true}))
|
||||
}
|
||||
return strings.TrimSpace(strings.Join(parts, "\nUNION ALL\n"))
|
||||
if input.PreviousPeriod {
|
||||
parts = append(parts, input.previousPeriod().toSQL1(3, toSQL1Options{
|
||||
skipWithClause: true,
|
||||
offsetedStart: input.Start,
|
||||
}))
|
||||
}
|
||||
if input.Bidirectional && input.PreviousPeriod {
|
||||
parts = append(parts, input.reverseDirection().previousPeriod().toSQL1(4, toSQL1Options{
|
||||
skipWithClause: true,
|
||||
offsetedStart: input.Start,
|
||||
}))
|
||||
}
|
||||
return strings.Join(parts, "\nUNION ALL\n")
|
||||
}
|
||||
|
||||
func (c *Component) graphHandlerFunc(gc *gin.Context) {
|
||||
@@ -246,6 +317,7 @@ func (c *Component) graphHandlerFunc(gc *gin.Context) {
|
||||
}
|
||||
output.Rows = make([][]string, totalRows)
|
||||
output.Axis = make([]int, totalRows)
|
||||
output.AxisNames = make(map[int]string)
|
||||
output.Points = make([][]int, totalRows)
|
||||
output.Average = make([]int, totalRows)
|
||||
output.Min = make([]int, totalRows)
|
||||
@@ -303,5 +375,17 @@ func (c *Component) graphHandlerFunc(gc *gin.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
for _, axis := range output.Axis {
|
||||
switch axis {
|
||||
case 1:
|
||||
output.AxisNames[axis] = "Direct"
|
||||
case 2:
|
||||
output.AxisNames[axis] = "Reverse"
|
||||
case 3, 4:
|
||||
diff := input.End.Sub(input.Start)
|
||||
_, name := nearestPeriod(diff)
|
||||
output.AxisNames[axis] = fmt.Sprintf("Previous %s", name)
|
||||
}
|
||||
}
|
||||
gc.JSON(http.StatusOK, output)
|
||||
}
|
||||
|
||||
@@ -55,6 +55,88 @@ func TestGraphInputReverseDirection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphPreviousPeriod(t *testing.T) {
|
||||
const longForm = "Jan 2, 2006 at 15:04"
|
||||
cases := []struct {
|
||||
Start string
|
||||
End string
|
||||
ExpectedStart string
|
||||
ExpectedEnd string
|
||||
}{
|
||||
{
|
||||
"Jan 2, 2020 at 15:04", "Jan 2, 2020 at 16:04",
|
||||
"Jan 2, 2020 at 14:04", "Jan 2, 2020 at 15:04",
|
||||
}, {
|
||||
"Jan 2, 2020 at 15:04", "Jan 2, 2020 at 16:34",
|
||||
"Jan 2, 2020 at 14:04", "Jan 2, 2020 at 15:34",
|
||||
}, {
|
||||
"Jan 2, 2020 at 15:04", "Jan 2, 2020 at 17:34",
|
||||
"Jan 1, 2020 at 15:04", "Jan 1, 2020 at 17:34",
|
||||
}, {
|
||||
"Jan 2, 2020 at 15:04", "Jan 3, 2020 at 17:34",
|
||||
"Jan 1, 2020 at 15:04", "Jan 2, 2020 at 17:34",
|
||||
}, {
|
||||
"Jan 10, 2020 at 15:04", "Jan 13, 2020 at 17:34",
|
||||
"Jan 3, 2020 at 15:04", "Jan 6, 2020 at 17:34",
|
||||
}, {
|
||||
"Jan 10, 2020 at 15:04", "Jan 15, 2020 at 17:34",
|
||||
"Jan 3, 2020 at 15:04", "Jan 8, 2020 at 17:34",
|
||||
}, {
|
||||
"Jan 10, 2020 at 15:04", "Jan 20, 2020 at 17:34",
|
||||
"Jan 3, 2020 at 15:04", "Jan 13, 2020 at 17:34",
|
||||
}, {
|
||||
"Feb 10, 2020 at 15:04", "Feb 25, 2020 at 17:34",
|
||||
"Jan 13, 2020 at 15:04", "Jan 28, 2020 at 17:34",
|
||||
}, {
|
||||
"Feb 10, 2020 at 15:04", "Mar 25, 2020 at 17:34",
|
||||
"Jan 13, 2020 at 15:04", "Feb 26, 2020 at 17:34",
|
||||
}, {
|
||||
"Feb 10, 2020 at 15:04", "Jul 25, 2020 at 17:34",
|
||||
"Feb 10, 2019 at 15:04", "Jul 25, 2019 at 17:34",
|
||||
}, {
|
||||
"Feb 10, 2019 at 15:04", "Jul 25, 2020 at 17:34",
|
||||
"Feb 10, 2018 at 15:04", "Jul 25, 2019 at 17:34",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(fmt.Sprintf("%s to %s", tc.Start, tc.End), func(t *testing.T) {
|
||||
start, err := time.Parse(longForm, tc.Start)
|
||||
if err != nil {
|
||||
t.Fatalf("time.Parse(%q) error:\n%+v", tc.Start, err)
|
||||
}
|
||||
end, err := time.Parse(longForm, tc.End)
|
||||
if err != nil {
|
||||
t.Fatalf("time.Parse(%q) error:\n%+v", tc.End, err)
|
||||
}
|
||||
expectedStart, err := time.Parse(longForm, tc.ExpectedStart)
|
||||
if err != nil {
|
||||
t.Fatalf("time.Parse(%q) error:\n%+v", tc.ExpectedStart, err)
|
||||
}
|
||||
expectedEnd, err := time.Parse(longForm, tc.ExpectedEnd)
|
||||
if err != nil {
|
||||
t.Fatalf("time.Parse(%q) error:\n%+v", tc.ExpectedEnd, err)
|
||||
}
|
||||
input := graphHandlerInput{
|
||||
Start: start,
|
||||
End: end,
|
||||
Dimensions: []queryColumn{
|
||||
queryColumnExporterAddress,
|
||||
queryColumnExporterName,
|
||||
},
|
||||
}
|
||||
got := input.previousPeriod()
|
||||
expected := graphHandlerInput{
|
||||
Start: expectedStart,
|
||||
End: expectedEnd,
|
||||
Dimensions: []queryColumn{},
|
||||
}
|
||||
if diff := helpers.Diff(got, expected); diff != "" {
|
||||
t.Fatalf("previousPeriod() (-got, +want):\n%s", diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphQuerySQL(t *testing.T) {
|
||||
cases := []struct {
|
||||
Description string
|
||||
@@ -308,6 +390,53 @@ ORDER BY time WITH FILL
|
||||
FROM {{ .TimefilterStart }}
|
||||
TO {{ .TimefilterEnd }} + INTERVAL 1 second
|
||||
STEP {{ .Interval }})
|
||||
{{ end }}`,
|
||||
}, {
|
||||
Description: "no filters, previous period",
|
||||
Input: graphHandlerInput{
|
||||
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
|
||||
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
|
||||
Points: 100,
|
||||
Limit: 20,
|
||||
Dimensions: []queryColumn{
|
||||
queryColumnExporterName,
|
||||
queryColumnInIfProvider,
|
||||
},
|
||||
Filter: queryFilter{},
|
||||
Units: "l3bps",
|
||||
PreviousPeriod: true,
|
||||
},
|
||||
Expected: `
|
||||
{{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
|
||||
WITH
|
||||
rows AS (SELECT ExporterName, InIfProvider FROM {{ .Table }} WHERE {{ .Timefilter }} GROUP BY ExporterName, InIfProvider ORDER BY SUM(Bytes) DESC LIMIT 20)
|
||||
SELECT 1 AS axis, * FROM (
|
||||
SELECT
|
||||
{{ call .ToStartOfInterval "TimeReceived" }} AS time,
|
||||
{{ .Units }}/{{ .Interval }} AS xps,
|
||||
if((ExporterName, InIfProvider) IN rows, [ExporterName, InIfProvider], ['Other', 'Other']) AS dimensions
|
||||
FROM {{ .Table }}
|
||||
WHERE {{ .Timefilter }}
|
||||
GROUP BY time, dimensions
|
||||
ORDER BY time WITH FILL
|
||||
FROM {{ .TimefilterStart }}
|
||||
TO {{ .TimefilterEnd }} + INTERVAL 1 second
|
||||
STEP {{ .Interval }})
|
||||
{{ end }}
|
||||
UNION ALL
|
||||
{{ with context @@{"start":"2022-04-09T15:45:10Z","end":"2022-04-10T15:45:10Z","start-for-interval":"2022-04-10T15:45:10Z","points":100,"units":"l3bps"}@@ }}
|
||||
SELECT 3 AS axis, * FROM (
|
||||
SELECT
|
||||
{{ call .ToStartOfInterval "TimeReceived" }} + INTERVAL 86400 second AS time,
|
||||
{{ .Units }}/{{ .Interval }} AS xps,
|
||||
emptyArrayString() AS dimensions
|
||||
FROM {{ .Table }}
|
||||
WHERE {{ .Timefilter }}
|
||||
GROUP BY time, dimensions
|
||||
ORDER BY time WITH FILL
|
||||
FROM {{ .TimefilterStart }} + INTERVAL 86400 second
|
||||
TO {{ .TimefilterEnd }} + INTERVAL 1 second + INTERVAL 86400 second
|
||||
STEP {{ .Interval }})
|
||||
{{ end }}`,
|
||||
},
|
||||
}
|
||||
@@ -400,6 +529,36 @@ func TestGraphHandler(t *testing.T) {
|
||||
SetArg(1, expectedSQL).
|
||||
Return(nil)
|
||||
|
||||
// Previous period
|
||||
expectedSQL = []struct {
|
||||
Axis uint8 `ch:"axis"`
|
||||
Time time.Time `ch:"time"`
|
||||
Xps float64 `ch:"xps"`
|
||||
Dimensions []string `ch:"dimensions"`
|
||||
}{
|
||||
{1, base, 1000, []string{"router1", "provider1"}},
|
||||
{1, base, 2000, []string{"router1", "provider2"}},
|
||||
{1, base, 1200, []string{"router2", "provider2"}},
|
||||
{1, base, 1100, []string{"router2", "provider3"}},
|
||||
{1, base, 1900, []string{"Other", "Other"}},
|
||||
{1, base.Add(time.Minute), 500, []string{"router1", "provider1"}},
|
||||
{1, base.Add(time.Minute), 5000, []string{"router1", "provider2"}},
|
||||
{1, base.Add(time.Minute), 900, []string{"router2", "provider4"}},
|
||||
{1, base.Add(time.Minute), 100, []string{"Other", "Other"}},
|
||||
{1, base.Add(2 * time.Minute), 100, []string{"router1", "provider1"}},
|
||||
{1, base.Add(2 * time.Minute), 3000, []string{"router1", "provider2"}},
|
||||
{1, base.Add(2 * time.Minute), 100, []string{"router2", "provider4"}},
|
||||
{1, base.Add(2 * time.Minute), 100, []string{"Other", "Other"}},
|
||||
|
||||
{3, base, 8000, []string{}},
|
||||
{3, base.Add(time.Minute), 6000, []string{}},
|
||||
{3, base.Add(2 * time.Minute), 4500, []string{}},
|
||||
}
|
||||
mockConn.EXPECT().
|
||||
Select(gomock.Any(), gomock.Any(), gomock.Any()).
|
||||
SetArg(1, expectedSQL).
|
||||
Return(nil)
|
||||
|
||||
helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
|
||||
{
|
||||
Description: "single direction",
|
||||
@@ -472,6 +631,9 @@ func TestGraphHandler(t *testing.T) {
|
||||
"axis": []int{
|
||||
1, 1, 1, 1, 1, 1,
|
||||
},
|
||||
"axis-names": map[int]string{
|
||||
1: "Direct",
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Description: "bidirectional",
|
||||
@@ -587,6 +749,94 @@ func TestGraphHandler(t *testing.T) {
|
||||
1, 1, 1, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2,
|
||||
},
|
||||
"axis-names": map[int]string{
|
||||
1: "Direct",
|
||||
2: "Reverse",
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Description: "previous period",
|
||||
URL: "/api/v0/console/graph",
|
||||
JSONInput: gin.H{
|
||||
"start": time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
|
||||
"end": time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
|
||||
"points": 100,
|
||||
"limit": 20,
|
||||
"dimensions": []string{"ExporterName", "InIfProvider"},
|
||||
"filter": "DstCountry = 'FR' AND SrcCountry = 'US'",
|
||||
"units": "l3bps",
|
||||
"bidirectional": false,
|
||||
"previous-period": true,
|
||||
},
|
||||
JSONOutput: gin.H{
|
||||
// Sorted by sum of bps
|
||||
"rows": [][]string{
|
||||
{"router1", "provider2"}, // 10000
|
||||
{"router1", "provider1"}, // 1600
|
||||
{"router2", "provider2"}, // 1200
|
||||
{"router2", "provider3"}, // 1100
|
||||
{"router2", "provider4"}, // 1000
|
||||
{"Other", "Other"}, // 2100
|
||||
{"Other", "Other"}, // Previous day
|
||||
},
|
||||
"t": []string{
|
||||
"2009-11-10T23:00:00Z",
|
||||
"2009-11-10T23:01:00Z",
|
||||
"2009-11-10T23:02:00Z",
|
||||
},
|
||||
"points": [][]int{
|
||||
{2000, 5000, 3000},
|
||||
{1000, 500, 100},
|
||||
{1200, 0, 0},
|
||||
{1100, 0, 0},
|
||||
{0, 900, 100},
|
||||
{1900, 100, 100},
|
||||
{8000, 6000, 4500},
|
||||
},
|
||||
"min": []int{
|
||||
2000,
|
||||
100,
|
||||
1200,
|
||||
1100,
|
||||
100,
|
||||
100,
|
||||
4500,
|
||||
},
|
||||
"max": []int{
|
||||
5000,
|
||||
1000,
|
||||
1200,
|
||||
1100,
|
||||
900,
|
||||
1900,
|
||||
8000,
|
||||
},
|
||||
"average": []int{
|
||||
3333,
|
||||
533,
|
||||
400,
|
||||
366,
|
||||
333,
|
||||
700,
|
||||
6166,
|
||||
},
|
||||
"95th": []int{
|
||||
4000,
|
||||
750,
|
||||
600,
|
||||
550,
|
||||
500,
|
||||
1000,
|
||||
7000,
|
||||
},
|
||||
"axis": []int{
|
||||
1, 1, 1, 1, 1, 1,
|
||||
3,
|
||||
},
|
||||
"axis-names": map[int]string{
|
||||
1: "Direct",
|
||||
3: "Previous day",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user