console: add visualize tab

Currently, there is no controls available.
This commit is contained in:
Vincent Bernat
2022-05-10 09:29:53 +02:00
parent 24848e029e
commit 2319262340
17 changed files with 1509 additions and 27 deletions

39
common/helpers/bimap.go Normal file
View File

@@ -0,0 +1,39 @@
package helpers
import "fmt"
// Bimap is a bidirectional map.
type Bimap[K, V comparable] struct {
forward map[K]V
inverse map[V]K
}
// NewBimap returns a new bimap from an existing map.
func NewBimap[K, V comparable](input map[K]V) *Bimap[K, V] {
output := &Bimap[K, V]{
forward: make(map[K]V),
inverse: make(map[V]K),
}
for key, value := range input {
output.forward[key] = value
output.inverse[value] = key
}
return output
}
// LoadValue returns the value stored in the bimap for a key.
func (bi *Bimap[K, V]) LoadValue(k K) (V, bool) {
v, ok := bi.forward[k]
return v, ok
}
// LoadKey returns the key stored in the bimap for a value.
func (bi *Bimap[K, V]) LoadKey(v V) (K, bool) {
k, ok := bi.inverse[v]
return k, ok
}
// String returns a string representation of the bimap.
func (bi *Bimap[K, V]) String() string {
return fmt.Sprintf("Bi%v", bi.forward)
}

View File

@@ -0,0 +1,61 @@
package helpers_test
import (
"testing"
"akvorado/common/helpers"
)
func TestBimapLoadValue(t *testing.T) {
input := helpers.NewBimap(map[int]string{
1: "hello",
2: "world",
3: "happy",
})
cases := []struct {
key int
value string
ok bool
}{
{1, "hello", true},
{2, "world", true},
{10, "", false},
{0, "", false},
}
for _, tc := range cases {
got, ok := input.LoadValue(tc.key)
if ok != tc.ok {
t.Errorf("LoadValue(%q) ok: %v but expected %v", tc.key, ok, tc.ok)
}
if got != tc.value {
t.Errorf("LoadValue(%q) got: %q but expected %q", tc.key, got, tc.value)
}
}
}
func TestBimapLoadKey(t *testing.T) {
input := helpers.NewBimap(map[int]string{
1: "hello",
2: "world",
3: "happy",
})
cases := []struct {
value string
key int
ok bool
}{
{"hello", 1, true},
{"happy", 3, true},
{"", 0, false},
{"nothing", 0, false},
}
for _, tc := range cases {
got, ok := input.LoadKey(tc.value)
if ok != tc.ok {
t.Errorf("LoadKey(%q) ok: %v but expected %v", tc.value, ok, tc.ok)
}
if got != tc.key {
t.Errorf("LoadKey(%q) got: %q but expected %q", tc.value, got, tc.value)
}
}
}

View File

@@ -0,0 +1,3 @@
((web-mode . ((mode . apheleia)))
(js2-mode . ((mode . apheleia)))
(js-mode . ((mode . apheleia))))

View File

@@ -14,8 +14,10 @@
"@heroicons/vue": "^1.0.6",
"echarts": "^5.3.2",
"notiwind": "^1.2.5",
"sugar-date": "^2.0.6",
"vue": "^3.2.25",
"vue-echarts": "^6.0.2",
"vue-resizer": "^1.1.9",
"vue-router": "^4.0.14"
},
"devDependencies": {

View File

@@ -63,12 +63,24 @@
import { computed } from "vue";
import { useRoute } from "vue-router";
import { Disclosure, DisclosureButton, DisclosurePanel } from "@headlessui/vue";
import { HomeIcon, BookOpenIcon, MenuIcon, XIcon } from "@heroicons/vue/solid";
import {
HomeIcon,
BookOpenIcon,
MenuIcon,
XIcon,
PresentationChartLineIcon,
} from "@heroicons/vue/solid";
import DarkMode from "./DarkMode.vue";
const route = useRoute();
const navigation = computed(() => [
{ name: "Home", icon: HomeIcon, link: "/", current: route.path == "/" },
{
name: "Visualize",
icon: PresentationChartLineIcon,
link: "/visualize",
current: route.path.startsWith("/visualize"),
},
{
name: "Documentation",
icon: BookOpenIcon,

View File

@@ -73,7 +73,7 @@ const option = ref({
watch(
() => props.refresh,
async () => {
const response = await fetch("/api/v0/console/widget/graph?width=200");
const response = await fetch("/api/v0/console/widget/graph");
if (!response.ok) {
// Keep current data
return;

View File

@@ -1,12 +1,14 @@
import { createRouter, createWebHistory } from "vue-router";
import HomePage from "../views/HomePage.vue";
import DocumentationPage from "../views/DocumentationPage.vue";
import NotFoundPage from "../views/NotFoundPage.vue";
import HomePage from "./views/HomePage.vue";
import VisualizePage from "./views/VisualizePage.vue";
import DocumentationPage from "./views/DocumentationPage.vue";
import NotFoundPage from "./views/NotFoundPage.vue";
const router = createRouter({
history: createWebHistory(),
routes: [
{ path: "/", name: "Home", component: HomePage },
{ path: "/visualize", name: "Visualize", component: VisualizePage },
{ path: "/docs", redirect: "/docs/intro" },
{ path: "/docs/:id", name: "Documentation", component: DocumentationPage },
{ path: "/:pathMatch(.*)", component: NotFoundPage },

View File

@@ -0,0 +1,109 @@
// See https://design.gitlab.com/data-visualization/color
const colors = {
blue: [
"#e9ebff", // 50
"#d4dcfa", // 100
"#b7c6ff", // 200
"#97acff", // 300
"#748eff", // 400
"#5772ff", // 500
"#445cf2", // 600
"#3547de", // 700
"#232fcf", // 800
"#1e23a8", // 900
"#11118a", // 950
],
orange: [
"#fae8d1",
"#f7d8b5",
"#f3c291",
"#eb9a5c",
"#e17223",
"#d14e00",
"#b24800",
"#944100",
"#6f3500",
"#5c2b00",
"#421e00",
],
aqua: [
"#b8fff2",
"#93fae7",
"#5eebdf",
"#25d2d2",
"#0bb6c6",
"#0094b6",
"#0080a1",
"#006887",
"#004d67",
"#003f57",
"#00293d",
],
green: [
"#ddfab7",
"#c9f097",
"#b0de73",
"#94c25e",
"#83ab4a",
"#608b2f",
"#487900",
"#366800",
"#275600",
"#1a4500",
"#0f3300",
],
magenta: [
"#ffe3eb",
"#ffc9d9",
"#fcacc5",
"#ff85af",
"#f2639a",
"#d84280",
"#c52c6b",
"#b31756",
"#950943",
"#7a0033",
"#570028",
],
};
const orderedColors = ["blue", "orange", "aqua", "green", "magenta"];
const lightPalette = [5, 6, 7, 8, 9, 10]
.map((idx) => orderedColors.map((colorName) => colors[colorName][idx]))
.flat();
const darkPalette = [5, 4, 3, 2, 1, 0]
.map((idx) => orderedColors.map((colorName) => colors[colorName][idx]))
.flat();
const lightenColor = (color, amount) =>
"#" +
color
.replace(/^#/, "")
.replace(/../g, (color) =>
(
"0" +
Math.min(255, Math.max(0, parseInt(color, 16) + amount)).toString(16)
).substr(-2)
);
export function dataColor(index, alternate = false, theme = "light") {
const palette = theme === "light" ? lightPalette : darkPalette;
const correctedIndex = index % 2 === 0 ? index : index + orderedColors.length;
const computed = palette[correctedIndex % palette.length];
if (!alternate) {
return computed;
}
return lightenColor(computed, 20);
}
export function dataColorGrey(index, alternate = false, theme = "light") {
const palette =
theme === "light"
? ["#aaaaaa", "#bbbbbb", "#999999", "#cccccc", "#888888"]
: ["#666666", "#777777", "#555555", "#888888", "#444444"];
const computed = palette[index % palette.length];
if (!alternate) {
return computed;
}
return lightenColor(computed, 10);
}

View File

@@ -0,0 +1,216 @@
<template>
<div class="container mx-auto">
<ResizeRow
:slider-width="10"
:height="graphHeight"
width="auto"
slider-bg-color="#eee3"
slider-bg-hover-color="#ccc3"
>
<v-chart :option="graph" autoresize />
</ResizeRow>
<div class="relative my-3 overflow-x-auto shadow-md sm:rounded-lg">
<table class="w-full text-left text-sm text-gray-500 dark:text-gray-400">
<thead
class="bg-gray-50 text-xs uppercase text-gray-700 dark:bg-gray-700 dark:text-gray-400"
>
<tr>
<th scope="col" class="px-6 py-2"></th>
<th
v-for="column in table.columns"
:key="column"
scope="col"
class="px-6 py-2"
>
{{ column }}
</th>
<th scope="col" class="px-6 py-2 text-right">Min</th>
<th scope="col" class="px-6 py-2 text-right">Max</th>
<th scope="col" class="px-6 py-2 text-right">Average</th>
</tr>
</thead>
<tbody>
<tr
v-for="row in table.rows"
:key="row.dimensions"
class="border-b odd:bg-white even:bg-gray-50 dark:border-gray-700 dark:bg-gray-800 odd:dark:bg-gray-800 even:dark:bg-gray-700"
>
<th
scope="row"
class="px-6 py-2 text-right font-medium text-gray-900 dark:text-white"
>
<div class="w-5" :style="row.style">&nbsp;</div>
</th>
<td
v-for="dimension in row.dimensions"
:key="dimension"
class="px-6 py-2"
>
{{ dimension }}
</td>
<td class="px-6 py-2 text-right tabular-nums">
{{ formatBps(row.min) }}bps
</td>
<td class="px-6 py-2 text-right tabular-nums">
{{ formatBps(row.max) }}bps
</td>
<td class="px-6 py-2 text-right tabular-nums">
{{ formatBps(row.average) }}bps
</td>
</tr>
</tbody>
</table>
</div>
</div>
</template>
<script setup>
import { ref, watch } from "vue";
import { notify } from "notiwind";
import { Date as SugarDate } from "sugar-date";
import { use, graphic } from "echarts/core";
import { CanvasRenderer } from "echarts/renderers";
import { LineChart } from "echarts/charts";
import { TooltipComponent, GridComponent } from "echarts/components";
import VChart from "vue-echarts";
import { ResizeRow } from "vue-resizer";
import { dataColor, dataColorGrey } from "../utils/palette.js";
use([CanvasRenderer, LineChart, TooltipComponent, GridComponent]);
const formatBps = (value) => {
const suffixes = ["", "K", "M", "G", "T"];
let idx = 0;
while (value >= 1000 && idx < suffixes.length) {
value /= 1000;
idx++;
}
value = value.toFixed(2);
return `${value}${suffixes[idx]}`;
};
const graphHeight = ref(500);
const graph = ref({
grid: {
left: 60,
top: 20,
right: "1%",
bottom: 20,
},
xAxis: {
type: "time",
},
yAxis: {
type: "value",
min: 0,
axisLabel: { formatter: formatBps },
axisPointer: { label: { formatter: ({ value }) => formatBps(value) } },
},
tooltip: {
confine: true,
trigger: "axis",
axisPointer: {
type: "cross",
label: { backgroundColor: "#6a7985" },
},
valueFormatter: formatBps,
},
series: [],
});
const table = ref({
columns: [],
rows: [],
});
const request = ref({
start: SugarDate.create("6 hours ago"),
end: SugarDate.create("now"),
points: 100,
"max-series": 10,
dimensions: ["SrcAS"],
filter: {
operator: "all",
rules: [
{
column: "InIfBoundary",
operator: "=",
value: "external",
},
],
},
});
watch(
request,
async () => {
const response = await fetch("/api/v0/console/graph", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(request.value),
});
if (!response.ok) {
notify(
{
group: "top",
kind: "error",
title: "Unable to fetch data",
text: `While retrieving data, got a fatal error.`,
},
60000
);
return;
}
const data = await response.json();
// Graphic
graph.value.xAxis.data = data.t.slice(1, -1);
graph.value.series = data.rows.map((rows, idx) => {
const color = rows.some((name) => name === "Other")
? dataColorGrey
: dataColor;
return {
type: "line",
name: rows.join(" — "),
symbol: "none",
itemStyle: {
color: color(idx),
},
lineStyle: {
color: color(idx),
width: 1,
},
areaStyle: {
opacity: 0.95,
color: new graphic.LinearGradient(0, 0, 0, 1, [
{ offset: 0, color: color(idx) },
{ offset: 1, color: color(idx, true) },
]),
},
emphasis: {
focus: "series",
},
stack: "all",
data: data.t.map((t, idx2) => [t, data.points[idx][idx2]]).slice(1, -1),
};
});
// Table
table.value = {
columns: request.value.dimensions,
rows: data.rows.map((rows, idx) => {
const color = rows.some((name) => name === "Other")
? dataColorGrey
: dataColor;
return {
dimensions: rows,
style: `background-color: ${color(idx)}`,
min: data.min[idx],
max: data.max[idx],
average: data.average[idx],
};
}),
};
},
{ immediate: true }
);
</script>

View File

@@ -1347,6 +1347,18 @@ strip-json-comments@^3.1.0, strip-json-comments@^3.1.1:
resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006"
integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==
sugar-core@^2.0.0:
version "2.0.6"
resolved "https://registry.yarnpkg.com/sugar-core/-/sugar-core-2.0.6.tgz#785e0cd64aa7302ea54d47bc1213efe52c006270"
integrity sha512-YmLFysR3Si6RImqL1+aB6JH81EXxvXn5iXhPf2PsjfoUYEwCxFDYCQY+zC3WqviuGWzxFaSkkJvkUE05Y03L5Q==
sugar-date@^2.0.6:
version "2.0.6"
resolved "https://registry.yarnpkg.com/sugar-date/-/sugar-date-2.0.6.tgz#cbfb25e1e1e5cfeec551aa9f302ed763a3778b02"
integrity sha512-5aPXcTl9pIgae3j8wOieRZOEbaowHHpL+MPgZwHHjXdhZz3FjzpacjzM+Aq7rZTjDsWyWuKHzkIALx2uUhnmyg==
dependencies:
sugar-core "^2.0.0"
supports-color@^5.3.0:
version "5.5.0"
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f"
@@ -1477,6 +1489,11 @@ vue-eslint-parser@^8.0.1:
lodash "^4.17.21"
semver "^7.3.5"
vue-resizer@^1.1.9:
version "1.1.9"
resolved "https://registry.yarnpkg.com/vue-resizer/-/vue-resizer-1.1.9.tgz#9c8676c90c8bae2d1e6b436bd1fe6544f9c87f33"
integrity sha512-cDGPIhpFxUYNHQ3e9rWISNckSvBqrKhNvR9MTmP3/cttTsz5sFkNU/NyO7GDXTxhQslzJKpCTf+PH65m3FEoyg==
vue-router@^4.0.14:
version "4.0.14"
resolved "https://registry.yarnpkg.com/vue-router/-/vue-router-4.0.14.tgz#ce2028c1c5c33e30c7287950c973f397fce1bd65"

505
console/graph.go Normal file
View File

@@ -0,0 +1,505 @@
package console
import (
"errors"
"fmt"
"net"
"net/http"
"sort"
"strconv"
"strings"
"time"
"github.com/gin-gonic/gin"
"akvorado/common/helpers"
)
// graphQuery describes the input for the /graph endpoint.
type graphQuery struct {
Start time.Time `json:"start" binding:"required"`
End time.Time `json:"end" binding:"required"`
Points int `json:"points" binding:"required"` // minimum number of points
Dimensions []graphColumn `json:"dimensions"` // group by ...
MaxSeries int `json:"max-series"` // limit product of dimensions
Filter graphFilterGroup `json:"filter"` // where ...
}
type graphColumn int
const (
graphColumnExporterAddress graphColumn = iota + 1
graphColumnExporterName
graphColumnExporterGroup
graphColumnSrcAddr
graphColumnDstAddr
graphColumnSrcAS
graphColumnDstAS
graphColumnSrcCountry
graphColumnDstCountry
graphColumnInIfName
graphColumnOutIfName
graphColumnInIfDescription
graphColumnOutIfDescription
graphColumnInIfSpeed
graphColumnOutIfSpeed
graphColumnInIfConnectivity
graphColumnOutIfConnectivity
graphColumnInIfProvider
graphColumnOutIfProvider
graphColumnInIfBoundary
graphColumnOutIfBoundary
graphColumnEType
graphColumnProto
graphColumnSrcPort
graphColumnDstPort
graphColumnForwardingStatus
)
var graphColumnMap = helpers.NewBimap(map[graphColumn]string{
graphColumnExporterAddress: "ExporterAddress",
graphColumnExporterName: "ExporterName",
graphColumnExporterGroup: "ExporterGroup",
graphColumnSrcAddr: "SrcAddr",
graphColumnDstAddr: "DstAddr",
graphColumnSrcAS: "SrcAS",
graphColumnDstAS: "DstAS",
graphColumnSrcCountry: "SrcCountry",
graphColumnDstCountry: "DstCountry",
graphColumnInIfName: "InIfName",
graphColumnOutIfName: "OutIfName",
graphColumnInIfDescription: "InIfDescription",
graphColumnOutIfDescription: "OutIfDescription",
graphColumnInIfSpeed: "InIfSpeed",
graphColumnOutIfSpeed: "OutIfSpeed",
graphColumnInIfConnectivity: "InIfConnectivity",
graphColumnOutIfConnectivity: "OutIfConnectivity",
graphColumnInIfProvider: "InIfProvider",
graphColumnOutIfProvider: "OutIfProvider",
graphColumnInIfBoundary: "InIfBoundary",
graphColumnOutIfBoundary: "OutIfBoundary",
graphColumnEType: "EType",
graphColumnProto: "Proto",
graphColumnSrcPort: "SrcPort",
graphColumnDstPort: "DstPort",
graphColumnForwardingStatus: "ForwardingStatus",
})
func (gc graphColumn) MarshalText() ([]byte, error) {
got, ok := graphColumnMap.LoadValue(gc)
if ok {
return []byte(got), nil
}
return nil, errors.New("unknown group operator")
}
func (gc graphColumn) String() string {
got, _ := graphColumnMap.LoadValue(gc)
return got
}
func (gc *graphColumn) UnmarshalText(input []byte) error {
got, ok := graphColumnMap.LoadKey(string(input))
if ok {
*gc = got
return nil
}
return errors.New("unknown group operator")
}
type graphFilterGroup struct {
Operator graphFilterGroupOperator `json:"operator" binding:"required"`
Children []graphFilterGroup `json:"children"`
Rules []graphFilterRule `json:"rules"`
}
type graphFilterGroupOperator int
const (
graphFilterGroupOperatorAny graphFilterGroupOperator = iota + 1
graphFilterGroupOperatorAll
)
var graphFilterGroupOperatorMap = helpers.NewBimap(map[graphFilterGroupOperator]string{
graphFilterGroupOperatorAny: "any",
graphFilterGroupOperatorAll: "all",
})
func (gfgo graphFilterGroupOperator) MarshalText() ([]byte, error) {
got, ok := graphFilterGroupOperatorMap.LoadValue(gfgo)
if ok {
return []byte(got), nil
}
return nil, errors.New("unknown group operator")
}
func (gfgo *graphFilterGroupOperator) UnmarshalText(input []byte) error {
got, ok := graphFilterGroupOperatorMap.LoadKey(string(input))
if ok {
*gfgo = got
return nil
}
return errors.New("unknown group operator")
}
type graphFilterRule struct {
Column graphColumn `json:"column" binding:"required"`
Operator graphFilterRuleOperator `json:"operator" binding:"required"`
Value string `json:"value" binding:"required"`
}
type graphFilterRuleOperator int
const (
graphFilterRuleOperatorEqual graphFilterRuleOperator = iota + 1
graphFilterRuleOperatorNotEqual
graphFilterRuleOperatorLessThan
graphFilterRuleOperatorGreaterThan
)
var graphFilterRuleOperatorMap = helpers.NewBimap(map[graphFilterRuleOperator]string{
graphFilterRuleOperatorEqual: "=",
graphFilterRuleOperatorNotEqual: "!=",
graphFilterRuleOperatorLessThan: "<",
graphFilterRuleOperatorGreaterThan: ">",
})
func (gfro graphFilterRuleOperator) MarshalText() ([]byte, error) {
got, ok := graphFilterRuleOperatorMap.LoadValue(gfro)
if ok {
return []byte(got), nil
}
return nil, errors.New("unknown rule operator")
}
func (gfro graphFilterRuleOperator) String() string {
got, _ := graphFilterRuleOperatorMap.LoadValue(gfro)
return got
}
func (gfro *graphFilterRuleOperator) UnmarshalText(input []byte) error {
got, ok := graphFilterRuleOperatorMap.LoadKey(string(input))
if ok {
*gfro = got
return nil
}
return errors.New("unknown rule operator")
}
// toSQLWhere translates a graphFilterGroup to SQL expression (to be used in WHERE)
func (gfg graphFilterGroup) toSQLWhere() (string, error) {
operator := map[graphFilterGroupOperator]string{
graphFilterGroupOperatorAll: " AND ",
graphFilterGroupOperatorAny: " OR ",
}[gfg.Operator]
expressions := []string{}
for _, expr := range gfg.Children {
subexpr, err := expr.toSQLWhere()
if err != nil {
return "", err
}
expressions = append(expressions, fmt.Sprintf("(%s)", subexpr))
}
for _, expr := range gfg.Rules {
subexpr, err := expr.toSQLWhere()
if err != nil {
return "", err
}
expressions = append(expressions, fmt.Sprintf("(%s)", subexpr))
}
return strings.Join(expressions, operator), nil
}
// toSQLWhere translates a graphFilterRule to an SQL expression (to be used in WHERE)
func (gfr graphFilterRule) toSQLWhere() (string, error) {
quote := func(v string) string {
return "'" + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(v) + "'"
}
switch gfr.Column {
case graphColumnExporterAddress, graphColumnSrcAddr, graphColumnDstAddr:
// IP
ip := net.ParseIP(gfr.Value)
if ip == nil {
return "", fmt.Errorf("cannot parse IP %q for %s", gfr.Value, gfr.Column)
}
switch gfr.Operator {
case graphFilterRuleOperatorEqual, graphFilterRuleOperatorNotEqual:
return fmt.Sprintf("%s %s IPv6StringToNum(%s)", gfr.Column, gfr.Operator, quote(ip.String())), nil
}
case graphColumnExporterName, graphColumnExporterGroup, graphColumnSrcCountry, graphColumnDstCountry, graphColumnInIfName, graphColumnOutIfName, graphColumnInIfDescription, graphColumnInIfConnectivity, graphColumnOutIfConnectivity, graphColumnInIfProvider, graphColumnOutIfProvider:
// String
switch gfr.Operator {
case graphFilterRuleOperatorEqual, graphFilterRuleOperatorNotEqual:
return fmt.Sprintf("%s %s %s", gfr.Column, gfr.Operator, quote(gfr.Value)), nil
}
case graphColumnInIfBoundary, graphColumnOutIfBoundary:
// Boundary
switch gfr.Value {
case "external", "internal":
default:
return "", fmt.Errorf("cannot parse boundary %q for %s", gfr.Value, gfr.Column)
}
switch gfr.Operator {
case graphFilterRuleOperatorEqual, graphFilterRuleOperatorNotEqual:
return fmt.Sprintf("%s %s %s", gfr.Column, gfr.Operator, quote(gfr.Value)), nil
}
case graphColumnInIfSpeed, graphColumnOutIfSpeed, graphColumnForwardingStatus:
// Integer (64 bit)
value, err := strconv.ParseUint(gfr.Value, 10, 64)
if err != nil {
return "", fmt.Errorf("cannot parse int %q for %s", gfr.Value, gfr.Column)
}
switch gfr.Operator {
case graphFilterRuleOperatorEqual, graphFilterRuleOperatorNotEqual, graphFilterRuleOperatorLessThan, graphFilterRuleOperatorGreaterThan:
return fmt.Sprintf("%s %s %d", gfr.Column, gfr.Operator, value), nil
}
case graphColumnSrcPort, graphColumnDstPort:
// Port
port, err := strconv.ParseUint(gfr.Value, 10, 16)
if err != nil {
return "", fmt.Errorf("cannot parse port %q for %s", gfr.Value, gfr.Column)
}
switch gfr.Operator {
case graphFilterRuleOperatorEqual, graphFilterRuleOperatorNotEqual, graphFilterRuleOperatorLessThan, graphFilterRuleOperatorGreaterThan:
return fmt.Sprintf("%s %s %d", gfr.Column, gfr.Operator, port), nil
}
case graphColumnSrcAS, graphColumnDstAS:
// AS number
value := strings.TrimPrefix(gfr.Value, "AS")
asn, err := strconv.ParseUint(value, 10, 32)
if err != nil {
return "", fmt.Errorf("cannot parse AS %q for %s", gfr.Value, gfr.Column)
}
switch gfr.Operator {
case graphFilterRuleOperatorEqual, graphFilterRuleOperatorNotEqual, graphFilterRuleOperatorLessThan, graphFilterRuleOperatorGreaterThan:
return fmt.Sprintf("%s %s %d", gfr.Column, gfr.Operator, asn), nil
}
case graphColumnEType:
// Ethernet Type
etypes := map[string]uint16{
"ipv4": 0x0800,
"ipv6": 0x86dd,
}
etype, ok := etypes[strings.ToLower(gfr.Value)]
if !ok {
return "", fmt.Errorf("cannot parse etype %q for %s", gfr.Value, gfr.Column)
}
switch gfr.Operator {
case graphFilterRuleOperatorEqual, graphFilterRuleOperatorNotEqual:
return fmt.Sprintf("%s %s %d", gfr.Column, gfr.Operator, etype), nil
}
case graphColumnProto:
// Protocol
// Case 1: int
proto, err := strconv.ParseUint(gfr.Value, 10, 8)
if err == nil {
switch gfr.Operator {
case graphFilterRuleOperatorEqual, graphFilterRuleOperatorNotEqual, graphFilterRuleOperatorLessThan, graphFilterRuleOperatorGreaterThan:
return fmt.Sprintf("%s %s %d", gfr.Column, gfr.Operator, proto), nil
}
break
}
// Case 2: string
switch gfr.Operator {
case graphFilterRuleOperatorEqual, graphFilterRuleOperatorNotEqual:
return fmt.Sprintf("dictGetOrDefault('protocols', 'name', Proto, '???') %s %s",
gfr.Operator, quote(gfr.Value)), nil
}
}
return "", fmt.Errorf("operator %s not supported for %s", gfr.Operator, gfr.Column)
}
// toSQLSelect transforms a column into an expression to use in SELECT
func (gc graphColumn) toSQLSelect() string {
subQuery := fmt.Sprintf(`SELECT %s FROM rows`, gc)
var strValue string
switch gc {
case graphColumnExporterAddress, graphColumnSrcAddr, graphColumnDstAddr:
strValue = fmt.Sprintf("IPv6NumToString(%s)", gc)
case graphColumnSrcAS, graphColumnDstAS:
strValue = fmt.Sprintf(`concat(toString(%s), ': ', dictGetOrDefault('asns', 'name', %s, '???'))`,
gc, gc)
case graphColumnEType:
strValue = `if(EType = 0x800, 'IPv4', if(EType = 0x86dd, 'IPv6', '???'))`
case graphColumnProto:
strValue = `dictGetOrDefault('protocols', 'name', Proto, '???')`
case graphColumnInIfSpeed, graphColumnOutIfSpeed, graphColumnSrcPort, graphColumnDstPort, graphColumnForwardingStatus:
strValue = fmt.Sprintf("toString(%s)", gc)
default:
strValue = gc.String()
}
return fmt.Sprintf(`if(%s IN (%s), %s, 'Other')`,
gc, subQuery, strValue)
}
// graphQueryToSQL converts a graph query to an SQL request
func (query graphQuery) toSQL() (string, error) {
interval := int64((query.End.Sub(query.Start).Seconds())) / int64(query.Points)
// Filter
where, err := query.Filter.toSQLWhere()
if err != nil {
return "", err
}
if where == "" {
where = "{timefilter}"
} else {
where = fmt.Sprintf("{timefilter} AND (%s)", where)
}
// Select
fields := []string{
`toStartOfInterval(TimeReceived, INTERVAL slot second) AS time`,
`SUM(Bytes*SamplingRate*8/slot) AS bps`,
}
selectFields := []string{}
dimensions := []string{}
for _, column := range query.Dimensions {
field := column.toSQLSelect()
selectFields = append(selectFields, field)
dimensions = append(dimensions, column.String())
}
if len(dimensions) > 0 {
fields = append(fields, fmt.Sprintf(`[%s] AS dimensions`, strings.Join(selectFields, ",\n ")))
} else {
fields = append(fields, "emptyArrayString() AS dimensions")
}
// With
with := []string{fmt.Sprintf(`intDiv(%d, {resolution})*{resolution} AS slot`, interval)}
if len(dimensions) > 0 {
with = append(with, fmt.Sprintf(
"rows AS (SELECT %s FROM {table} WHERE %s GROUP BY %s ORDER BY SUM(Bytes) DESC LIMIT %d)",
strings.Join(dimensions, ", "),
where,
strings.Join(dimensions, ", "),
query.MaxSeries))
}
sqlQuery := fmt.Sprintf(`
WITH
%s
SELECT
%s
FROM {table}
WHERE %s
GROUP BY time, dimensions
ORDER BY time`, strings.Join(with, ",\n "), strings.Join(fields, ",\n "), where)
return sqlQuery, nil
}
type graphHandlerOutput struct {
Rows [][]string `json:"rows"`
Time []time.Time `json:"t"`
Points [][]int `json:"points"` // t → row → bps
Average []int `json:"average"` // row → bps
Min []int `json:"min"`
Max []int `json:"max"`
}
func (c *Component) graphHandlerFunc(gc *gin.Context) {
ctx := c.t.Context(gc.Request.Context())
var query graphQuery
if err := gc.ShouldBindJSON(&query); err != nil {
gc.JSON(http.StatusBadRequest, gin.H{"message": err.Error()})
return
}
if query.Start.After(query.End) {
gc.JSON(http.StatusBadRequest, gin.H{"message": "start should not be after end"})
return
}
if query.Points < 5 || query.Points > 2000 {
gc.JSON(http.StatusBadRequest, gin.H{"message": "points should be >= 5 and <= 2000"})
return
}
if query.MaxSeries == 0 {
query.MaxSeries = 10
}
if query.MaxSeries < 5 || query.MaxSeries > 50 {
gc.JSON(http.StatusBadRequest, gin.H{"message": "max-series should be >= 5 and <= 50"})
return
}
sqlQuery, err := query.toSQL()
if err != nil {
gc.JSON(http.StatusBadRequest, gin.H{"message": err.Error()})
return
}
resolution := time.Duration(int64(query.End.Sub(query.Start).Nanoseconds()) / int64(query.Points))
if resolution < time.Second {
resolution = time.Second
}
sqlQuery = c.queryFlowsTable(sqlQuery,
query.Start, query.End, resolution)
gc.Header("X-SQL-Query", sqlQuery)
results := []struct {
Time time.Time `ch:"time"`
Bps float64 `ch:"bps"`
Dimensions []string `ch:"dimensions"`
}{}
if err := c.d.ClickHouseDB.Conn.Select(ctx, &results, sqlQuery); err != nil {
c.r.Err(err).Msg("unable to query database")
gc.JSON(http.StatusInternalServerError, gin.H{"message": "Unable to query database."})
return
}
// We want to sort rows depending on how much data they gather each
output := graphHandlerOutput{
Time: []time.Time{},
}
rowValues := map[string][]int{} // values for each row (indexed by internal key)
rowKeys := map[string][]string{} // mapping from keys to dimensions
rowSums := map[string]uint64{} // sum for a given row (to sort)
lastTime := time.Time{}
for _, result := range results {
if result.Time != lastTime {
output.Time = append(output.Time, result.Time)
lastTime = result.Time
}
}
lastTime = time.Time{}
idx := -1
for _, result := range results {
if result.Time != lastTime {
idx++
lastTime = result.Time
}
rowKey := fmt.Sprintf("%s", result.Dimensions)
row, ok := rowValues[rowKey]
if !ok {
rowKeys[rowKey] = result.Dimensions
row = make([]int, len(output.Time))
rowValues[rowKey] = row
}
rowValues[rowKey][idx] = int(result.Bps)
sum, _ := rowSums[rowKey]
rowSums[rowKey] = sum + uint64(result.Bps)
}
rows := make([]string, len(rowKeys))
i := 0
for k := range rowKeys {
rows[i] = k
i++
}
sort.Slice(rows, func(i, j int) bool {
return rowSums[rows[i]] > rowSums[rows[j]]
})
output.Rows = make([][]string, len(rows))
output.Points = make([][]int, len(rows))
output.Average = make([]int, len(rows))
output.Min = make([]int, len(rows))
output.Max = make([]int, len(rows))
for idx, r := range rows {
output.Rows[idx] = rowKeys[r]
output.Points[idx] = rowValues[r]
output.Average[idx] = int(rowSums[r] / uint64(len(output.Time)))
for j, v := range rowValues[r] {
if j == 0 || output.Min[idx] > v {
output.Min[idx] = v
}
if j == 0 || output.Max[idx] < v {
output.Max[idx] = v
}
}
}
gc.JSON(http.StatusOK, output)
}

530
console/graph_test.go Normal file
View File

@@ -0,0 +1,530 @@
package console
import (
"bytes"
"encoding/json"
"fmt"
netHTTP "net/http"
"strings"
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/golang/mock/gomock"
"akvorado/common/clickhousedb"
"akvorado/common/daemon"
"akvorado/common/helpers"
"akvorado/common/http"
"akvorado/common/reporter"
)
func TestGraphFilterGroupSQLWhere(t *testing.T) {
cases := []struct {
Description string
Input graphFilterGroup
Expected string
}{
{
Description: "empty group",
Expected: "",
}, {
Description: "all group",
Input: graphFilterGroup{
Operator: graphFilterGroupOperatorAll,
Rules: []graphFilterRule{
{
Column: graphColumnDstCountry,
Operator: graphFilterRuleOperatorEqual,
Value: "FR",
}, {
Column: graphColumnSrcCountry,
Operator: graphFilterRuleOperatorEqual,
Value: "US",
},
},
},
Expected: `(DstCountry = 'FR') AND (SrcCountry = 'US')`,
}, {
Description: "any group",
Input: graphFilterGroup{
Operator: graphFilterGroupOperatorAny,
Rules: []graphFilterRule{
{
Column: graphColumnDstCountry,
Operator: graphFilterRuleOperatorEqual,
Value: "FR",
}, {
Column: graphColumnSrcCountry,
Operator: graphFilterRuleOperatorEqual,
Value: "US",
},
},
},
Expected: `(DstCountry = 'FR') OR (SrcCountry = 'US')`,
}, {
Description: "nested group",
Input: graphFilterGroup{
Operator: graphFilterGroupOperatorAll,
Rules: []graphFilterRule{
{
Column: graphColumnDstCountry,
Operator: graphFilterRuleOperatorEqual,
Value: "FR",
},
},
Children: []graphFilterGroup{
{
Operator: graphFilterGroupOperatorAny,
Rules: []graphFilterRule{
{
Column: graphColumnSrcCountry,
Operator: graphFilterRuleOperatorEqual,
Value: "US",
}, {
Column: graphColumnSrcCountry,
Operator: graphFilterRuleOperatorEqual,
Value: "IE",
},
},
},
},
},
Expected: `((SrcCountry = 'US') OR (SrcCountry = 'IE')) AND (DstCountry = 'FR')`,
},
}
for _, tc := range cases {
t.Run(tc.Description, func(t *testing.T) {
got, _ := tc.Input.toSQLWhere()
if diff := helpers.Diff(got, tc.Expected); diff != "" {
t.Errorf("toSQLWhere (-got, +want):\n%s", diff)
}
})
}
}
func TestGraphFilterRuleSQLWhere(t *testing.T) {
cases := []struct {
Description string
Input graphFilterRule
Expected string
}{
{
Description: "source IP (v4)",
Input: graphFilterRule{
Column: graphColumnSrcAddr,
Operator: graphFilterRuleOperatorEqual,
Value: "192.0.2.11",
},
Expected: `SrcAddr = IPv6StringToNum('192.0.2.11')`,
}, {
Description: "source IP (v6)",
Input: graphFilterRule{
Column: graphColumnSrcAddr,
Operator: graphFilterRuleOperatorEqual,
Value: "2001:db8::1",
},
Expected: `SrcAddr = IPv6StringToNum('2001:db8::1')`,
}, {
Description: "source IP (bad)",
Input: graphFilterRule{
Column: graphColumnSrcAddr,
Operator: graphFilterRuleOperatorEqual,
Value: "alfred",
},
Expected: "",
}, {
Description: "boundary",
Input: graphFilterRule{
Column: graphColumnInIfBoundary,
Operator: graphFilterRuleOperatorNotEqual,
Value: "external",
},
Expected: `InIfBoundary != 'external'`,
}, {
Description: "boundary (bad)",
Input: graphFilterRule{
Column: graphColumnInIfBoundary,
Operator: graphFilterRuleOperatorNotEqual,
Value: "eternal",
},
Expected: "",
}, {
Description: "speed",
Input: graphFilterRule{
Column: graphColumnInIfSpeed,
Operator: graphFilterRuleOperatorLessThan,
Value: "1000",
},
Expected: `InIfSpeed < 1000`,
}, {
Description: "speed (bad)",
Input: graphFilterRule{
Column: graphColumnInIfSpeed,
Operator: graphFilterRuleOperatorLessThan,
Value: "-1000",
},
Expected: "",
}, {
Description: "source port",
Input: graphFilterRule{
Column: graphColumnSrcPort,
Operator: graphFilterRuleOperatorLessThan,
Value: "1000",
},
Expected: `SrcPort < 1000`,
}, {
Description: "source port (bad)",
Input: graphFilterRule{
Column: graphColumnSrcPort,
Operator: graphFilterRuleOperatorLessThan,
Value: "10000000",
},
Expected: "",
}, {
Description: "source AS",
Input: graphFilterRule{
Column: graphColumnSrcAS,
Operator: graphFilterRuleOperatorEqual,
Value: "2906",
},
Expected: "SrcAS = 2906",
}, {
Description: "source AS (prefixed)",
Input: graphFilterRule{
Column: graphColumnSrcAS,
Operator: graphFilterRuleOperatorEqual,
Value: "AS2906",
},
Expected: "SrcAS = 2906",
}, {
Description: "source AS (bad)",
Input: graphFilterRule{
Column: graphColumnSrcAS,
Operator: graphFilterRuleOperatorEqual,
Value: "ASMN2906",
},
Expected: "",
}, {
Description: "EType",
Input: graphFilterRule{
Column: graphColumnEType,
Operator: graphFilterRuleOperatorEqual,
Value: "IPv6",
},
Expected: "EType = 34525",
}, {
Description: "EType (bad)",
Input: graphFilterRule{
Column: graphColumnEType,
Operator: graphFilterRuleOperatorEqual,
Value: "IPv4+",
},
Expected: "",
}, {
Description: "Proto (string)",
Input: graphFilterRule{
Column: graphColumnProto,
Operator: graphFilterRuleOperatorEqual,
Value: "TCP",
},
Expected: `dictGetOrDefault('protocols', 'name', Proto, '???') = 'TCP'`,
}, {
Description: "Proto (int)",
Input: graphFilterRule{
Column: graphColumnProto,
Operator: graphFilterRuleOperatorEqual,
Value: "47",
},
Expected: `Proto = 47`,
},
}
for _, tc := range cases {
t.Run(tc.Description, func(t *testing.T) {
got, _ := tc.Input.toSQLWhere()
if diff := helpers.Diff(got, tc.Expected); diff != "" {
t.Errorf("toSQLWhere (-got, +want):\n%s", diff)
}
})
}
}
func TestGraphColumnSQLSelect(t *testing.T) {
cases := []struct {
Input graphColumn
Expected string
}{
{
Input: graphColumnSrcAddr,
Expected: `if(SrcAddr IN (SELECT SrcAddr FROM rows), IPv6NumToString(SrcAddr), 'Other')`,
}, {
Input: graphColumnDstAS,
Expected: `if(DstAS IN (SELECT DstAS FROM rows), concat(toString(DstAS), ': ', dictGetOrDefault('asns', 'name', DstAS, '???')), 'Other')`,
}, {
Input: graphColumnProto,
Expected: `if(Proto IN (SELECT Proto FROM rows), dictGetOrDefault('protocols', 'name', Proto, '???'), 'Other')`,
}, {
Input: graphColumnEType,
Expected: `if(EType IN (SELECT EType FROM rows), if(EType = 0x800, 'IPv4', if(EType = 0x86dd, 'IPv6', '???')), 'Other')`,
}, {
Input: graphColumnOutIfSpeed,
Expected: `if(OutIfSpeed IN (SELECT OutIfSpeed FROM rows), toString(OutIfSpeed), 'Other')`,
}, {
Input: graphColumnExporterName,
Expected: `if(ExporterName IN (SELECT ExporterName FROM rows), ExporterName, 'Other')`,
},
}
for _, tc := range cases {
t.Run(tc.Input.String(), func(t *testing.T) {
got := tc.Input.toSQLSelect()
if diff := helpers.Diff(got, tc.Expected); diff != "" {
t.Errorf("toSQLWhere (-got, +want):\n%s", diff)
}
})
}
}
func TestGraphQuerySQL(t *testing.T) {
cases := []struct {
Description string
Input graphQuery
Expected string
}{
{
Description: "no dimensions, no filters",
Input: graphQuery{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Dimensions: []graphColumn{},
Filter: graphFilterGroup{},
},
Expected: `
WITH
intDiv(864, {resolution})*{resolution} AS slot
SELECT
toStartOfInterval(TimeReceived, INTERVAL slot second) AS time,
SUM(Bytes*SamplingRate*8/slot) AS bps,
emptyArrayString() AS dimensions
FROM {table}
WHERE {timefilter}
GROUP BY time, dimensions
ORDER BY time`,
}, {
Description: "no dimensions",
Input: graphQuery{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Dimensions: []graphColumn{},
Filter: graphFilterGroup{
Operator: graphFilterGroupOperatorAll,
Rules: []graphFilterRule{
{
Column: graphColumnDstCountry,
Operator: graphFilterRuleOperatorEqual,
Value: "FR",
}, {
Column: graphColumnSrcCountry,
Operator: graphFilterRuleOperatorEqual,
Value: "US",
},
},
},
},
Expected: `
WITH
intDiv(864, {resolution})*{resolution} AS slot
SELECT
toStartOfInterval(TimeReceived, INTERVAL slot second) AS time,
SUM(Bytes*SamplingRate*8/slot) AS bps,
emptyArrayString() AS dimensions
FROM {table}
WHERE {timefilter} AND ((DstCountry = 'FR') AND (SrcCountry = 'US'))
GROUP BY time, dimensions
ORDER BY time`,
}, {
Description: "no filters",
Input: graphQuery{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
MaxSeries: 20,
Dimensions: []graphColumn{
graphColumnExporterName,
graphColumnInIfProvider,
},
Filter: graphFilterGroup{},
},
Expected: `
WITH
intDiv(864, {resolution})*{resolution} AS slot,
rows AS (SELECT ExporterName, InIfProvider FROM {table} WHERE {timefilter} GROUP BY ExporterName, InIfProvider ORDER BY SUM(Bytes) DESC LIMIT 20)
SELECT
toStartOfInterval(TimeReceived, INTERVAL slot second) AS time,
SUM(Bytes*SamplingRate*8/slot) AS bps,
[if(ExporterName IN (SELECT ExporterName FROM rows), ExporterName, 'Other'),
if(InIfProvider IN (SELECT InIfProvider FROM rows), InIfProvider, 'Other')] AS dimensions
FROM {table}
WHERE {timefilter}
GROUP BY time, dimensions
ORDER BY time`,
},
}
for _, tc := range cases {
t.Run(tc.Description, func(t *testing.T) {
got, _ := tc.Input.toSQL()
if diff := helpers.Diff(strings.Split(got, "\n"), strings.Split(tc.Expected, "\n")); diff != "" {
t.Errorf("toSQL (-got, +want):\n%s", diff)
}
})
}
}
func TestGraphHandler(t *testing.T) {
r := reporter.NewMock(t)
ch, mockConn := clickhousedb.NewMock(t, r)
h := http.NewMock(t, r)
c, err := New(r, Configuration{}, Dependencies{
Daemon: daemon.NewMock(t),
HTTP: h,
ClickHouseDB: ch,
})
if err != nil {
t.Fatalf("New() error:\n%+v", err)
}
helpers.StartStop(t, c)
base := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
expectedSQL := []struct {
Time time.Time `ch:"time"`
Bps float64 `ch:"bps"`
Dimensions []string `ch:"dimensions"`
}{
{base, 1000, []string{"router1", "provider1"}},
{base, 2000, []string{"router1", "provider2"}},
{base, 1500, []string{"router1", "Others"}},
{base, 1200, []string{"router2", "provider2"}},
{base, 1100, []string{"router2", "provider3"}},
{base, 1900, []string{"Others", "Others"}},
{base.Add(time.Minute), 500, []string{"router1", "provider1"}},
{base.Add(time.Minute), 5000, []string{"router1", "provider2"}},
{base.Add(time.Minute), 900, []string{"router2", "provider4"}},
{base.Add(time.Minute), 100, []string{"Others", "Others"}},
{base.Add(2 * time.Minute), 100, []string{"router1", "provider1"}},
{base.Add(2 * time.Minute), 3000, []string{"router1", "provider2"}},
{base.Add(2 * time.Minute), 1500, []string{"router1", "Others"}},
{base.Add(2 * time.Minute), 100, []string{"router2", "provider4"}},
{base.Add(2 * time.Minute), 100, []string{"Others", "Others"}},
}
expected := gin.H{
// Sorted by sum of bps
"rows": [][]string{
{"router1", "provider2"}, // 10000
{"router1", "Others"}, // 3000
{"Others", "Others"}, // 2000
{"router1", "provider1"}, // 1600
{"router2", "provider2"}, // 1200
{"router2", "provider3"}, // 1100
{"router2", "provider4"}, // 1000
},
"t": []string{
"2009-11-10T23:00:00Z",
"2009-11-10T23:01:00Z",
"2009-11-10T23:02:00Z",
},
"points": [][]int{
{2000, 5000, 3000},
{1500, 0, 1500},
{1900, 100, 100},
{1000, 500, 100},
{1200, 0, 0},
{1100, 0, 0},
{0, 900, 100},
},
"min": []int{
2000,
0,
100,
100,
0,
0,
0,
},
"max": []int{
5000,
1500,
1900,
1000,
1200,
1100,
900,
},
"average": []int{
3333,
1000,
700,
533,
400,
366,
333,
},
}
mockConn.EXPECT().
Select(gomock.Any(), gomock.Any(), gomock.Any()).
SetArg(1, expectedSQL).
Return(nil)
input := graphQuery{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
MaxSeries: 20,
Dimensions: []graphColumn{
graphColumnExporterName,
graphColumnInIfProvider,
},
Filter: graphFilterGroup{
Operator: graphFilterGroupOperatorAll,
Rules: []graphFilterRule{
{
Column: graphColumnDstCountry,
Operator: graphFilterRuleOperatorEqual,
Value: "FR",
}, {
Column: graphColumnSrcCountry,
Operator: graphFilterRuleOperatorEqual,
Value: "US",
},
},
},
}
payload := new(bytes.Buffer)
err = json.NewEncoder(payload).Encode(input)
if err != nil {
t.Fatalf("Encode() error:\n%+v", err)
}
resp, err := netHTTP.Post(fmt.Sprintf("http://%s/api/v0/console/graph", h.Address),
"application/json", payload)
if err != nil {
t.Fatalf("POST /api/v0/console/graph:\n%+v", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Errorf("POST /api/v0/console/graph: got status code %d, not 200", resp.StatusCode)
}
gotContentType := resp.Header.Get("Content-Type")
if gotContentType != "application/json; charset=utf-8" {
t.Errorf("POST /api/v0/console/graph Content-Type (-got, +want):\n-%s\n+%s",
gotContentType, "application/json; charset=utf-8")
}
decoder := json.NewDecoder(resp.Body)
var got gin.H
if err := decoder.Decode(&got); err != nil {
t.Fatalf("POST /api/v0/console/graph error:\n%+v", err)
}
if diff := helpers.Diff(got, expected); diff != "" {
t.Fatalf("POST /api/v0/console/graph (-got, +want):\n%s", diff)
}
}

View File

@@ -81,6 +81,7 @@ func (c *Component) Start() error {
c.d.HTTP.GinRouter.GET("/api/v0/console/widget/exporters", c.widgetExportersHandlerFunc)
c.d.HTTP.GinRouter.GET("/api/v0/console/widget/top/:name", c.widgetTopHandlerFunc)
c.d.HTTP.GinRouter.GET("/api/v0/console/widget/graph", c.widgetGraphHandlerFunc)
c.d.HTTP.GinRouter.POST("/api/v0/console/graph", c.graphHandlerFunc)
c.t.Go(func() error {
ticker := time.NewTicker(10 * time.Second)

View File

@@ -172,17 +172,16 @@ LIMIT 5
func (c *Component) widgetGraphHandlerFunc(gc *gin.Context) {
ctx := c.t.Context(gc.Request.Context())
width, err := strconv.ParseUint(gc.DefaultQuery("width", "500"), 10, 16)
points, err := strconv.ParseUint(gc.DefaultQuery("points", "200"), 10, 16)
if err != nil {
c.r.Err(err).Msg("invalid width parameter")
gc.JSON(http.StatusBadRequest, gin.H{"message": "Invalid width value."})
gc.JSON(http.StatusBadRequest, gin.H{"message": "Invalid value for points."})
return
}
if width < 5 || width > 1000 {
gc.JSON(http.StatusBadRequest, gin.H{"message": "Width should be > 5 and < 1000"})
if points < 5 || points > 1000 {
gc.JSON(http.StatusBadRequest, gin.H{"message": "Points should be > 5 and < 1000"})
return
}
interval := int64((24 * time.Hour).Seconds()) / int64(width)
interval := int64((24 * time.Hour).Seconds()) / int64(points)
now := c.d.Clock.Now()
query := c.queryFlowsTable(fmt.Sprintf(`
WITH

View File

@@ -308,7 +308,7 @@ ORDER BY Time`).
helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
{
URL: "/api/v0/console/widget/graph?width=100",
URL: "/api/v0/console/widget/graph?points=100",
ContentType: "application/json; charset=utf-8",
FirstLines: []string{
`{"data":[{"t":"2009-11-10T23:00:00Z","gbps":25.3},{"t":"2009-11-10T23:01:00Z","gbps":27.8},{"t":"2009-11-10T23:02:00Z","gbps":26.4},{"t":"2009-11-10T23:03:00Z","gbps":29.2},{"t":"2009-11-10T23:04:00Z","gbps":21.3},{"t":"2009-11-10T23:05:00Z","gbps":24.7}]}`,

2
go.mod
View File

@@ -1,6 +1,6 @@
module akvorado
go 1.17
go 1.18
require (
github.com/ClickHouse/clickhouse-go/v2 v2.0.12

14
go.sum
View File

@@ -48,15 +48,12 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/ClickHouse/clickhouse-go v1.5.3 h1:Vok8zUb/wlqc9u8oEqQzBMBRDoFd8NxPRqgYEqMnV88=
github.com/ClickHouse/clickhouse-go v1.5.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/Shopify/sarama v1.30.1/go.mod h1:hGgx05L/DiW8XYBXeJdKIN6V2QUy2H6JqME5VT1NLRw=
github.com/Shopify/sarama v1.32.1-0.20220321223103-27b8f1b5973b h1:Migey8dJIiByMK+ZNhgX0UOVhI4e4H2eoDDcrTDWDxw=
github.com/Shopify/sarama v1.32.1-0.20220321223103-27b8f1b5973b/go.mod h1:/+RbbDR4gY1hgLuBERUgPznvftUnWnHKHMzjRF0TYa4=
github.com/Shopify/toxiproxy/v2 v2.1.6-0.20210914104332-15ea381dcdae/go.mod h1:/cvHQkZ1fst0EmZnA5dFtiQdWCNCFYzb+uE2vqVgvx0=
github.com/Shopify/toxiproxy/v2 v2.3.0 h1:62YkpiP4bzdhKMH+6uC5E95y608k3zDwdzuBMsnn3uQ=
github.com/Shopify/toxiproxy/v2 v2.3.0/go.mod h1:KvQTtB6RjCJY4zqNJn7C7JDFgsG5uoHYDirfUfpIm0c=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
@@ -87,7 +84,6 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
@@ -148,7 +144,6 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/frankban/quicktest v1.14.2 h1:SPb1KFFmM+ybpEjPUhCCkZOM5xlovT5UbrMvWnXyBns=
github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
@@ -342,7 +337,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A=
github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
@@ -364,7 +358,6 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/libp2p/go-reuseport v0.1.0/go.mod h1:bQVn9hmfcTaoo0c9v5pBhOarsU1eNOBZdaAd2hzXRKU=
github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s=
github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
@@ -409,7 +402,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/netsampler/goflow2 v1.0.5-0.20220314051046-58f0f97a629c h1:A1TZ4ZpqyBEGxbGXAyAvxKSShwjR4SObXYfsgCehkTU=
github.com/netsampler/goflow2 v1.0.5-0.20220314051046-58f0f97a629c/go.mod h1:yqw2cLe+lbnDN1+JKBqxoj2FKOA83iB8wV0aCKnlesg=
github.com/oschwald/geoip2-golang v1.5.0/go.mod h1:xdvYt5xQzB8ORWFqPnqMwZpCpgNagttWdoZLlJQzg7s=
github.com/oschwald/geoip2-golang v1.6.1 h1:GKxT3yaWWNXSb7vj6D7eoJBns+lGYgx08QO0UcNm0YY=
github.com/oschwald/geoip2-golang v1.6.1/go.mod h1:xdvYt5xQzB8ORWFqPnqMwZpCpgNagttWdoZLlJQzg7s=
github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph9p/UMXK/Hk=
@@ -517,7 +509,6 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
@@ -526,9 +517,7 @@ github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/X
github.com/vincentbernat/clickhouse-go/v2 v2.0.13-0.20220422164106-2b7b2c2786d2 h1:Risx2cIItdQxF2SIs+cnn6OjR/86B2ukpMgfc0VIR9U=
github.com/vincentbernat/clickhouse-go/v2 v2.0.13-0.20220422164106-2b7b2c2786d2/go.mod h1:u4RoNQLLM2W6hNSPYrIESLJqaWSInZVmfM+MlaAhXcg=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -569,7 +558,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 h1:S25/rfnfsMVgORT4/J61MJ7rdyseOZOyvLIrZEZ7s6s=
@@ -653,7 +641,6 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
@@ -692,7 +679,6 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=