johanels/akvorado
1
0
Fork 0
mirror of https://github.com/akvorado/akvorado.git synced 2025-12-11 22:14:02 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

tests: handle JSON in TestHTTPEndpoints

Browse Source
This commit is contained in:
Vincent Bernat
2022-05-26 10:40:55 +02:00
parent a6a13a16ab
commit 2d4f5214bd
10 changed files with 393 additions and 402 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
common/helpers/tests.go
View File

@@ -4,7 +4,9 @@ package helpers
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net"
@@ -14,6 +16,7 @@ import (
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/kylelemons/godebug/pretty"
)
@@ -35,22 +38,48 @@ func Diff(a, b interface{}) string {
// HTTPEndpointCases describes case for TestHTTPEndpoints
type HTTPEndpointCases []struct {
Description string
URL string
ContentType string
StatusCode int
FirstLines []string
JSONInput interface{}
JSONOutput interface{}
}
// TestHTTPEndpoints test a few HTTP endpoints
func TestHTTPEndpoints(t *testing.T, serverAddr net.Addr, cases HTTPEndpointCases) {
t.Helper()
for _, tc := range cases {
t.Run(tc.URL, func(t *testing.T) {
desc := tc.Description
if desc == "" {
desc = tc.URL
}
t.Run(desc, func(t *testing.T) {
t.Helper()
resp, err := http.Get(fmt.Sprintf("http://%s%s", serverAddr, tc.URL))
if err != nil {
t.Fatalf("GET %s:\n%+v", tc.URL, err)
if tc.FirstLines != nil && tc.JSONOutput != nil {
t.Fatalf("Cannot have both FirstLines and JSONOutput")
}
var resp *http.Response
var err error
if tc.JSONInput == nil {
resp, err = http.Get(fmt.Sprintf("http://%s%s", serverAddr, tc.URL))
if err != nil {
t.Fatalf("GET %s:\n%+v", tc.URL, err)
}
} else {
payload := new(bytes.Buffer)
err = json.NewEncoder(payload).Encode(tc.JSONInput)
if err != nil {
t.Fatalf("Encode() error:\n%+v", err)
}
resp, err = http.Post(fmt.Sprintf("http://%s%s", serverAddr, tc.URL),
"application/json", payload)
if err != nil {
t.Fatalf("POST %s:\n%+v", tc.URL, err)
}
}
defer resp.Body.Close()
if tc.StatusCode == 0 {
tc.StatusCode = 200
@@ -59,18 +88,32 @@ func TestHTTPEndpoints(t *testing.T, serverAddr net.Addr, cases HTTPEndpointCase
t.Fatalf("GET %s: got status code %d, not %d", tc.URL,
resp.StatusCode, tc.StatusCode)
}
if tc.JSONOutput != nil {
tc.ContentType = "application/json; charset=utf-8"
}
gotContentType := resp.Header.Get("Content-Type")
if gotContentType != tc.ContentType {
t.Errorf("GET %s Content-Type (-got, +want):\n-%s\n+%s",
tc.URL, gotContentType, tc.ContentType)
}
reader := bufio.NewScanner(resp.Body)
got := []string{}
for reader.Scan() && len(got) < len(tc.FirstLines) {
got = append(got, reader.Text())
}
if diff := Diff(got, tc.FirstLines); diff != "" {
t.Errorf("GET %s (-got, +want):\n%s", tc.URL, diff)
if tc.JSONOutput == nil {
reader := bufio.NewScanner(resp.Body)
got := []string{}
for reader.Scan() && len(got) < len(tc.FirstLines) {
got = append(got, reader.Text())
}
if diff := Diff(got, tc.FirstLines); diff != "" {
t.Errorf("GET %s (-got, +want):\n%s", tc.URL, diff)
}
} else {
decoder := json.NewDecoder(resp.Body)
var got gin.H
if err := decoder.Decode(&got); err != nil {
t.Fatalf("POST %s:\n%+v", tc.URL, err)
}
if diff := Diff(got, tc.JSONOutput); diff != "" {
t.Fatalf("POST %s (-got, +want):\n%s", tc.URL, diff)
}
}
})
}

3
common/http/root_test.go
View File

@@ -63,6 +63,9 @@ func TestGinRouter(t *testing.T) {
URL: "/api/v0/test",
ContentType: "application/json; charset=utf-8",
FirstLines: []string{`{"message":"ping"}`},
}, {
URL: "/api/v0/test",
JSONOutput: gin.H{"message": "ping"},
},
})
}

49
console/graph.go
View File

@@ -12,8 +12,8 @@ import (
"akvorado/common/helpers"
)
// graphQuery describes the input for the /graph endpoint.
type graphQuery struct {
// graphHandlerInput describes the input for the /graph endpoint.
type graphHandlerInput struct {
Start time.Time `json:"start" binding:"required"`
End time.Time `json:"end" binding:"required,gtfield=Start"`
Points int `json:"points" binding:"required,min=5,max=2000"` // minimum number of points
@@ -23,12 +23,23 @@ type graphQuery struct {
Units string `json:"units" binding:"required,oneof=pps bps"`
}
// graphQueryToSQL converts a graph query to an SQL request
func (query graphQuery) toSQL() (string, error) {
interval := int64((query.End.Sub(query.Start).Seconds())) / int64(query.Points)
// graphHandlerOutput describes the output for the /graph endpoint.
type graphHandlerOutput struct {
Rows [][]string `json:"rows"`
Time []time.Time `json:"t"`
Points [][]int `json:"points"` // t → row → xps
Average []int `json:"average"` // row → xps
Min []int `json:"min"`
Max []int `json:"max"`
NinetyFivePercentile []int `json:"95th"`
}
// graphHandlerInputToSQL converts a graph input to an SQL request
func (input graphHandlerInput) toSQL() (string, error) {
interval := int64((input.End.Sub(input.Start).Seconds())) / int64(input.Points)
// Filter
where := query.Filter.filter
where := input.Filter.filter
if where == "" {
where = "{timefilter}"
} else {
@@ -39,7 +50,7 @@ func (query graphQuery) toSQL() (string, error) {
fields := []string{
`toStartOfInterval(TimeReceived, INTERVAL slot second) AS time`,
}
if query.Units == "pps" {
if input.Units == "pps" {
fields = append(fields, `SUM(Packets*SamplingRate/slot) AS xps`)
} else {
fields = append(fields, `SUM(Bytes*SamplingRate*8/slot) AS xps`)
@@ -47,7 +58,7 @@ func (query graphQuery) toSQL() (string, error) {
selectFields := []string{}
dimensions := []string{}
others := []string{}
for _, column := range query.Dimensions {
for _, column := range input.Dimensions {
field := column.toSQLSelect()
selectFields = append(selectFields, field)
dimensions = append(dimensions, column.String())
@@ -70,7 +81,7 @@ func (query graphQuery) toSQL() (string, error) {
strings.Join(dimensions, ", "),
where,
strings.Join(dimensions, ", "),
query.Limit))
input.Limit))
}
sqlQuery := fmt.Sprintf(`
@@ -85,35 +96,25 @@ ORDER BY time`, strings.Join(with, ",\n "), strings.Join(fields, ",\n "), where)
return sqlQuery, nil
}
type graphHandlerOutput struct {
Rows [][]string `json:"rows"`
Time []time.Time `json:"t"`
Points [][]int `json:"points"` // t → row → xps
Average []int `json:"average"` // row → xps
Min []int `json:"min"`
Max []int `json:"max"`
NinetyFivePercentile []int `json:"95th"`
}
func (c *Component) graphHandlerFunc(gc *gin.Context) {
ctx := c.t.Context(gc.Request.Context())
var query graphQuery
if err := gc.ShouldBindJSON(&query); err != nil {
var input graphHandlerInput
if err := gc.ShouldBindJSON(&input); err != nil {
gc.JSON(http.StatusBadRequest, gin.H{"message": helpers.Capitalize(err.Error())})
return
}
sqlQuery, err := query.toSQL()
sqlQuery, err := input.toSQL()
if err != nil {
gc.JSON(http.StatusBadRequest, gin.H{"message": helpers.Capitalize(err.Error())})
return
}
resolution := time.Duration(int64(query.End.Sub(query.Start).Nanoseconds()) / int64(query.Points))
resolution := time.Duration(int64(input.End.Sub(input.Start).Nanoseconds()) / int64(input.Points))
if resolution < time.Second {
resolution = time.Second
}
sqlQuery = c.queryFlowsTable(sqlQuery,
query.Start, query.End, resolution)
input.Start, input.End, resolution)
gc.Header("X-SQL-Query", strings.ReplaceAll(sqlQuery, "\n", " "))
results := []struct {

178
console/graph_test.go
View File

@@ -1,10 +1,6 @@
package console
import (
"bytes"
"encoding/json"
"fmt"
netHTTP "net/http"
"strings"
"testing"
"time"
@@ -22,12 +18,12 @@ import (
func TestGraphQuerySQL(t *testing.T) {
cases := []struct {
Description string
Input graphQuery
Input graphHandlerInput
Expected string
}{
{
Description: "no dimensions, no filters, bps",
Input: graphQuery{
Input: graphHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
@@ -48,7 +44,7 @@ GROUP BY time, dimensions
ORDER BY time`,
}, {
Description: "no dimensions, no filters, pps",
Input: graphQuery{
Input: graphHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
@@ -69,7 +65,7 @@ GROUP BY time, dimensions
ORDER BY time`,
}, {
Description: "no dimensions",
Input: graphQuery{
Input: graphHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
@@ -90,7 +86,7 @@ GROUP BY time, dimensions
ORDER BY time`,
}, {
Description: "no filters",
Input: graphQuery{
Input: graphHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
@@ -160,105 +156,79 @@ func TestGraphHandler(t *testing.T) {
{base.Add(2 * time.Minute), 100, []string{"router2", "provider4"}},
{base.Add(2 * time.Minute), 100, []string{"Other", "Other"}},
}
expected := gin.H{
// Sorted by sum of bps
"rows": [][]string{
{"router1", "provider2"}, // 10000
{"router1", "provider1"}, // 1600
{"router2", "provider2"}, // 1200
{"router2", "provider3"}, // 1100
{"router2", "provider4"}, // 1000
{"Other", "Other"}, // 2100
},
"t": []string{
"2009-11-10T23:00:00Z",
"2009-11-10T23:01:00Z",
"2009-11-10T23:02:00Z",
},
"points": [][]int{
{2000, 5000, 3000},
{1000, 500, 100},
{1200, 0, 0},
{1100, 0, 0},
{0, 900, 100},
{1900, 100, 100},
},
"min": []int{
2000,
100,
0,
0,
0,
100,
},
"max": []int{
5000,
1000,
1200,
1100,
900,
1900,
},
"average": []int{
3333,
533,
400,
366,
333,
700,
},
"95th": []int{
4000,
750,
600,
550,
500,
1000,
},
}
mockConn.EXPECT().
Select(gomock.Any(), gomock.Any(), gomock.Any()).
SetArg(1, expectedSQL).
Return(nil)
input := graphQuery{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Points: 100,
Limit: 20,
Dimensions: []queryColumn{
queryColumnExporterName,
queryColumnInIfProvider,
helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
{
URL: "/api/v0/console/graph",
JSONInput: gin.H{
"start": time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
"end": time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
"points": 100,
"limit": 20,
"dimensions": []string{"ExporterName", "InIfProvider"},
"filter": "DstCountry = 'FR' AND SrcCountry = 'US'",
"units": "bps",
},
JSONOutput: gin.H{
// Sorted by sum of bps
"rows": [][]string{
{"router1", "provider2"}, // 10000
{"router1", "provider1"}, // 1600
{"router2", "provider2"}, // 1200
{"router2", "provider3"}, // 1100
{"router2", "provider4"}, // 1000
{"Other", "Other"}, // 2100
},
"t": []string{
"2009-11-10T23:00:00Z",
"2009-11-10T23:01:00Z",
"2009-11-10T23:02:00Z",
},
"points": [][]int{
{2000, 5000, 3000},
{1000, 500, 100},
{1200, 0, 0},
{1100, 0, 0},
{0, 900, 100},
{1900, 100, 100},
},
"min": []int{
2000,
100,
0,
0,
0,
100,
},
"max": []int{
5000,
1000,
1200,
1100,
900,
1900,
},
"average": []int{
3333,
533,
400,
366,
333,
700,
},
"95th": []int{
4000,
750,
600,
550,
500,
1000,
},
},
},
Filter: queryFilter{"DstCountry = 'FR' AND SrcCountry = 'US'"},
Units: "bps",
}
payload := new(bytes.Buffer)
err = json.NewEncoder(payload).Encode(input)
if err != nil {
t.Fatalf("Encode() error:\n%+v", err)
}
resp, err := netHTTP.Post(fmt.Sprintf("http://%s/api/v0/console/graph", h.Address),
"application/json", payload)
if err != nil {
t.Fatalf("POST /api/v0/console/graph:\n%+v", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Errorf("POST /api/v0/console/graph: got status code %d, not 200", resp.StatusCode)
}
gotContentType := resp.Header.Get("Content-Type")
if gotContentType != "application/json; charset=utf-8" {
t.Errorf("POST /api/v0/console/graph Content-Type (-got, +want):\n-%s\n+%s",
gotContentType, "application/json; charset=utf-8")
}
decoder := json.NewDecoder(resp.Body)
var got gin.H
if err := decoder.Decode(&got); err != nil {
t.Fatalf("POST /api/v0/console/graph error:\n%+v", err)
}
if diff := helpers.Diff(got, expected); diff != "" {
t.Fatalf("POST /api/v0/console/graph (-got, +want):\n%s", diff)
}
})
}

1
console/root.go
View File

@@ -83,6 +83,7 @@ func (c *Component) Start() error {
c.d.HTTP.GinRouter.GET("/api/v0/console/widget/graph", c.widgetGraphHandlerFunc)
c.d.HTTP.GinRouter.POST("/api/v0/console/graph", c.graphHandlerFunc)
c.d.HTTP.GinRouter.POST("/api/v0/console/sankey", c.sankeyHandlerFunc)
c.d.HTTP.GinRouter.POST("/api/v0/console/filter/validate", c.filterValidateHandlerFunc)
c.t.Go(func() error {
ticker := time.NewTicker(10 * time.Second)

121
console/sankey.go
View File

@@ -12,8 +12,8 @@ import (
"akvorado/common/helpers"
)
// sankeyQuery describes the input for the /sankey endpoint.
type sankeyQuery struct {
// sankeyHandlerInput describes the input for the /sankey endpoint.
type sankeyHandlerInput struct {
Start time.Time `json:"start" binding:"required"`
End time.Time `json:"end" binding:"required,gtfield=Start"`
Dimensions []queryColumn `json:"dimensions" binding:"required,min=2"` // group by ...
@@ -22,57 +22,7 @@ type sankeyQuery struct {
Units string `json:"units" binding:"required,oneof=pps bps"`
}
// sankeyQueryToSQL converts a sankey query to an SQL request
func (query sankeyQuery) toSQL() (string, error) {
// Filter
where := query.Filter.filter
if where == "" {
where = "{timefilter}"
} else {
where = fmt.Sprintf("{timefilter} AND (%s)", where)
}
// Select
arrayFields := []string{}
dimensions := []string{}
for _, column := range query.Dimensions {
arrayFields = append(arrayFields, fmt.Sprintf(`if(%s IN (SELECT %s FROM rows), %s, 'Other')`,
column.String(),
column.String(),
column.toSQLSelect()))
dimensions = append(dimensions, column.String())
}
fields := []string{}
if query.Units == "pps" {
fields = append(fields, `SUM(Packets*SamplingRate/range) AS xps`)
} else {
fields = append(fields, `SUM(Bytes*SamplingRate*8/range) AS xps`)
}
fields = append(fields, fmt.Sprintf("[%s] AS dimensions", strings.Join(arrayFields, ",\n ")))
// With
with := []string{
fmt.Sprintf(`(SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM {table} WHERE %s) AS range`, where),
fmt.Sprintf(
"rows AS (SELECT %s FROM {table} WHERE %s GROUP BY %s ORDER BY SUM(Bytes) DESC LIMIT %d)",
strings.Join(dimensions, ", "),
where,
strings.Join(dimensions, ", "),
query.Limit),
}
sqlQuery := fmt.Sprintf(`
WITH
%s
SELECT
%s
FROM {table}
WHERE %s
GROUP BY dimensions
ORDER BY xps DESC`, strings.Join(with, ",\n "), strings.Join(fields, ",\n "), where)
return sqlQuery, nil
}
// sankeyHandlerOutput describes the output for the /sankey endpoint.
type sankeyHandlerOutput struct {
// Unprocessed data for table view
Rows [][]string `json:"rows"`
@@ -87,29 +37,80 @@ type sankeyLink struct {
Xps int `json:"xps"`
}
// sankeyHandlerInputToSQL converts a sankey query to an SQL request
func (input sankeyHandlerInput) toSQL() (string, error) {
// Filter
where := input.Filter.filter
if where == "" {
where = "{timefilter}"
} else {
where = fmt.Sprintf("{timefilter} AND (%s)", where)
}
// Select
arrayFields := []string{}
dimensions := []string{}
for _, column := range input.Dimensions {
arrayFields = append(arrayFields, fmt.Sprintf(`if(%s IN (SELECT %s FROM rows), %s, 'Other')`,
column.String(),
column.String(),
column.toSQLSelect()))
dimensions = append(dimensions, column.String())
}
fields := []string{}
if input.Units == "pps" {
fields = append(fields, `SUM(Packets*SamplingRate/range) AS xps`)
} else {
fields = append(fields, `SUM(Bytes*SamplingRate*8/range) AS xps`)
}
fields = append(fields, fmt.Sprintf("[%s] AS dimensions", strings.Join(arrayFields, ",\n ")))
// With
with := []string{
fmt.Sprintf(`(SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM {table} WHERE %s) AS range`, where),
fmt.Sprintf(
"rows AS (SELECT %s FROM {table} WHERE %s GROUP BY %s ORDER BY SUM(Bytes) DESC LIMIT %d)",
strings.Join(dimensions, ", "),
where,
strings.Join(dimensions, ", "),
input.Limit),
}
sqlQuery := fmt.Sprintf(`
WITH
%s
SELECT
%s
FROM {table}
WHERE %s
GROUP BY dimensions
ORDER BY xps DESC`, strings.Join(with, ",\n "), strings.Join(fields, ",\n "), where)
return sqlQuery, nil
}
func (c *Component) sankeyHandlerFunc(gc *gin.Context) {
ctx := c.t.Context(gc.Request.Context())
var query sankeyQuery
if err := gc.ShouldBindJSON(&query); err != nil {
var input sankeyHandlerInput
if err := gc.ShouldBindJSON(&input); err != nil {
gc.JSON(http.StatusBadRequest, gin.H{"message": helpers.Capitalize(err.Error())})
return
}
sqlQuery, err := query.toSQL()
sqlQuery, err := input.toSQL()
if err != nil {
gc.JSON(http.StatusBadRequest, gin.H{"message": helpers.Capitalize(err.Error())})
return
}
// We need to select a resolution allowing us to have a somewhat accurate timespan
resolution := time.Duration(int64(query.End.Sub(query.Start).Nanoseconds()) / 20)
resolution := time.Duration(int64(input.End.Sub(input.Start).Nanoseconds()) / 20)
if resolution < time.Second {
resolution = time.Second
}
// Prepare and execute query
sqlQuery = c.queryFlowsTable(sqlQuery,
query.Start, query.End, resolution)
input.Start, input.End, resolution)
gc.Header("X-SQL-Query", strings.ReplaceAll(sqlQuery, "\n", " "))
results := []struct {
Xps float64 `ch:"xps"`
@@ -132,7 +133,7 @@ func (c *Component) sankeyHandlerFunc(gc *gin.Context) {
if name != "Other" {
return name
}
return fmt.Sprintf("Other %s", query.Dimensions[index].String())
return fmt.Sprintf("Other %s", input.Dimensions[index].String())
}
addedNodes := map[string]bool{}
addNode := func(name string) {
@@ -154,7 +155,7 @@ func (c *Component) sankeyHandlerFunc(gc *gin.Context) {
output.Rows = append(output.Rows, result.Dimensions)
output.Xps = append(output.Xps, int(result.Xps))
// Consider each pair of successive dimensions
for i := 0; i < len(query.Dimensions)-1; i++ {
for i := 0; i < len(input.Dimensions)-1; i++ {
dimension1 := completeName(result.Dimensions[i], i)
dimension2 := completeName(result.Dimensions[i+1], i+1)
addNode(dimension1)

239
console/sankey_test.go
View File

@@ -1,10 +1,6 @@
package console
import (
"bytes"
"encoding/json"
"fmt"
netHTTP "net/http"
"strings"
"testing"
"time"
@@ -22,12 +18,12 @@ import (
func TestSankeyQuerySQL(t *testing.T) {
cases := []struct {
Description string
Input sankeyQuery
Input sankeyHandlerInput
Expected string
}{
{
Description: "two dimensions, no filters, bps",
Input: sankeyQuery{
Input: sankeyHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Dimensions: []queryColumn{queryColumnSrcAS, queryColumnExporterName},
@@ -49,7 +45,7 @@ GROUP BY dimensions
ORDER BY xps DESC`,
}, {
Description: "two dimensions, no filters, pps",
Input: sankeyQuery{
Input: sankeyHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Dimensions: []queryColumn{queryColumnSrcAS, queryColumnExporterName},
@@ -71,7 +67,7 @@ GROUP BY dimensions
ORDER BY xps DESC`,
}, {
Description: "two dimensions, with filter",
Input: sankeyQuery{
Input: sankeyHandlerInput{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Dimensions: []queryColumn{queryColumnSrcAS, queryColumnExporterName},
@@ -147,134 +143,111 @@ func TestSankeyHandler(t *testing.T) {
{621, []string{"Other", "Other", "Other"}},
{159, []string{"Other", "provider1", "router1"}},
}
expected := gin.H{
// Raw data
"rows": [][]string{
{"AS100", "Other", "router1"},
{"AS300", "provider1", "Other"},
{"AS300", "provider2", "router1"},
{"AS200", "provider1", "Other"},
{"AS100", "provider1", "Other"},
{"Other", "provider1", "Other"},
{"AS200", "provider3", "Other"},
{"AS200", "Other", "router2"},
{"AS100", "provider3", "Other"},
{"AS100", "provider3", "router2"},
{"Other", "Other", "router1"},
{"AS300", "provider3", "router2"},
{"AS300", "Other", "router1"},
{"AS100", "provider1", "router1"},
{"AS200", "provider2", "router2"},
{"AS100", "provider2", "Other"},
{"AS200", "Other", "router1"},
{"AS300", "Other", "Other"},
{"AS200", "provider3", "router2"},
{"Other", "Other", "Other"},
{"Other", "provider1", "router1"},
},
"xps": []int{
9677,
9472,
7593,
7234,
6006,
5988,
4675,
4348,
3999,
3978,
3623,
3080,
2915,
2623,
2482,
2234,
1360,
975,
717,
621,
159,
},
// For graph
"nodes": []string{
"AS100",
"Other InIfProvider",
"router1",
"AS300",
"provider1",
"Other ExporterName",
"provider2",
"AS200",
"Other SrcAS",
"provider3",
"router2",
},
"links": []gin.H{
{"source": "provider1", "target": "Other ExporterName", "xps": 9472 + 7234 + 6006 + 5988},
{"source": "Other InIfProvider", "target": "router1", "xps": 9677 + 3623 + 2915 + 1360},
{"source": "AS100", "target": "Other InIfProvider", "xps": 9677},
{"source": "AS300", "target": "provider1", "xps": 9472},
{"source": "provider3", "target": "Other ExporterName", "xps": 4675 + 3999},
{"source": "AS100", "target": "provider1", "xps": 6006 + 2623},
{"source": "AS100", "target": "provider3", "xps": 3999 + 3978},
{"source": "provider3", "target": "router2", "xps": 3978 + 3080 + 717},
{"source": "AS300", "target": "provider2", "xps": 7593},
{"source": "provider2", "target": "router1", "xps": 7593},
{"source": "AS200", "target": "provider1", "xps": 7234},
{"source": "Other SrcAS", "target": "provider1", "xps": 5988 + 159},
{"source": "AS200", "target": "Other InIfProvider", "xps": 4348 + 1360},
{"source": "AS200", "target": "provider3", "xps": 4675 + 717},
{"source": "Other InIfProvider", "target": "router2", "xps": 4348},
{"source": "Other SrcAS", "target": "Other InIfProvider", "xps": 3623 + 621},
{"source": "AS300", "target": "Other InIfProvider", "xps": 2915 + 975},
{"source": "AS300", "target": "provider3", "xps": 3080},
{"source": "provider1", "target": "router1", "xps": 2623 + 159},
{"source": "AS200", "target": "provider2", "xps": 2482},
{"source": "provider2", "target": "router2", "xps": 2482},
{"source": "AS100", "target": "provider2", "xps": 2234},
{"source": "provider2", "target": "Other ExporterName", "xps": 2234},
{"source": "Other InIfProvider", "target": "Other ExporterName", "xps": 975 + 621},
},
}
mockConn.EXPECT().
Select(gomock.Any(), gomock.Any(), gomock.Any()).
SetArg(1, expectedSQL).
Return(nil)
input := sankeyQuery{
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Dimensions: []queryColumn{queryColumnSrcAS, queryColumnInIfProvider, queryColumnExporterName},
Limit: 10,
Filter: queryFilter{"DstCountry = 'FR'"},
Units: "bps",
}
payload := new(bytes.Buffer)
err = json.NewEncoder(payload).Encode(input)
if err != nil {
t.Fatalf("Encode() error:\n%+v", err)
}
resp, err := netHTTP.Post(fmt.Sprintf("http://%s/api/v0/console/sankey", h.Address),
"application/json", payload)
if err != nil {
t.Fatalf("POST /api/v0/console/sankey:\n%+v", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Errorf("POST /api/v0/console/sankey: got status code %d, not 200", resp.StatusCode)
}
gotContentType := resp.Header.Get("Content-Type")
if gotContentType != "application/json; charset=utf-8" {
t.Errorf("POST /api/v0/console/sankey Content-Type (-got, +want):\n-%s\n+%s",
gotContentType, "application/json; charset=utf-8")
}
decoder := json.NewDecoder(resp.Body)
var got gin.H
if err := decoder.Decode(&got); err != nil {
t.Fatalf("POST /api/v0/console/sankey error:\n%+v", err)
}
if diff := helpers.Diff(got, expected); diff != "" {
t.Fatalf("POST /api/v0/console/sankey (-got, +want):\n%s", diff)
}
helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
{
URL: "/api/v0/console/sankey",
JSONInput: gin.H{
"start": time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
"end": time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
"dimensions": []string{"SrcAS", "InIfProvider", "ExporterName"},
"limit": 10,
"filter": "DstCountry = 'FR'",
"units": "bps",
},
JSONOutput: gin.H{
// Raw data
"rows": [][]string{
{"AS100", "Other", "router1"},
{"AS300", "provider1", "Other"},
{"AS300", "provider2", "router1"},
{"AS200", "provider1", "Other"},
{"AS100", "provider1", "Other"},
{"Other", "provider1", "Other"},
{"AS200", "provider3", "Other"},
{"AS200", "Other", "router2"},
{"AS100", "provider3", "Other"},
{"AS100", "provider3", "router2"},
{"Other", "Other", "router1"},
{"AS300", "provider3", "router2"},
{"AS300", "Other", "router1"},
{"AS100", "provider1", "router1"},
{"AS200", "provider2", "router2"},
{"AS100", "provider2", "Other"},
{"AS200", "Other", "router1"},
{"AS300", "Other", "Other"},
{"AS200", "provider3", "router2"},
{"Other", "Other", "Other"},
{"Other", "provider1", "router1"},
},
"xps": []int{
9677,
9472,
7593,
7234,
6006,
5988,
4675,
4348,
3999,
3978,
3623,
3080,
2915,
2623,
2482,
2234,
1360,
975,
717,
621,
159,
},
// For graph
"nodes": []string{
"AS100",
"Other InIfProvider",
"router1",
"AS300",
"provider1",
"Other ExporterName",
"provider2",
"AS200",
"Other SrcAS",
"provider3",
"router2",
},
"links": []gin.H{
{"source": "provider1", "target": "Other ExporterName", "xps": 9472 + 7234 + 6006 + 5988},
{"source": "Other InIfProvider", "target": "router1", "xps": 9677 + 3623 + 2915 + 1360},
{"source": "AS100", "target": "Other InIfProvider", "xps": 9677},
{"source": "AS300", "target": "provider1", "xps": 9472},
{"source": "provider3", "target": "Other ExporterName", "xps": 4675 + 3999},
{"source": "AS100", "target": "provider1", "xps": 6006 + 2623},
{"source": "AS100", "target": "provider3", "xps": 3999 + 3978},
{"source": "provider3", "target": "router2", "xps": 3978 + 3080 + 717},
{"source": "AS300", "target": "provider2", "xps": 7593},
{"source": "provider2", "target": "router1", "xps": 7593},
{"source": "AS200", "target": "provider1", "xps": 7234},
{"source": "Other SrcAS", "target": "provider1", "xps": 5988 + 159},
{"source": "AS200", "target": "Other InIfProvider", "xps": 4348 + 1360},
{"source": "AS200", "target": "provider3", "xps": 4675 + 717},
{"source": "Other InIfProvider", "target": "router2", "xps": 4348},
{"source": "Other SrcAS", "target": "Other InIfProvider", "xps": 3623 + 621},
{"source": "AS300", "target": "Other InIfProvider", "xps": 2915 + 975},
{"source": "AS300", "target": "provider3", "xps": 3080},
{"source": "provider1", "target": "router1", "xps": 2623 + 159},
{"source": "AS200", "target": "provider2", "xps": 2482},
{"source": "provider2", "target": "router2", "xps": 2482},
{"source": "AS100", "target": "provider2", "xps": 2234},
{"source": "provider2", "target": "Other ExporterName", "xps": 2234},
{"source": "Other InIfProvider", "target": "Other ExporterName", "xps": 975 + 621},
},
},
},
})
}

107
console/widgets_test.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
"github.com/benbjohnson/clock"
"github.com/gin-gonic/gin"
"github.com/golang/mock/gomock"
"akvorado/common/clickhousedb"
@@ -90,18 +91,15 @@ func TestWidgetLastFlow(t *testing.T) {
helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
{
URL: "/api/v0/console/widget/flow-last",
ContentType: "application/json; charset=utf-8",
FirstLines: []string{
`{`,
` "InIfBoundary": "external",`,
` "InIfName": "Hu0/0/1/10",`,
` "InIfSpeed": 100000,`,
` "SamplingRate": 10000,`,
` "SrcAddr": "2001:db8::22",`,
` "SrcCountry": "FR",`,
` "TimeReceived": "2022-04-04T08:36:11.00000001Z"`,
`}`,
URL: "/api/v0/console/widget/flow-last",
JSONOutput: gin.H{
"InIfBoundary": "external",
"InIfName": "Hu0/0/1/10",
"InIfSpeed": 100000,
"SamplingRate": 10000,
"SrcAddr": "2001:db8::22",
"SrcCountry": "FR",
"TimeReceived": "2022-04-04T08:36:11.00000001Z",
},
},
})
@@ -132,13 +130,10 @@ func TestFlowRate(t *testing.T) {
helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
{
URL: "/api/v0/console/widget/flow-rate",
ContentType: "application/json; charset=utf-8",
FirstLines: []string{
`{`,
` "period": "second",`,
` "rate": 100.1`,
`}`,
URL: "/api/v0/console/widget/flow-rate",
JSONOutput: gin.H{
"period": "second",
"rate": 100.1,
},
},
})
@@ -173,16 +168,13 @@ func TestWidgetExporters(t *testing.T) {
helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
{
URL: "/api/v0/console/widget/exporters",
ContentType: "application/json; charset=utf-8",
FirstLines: []string{
`{`,
` "exporters": [`,
` "exporter1",`,
` "exporter2",`,
` "exporter3"`,
` ]`,
`}`,
URL: "/api/v0/console/widget/exporters",
JSONOutput: gin.H{
"exporters": []string{
"exporter1",
"exporter2",
"exporter3",
},
},
},
})
@@ -235,29 +227,33 @@ func TestWidgetTop(t *testing.T) {
helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
{
URL: "/api/v0/console/widget/top/src-port",
ContentType: "application/json; charset=utf-8",
FirstLines: []string{
`{"top":[{"name":"TCP/443","percent":51},{"name":"UDP/443","percent":20},{"name":"TCP/80","percent":18}]}`,
},
URL: "/api/v0/console/widget/top/src-port",
JSONOutput: gin.H{
"top": []gin.H{
gin.H{"name": "TCP/443", "percent": 51},
gin.H{"name": "UDP/443", "percent": 20},
gin.H{"name": "TCP/80", "percent": 18}}},
}, {
URL: "/api/v0/console/widget/top/protocol",
ContentType: "application/json; charset=utf-8",
FirstLines: []string{
`{"top":[{"name":"TCP","percent":75},{"name":"UDP","percent":24},{"name":"ESP","percent":1}]}`,
},
URL: "/api/v0/console/widget/top/protocol",
JSONOutput: gin.H{
"top": []gin.H{
gin.H{"name": "TCP", "percent": 75},
gin.H{"name": "UDP", "percent": 24},
gin.H{"name": "ESP", "percent": 1}}},
}, {
URL: "/api/v0/console/widget/top/exporter",
ContentType: "application/json; charset=utf-8",
FirstLines: []string{
`{"top":[{"name":"exporter1","percent":20},{"name":"exporter3","percent":10},{"name":"exporter5","percent":3}]}`,
},
URL: "/api/v0/console/widget/top/exporter",
JSONOutput: gin.H{
"top": []gin.H{
gin.H{"name": "exporter1", "percent": 20},
gin.H{"name": "exporter3", "percent": 10},
gin.H{"name": "exporter5", "percent": 3}}},
}, {
URL: "/api/v0/console/widget/top/src-as",
ContentType: "application/json; charset=utf-8",
FirstLines: []string{
`{"top":[{"name":"2906: Netflix","percent":12},{"name":"36040: Youtube","percent":10},{"name":"20940: Akamai","percent":9}]}`,
},
URL: "/api/v0/console/widget/top/src-as",
JSONOutput: gin.H{
"top": []gin.H{
gin.H{"name": "2906: Netflix", "percent": 12},
gin.H{"name": "36040: Youtube", "percent": 10},
gin.H{"name": "20940: Akamai", "percent": 9}}},
},
})
}
@@ -308,10 +304,15 @@ ORDER BY Time`).
helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
{
URL: "/api/v0/console/widget/graph?points=100",
ContentType: "application/json; charset=utf-8",
FirstLines: []string{
`{"data":[{"t":"2009-11-10T23:00:00Z","gbps":25.3},{"t":"2009-11-10T23:01:00Z","gbps":27.8},{"t":"2009-11-10T23:02:00Z","gbps":26.4},{"t":"2009-11-10T23:03:00Z","gbps":29.2},{"t":"2009-11-10T23:04:00Z","gbps":21.3},{"t":"2009-11-10T23:05:00Z","gbps":24.7}]}`,
URL: "/api/v0/console/widget/graph?points=100",
JSONOutput: gin.H{
"data": []gin.H{
gin.H{"t": "2009-11-10T23:00:00Z", "gbps": 25.3},
gin.H{"t": "2009-11-10T23:01:00Z", "gbps": 27.8},
gin.H{"t": "2009-11-10T23:02:00Z", "gbps": 26.4},
gin.H{"t": "2009-11-10T23:03:00Z", "gbps": 29.2},
gin.H{"t": "2009-11-10T23:04:00Z", "gbps": 21.3},
gin.H{"t": "2009-11-10T23:05:00Z", "gbps": 24.7}},
},
},
})

19
inlet/flow/schemas_test.go
View File

@@ -5,6 +5,8 @@ import (
"akvorado/common/helpers"
"akvorado/common/reporter"
"github.com/gin-gonic/gin"
)
func TestHTTPEndpoints(t *testing.T) {
@@ -20,16 +22,13 @@ func TestHTTPEndpoints(t *testing.T) {
`package decoder;`,
},
}, {
URL: "/api/v0/inlet/flow/schemas.json",
ContentType: "application/json; charset=utf-8",
FirstLines: []string{
`{`,
` "current-version": 1,`,
` "versions": {`,
` "0": "/api/v0/inlet/flow/schema-0.proto",`,
` "1": "/api/v0/inlet/flow/schema-1.proto"`,
` }`,
`}`,
URL: "/api/v0/inlet/flow/schemas.json",
JSONOutput: gin.H{
"current-version": 1,
"versions": gin.H{
"0": "/api/v0/inlet/flow/schema-0.proto",
"1": "/api/v0/inlet/flow/schema-1.proto",
},
},
},
}

13
orchestrator/broker/http_test.go
View File

@@ -6,6 +6,8 @@ import (
"akvorado/common/helpers"
"akvorado/common/http"
"akvorado/common/reporter"
"github.com/gin-gonic/gin"
)
func TestConfigurationEndpoint(t *testing.T) {
@@ -24,13 +26,10 @@ func TestConfigurationEndpoint(t *testing.T) {
helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
{
URL: "/api/v0/orchestrator/broker/configuration/inlet",
ContentType: "application/json; charset=utf-8",
FirstLines: []string{
"{",
` "bye": "Goodbye world!",`,
` "hello": "Hello world!"`,
"}",
URL: "/api/v0/orchestrator/broker/configuration/inlet",
JSONOutput: gin.H{
"bye": "Goodbye world!",
"hello": "Hello world!",
},
},
})