tests: handle JSON in TestHTTPEndpoints

common/helpers/tests.go

@@ -4,7 +4,9 @@ package helpers
 
 import (
 	"bufio"
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net"
@@ -14,6 +16,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/gin-gonic/gin"
 	"github.com/kylelemons/godebug/pretty"
 )
 
@@ -35,22 +38,48 @@ func Diff(a, b interface{}) string {
 
 // HTTPEndpointCases describes case for TestHTTPEndpoints
 type HTTPEndpointCases []struct {
+	Description string
 	URL         string
 	ContentType string
 	StatusCode  int
 	FirstLines  []string
+	JSONInput   interface{}
+	JSONOutput  interface{}
 }
 
 // TestHTTPEndpoints test a few HTTP endpoints
 func TestHTTPEndpoints(t *testing.T, serverAddr net.Addr, cases HTTPEndpointCases) {
 	t.Helper()
 	for _, tc := range cases {
-		t.Run(tc.URL, func(t *testing.T) {
+		desc := tc.Description
+		if desc == "" {
+			desc = tc.URL
+		}
+		t.Run(desc, func(t *testing.T) {
 			t.Helper()
-			resp, err := http.Get(fmt.Sprintf("http://%s%s", serverAddr, tc.URL))
+			if tc.FirstLines != nil && tc.JSONOutput != nil {
-			if err != nil {
+				t.Fatalf("Cannot have both FirstLines and JSONOutput")
-				t.Fatalf("GET %s:\n%+v", tc.URL, err)
 			}
+			var resp *http.Response
+			var err error
+			if tc.JSONInput == nil {
+				resp, err = http.Get(fmt.Sprintf("http://%s%s", serverAddr, tc.URL))
+				if err != nil {
+					t.Fatalf("GET %s:\n%+v", tc.URL, err)
+				}
+			} else {
+				payload := new(bytes.Buffer)
+				err = json.NewEncoder(payload).Encode(tc.JSONInput)
+				if err != nil {
+					t.Fatalf("Encode() error:\n%+v", err)
+				}
+				resp, err = http.Post(fmt.Sprintf("http://%s%s", serverAddr, tc.URL),
+					"application/json", payload)
+				if err != nil {
+					t.Fatalf("POST %s:\n%+v", tc.URL, err)
+				}
+			}
+
 			defer resp.Body.Close()
 			if tc.StatusCode == 0 {
 				tc.StatusCode = 200
@@ -59,18 +88,32 @@ func TestHTTPEndpoints(t *testing.T, serverAddr net.Addr, cases HTTPEndpointCase
 				t.Fatalf("GET %s: got status code %d, not %d", tc.URL,
 					resp.StatusCode, tc.StatusCode)
 			}
+			if tc.JSONOutput != nil {
+				tc.ContentType = "application/json; charset=utf-8"
+			}
 			gotContentType := resp.Header.Get("Content-Type")
 			if gotContentType != tc.ContentType {
 				t.Errorf("GET %s Content-Type (-got, +want):\n-%s\n+%s",
 					tc.URL, gotContentType, tc.ContentType)
 			}
-			reader := bufio.NewScanner(resp.Body)
+			if tc.JSONOutput == nil {
-			got := []string{}
+				reader := bufio.NewScanner(resp.Body)
-			for reader.Scan() && len(got) < len(tc.FirstLines) {
+				got := []string{}
-				got = append(got, reader.Text())
+				for reader.Scan() && len(got) < len(tc.FirstLines) {
-			}
+					got = append(got, reader.Text())
-			if diff := Diff(got, tc.FirstLines); diff != "" {
+				}
-				t.Errorf("GET %s (-got, +want):\n%s", tc.URL, diff)
+				if diff := Diff(got, tc.FirstLines); diff != "" {
+					t.Errorf("GET %s (-got, +want):\n%s", tc.URL, diff)
+				}
+			} else {
+				decoder := json.NewDecoder(resp.Body)
+				var got gin.H
+				if err := decoder.Decode(&got); err != nil {
+					t.Fatalf("POST %s:\n%+v", tc.URL, err)
+				}
+				if diff := Diff(got, tc.JSONOutput); diff != "" {
+					t.Fatalf("POST %s (-got, +want):\n%s", tc.URL, diff)
+				}
 			}
 		})
 	}

common/http/root_test.go

@@ -63,6 +63,9 @@ func TestGinRouter(t *testing.T) {
 			URL:         "/api/v0/test",
 			ContentType: "application/json; charset=utf-8",
 			FirstLines:  []string{`{"message":"ping"}`},
+		}, {
+			URL:        "/api/v0/test",
+			JSONOutput: gin.H{"message": "ping"},
 		},
 	})
 }

console/graph.go

@@ -12,8 +12,8 @@ import (
 	"akvorado/common/helpers"
 )
 
-// graphQuery describes the input for the /graph endpoint.
+// graphHandlerInput describes the input for the /graph endpoint.
-type graphQuery struct {
+type graphHandlerInput struct {
 	Start      time.Time     `json:"start" binding:"required"`
 	End        time.Time     `json:"end" binding:"required,gtfield=Start"`
 	Points     int           `json:"points" binding:"required,min=5,max=2000"` // minimum number of points
@@ -23,12 +23,23 @@ type graphQuery struct {
 	Units      string        `json:"units" binding:"required,oneof=pps bps"`
 }
 
-// graphQueryToSQL converts a graph query to an SQL request
+// graphHandlerOutput describes the output for the /graph endpoint.
-func (query graphQuery) toSQL() (string, error) {
+type graphHandlerOutput struct {
-	interval := int64((query.End.Sub(query.Start).Seconds())) / int64(query.Points)
+	Rows                 [][]string  `json:"rows"`
+	Time                 []time.Time `json:"t"`
+	Points               [][]int     `json:"points"`  // t → row → xps
+	Average              []int       `json:"average"` // row → xps
+	Min                  []int       `json:"min"`
+	Max                  []int       `json:"max"`
+	NinetyFivePercentile []int       `json:"95th"`
+}
+
+// graphHandlerInputToSQL converts a graph input to an SQL request
+func (input graphHandlerInput) toSQL() (string, error) {
+	interval := int64((input.End.Sub(input.Start).Seconds())) / int64(input.Points)
 
 	// Filter
-	where := query.Filter.filter
+	where := input.Filter.filter
 	if where == "" {
 		where = "{timefilter}"
 	} else {
@@ -39,7 +50,7 @@ func (query graphQuery) toSQL() (string, error) {
 	fields := []string{
 		`toStartOfInterval(TimeReceived, INTERVAL slot second) AS time`,
 	}
-	if query.Units == "pps" {
+	if input.Units == "pps" {
 		fields = append(fields, `SUM(Packets*SamplingRate/slot) AS xps`)
 	} else {
 		fields = append(fields, `SUM(Bytes*SamplingRate*8/slot) AS xps`)
@@ -47,7 +58,7 @@ func (query graphQuery) toSQL() (string, error) {
 	selectFields := []string{}
 	dimensions := []string{}
 	others := []string{}
-	for _, column := range query.Dimensions {
+	for _, column := range input.Dimensions {
 		field := column.toSQLSelect()
 		selectFields = append(selectFields, field)
 		dimensions = append(dimensions, column.String())
@@ -70,7 +81,7 @@ func (query graphQuery) toSQL() (string, error) {
 			strings.Join(dimensions, ", "),
 			where,
 			strings.Join(dimensions, ", "),
-			query.Limit))
+			input.Limit))
 	}
 
 	sqlQuery := fmt.Sprintf(`
@@ -85,35 +96,25 @@ ORDER BY time`, strings.Join(with, ",\n "), strings.Join(fields, ",\n "), where)
 	return sqlQuery, nil
 }
 
-type graphHandlerOutput struct {
-	Rows                 [][]string  `json:"rows"`
-	Time                 []time.Time `json:"t"`
-	Points               [][]int     `json:"points"`  // t → row → xps
-	Average              []int       `json:"average"` // row → xps
-	Min                  []int       `json:"min"`
-	Max                  []int       `json:"max"`
-	NinetyFivePercentile []int       `json:"95th"`
-}
-
 func (c *Component) graphHandlerFunc(gc *gin.Context) {
 	ctx := c.t.Context(gc.Request.Context())
-	var query graphQuery
+	var input graphHandlerInput
-	if err := gc.ShouldBindJSON(&query); err != nil {
+	if err := gc.ShouldBindJSON(&input); err != nil {
 		gc.JSON(http.StatusBadRequest, gin.H{"message": helpers.Capitalize(err.Error())})
 		return
 	}
 
-	sqlQuery, err := query.toSQL()
+	sqlQuery, err := input.toSQL()
 	if err != nil {
 		gc.JSON(http.StatusBadRequest, gin.H{"message": helpers.Capitalize(err.Error())})
 		return
 	}
-	resolution := time.Duration(int64(query.End.Sub(query.Start).Nanoseconds()) / int64(query.Points))
+	resolution := time.Duration(int64(input.End.Sub(input.Start).Nanoseconds()) / int64(input.Points))
 	if resolution < time.Second {
 		resolution = time.Second
 	}
 	sqlQuery = c.queryFlowsTable(sqlQuery,
-		query.Start, query.End, resolution)
+		input.Start, input.End, resolution)
 	gc.Header("X-SQL-Query", strings.ReplaceAll(sqlQuery, "\n", "  "))
 
 	results := []struct {

console/graph_test.go

@@ -1,10 +1,6 @@
 package console
 
 import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	netHTTP "net/http"
 	"strings"
 	"testing"
 	"time"
@@ -22,12 +18,12 @@ import (
 func TestGraphQuerySQL(t *testing.T) {
 	cases := []struct {
 		Description string
-		Input       graphQuery
+		Input       graphHandlerInput
 		Expected    string
 	}{
 		{
 			Description: "no dimensions, no filters, bps",
-			Input: graphQuery{
+			Input: graphHandlerInput{
 				Start:      time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
 				End:        time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
 				Points:     100,
@@ -48,7 +44,7 @@ GROUP BY time, dimensions
 ORDER BY time`,
 		}, {
 			Description: "no dimensions, no filters, pps",
-			Input: graphQuery{
+			Input: graphHandlerInput{
 				Start:      time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
 				End:        time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
 				Points:     100,
@@ -69,7 +65,7 @@ GROUP BY time, dimensions
 ORDER BY time`,
 		}, {
 			Description: "no dimensions",
-			Input: graphQuery{
+			Input: graphHandlerInput{
 				Start:      time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
 				End:        time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
 				Points:     100,
@@ -90,7 +86,7 @@ GROUP BY time, dimensions
 ORDER BY time`,
 		}, {
 			Description: "no filters",
-			Input: graphQuery{
+			Input: graphHandlerInput{
 				Start:  time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
 				End:    time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
 				Points: 100,
@@ -160,105 +156,79 @@ func TestGraphHandler(t *testing.T) {
 		{base.Add(2 * time.Minute), 100, []string{"router2", "provider4"}},
 		{base.Add(2 * time.Minute), 100, []string{"Other", "Other"}},
 	}
-	expected := gin.H{
-		// Sorted by sum of bps
-		"rows": [][]string{
-			{"router1", "provider2"}, // 10000
-			{"router1", "provider1"}, // 1600
-			{"router2", "provider2"}, // 1200
-			{"router2", "provider3"}, // 1100
-			{"router2", "provider4"}, // 1000
-			{"Other", "Other"},       // 2100
-		},
-		"t": []string{
-			"2009-11-10T23:00:00Z",
-			"2009-11-10T23:01:00Z",
-			"2009-11-10T23:02:00Z",
-		},
-		"points": [][]int{
-			{2000, 5000, 3000},
-			{1000, 500, 100},
-			{1200, 0, 0},
-			{1100, 0, 0},
-			{0, 900, 100},
-			{1900, 100, 100},
-		},
-		"min": []int{
-			2000,
-			100,
-			0,
-			0,
-			0,
-			100,
-		},
-		"max": []int{
-			5000,
-			1000,
-			1200,
-			1100,
-			900,
-			1900,
-		},
-		"average": []int{
-			3333,
-			533,
-			400,
-			366,
-			333,
-			700,
-		},
-		"95th": []int{
-			4000,
-			750,
-			600,
-			550,
-			500,
-			1000,
-		},
-	}
 	mockConn.EXPECT().
 		Select(gomock.Any(), gomock.Any(), gomock.Any()).
 		SetArg(1, expectedSQL).
 		Return(nil)
 
-	input := graphQuery{
+	helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
-		Start:  time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
+		{
-		End:    time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
+			URL: "/api/v0/console/graph",
-		Points: 100,
+			JSONInput: gin.H{
-		Limit:  20,
+				"start":      time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
-		Dimensions: []queryColumn{
+				"end":        time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
-			queryColumnExporterName,
+				"points":     100,
-			queryColumnInIfProvider,
+				"limit":      20,
+				"dimensions": []string{"ExporterName", "InIfProvider"},
+				"filter":     "DstCountry = 'FR' AND SrcCountry = 'US'",
+				"units":      "bps",
+			},
+			JSONOutput: gin.H{
+				// Sorted by sum of bps
+				"rows": [][]string{
+					{"router1", "provider2"}, // 10000
+					{"router1", "provider1"}, // 1600
+					{"router2", "provider2"}, // 1200
+					{"router2", "provider3"}, // 1100
+					{"router2", "provider4"}, // 1000
+					{"Other", "Other"},       // 2100
+				},
+				"t": []string{
+					"2009-11-10T23:00:00Z",
+					"2009-11-10T23:01:00Z",
+					"2009-11-10T23:02:00Z",
+				},
+				"points": [][]int{
+					{2000, 5000, 3000},
+					{1000, 500, 100},
+					{1200, 0, 0},
+					{1100, 0, 0},
+					{0, 900, 100},
+					{1900, 100, 100},
+				},
+				"min": []int{
+					2000,
+					100,
+					0,
+					0,
+					0,
+					100,
+				},
+				"max": []int{
+					5000,
+					1000,
+					1200,
+					1100,
+					900,
+					1900,
+				},
+				"average": []int{
+					3333,
+					533,
+					400,
+					366,
+					333,
+					700,
+				},
+				"95th": []int{
+					4000,
+					750,
+					600,
+					550,
+					500,
+					1000,
+				},
+			},
 		},
-		Filter: queryFilter{"DstCountry = 'FR' AND SrcCountry = 'US'"},
+	})
-		Units:  "bps",
-	}
-	payload := new(bytes.Buffer)
-	err = json.NewEncoder(payload).Encode(input)
-	if err != nil {
-		t.Fatalf("Encode() error:\n%+v", err)
-	}
-	resp, err := netHTTP.Post(fmt.Sprintf("http://%s/api/v0/console/graph", h.Address),
-		"application/json", payload)
-	if err != nil {
-		t.Fatalf("POST /api/v0/console/graph:\n%+v", err)
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != 200 {
-		t.Errorf("POST /api/v0/console/graph: got status code %d, not 200", resp.StatusCode)
-	}
-	gotContentType := resp.Header.Get("Content-Type")
-	if gotContentType != "application/json; charset=utf-8" {
-		t.Errorf("POST /api/v0/console/graph Content-Type (-got, +want):\n-%s\n+%s",
-			gotContentType, "application/json; charset=utf-8")
-	}
-	decoder := json.NewDecoder(resp.Body)
-	var got gin.H
-	if err := decoder.Decode(&got); err != nil {
-		t.Fatalf("POST /api/v0/console/graph error:\n%+v", err)
-	}
-
-	if diff := helpers.Diff(got, expected); diff != "" {
-		t.Fatalf("POST /api/v0/console/graph (-got, +want):\n%s", diff)
-	}
 }

console/root.go

@@ -83,6 +83,7 @@ func (c *Component) Start() error {
 	c.d.HTTP.GinRouter.GET("/api/v0/console/widget/graph", c.widgetGraphHandlerFunc)
 	c.d.HTTP.GinRouter.POST("/api/v0/console/graph", c.graphHandlerFunc)
 	c.d.HTTP.GinRouter.POST("/api/v0/console/sankey", c.sankeyHandlerFunc)
+	c.d.HTTP.GinRouter.POST("/api/v0/console/filter/validate", c.filterValidateHandlerFunc)
 
 	c.t.Go(func() error {
 		ticker := time.NewTicker(10 * time.Second)

console/sankey.go

@@ -12,8 +12,8 @@ import (
 	"akvorado/common/helpers"
 )
 
-// sankeyQuery describes the input for the /sankey endpoint.
+// sankeyHandlerInput describes the input for the /sankey endpoint.
-type sankeyQuery struct {
+type sankeyHandlerInput struct {
 	Start      time.Time     `json:"start" binding:"required"`
 	End        time.Time     `json:"end" binding:"required,gtfield=Start"`
 	Dimensions []queryColumn `json:"dimensions" binding:"required,min=2"` // group by ...
@@ -22,57 +22,7 @@ type sankeyQuery struct {
 	Units      string        `json:"units" binding:"required,oneof=pps bps"`
 }
 
-// sankeyQueryToSQL converts a sankey query to an SQL request
+// sankeyHandlerOutput describes the output for the /sankey endpoint.
-func (query sankeyQuery) toSQL() (string, error) {
-	// Filter
-	where := query.Filter.filter
-	if where == "" {
-		where = "{timefilter}"
-	} else {
-		where = fmt.Sprintf("{timefilter} AND (%s)", where)
-	}
-
-	// Select
-	arrayFields := []string{}
-	dimensions := []string{}
-	for _, column := range query.Dimensions {
-		arrayFields = append(arrayFields, fmt.Sprintf(`if(%s IN (SELECT %s FROM rows), %s, 'Other')`,
-			column.String(),
-			column.String(),
-			column.toSQLSelect()))
-		dimensions = append(dimensions, column.String())
-	}
-	fields := []string{}
-	if query.Units == "pps" {
-		fields = append(fields, `SUM(Packets*SamplingRate/range) AS xps`)
-	} else {
-		fields = append(fields, `SUM(Bytes*SamplingRate*8/range) AS xps`)
-	}
-	fields = append(fields, fmt.Sprintf("[%s] AS dimensions", strings.Join(arrayFields, ",\n  ")))
-
-	// With
-	with := []string{
-		fmt.Sprintf(`(SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM {table} WHERE %s) AS range`, where),
-		fmt.Sprintf(
-			"rows AS (SELECT %s FROM {table} WHERE %s GROUP BY %s ORDER BY SUM(Bytes) DESC LIMIT %d)",
-			strings.Join(dimensions, ", "),
-			where,
-			strings.Join(dimensions, ", "),
-			query.Limit),
-	}
-
-	sqlQuery := fmt.Sprintf(`
-WITH
- %s
-SELECT
- %s
-FROM {table}
-WHERE %s
-GROUP BY dimensions
-ORDER BY xps DESC`, strings.Join(with, ",\n "), strings.Join(fields, ",\n "), where)
-	return sqlQuery, nil
-}
-
 type sankeyHandlerOutput struct {
 	// Unprocessed data for table view
 	Rows [][]string `json:"rows"`
@@ -87,29 +37,80 @@ type sankeyLink struct {
 	Xps    int    `json:"xps"`
 }
 
+// sankeyHandlerInputToSQL converts a sankey query to an SQL request
+func (input sankeyHandlerInput) toSQL() (string, error) {
+	// Filter
+	where := input.Filter.filter
+	if where == "" {
+		where = "{timefilter}"
+	} else {
+		where = fmt.Sprintf("{timefilter} AND (%s)", where)
+	}
+
+	// Select
+	arrayFields := []string{}
+	dimensions := []string{}
+	for _, column := range input.Dimensions {
+		arrayFields = append(arrayFields, fmt.Sprintf(`if(%s IN (SELECT %s FROM rows), %s, 'Other')`,
+			column.String(),
+			column.String(),
+			column.toSQLSelect()))
+		dimensions = append(dimensions, column.String())
+	}
+	fields := []string{}
+	if input.Units == "pps" {
+		fields = append(fields, `SUM(Packets*SamplingRate/range) AS xps`)
+	} else {
+		fields = append(fields, `SUM(Bytes*SamplingRate*8/range) AS xps`)
+	}
+	fields = append(fields, fmt.Sprintf("[%s] AS dimensions", strings.Join(arrayFields, ",\n  ")))
+
+	// With
+	with := []string{
+		fmt.Sprintf(`(SELECT MAX(TimeReceived) - MIN(TimeReceived) FROM {table} WHERE %s) AS range`, where),
+		fmt.Sprintf(
+			"rows AS (SELECT %s FROM {table} WHERE %s GROUP BY %s ORDER BY SUM(Bytes) DESC LIMIT %d)",
+			strings.Join(dimensions, ", "),
+			where,
+			strings.Join(dimensions, ", "),
+			input.Limit),
+	}
+
+	sqlQuery := fmt.Sprintf(`
+WITH
+ %s
+SELECT
+ %s
+FROM {table}
+WHERE %s
+GROUP BY dimensions
+ORDER BY xps DESC`, strings.Join(with, ",\n "), strings.Join(fields, ",\n "), where)
+	return sqlQuery, nil
+}
+
 func (c *Component) sankeyHandlerFunc(gc *gin.Context) {
 	ctx := c.t.Context(gc.Request.Context())
-	var query sankeyQuery
+	var input sankeyHandlerInput
-	if err := gc.ShouldBindJSON(&query); err != nil {
+	if err := gc.ShouldBindJSON(&input); err != nil {
 		gc.JSON(http.StatusBadRequest, gin.H{"message": helpers.Capitalize(err.Error())})
 		return
 	}
 
-	sqlQuery, err := query.toSQL()
+	sqlQuery, err := input.toSQL()
 	if err != nil {
 		gc.JSON(http.StatusBadRequest, gin.H{"message": helpers.Capitalize(err.Error())})
 		return
 	}
 
 	// We need to select a resolution allowing us to have a somewhat accurate timespan
-	resolution := time.Duration(int64(query.End.Sub(query.Start).Nanoseconds()) / 20)
+	resolution := time.Duration(int64(input.End.Sub(input.Start).Nanoseconds()) / 20)
 	if resolution < time.Second {
 		resolution = time.Second
 	}
 
 	// Prepare and execute query
 	sqlQuery = c.queryFlowsTable(sqlQuery,
-		query.Start, query.End, resolution)
+		input.Start, input.End, resolution)
 	gc.Header("X-SQL-Query", strings.ReplaceAll(sqlQuery, "\n", "  "))
 	results := []struct {
 		Xps        float64  `ch:"xps"`
@@ -132,7 +133,7 @@ func (c *Component) sankeyHandlerFunc(gc *gin.Context) {
 		if name != "Other" {
 			return name
 		}
-		return fmt.Sprintf("Other %s", query.Dimensions[index].String())
+		return fmt.Sprintf("Other %s", input.Dimensions[index].String())
 	}
 	addedNodes := map[string]bool{}
 	addNode := func(name string) {
@@ -154,7 +155,7 @@ func (c *Component) sankeyHandlerFunc(gc *gin.Context) {
 		output.Rows = append(output.Rows, result.Dimensions)
 		output.Xps = append(output.Xps, int(result.Xps))
 		// Consider each pair of successive dimensions
-		for i := 0; i < len(query.Dimensions)-1; i++ {
+		for i := 0; i < len(input.Dimensions)-1; i++ {
 			dimension1 := completeName(result.Dimensions[i], i)
 			dimension2 := completeName(result.Dimensions[i+1], i+1)
 			addNode(dimension1)

console/sankey_test.go

@@ -1,10 +1,6 @@
 package console
 
 import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	netHTTP "net/http"
 	"strings"
 	"testing"
 	"time"
@@ -22,12 +18,12 @@ import (
 func TestSankeyQuerySQL(t *testing.T) {
 	cases := []struct {
 		Description string
-		Input       sankeyQuery
+		Input       sankeyHandlerInput
 		Expected    string
 	}{
 		{
 			Description: "two dimensions, no filters, bps",
-			Input: sankeyQuery{
+			Input: sankeyHandlerInput{
 				Start:      time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
 				End:        time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
 				Dimensions: []queryColumn{queryColumnSrcAS, queryColumnExporterName},
@@ -49,7 +45,7 @@ GROUP BY dimensions
 ORDER BY xps DESC`,
 		}, {
 			Description: "two dimensions, no filters, pps",
-			Input: sankeyQuery{
+			Input: sankeyHandlerInput{
 				Start:      time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
 				End:        time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
 				Dimensions: []queryColumn{queryColumnSrcAS, queryColumnExporterName},
@@ -71,7 +67,7 @@ GROUP BY dimensions
 ORDER BY xps DESC`,
 		}, {
 			Description: "two dimensions, with filter",
-			Input: sankeyQuery{
+			Input: sankeyHandlerInput{
 				Start:      time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
 				End:        time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
 				Dimensions: []queryColumn{queryColumnSrcAS, queryColumnExporterName},
@@ -147,134 +143,111 @@ func TestSankeyHandler(t *testing.T) {
 		{621, []string{"Other", "Other", "Other"}},
 		{159, []string{"Other", "provider1", "router1"}},
 	}
-	expected := gin.H{
-		// Raw data
-		"rows": [][]string{
-			{"AS100", "Other", "router1"},
-			{"AS300", "provider1", "Other"},
-			{"AS300", "provider2", "router1"},
-			{"AS200", "provider1", "Other"},
-			{"AS100", "provider1", "Other"},
-			{"Other", "provider1", "Other"},
-			{"AS200", "provider3", "Other"},
-			{"AS200", "Other", "router2"},
-			{"AS100", "provider3", "Other"},
-			{"AS100", "provider3", "router2"},
-			{"Other", "Other", "router1"},
-			{"AS300", "provider3", "router2"},
-			{"AS300", "Other", "router1"},
-			{"AS100", "provider1", "router1"},
-			{"AS200", "provider2", "router2"},
-			{"AS100", "provider2", "Other"},
-			{"AS200", "Other", "router1"},
-			{"AS300", "Other", "Other"},
-			{"AS200", "provider3", "router2"},
-			{"Other", "Other", "Other"},
-			{"Other", "provider1", "router1"},
-		},
-		"xps": []int{
-			9677,
-			9472,
-			7593,
-			7234,
-			6006,
-			5988,
-			4675,
-			4348,
-			3999,
-			3978,
-			3623,
-			3080,
-			2915,
-			2623,
-			2482,
-			2234,
-			1360,
-			975,
-			717,
-			621,
-			159,
-		},
-		// For graph
-		"nodes": []string{
-			"AS100",
-			"Other InIfProvider",
-			"router1",
-			"AS300",
-			"provider1",
-			"Other ExporterName",
-			"provider2",
-			"AS200",
-			"Other SrcAS",
-			"provider3",
-			"router2",
-		},
-		"links": []gin.H{
-			{"source": "provider1", "target": "Other ExporterName", "xps": 9472 + 7234 + 6006 + 5988},
-			{"source": "Other InIfProvider", "target": "router1", "xps": 9677 + 3623 + 2915 + 1360},
-			{"source": "AS100", "target": "Other InIfProvider", "xps": 9677},
-			{"source": "AS300", "target": "provider1", "xps": 9472},
-			{"source": "provider3", "target": "Other ExporterName", "xps": 4675 + 3999},
-			{"source": "AS100", "target": "provider1", "xps": 6006 + 2623},
-			{"source": "AS100", "target": "provider3", "xps": 3999 + 3978},
-			{"source": "provider3", "target": "router2", "xps": 3978 + 3080 + 717},
-			{"source": "AS300", "target": "provider2", "xps": 7593},
-			{"source": "provider2", "target": "router1", "xps": 7593},
-			{"source": "AS200", "target": "provider1", "xps": 7234},
-			{"source": "Other SrcAS", "target": "provider1", "xps": 5988 + 159},
-			{"source": "AS200", "target": "Other InIfProvider", "xps": 4348 + 1360},
-			{"source": "AS200", "target": "provider3", "xps": 4675 + 717},
-			{"source": "Other InIfProvider", "target": "router2", "xps": 4348},
-			{"source": "Other SrcAS", "target": "Other InIfProvider", "xps": 3623 + 621},
-			{"source": "AS300", "target": "Other InIfProvider", "xps": 2915 + 975},
-			{"source": "AS300", "target": "provider3", "xps": 3080},
-			{"source": "provider1", "target": "router1", "xps": 2623 + 159},
-			{"source": "AS200", "target": "provider2", "xps": 2482},
-			{"source": "provider2", "target": "router2", "xps": 2482},
-			{"source": "AS100", "target": "provider2", "xps": 2234},
-			{"source": "provider2", "target": "Other ExporterName", "xps": 2234},
-			{"source": "Other InIfProvider", "target": "Other ExporterName", "xps": 975 + 621},
-		},
-	}
 	mockConn.EXPECT().
 		Select(gomock.Any(), gomock.Any(), gomock.Any()).
 		SetArg(1, expectedSQL).
 		Return(nil)
 
-	input := sankeyQuery{
+	helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
-		Start:      time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
+		{
-		End:        time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
+			URL: "/api/v0/console/sankey",
-		Dimensions: []queryColumn{queryColumnSrcAS, queryColumnInIfProvider, queryColumnExporterName},
+			JSONInput: gin.H{
-		Limit:      10,
+				"start":      time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
-		Filter:     queryFilter{"DstCountry = 'FR'"},
+				"end":        time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
-		Units:      "bps",
+				"dimensions": []string{"SrcAS", "InIfProvider", "ExporterName"},
-	}
+				"limit":      10,
-	payload := new(bytes.Buffer)
+				"filter":     "DstCountry = 'FR'",
-	err = json.NewEncoder(payload).Encode(input)
+				"units":      "bps",
-	if err != nil {
+			},
-		t.Fatalf("Encode() error:\n%+v", err)
+			JSONOutput: gin.H{
-	}
+				// Raw data
-	resp, err := netHTTP.Post(fmt.Sprintf("http://%s/api/v0/console/sankey", h.Address),
+				"rows": [][]string{
-		"application/json", payload)
+					{"AS100", "Other", "router1"},
-	if err != nil {
+					{"AS300", "provider1", "Other"},
-		t.Fatalf("POST /api/v0/console/sankey:\n%+v", err)
+					{"AS300", "provider2", "router1"},
-	}
+					{"AS200", "provider1", "Other"},
-	defer resp.Body.Close()
+					{"AS100", "provider1", "Other"},
-	if resp.StatusCode != 200 {
+					{"Other", "provider1", "Other"},
-		t.Errorf("POST /api/v0/console/sankey: got status code %d, not 200", resp.StatusCode)
+					{"AS200", "provider3", "Other"},
-	}
+					{"AS200", "Other", "router2"},
-	gotContentType := resp.Header.Get("Content-Type")
+					{"AS100", "provider3", "Other"},
-	if gotContentType != "application/json; charset=utf-8" {
+					{"AS100", "provider3", "router2"},
-		t.Errorf("POST /api/v0/console/sankey Content-Type (-got, +want):\n-%s\n+%s",
+					{"Other", "Other", "router1"},
-			gotContentType, "application/json; charset=utf-8")
+					{"AS300", "provider3", "router2"},
-	}
+					{"AS300", "Other", "router1"},
-	decoder := json.NewDecoder(resp.Body)
+					{"AS100", "provider1", "router1"},
-	var got gin.H
+					{"AS200", "provider2", "router2"},
-	if err := decoder.Decode(&got); err != nil {
+					{"AS100", "provider2", "Other"},
-		t.Fatalf("POST /api/v0/console/sankey error:\n%+v", err)
+					{"AS200", "Other", "router1"},
-	}
+					{"AS300", "Other", "Other"},
-
+					{"AS200", "provider3", "router2"},
-	if diff := helpers.Diff(got, expected); diff != "" {
+					{"Other", "Other", "Other"},
-		t.Fatalf("POST /api/v0/console/sankey (-got, +want):\n%s", diff)
+					{"Other", "provider1", "router1"},
-	}
+				},
+				"xps": []int{
+					9677,
+					9472,
+					7593,
+					7234,
+					6006,
+					5988,
+					4675,
+					4348,
+					3999,
+					3978,
+					3623,
+					3080,
+					2915,
+					2623,
+					2482,
+					2234,
+					1360,
+					975,
+					717,
+					621,
+					159,
+				},
+				// For graph
+				"nodes": []string{
+					"AS100",
+					"Other InIfProvider",
+					"router1",
+					"AS300",
+					"provider1",
+					"Other ExporterName",
+					"provider2",
+					"AS200",
+					"Other SrcAS",
+					"provider3",
+					"router2",
+				},
+				"links": []gin.H{
+					{"source": "provider1", "target": "Other ExporterName", "xps": 9472 + 7234 + 6006 + 5988},
+					{"source": "Other InIfProvider", "target": "router1", "xps": 9677 + 3623 + 2915 + 1360},
+					{"source": "AS100", "target": "Other InIfProvider", "xps": 9677},
+					{"source": "AS300", "target": "provider1", "xps": 9472},
+					{"source": "provider3", "target": "Other ExporterName", "xps": 4675 + 3999},
+					{"source": "AS100", "target": "provider1", "xps": 6006 + 2623},
+					{"source": "AS100", "target": "provider3", "xps": 3999 + 3978},
+					{"source": "provider3", "target": "router2", "xps": 3978 + 3080 + 717},
+					{"source": "AS300", "target": "provider2", "xps": 7593},
+					{"source": "provider2", "target": "router1", "xps": 7593},
+					{"source": "AS200", "target": "provider1", "xps": 7234},
+					{"source": "Other SrcAS", "target": "provider1", "xps": 5988 + 159},
+					{"source": "AS200", "target": "Other InIfProvider", "xps": 4348 + 1360},
+					{"source": "AS200", "target": "provider3", "xps": 4675 + 717},
+					{"source": "Other InIfProvider", "target": "router2", "xps": 4348},
+					{"source": "Other SrcAS", "target": "Other InIfProvider", "xps": 3623 + 621},
+					{"source": "AS300", "target": "Other InIfProvider", "xps": 2915 + 975},
+					{"source": "AS300", "target": "provider3", "xps": 3080},
+					{"source": "provider1", "target": "router1", "xps": 2623 + 159},
+					{"source": "AS200", "target": "provider2", "xps": 2482},
+					{"source": "provider2", "target": "router2", "xps": 2482},
+					{"source": "AS100", "target": "provider2", "xps": 2234},
+					{"source": "provider2", "target": "Other ExporterName", "xps": 2234},
+					{"source": "Other InIfProvider", "target": "Other ExporterName", "xps": 975 + 621},
+				},
+			},
+		},
+	})
 }

console/widgets_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 	"github.com/benbjohnson/clock"
+	"github.com/gin-gonic/gin"
 	"github.com/golang/mock/gomock"
 
 	"akvorado/common/clickhousedb"
@@ -90,18 +91,15 @@ func TestWidgetLastFlow(t *testing.T) {
 
 	helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
 		{
-			URL:         "/api/v0/console/widget/flow-last",
+			URL: "/api/v0/console/widget/flow-last",
-			ContentType: "application/json; charset=utf-8",
+			JSONOutput: gin.H{
-			FirstLines: []string{
+				"InIfBoundary": "external",
-				`{`,
+				"InIfName":     "Hu0/0/1/10",
-				`    "InIfBoundary": "external",`,
+				"InIfSpeed":    100000,
-				`    "InIfName": "Hu0/0/1/10",`,
+				"SamplingRate": 10000,
-				`    "InIfSpeed": 100000,`,
+				"SrcAddr":      "2001:db8::22",
-				`    "SamplingRate": 10000,`,
+				"SrcCountry":   "FR",
-				`    "SrcAddr": "2001:db8::22",`,
+				"TimeReceived": "2022-04-04T08:36:11.00000001Z",
-				`    "SrcCountry": "FR",`,
-				`    "TimeReceived": "2022-04-04T08:36:11.00000001Z"`,
-				`}`,
 			},
 		},
 	})
@@ -132,13 +130,10 @@ func TestFlowRate(t *testing.T) {
 
 	helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
 		{
-			URL:         "/api/v0/console/widget/flow-rate",
+			URL: "/api/v0/console/widget/flow-rate",
-			ContentType: "application/json; charset=utf-8",
+			JSONOutput: gin.H{
-			FirstLines: []string{
+				"period": "second",
-				`{`,
+				"rate":   100.1,
-				`    "period": "second",`,
-				`    "rate": 100.1`,
-				`}`,
 			},
 		},
 	})
@@ -173,16 +168,13 @@ func TestWidgetExporters(t *testing.T) {
 
 	helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
 		{
-			URL:         "/api/v0/console/widget/exporters",
+			URL: "/api/v0/console/widget/exporters",
-			ContentType: "application/json; charset=utf-8",
+			JSONOutput: gin.H{
-			FirstLines: []string{
+				"exporters": []string{
-				`{`,
+					"exporter1",
-				`    "exporters": [`,
+					"exporter2",
-				`        "exporter1",`,
+					"exporter3",
-				`        "exporter2",`,
+				},
-				`        "exporter3"`,
-				`    ]`,
-				`}`,
 			},
 		},
 	})
@@ -235,29 +227,33 @@ func TestWidgetTop(t *testing.T) {
 
 	helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
 		{
-			URL:         "/api/v0/console/widget/top/src-port",
+			URL: "/api/v0/console/widget/top/src-port",
-			ContentType: "application/json; charset=utf-8",
+			JSONOutput: gin.H{
-			FirstLines: []string{
+				"top": []gin.H{
-				`{"top":[{"name":"TCP/443","percent":51},{"name":"UDP/443","percent":20},{"name":"TCP/80","percent":18}]}`,
+					gin.H{"name": "TCP/443", "percent": 51},
-			},
+					gin.H{"name": "UDP/443", "percent": 20},
+					gin.H{"name": "TCP/80", "percent": 18}}},
 		}, {
-			URL:         "/api/v0/console/widget/top/protocol",
+			URL: "/api/v0/console/widget/top/protocol",
-			ContentType: "application/json; charset=utf-8",
+			JSONOutput: gin.H{
-			FirstLines: []string{
+				"top": []gin.H{
-				`{"top":[{"name":"TCP","percent":75},{"name":"UDP","percent":24},{"name":"ESP","percent":1}]}`,
+					gin.H{"name": "TCP", "percent": 75},
-			},
+					gin.H{"name": "UDP", "percent": 24},
+					gin.H{"name": "ESP", "percent": 1}}},
 		}, {
-			URL:         "/api/v0/console/widget/top/exporter",
+			URL: "/api/v0/console/widget/top/exporter",
-			ContentType: "application/json; charset=utf-8",
+			JSONOutput: gin.H{
-			FirstLines: []string{
+				"top": []gin.H{
-				`{"top":[{"name":"exporter1","percent":20},{"name":"exporter3","percent":10},{"name":"exporter5","percent":3}]}`,
+					gin.H{"name": "exporter1", "percent": 20},
-			},
+					gin.H{"name": "exporter3", "percent": 10},
+					gin.H{"name": "exporter5", "percent": 3}}},
 		}, {
-			URL:         "/api/v0/console/widget/top/src-as",
+			URL: "/api/v0/console/widget/top/src-as",
-			ContentType: "application/json; charset=utf-8",
+			JSONOutput: gin.H{
-			FirstLines: []string{
+				"top": []gin.H{
-				`{"top":[{"name":"2906: Netflix","percent":12},{"name":"36040: Youtube","percent":10},{"name":"20940: Akamai","percent":9}]}`,
+					gin.H{"name": "2906: Netflix", "percent": 12},
-			},
+					gin.H{"name": "36040: Youtube", "percent": 10},
+					gin.H{"name": "20940: Akamai", "percent": 9}}},
 		},
 	})
 }
@@ -308,10 +304,15 @@ ORDER BY Time`).
 
 	helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
 		{
-			URL:         "/api/v0/console/widget/graph?points=100",
+			URL: "/api/v0/console/widget/graph?points=100",
-			ContentType: "application/json; charset=utf-8",
+			JSONOutput: gin.H{
-			FirstLines: []string{
+				"data": []gin.H{
-				`{"data":[{"t":"2009-11-10T23:00:00Z","gbps":25.3},{"t":"2009-11-10T23:01:00Z","gbps":27.8},{"t":"2009-11-10T23:02:00Z","gbps":26.4},{"t":"2009-11-10T23:03:00Z","gbps":29.2},{"t":"2009-11-10T23:04:00Z","gbps":21.3},{"t":"2009-11-10T23:05:00Z","gbps":24.7}]}`,
+					gin.H{"t": "2009-11-10T23:00:00Z", "gbps": 25.3},
+					gin.H{"t": "2009-11-10T23:01:00Z", "gbps": 27.8},
+					gin.H{"t": "2009-11-10T23:02:00Z", "gbps": 26.4},
+					gin.H{"t": "2009-11-10T23:03:00Z", "gbps": 29.2},
+					gin.H{"t": "2009-11-10T23:04:00Z", "gbps": 21.3},
+					gin.H{"t": "2009-11-10T23:05:00Z", "gbps": 24.7}},
 			},
 		},
 	})

inlet/flow/schemas_test.go

@@ -5,6 +5,8 @@ import (
 
 	"akvorado/common/helpers"
 	"akvorado/common/reporter"
+
+	"github.com/gin-gonic/gin"
 )
 
 func TestHTTPEndpoints(t *testing.T) {
@@ -20,16 +22,13 @@ func TestHTTPEndpoints(t *testing.T) {
 				`package decoder;`,
 			},
 		}, {
-			URL:         "/api/v0/inlet/flow/schemas.json",
+			URL: "/api/v0/inlet/flow/schemas.json",
-			ContentType: "application/json; charset=utf-8",
+			JSONOutput: gin.H{
-			FirstLines: []string{
+				"current-version": 1,
-				`{`,
+				"versions": gin.H{
-				`    "current-version": 1,`,
+					"0": "/api/v0/inlet/flow/schema-0.proto",
-				`    "versions": {`,
+					"1": "/api/v0/inlet/flow/schema-1.proto",
-				`        "0": "/api/v0/inlet/flow/schema-0.proto",`,
+				},
-				`        "1": "/api/v0/inlet/flow/schema-1.proto"`,
-				`    }`,
-				`}`,
 			},
 		},
 	}

orchestrator/broker/http_test.go

@@ -6,6 +6,8 @@ import (
 	"akvorado/common/helpers"
 	"akvorado/common/http"
 	"akvorado/common/reporter"
+
+	"github.com/gin-gonic/gin"
 )
 
 func TestConfigurationEndpoint(t *testing.T) {
@@ -24,13 +26,10 @@ func TestConfigurationEndpoint(t *testing.T) {
 
 	helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
 		{
-			URL:         "/api/v0/orchestrator/broker/configuration/inlet",
+			URL: "/api/v0/orchestrator/broker/configuration/inlet",
-			ContentType: "application/json; charset=utf-8",
+			JSONOutput: gin.H{
-			FirstLines: []string{
+				"bye":   "Goodbye world!",
-				"{",
+				"hello": "Hello world!",
-				`    "bye": "Goodbye world!",`,
-				`    "hello": "Hello world!"`,
-				"}",
 			},
 		},
 	})