tests: replace godebug by go-cmp for structure diffs

go-cmp is stricter and allow to catch more problems. Moreover, the
output is a bit nicer.

common/clickhousedb/root_test.go

@@ -130,8 +130,8 @@ func TestRealClickHouse(t *testing.T) {
 		}
 
 		expected := []struct {
-			N uint64
-			M uint64
+			N uint64 `ch:"n"`
+			M uint64 `ch:"m"`
 		}{
 			{0, 1},
 			{1, 2},

common/helpers/intern/intern_test.go

@@ -7,6 +7,8 @@ import (
 	"testing"
 
 	"akvorado/common/helpers"
+
+	"github.com/google/go-cmp/cmp/cmpopts"
 )
 
 type likeInt int
@@ -72,6 +74,7 @@ func TestPutCollision(t *testing.T) {
 
 func TestTake(t *testing.T) {
 	p := NewPool[likeInt]()
+	diffOpt := cmpopts.EquateComparable(internValue[likeInt]{})
 
 	val1 := likeInt(10)
 	ref1 := p.Put(val1)
@@ -91,7 +94,7 @@ func TestTake(t *testing.T) {
 		{value: 22, refCount: 1, previous: 2, next: 4},
 		{value: 32, refCount: 1, previous: 3},
 	}
-	if diff := helpers.Diff(p.values, expectedValues, helpers.DiffUnexported); diff != "" {
+	if diff := helpers.Diff(p.values, expectedValues, diffOpt); diff != "" {
 		t.Fatalf("p.values (-got, +want):\n%s", diff)
 	}
 
@@ -104,7 +107,7 @@ func TestTake(t *testing.T) {
 		{value: 22, refCount: 0, previous: 2, next: 4}, // free
 		{value: 32, refCount: 1, previous: 2},
 	}
-	if diff := helpers.Diff(p.values, expectedValues, helpers.DiffUnexported); diff != "" {
+	if diff := helpers.Diff(p.values, expectedValues, diffOpt); diff != "" {
 		t.Fatalf("p.values (-got, +want):\n%s", diff)
 	}
 
@@ -120,7 +123,7 @@ func TestTake(t *testing.T) {
 		{value: 42, refCount: 1, previous: 4},
 		{value: 32, refCount: 1, previous: 2, next: 3},
 	}
-	if diff := helpers.Diff(p.values, expectedValues, helpers.DiffUnexported); diff != "" {
+	if diff := helpers.Diff(p.values, expectedValues, diffOpt); diff != "" {
 		t.Fatalf("p.values (-got, +want):\n%s", diff)
 	}
 
@@ -133,7 +136,7 @@ func TestTake(t *testing.T) {
 		{value: 42, refCount: 1, previous: 4},
 		{value: 32, refCount: 1, next: 3},
 	}
-	if diff := helpers.Diff(p.values, expectedValues, helpers.DiffUnexported); diff != "" {
+	if diff := helpers.Diff(p.values, expectedValues, diffOpt); diff != "" {
 		t.Fatalf("p.values (-got, +want):\n%s", diff)
 	}
 
@@ -146,7 +149,7 @@ func TestTake(t *testing.T) {
 		{value: 42, refCount: 1},
 		{value: 32, refCount: 0, next: 3}, // free
 	}
-	if diff := helpers.Diff(p.values, expectedValues, helpers.DiffUnexported); diff != "" {
+	if diff := helpers.Diff(p.values, expectedValues, diffOpt); diff != "" {
 		t.Fatalf("p.values (-got, +want):\n%s", diff)
 	}
 
@@ -159,7 +162,7 @@ func TestTake(t *testing.T) {
 		{value: 42, refCount: 0},          // free
 		{value: 32, refCount: 0, next: 3}, // free
 	}
-	if diff := helpers.Diff(p.values, expectedValues, helpers.DiffUnexported); diff != "" {
+	if diff := helpers.Diff(p.values, expectedValues, diffOpt); diff != "" {
 		t.Fatalf("p.values (-got, +want):\n%s", diff)
 	}
 

common/helpers/mapstructure_test.go

@@ -236,7 +236,7 @@ func TestParametrizedConfig(t *testing.T) {
 				Expected: OuterConfiguration{
 					AA: "a1",
 					BB: "b1",
-					Config: InnerConfigurationType1{
+					Config: &InnerConfigurationType1{
 						CC: "c1",
 						DD: "d1",
 					},
@@ -256,7 +256,7 @@ func TestParametrizedConfig(t *testing.T) {
 				Expected: OuterConfiguration{
 					AA: "a2",
 					BB: "b2",
-					Config: InnerConfigurationType2{
+					Config: &InnerConfigurationType2{
 						CC: "c2",
 						EE: "e2",
 					},

common/helpers/subnetmap_test.go

@@ -6,6 +6,7 @@ package helpers_test
 import (
 	"net/netip"
 	"slices"
+	"strings"
 	"testing"
 
 	"github.com/gin-gonic/gin"
@@ -19,6 +20,7 @@ import (
 func TestSubnetMapUnmarshalHook(t *testing.T) {
 	var nilMap map[string]string
 	cases := []struct {
+		Pos         helpers.Pos
 		Description string
 		Input       any
 		Tests       map[string]string
@@ -26,20 +28,24 @@ func TestSubnetMapUnmarshalHook(t *testing.T) {
 		YAML        any
 	}{
 		{
+			Pos:         helpers.Mark(),
 			Description: "nil",
 			Input:       nilMap,
 			Tests: map[string]string{
 				"::ffff:203.0.113.1": "",
 			},
+			YAML: map[string]string{},
 		}, {
+			Pos:         helpers.Mark(),
 			Description: "empty",
-			Input:       gin.H{},
+			Input:       map[string]string{},
 			Tests: map[string]string{
 				"::ffff:203.0.113.1": "",
 			},
 		}, {
+			Pos:         helpers.Mark(),
 			Description: "IPv4 subnet",
-			Input:       gin.H{"203.0.113.0/24": "customer1"},
+			Input:       map[string]string{"203.0.113.0/24": "customer1"},
 			Tests: map[string]string{
 				"::ffff:203.0.113.18": "customer1",
 				"::ffff:203.0.113.16": "customer1",
@@ -49,59 +55,69 @@ func TestSubnetMapUnmarshalHook(t *testing.T) {
 				"2001:db8:1::12":      "",
 			},
 		}, {
+			Pos:         helpers.Mark(),
 			Description: "IPv4 IP",
-			Input:       gin.H{"203.0.113.1": "customer1"},
+			Input:       map[string]string{"203.0.113.1": "customer1"},
 			Tests: map[string]string{
 				"::ffff:203.0.113.1": "customer1",
 				"2001:db8:1::12":     "",
 			},
-			YAML: gin.H{"203.0.113.1/32": "customer1"},
+			YAML: map[string]string{"203.0.113.1/32": "customer1"},
 		}, {
+			Pos:         helpers.Mark(),
 			Description: "IPv6 subnet",
-			Input:       gin.H{"2001:db8:1::/64": "customer2"},
+			Input:       map[string]string{"2001:db8:1::/64": "customer2"},
 			Tests: map[string]string{
 				"2001:db8:1::1": "customer2",
 				"2001:db8:1::2": "customer2",
 				"2001:db8:2::2": "",
 			},
 		}, {
+			Pos:         helpers.Mark(),
 			Description: "IPv6-mapped-IPv4 subnet",
-			Input:       gin.H{"::ffff:203.0.113.0/120": "customer2"},
+			Input:       map[string]string{"::ffff:203.0.113.0/120": "customer2"},
 			Tests: map[string]string{
 				"::ffff:203.0.113.10": "customer2",
 				"::ffff:203.0.112.10": "",
 			},
-			YAML: gin.H{"203.0.113.0/24": "customer2"},
+			YAML: map[string]string{"203.0.113.0/24": "customer2"},
 		}, {
+			Pos:         helpers.Mark(),
 			Description: "IPv6 IP",
-			Input:       gin.H{"2001:db8:1::1": "customer2"},
+			Input:       map[string]string{"2001:db8:1::1": "customer2"},
 			Tests: map[string]string{
 				"2001:db8:1::1": "customer2",
 				"2001:db8:1::2": "",
 				"2001:db8:2::2": "",
 			},
-			YAML: gin.H{"2001:db8:1::1/128": "customer2"},
+			YAML: map[string]string{"2001:db8:1::1/128": "customer2"},
 		}, {
+			Pos:         helpers.Mark(),
 			Description: "Invalid subnet (1)",
-			Input:       gin.H{"192.0.2.1/38": "customer"},
+			Input:       map[string]string{"192.0.2.1/38": "customer"},
 			Error:       true,
 		}, {
+			Pos:         helpers.Mark(),
 			Description: "Invalid subnet (2)",
-			Input:       gin.H{"192.0.2.1/255.0.255.0": "customer"},
+			Input:       map[string]string{"192.0.2.1/255.0.255.0": "customer"},
 			Error:       true,
 		}, {
+			Pos:         helpers.Mark(),
 			Description: "Invalid subnet (3)",
-			Input:       gin.H{"2001:db8::/1000": "customer"},
+			Input:       map[string]string{"2001:db8::/1000": "customer"},
 			Error:       true,
 		}, {
+			Pos:         helpers.Mark(),
 			Description: "Invalid IP",
-			Input:       gin.H{"200.33.300.1": "customer"},
+			Input:       map[string]string{"200.33.300.1": "customer"},
 			Error:       true,
 		}, {
+			Pos:         helpers.Mark(),
 			Description: "Random key",
-			Input:       gin.H{"kfgdjgkfj": "customer"},
+			Input:       map[string]string{"kfgdjgkfj": "customer"},
 			Error:       true,
 		}, {
+			Pos:         helpers.Mark(),
 			Description: "Single value",
 			Input:       "customer",
 			Tests: map[string]string{
@@ -115,11 +131,10 @@ func TestSubnetMapUnmarshalHook(t *testing.T) {
 	}
 	for _, tc := range cases {
 		if tc.YAML == nil {
-			if tc.Error {
-				tc.YAML = map[string]string{}
-			} else {
-				tc.YAML = tc.Input
-			}
+			tc.YAML = tc.Input
+		}
+		if tc.Tests == nil {
+			tc.Tests = map[string]string{}
 		}
 		t.Run(tc.Description, func(t *testing.T) {
 			var tree helpers.SubnetMap[string]
@@ -134,9 +149,11 @@ func TestSubnetMapUnmarshalHook(t *testing.T) {
 			}
 			err = decoder.Decode(tc.Input)
 			if err != nil && !tc.Error {
-				t.Fatalf("Decode() error:\n%+v", err)
+				t.Fatalf("%sDecode() error:\n%+v", tc.Pos, err)
 			} else if err == nil && tc.Error {
-				t.Fatal("Decode() did not return an error")
+				t.Fatalf("%sDecode() did not return an error", tc.Pos)
+			} else if tc.Error {
+				return
 			}
 			got := map[string]string{}
 			for k := range tc.Tests {
@@ -144,20 +161,20 @@ func TestSubnetMapUnmarshalHook(t *testing.T) {
 				got[k] = v
 			}
 			if diff := helpers.Diff(got, tc.Tests); diff != "" {
-				t.Fatalf("Decode() (-got, +want):\n%s", diff)
+				t.Fatalf("%sDecode() (-got, +want):\n%s", tc.Pos, diff)
 			}
 
 			// Try to unmarshal with YAML
 			buf, err := yaml.Marshal(tree)
 			if err != nil {
-				t.Fatalf("yaml.Marshal() error:\n%+v", err)
+				t.Fatalf("%syaml.Marshal() error:\n%+v", tc.Pos, err)
 			}
 			got = map[string]string{}
 			if err := yaml.Unmarshal(buf, &got); err != nil {
-				t.Fatalf("yaml.Unmarshal() error:\n%+v", err)
+				t.Fatalf("%syaml.Unmarshal() error:\n%+v", tc.Pos, err)
 			}
 			if diff := helpers.Diff(got, tc.YAML); diff != "" {
-				t.Fatalf("MarshalYAML() (-got, +want):\n%s", diff)
+				t.Fatalf("%sMarshalYAML() (-got, +want):\n%s", tc.Pos, diff)
 			}
 		})
 	}
@@ -171,7 +188,7 @@ func TestSubnetMapUnmarshalHookWithMapValue(t *testing.T) {
 	cases := []struct {
 		Pos      helpers.Pos
 		Input    gin.H
-		Expected gin.H
+		Expected any
 	}{
 		{
 			Pos: helpers.Mark(),
@@ -179,10 +196,10 @@ func TestSubnetMapUnmarshalHookWithMapValue(t *testing.T) {
 				"blip": "some",
 				"blop": "thing",
 			},
-			Expected: gin.H{
-				"::/0": gin.H{
-					"Blip": "some",
-					"Blop": "thing",
+			Expected: map[string]SomeStruct{
+				"::/0": {
+					Blip: "some",
+					Blop: "thing",
 				},
 			},
 		}, {
@@ -197,14 +214,14 @@ func TestSubnetMapUnmarshalHookWithMapValue(t *testing.T) {
 					"blop": "stuff",
 				},
 			},
-			Expected: gin.H{
-				"::/0": gin.H{
-					"Blip": "some",
-					"Blop": "thing",
+			Expected: map[string]SomeStruct{
+				"::/0": {
+					Blip: "some",
+					Blop: "thing",
 				},
-				"203.0.113.14/32": gin.H{
-					"Blip": "other",
-					"Blop": "stuff",
+				"203.0.113.14/32": {
+					Blip: "other",
+					Blop: "stuff",
 				},
 			},
 		},
@@ -717,3 +734,31 @@ func TestSubnetMapAllMaybeSorted(t *testing.T) {
 		}
 	})
 }
+
+func TestSubnetmapDiff(t *testing.T) {
+	got := helpers.MustNewSubnetMap(map[string]string{
+		"2001:db8::/64":        "hello",
+		"::ffff:192.0.2.0/120": "bye",
+	})
+	expected := helpers.MustNewSubnetMap(map[string]string{
+		"2001:db8::/64":        "hello",
+		"::ffff:192.0.2.0/120": "bye",
+	})
+
+	if diff := helpers.Diff(got, expected); diff != "" {
+		t.Fatalf("Diff():\n%+v", diff)
+	}
+
+	got.Set(netip.MustParsePrefix("2001:db8:1::/64"), "bye")
+	diffGot := helpers.Diff(got, expected)
+	diffGot = strings.ReplaceAll(diffGot, "\u00a0", " ")
+	diffExpected := `  (*helpers.SubnetMap[string])(Inverse(subnetmap.Transform, map[string]string{
+  	"192.0.2.0/24":    "bye",
+- 	"2001:db8:1::/64": "bye",
+  	"2001:db8::/64":   "hello",
+  }))
+`
+	if diff := helpers.Diff(diffGot, diffExpected); diff != "" {
+		t.Fatalf("Diff() (-got, +want):\n%+v", diff)
+	}
+}

common/helpers/tests.go

@@ -141,7 +141,7 @@ func Mark() Pos {
 // String returns a textual representation of a Pos.
 func (p Pos) String() string {
 	if p.file != "" {
-		return fmt.Sprintf("%s:%d", p.file, p.line)
+		return fmt.Sprintf("%s:%d: ", p.file, p.line)
 	}
 	return ""
 }

common/helpers/tests_config.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/go-viper/mapstructure/v2"
+	"github.com/google/go-cmp/cmp"
 
 	"akvorado/common/helpers/yaml"
 )
@@ -28,7 +29,7 @@ type ConfigurationDecodeCases []struct {
 }
 
 // TestConfigurationDecode helps decoding configuration. It also test decoding from YAML.
-func TestConfigurationDecode(t *testing.T, cases ConfigurationDecodeCases, options ...DiffOption) {
+func TestConfigurationDecode(t *testing.T, cases ConfigurationDecodeCases, options ...cmp.Option) {
 	t.Helper()
 	for _, tc := range cases {
 		for _, fromYAML := range []bool{false, true} {

common/helpers/tests_diff.go

@@ -6,85 +6,28 @@
 package helpers
 
 import (
-	"fmt"
-	"net"
 	"net/netip"
-	"reflect"
-	"time"
 
-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 )
 
-var prettyC = pretty.Config{
-	Diffable:          true,
-	PrintStringers:    false,
-	SkipZeroFields:    true,
-	IncludeUnexported: false,
-}
+var diffCmpOptions cmp.Options
 
-func formatByte(v any) string {
-	return fmt.Sprintf("0x%x", v)
-}
-
-func defaultPrettyFormatters() map[reflect.Type]any {
-	result := map[reflect.Type]any{
-		reflect.TypeOf(net.IP{}):            fmt.Sprint,
-		reflect.TypeOf(netip.Addr{}):        fmt.Sprint,
-		reflect.TypeOf(netip.Prefix{}):      fmt.Sprint,
-		reflect.TypeOf(time.Time{}):         fmt.Sprint,
-		reflect.TypeOf(SubnetMap[string]{}): fmt.Sprint,
-		reflect.TypeOf(SubnetMap[uint]{}):   fmt.Sprint,
-		reflect.TypeOf(SubnetMap[uint16]{}): fmt.Sprint,
-		reflect.TypeOf(byte(0)):             formatByte,
-	}
-	for t, fn := range nonDefaultPrettyFormatters {
-		result[t] = fn
-	}
-	return result
-}
-
-var nonDefaultPrettyFormatters = map[reflect.Type]any{}
-
-// AddPrettyFormatter add a global pretty formatter. We cannot put everything in
-// the default map due to cycles.
-func AddPrettyFormatter(t reflect.Type, fn any) {
-	nonDefaultPrettyFormatters[t] = fn
-}
-
-// DiffOption changes the behavior of the Diff function.
-type DiffOption struct {
-	kind int
-	// When this is a formatter
-	t  reflect.Type
-	fn any
+// RegisterCmpOption adds an option that will be used in all call to Diff().
+func RegisterCmpOption(option cmp.Option) {
+	diffCmpOptions = append(diffCmpOptions, option)
 }
 
 // Diff return a diff of two objects. If no diff, an empty string is
 // returned.
-func Diff(a, b any, options ...DiffOption) string {
-	prettyC := prettyC
-	prettyC.Formatter = defaultPrettyFormatters()
-	for _, option := range options {
-		switch option.kind {
-		case DiffUnexported.kind:
-			prettyC.IncludeUnexported = true
-		case DiffZero.kind:
-			prettyC.SkipZeroFields = false
-		case DiffFormatter(nil, nil).kind:
-			prettyC.Formatter[option.t] = option.fn
-		}
-	}
-	return prettyC.Compare(a, b)
+func Diff(a, b any, options ...cmp.Option) string {
+	options = append(options, diffCmpOptions...)
+	return cmp.Diff(a, b, options...)
 }
 
-var (
-	// DiffUnexported will display diff of unexported fields too.
-	DiffUnexported = DiffOption{kind: 1}
-	// DiffZero will include zero-field in diff
-	DiffZero = DiffOption{kind: 2}
-)
-
-// DiffFormatter adds a new formatter
-func DiffFormatter(t reflect.Type, fn any) DiffOption {
-	return DiffOption{kind: 3, t: t, fn: fn}
+func init() {
+	RegisterCmpOption(cmpopts.EquateComparable(netip.Addr{}))
+	RegisterCmpOption(cmpopts.EquateComparable(netip.Prefix{}))
+	RegisterCmpOption(cmpopts.EquateErrors())
 }

common/helpers/tests_http.go

@@ -24,12 +24,12 @@ type HTTPEndpointCases []struct {
 	Method      string
 	URL         string
 	Header      http.Header
-	JSONInput   any
+	JSONInput   gin.H
 
 	ContentType string
 	StatusCode  int
 	FirstLines  []string
-	JSONOutput  any
+	JSONOutput  gin.H
 }
 
 // TestHTTPEndpoints test a few HTTP endpoints
@@ -108,6 +108,9 @@ func TestHTTPEndpoints(t *testing.T, serverAddr net.Addr, cases HTTPEndpointCase
 				for reader.Scan() && len(got) < len(tc.FirstLines) {
 					got = append(got, reader.Text())
 				}
+				if tc.FirstLines == nil {
+					tc.FirstLines = []string{}
+				}
 				if diff := Diff(got, tc.FirstLines); diff != "" {
 					t.Errorf("%s%s %s (-got, +want):\n%s", tc.Pos, tc.Method, tc.URL, diff)
 				}
@@ -117,7 +120,18 @@ func TestHTTPEndpoints(t *testing.T, serverAddr net.Addr, cases HTTPEndpointCase
 				if err := decoder.Decode(&got); err != nil {
 					t.Fatalf("%s%s %s:\n%+v", tc.Pos, tc.Method, tc.URL, err)
 				}
-				if diff := Diff(got, tc.JSONOutput); diff != "" {
+
+				// Encode/decode expected to compare JSON stuff
+				var expected gin.H
+				expectedBytes, err := json.Marshal(tc.JSONOutput)
+				if err != nil {
+					t.Fatalf("json.Marshal() error:\n%+v", err)
+				}
+				if err := json.Unmarshal(expectedBytes, &expected); err != nil {
+					t.Fatalf("json.Unmarshal() error:\n%+v", err)
+				}
+
+				if diff := Diff(got, expected); diff != "" {
 					t.Fatalf("%s%s %s (-got, +want):\n%s", tc.Pos, tc.Method, tc.URL, diff)
 				}
 			}

common/helpers/tests_subnetmap.go

@@ -0,0 +1,23 @@
+// SPDX-FileCopyrightText: 2022 Free Mobile
+// SPDX-License-Identifier: AGPL-3.0-only
+
+//go:build !release
+
+package helpers
+
+import "github.com/google/go-cmp/cmp"
+
+// RegisterSubnetMapCmp register a subnetmap to work with cmp.Equal()/cmp.Diff()
+func RegisterSubnetMapCmp[T any]() {
+	RegisterCmpOption(
+		cmp.Transformer("subnetmap.Transform",
+			func(sm *SubnetMap[T]) map[string]T {
+				return sm.ToMap()
+			}))
+}
+
+func init() {
+	RegisterSubnetMapCmp[uint16]()
+	RegisterSubnetMapCmp[uint]()
+	RegisterSubnetMapCmp[string]()
+}

common/helpers/yaml/unmarshal_test.go

@@ -9,8 +9,6 @@ import (
 
 	"akvorado/common/helpers"
 	"akvorado/common/helpers/yaml"
-
-	"github.com/gin-gonic/gin"
 )
 
 func TestUnmarshalWithIn(t *testing.T) {
@@ -19,14 +17,14 @@ func TestUnmarshalWithIn(t *testing.T) {
 	if err := yaml.UnmarshalWithInclude(fsys, "base.yaml", &got); err != nil {
 		t.Fatalf("UnmarshalWithInclude() error:\n%+v", err)
 	}
-	expected := gin.H{
-		"file1": gin.H{"name": "1.yaml"},
-		"file2": gin.H{"name": "2.yaml"},
-		"nested": gin.H{
-			"file1": gin.H{"name": "1.yaml"},
+	expected := map[string]any{
+		"file1": map[string]any{"name": "1.yaml"},
+		"file2": map[string]any{"name": "2.yaml"},
+		"nested": map[string]any{
+			"file1": map[string]any{"name": "1.yaml"},
 		},
-		"list1": []string{"el1", "el2", "el3"},
-		"list2": []any{gin.H{
+		"list1": []any{"el1", "el2", "el3"},
+		"list2": []any{map[string]any{
 			"protocol": "tcp",
 			"size":     1300,
 		}, "el2", "el3"},

common/pb/tests.go

@@ -0,0 +1,16 @@
+// SPDX-FileCopyrightText: 2022 Free Mobile
+// SPDX-License-Identifier: AGPL-3.0-only
+
+//go:build !release
+
+package pb
+
+import (
+	"akvorado/common/helpers"
+
+	"github.com/google/go-cmp/cmp/cmpopts"
+)
+
+func init() {
+	helpers.RegisterCmpOption(cmpopts.IgnoreUnexported(RawFlow{}))
+}

common/remotedatasource/config_test.go

@@ -4,8 +4,6 @@
 package remotedatasource
 
 import (
-	"fmt"
-	"reflect"
 	"testing"
 	"time"
 
@@ -100,5 +98,5 @@ func TestSourceDecode(t *testing.T) {
 			},
 			Error: true,
 		},
-	}, helpers.DiffFormatter(reflect.TypeOf(TransformQuery{}), fmt.Sprint), helpers.DiffZero)
+	})
 }

common/remotedatasource/root_test.go

@@ -136,7 +136,7 @@ func TestSource(t *testing.T) {
 	handler := remoteDataHandler{
 		data: []remoteData{},
 	}
-	var expected []remoteData
+	expected := []remoteData{}
 	handler.fetcher, _ = New[remoteData](r, handler.UpdateData, "test", config)
 
 	handler.fetcher.Start()

common/reporter/healthcheck_test.go

@@ -139,12 +139,12 @@ func TestHealthcheckHTTPHandler(t *testing.T) {
 	}
 	expected := gin.H{
 		"status": "error",
-		"details": gin.H{
-			"hc1": map[string]string{
+		"details": map[string]any{
+			"hc1": map[string]any{
 				"status": "ok",
 				"reason": "all well",
 			},
-			"hc2": map[string]string{
+			"hc2": map[string]any{
 				"status": "error",
 				"reason": "trying to be better",
 			},

common/schema/clickhouse.go

@@ -161,19 +161,23 @@ func (bf *FlowMessage) AppendUint(columnKey ColumnKey, value uint64) {
 	switch col := col.(type) {
 	case *proto.ColUInt64:
 		col.Append(value)
+		bf.appendDebug(columnKey, value)
 	case *proto.ColUInt32:
 		col.Append(uint32(value))
+		bf.appendDebug(columnKey, uint32(value))
 	case *proto.ColUInt16:
 		col.Append(uint16(value))
+		bf.appendDebug(columnKey, uint16(value))
 	case *proto.ColUInt8:
 		col.Append(uint8(value))
+		bf.appendDebug(columnKey, uint8(value))
 	case *proto.ColEnum8:
 		col.Append(proto.Enum8(value))
+		bf.appendDebug(columnKey, uint8(value))
 	default:
 		panic(fmt.Sprintf("unhandled uint type %q", col.Type()))
 	}
 	bf.batch.columnSet.Set(uint(columnKey))
-	bf.appendDebug(columnKey, value)
 }
 
 // AppendString adds a String value to the provided column

common/schema/clickhouse_test.go

@@ -6,11 +6,13 @@ package schema
 import (
 	"net/netip"
 	"slices"
+	"strings"
 	"testing"
 
 	"akvorado/common/helpers"
 
 	"github.com/ClickHouse/ch-go/proto"
+	"github.com/google/go-cmp/cmp"
 )
 
 func TestAppendDefault(t *testing.T) {
@@ -50,10 +52,10 @@ func TestAppendBasics(t *testing.T) {
 	bf.AppendUint(ColumnPackets, 200)
 
 	expected := map[ColumnKey]any{
-		ColumnTimeReceived: 1000,
-		ColumnSamplingRate: 20000,
-		ColumnDstAS:        65000,
-		ColumnPackets:      100,
+		ColumnTimeReceived: uint32(1000),
+		ColumnSamplingRate: uint64(20000),
+		ColumnDstAS:        uint32(65000),
+		ColumnPackets:      uint64(100),
 	}
 	got := bf.OtherColumns
 	if diff := helpers.Diff(got, expected); diff != "" {
@@ -94,7 +96,7 @@ func TestAppendArrayUInt32Columns(t *testing.T) {
 		Offsets: proto.ColUInt64{3, 6},
 		Data:    &proto.ColUInt32{65400, 65500, 65001, 65403, 65503, 65003},
 	}
-	if diff := helpers.Diff(got, expected); diff != "" {
+	if diff := helpers.Diff(got, &expected); diff != "" {
 		t.Errorf("AppendArrayUInt32 (-got, +want):\n%s", diff)
 	}
 }
@@ -122,7 +124,7 @@ func TestAppendArrayUInt128Columns(t *testing.T) {
 			{High: (65401 << 32) + 100, Low: 201},
 		},
 	}
-	if diff := helpers.Diff(got, expected); diff != "" {
+	if diff := helpers.Diff(got, &expected); diff != "" {
 		t.Errorf("AppendArrayUInt128 (-got, +want):\n%s", diff)
 	}
 }
@@ -142,10 +144,10 @@ func TestUndoUInt64(t *testing.T) {
 	expectedBytes := proto.ColUInt64{100}
 	expectedPackets := proto.ColUInt64{200}
 
-	if diff := helpers.Diff(bytesCol, expectedBytes); diff != "" {
+	if diff := helpers.Diff(bytesCol, &expectedBytes); diff != "" {
 		t.Errorf("Initial bytes column state (-got, +want):\n%s", diff)
 	}
-	if diff := helpers.Diff(packetsCol, expectedPackets); diff != "" {
+	if diff := helpers.Diff(packetsCol, &expectedPackets); diff != "" {
 		t.Errorf("Initial packets column state (-got, +want):\n%s", diff)
 	}
 
@@ -155,10 +157,10 @@ func TestUndoUInt64(t *testing.T) {
 	expectedBytesAfter := proto.ColUInt64{}
 	expectedPacketsAfter := proto.ColUInt64{}
 
-	if diff := helpers.Diff(bytesCol, expectedBytesAfter); diff != "" {
+	if diff := helpers.Diff(bytesCol, &expectedBytesAfter); diff != "" {
 		t.Errorf("Bytes column after undo (-got, +want):\n%s", diff)
 	}
-	if diff := helpers.Diff(packetsCol, expectedPacketsAfter); diff != "" {
+	if diff := helpers.Diff(packetsCol, &expectedPacketsAfter); diff != "" {
 		t.Errorf("Packets column after undo (-got, +want):\n%s", diff)
 	}
 }
@@ -178,10 +180,10 @@ func TestUndoUInt32(t *testing.T) {
 	expectedSrc := proto.ColUInt32{65001}
 	expectedDst := proto.ColUInt32{65002}
 
-	if diff := helpers.Diff(srcCol, expectedSrc); diff != "" {
+	if diff := helpers.Diff(srcCol, &expectedSrc); diff != "" {
 		t.Errorf("Initial SrcAS column state (-got, +want):\n%s", diff)
 	}
-	if diff := helpers.Diff(dstCol, expectedDst); diff != "" {
+	if diff := helpers.Diff(dstCol, &expectedDst); diff != "" {
 		t.Errorf("Initial DstAS column state (-got, +want):\n%s", diff)
 	}
 
@@ -191,10 +193,10 @@ func TestUndoUInt32(t *testing.T) {
 	expectedSrcAfter := proto.ColUInt32{}
 	expectedDstAfter := proto.ColUInt32{}
 
-	if diff := helpers.Diff(srcCol, expectedSrcAfter); diff != "" {
+	if diff := helpers.Diff(srcCol, &expectedSrcAfter); diff != "" {
 		t.Errorf("SrcAS column after undo (-got, +want):\n%s", diff)
 	}
-	if diff := helpers.Diff(dstCol, expectedDstAfter); diff != "" {
+	if diff := helpers.Diff(dstCol, &expectedDstAfter); diff != "" {
 		t.Errorf("DstAS column after undo (-got, +want):\n%s", diff)
 	}
 }
@@ -214,10 +216,10 @@ func TestUndoUInt16(t *testing.T) {
 	expectedSrc := proto.ColUInt16{80}
 	expectedDst := proto.ColUInt16{443}
 
-	if diff := helpers.Diff(srcCol, expectedSrc); diff != "" {
+	if diff := helpers.Diff(srcCol, &expectedSrc); diff != "" {
 		t.Errorf("Initial SrcPort column state (-got, +want):\n%s", diff)
 	}
-	if diff := helpers.Diff(dstCol, expectedDst); diff != "" {
+	if diff := helpers.Diff(dstCol, &expectedDst); diff != "" {
 		t.Errorf("Initial DstPort column state (-got, +want):\n%s", diff)
 	}
 
@@ -227,10 +229,10 @@ func TestUndoUInt16(t *testing.T) {
 	expectedSrcAfter := proto.ColUInt16{}
 	expectedDstAfter := proto.ColUInt16{}
 
-	if diff := helpers.Diff(srcCol, expectedSrcAfter); diff != "" {
+	if diff := helpers.Diff(srcCol, &expectedSrcAfter); diff != "" {
 		t.Errorf("SrcPort column after undo (-got, +want):\n%s", diff)
 	}
-	if diff := helpers.Diff(dstCol, expectedDstAfter); diff != "" {
+	if diff := helpers.Diff(dstCol, &expectedDstAfter); diff != "" {
 		t.Errorf("DstPort column after undo (-got, +want):\n%s", diff)
 	}
 }
@@ -246,7 +248,7 @@ func TestUndoUInt8(t *testing.T) {
 	col := bf.batch.columns[ColumnSrcNetMask].(*proto.ColUInt8)
 	expected := proto.ColUInt8{6}
 
-	if diff := helpers.Diff(col, expected); diff != "" {
+	if diff := helpers.Diff(col, &expected); diff != "" {
 		t.Errorf("Initial Proto column state (-got, +want):\n%s", diff)
 	}
 
@@ -255,7 +257,7 @@ func TestUndoUInt8(t *testing.T) {
 
 	expectedAfter := proto.ColUInt8{}
 
-	if diff := helpers.Diff(col, expectedAfter); diff != "" {
+	if diff := helpers.Diff(col, &expectedAfter); diff != "" {
 		t.Errorf("Proto column after undo (-got, +want):\n%s", diff)
 	}
 }
@@ -278,10 +280,10 @@ func TestUndoIPv6(t *testing.T) {
 	expectedSrc := proto.ColIPv6{srcAddr.As16()}
 	expectedDst := proto.ColIPv6{dstAddr.As16()}
 
-	if diff := helpers.Diff(srcCol, expectedSrc); diff != "" {
+	if diff := helpers.Diff(srcCol, &expectedSrc); diff != "" {
 		t.Errorf("Initial SrcAddr column state (-got, +want):\n%s", diff)
 	}
-	if diff := helpers.Diff(dstCol, expectedDst); diff != "" {
+	if diff := helpers.Diff(dstCol, &expectedDst); diff != "" {
 		t.Errorf("Initial DstAddr column state (-got, +want):\n%s", diff)
 	}
 
@@ -291,10 +293,10 @@ func TestUndoIPv6(t *testing.T) {
 	expectedSrcAfter := proto.ColIPv6{}
 	expectedDstAfter := proto.ColIPv6{}
 
-	if diff := helpers.Diff(srcCol, expectedSrcAfter); diff != "" {
+	if diff := helpers.Diff(srcCol, &expectedSrcAfter); diff != "" {
 		t.Errorf("SrcAddr column after undo (-got, +want):\n%s", diff)
 	}
-	if diff := helpers.Diff(dstCol, expectedDstAfter); diff != "" {
+	if diff := helpers.Diff(dstCol, &expectedDstAfter); diff != "" {
 		t.Errorf("DstAddr column after undo (-got, +want):\n%s", diff)
 	}
 }
@@ -310,7 +312,7 @@ func TestUndoDateTime(t *testing.T) {
 	col := bf.batch.columns[ColumnTimeReceived].(*proto.ColDateTime)
 	expected := proto.ColDateTime{Data: []proto.DateTime{1000}}
 
-	if diff := helpers.Diff(col, expected); diff != "" {
+	if diff := helpers.Diff(col, &expected); diff != "" {
 		t.Errorf("Initial TimeReceived column state (-got, +want):\n%s", diff)
 	}
 
@@ -319,7 +321,7 @@ func TestUndoDateTime(t *testing.T) {
 
 	expectedAfter := proto.ColDateTime{Data: []proto.DateTime{}}
 
-	if diff := helpers.Diff(col, expectedAfter); diff != "" {
+	if diff := helpers.Diff(col, &expectedAfter); diff != "" {
 		t.Errorf("TimeReceived column after undo (-got, +want):\n%s", diff)
 	}
 }
@@ -335,7 +337,7 @@ func TestUndoEnum8(t *testing.T) {
 	col := bf.batch.columns[ColumnInIfBoundary].(*proto.ColEnum8)
 	expected := proto.ColEnum8{proto.Enum8(InterfaceBoundaryExternal)}
 
-	if diff := helpers.Diff(col, expected); diff != "" {
+	if diff := helpers.Diff(col, &expected); diff != "" {
 		t.Errorf("Initial InIfBoundary column state (-got, +want):\n%s", diff)
 	}
 
@@ -344,7 +346,7 @@ func TestUndoEnum8(t *testing.T) {
 
 	expectedAfter := proto.ColEnum8{}
 
-	if diff := helpers.Diff(col, expectedAfter); diff != "" {
+	if diff := helpers.Diff(col, &expectedAfter); diff != "" {
 		t.Errorf("InIfBoundary column after undo (-got, +want):\n%s", diff)
 	}
 }
@@ -364,10 +366,14 @@ func TestUndoLowCardinalityString(t *testing.T) {
 	expectedName := proto.ColLowCardinality[string]{Values: []string{"router1"}}
 	expectedRole := proto.ColLowCardinality[string]{Values: []string{"edge"}}
 
-	if diff := helpers.Diff(nameCol, expectedName); diff != "" {
+	diffOpt := cmp.Comparer(func(x, y proto.ColLowCardinality[string]) bool {
+		return slices.Compare(x.Values, y.Values) == 0
+	})
+
+	if diff := helpers.Diff(nameCol, &expectedName, diffOpt); diff != "" {
 		t.Errorf("Initial ExporterName column state (-got, +want):\n%s", diff)
 	}
-	if diff := helpers.Diff(roleCol, expectedRole); diff != "" {
+	if diff := helpers.Diff(roleCol, &expectedRole, diffOpt); diff != "" {
 		t.Errorf("Initial ExporterRole column state (-got, +want):\n%s", diff)
 	}
 
@@ -377,10 +383,10 @@ func TestUndoLowCardinalityString(t *testing.T) {
 	expectedNameAfter := proto.ColLowCardinality[string]{Values: []string{}}
 	expectedRoleAfter := proto.ColLowCardinality[string]{Values: []string{}}
 
-	if diff := helpers.Diff(nameCol, expectedNameAfter); diff != "" {
+	if diff := helpers.Diff(nameCol, &expectedNameAfter, diffOpt); diff != "" {
 		t.Errorf("ExporterName column after undo (-got, +want):\n%s", diff)
 	}
-	if diff := helpers.Diff(roleCol, expectedRoleAfter); diff != "" {
+	if diff := helpers.Diff(roleCol, &expectedRoleAfter, diffOpt); diff != "" {
 		t.Errorf("ExporterRole column after undo (-got, +want):\n%s", diff)
 	}
 }
@@ -393,11 +399,17 @@ func TestUndoLowCardinalityIPv6(t *testing.T) {
 	addr := netip.MustParseAddr("2001:db8::1")
 	bf.AppendIPv6(ColumnExporterAddress, addr)
 
+	diffOpt := cmp.Comparer(func(x, y proto.ColLowCardinality[proto.IPv6]) bool {
+		return slices.CompareFunc(x.Values, y.Values, func(a, b proto.IPv6) int {
+			return strings.Compare(a.String(), b.String())
+		}) == 0
+	})
+
 	// Check we have the expected initial state
 	col := bf.batch.columns[ColumnExporterAddress].(*proto.ColLowCardinality[proto.IPv6])
 	expected := proto.ColLowCardinality[proto.IPv6]{Values: []proto.IPv6{addr.As16()}}
 
-	if diff := helpers.Diff(col, expected); diff != "" {
+	if diff := helpers.Diff(col, &expected, diffOpt); diff != "" {
 		t.Errorf("Initial ExporterAddress column state (-got, +want):\n%s", diff)
 	}
 
@@ -406,7 +418,7 @@ func TestUndoLowCardinalityIPv6(t *testing.T) {
 
 	expectedAfter := proto.ColLowCardinality[proto.IPv6]{Values: []proto.IPv6{}}
 
-	if diff := helpers.Diff(col, expectedAfter); diff != "" {
+	if diff := helpers.Diff(col, &expectedAfter, diffOpt); diff != "" {
 		t.Errorf("ExporterAddress column after undo (-got, +want):\n%s", diff)
 	}
 }
@@ -593,11 +605,11 @@ func TestBuildProtoInput(t *testing.T) {
 
 	// Let's compare a subset
 	expected := proto.Input{
-		{Name: "TimeReceived", Data: proto.ColDateTime{Data: []proto.DateTime{1002, 1003}}},
-		{Name: "SrcAS", Data: proto.ColUInt32{65000, 65001}},
-		{Name: "DstAS", Data: proto.ColUInt32{0, 0}},
-		{Name: "Bytes", Data: proto.ColUInt64{2000, 202}},
-		{Name: "Packets", Data: proto.ColUInt64{30, 3}},
+		{Name: "TimeReceived", Data: &proto.ColDateTime{Data: []proto.DateTime{1002, 1003}}},
+		{Name: "SrcAS", Data: &proto.ColUInt32{65000, 65001}},
+		{Name: "DstAS", Data: &proto.ColUInt32{0, 0}},
+		{Name: "Bytes", Data: &proto.ColUInt64{2000, 202}},
+		{Name: "Packets", Data: &proto.ColUInt64{30, 3}},
 	}
 	got = slices.DeleteFunc(got, func(col proto.InputColumn) bool {
 		return !slices.Contains([]string{"TimeReceived", "SrcAS", "DstAS", "Packets", "Bytes"}, col.Name)

common/schema/definition_test.go

@@ -7,6 +7,9 @@ import (
 	"testing"
 
 	"akvorado/common/helpers"
+
+	"github.com/bits-and-blooms/bitset"
+	"github.com/google/go-cmp/cmp"
 )
 
 func TestFlowsClickHouse(t *testing.T) {
@@ -35,7 +38,11 @@ func TestFinalizeTwice(t *testing.T) {
 	c := NewMock(t)
 	oldSchema := c.Schema
 	newSchema := c.finalize()
-	if diff := helpers.Diff(oldSchema, newSchema, helpers.DiffUnexported); diff != "" {
+	if diff := helpers.Diff(oldSchema, newSchema,
+		cmp.AllowUnexported(Schema{}),
+		cmp.Comparer(func(x, y bitset.BitSet) bool {
+			return x.Equal(&y)
+		})); diff != "" {
 		t.Fatalf("finalize() (-old, +new):\n%s", diff)
 	}
 }

common/schema/insert_test.go

@@ -160,29 +160,29 @@ func TestInsertMemory(t *testing.T) {
 				"SamplingRate":    "20000",
 				"ExporterAddress": "::ffff:203.0.113.14",
 				"ExporterName":    "router1.example.net",
-				"SrcAS":           65000,
-				"DstAS":           12322,
+				"SrcAS":           float64(65000),
+				"DstAS":           float64(12322),
 				"Bytes":           "20",
 				"Packets":         "3",
 				"InIfBoundary":    "internal",
 				"OutIfBoundary":   "external",
-				"InIfSpeed":       10000,
-				"EType":           helpers.ETypeIPv4,
+				"InIfSpeed":       float64(10000),
+				"EType":           float64(helpers.ETypeIPv4),
 			}, {
 				"TimeReceived":    "1970-01-01 00:16:41",
 				"SamplingRate":    "20000",
 				"ExporterAddress": "::ffff:203.0.113.14",
 				"ExporterName":    "router1.example.net",
-				"SrcAS":           12322,
-				"DstAS":           65000,
+				"SrcAS":           float64(12322),
+				"DstAS":           float64(65000),
 				"Bytes":           "200",
 				"Packets":         "3",
 				"InIfBoundary":    "external",
 				"OutIfBoundary":   "undefined",
-				"OutIfSpeed":      10000,
-				"EType":           helpers.ETypeIPv4,
-				"DstASPath":       []uint32{65400, 65500, 65001},
-				"DstLargeCommunities": []string{
+				"OutIfSpeed":      float64(10000),
+				"EType":           float64(helpers.ETypeIPv4),
+				"DstASPath":       []any{float64(65400), float64(65500), float64(65001)},
+				"DstLargeCommunities": []any{
 					"1206435509165107881967816", // 65401:100:200
 					"1206435509165107881967817", // 65401:100:201
 				},

common/schema/tests.go

@@ -6,11 +6,11 @@
 package schema
 
 import (
-	"fmt"
-	"reflect"
 	"testing"
 
 	"akvorado/common/helpers"
+
+	"github.com/google/go-cmp/cmp/cmpopts"
 )
 
 var debug = true
@@ -44,5 +44,5 @@ func (c *Component) EnableAllColumns() *Component {
 }
 
 func init() {
-	helpers.AddPrettyFormatter(reflect.TypeOf(ColumnBytes), fmt.Sprint)
+	helpers.RegisterCmpOption(cmpopts.IgnoreUnexported(FlowMessage{}))
 }