common/helpers: convert SubnetMap to github.com/gaissmai/bart

I did not benchmark it myself, but it was benchmarked here:
 https://github.com/osrg/gobgp/issues/1414#issuecomment-3067255941

Of course, no guarantee that this benchmark matches our use cases.
Moreover, SubnetMap have been optimized to avoid parsing keys all
the time.

Also, the interface is a bit nicer and it uses netip.Prefix directly.

The next step is to convert outlet/routing/provider/bmp.

common/helpers/subnetmap.go

@@ -5,34 +5,30 @@ package helpers
 
 import (
 	"fmt"
-	"net"
+	"iter"
 	"net/netip"
 	"reflect"
 	"regexp"
 	"strings"
 
+	"github.com/gaissmai/bart"
 	"github.com/gin-gonic/gin"
 	"github.com/go-viper/mapstructure/v2"
-	"github.com/kentik/patricia"
-	tree "github.com/kentik/patricia/generics_tree"
 )
 
 // SubnetMap maps subnets to values and allow to lookup by IP address.
-// Internally, everything is stored as an IPv6 (using v6-mapped IPv4
-// addresses).
 type SubnetMap[V any] struct {
-	tree *tree.TreeV6[V]
+	table *bart.Table[V]
 }
 
 // Lookup will search for the most specific subnet matching the
 // provided IP address and return the value associated with it.
 func (sm *SubnetMap[V]) Lookup(ip netip.Addr) (V, bool) {
-	if sm == nil || sm.tree == nil {
+	if sm == nil || sm.table == nil {
 		var value V
 		return value, false
 	}
-	ok, value := sm.tree.FindDeepestTag(patricia.NewIPv6Address(ip.AsSlice(), 128))
+	return sm.table.Lookup(ip)
-	return value, ok
 }
 
 // LookupOrDefault calls lookup and if not found, will return the
@@ -44,82 +40,98 @@ func (sm *SubnetMap[V]) LookupOrDefault(ip netip.Addr, fallback V) V {
 	return fallback
 }
 
-// ToMap return a map of the tree.
+// ToMap return a map of the tree. This should be used only when handling user
+// configuration or for debugging. Otherwise, it is better to use Iter().
 func (sm *SubnetMap[V]) ToMap() map[string]V {
 	output := map[string]V{}
-	if sm == nil || sm.tree == nil {
+	for prefix, value := range sm.All() {
-		return output
+		if prefix.Addr().Is4In6() {
-	}
+			ipv4Addr := prefix.Addr().Unmap()
-	iter := sm.tree.Iterate()
+			ipv4Prefix := netip.PrefixFrom(ipv4Addr, prefix.Bits()-96)
-	for iter.Next() {
+			output[ipv4Prefix.String()] = value
-		output[iter.Address().String()] = iter.Tags()[0]
+			continue
+		}
+		output[prefix.String()] = value
 	}
 	return output
 }
 
-// Set inserts the given key k into the SubnetMap, replacing any existing value if it exists.
+// Set inserts the given key k into the SubnetMap, replacing any existing value
-func (sm *SubnetMap[V]) Set(k string, v V) error {
+// if it exists. It requires an IPv6 prefix or it will panic.
-	subnetK, err := SubnetMapParseKey(k)
+func (sm *SubnetMap[V]) Set(prefix netip.Prefix, v V) {
-	if err != nil {
+	if !prefix.Addr().Is6() {
-		return err
+		panic(fmt.Errorf("%q is not an IPv6 subnet", prefix))
 	}
-	_, ipNet, err := net.ParseCIDR(subnetK)
+	if sm.table == nil {
-	if err != nil {
+		sm.table = &bart.Table[V]{}
-		// Should not happen
-		return err
 	}
-	_, bits := ipNet.Mask.Size()
+	sm.table.Insert(prefix, v)
-	if bits != 128 {
-		return fmt.Errorf("%q is not an IPv6 subnet", ipNet)
-	}
-	plen, _ := ipNet.Mask.Size()
-	sm.tree.Set(patricia.NewIPv6Address(ipNet.IP.To16(), uint(plen)), v)
-	return nil
 }
 
-// Update inserts the given key k into the SubnetMap, calling updateFunc with the existing value.
+// Update inserts the given key k into the SubnetMap, calling cb with the
-func (sm *SubnetMap[V]) Update(k string, v V, updateFunc tree.UpdatesFunc[V]) error {
+// existing value. It requires an IPv6 prefix or it will panic.
-	subnetK, err := SubnetMapParseKey(k)
+func (sm *SubnetMap[V]) Update(prefix netip.Prefix, cb func(V, bool) V) {
-	if err != nil {
+	if !prefix.Addr().Is6() {
-		return err
+		panic(fmt.Errorf("%q is not an IPv6 subnet", prefix))
 	}
-	_, ipNet, err := net.ParseCIDR(subnetK)
+	if sm.table == nil {
-	if err != nil {
+		sm.table = &bart.Table[V]{}
-		// Should not happen
-		return err
 	}
-	_, bits := ipNet.Mask.Size()
+	sm.table.Update(prefix, cb)
-	if bits != 128 {
-		return fmt.Errorf("%q is not an IPv6 subnet", ipNet)
-	}
-	plen, _ := ipNet.Mask.Size()
-	sm.tree.SetOrUpdate(patricia.NewIPv6Address(ipNet.IP.To16(), uint(plen)), v, updateFunc)
-	return nil
 }
 
-// Iter enables iteration of the SubnetMap, calling f for every entry. If f returns an error, the iteration is aborted.
+// All walks the whole subnet map.
-func (sm *SubnetMap[V]) Iter(f func(address patricia.IPv6Address, tags [][]V) error) error {
+func (sm *SubnetMap[V]) All() iter.Seq2[netip.Prefix, V] {
-	iter := sm.tree.Iterate()
+	return func(yield func(netip.Prefix, V) bool) {
-	for iter.Next() {
+		if sm == nil || sm.table == nil {
-		if err := f(iter.Address(), iter.TagsFromRoot()); err != nil {
+			return
-			return err
+		}
+		sm.table.All6()(yield)
+	}
+}
+
+// AllMaybeSorted walks the whole subnet map in sorted order during tests but
+// not when running tests.
+func (sm *SubnetMap[V]) AllMaybeSorted() iter.Seq2[netip.Prefix, V] {
+	return func(yield func(netip.Prefix, V) bool) {
+		if sm == nil || sm.table == nil {
+			return
+		}
+		if Testing() {
+			sm.table.AllSorted6()(yield)
+		} else {
+			sm.table.All6()(yield)
 		}
 	}
-	return nil
 }
 
-// NewSubnetMap creates a subnetmap from a map. Unlike user-provided
+// Supernets returns an iterator over all supernet routes that cover the given
-// configuration, this function is stricter and require everything to
+// prefix. The iteration order is reverse-CIDR: from longest prefix match (LPM)
-// be IPv6 subnets.
+// towards least-specific routes.
+func (sm *SubnetMap[V]) Supernets(prefix netip.Prefix) iter.Seq2[netip.Prefix, V] {
+	return func(yield func(netip.Prefix, V) bool) {
+		if sm == nil || sm.table == nil {
+			return
+		}
+		sm.table.Supernets(prefix)(yield)
+	}
+}
+
+// NewSubnetMap creates a subnetmap from a map. It should not be used in a hot
+// path as it builds the subnet from a map keyed by strings.
 func NewSubnetMap[V any](from map[string]V) (*SubnetMap[V], error) {
-	trie := &SubnetMap[V]{tree.NewTreeV6[V]()}
+	sm := &SubnetMap[V]{table: &bart.Table[V]{}}
 	if from == nil {
-		return trie, nil
+		return sm, nil
 	}
 	for k, v := range from {
-		trie.Set(k, v)
+		key, err := SubnetMapParseKey(k)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse key %s: %w", k, err)
+		}
+		sm.Set(key, v)
 	}
-	return trie, nil
+	return sm, nil
 }
 
 // MustNewSubnetMap creates a subnet from a map and panic in case of a
@@ -157,9 +169,10 @@ func LooksLikeSubnetMap(v reflect.Value) (result bool) {
 	return
 }
 
-// SubnetMapUnmarshallerHook decodes SubnetMap and notably check that
+// SubnetMapUnmarshallerHook decodes SubnetMap and notably check that valid
-// valid networks are provided as key. It also accepts a single value
+// networks are provided as key. It also accepts a single value instead of a map
-// instead of a map for backward compatibility.
+// for backward compatibility. It should not be used in hot paths as it builds
+// an intermediate map.
 func SubnetMapUnmarshallerHook[V any]() mapstructure.DecodeHookFunc {
 	return func(from, to reflect.Value) (any, error) {
 		if to.Type() != reflect.TypeOf(SubnetMap[V]{}) {
@@ -184,7 +197,7 @@ func SubnetMapUnmarshallerHook[V any]() mapstructure.DecodeHookFunc {
 				if err != nil {
 					return nil, fmt.Errorf("failed to parse key %s: %w", key, err)
 				}
-				output[key] = v.Interface()
+				output[key.String()] = v.Interface()
 			}
 		} else {
 			// Second case, we have a single value and we let mapstructure handles it
@@ -211,40 +224,36 @@ func SubnetMapUnmarshallerHook[V any]() mapstructure.DecodeHookFunc {
 	}
 }
 
-// SubnetMapParseKey decodes and validates a key used in SubnetMap from a network string.
+// PrefixTo16 converts an IPv4 prefix to an IPv4-mapped IPv6 prefix.
-func SubnetMapParseKey(k string) (string, error) {
+// IPv6 prefixes are returned as-is.
-	var key string
+func PrefixTo16(prefix netip.Prefix) netip.Prefix {
-	if strings.Contains(k, "/") {
+	if prefix.Addr().Is6() {
-		// Subnet
+		return prefix
-		_, ipNet, err := net.ParseCIDR(k)
-		if err != nil {
-			return "", err
-		}
-		// Convert key to IPv6
-		ones, bits := ipNet.Mask.Size()
-		if bits != 32 && bits != 128 {
-			return "", fmt.Errorf("key %s has invalid netmask", k)
-		}
-		if bits == 32 {
-			key = fmt.Sprintf("::ffff:%s/%d", ipNet.IP.String(), ones+96)
-		} else if ipNet.IP.To4() != nil {
-			key = fmt.Sprintf("::ffff:%s/%d", ipNet.IP.String(), ones)
-		} else {
-			key = ipNet.String()
-		}
-	} else {
-		// IP
-		ip := net.ParseIP(k)
-		if ip == nil {
-			return "", fmt.Errorf("key %s is not a valid subnet", k)
-		}
-		if ipv4 := ip.To4(); ipv4 != nil {
-			key = fmt.Sprintf("::ffff:%s/128", ipv4.String())
-		} else {
-			key = fmt.Sprintf("%s/128", ip.String())
-		}
 	}
-	return key, nil
+	// Convert IPv4 to IPv4-mapped IPv6
+	return netip.PrefixFrom(netip.AddrFrom16(prefix.Addr().As16()), prefix.Bits()+96)
+}
+
+// SubnetMapParseKey parses a prefix or an IP address into a netip.Prefix that
+// can be used in a map.
+func SubnetMapParseKey(k string) (netip.Prefix, error) {
+	// Subnet
+	if strings.Contains(k, "/") {
+		key, err := netip.ParsePrefix(k)
+		if err != nil {
+			return netip.Prefix{}, err
+		}
+		return PrefixTo16(key), nil
+	}
+	// IP address
+	key, err := netip.ParseAddr(k)
+	if err != nil {
+		return netip.Prefix{}, err
+	}
+	if key.Is4() {
+		return PrefixTo16(netip.PrefixFrom(key, 32)), nil
+	}
+	return netip.PrefixFrom(key, 128), nil
 }
 
 // MarshalYAML turns a subnet into a map that can be marshaled.

common/helpers/subnetmap_test.go

@@ -5,6 +5,7 @@ package helpers_test
 
 import (
 	"net/netip"
+	"slices"
 	"testing"
 
 	"github.com/gin-gonic/gin"
@@ -282,7 +283,7 @@ func TestSubnetMapParseKey(t *testing.T) {
 			} else if err == nil && tc.Error {
 				t.Fatal("SubnetMapParseKey() did not return an error")
 			}
-			if diff := helpers.Diff(res, tc.Expected); diff != "" {
+			if diff := helpers.Diff(res.String(), tc.Expected); err == nil && diff != "" {
 				t.Fatalf("Decode() (-got, +want):\n%s", diff)
 			}
 		})
@@ -303,3 +304,416 @@ func TestToMap(t *testing.T) {
 		t.Fatalf("ToMap() (-got, +want):\n%s", diff)
 	}
 }
+
+func TestSubnetMapLookup(t *testing.T) {
+	t.Run("nil", func(t *testing.T) {
+		var sm *helpers.SubnetMap[string]
+		value, ok := sm.Lookup(netip.MustParseAddr("::ffff:192.0.2.1"))
+		if ok || value != "" {
+			t.Fatalf("Lookup() == value=%q, ok=%v", value, ok)
+		}
+	})
+
+	t.Run("empty", func(t *testing.T) {
+		sm := &helpers.SubnetMap[string]{}
+		value, ok := sm.Lookup(netip.MustParseAddr("::ffff:192.0.2.1"))
+		if ok || value != "" {
+			t.Fatalf("Lookup() == value=%q, ok=%v", value, ok)
+		}
+	})
+
+	t.Run("populated", func(t *testing.T) {
+		sm := helpers.MustNewSubnetMap(map[string]string{
+			"192.0.2.0/24":  "customer1",
+			"2001:db8::/64": "customer2",
+			"10.0.0.1":      "specific",
+		})
+
+		cases := []struct {
+			ip       string
+			expected string
+			found    bool
+		}{
+			{"::ffff:192.0.2.1", "customer1", true},
+			{"::ffff:192.0.2.255", "customer1", true},
+			{"::ffff:192.0.3.1", "", false},
+			{"::ffff:10.0.0.1", "specific", true},
+			{"::ffff:10.0.0.2", "", false},
+			{"2001:db8::1", "customer2", true},
+			{"2001:db8:1::1", "", false},
+		}
+
+		for _, tc := range cases {
+			t.Run(tc.ip, func(t *testing.T) {
+				value, ok := sm.Lookup(netip.MustParseAddr(tc.ip))
+				if ok != tc.found || value != tc.expected {
+					t.Fatalf("Lookup(%s) = (%q, %v), want (%q, %v)", tc.ip, value, ok, tc.expected, tc.found)
+				}
+			})
+		}
+	})
+}
+
+func TestSubnetMapLookupOrDefault(t *testing.T) {
+	sm := helpers.MustNewSubnetMap(map[string]string{
+		"192.0.2.0/24": "customer1",
+	})
+
+	t.Run("found", func(t *testing.T) {
+		value := sm.LookupOrDefault(netip.MustParseAddr("::ffff:192.0.2.1"), "default")
+		if value != "customer1" {
+			t.Fatalf("LookupOrDefault() = %q, want %q", value, "customer1")
+		}
+	})
+
+	t.Run("not found", func(t *testing.T) {
+		value := sm.LookupOrDefault(netip.MustParseAddr("::ffff:192.0.3.1"), "default")
+		if value != "default" {
+			t.Fatalf("LookupOrDefault() = %q, want %q", value, "default")
+		}
+	})
+
+	t.Run("nil", func(t *testing.T) {
+		var sm *helpers.SubnetMap[string]
+		value := sm.LookupOrDefault(netip.MustParseAddr("::ffff:192.0.2.1"), "default")
+		if value != "default" {
+			t.Fatalf("LookupOrDefault() = %q, want %q", value, "default")
+		}
+	})
+}
+
+func TestSubnetMapSet(t *testing.T) {
+	sm := &helpers.SubnetMap[string]{}
+
+	t.Run("set IPv6 subnet", func(t *testing.T) {
+		prefix := netip.MustParsePrefix("2001:db8::/64")
+		sm.Set(prefix, "test-value")
+
+		value, ok := sm.Lookup(netip.MustParseAddr("2001:db8::1"))
+		if !ok || value != "test-value" {
+			t.Fatalf("Lookup() = (%q, %v), want (%q, %v)", value, ok, "test-value", true)
+		}
+	})
+
+	t.Run("set IPv4-mapped IPv6 subnet", func(t *testing.T) {
+		prefix := netip.MustParsePrefix("::ffff:192.0.2.0/120")
+		sm.Set(prefix, "ipv4-mapped")
+
+		value, ok := sm.Lookup(netip.MustParseAddr("::ffff:192.0.2.1"))
+		if !ok || value != "ipv4-mapped" {
+			t.Fatalf("Lookup() = (%q, %v), want (%q, %v)", value, ok, "ipv4-mapped", true)
+		}
+	})
+
+	t.Run("overwrite existing value", func(t *testing.T) {
+		prefix := netip.MustParsePrefix("2001:db8::/64")
+		sm.Set(prefix, "new-value")
+
+		value, ok := sm.Lookup(netip.MustParseAddr("2001:db8::1"))
+		if !ok || value != "new-value" {
+			t.Fatalf("Lookup() = (%q, %v), want (%q, %v)", value, ok, "new-value", true)
+		}
+	})
+}
+
+func TestSubnetMapSetPanic(t *testing.T) {
+	sm := &helpers.SubnetMap[string]{}
+
+	t.Run("panic on IPv4 subnet", func(t *testing.T) {
+		defer func() {
+			if r := recover(); r == nil {
+				t.Fatal("Set() should panic with IPv4")
+			}
+		}()
+		prefix := netip.MustParsePrefix("192.0.2.0/24")
+		sm.Set(prefix, "should-panic")
+	})
+}
+
+func TestSubnetMapUpdate(t *testing.T) {
+	sm := &helpers.SubnetMap[int]{}
+
+	t.Run("update new value", func(t *testing.T) {
+		prefix := netip.MustParsePrefix("2001:db8::/64")
+		sm.Update(prefix, func(old int, exists bool) int {
+			if !exists {
+				return 42
+			}
+			return old + 1
+		})
+
+		value, ok := sm.Lookup(netip.MustParseAddr("2001:db8::1"))
+		if !ok || value != 42 {
+			t.Fatalf("Lookup() = (%d, %v), want (%d, %v)", value, ok, 42, true)
+		}
+	})
+
+	t.Run("update existing value", func(t *testing.T) {
+		prefix := netip.MustParsePrefix("2001:db8::/64")
+		sm.Update(prefix, func(old int, exists bool) int {
+			if exists {
+				return old + 10
+			}
+			return 0
+		})
+
+		value, ok := sm.Lookup(netip.MustParseAddr("2001:db8::1"))
+		if !ok || value != 52 {
+			t.Fatalf("Lookup() = (%d, %v), want (%d, %v)", value, ok, 52, true)
+		}
+	})
+}
+
+func TestSubnetMapUpdatePanic(t *testing.T) {
+	sm := &helpers.SubnetMap[int]{}
+
+	t.Run("panic on IPv4 subnet", func(t *testing.T) {
+		defer func() {
+			if r := recover(); r == nil {
+				t.Fatal("Update() should panic with IPv4")
+			}
+		}()
+		prefix := netip.MustParsePrefix("192.0.2.0/24")
+		sm.Update(prefix, func(old int, exists bool) int { return 1 })
+	})
+}
+
+func TestSubnetMapAll(t *testing.T) {
+	t.Run("nil", func(t *testing.T) {
+		var sm *helpers.SubnetMap[string]
+		count := 0
+		for range sm.All() {
+			count++
+		}
+		if count != 0 {
+			t.Fatalf("All() count = %d, want %d", count, 0)
+		}
+	})
+
+	t.Run("empty", func(t *testing.T) {
+		sm := &helpers.SubnetMap[string]{}
+		count := 0
+		for range sm.All() {
+			count++
+		}
+		if count != 0 {
+			t.Fatalf("All() count = %d, want %d", count, 0)
+		}
+	})
+
+	t.Run("populated", func(t *testing.T) {
+		sm := helpers.MustNewSubnetMap(map[string]string{
+			"2001:db8::/64":        "ipv6",
+			"::ffff:192.0.2.0/120": "ipv4-mapped",
+		})
+
+		items := make(map[string]string)
+		for prefix, value := range sm.All() {
+			items[prefix.String()] = value
+		}
+
+		expected := map[string]string{
+			"2001:db8::/64":        "ipv6",
+			"::ffff:192.0.2.0/120": "ipv4-mapped",
+		}
+
+		if diff := helpers.Diff(items, expected); diff != "" {
+			t.Fatalf("All() (-got, +want):\n%s", diff)
+		}
+	})
+}
+
+func TestSubnetMapString(t *testing.T) {
+	t.Run("empty", func(t *testing.T) {
+		sm := &helpers.SubnetMap[string]{}
+		str := sm.String()
+		expected := "map[]"
+		if str != expected {
+			t.Fatalf("String() = %q, want %q", str, expected)
+		}
+	})
+
+	t.Run("populated", func(t *testing.T) {
+		sm := helpers.MustNewSubnetMap(map[string]string{
+			"192.0.2.0/24": "customer",
+		})
+		str := sm.String()
+		if diff := helpers.Diff(str, "map[192.0.2.0/24:customer]"); diff != "" {
+			t.Fatalf("String() (-got, +want):\n%s", diff)
+		}
+	})
+}
+
+func TestPrefixTo16(t *testing.T) {
+	cases := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name:     "IPv4 prefix",
+			input:    "192.0.2.0/24",
+			expected: "::ffff:192.0.2.0/120",
+		},
+		{
+			name:     "IPv4 host",
+			input:    "192.0.2.1/32",
+			expected: "::ffff:192.0.2.1/128",
+		},
+		{
+			name:     "IPv6 prefix unchanged",
+			input:    "2001:db8::/64",
+			expected: "2001:db8::/64",
+		},
+		{
+			name:     "IPv6 host unchanged",
+			input:    "2001:db8::1/128",
+			expected: "2001:db8::1/128",
+		},
+		{
+			name:     "IPv4-mapped IPv6 unchanged",
+			input:    "::ffff:192.0.2.0/120",
+			expected: "::ffff:192.0.2.0/120",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			prefix := netip.MustParsePrefix(tc.input)
+			result := helpers.PrefixTo16(prefix)
+			if result.String() != tc.expected {
+				t.Fatalf("PrefixTo16(%s) = %s, want %s", tc.input, result, tc.expected)
+			}
+		})
+	}
+}
+
+func TestSubnetMapSupernets(t *testing.T) {
+	t.Run("nil", func(t *testing.T) {
+		var sm *helpers.SubnetMap[string]
+		count := 0
+		for range sm.Supernets(netip.MustParsePrefix("::ffff:192.0.2.1/128")) {
+			count++
+		}
+		if count != 0 {
+			t.Fatalf("Supernets() count = %d, want %d", count, 0)
+		}
+	})
+
+	t.Run("empty", func(t *testing.T) {
+		sm := &helpers.SubnetMap[string]{}
+		count := 0
+		for range sm.Supernets(netip.MustParsePrefix("::ffff:192.0.2.1/128")) {
+			count++
+		}
+		if count != 0 {
+			t.Fatalf("Supernets() count = %d, want %d", count, 0)
+		}
+	})
+
+	t.Run("hierarchical supernets", func(t *testing.T) {
+		sm := helpers.MustNewSubnetMap(map[string]string{
+			"192.0.0.0/16": "region",
+			"192.0.2.0/24": "site",
+			"192.0.2.0/28": "rack",
+		})
+
+		// Query for 192.0.2.1/32 should find supernets in reverse-CIDR order
+		var results []string
+		var prefixes []string
+		for prefix, value := range sm.Supernets(netip.MustParsePrefix("::ffff:192.0.2.1/128")) {
+			results = append(results, value)
+			prefixes = append(prefixes, prefix.String())
+		}
+
+		expectedValues := []string{"rack", "site", "region"}
+		expectedPrefixes := []string{"::ffff:192.0.2.0/124", "::ffff:192.0.2.0/120", "::ffff:192.0.0.0/112"}
+
+		if diff := helpers.Diff(results, expectedValues); diff != "" {
+			t.Errorf("Supernets() values (-got, +want):\n%s", diff)
+		}
+		if diff := helpers.Diff(prefixes, expectedPrefixes); diff != "" {
+			t.Errorf("Supernets() prefixes (-got, +want):\n%s", diff)
+		}
+
+		count := 0
+		for range sm.Supernets(netip.MustParsePrefix("::ffff:10.0.0.1/128")) {
+			count++
+		}
+		if count != 0 {
+			t.Fatalf("Supernets() count = %d, want %d", count, 0)
+		}
+	})
+}
+
+func TestSubnetMapAllMaybeSorted(t *testing.T) {
+	t.Run("nil", func(t *testing.T) {
+		var sm *helpers.SubnetMap[string]
+		count := 0
+		for range sm.AllMaybeSorted() {
+			count++
+		}
+		if count != 0 {
+			t.Fatalf("AllMaybeSorted() count = %d, want %d", count, 0)
+		}
+	})
+
+	t.Run("empty", func(t *testing.T) {
+		sm := &helpers.SubnetMap[string]{}
+		count := 0
+		for range sm.AllMaybeSorted() {
+			count++
+		}
+		if count != 0 {
+			t.Fatalf("AllMaybeSorted() count = %d, want %d", count, 0)
+		}
+	})
+
+	t.Run("sorted vs unsorted", func(t *testing.T) {
+		sm := helpers.MustNewSubnetMap(map[string]string{
+			"192.0.2.0/28":    "rack",
+			"192.0.0.0/16":    "region",
+			"192.0.2.0/24":    "site",
+			"2001:db8:1::/64": "ipv6-site1",
+			"2001:db8::/32":   "ipv6-region",
+			"2001:db8:2::/64": "ipv6-site2",
+		})
+
+		// Collect results from All() (potentially unsorted)
+		var allPrefixes []string
+		for prefix := range sm.All() {
+			allPrefixes = append(allPrefixes, prefix.String())
+		}
+
+		// Collect results from AllMaybeSorted() (sorted during tests)
+		var sortedPrefixes []string
+		for prefix := range sm.AllMaybeSorted() {
+			sortedPrefixes = append(sortedPrefixes, prefix.String())
+		}
+
+		// Expected sorted order: IPv6 addresses first (sorted), then IPv4-mapped IPv6 (sorted)
+		expectedPrefixes := []string{
+			"::ffff:192.0.0.0/112",
+			"::ffff:192.0.2.0/120",
+			"::ffff:192.0.2.0/124",
+			"2001:db8::/32",
+			"2001:db8:1::/64",
+			"2001:db8:2::/64",
+		}
+
+		// AllMaybeSorted() should be sorted during tests
+		if diff := helpers.Diff(sortedPrefixes, expectedPrefixes); diff != "" {
+			t.Errorf("AllMaybeSorted() prefixes (-got, +want):\n%s", diff)
+		}
+
+		// All() and AllMaybeSorted() should contain the same elements (but potentially different order)
+		if len(allPrefixes) != len(sortedPrefixes) {
+			t.Errorf("All() returned %d prefixes, AllMaybeSorted() returned %d", len(allPrefixes), len(sortedPrefixes))
+		}
+
+		// Verify that All() and AllMaybeSorted() return different orders (otherwise test is meaningless)
+		if slices.Equal(allPrefixes, sortedPrefixes) {
+			t.Skip("All() and AllMaybeSorted() returned identical order")
+		}
+	})
+}

common/helpers/tests_diff.go

@@ -30,6 +30,7 @@ func defaultPrettyFormatters() map[reflect.Type]any {
 	result := map[reflect.Type]any{
 		reflect.TypeOf(net.IP{}):            fmt.Sprint,
 		reflect.TypeOf(netip.Addr{}):        fmt.Sprint,
+		reflect.TypeOf(netip.Prefix{}):      fmt.Sprint,
 		reflect.TypeOf(time.Time{}):         fmt.Sprint,
 		reflect.TypeOf(SubnetMap[string]{}): fmt.Sprint,
 		reflect.TypeOf(SubnetMap[uint]{}):   fmt.Sprint,

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/eapache/go-resiliency v1.7.0
 	github.com/expr-lang/expr v1.17.5
 	github.com/fsnotify/fsnotify v1.9.0
+	github.com/gaissmai/bart v0.23.0
 	github.com/gin-gonic/gin v1.10.1
 	github.com/glebarez/sqlite v1.11.0
 	github.com/go-playground/validator/v10 v10.20.0

go.sum

@@ -112,6 +112,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
 github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/gaissmai/bart v0.23.0 h1:ct+78nySK5MaO+citQAUeef7QZ0ApXM3b+AYuCZYGIk=
+github.com/gaissmai/bart v0.23.0/go.mod h1:RpLtt3lWq1BoRz3AAyDAJ7jhLWBkYhVCfi+ximB2t68=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U=

orchestrator/clickhouse/http_test.go

@@ -71,7 +71,7 @@ func TestHTTPEndpoints(t *testing.T) {
 			ContentType: "text/csv; charset=utf-8",
 			FirstLines: []string{
 				`network,name,role,site,region,country,state,city,tenant,asn`,
-				`192.0.2.0/24,infra,,,,,,,,`,
+				`::ffff:192.0.2.0/120,infra,,,,,,,,`,
 			},
 		}, {
 			URL:         "/api/v0/orchestrator/clickhouse/custom_dict_none.csv",

orchestrator/clickhouse/networks.go

@@ -13,8 +13,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/kentik/patricia"
-
 	"akvorado/common/helpers"
 	"akvorado/common/schema"
 	"akvorado/orchestrator/geoip"
@@ -58,30 +56,30 @@ func (c *Component) networksCSVRefresher() {
 
 		// Add content of all geoip databases
 		err := c.d.GeoIP.IterASNDatabases(func(prefix netip.Prefix, data geoip.ASNInfo) error {
-			subV6Str, err := helpers.SubnetMapParseKey(prefix.String())
+			subV6Prefix := helpers.PrefixTo16(prefix)
-			if err != nil {
-				return err
-			}
 			attrs := NetworkAttributes{
 				ASN: data.ASNumber,
 			}
-			return networks.Update(subV6Str, attrs, overrideNetworkAttrs(attrs))
+			networks.Update(subV6Prefix, func(existing NetworkAttributes, _ bool) NetworkAttributes {
+				return mergeNetworkAttrs(existing, attrs)
+			})
+			return nil
 		})
 		if err != nil {
 			c.r.Err(err).Msg("unable to iter over ASN databases")
 			return
 		}
 		err = c.d.GeoIP.IterGeoDatabases(func(prefix netip.Prefix, data geoip.GeoInfo) error {
-			subV6Str, err := helpers.SubnetMapParseKey(prefix.String())
+			subV6Prefix := helpers.PrefixTo16(prefix)
-			if err != nil {
-				return err
-			}
 			attrs := NetworkAttributes{
 				State:   data.State,
 				Country: data.Country,
 				City:    data.City,
 			}
-			return networks.Update(subV6Str, attrs, overrideNetworkAttrs(attrs))
+			networks.Update(subV6Prefix, func(existing NetworkAttributes, _ bool) NetworkAttributes {
+				return mergeNetworkAttrs(existing, attrs)
+			})
+			return nil
 		})
 		if err != nil {
 			c.r.Err(err).Msg("unable to iter over geo databases")
@@ -93,13 +91,10 @@ func (c *Component) networksCSVRefresher() {
 			defer c.networkSourcesLock.RUnlock()
 			for _, networkList := range c.networkSources {
 				for _, val := range networkList {
-					if err := networks.Update(
+					subV6Prefix := helpers.PrefixTo16(val.Prefix)
-						val.Prefix.String(),
+					networks.Update(subV6Prefix, func(existing NetworkAttributes, _ bool) NetworkAttributes {
-						val.NetworkAttributes,
+						return mergeNetworkAttrs(existing, val.NetworkAttributes)
-						overrideNetworkAttrs(val.NetworkAttributes),
+					})
-					); err != nil {
-						return err
-					}
 				}
 			}
 			return nil
@@ -110,16 +105,10 @@ func (c *Component) networksCSVRefresher() {
 		// Add static network sources
 		if c.config.Networks != nil {
 			// Update networks with static network source
-			err := c.config.Networks.Iter(func(address patricia.IPv6Address, tags [][]NetworkAttributes) error {
+			for prefix, attrs := range c.config.Networks.All() {
-				return networks.Update(
+				networks.Update(prefix, func(existing NetworkAttributes, _ bool) NetworkAttributes {
-					address.String(),
+					return mergeNetworkAttrs(existing, attrs)
-					tags[len(tags)-1][0],
+				})
-					overrideNetworkAttrs(tags[len(tags)-1][0]),
-				)
-			})
-			if err != nil {
-				c.r.Err(err).Msg("unable to update with static network sources")
-				return
 			}
 		}
 
@@ -142,12 +131,13 @@ func (c *Component) networksCSVRefresher() {
 		gzipWriter := gzip.NewWriter(tmpfile)
 		csvWriter := csv.NewWriter(gzipWriter)
 		csvWriter.Write([]string{"network", "name", "role", "site", "region", "country", "state", "city", "tenant", "asn"})
-		networks.Iter(func(address patricia.IPv6Address, tags [][]NetworkAttributes) error {
+		for prefix, leafAttrs := range networks.AllMaybeSorted() {
-			current := NetworkAttributes{}
+			// Merge attributes from root to leaf for hierarchical inheritance.
-			for _, nodeTags := range tags {
+			// Supernets() returns in reverse-CIDR order (LPM to root), so we
-				for _, tag := range nodeTags {
+			// merge in that order.
-					current = mergeNetworkAttrs(current, tag)
+			current := leafAttrs
-				}
+			for _, attrs := range networks.Supernets(prefix) {
+				current = mergeNetworkAttrs(attrs, current)
 			}
 
 			var asnVal string
@@ -155,7 +145,7 @@ func (c *Component) networksCSVRefresher() {
 				asnVal = strconv.Itoa(int(current.ASN))
 			}
 			csvWriter.Write([]string{
-				address.String(),
+				prefix.String(),
 				current.Name,
 				current.Role,
 				current.Site,
@@ -166,8 +156,7 @@ func (c *Component) networksCSVRefresher() {
 				current.Tenant,
 				asnVal,
 			})
-			return nil
+		}
-		})
 		csvWriter.Flush()
 		gzipWriter.Close()
 
@@ -195,12 +184,6 @@ func (c *Component) networksCSVRefresher() {
 	}
 }
 
-func overrideNetworkAttrs(newAttrs NetworkAttributes) func(existing NetworkAttributes) NetworkAttributes {
-	return func(existing NetworkAttributes) NetworkAttributes {
-		return mergeNetworkAttrs(existing, newAttrs)
-	}
-}
-
 func mergeNetworkAttrs(existing, newAttrs NetworkAttributes) NetworkAttributes {
 	if newAttrs.ASN != 0 {
 		existing.ASN = newAttrs.ASN

orchestrator/clickhouse/networks_test.go

@@ -44,22 +44,22 @@ func TestNetworksCSVWithGeoip(t *testing.T) {
 				ContentType: "text/csv; charset=utf-8",
 				FirstLines: []string{
 					"network,name,role,site,region,country,state,city,tenant,asn",
-					"1.0.0.0/24,,,,,AU,Queensland,Brisbane,,15169",
+					"::ffff:1.0.0.0/120,,,,,AU,Queensland,Brisbane,,15169",
-					"1.0.1.0/24,,,,,CN,Fujian,Xiamen,,",
+					"::ffff:1.0.1.0/120,,,,,CN,Fujian,Xiamen,,",
-					"1.0.2.0/23,,,,,CN,Fujian,Xiamen,,",
+					"::ffff:1.0.2.0/119,,,,,CN,Fujian,Xiamen,,",
-					"1.0.4.0/22,,,,,AU,Victoria,Melbourne,,",
+					"::ffff:1.0.4.0/118,,,,,AU,Victoria,Melbourne,,",
-					"1.0.8.0/21,,,,,CN,Guangdong,Shenzhen,,",
+					"::ffff:1.0.8.0/117,,,,,CN,Guangdong,Shenzhen,,",
-					"1.0.16.0/29,,,,,JP,Tokyo,Tokyo,,",
+					"::ffff:1.0.16.0/125,,,,,JP,Tokyo,Tokyo,,",
-					"1.0.16.8/30,,,,,JP,Tokyo,Tokyo,,",
+					"::ffff:1.0.16.8/126,,,,,JP,Tokyo,Tokyo,,",
-					"1.0.16.12/31,,,,,JP,Tokyo,Tokyo,,",
+					"::ffff:1.0.16.12/127,,,,,JP,Tokyo,Tokyo,,",
-					"1.0.16.14/32,,,,,JP,Tokyo,Asagaya-minami,,",
+					"::ffff:1.0.16.14/128,,,,,JP,Tokyo,Asagaya-minami,,",
-					"1.0.16.15/32,,,,,JP,Tokyo,Tokyo,,",
+					"::ffff:1.0.16.15/128,,,,,JP,Tokyo,Tokyo,,",
-					"1.0.16.16/28,,,,,JP,Tokyo,Tokyo,,",
+					"::ffff:1.0.16.16/124,,,,,JP,Tokyo,Tokyo,,",
-					"1.0.16.32/27,,,,,JP,Tokyo,Tokyo,,",
+					"::ffff:1.0.16.32/123,,,,,JP,Tokyo,Tokyo,,",
-					"1.0.16.64/26,,,,,JP,Tokyo,Tokyo,,",
+					"::ffff:1.0.16.64/122,,,,,JP,Tokyo,Tokyo,,",
-					"1.0.16.128/25,,,,,JP,Tokyo,Tokyo,,",
+					"::ffff:1.0.16.128/121,,,,,JP,Tokyo,Tokyo,,",
-					"1.0.17.0/24,,,,,JP,Tokyo,Asagaya-minami,,",
+					"::ffff:1.0.17.0/120,,,,,JP,Tokyo,Asagaya-minami,,",
-					"1.0.18.0/23,,,,,JP,Tokyo,Asagaya-minami,,",
+					"::ffff:1.0.18.0/119,,,,,JP,Tokyo,Asagaya-minami,,",
 				},
 			},
 		})
@@ -92,29 +92,29 @@ func TestNetworksCSVWithGeoip(t *testing.T) {
 				ContentType: "text/csv; charset=utf-8",
 				FirstLines: []string{
 					"network,name,role,site,region,country,state,city,tenant,asn",
-					"0.80.0.0/16,,,,,,,,Alfred,",                        // not covered by GeoIP
+					"::ffff:0.80.0.0/112,,,,,,,,Alfred,",                        // not covered by GeoIP
-					"1.0.0.0/20,infra,,,,,,,,",                          // not covered by GeoIP...
+					"::ffff:1.0.0.0/116,infra,,,,,,,,",                          // not covered by GeoIP...
-					"1.0.0.0/24,infra,,,,AU,Queensland,Brisbane,,15169", // but covers GeoIP entries
+					"::ffff:1.0.0.0/120,infra,,,,AU,Queensland,Brisbane,,15169", // but covers GeoIP entries
-					"1.0.1.0/24,infra,,,,CN,Fujian,Xiamen,,",            // but covers GeoIP entries
+					"::ffff:1.0.1.0/120,infra,,,,CN,Fujian,Xiamen,,",            // but covers GeoIP entries
-					"1.0.2.0/23,infra,,,,CN,Fujian,Xiamen,,",            // but covers GeoIP entries
+					"::ffff:1.0.2.0/119,infra,,,,CN,Fujian,Xiamen,,",            // but covers GeoIP entries
-					"1.0.4.0/22,infra,,,,AU,Victoria,Melbourne,,",       // but covers GeoIP entries
+					"::ffff:1.0.4.0/118,infra,,,,AU,Victoria,Melbourne,,",       // but covers GeoIP entries
-					"1.0.8.0/21,infra,,,,CN,Guangdong,Shenzhen,,",       // but covers GeoIP entries
+					"::ffff:1.0.8.0/117,infra,,,,CN,Guangdong,Shenzhen,,",       // but covers GeoIP entries
-					"1.0.16.0/29,,,,,JP,Tokyo,Tokyo,,",
+					"::ffff:1.0.16.0/125,,,,,JP,Tokyo,Tokyo,,",
-					"1.0.16.8/30,,,,,JP,Tokyo,Tokyo,,",
+					"::ffff:1.0.16.8/126,,,,,JP,Tokyo,Tokyo,,",
-					"1.0.16.12/31,,,,,JP,Tokyo,Tokyo,,",
+					"::ffff:1.0.16.12/127,,,,,JP,Tokyo,Tokyo,,",
-					"1.0.16.14/32,,,,,JP,Tokyo,Asagaya-minami,,",
+					"::ffff:1.0.16.14/128,,,,,JP,Tokyo,Asagaya-minami,,",
-					"1.0.16.15/32,,,,,JP,Tokyo,Tokyo,,",
+					"::ffff:1.0.16.15/128,,,,,JP,Tokyo,Tokyo,,",
-					"1.0.16.16/28,,,,,JP,Tokyo,Tokyo,,",
+					"::ffff:1.0.16.16/124,,,,,JP,Tokyo,Tokyo,,",
-					"1.0.16.32/27,,,,,JP,Tokyo,Tokyo,,",
+					"::ffff:1.0.16.32/123,,,,,JP,Tokyo,Tokyo,,",
-					"1.0.16.64/26,infra,,,,JP,Tokyo,Tokyo,,",       // matching a GeoIP entry
+					"::ffff:1.0.16.64/122,infra,,,,JP,Tokyo,Tokyo,,",       // matching a GeoIP entry
-					"1.0.16.66/32,infra,,,,JP,Tokyo,Tokyo,Alfred,", // nested in previous one
+					"::ffff:1.0.16.66/128,infra,,,,JP,Tokyo,Tokyo,Alfred,", // nested in previous one
-					"1.0.16.128/25,,,,,JP,Tokyo,Tokyo,,",
+					"::ffff:1.0.16.128/121,,,,,JP,Tokyo,Tokyo,,",
-					"1.0.17.0/24,,,,,JP,Tokyo,Asagaya-minami,,",
+					"::ffff:1.0.17.0/120,,,,,JP,Tokyo,Asagaya-minami,,",
-					"1.0.18.0/23,,,,,JP,Tokyo,Asagaya-minami,,",
+					"::ffff:1.0.18.0/119,,,,,JP,Tokyo,Asagaya-minami,,",
-					"1.0.20.0/22,,,,,JP,Tokyo,Asagaya-minami,,",
+					"::ffff:1.0.20.0/118,,,,,JP,Tokyo,Asagaya-minami,,",
-					"1.0.24.0/21,,,,,JP,Tokyo,Asagaya-minami,,",
+					"::ffff:1.0.24.0/117,,,,,JP,Tokyo,Asagaya-minami,,",
-					"1.0.32.0/19,,,,,CN,Guangdong,Shenzhen,,",
+					"::ffff:1.0.32.0/115,,,,,CN,Guangdong,Shenzhen,,",
-					"1.0.64.0/20,,,,,JP,Hiroshima,Hiroshima,,",
+					"::ffff:1.0.64.0/116,,,,,JP,Hiroshima,Hiroshima,,",
 				},
 			},
 		})

orchestrator/clickhouse/source_test.go

@@ -128,7 +128,7 @@ func TestNetworkSources(t *testing.T) {
 			ContentType: "text/csv; charset=utf-8",
 			FirstLines: []string{
 				`network,name,role,site,region,country,state,city,tenant,asn`,
-				`3.2.34.0/26,,amazon,,af-south-1,,,,amazon,`,
+				`::ffff:3.2.34.0/122,,amazon,,af-south-1,,,,amazon,`,
 				`2600:1f14:fff:f800::/56,,route53_healthchecks,,us-west-2,,,,amazon,`,
 				`2600:1ff2:4000::/40,,amazon,,us-west-2,,,,amazon,`,
 			},

outlet/metadata/provider/static/source.go

@@ -42,9 +42,9 @@ func (i exporterInfo) toExporterConfiguration() ExporterConfiguration {
 // initStaticExporters initializes the reconciliation map for exporter configurations
 // with the static prioritized data from exporters' Configuration.
 func (p *Provider) initStaticExporters() {
-	staticExportersMap := p.exporters.Load().ToMap()
+	staticExporters := make([]exporterInfo, 0)
-	staticExporters := make([]exporterInfo, 0, len(staticExportersMap))
+	staticExportersMap := p.exporters.Load()
-	for subnet, config := range staticExportersMap {
+	for subnet, config := range staticExportersMap.All() {
 		interfaces := make([]exporterInterface, 0, len(config.IfIndexes))
 		for ifindex, iface := range config.IfIndexes {
 			interfaces = append(interfaces, exporterInterface{
@@ -58,7 +58,7 @@ func (p *Provider) initStaticExporters() {
 				Exporter: provider.Exporter{
 					Name: config.Name,
 				},
-				ExporterSubnet: subnet,
+				ExporterSubnet: subnet.String(),
 				Default:        config.Default,
 				Interfaces:     interfaces,
 			},
@@ -88,7 +88,7 @@ func (p *Provider) UpdateSource(ctx context.Context, name string, source remoted
 				continue
 			}
 			// Concurrency for same Exporter config across multiple remote data sources is not handled
-			finalMap[exporterSubnet] = exporterData.toExporterConfiguration()
+			finalMap[exporterSubnet.String()] = exporterData.toExporterConfiguration()
 		}
 	}
 	for _, exporterData := range p.exportersMap["static"] {
@@ -98,10 +98,10 @@ func (p *Provider) UpdateSource(ctx context.Context, name string, source remoted
 			continue
 		}
 		// This overrides duplicates config for an Exporter if it's also defined as static
-		finalMap[exporterSubnet] = exporterData.toExporterConfiguration()
+		finalMap[exporterSubnet.String()] = exporterData.toExporterConfiguration()
 	}
 	p.exportersLock.Unlock()
-	exporters, err := helpers.NewSubnetMap[ExporterConfiguration](finalMap)
+	exporters, err := helpers.NewSubnetMap(finalMap)
 	if err != nil {
 		return 0, errors.New("cannot create subnetmap")
 	}

outlet/metadata/provider/static/source_test.go

@@ -50,7 +50,7 @@ func TestInitStaticExporters(t *testing.T) {
 
 	expected["static"] = []exporterInfo{
 		{
-			ExporterSubnet: "203.0.113.0/24",
+			ExporterSubnet: "::ffff:203.0.113.0/120",
 			Exporter: provider.Exporter{
 				Name: "something",
 			},