diff --git a/common/helpers/subnetmap.go b/common/helpers/subnetmap.go index 415f0ae7..29a89344 100644 --- a/common/helpers/subnetmap.go +++ b/common/helpers/subnetmap.go @@ -5,34 +5,30 @@ package helpers import ( "fmt" - "net" + "iter" "net/netip" "reflect" "regexp" "strings" + "github.com/gaissmai/bart" "github.com/gin-gonic/gin" "github.com/go-viper/mapstructure/v2" - "github.com/kentik/patricia" - tree "github.com/kentik/patricia/generics_tree" ) // SubnetMap maps subnets to values and allow to lookup by IP address. -// Internally, everything is stored as an IPv6 (using v6-mapped IPv4 -// addresses). type SubnetMap[V any] struct { - tree *tree.TreeV6[V] + table *bart.Table[V] } // Lookup will search for the most specific subnet matching the // provided IP address and return the value associated with it. func (sm *SubnetMap[V]) Lookup(ip netip.Addr) (V, bool) { - if sm == nil || sm.tree == nil { + if sm == nil || sm.table == nil { var value V return value, false } - ok, value := sm.tree.FindDeepestTag(patricia.NewIPv6Address(ip.AsSlice(), 128)) - return value, ok + return sm.table.Lookup(ip) } // LookupOrDefault calls lookup and if not found, will return the @@ -44,82 +40,98 @@ func (sm *SubnetMap[V]) LookupOrDefault(ip netip.Addr, fallback V) V { return fallback } -// ToMap return a map of the tree. +// ToMap return a map of the tree. This should be used only when handling user +// configuration or for debugging. Otherwise, it is better to use Iter(). func (sm *SubnetMap[V]) ToMap() map[string]V { output := map[string]V{} - if sm == nil || sm.tree == nil { - return output - } - iter := sm.tree.Iterate() - for iter.Next() { - output[iter.Address().String()] = iter.Tags()[0] + for prefix, value := range sm.All() { + if prefix.Addr().Is4In6() { + ipv4Addr := prefix.Addr().Unmap() + ipv4Prefix := netip.PrefixFrom(ipv4Addr, prefix.Bits()-96) + output[ipv4Prefix.String()] = value + continue + } + output[prefix.String()] = value } return output } -// Set inserts the given key k into the SubnetMap, replacing any existing value if it exists. -func (sm *SubnetMap[V]) Set(k string, v V) error { - subnetK, err := SubnetMapParseKey(k) - if err != nil { - return err +// Set inserts the given key k into the SubnetMap, replacing any existing value +// if it exists. It requires an IPv6 prefix or it will panic. +func (sm *SubnetMap[V]) Set(prefix netip.Prefix, v V) { + if !prefix.Addr().Is6() { + panic(fmt.Errorf("%q is not an IPv6 subnet", prefix)) } - _, ipNet, err := net.ParseCIDR(subnetK) - if err != nil { - // Should not happen - return err + if sm.table == nil { + sm.table = &bart.Table[V]{} } - _, bits := ipNet.Mask.Size() - if bits != 128 { - return fmt.Errorf("%q is not an IPv6 subnet", ipNet) - } - plen, _ := ipNet.Mask.Size() - sm.tree.Set(patricia.NewIPv6Address(ipNet.IP.To16(), uint(plen)), v) - return nil + sm.table.Insert(prefix, v) } -// Update inserts the given key k into the SubnetMap, calling updateFunc with the existing value. -func (sm *SubnetMap[V]) Update(k string, v V, updateFunc tree.UpdatesFunc[V]) error { - subnetK, err := SubnetMapParseKey(k) - if err != nil { - return err +// Update inserts the given key k into the SubnetMap, calling cb with the +// existing value. It requires an IPv6 prefix or it will panic. +func (sm *SubnetMap[V]) Update(prefix netip.Prefix, cb func(V, bool) V) { + if !prefix.Addr().Is6() { + panic(fmt.Errorf("%q is not an IPv6 subnet", prefix)) } - _, ipNet, err := net.ParseCIDR(subnetK) - if err != nil { - // Should not happen - return err + if sm.table == nil { + sm.table = &bart.Table[V]{} } - _, bits := ipNet.Mask.Size() - if bits != 128 { - return fmt.Errorf("%q is not an IPv6 subnet", ipNet) - } - plen, _ := ipNet.Mask.Size() - sm.tree.SetOrUpdate(patricia.NewIPv6Address(ipNet.IP.To16(), uint(plen)), v, updateFunc) - return nil + sm.table.Update(prefix, cb) } -// Iter enables iteration of the SubnetMap, calling f for every entry. If f returns an error, the iteration is aborted. -func (sm *SubnetMap[V]) Iter(f func(address patricia.IPv6Address, tags [][]V) error) error { - iter := sm.tree.Iterate() - for iter.Next() { - if err := f(iter.Address(), iter.TagsFromRoot()); err != nil { - return err +// All walks the whole subnet map. +func (sm *SubnetMap[V]) All() iter.Seq2[netip.Prefix, V] { + return func(yield func(netip.Prefix, V) bool) { + if sm == nil || sm.table == nil { + return + } + sm.table.All6()(yield) + } +} + +// AllMaybeSorted walks the whole subnet map in sorted order during tests but +// not when running tests. +func (sm *SubnetMap[V]) AllMaybeSorted() iter.Seq2[netip.Prefix, V] { + return func(yield func(netip.Prefix, V) bool) { + if sm == nil || sm.table == nil { + return + } + if Testing() { + sm.table.AllSorted6()(yield) + } else { + sm.table.All6()(yield) } } - return nil } -// NewSubnetMap creates a subnetmap from a map. Unlike user-provided -// configuration, this function is stricter and require everything to -// be IPv6 subnets. +// Supernets returns an iterator over all supernet routes that cover the given +// prefix. The iteration order is reverse-CIDR: from longest prefix match (LPM) +// towards least-specific routes. +func (sm *SubnetMap[V]) Supernets(prefix netip.Prefix) iter.Seq2[netip.Prefix, V] { + return func(yield func(netip.Prefix, V) bool) { + if sm == nil || sm.table == nil { + return + } + sm.table.Supernets(prefix)(yield) + } +} + +// NewSubnetMap creates a subnetmap from a map. It should not be used in a hot +// path as it builds the subnet from a map keyed by strings. func NewSubnetMap[V any](from map[string]V) (*SubnetMap[V], error) { - trie := &SubnetMap[V]{tree.NewTreeV6[V]()} + sm := &SubnetMap[V]{table: &bart.Table[V]{}} if from == nil { - return trie, nil + return sm, nil } for k, v := range from { - trie.Set(k, v) + key, err := SubnetMapParseKey(k) + if err != nil { + return nil, fmt.Errorf("failed to parse key %s: %w", k, err) + } + sm.Set(key, v) } - return trie, nil + return sm, nil } // MustNewSubnetMap creates a subnet from a map and panic in case of a @@ -157,9 +169,10 @@ func LooksLikeSubnetMap(v reflect.Value) (result bool) { return } -// SubnetMapUnmarshallerHook decodes SubnetMap and notably check that -// valid networks are provided as key. It also accepts a single value -// instead of a map for backward compatibility. +// SubnetMapUnmarshallerHook decodes SubnetMap and notably check that valid +// networks are provided as key. It also accepts a single value instead of a map +// for backward compatibility. It should not be used in hot paths as it builds +// an intermediate map. func SubnetMapUnmarshallerHook[V any]() mapstructure.DecodeHookFunc { return func(from, to reflect.Value) (any, error) { if to.Type() != reflect.TypeOf(SubnetMap[V]{}) { @@ -184,7 +197,7 @@ func SubnetMapUnmarshallerHook[V any]() mapstructure.DecodeHookFunc { if err != nil { return nil, fmt.Errorf("failed to parse key %s: %w", key, err) } - output[key] = v.Interface() + output[key.String()] = v.Interface() } } else { // Second case, we have a single value and we let mapstructure handles it @@ -211,40 +224,36 @@ func SubnetMapUnmarshallerHook[V any]() mapstructure.DecodeHookFunc { } } -// SubnetMapParseKey decodes and validates a key used in SubnetMap from a network string. -func SubnetMapParseKey(k string) (string, error) { - var key string - if strings.Contains(k, "/") { - // Subnet - _, ipNet, err := net.ParseCIDR(k) - if err != nil { - return "", err - } - // Convert key to IPv6 - ones, bits := ipNet.Mask.Size() - if bits != 32 && bits != 128 { - return "", fmt.Errorf("key %s has invalid netmask", k) - } - if bits == 32 { - key = fmt.Sprintf("::ffff:%s/%d", ipNet.IP.String(), ones+96) - } else if ipNet.IP.To4() != nil { - key = fmt.Sprintf("::ffff:%s/%d", ipNet.IP.String(), ones) - } else { - key = ipNet.String() - } - } else { - // IP - ip := net.ParseIP(k) - if ip == nil { - return "", fmt.Errorf("key %s is not a valid subnet", k) - } - if ipv4 := ip.To4(); ipv4 != nil { - key = fmt.Sprintf("::ffff:%s/128", ipv4.String()) - } else { - key = fmt.Sprintf("%s/128", ip.String()) - } +// PrefixTo16 converts an IPv4 prefix to an IPv4-mapped IPv6 prefix. +// IPv6 prefixes are returned as-is. +func PrefixTo16(prefix netip.Prefix) netip.Prefix { + if prefix.Addr().Is6() { + return prefix } - return key, nil + // Convert IPv4 to IPv4-mapped IPv6 + return netip.PrefixFrom(netip.AddrFrom16(prefix.Addr().As16()), prefix.Bits()+96) +} + +// SubnetMapParseKey parses a prefix or an IP address into a netip.Prefix that +// can be used in a map. +func SubnetMapParseKey(k string) (netip.Prefix, error) { + // Subnet + if strings.Contains(k, "/") { + key, err := netip.ParsePrefix(k) + if err != nil { + return netip.Prefix{}, err + } + return PrefixTo16(key), nil + } + // IP address + key, err := netip.ParseAddr(k) + if err != nil { + return netip.Prefix{}, err + } + if key.Is4() { + return PrefixTo16(netip.PrefixFrom(key, 32)), nil + } + return netip.PrefixFrom(key, 128), nil } // MarshalYAML turns a subnet into a map that can be marshaled. diff --git a/common/helpers/subnetmap_test.go b/common/helpers/subnetmap_test.go index 17d9832e..c40794cf 100644 --- a/common/helpers/subnetmap_test.go +++ b/common/helpers/subnetmap_test.go @@ -5,6 +5,7 @@ package helpers_test import ( "net/netip" + "slices" "testing" "github.com/gin-gonic/gin" @@ -282,7 +283,7 @@ func TestSubnetMapParseKey(t *testing.T) { } else if err == nil && tc.Error { t.Fatal("SubnetMapParseKey() did not return an error") } - if diff := helpers.Diff(res, tc.Expected); diff != "" { + if diff := helpers.Diff(res.String(), tc.Expected); err == nil && diff != "" { t.Fatalf("Decode() (-got, +want):\n%s", diff) } }) @@ -303,3 +304,416 @@ func TestToMap(t *testing.T) { t.Fatalf("ToMap() (-got, +want):\n%s", diff) } } + +func TestSubnetMapLookup(t *testing.T) { + t.Run("nil", func(t *testing.T) { + var sm *helpers.SubnetMap[string] + value, ok := sm.Lookup(netip.MustParseAddr("::ffff:192.0.2.1")) + if ok || value != "" { + t.Fatalf("Lookup() == value=%q, ok=%v", value, ok) + } + }) + + t.Run("empty", func(t *testing.T) { + sm := &helpers.SubnetMap[string]{} + value, ok := sm.Lookup(netip.MustParseAddr("::ffff:192.0.2.1")) + if ok || value != "" { + t.Fatalf("Lookup() == value=%q, ok=%v", value, ok) + } + }) + + t.Run("populated", func(t *testing.T) { + sm := helpers.MustNewSubnetMap(map[string]string{ + "192.0.2.0/24": "customer1", + "2001:db8::/64": "customer2", + "10.0.0.1": "specific", + }) + + cases := []struct { + ip string + expected string + found bool + }{ + {"::ffff:192.0.2.1", "customer1", true}, + {"::ffff:192.0.2.255", "customer1", true}, + {"::ffff:192.0.3.1", "", false}, + {"::ffff:10.0.0.1", "specific", true}, + {"::ffff:10.0.0.2", "", false}, + {"2001:db8::1", "customer2", true}, + {"2001:db8:1::1", "", false}, + } + + for _, tc := range cases { + t.Run(tc.ip, func(t *testing.T) { + value, ok := sm.Lookup(netip.MustParseAddr(tc.ip)) + if ok != tc.found || value != tc.expected { + t.Fatalf("Lookup(%s) = (%q, %v), want (%q, %v)", tc.ip, value, ok, tc.expected, tc.found) + } + }) + } + }) +} + +func TestSubnetMapLookupOrDefault(t *testing.T) { + sm := helpers.MustNewSubnetMap(map[string]string{ + "192.0.2.0/24": "customer1", + }) + + t.Run("found", func(t *testing.T) { + value := sm.LookupOrDefault(netip.MustParseAddr("::ffff:192.0.2.1"), "default") + if value != "customer1" { + t.Fatalf("LookupOrDefault() = %q, want %q", value, "customer1") + } + }) + + t.Run("not found", func(t *testing.T) { + value := sm.LookupOrDefault(netip.MustParseAddr("::ffff:192.0.3.1"), "default") + if value != "default" { + t.Fatalf("LookupOrDefault() = %q, want %q", value, "default") + } + }) + + t.Run("nil", func(t *testing.T) { + var sm *helpers.SubnetMap[string] + value := sm.LookupOrDefault(netip.MustParseAddr("::ffff:192.0.2.1"), "default") + if value != "default" { + t.Fatalf("LookupOrDefault() = %q, want %q", value, "default") + } + }) +} + +func TestSubnetMapSet(t *testing.T) { + sm := &helpers.SubnetMap[string]{} + + t.Run("set IPv6 subnet", func(t *testing.T) { + prefix := netip.MustParsePrefix("2001:db8::/64") + sm.Set(prefix, "test-value") + + value, ok := sm.Lookup(netip.MustParseAddr("2001:db8::1")) + if !ok || value != "test-value" { + t.Fatalf("Lookup() = (%q, %v), want (%q, %v)", value, ok, "test-value", true) + } + }) + + t.Run("set IPv4-mapped IPv6 subnet", func(t *testing.T) { + prefix := netip.MustParsePrefix("::ffff:192.0.2.0/120") + sm.Set(prefix, "ipv4-mapped") + + value, ok := sm.Lookup(netip.MustParseAddr("::ffff:192.0.2.1")) + if !ok || value != "ipv4-mapped" { + t.Fatalf("Lookup() = (%q, %v), want (%q, %v)", value, ok, "ipv4-mapped", true) + } + }) + + t.Run("overwrite existing value", func(t *testing.T) { + prefix := netip.MustParsePrefix("2001:db8::/64") + sm.Set(prefix, "new-value") + + value, ok := sm.Lookup(netip.MustParseAddr("2001:db8::1")) + if !ok || value != "new-value" { + t.Fatalf("Lookup() = (%q, %v), want (%q, %v)", value, ok, "new-value", true) + } + }) +} + +func TestSubnetMapSetPanic(t *testing.T) { + sm := &helpers.SubnetMap[string]{} + + t.Run("panic on IPv4 subnet", func(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Fatal("Set() should panic with IPv4") + } + }() + prefix := netip.MustParsePrefix("192.0.2.0/24") + sm.Set(prefix, "should-panic") + }) +} + +func TestSubnetMapUpdate(t *testing.T) { + sm := &helpers.SubnetMap[int]{} + + t.Run("update new value", func(t *testing.T) { + prefix := netip.MustParsePrefix("2001:db8::/64") + sm.Update(prefix, func(old int, exists bool) int { + if !exists { + return 42 + } + return old + 1 + }) + + value, ok := sm.Lookup(netip.MustParseAddr("2001:db8::1")) + if !ok || value != 42 { + t.Fatalf("Lookup() = (%d, %v), want (%d, %v)", value, ok, 42, true) + } + }) + + t.Run("update existing value", func(t *testing.T) { + prefix := netip.MustParsePrefix("2001:db8::/64") + sm.Update(prefix, func(old int, exists bool) int { + if exists { + return old + 10 + } + return 0 + }) + + value, ok := sm.Lookup(netip.MustParseAddr("2001:db8::1")) + if !ok || value != 52 { + t.Fatalf("Lookup() = (%d, %v), want (%d, %v)", value, ok, 52, true) + } + }) +} + +func TestSubnetMapUpdatePanic(t *testing.T) { + sm := &helpers.SubnetMap[int]{} + + t.Run("panic on IPv4 subnet", func(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Fatal("Update() should panic with IPv4") + } + }() + prefix := netip.MustParsePrefix("192.0.2.0/24") + sm.Update(prefix, func(old int, exists bool) int { return 1 }) + }) +} + +func TestSubnetMapAll(t *testing.T) { + t.Run("nil", func(t *testing.T) { + var sm *helpers.SubnetMap[string] + count := 0 + for range sm.All() { + count++ + } + if count != 0 { + t.Fatalf("All() count = %d, want %d", count, 0) + } + }) + + t.Run("empty", func(t *testing.T) { + sm := &helpers.SubnetMap[string]{} + count := 0 + for range sm.All() { + count++ + } + if count != 0 { + t.Fatalf("All() count = %d, want %d", count, 0) + } + }) + + t.Run("populated", func(t *testing.T) { + sm := helpers.MustNewSubnetMap(map[string]string{ + "2001:db8::/64": "ipv6", + "::ffff:192.0.2.0/120": "ipv4-mapped", + }) + + items := make(map[string]string) + for prefix, value := range sm.All() { + items[prefix.String()] = value + } + + expected := map[string]string{ + "2001:db8::/64": "ipv6", + "::ffff:192.0.2.0/120": "ipv4-mapped", + } + + if diff := helpers.Diff(items, expected); diff != "" { + t.Fatalf("All() (-got, +want):\n%s", diff) + } + }) +} + +func TestSubnetMapString(t *testing.T) { + t.Run("empty", func(t *testing.T) { + sm := &helpers.SubnetMap[string]{} + str := sm.String() + expected := "map[]" + if str != expected { + t.Fatalf("String() = %q, want %q", str, expected) + } + }) + + t.Run("populated", func(t *testing.T) { + sm := helpers.MustNewSubnetMap(map[string]string{ + "192.0.2.0/24": "customer", + }) + str := sm.String() + if diff := helpers.Diff(str, "map[192.0.2.0/24:customer]"); diff != "" { + t.Fatalf("String() (-got, +want):\n%s", diff) + } + }) +} + +func TestPrefixTo16(t *testing.T) { + cases := []struct { + name string + input string + expected string + }{ + { + name: "IPv4 prefix", + input: "192.0.2.0/24", + expected: "::ffff:192.0.2.0/120", + }, + { + name: "IPv4 host", + input: "192.0.2.1/32", + expected: "::ffff:192.0.2.1/128", + }, + { + name: "IPv6 prefix unchanged", + input: "2001:db8::/64", + expected: "2001:db8::/64", + }, + { + name: "IPv6 host unchanged", + input: "2001:db8::1/128", + expected: "2001:db8::1/128", + }, + { + name: "IPv4-mapped IPv6 unchanged", + input: "::ffff:192.0.2.0/120", + expected: "::ffff:192.0.2.0/120", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + prefix := netip.MustParsePrefix(tc.input) + result := helpers.PrefixTo16(prefix) + if result.String() != tc.expected { + t.Fatalf("PrefixTo16(%s) = %s, want %s", tc.input, result, tc.expected) + } + }) + } +} + +func TestSubnetMapSupernets(t *testing.T) { + t.Run("nil", func(t *testing.T) { + var sm *helpers.SubnetMap[string] + count := 0 + for range sm.Supernets(netip.MustParsePrefix("::ffff:192.0.2.1/128")) { + count++ + } + if count != 0 { + t.Fatalf("Supernets() count = %d, want %d", count, 0) + } + }) + + t.Run("empty", func(t *testing.T) { + sm := &helpers.SubnetMap[string]{} + count := 0 + for range sm.Supernets(netip.MustParsePrefix("::ffff:192.0.2.1/128")) { + count++ + } + if count != 0 { + t.Fatalf("Supernets() count = %d, want %d", count, 0) + } + }) + + t.Run("hierarchical supernets", func(t *testing.T) { + sm := helpers.MustNewSubnetMap(map[string]string{ + "192.0.0.0/16": "region", + "192.0.2.0/24": "site", + "192.0.2.0/28": "rack", + }) + + // Query for 192.0.2.1/32 should find supernets in reverse-CIDR order + var results []string + var prefixes []string + for prefix, value := range sm.Supernets(netip.MustParsePrefix("::ffff:192.0.2.1/128")) { + results = append(results, value) + prefixes = append(prefixes, prefix.String()) + } + + expectedValues := []string{"rack", "site", "region"} + expectedPrefixes := []string{"::ffff:192.0.2.0/124", "::ffff:192.0.2.0/120", "::ffff:192.0.0.0/112"} + + if diff := helpers.Diff(results, expectedValues); diff != "" { + t.Errorf("Supernets() values (-got, +want):\n%s", diff) + } + if diff := helpers.Diff(prefixes, expectedPrefixes); diff != "" { + t.Errorf("Supernets() prefixes (-got, +want):\n%s", diff) + } + + count := 0 + for range sm.Supernets(netip.MustParsePrefix("::ffff:10.0.0.1/128")) { + count++ + } + if count != 0 { + t.Fatalf("Supernets() count = %d, want %d", count, 0) + } + }) +} + +func TestSubnetMapAllMaybeSorted(t *testing.T) { + t.Run("nil", func(t *testing.T) { + var sm *helpers.SubnetMap[string] + count := 0 + for range sm.AllMaybeSorted() { + count++ + } + if count != 0 { + t.Fatalf("AllMaybeSorted() count = %d, want %d", count, 0) + } + }) + + t.Run("empty", func(t *testing.T) { + sm := &helpers.SubnetMap[string]{} + count := 0 + for range sm.AllMaybeSorted() { + count++ + } + if count != 0 { + t.Fatalf("AllMaybeSorted() count = %d, want %d", count, 0) + } + }) + + t.Run("sorted vs unsorted", func(t *testing.T) { + sm := helpers.MustNewSubnetMap(map[string]string{ + "192.0.2.0/28": "rack", + "192.0.0.0/16": "region", + "192.0.2.0/24": "site", + "2001:db8:1::/64": "ipv6-site1", + "2001:db8::/32": "ipv6-region", + "2001:db8:2::/64": "ipv6-site2", + }) + + // Collect results from All() (potentially unsorted) + var allPrefixes []string + for prefix := range sm.All() { + allPrefixes = append(allPrefixes, prefix.String()) + } + + // Collect results from AllMaybeSorted() (sorted during tests) + var sortedPrefixes []string + for prefix := range sm.AllMaybeSorted() { + sortedPrefixes = append(sortedPrefixes, prefix.String()) + } + + // Expected sorted order: IPv6 addresses first (sorted), then IPv4-mapped IPv6 (sorted) + expectedPrefixes := []string{ + "::ffff:192.0.0.0/112", + "::ffff:192.0.2.0/120", + "::ffff:192.0.2.0/124", + "2001:db8::/32", + "2001:db8:1::/64", + "2001:db8:2::/64", + } + + // AllMaybeSorted() should be sorted during tests + if diff := helpers.Diff(sortedPrefixes, expectedPrefixes); diff != "" { + t.Errorf("AllMaybeSorted() prefixes (-got, +want):\n%s", diff) + } + + // All() and AllMaybeSorted() should contain the same elements (but potentially different order) + if len(allPrefixes) != len(sortedPrefixes) { + t.Errorf("All() returned %d prefixes, AllMaybeSorted() returned %d", len(allPrefixes), len(sortedPrefixes)) + } + + // Verify that All() and AllMaybeSorted() return different orders (otherwise test is meaningless) + if slices.Equal(allPrefixes, sortedPrefixes) { + t.Skip("All() and AllMaybeSorted() returned identical order") + } + }) +} diff --git a/common/helpers/tests_diff.go b/common/helpers/tests_diff.go index f0bff2af..d33747fc 100644 --- a/common/helpers/tests_diff.go +++ b/common/helpers/tests_diff.go @@ -30,6 +30,7 @@ func defaultPrettyFormatters() map[reflect.Type]any { result := map[reflect.Type]any{ reflect.TypeOf(net.IP{}): fmt.Sprint, reflect.TypeOf(netip.Addr{}): fmt.Sprint, + reflect.TypeOf(netip.Prefix{}): fmt.Sprint, reflect.TypeOf(time.Time{}): fmt.Sprint, reflect.TypeOf(SubnetMap[string]{}): fmt.Sprint, reflect.TypeOf(SubnetMap[uint]{}): fmt.Sprint, diff --git a/go.mod b/go.mod index 98698628..3a51a45d 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/eapache/go-resiliency v1.7.0 github.com/expr-lang/expr v1.17.5 github.com/fsnotify/fsnotify v1.9.0 + github.com/gaissmai/bart v0.23.0 github.com/gin-gonic/gin v1.10.1 github.com/glebarez/sqlite v1.11.0 github.com/go-playground/validator/v10 v10.20.0 diff --git a/go.sum b/go.sum index 97d508f3..2192417f 100644 --- a/go.sum +++ b/go.sum @@ -112,6 +112,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gaissmai/bart v0.23.0 h1:ct+78nySK5MaO+citQAUeef7QZ0ApXM3b+AYuCZYGIk= +github.com/gaissmai/bart v0.23.0/go.mod h1:RpLtt3lWq1BoRz3AAyDAJ7jhLWBkYhVCfi+ximB2t68= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= diff --git a/orchestrator/clickhouse/http_test.go b/orchestrator/clickhouse/http_test.go index c14ed6a0..f46d571c 100644 --- a/orchestrator/clickhouse/http_test.go +++ b/orchestrator/clickhouse/http_test.go @@ -71,7 +71,7 @@ func TestHTTPEndpoints(t *testing.T) { ContentType: "text/csv; charset=utf-8", FirstLines: []string{ `network,name,role,site,region,country,state,city,tenant,asn`, - `192.0.2.0/24,infra,,,,,,,,`, + `::ffff:192.0.2.0/120,infra,,,,,,,,`, }, }, { URL: "/api/v0/orchestrator/clickhouse/custom_dict_none.csv", diff --git a/orchestrator/clickhouse/networks.go b/orchestrator/clickhouse/networks.go index f356ba94..d9c9f4d2 100644 --- a/orchestrator/clickhouse/networks.go +++ b/orchestrator/clickhouse/networks.go @@ -13,8 +13,6 @@ import ( "strconv" "time" - "github.com/kentik/patricia" - "akvorado/common/helpers" "akvorado/common/schema" "akvorado/orchestrator/geoip" @@ -58,30 +56,30 @@ func (c *Component) networksCSVRefresher() { // Add content of all geoip databases err := c.d.GeoIP.IterASNDatabases(func(prefix netip.Prefix, data geoip.ASNInfo) error { - subV6Str, err := helpers.SubnetMapParseKey(prefix.String()) - if err != nil { - return err - } + subV6Prefix := helpers.PrefixTo16(prefix) attrs := NetworkAttributes{ ASN: data.ASNumber, } - return networks.Update(subV6Str, attrs, overrideNetworkAttrs(attrs)) + networks.Update(subV6Prefix, func(existing NetworkAttributes, _ bool) NetworkAttributes { + return mergeNetworkAttrs(existing, attrs) + }) + return nil }) if err != nil { c.r.Err(err).Msg("unable to iter over ASN databases") return } err = c.d.GeoIP.IterGeoDatabases(func(prefix netip.Prefix, data geoip.GeoInfo) error { - subV6Str, err := helpers.SubnetMapParseKey(prefix.String()) - if err != nil { - return err - } + subV6Prefix := helpers.PrefixTo16(prefix) attrs := NetworkAttributes{ State: data.State, Country: data.Country, City: data.City, } - return networks.Update(subV6Str, attrs, overrideNetworkAttrs(attrs)) + networks.Update(subV6Prefix, func(existing NetworkAttributes, _ bool) NetworkAttributes { + return mergeNetworkAttrs(existing, attrs) + }) + return nil }) if err != nil { c.r.Err(err).Msg("unable to iter over geo databases") @@ -93,13 +91,10 @@ func (c *Component) networksCSVRefresher() { defer c.networkSourcesLock.RUnlock() for _, networkList := range c.networkSources { for _, val := range networkList { - if err := networks.Update( - val.Prefix.String(), - val.NetworkAttributes, - overrideNetworkAttrs(val.NetworkAttributes), - ); err != nil { - return err - } + subV6Prefix := helpers.PrefixTo16(val.Prefix) + networks.Update(subV6Prefix, func(existing NetworkAttributes, _ bool) NetworkAttributes { + return mergeNetworkAttrs(existing, val.NetworkAttributes) + }) } } return nil @@ -110,16 +105,10 @@ func (c *Component) networksCSVRefresher() { // Add static network sources if c.config.Networks != nil { // Update networks with static network source - err := c.config.Networks.Iter(func(address patricia.IPv6Address, tags [][]NetworkAttributes) error { - return networks.Update( - address.String(), - tags[len(tags)-1][0], - overrideNetworkAttrs(tags[len(tags)-1][0]), - ) - }) - if err != nil { - c.r.Err(err).Msg("unable to update with static network sources") - return + for prefix, attrs := range c.config.Networks.All() { + networks.Update(prefix, func(existing NetworkAttributes, _ bool) NetworkAttributes { + return mergeNetworkAttrs(existing, attrs) + }) } } @@ -142,12 +131,13 @@ func (c *Component) networksCSVRefresher() { gzipWriter := gzip.NewWriter(tmpfile) csvWriter := csv.NewWriter(gzipWriter) csvWriter.Write([]string{"network", "name", "role", "site", "region", "country", "state", "city", "tenant", "asn"}) - networks.Iter(func(address patricia.IPv6Address, tags [][]NetworkAttributes) error { - current := NetworkAttributes{} - for _, nodeTags := range tags { - for _, tag := range nodeTags { - current = mergeNetworkAttrs(current, tag) - } + for prefix, leafAttrs := range networks.AllMaybeSorted() { + // Merge attributes from root to leaf for hierarchical inheritance. + // Supernets() returns in reverse-CIDR order (LPM to root), so we + // merge in that order. + current := leafAttrs + for _, attrs := range networks.Supernets(prefix) { + current = mergeNetworkAttrs(attrs, current) } var asnVal string @@ -155,7 +145,7 @@ func (c *Component) networksCSVRefresher() { asnVal = strconv.Itoa(int(current.ASN)) } csvWriter.Write([]string{ - address.String(), + prefix.String(), current.Name, current.Role, current.Site, @@ -166,8 +156,7 @@ func (c *Component) networksCSVRefresher() { current.Tenant, asnVal, }) - return nil - }) + } csvWriter.Flush() gzipWriter.Close() @@ -195,12 +184,6 @@ func (c *Component) networksCSVRefresher() { } } -func overrideNetworkAttrs(newAttrs NetworkAttributes) func(existing NetworkAttributes) NetworkAttributes { - return func(existing NetworkAttributes) NetworkAttributes { - return mergeNetworkAttrs(existing, newAttrs) - } -} - func mergeNetworkAttrs(existing, newAttrs NetworkAttributes) NetworkAttributes { if newAttrs.ASN != 0 { existing.ASN = newAttrs.ASN diff --git a/orchestrator/clickhouse/networks_test.go b/orchestrator/clickhouse/networks_test.go index d8bec29f..4a1be273 100644 --- a/orchestrator/clickhouse/networks_test.go +++ b/orchestrator/clickhouse/networks_test.go @@ -44,22 +44,22 @@ func TestNetworksCSVWithGeoip(t *testing.T) { ContentType: "text/csv; charset=utf-8", FirstLines: []string{ "network,name,role,site,region,country,state,city,tenant,asn", - "1.0.0.0/24,,,,,AU,Queensland,Brisbane,,15169", - "1.0.1.0/24,,,,,CN,Fujian,Xiamen,,", - "1.0.2.0/23,,,,,CN,Fujian,Xiamen,,", - "1.0.4.0/22,,,,,AU,Victoria,Melbourne,,", - "1.0.8.0/21,,,,,CN,Guangdong,Shenzhen,,", - "1.0.16.0/29,,,,,JP,Tokyo,Tokyo,,", - "1.0.16.8/30,,,,,JP,Tokyo,Tokyo,,", - "1.0.16.12/31,,,,,JP,Tokyo,Tokyo,,", - "1.0.16.14/32,,,,,JP,Tokyo,Asagaya-minami,,", - "1.0.16.15/32,,,,,JP,Tokyo,Tokyo,,", - "1.0.16.16/28,,,,,JP,Tokyo,Tokyo,,", - "1.0.16.32/27,,,,,JP,Tokyo,Tokyo,,", - "1.0.16.64/26,,,,,JP,Tokyo,Tokyo,,", - "1.0.16.128/25,,,,,JP,Tokyo,Tokyo,,", - "1.0.17.0/24,,,,,JP,Tokyo,Asagaya-minami,,", - "1.0.18.0/23,,,,,JP,Tokyo,Asagaya-minami,,", + "::ffff:1.0.0.0/120,,,,,AU,Queensland,Brisbane,,15169", + "::ffff:1.0.1.0/120,,,,,CN,Fujian,Xiamen,,", + "::ffff:1.0.2.0/119,,,,,CN,Fujian,Xiamen,,", + "::ffff:1.0.4.0/118,,,,,AU,Victoria,Melbourne,,", + "::ffff:1.0.8.0/117,,,,,CN,Guangdong,Shenzhen,,", + "::ffff:1.0.16.0/125,,,,,JP,Tokyo,Tokyo,,", + "::ffff:1.0.16.8/126,,,,,JP,Tokyo,Tokyo,,", + "::ffff:1.0.16.12/127,,,,,JP,Tokyo,Tokyo,,", + "::ffff:1.0.16.14/128,,,,,JP,Tokyo,Asagaya-minami,,", + "::ffff:1.0.16.15/128,,,,,JP,Tokyo,Tokyo,,", + "::ffff:1.0.16.16/124,,,,,JP,Tokyo,Tokyo,,", + "::ffff:1.0.16.32/123,,,,,JP,Tokyo,Tokyo,,", + "::ffff:1.0.16.64/122,,,,,JP,Tokyo,Tokyo,,", + "::ffff:1.0.16.128/121,,,,,JP,Tokyo,Tokyo,,", + "::ffff:1.0.17.0/120,,,,,JP,Tokyo,Asagaya-minami,,", + "::ffff:1.0.18.0/119,,,,,JP,Tokyo,Asagaya-minami,,", }, }, }) @@ -92,29 +92,29 @@ func TestNetworksCSVWithGeoip(t *testing.T) { ContentType: "text/csv; charset=utf-8", FirstLines: []string{ "network,name,role,site,region,country,state,city,tenant,asn", - "0.80.0.0/16,,,,,,,,Alfred,", // not covered by GeoIP - "1.0.0.0/20,infra,,,,,,,,", // not covered by GeoIP... - "1.0.0.0/24,infra,,,,AU,Queensland,Brisbane,,15169", // but covers GeoIP entries - "1.0.1.0/24,infra,,,,CN,Fujian,Xiamen,,", // but covers GeoIP entries - "1.0.2.0/23,infra,,,,CN,Fujian,Xiamen,,", // but covers GeoIP entries - "1.0.4.0/22,infra,,,,AU,Victoria,Melbourne,,", // but covers GeoIP entries - "1.0.8.0/21,infra,,,,CN,Guangdong,Shenzhen,,", // but covers GeoIP entries - "1.0.16.0/29,,,,,JP,Tokyo,Tokyo,,", - "1.0.16.8/30,,,,,JP,Tokyo,Tokyo,,", - "1.0.16.12/31,,,,,JP,Tokyo,Tokyo,,", - "1.0.16.14/32,,,,,JP,Tokyo,Asagaya-minami,,", - "1.0.16.15/32,,,,,JP,Tokyo,Tokyo,,", - "1.0.16.16/28,,,,,JP,Tokyo,Tokyo,,", - "1.0.16.32/27,,,,,JP,Tokyo,Tokyo,,", - "1.0.16.64/26,infra,,,,JP,Tokyo,Tokyo,,", // matching a GeoIP entry - "1.0.16.66/32,infra,,,,JP,Tokyo,Tokyo,Alfred,", // nested in previous one - "1.0.16.128/25,,,,,JP,Tokyo,Tokyo,,", - "1.0.17.0/24,,,,,JP,Tokyo,Asagaya-minami,,", - "1.0.18.0/23,,,,,JP,Tokyo,Asagaya-minami,,", - "1.0.20.0/22,,,,,JP,Tokyo,Asagaya-minami,,", - "1.0.24.0/21,,,,,JP,Tokyo,Asagaya-minami,,", - "1.0.32.0/19,,,,,CN,Guangdong,Shenzhen,,", - "1.0.64.0/20,,,,,JP,Hiroshima,Hiroshima,,", + "::ffff:0.80.0.0/112,,,,,,,,Alfred,", // not covered by GeoIP + "::ffff:1.0.0.0/116,infra,,,,,,,,", // not covered by GeoIP... + "::ffff:1.0.0.0/120,infra,,,,AU,Queensland,Brisbane,,15169", // but covers GeoIP entries + "::ffff:1.0.1.0/120,infra,,,,CN,Fujian,Xiamen,,", // but covers GeoIP entries + "::ffff:1.0.2.0/119,infra,,,,CN,Fujian,Xiamen,,", // but covers GeoIP entries + "::ffff:1.0.4.0/118,infra,,,,AU,Victoria,Melbourne,,", // but covers GeoIP entries + "::ffff:1.0.8.0/117,infra,,,,CN,Guangdong,Shenzhen,,", // but covers GeoIP entries + "::ffff:1.0.16.0/125,,,,,JP,Tokyo,Tokyo,,", + "::ffff:1.0.16.8/126,,,,,JP,Tokyo,Tokyo,,", + "::ffff:1.0.16.12/127,,,,,JP,Tokyo,Tokyo,,", + "::ffff:1.0.16.14/128,,,,,JP,Tokyo,Asagaya-minami,,", + "::ffff:1.0.16.15/128,,,,,JP,Tokyo,Tokyo,,", + "::ffff:1.0.16.16/124,,,,,JP,Tokyo,Tokyo,,", + "::ffff:1.0.16.32/123,,,,,JP,Tokyo,Tokyo,,", + "::ffff:1.0.16.64/122,infra,,,,JP,Tokyo,Tokyo,,", // matching a GeoIP entry + "::ffff:1.0.16.66/128,infra,,,,JP,Tokyo,Tokyo,Alfred,", // nested in previous one + "::ffff:1.0.16.128/121,,,,,JP,Tokyo,Tokyo,,", + "::ffff:1.0.17.0/120,,,,,JP,Tokyo,Asagaya-minami,,", + "::ffff:1.0.18.0/119,,,,,JP,Tokyo,Asagaya-minami,,", + "::ffff:1.0.20.0/118,,,,,JP,Tokyo,Asagaya-minami,,", + "::ffff:1.0.24.0/117,,,,,JP,Tokyo,Asagaya-minami,,", + "::ffff:1.0.32.0/115,,,,,CN,Guangdong,Shenzhen,,", + "::ffff:1.0.64.0/116,,,,,JP,Hiroshima,Hiroshima,,", }, }, }) diff --git a/orchestrator/clickhouse/source_test.go b/orchestrator/clickhouse/source_test.go index 50143b15..61c014c7 100644 --- a/orchestrator/clickhouse/source_test.go +++ b/orchestrator/clickhouse/source_test.go @@ -128,7 +128,7 @@ func TestNetworkSources(t *testing.T) { ContentType: "text/csv; charset=utf-8", FirstLines: []string{ `network,name,role,site,region,country,state,city,tenant,asn`, - `3.2.34.0/26,,amazon,,af-south-1,,,,amazon,`, + `::ffff:3.2.34.0/122,,amazon,,af-south-1,,,,amazon,`, `2600:1f14:fff:f800::/56,,route53_healthchecks,,us-west-2,,,,amazon,`, `2600:1ff2:4000::/40,,amazon,,us-west-2,,,,amazon,`, }, diff --git a/outlet/metadata/provider/static/source.go b/outlet/metadata/provider/static/source.go index 6999cbac..54c0920d 100644 --- a/outlet/metadata/provider/static/source.go +++ b/outlet/metadata/provider/static/source.go @@ -42,9 +42,9 @@ func (i exporterInfo) toExporterConfiguration() ExporterConfiguration { // initStaticExporters initializes the reconciliation map for exporter configurations // with the static prioritized data from exporters' Configuration. func (p *Provider) initStaticExporters() { - staticExportersMap := p.exporters.Load().ToMap() - staticExporters := make([]exporterInfo, 0, len(staticExportersMap)) - for subnet, config := range staticExportersMap { + staticExporters := make([]exporterInfo, 0) + staticExportersMap := p.exporters.Load() + for subnet, config := range staticExportersMap.All() { interfaces := make([]exporterInterface, 0, len(config.IfIndexes)) for ifindex, iface := range config.IfIndexes { interfaces = append(interfaces, exporterInterface{ @@ -58,7 +58,7 @@ func (p *Provider) initStaticExporters() { Exporter: provider.Exporter{ Name: config.Name, }, - ExporterSubnet: subnet, + ExporterSubnet: subnet.String(), Default: config.Default, Interfaces: interfaces, }, @@ -88,7 +88,7 @@ func (p *Provider) UpdateSource(ctx context.Context, name string, source remoted continue } // Concurrency for same Exporter config across multiple remote data sources is not handled - finalMap[exporterSubnet] = exporterData.toExporterConfiguration() + finalMap[exporterSubnet.String()] = exporterData.toExporterConfiguration() } } for _, exporterData := range p.exportersMap["static"] { @@ -98,10 +98,10 @@ func (p *Provider) UpdateSource(ctx context.Context, name string, source remoted continue } // This overrides duplicates config for an Exporter if it's also defined as static - finalMap[exporterSubnet] = exporterData.toExporterConfiguration() + finalMap[exporterSubnet.String()] = exporterData.toExporterConfiguration() } p.exportersLock.Unlock() - exporters, err := helpers.NewSubnetMap[ExporterConfiguration](finalMap) + exporters, err := helpers.NewSubnetMap(finalMap) if err != nil { return 0, errors.New("cannot create subnetmap") } diff --git a/outlet/metadata/provider/static/source_test.go b/outlet/metadata/provider/static/source_test.go index ab132fb1..1248dc05 100644 --- a/outlet/metadata/provider/static/source_test.go +++ b/outlet/metadata/provider/static/source_test.go @@ -50,7 +50,7 @@ func TestInitStaticExporters(t *testing.T) { expected["static"] = []exporterInfo{ { - ExporterSubnet: "203.0.113.0/24", + ExporterSubnet: "::ffff:203.0.113.0/120", Exporter: provider.Exporter{ Name: "something", },