johanels/akvorado
1
0
Fork 0
mirror of https://github.com/akvorado/akvorado.git synced 2025-12-11 22:14:02 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

common/helpers: convert SubnetMap to github.com/gaissmai/bart

Browse Source 
I did not benchmark it myself, but it was benchmarked here:
 https://github.com/osrg/gobgp/issues/1414#issuecomment-3067255941

Of course, no guarantee that this benchmark matches our use cases.
Moreover, SubnetMap have been optimized to avoid parsing keys all
the time.

Also, the interface is a bit nicer and it uses netip.Prefix directly.

The next step is to convert outlet/routing/provider/bmp.
This commit is contained in:
Vincent Bernat
2025-08-11 23:36:42 +02:00
parent 2692e5308c
commit 6118bb7aac
11 changed files with 603 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

207
common/helpers/subnetmap.go
View File

@@ -5,34 +5,30 @@ package helpers
import (
"fmt"
"net"
"iter"
"net/netip"
"reflect"
"regexp"
"strings"
"github.com/gaissmai/bart"
"github.com/gin-gonic/gin"
"github.com/go-viper/mapstructure/v2"
"github.com/kentik/patricia"
tree "github.com/kentik/patricia/generics_tree"
)
// SubnetMap maps subnets to values and allow to lookup by IP address.
// Internally, everything is stored as an IPv6 (using v6-mapped IPv4
// addresses).
type SubnetMap[V any] struct {
tree *tree.TreeV6[V]
table *bart.Table[V]
}
// Lookup will search for the most specific subnet matching the
// provided IP address and return the value associated with it.
func (sm *SubnetMap[V]) Lookup(ip netip.Addr) (V, bool) {
if sm == nil || sm.tree == nil {
if sm == nil || sm.table == nil {
var value V
return value, false
}
ok, value := sm.tree.FindDeepestTag(patricia.NewIPv6Address(ip.AsSlice(), 128))
return value, ok
return sm.table.Lookup(ip)
}
// LookupOrDefault calls lookup and if not found, will return the
@@ -44,82 +40,98 @@ func (sm *SubnetMap[V]) LookupOrDefault(ip netip.Addr, fallback V) V {
return fallback
}
// ToMap return a map of the tree.
// ToMap return a map of the tree. This should be used only when handling user
// configuration or for debugging. Otherwise, it is better to use Iter().
func (sm *SubnetMap[V]) ToMap() map[string]V {
output := map[string]V{}
if sm == nil || sm.tree == nil {
return output
}
iter := sm.tree.Iterate()
for iter.Next() {
output[iter.Address().String()] = iter.Tags()[0]
for prefix, value := range sm.All() {
if prefix.Addr().Is4In6() {
ipv4Addr := prefix.Addr().Unmap()
ipv4Prefix := netip.PrefixFrom(ipv4Addr, prefix.Bits()-96)
output[ipv4Prefix.String()] = value
continue
}
output[prefix.String()] = value
}
return output
}
// Set inserts the given key k into the SubnetMap, replacing any existing value if it exists.
func (sm *SubnetMap[V]) Set(k string, v V) error {
subnetK, err := SubnetMapParseKey(k)
if err != nil {
return err
// Set inserts the given key k into the SubnetMap, replacing any existing value
// if it exists. It requires an IPv6 prefix or it will panic.
func (sm *SubnetMap[V]) Set(prefix netip.Prefix, v V) {
if !prefix.Addr().Is6() {
panic(fmt.Errorf("%q is not an IPv6 subnet", prefix))
}
_, ipNet, err := net.ParseCIDR(subnetK)
if err != nil {
// Should not happen
return err
if sm.table == nil {
sm.table = &bart.Table[V]{}
}
_, bits := ipNet.Mask.Size()
if bits != 128 {
return fmt.Errorf("%q is not an IPv6 subnet", ipNet)
}
plen, _ := ipNet.Mask.Size()
sm.tree.Set(patricia.NewIPv6Address(ipNet.IP.To16(), uint(plen)), v)
return nil
sm.table.Insert(prefix, v)
}
// Update inserts the given key k into the SubnetMap, calling updateFunc with the existing value.
func (sm *SubnetMap[V]) Update(k string, v V, updateFunc tree.UpdatesFunc[V]) error {
subnetK, err := SubnetMapParseKey(k)
if err != nil {
return err
// Update inserts the given key k into the SubnetMap, calling cb with the
// existing value. It requires an IPv6 prefix or it will panic.
func (sm *SubnetMap[V]) Update(prefix netip.Prefix, cb func(V, bool) V) {
if !prefix.Addr().Is6() {
panic(fmt.Errorf("%q is not an IPv6 subnet", prefix))
}
_, ipNet, err := net.ParseCIDR(subnetK)
if err != nil {
// Should not happen
return err
if sm.table == nil {
sm.table = &bart.Table[V]{}
}
_, bits := ipNet.Mask.Size()
if bits != 128 {
return fmt.Errorf("%q is not an IPv6 subnet", ipNet)
}
plen, _ := ipNet.Mask.Size()
sm.tree.SetOrUpdate(patricia.NewIPv6Address(ipNet.IP.To16(), uint(plen)), v, updateFunc)
return nil
sm.table.Update(prefix, cb)
}
// Iter enables iteration of the SubnetMap, calling f for every entry. If f returns an error, the iteration is aborted.
func (sm *SubnetMap[V]) Iter(f func(address patricia.IPv6Address, tags [][]V) error) error {
iter := sm.tree.Iterate()
for iter.Next() {
if err := f(iter.Address(), iter.TagsFromRoot()); err != nil {
return err
// All walks the whole subnet map.
func (sm *SubnetMap[V]) All() iter.Seq2[netip.Prefix, V] {
return func(yield func(netip.Prefix, V) bool) {
if sm == nil || sm.table == nil {
return
}
sm.table.All6()(yield)
}
}
// AllMaybeSorted walks the whole subnet map in sorted order during tests but
// not when running tests.
func (sm *SubnetMap[V]) AllMaybeSorted() iter.Seq2[netip.Prefix, V] {
return func(yield func(netip.Prefix, V) bool) {
if sm == nil || sm.table == nil {
return
}
if Testing() {
sm.table.AllSorted6()(yield)
} else {
sm.table.All6()(yield)
}
}
return nil
}
// NewSubnetMap creates a subnetmap from a map. Unlike user-provided
// configuration, this function is stricter and require everything to
// be IPv6 subnets.
// Supernets returns an iterator over all supernet routes that cover the given
// prefix. The iteration order is reverse-CIDR: from longest prefix match (LPM)
// towards least-specific routes.
func (sm *SubnetMap[V]) Supernets(prefix netip.Prefix) iter.Seq2[netip.Prefix, V] {
return func(yield func(netip.Prefix, V) bool) {
if sm == nil || sm.table == nil {
return
}
sm.table.Supernets(prefix)(yield)
}
}
// NewSubnetMap creates a subnetmap from a map. It should not be used in a hot
// path as it builds the subnet from a map keyed by strings.
func NewSubnetMap[V any](from map[string]V) (*SubnetMap[V], error) {
trie := &SubnetMap[V]{tree.NewTreeV6[V]()}
sm := &SubnetMap[V]{table: &bart.Table[V]{}}
if from == nil {
return trie, nil
return sm, nil
}
for k, v := range from {
trie.Set(k, v)
key, err := SubnetMapParseKey(k)
if err != nil {
return nil, fmt.Errorf("failed to parse key %s: %w", k, err)
}
sm.Set(key, v)
}
return trie, nil
return sm, nil
}
// MustNewSubnetMap creates a subnet from a map and panic in case of a
@@ -157,9 +169,10 @@ func LooksLikeSubnetMap(v reflect.Value) (result bool) {
return
}
// SubnetMapUnmarshallerHook decodes SubnetMap and notably check that
// valid networks are provided as key. It also accepts a single value
// instead of a map for backward compatibility.
// SubnetMapUnmarshallerHook decodes SubnetMap and notably check that valid
// networks are provided as key. It also accepts a single value instead of a map
// for backward compatibility. It should not be used in hot paths as it builds
// an intermediate map.
func SubnetMapUnmarshallerHook[V any]() mapstructure.DecodeHookFunc {
return func(from, to reflect.Value) (any, error) {
if to.Type() != reflect.TypeOf(SubnetMap[V]{}) {
@@ -184,7 +197,7 @@ func SubnetMapUnmarshallerHook[V any]() mapstructure.DecodeHookFunc {
if err != nil {
return nil, fmt.Errorf("failed to parse key %s: %w", key, err)
}
output[key] = v.Interface()
output[key.String()] = v.Interface()
}
} else {
// Second case, we have a single value and we let mapstructure handles it
@@ -211,40 +224,36 @@ func SubnetMapUnmarshallerHook[V any]() mapstructure.DecodeHookFunc {
}
}
// SubnetMapParseKey decodes and validates a key used in SubnetMap from a network string.
func SubnetMapParseKey(k string) (string, error) {
var key string
if strings.Contains(k, "/") {
// Subnet
_, ipNet, err := net.ParseCIDR(k)
if err != nil {
return "", err
}
// Convert key to IPv6
ones, bits := ipNet.Mask.Size()
if bits != 32 && bits != 128 {
return "", fmt.Errorf("key %s has invalid netmask", k)
}
if bits == 32 {
key = fmt.Sprintf("::ffff:%s/%d", ipNet.IP.String(), ones+96)
} else if ipNet.IP.To4() != nil {
key = fmt.Sprintf("::ffff:%s/%d", ipNet.IP.String(), ones)
} else {
key = ipNet.String()
}
} else {
// IP
ip := net.ParseIP(k)
if ip == nil {
return "", fmt.Errorf("key %s is not a valid subnet", k)
}
if ipv4 := ip.To4(); ipv4 != nil {
key = fmt.Sprintf("::ffff:%s/128", ipv4.String())
} else {
key = fmt.Sprintf("%s/128", ip.String())
}
// PrefixTo16 converts an IPv4 prefix to an IPv4-mapped IPv6 prefix.
// IPv6 prefixes are returned as-is.
func PrefixTo16(prefix netip.Prefix) netip.Prefix {
if prefix.Addr().Is6() {
return prefix
}
return key, nil
// Convert IPv4 to IPv4-mapped IPv6
return netip.PrefixFrom(netip.AddrFrom16(prefix.Addr().As16()), prefix.Bits()+96)
}
// SubnetMapParseKey parses a prefix or an IP address into a netip.Prefix that
// can be used in a map.
func SubnetMapParseKey(k string) (netip.Prefix, error) {
// Subnet
if strings.Contains(k, "/") {
key, err := netip.ParsePrefix(k)
if err != nil {
return netip.Prefix{}, err
}
return PrefixTo16(key), nil
}
// IP address
key, err := netip.ParseAddr(k)
if err != nil {
return netip.Prefix{}, err
}
if key.Is4() {
return PrefixTo16(netip.PrefixFrom(key, 32)), nil
}
return netip.PrefixFrom(key, 128), nil
}
// MarshalYAML turns a subnet into a map that can be marshaled.

416
common/helpers/subnetmap_test.go
View File

@@ -5,6 +5,7 @@ package helpers_test
import (
"net/netip"
"slices"
"testing"
"github.com/gin-gonic/gin"
@@ -282,7 +283,7 @@ func TestSubnetMapParseKey(t *testing.T) {
} else if err == nil && tc.Error {
t.Fatal("SubnetMapParseKey() did not return an error")
}
if diff := helpers.Diff(res, tc.Expected); diff != "" {
if diff := helpers.Diff(res.String(), tc.Expected); err == nil && diff != "" {
t.Fatalf("Decode() (-got, +want):\n%s", diff)
}
})
@@ -303,3 +304,416 @@ func TestToMap(t *testing.T) {
t.Fatalf("ToMap() (-got, +want):\n%s", diff)
}
}
func TestSubnetMapLookup(t *testing.T) {
t.Run("nil", func(t *testing.T) {
var sm *helpers.SubnetMap[string]
value, ok := sm.Lookup(netip.MustParseAddr("::ffff:192.0.2.1"))
if ok || value != "" {
t.Fatalf("Lookup() == value=%q, ok=%v", value, ok)
}
})
t.Run("empty", func(t *testing.T) {
sm := &helpers.SubnetMap[string]{}
value, ok := sm.Lookup(netip.MustParseAddr("::ffff:192.0.2.1"))
if ok || value != "" {
t.Fatalf("Lookup() == value=%q, ok=%v", value, ok)
}
})
t.Run("populated", func(t *testing.T) {
sm := helpers.MustNewSubnetMap(map[string]string{
"192.0.2.0/24": "customer1",
"2001:db8::/64": "customer2",
"10.0.0.1": "specific",
})
cases := []struct {
ip string
expected string
found bool
}{
{"::ffff:192.0.2.1", "customer1", true},
{"::ffff:192.0.2.255", "customer1", true},
{"::ffff:192.0.3.1", "", false},
{"::ffff:10.0.0.1", "specific", true},
{"::ffff:10.0.0.2", "", false},
{"2001:db8::1", "customer2", true},
{"2001:db8:1::1", "", false},
}
for _, tc := range cases {
t.Run(tc.ip, func(t *testing.T) {
value, ok := sm.Lookup(netip.MustParseAddr(tc.ip))
if ok != tc.found || value != tc.expected {
t.Fatalf("Lookup(%s) = (%q, %v), want (%q, %v)", tc.ip, value, ok, tc.expected, tc.found)
}
})
}
})
}
func TestSubnetMapLookupOrDefault(t *testing.T) {
sm := helpers.MustNewSubnetMap(map[string]string{
"192.0.2.0/24": "customer1",
})
t.Run("found", func(t *testing.T) {
value := sm.LookupOrDefault(netip.MustParseAddr("::ffff:192.0.2.1"), "default")
if value != "customer1" {
t.Fatalf("LookupOrDefault() = %q, want %q", value, "customer1")
}
})
t.Run("not found", func(t *testing.T) {
value := sm.LookupOrDefault(netip.MustParseAddr("::ffff:192.0.3.1"), "default")
if value != "default" {
t.Fatalf("LookupOrDefault() = %q, want %q", value, "default")
}
})
t.Run("nil", func(t *testing.T) {
var sm *helpers.SubnetMap[string]
value := sm.LookupOrDefault(netip.MustParseAddr("::ffff:192.0.2.1"), "default")
if value != "default" {
t.Fatalf("LookupOrDefault() = %q, want %q", value, "default")
}
})
}
func TestSubnetMapSet(t *testing.T) {
sm := &helpers.SubnetMap[string]{}
t.Run("set IPv6 subnet", func(t *testing.T) {
prefix := netip.MustParsePrefix("2001:db8::/64")
sm.Set(prefix, "test-value")
value, ok := sm.Lookup(netip.MustParseAddr("2001:db8::1"))
if !ok || value != "test-value" {
t.Fatalf("Lookup() = (%q, %v), want (%q, %v)", value, ok, "test-value", true)
}
})
t.Run("set IPv4-mapped IPv6 subnet", func(t *testing.T) {
prefix := netip.MustParsePrefix("::ffff:192.0.2.0/120")
sm.Set(prefix, "ipv4-mapped")
value, ok := sm.Lookup(netip.MustParseAddr("::ffff:192.0.2.1"))
if !ok || value != "ipv4-mapped" {
t.Fatalf("Lookup() = (%q, %v), want (%q, %v)", value, ok, "ipv4-mapped", true)
}
})
t.Run("overwrite existing value", func(t *testing.T) {
prefix := netip.MustParsePrefix("2001:db8::/64")
sm.Set(prefix, "new-value")
value, ok := sm.Lookup(netip.MustParseAddr("2001:db8::1"))
if !ok || value != "new-value" {
t.Fatalf("Lookup() = (%q, %v), want (%q, %v)", value, ok, "new-value", true)
}
})
}
func TestSubnetMapSetPanic(t *testing.T) {
sm := &helpers.SubnetMap[string]{}
t.Run("panic on IPv4 subnet", func(t *testing.T) {
defer func() {
if r := recover(); r == nil {
t.Fatal("Set() should panic with IPv4")
}
}()
prefix := netip.MustParsePrefix("192.0.2.0/24")
sm.Set(prefix, "should-panic")
})
}
func TestSubnetMapUpdate(t *testing.T) {
sm := &helpers.SubnetMap[int]{}
t.Run("update new value", func(t *testing.T) {
prefix := netip.MustParsePrefix("2001:db8::/64")
sm.Update(prefix, func(old int, exists bool) int {
if !exists {
return 42
}
return old + 1
})
value, ok := sm.Lookup(netip.MustParseAddr("2001:db8::1"))
if !ok || value != 42 {
t.Fatalf("Lookup() = (%d, %v), want (%d, %v)", value, ok, 42, true)
}
})
t.Run("update existing value", func(t *testing.T) {
prefix := netip.MustParsePrefix("2001:db8::/64")
sm.Update(prefix, func(old int, exists bool) int {
if exists {
return old + 10
}
return 0
})
value, ok := sm.Lookup(netip.MustParseAddr("2001:db8::1"))
if !ok || value != 52 {
t.Fatalf("Lookup() = (%d, %v), want (%d, %v)", value, ok, 52, true)
}
})
}
func TestSubnetMapUpdatePanic(t *testing.T) {
sm := &helpers.SubnetMap[int]{}
t.Run("panic on IPv4 subnet", func(t *testing.T) {
defer func() {
if r := recover(); r == nil {
t.Fatal("Update() should panic with IPv4")
}
}()
prefix := netip.MustParsePrefix("192.0.2.0/24")
sm.Update(prefix, func(old int, exists bool) int { return 1 })
})
}
func TestSubnetMapAll(t *testing.T) {
t.Run("nil", func(t *testing.T) {
var sm *helpers.SubnetMap[string]
count := 0
for range sm.All() {
count++
}
if count != 0 {
t.Fatalf("All() count = %d, want %d", count, 0)
}
})
t.Run("empty", func(t *testing.T) {
sm := &helpers.SubnetMap[string]{}
count := 0
for range sm.All() {
count++
}
if count != 0 {
t.Fatalf("All() count = %d, want %d", count, 0)
}
})
t.Run("populated", func(t *testing.T) {
sm := helpers.MustNewSubnetMap(map[string]string{
"2001:db8::/64": "ipv6",
"::ffff:192.0.2.0/120": "ipv4-mapped",
})
items := make(map[string]string)
for prefix, value := range sm.All() {
items[prefix.String()] = value
}
expected := map[string]string{
"2001:db8::/64": "ipv6",
"::ffff:192.0.2.0/120": "ipv4-mapped",
}
if diff := helpers.Diff(items, expected); diff != "" {
t.Fatalf("All() (-got, +want):\n%s", diff)
}
})
}
func TestSubnetMapString(t *testing.T) {
t.Run("empty", func(t *testing.T) {
sm := &helpers.SubnetMap[string]{}
str := sm.String()
expected := "map[]"
if str != expected {
t.Fatalf("String() = %q, want %q", str, expected)
}
})
t.Run("populated", func(t *testing.T) {
sm := helpers.MustNewSubnetMap(map[string]string{
"192.0.2.0/24": "customer",
})
str := sm.String()
if diff := helpers.Diff(str, "map[192.0.2.0/24:customer]"); diff != "" {
t.Fatalf("String() (-got, +want):\n%s", diff)
}
})
}
func TestPrefixTo16(t *testing.T) {
cases := []struct {
name string
input string
expected string
}{
{
name: "IPv4 prefix",
input: "192.0.2.0/24",
expected: "::ffff:192.0.2.0/120",
},
{
name: "IPv4 host",
input: "192.0.2.1/32",
expected: "::ffff:192.0.2.1/128",
},
{
name: "IPv6 prefix unchanged",
input: "2001:db8::/64",
expected: "2001:db8::/64",
},
{
name: "IPv6 host unchanged",
input: "2001:db8::1/128",
expected: "2001:db8::1/128",
},
{
name: "IPv4-mapped IPv6 unchanged",
input: "::ffff:192.0.2.0/120",
expected: "::ffff:192.0.2.0/120",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
prefix := netip.MustParsePrefix(tc.input)
result := helpers.PrefixTo16(prefix)
if result.String() != tc.expected {
t.Fatalf("PrefixTo16(%s) = %s, want %s", tc.input, result, tc.expected)
}
})
}
}
func TestSubnetMapSupernets(t *testing.T) {
t.Run("nil", func(t *testing.T) {
var sm *helpers.SubnetMap[string]
count := 0
for range sm.Supernets(netip.MustParsePrefix("::ffff:192.0.2.1/128")) {
count++
}
if count != 0 {
t.Fatalf("Supernets() count = %d, want %d", count, 0)
}
})
t.Run("empty", func(t *testing.T) {
sm := &helpers.SubnetMap[string]{}
count := 0
for range sm.Supernets(netip.MustParsePrefix("::ffff:192.0.2.1/128")) {
count++
}
if count != 0 {
t.Fatalf("Supernets() count = %d, want %d", count, 0)
}
})
t.Run("hierarchical supernets", func(t *testing.T) {
sm := helpers.MustNewSubnetMap(map[string]string{
"192.0.0.0/16": "region",
"192.0.2.0/24": "site",
"192.0.2.0/28": "rack",
})
// Query for 192.0.2.1/32 should find supernets in reverse-CIDR order
var results []string
var prefixes []string
for prefix, value := range sm.Supernets(netip.MustParsePrefix("::ffff:192.0.2.1/128")) {
results = append(results, value)
prefixes = append(prefixes, prefix.String())
}
expectedValues := []string{"rack", "site", "region"}
expectedPrefixes := []string{"::ffff:192.0.2.0/124", "::ffff:192.0.2.0/120", "::ffff:192.0.0.0/112"}
if diff := helpers.Diff(results, expectedValues); diff != "" {
t.Errorf("Supernets() values (-got, +want):\n%s", diff)
}
if diff := helpers.Diff(prefixes, expectedPrefixes); diff != "" {
t.Errorf("Supernets() prefixes (-got, +want):\n%s", diff)
}
count := 0
for range sm.Supernets(netip.MustParsePrefix("::ffff:10.0.0.1/128")) {
count++
}
if count != 0 {
t.Fatalf("Supernets() count = %d, want %d", count, 0)
}
})
}
func TestSubnetMapAllMaybeSorted(t *testing.T) {
t.Run("nil", func(t *testing.T) {
var sm *helpers.SubnetMap[string]
count := 0
for range sm.AllMaybeSorted() {
count++
}
if count != 0 {
t.Fatalf("AllMaybeSorted() count = %d, want %d", count, 0)
}
})
t.Run("empty", func(t *testing.T) {
sm := &helpers.SubnetMap[string]{}
count := 0
for range sm.AllMaybeSorted() {
count++
}
if count != 0 {
t.Fatalf("AllMaybeSorted() count = %d, want %d", count, 0)
}
})
t.Run("sorted vs unsorted", func(t *testing.T) {
sm := helpers.MustNewSubnetMap(map[string]string{
"192.0.2.0/28": "rack",
"192.0.0.0/16": "region",
"192.0.2.0/24": "site",
"2001:db8:1::/64": "ipv6-site1",
"2001:db8::/32": "ipv6-region",
"2001:db8:2::/64": "ipv6-site2",
})
// Collect results from All() (potentially unsorted)
var allPrefixes []string
for prefix := range sm.All() {
allPrefixes = append(allPrefixes, prefix.String())
}
// Collect results from AllMaybeSorted() (sorted during tests)
var sortedPrefixes []string
for prefix := range sm.AllMaybeSorted() {
sortedPrefixes = append(sortedPrefixes, prefix.String())
}
// Expected sorted order: IPv6 addresses first (sorted), then IPv4-mapped IPv6 (sorted)
expectedPrefixes := []string{
"::ffff:192.0.0.0/112",
"::ffff:192.0.2.0/120",
"::ffff:192.0.2.0/124",
"2001:db8::/32",
"2001:db8:1::/64",
"2001:db8:2::/64",
}
// AllMaybeSorted() should be sorted during tests
if diff := helpers.Diff(sortedPrefixes, expectedPrefixes); diff != "" {
t.Errorf("AllMaybeSorted() prefixes (-got, +want):\n%s", diff)
}
// All() and AllMaybeSorted() should contain the same elements (but potentially different order)
if len(allPrefixes) != len(sortedPrefixes) {
t.Errorf("All() returned %d prefixes, AllMaybeSorted() returned %d", len(allPrefixes), len(sortedPrefixes))
}
// Verify that All() and AllMaybeSorted() return different orders (otherwise test is meaningless)
if slices.Equal(allPrefixes, sortedPrefixes) {
t.Skip("All() and AllMaybeSorted() returned identical order")
}
})
}

1
common/helpers/tests_diff.go
View File

@@ -30,6 +30,7 @@ func defaultPrettyFormatters() map[reflect.Type]any {
result := map[reflect.Type]any{
reflect.TypeOf(net.IP{}): fmt.Sprint,
reflect.TypeOf(netip.Addr{}): fmt.Sprint,
reflect.TypeOf(netip.Prefix{}): fmt.Sprint,
reflect.TypeOf(time.Time{}): fmt.Sprint,
reflect.TypeOf(SubnetMap[string]{}): fmt.Sprint,
reflect.TypeOf(SubnetMap[uint]{}): fmt.Sprint,

1
go.mod
View File

@@ -16,6 +16,7 @@ require (
github.com/eapache/go-resiliency v1.7.0
github.com/expr-lang/expr v1.17.5
github.com/fsnotify/fsnotify v1.9.0
github.com/gaissmai/bart v0.23.0
github.com/gin-gonic/gin v1.10.1
github.com/glebarez/sqlite v1.11.0
github.com/go-playground/validator/v10 v10.20.0

2
go.sum
View File

@@ -112,6 +112,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
github.com/gaissmai/bart v0.23.0 h1:ct+78nySK5MaO+citQAUeef7QZ0ApXM3b+AYuCZYGIk=
github.com/gaissmai/bart v0.23.0/go.mod h1:RpLtt3lWq1BoRz3AAyDAJ7jhLWBkYhVCfi+ximB2t68=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U=

2
orchestrator/clickhouse/http_test.go
View File

@@ -71,7 +71,7 @@ func TestHTTPEndpoints(t *testing.T) {
ContentType: "text/csv; charset=utf-8",
FirstLines: []string{
`network,name,role,site,region,country,state,city,tenant,asn`,
`192.0.2.0/24,infra,,,,,,,,`,
`::ffff:192.0.2.0/120,infra,,,,,,,,`,
},
}, {
URL: "/api/v0/orchestrator/clickhouse/custom_dict_none.csv",

71
orchestrator/clickhouse/networks.go
View File

@@ -13,8 +13,6 @@ import (
"strconv"
"time"
"github.com/kentik/patricia"
"akvorado/common/helpers"
"akvorado/common/schema"
"akvorado/orchestrator/geoip"
@@ -58,30 +56,30 @@ func (c *Component) networksCSVRefresher() {
// Add content of all geoip databases
err := c.d.GeoIP.IterASNDatabases(func(prefix netip.Prefix, data geoip.ASNInfo) error {
subV6Str, err := helpers.SubnetMapParseKey(prefix.String())
if err != nil {
return err
}
subV6Prefix := helpers.PrefixTo16(prefix)
attrs := NetworkAttributes{
ASN: data.ASNumber,
}
return networks.Update(subV6Str, attrs, overrideNetworkAttrs(attrs))
networks.Update(subV6Prefix, func(existing NetworkAttributes, _ bool) NetworkAttributes {
return mergeNetworkAttrs(existing, attrs)
})
return nil
})
if err != nil {
c.r.Err(err).Msg("unable to iter over ASN databases")
return
}
err = c.d.GeoIP.IterGeoDatabases(func(prefix netip.Prefix, data geoip.GeoInfo) error {
subV6Str, err := helpers.SubnetMapParseKey(prefix.String())
if err != nil {
return err
}
subV6Prefix := helpers.PrefixTo16(prefix)
attrs := NetworkAttributes{
State: data.State,
Country: data.Country,
City: data.City,
}
return networks.Update(subV6Str, attrs, overrideNetworkAttrs(attrs))
networks.Update(subV6Prefix, func(existing NetworkAttributes, _ bool) NetworkAttributes {
return mergeNetworkAttrs(existing, attrs)
})
return nil
})
if err != nil {
c.r.Err(err).Msg("unable to iter over geo databases")
@@ -93,13 +91,10 @@ func (c *Component) networksCSVRefresher() {
defer c.networkSourcesLock.RUnlock()
for _, networkList := range c.networkSources {
for _, val := range networkList {
if err := networks.Update(
val.Prefix.String(),
val.NetworkAttributes,
overrideNetworkAttrs(val.NetworkAttributes),
); err != nil {
return err
}
subV6Prefix := helpers.PrefixTo16(val.Prefix)
networks.Update(subV6Prefix, func(existing NetworkAttributes, _ bool) NetworkAttributes {
return mergeNetworkAttrs(existing, val.NetworkAttributes)
})
}
}
return nil
@@ -110,16 +105,10 @@ func (c *Component) networksCSVRefresher() {
// Add static network sources
if c.config.Networks != nil {
// Update networks with static network source
err := c.config.Networks.Iter(func(address patricia.IPv6Address, tags [][]NetworkAttributes) error {
return networks.Update(
address.String(),
tags[len(tags)-1][0],
overrideNetworkAttrs(tags[len(tags)-1][0]),
)
})
if err != nil {
c.r.Err(err).Msg("unable to update with static network sources")
return
for prefix, attrs := range c.config.Networks.All() {
networks.Update(prefix, func(existing NetworkAttributes, _ bool) NetworkAttributes {
return mergeNetworkAttrs(existing, attrs)
})
}
}
@@ -142,12 +131,13 @@ func (c *Component) networksCSVRefresher() {
gzipWriter := gzip.NewWriter(tmpfile)
csvWriter := csv.NewWriter(gzipWriter)
csvWriter.Write([]string{"network", "name", "role", "site", "region", "country", "state", "city", "tenant", "asn"})
networks.Iter(func(address patricia.IPv6Address, tags [][]NetworkAttributes) error {
current := NetworkAttributes{}
for _, nodeTags := range tags {
for _, tag := range nodeTags {
current = mergeNetworkAttrs(current, tag)
}
for prefix, leafAttrs := range networks.AllMaybeSorted() {
// Merge attributes from root to leaf for hierarchical inheritance.
// Supernets() returns in reverse-CIDR order (LPM to root), so we
// merge in that order.
current := leafAttrs
for _, attrs := range networks.Supernets(prefix) {
current = mergeNetworkAttrs(attrs, current)
}
var asnVal string
@@ -155,7 +145,7 @@ func (c *Component) networksCSVRefresher() {
asnVal = strconv.Itoa(int(current.ASN))
}
csvWriter.Write([]string{
address.String(),
prefix.String(),
current.Name,
current.Role,
current.Site,
@@ -166,8 +156,7 @@ func (c *Component) networksCSVRefresher() {
current.Tenant,
asnVal,
})
return nil
})
}
csvWriter.Flush()
gzipWriter.Close()
@@ -195,12 +184,6 @@ func (c *Component) networksCSVRefresher() {
}
}
func overrideNetworkAttrs(newAttrs NetworkAttributes) func(existing NetworkAttributes) NetworkAttributes {
return func(existing NetworkAttributes) NetworkAttributes {
return mergeNetworkAttrs(existing, newAttrs)
}
}
func mergeNetworkAttrs(existing, newAttrs NetworkAttributes) NetworkAttributes {
if newAttrs.ASN != 0 {
existing.ASN = newAttrs.ASN

78
orchestrator/clickhouse/networks_test.go
View File

@@ -44,22 +44,22 @@ func TestNetworksCSVWithGeoip(t *testing.T) {
ContentType: "text/csv; charset=utf-8",
FirstLines: []string{
"network,name,role,site,region,country,state,city,tenant,asn",
"1.0.0.0/24,,,,,AU,Queensland,Brisbane,,15169",
"1.0.1.0/24,,,,,CN,Fujian,Xiamen,,",
"1.0.2.0/23,,,,,CN,Fujian,Xiamen,,",
"1.0.4.0/22,,,,,AU,Victoria,Melbourne,,",
"1.0.8.0/21,,,,,CN,Guangdong,Shenzhen,,",
"1.0.16.0/29,,,,,JP,Tokyo,Tokyo,,",
"1.0.16.8/30,,,,,JP,Tokyo,Tokyo,,",
"1.0.16.12/31,,,,,JP,Tokyo,Tokyo,,",
"1.0.16.14/32,,,,,JP,Tokyo,Asagaya-minami,,",
"1.0.16.15/32,,,,,JP,Tokyo,Tokyo,,",
"1.0.16.16/28,,,,,JP,Tokyo,Tokyo,,",
"1.0.16.32/27,,,,,JP,Tokyo,Tokyo,,",
"1.0.16.64/26,,,,,JP,Tokyo,Tokyo,,",
"1.0.16.128/25,,,,,JP,Tokyo,Tokyo,,",
"1.0.17.0/24,,,,,JP,Tokyo,Asagaya-minami,,",
"1.0.18.0/23,,,,,JP,Tokyo,Asagaya-minami,,",
"::ffff:1.0.0.0/120,,,,,AU,Queensland,Brisbane,,15169",
"::ffff:1.0.1.0/120,,,,,CN,Fujian,Xiamen,,",
"::ffff:1.0.2.0/119,,,,,CN,Fujian,Xiamen,,",
"::ffff:1.0.4.0/118,,,,,AU,Victoria,Melbourne,,",
"::ffff:1.0.8.0/117,,,,,CN,Guangdong,Shenzhen,,",
"::ffff:1.0.16.0/125,,,,,JP,Tokyo,Tokyo,,",
"::ffff:1.0.16.8/126,,,,,JP,Tokyo,Tokyo,,",
"::ffff:1.0.16.12/127,,,,,JP,Tokyo,Tokyo,,",
"::ffff:1.0.16.14/128,,,,,JP,Tokyo,Asagaya-minami,,",
"::ffff:1.0.16.15/128,,,,,JP,Tokyo,Tokyo,,",
"::ffff:1.0.16.16/124,,,,,JP,Tokyo,Tokyo,,",
"::ffff:1.0.16.32/123,,,,,JP,Tokyo,Tokyo,,",
"::ffff:1.0.16.64/122,,,,,JP,Tokyo,Tokyo,,",
"::ffff:1.0.16.128/121,,,,,JP,Tokyo,Tokyo,,",
"::ffff:1.0.17.0/120,,,,,JP,Tokyo,Asagaya-minami,,",
"::ffff:1.0.18.0/119,,,,,JP,Tokyo,Asagaya-minami,,",
},
},
})
@@ -92,29 +92,29 @@ func TestNetworksCSVWithGeoip(t *testing.T) {
ContentType: "text/csv; charset=utf-8",
FirstLines: []string{
"network,name,role,site,region,country,state,city,tenant,asn",
"0.80.0.0/16,,,,,,,,Alfred,", // not covered by GeoIP
"1.0.0.0/20,infra,,,,,,,,", // not covered by GeoIP...
"1.0.0.0/24,infra,,,,AU,Queensland,Brisbane,,15169", // but covers GeoIP entries
"1.0.1.0/24,infra,,,,CN,Fujian,Xiamen,,", // but covers GeoIP entries
"1.0.2.0/23,infra,,,,CN,Fujian,Xiamen,,", // but covers GeoIP entries
"1.0.4.0/22,infra,,,,AU,Victoria,Melbourne,,", // but covers GeoIP entries
"1.0.8.0/21,infra,,,,CN,Guangdong,Shenzhen,,", // but covers GeoIP entries
"1.0.16.0/29,,,,,JP,Tokyo,Tokyo,,",
"1.0.16.8/30,,,,,JP,Tokyo,Tokyo,,",
"1.0.16.12/31,,,,,JP,Tokyo,Tokyo,,",
"1.0.16.14/32,,,,,JP,Tokyo,Asagaya-minami,,",
"1.0.16.15/32,,,,,JP,Tokyo,Tokyo,,",
"1.0.16.16/28,,,,,JP,Tokyo,Tokyo,,",
"1.0.16.32/27,,,,,JP,Tokyo,Tokyo,,",
"1.0.16.64/26,infra,,,,JP,Tokyo,Tokyo,,", // matching a GeoIP entry
"1.0.16.66/32,infra,,,,JP,Tokyo,Tokyo,Alfred,", // nested in previous one
"1.0.16.128/25,,,,,JP,Tokyo,Tokyo,,",
"1.0.17.0/24,,,,,JP,Tokyo,Asagaya-minami,,",
"1.0.18.0/23,,,,,JP,Tokyo,Asagaya-minami,,",
"1.0.20.0/22,,,,,JP,Tokyo,Asagaya-minami,,",
"1.0.24.0/21,,,,,JP,Tokyo,Asagaya-minami,,",
"1.0.32.0/19,,,,,CN,Guangdong,Shenzhen,,",
"1.0.64.0/20,,,,,JP,Hiroshima,Hiroshima,,",
"::ffff:0.80.0.0/112,,,,,,,,Alfred,", // not covered by GeoIP
"::ffff:1.0.0.0/116,infra,,,,,,,,", // not covered by GeoIP...
"::ffff:1.0.0.0/120,infra,,,,AU,Queensland,Brisbane,,15169", // but covers GeoIP entries
"::ffff:1.0.1.0/120,infra,,,,CN,Fujian,Xiamen,,", // but covers GeoIP entries
"::ffff:1.0.2.0/119,infra,,,,CN,Fujian,Xiamen,,", // but covers GeoIP entries
"::ffff:1.0.4.0/118,infra,,,,AU,Victoria,Melbourne,,", // but covers GeoIP entries
"::ffff:1.0.8.0/117,infra,,,,CN,Guangdong,Shenzhen,,", // but covers GeoIP entries
"::ffff:1.0.16.0/125,,,,,JP,Tokyo,Tokyo,,",
"::ffff:1.0.16.8/126,,,,,JP,Tokyo,Tokyo,,",
"::ffff:1.0.16.12/127,,,,,JP,Tokyo,Tokyo,,",
"::ffff:1.0.16.14/128,,,,,JP,Tokyo,Asagaya-minami,,",
"::ffff:1.0.16.15/128,,,,,JP,Tokyo,Tokyo,,",
"::ffff:1.0.16.16/124,,,,,JP,Tokyo,Tokyo,,",
"::ffff:1.0.16.32/123,,,,,JP,Tokyo,Tokyo,,",
"::ffff:1.0.16.64/122,infra,,,,JP,Tokyo,Tokyo,,", // matching a GeoIP entry
"::ffff:1.0.16.66/128,infra,,,,JP,Tokyo,Tokyo,Alfred,", // nested in previous one
"::ffff:1.0.16.128/121,,,,,JP,Tokyo,Tokyo,,",
"::ffff:1.0.17.0/120,,,,,JP,Tokyo,Asagaya-minami,,",
"::ffff:1.0.18.0/119,,,,,JP,Tokyo,Asagaya-minami,,",
"::ffff:1.0.20.0/118,,,,,JP,Tokyo,Asagaya-minami,,",
"::ffff:1.0.24.0/117,,,,,JP,Tokyo,Asagaya-minami,,",
"::ffff:1.0.32.0/115,,,,,CN,Guangdong,Shenzhen,,",
"::ffff:1.0.64.0/116,,,,,JP,Hiroshima,Hiroshima,,",
},
},
})

2
orchestrator/clickhouse/source_test.go
View File

@@ -128,7 +128,7 @@ func TestNetworkSources(t *testing.T) {
ContentType: "text/csv; charset=utf-8",
FirstLines: []string{
`network,name,role,site,region,country,state,city,tenant,asn`,
`3.2.34.0/26,,amazon,,af-south-1,,,,amazon,`,
`::ffff:3.2.34.0/122,,amazon,,af-south-1,,,,amazon,`,
`2600:1f14:fff:f800::/56,,route53_healthchecks,,us-west-2,,,,amazon,`,
`2600:1ff2:4000::/40,,amazon,,us-west-2,,,,amazon,`,
},

14
outlet/metadata/provider/static/source.go
View File

@@ -42,9 +42,9 @@ func (i exporterInfo) toExporterConfiguration() ExporterConfiguration {
// initStaticExporters initializes the reconciliation map for exporter configurations
// with the static prioritized data from exporters' Configuration.
func (p *Provider) initStaticExporters() {
staticExportersMap := p.exporters.Load().ToMap()
staticExporters := make([]exporterInfo, 0, len(staticExportersMap))
for subnet, config := range staticExportersMap {
staticExporters := make([]exporterInfo, 0)
staticExportersMap := p.exporters.Load()
for subnet, config := range staticExportersMap.All() {
interfaces := make([]exporterInterface, 0, len(config.IfIndexes))
for ifindex, iface := range config.IfIndexes {
interfaces = append(interfaces, exporterInterface{
@@ -58,7 +58,7 @@ func (p *Provider) initStaticExporters() {
Exporter: provider.Exporter{
Name: config.Name,
},
ExporterSubnet: subnet,
ExporterSubnet: subnet.String(),
Default: config.Default,
Interfaces: interfaces,
},
@@ -88,7 +88,7 @@ func (p *Provider) UpdateSource(ctx context.Context, name string, source remoted
continue
}
// Concurrency for same Exporter config across multiple remote data sources is not handled
finalMap[exporterSubnet] = exporterData.toExporterConfiguration()
finalMap[exporterSubnet.String()] = exporterData.toExporterConfiguration()
}
}
for _, exporterData := range p.exportersMap["static"] {
@@ -98,10 +98,10 @@ func (p *Provider) UpdateSource(ctx context.Context, name string, source remoted
continue
}
// This overrides duplicates config for an Exporter if it's also defined as static
finalMap[exporterSubnet] = exporterData.toExporterConfiguration()
finalMap[exporterSubnet.String()] = exporterData.toExporterConfiguration()
}
p.exportersLock.Unlock()
exporters, err := helpers.NewSubnetMap[ExporterConfiguration](finalMap)
exporters, err := helpers.NewSubnetMap(finalMap)
if err != nil {
return 0, errors.New("cannot create subnetmap")
}

2
outlet/metadata/provider/static/source_test.go
View File

@@ -50,7 +50,7 @@ func TestInitStaticExporters(t *testing.T) {
expected["static"] = []exporterInfo{
{
ExporterSubnet: "203.0.113.0/24",
ExporterSubnet: "::ffff:203.0.113.0/120",
Exporter: provider.Exporter{
Name: "something",
},