inlet/snmp: accept subnets for communities

Also deprecate `default-community`.

common/helpers/subnetmap.go

@@ -33,6 +33,37 @@ func (sm *SubnetMap[V]) Lookup(ip net.IP) (V, bool) {
 	return value, ok
 }
 
+// NewSubnetMap creates a subnetmap from a map. Unlike user-provided
+// configuration, this function is stricter and require everything to
+// be IPv6 subnets.
+func NewSubnetMap[V any](from map[string]V) (*SubnetMap[V], error) {
+	trie := tree.NewTreeV6[V]()
+	for k, v := range from {
+		_, ipNet, err := net.ParseCIDR(k)
+		if err != nil {
+			// Should not happen
+			return nil, err
+		}
+		_, bits := ipNet.Mask.Size()
+		if bits != 128 {
+			return nil, fmt.Errorf("%q is not an IPv6 subnet", ipNet)
+		}
+		plen, _ := ipNet.Mask.Size()
+		trie.Set(patricia.NewIPv6Address(ipNet.IP.To16(), uint(plen)), v)
+	}
+	return &SubnetMap[V]{trie}, nil
+}
+
+// MustNewSubnetMap creates a subnet from a map and panic in case of a
+// problem. This should only be used with tests.
+func MustNewSubnetMap[V any](from map[string]V) *SubnetMap[V] {
+	trie, err := NewSubnetMap(from)
+	if err != nil {
+		panic(err)
+	}
+	return trie
+}
+
 // SubnetMapUnmarshallerHook decodes SubnetMap and notably check that
 // valid networks are provided as key. It also accepts a single value
 // instead of a map for backward compatibility.
@@ -106,18 +137,13 @@ func SubnetMapUnmarshallerHook[V any]() mapstructure.DecodeHookFunc {
 		if err := intermediateDecoder.Decode(output); err != nil {
 			return nil, fmt.Errorf("unable to decode %q: %w", reflect.TypeOf(zero).Name(), err)
 		}
-		trie := tree.NewTreeV6[V]()
+		trie, err := NewSubnetMap[V](intermediate)
-		for k, v := range intermediate {
+		if err != nil {
-			_, ipNet, err := net.ParseCIDR(k)
+			// Should not happen
-			if err != nil {
+			return nil, err
-				// Should not happen
-				return nil, err
-			}
-			plen, _ := ipNet.Mask.Size()
-			trie.Set(patricia.NewIPv6Address(ipNet.IP.To16(), uint(plen)), v)
 		}
 
-		return SubnetMap[V]{trie}, nil
+		return trie, nil
 	}
 }
 
@@ -132,3 +158,11 @@ func (sm SubnetMap[V]) MarshalYAML() (interface{}, error) {
 	}
 	return output, nil
 }
+
+func (sm SubnetMap[V]) String() string {
+	out, err := sm.MarshalYAML()
+	if err != nil {
+		return "SubnetMap???"
+	}
+	return fmt.Sprintf("%v", out)
+}

common/helpers/tests.go

@@ -29,7 +29,8 @@ var prettyC = pretty.Config{
 	SkipZeroFields:    true,
 	IncludeUnexported: false,
 	Formatter: map[reflect.Type]interface{}{
-		reflect.TypeOf(net.IP{}): fmt.Sprint,
+		reflect.TypeOf(net.IP{}):            fmt.Sprint,
+		reflect.TypeOf(SubnetMap[string]{}): fmt.Sprint,
 	},
 }
 

console/data/docs/00-intro.md

@@ -40,8 +40,8 @@ documentation.
 
 - `clickhouse` → `asns` to give names to your internal AS numbers
 - `clickhouse` → `networks` to attach attributes to your networks
-- `inlet` → `snmp` → `default-community` to set the default community
+- `inlet` → `snmp` → `communities` to set the communities to use for
-  to use for SNMP queries
+  SNMP queries
 - `inlet` → `core` → `exporter-classifiers` to define rules to attach
   attributes to your exporters
 - `inlet` → `core` → `interface-classifiers` to define rules to attach

console/data/docs/02-configuration.md

@@ -255,9 +255,10 @@ continuously the exporters. The following keys are accepted:
   about to expire or need an update
 - `cache-persist-file` tells where to store cached data on shutdown and
   read them back on startup
-- `default-community` tells which community to use when polling exporters
+- `communities` is a map from a subnets to the community to use for
-- `communities` is a map from a exporter IP address to the community to
+  exporters in the provided subnet. Use `::/0` to set the default
-  use for a exporter, overriding the default value set above,
+  value. Alternatively, it also accepts a string to use for all
+  exporters.
 - `poller-retries` is the number of retries on unsuccessful SNMP requests.
 - `poller-timeout` tells how much time should the poller wait for an answer.
 - `workers` tell how many workers to spawn to handle SNMP polling.

console/data/docs/99-changelog.md

@@ -23,6 +23,7 @@ details.
 - 🩹 *orchestrator*: fix `SrcCountry`/`DstCountry` columns in aggregated tables [PR #61][]
 - 🌱 *inlet*: `inlet.geoip.country-database` has been renamed to `inlet.geoip.geo-database`
 - 🌱 *inlet*: add counters for GeoIP database hit/miss
+- 🌱 *inlet*: `inlet.snmp.communities` accepts subnets as keys, `inlet.snmp.default-community` is now deprecated
 - 🌱 *docker-compose*: disable healthcheck for the conntrack-fixer container
 
 [PR #61]: https://github.com/vincentbernat/akvorado/pull/61

inlet/geoip/config.go

@@ -38,13 +38,13 @@ func ConfigurationUnmarshallerHook() mapstructure.DecodeHookFunc {
 		}
 
 		// country-database → geo-database
-		var countryKey *reflect.Value
+		var countryKey, geoKey *reflect.Value
-		var geoKey *reflect.Value
+		fromMap := from.MapKeys()
-		for _, k := range from.MapKeys() {
+		for i, k := range from.MapKeys() {
 			if helpers.MapStructureMatchName(k.String(), "CountryDatabase") {
-				countryKey = &k
+				countryKey = &fromMap[i]
 			} else if helpers.MapStructureMatchName(k.String(), "GeoDatabase") {
-				geoKey = &k
+				geoKey = &fromMap[i]
 			}
 		}
 		if countryKey != nil && geoKey != nil {

inlet/snmp/config.go

@@ -4,7 +4,12 @@
 package snmp
 
 import (
+	"reflect"
 	"time"
+
+	"akvorado/common/helpers"
+
+	"github.com/mitchellh/mapstructure"
 )
 
 // Configuration describes the configuration for the SNMP client
@@ -17,10 +22,8 @@ type Configuration struct {
 	CacheCheckInterval time.Duration `validate:"ltefield=CacheRefresh"`
 	// CachePersist defines a file to store cache and survive restarts
 	CachePersistFile string
-	// DefaultCommunity is the default SNMP community to use
-	DefaultCommunity string `validate:"required"`
 	// Communities is a mapping from exporter IPs to communities
-	Communities map[string]string
+	Communities *helpers.SubnetMap[string]
 	// PollerRetries tell how many time a poller should retry before giving up
 	PollerRetries int `validate:"min=0"`
 	// PollerTimeout tell how much time a poller should wait for an answer
@@ -33,16 +36,61 @@ type Configuration struct {
 
 // DefaultConfiguration represents the default configuration for the SNMP client.
 func DefaultConfiguration() Configuration {
+	communities, err := helpers.NewSubnetMap(map[string]string{
+		"::/0": "public",
+	})
+	if err != nil {
+		panic(err)
+	}
 	return Configuration{
 		CacheDuration:      30 * time.Minute,
 		CacheRefresh:       time.Hour,
 		CacheCheckInterval: 2 * time.Minute,
 		CachePersistFile:   "",
-		DefaultCommunity:   "public",
+		Communities:        communities,
-		Communities:        map[string]string{},
 		PollerRetries:      1,
 		PollerTimeout:      time.Second,
 		PollerCoalesce:     10,
 		Workers:            1,
 	}
 }
+
+// ConfigurationUnmarshallerHook normalize SNMP configuration:
+//  - append default-community to communities (as ::/0)
+func ConfigurationUnmarshallerHook() mapstructure.DecodeHookFunc {
+	return func(from, to reflect.Value) (interface{}, error) {
+		if from.Kind() != reflect.Map || from.IsNil() || from.Type().Key().Kind() != reflect.String || to.Type() != reflect.TypeOf(Configuration{}) {
+			return from.Interface(), nil
+		}
+
+		// default-community → communities
+		var defaultKey, mapKey *reflect.Value
+		fromMap := from.MapKeys()
+		for i, k := range fromMap {
+			if helpers.MapStructureMatchName(k.String(), "DefaultCommunity") {
+				defaultKey = &fromMap[i]
+			} else if helpers.MapStructureMatchName(k.String(), "Communities") {
+				mapKey = &fromMap[i]
+			}
+		}
+		if defaultKey != nil {
+			if mapKey == nil {
+				from.SetMapIndex(reflect.ValueOf("communities"), from.MapIndex(*defaultKey))
+			} else {
+				communities := from.MapIndex(*mapKey)
+				if communities.Kind() == reflect.Interface {
+					communities = communities.Elem()
+				}
+				communities.SetMapIndex(reflect.ValueOf("::/0"), from.MapIndex(*defaultKey))
+			}
+			from.SetMapIndex(*defaultKey, reflect.Value{})
+		}
+
+		return from.Interface(), nil
+	}
+}
+
+func init() {
+	helpers.AddMapstructureUnmarshallerHook(ConfigurationUnmarshallerHook())
+	helpers.AddMapstructureUnmarshallerHook(helpers.SubnetMapUnmarshallerHook[string]())
+}

inlet/snmp/config_test.go

@@ -5,8 +5,12 @@ package snmp
 
 import (
 	"testing"
+	"time"
 
 	"akvorado/common/helpers"
+
+	"github.com/gin-gonic/gin"
+	"github.com/mitchellh/mapstructure"
 )
 
 func TestDefaultConfiguration(t *testing.T) {
@@ -14,3 +18,84 @@ func TestDefaultConfiguration(t *testing.T) {
 		t.Fatalf("validate.Struct() error:\n%+v", err)
 	}
 }
+
+func TestConfigurationUnmarshallerHook(t *testing.T) {
+	cases := []struct {
+		Description string
+		Input       gin.H
+		Output      Configuration
+	}{
+		{
+			Description: "nil",
+			Input:       nil,
+		}, {
+			Description: "empty",
+			Input:       gin.H{},
+		}, {
+			Description: "no communities, no default community",
+			Input: gin.H{
+				"cache-refresh":  "10s",
+				"poller-retries": 10,
+			},
+			Output: Configuration{
+				CacheRefresh:  10 * time.Second,
+				PollerRetries: 10,
+			},
+		}, {
+			Description: "communities, no default community",
+			Input: gin.H{
+				"communities": gin.H{
+					"203.0.113.0/25":   "public",
+					"203.0.113.128/25": "private",
+				},
+			},
+			Output: Configuration{
+				Communities: helpers.MustNewSubnetMap(map[string]string{
+					"::ffff:203.0.113.0/121":   "public",
+					"::ffff:203.0.113.128/121": "private",
+				}),
+			},
+		}, {
+			Description: "no communities, default community",
+			Input: gin.H{
+				"default-community": "private",
+			},
+			Output: Configuration{
+				Communities: helpers.MustNewSubnetMap(map[string]string{
+					"::/0": "private",
+				}),
+			},
+		}, {
+			Description: "communities, default community",
+			Input: gin.H{
+				"default-community": "private",
+				"communities": gin.H{
+					"203.0.113.0/25":   "public",
+					"203.0.113.128/25": "private",
+				},
+			},
+			Output: Configuration{
+				Communities: helpers.MustNewSubnetMap(map[string]string{
+					"::/0":                     "private",
+					"::ffff:203.0.113.0/121":   "public",
+					"::ffff:203.0.113.128/121": "private",
+				}),
+			},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.Description, func(t *testing.T) {
+			var got Configuration
+			decoder, err := mapstructure.NewDecoder(helpers.GetMapStructureDecoderConfig(&got))
+			if err != nil {
+				t.Fatalf("NewDecoder() error:\n%+v", err)
+			}
+			err = decoder.Decode(tc.Input)
+			if err != nil {
+				t.Fatalf("Decode() error:\n%+v", err)
+			} else if diff := helpers.Diff(got, tc.Output); diff != "" {
+				t.Fatalf("Decode() (-got, +want):\n%s", diff)
+			}
+		})
+	}
+}

inlet/snmp/root.go

@@ -9,6 +9,7 @@ package snmp
 import (
 	"errors"
 	"fmt"
+	"net"
 	"strconv"
 	"sync"
 	"time"
@@ -282,9 +283,9 @@ func (c *Component) dispatchIncomingRequest(request lookupRequest) {
 // pollerIncomingRequest handles an incoming request to the poller. It
 // uses a breaker to avoid pushing working on non-responsive exporters.
 func (c *Component) pollerIncomingRequest(request lookupRequest) {
-	community, ok := c.config.Communities[request.ExporterIP]
+	community, ok := c.config.Communities.Lookup(net.ParseIP(request.ExporterIP))
 	if !ok {
-		community = c.config.DefaultCommunity
+		community = "public"
 	}
 
 	// Avoid querying too much exporters with errors

inlet/snmp/root_test.go

@@ -41,11 +41,11 @@ func TestLookup(t *testing.T) {
 func TestSNMPCommunities(t *testing.T) {
 	r := reporter.NewMock(t)
 	configuration := DefaultConfiguration()
-	configuration.DefaultCommunity = "notpublic"
+	configuration.Communities, _ = helpers.NewSubnetMap(map[string]string{
-	configuration.Communities = map[string]string{
+		"::/0":                 "notpublic",
-		"127.0.0.1": "public",
+		"::ffff:127.0.0.1/128": "public",
-		"127.0.0.2": "private",
+		"::ffff:127.0.0.2/128": "private",
-	}
+	})
 	c := NewMock(t, r, configuration, Dependencies{Daemon: daemon.NewMock(t)})
 
 	// Use "public" as a community. Should work.