common/helpers: move parametrized configuration to helpers package

common/helpers/mapstructure.go

@@ -4,10 +4,13 @@
 package helpers
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
 	"reflect"
 	"strings"
 
+	"github.com/gin-gonic/gin"
 	"github.com/mitchellh/mapstructure"
 )
 
@@ -59,3 +62,140 @@ func MapStructureMatchName(mapKey, fieldName string) bool {
 	field := strings.ToLower(fieldName)
 	return key == field
 }
+
+// ParametrizedConfigurationUnmarshallerHook will help decode a configuration
+// structure parametrized by a type by selecting the appropriate concrete type
+// depending on the type contained in the source. We have two configuration
+// structures: the outer one should contain a "Config" field using the inner
+// type. A map from configuration types to a function providing the inner
+// default config should be provided.
+func ParametrizedConfigurationUnmarshallerHook[OuterConfiguration any, InnerConfiguration any](zeroOuterConfiguration OuterConfiguration, innerConfigurationMap map[string](func() InnerConfiguration)) mapstructure.DecodeHookFunc {
+	return func(from, to reflect.Value) (interface{}, error) {
+		if to.Type() != reflect.TypeOf(zeroOuterConfiguration) {
+			return from.Interface(), nil
+		}
+		configField := to.FieldByName("Config")
+		fromConfig := reflect.MakeMap(reflect.TypeOf(gin.H{}))
+
+		// Find "type" key in map to get input type. Keep existing fields as is.
+		// Move everything else in "config".
+		var innerConfigurationType string
+		if from.Kind() != reflect.Map {
+			return nil, errors.New("configuration should be a map")
+		}
+		mapKeys := from.MapKeys()
+	outer:
+		for _, key := range mapKeys {
+			var keyStr string
+			// YAML may unmarshal keys to interfaces
+			if ElemOrIdentity(key).Kind() == reflect.String {
+				keyStr = ElemOrIdentity(key).String()
+			} else {
+				continue
+			}
+			switch strings.ToLower(keyStr) {
+			case "type":
+				inputTypeVal := ElemOrIdentity(from.MapIndex(key))
+				if inputTypeVal.Kind() != reflect.String {
+					return nil, fmt.Errorf("type should be a string not %s", inputTypeVal.Kind())
+				}
+				innerConfigurationType = strings.ToLower(inputTypeVal.String())
+				from.SetMapIndex(key, reflect.Value{})
+			case "config":
+				return nil, errors.New("configuration should not have a `config' key")
+			default:
+				t := to.Type()
+				for i := 0; i < t.NumField(); i++ {
+					if MapStructureMatchName(keyStr, t.Field(i).Name) {
+						// Don't touch
+						continue outer
+					}
+				}
+				fromConfig.SetMapIndex(reflect.ValueOf(keyStr), from.MapIndex(key))
+				from.SetMapIndex(key, reflect.Value{})
+			}
+		}
+		from.SetMapIndex(reflect.ValueOf("config"), fromConfig)
+
+		if !configField.IsNil() && innerConfigurationType == "" {
+			// Get current type.
+			currentType := configField.Elem().Type().Elem()
+			for k, v := range innerConfigurationMap {
+				typeOf := reflect.TypeOf(v())
+				if typeOf.Kind() == reflect.Pointer {
+					typeOf = typeOf.Elem()
+				}
+				if typeOf == currentType {
+					innerConfigurationType = k
+					break
+				}
+			}
+		}
+		if innerConfigurationType == "" {
+			return nil, errors.New("configuration has no type")
+		}
+
+		// Get the appropriate input.Configuration for the string
+		innerConfiguration, ok := innerConfigurationMap[innerConfigurationType]
+		if !ok {
+			return nil, fmt.Errorf("%q is not a known input type", innerConfigurationType)
+		}
+
+		// Alter config with a copy of the concrete type
+		defaultV := innerConfiguration()
+		original := reflect.Indirect(reflect.ValueOf(defaultV))
+		if !configField.IsNil() && configField.Elem().Type().Elem() == reflect.TypeOf(defaultV).Elem() {
+			// Use the value we already have instead of default.
+			original = reflect.Indirect(configField.Elem())
+		}
+		copy := reflect.New(original.Type())
+		copy.Elem().Set(reflect.ValueOf(original.Interface()))
+		configField.Set(copy)
+
+		// Resume decoding
+		return from.Interface(), nil
+	}
+}
+
+// ParametrizedConfigurationMarshalYAML undoes ParametrizedConfigurationUnmarshallerHook().
+func ParametrizedConfigurationMarshalYAML[OuterConfiguration any, InnerConfiguration any](oc OuterConfiguration, innerConfigurationMap map[string](func() InnerConfiguration)) (interface{}, error) {
+	var innerConfigStruct reflect.Value
+	outerConfigStruct := ElemOrIdentity(reflect.ValueOf(oc))
+	result := gin.H{}
+	for i, field := range reflect.VisibleFields(outerConfigStruct.Type()) {
+		if field.Name != "Config" {
+			result[strings.ToLower(field.Name)] = outerConfigStruct.Field(i).Interface()
+		} else {
+			innerConfigStruct = outerConfigStruct.Field(i).Elem()
+			if innerConfigStruct.Kind() == reflect.Pointer {
+				innerConfigStruct = innerConfigStruct.Elem()
+			}
+		}
+	}
+	for k, v := range innerConfigurationMap {
+		typeOf := reflect.TypeOf(v())
+		if typeOf.Kind() == reflect.Pointer {
+			typeOf = typeOf.Elem()
+		}
+		if typeOf == innerConfigStruct.Type() {
+			result["type"] = k
+			break
+		}
+	}
+	if result["type"] == nil {
+		return nil, errors.New("unable to guess configuration type")
+	}
+	for i, field := range reflect.VisibleFields(innerConfigStruct.Type()) {
+		result[strings.ToLower(field.Name)] = innerConfigStruct.Field(i).Interface()
+	}
+	return result, nil
+}
+
+// ParametrizedConfigurationMarshalJSON undoes ParametrizedConfigurationUnmarshallerHook().
+func ParametrizedConfigurationMarshalJSON[OuterConfiguration any, InnerConfiguration any](oc OuterConfiguration, innerConfigurationMap map[string](func() InnerConfiguration)) ([]byte, error) {
+	result, err := ParametrizedConfigurationMarshalYAML(oc, innerConfigurationMap)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(result)
+}

common/helpers/mapstructure_test.go

@@ -67,3 +67,152 @@ func TestProtectedDecodeHook(t *testing.T) {
 		}
 	}
 }
+
+func TestParametrizedConfig(t *testing.T) {
+	type InnerConfigurationType1 struct {
+		CC string
+		DD string
+	}
+	type InnerConfigurationType2 struct {
+		CC string
+		EE string
+	}
+	type OuterConfiguration struct {
+		AA     string
+		BB     string
+		Config interface{}
+	}
+	available := map[string](func() interface{}){
+		"type1": func() interface{} {
+			return InnerConfigurationType1{
+				CC: "cc1",
+				DD: "dd1",
+			}
+		},
+		"type2": func() interface{} {
+			return InnerConfigurationType2{
+				CC: "cc2",
+				EE: "ee2",
+			}
+		},
+	}
+	RegisterMapstructureUnmarshallerHook(ParametrizedConfigurationUnmarshallerHook(OuterConfiguration{}, available))
+
+	t.Run("unmarshal", func(t *testing.T) {
+		TestConfigurationDecode(t, ConfigurationDecodeCases{
+			{
+				Description: "type1",
+				Initial:     func() interface{} { return OuterConfiguration{} },
+				Configuration: func() interface{} {
+					return gin.H{
+						"type": "type1",
+						"aa":   "a1",
+						"bb":   "b1",
+						"cc":   "c1",
+						"dd":   "d1",
+					}
+				},
+				Expected: OuterConfiguration{
+					AA: "a1",
+					BB: "b1",
+					Config: InnerConfigurationType1{
+						CC: "c1",
+						DD: "d1",
+					},
+				},
+			}, {
+				Description: "type2",
+				Initial:     func() interface{} { return OuterConfiguration{} },
+				Configuration: func() interface{} {
+					return gin.H{
+						"type": "type2",
+						"aa":   "a2",
+						"bb":   "b2",
+						"cc":   "c2",
+						"ee":   "e2",
+					}
+				},
+				Expected: OuterConfiguration{
+					AA: "a2",
+					BB: "b2",
+					Config: InnerConfigurationType2{
+						CC: "c2",
+						EE: "e2",
+					},
+				},
+			}, {
+				Description: "unknown type",
+				Initial:     func() interface{} { return OuterConfiguration{} },
+				Configuration: func() interface{} {
+					return gin.H{
+						"type": "type3",
+						"aa":   "a2",
+						"bb":   "b2",
+						"cc":   "c2",
+						"ee":   "e2",
+					}
+				},
+				Error: true,
+			},
+		})
+
+	})
+	t.Run("marshal", func(t *testing.T) {
+		config1 := OuterConfiguration{
+			AA: "a1",
+			BB: "b1",
+			Config: InnerConfigurationType1{
+				CC: "c1",
+				DD: "d1",
+			},
+		}
+		expected1 := gin.H{
+			"type": "type1",
+			"aa":   "a1",
+			"bb":   "b1",
+			"cc":   "c1",
+			"dd":   "d1",
+		}
+		got1, err := ParametrizedConfigurationMarshalYAML(config1, available)
+		if err != nil {
+			t.Fatalf("ParametrizedConfigurationMarshalYAML() error:\n%+v", err)
+		}
+		if diff := Diff(got1, expected1); diff != "" {
+			t.Fatalf("ParametrizedConfigurationMarshalYAML() (-got, +want):\n%s", diff)
+		}
+
+		config2 := OuterConfiguration{
+			AA: "a2",
+			BB: "b2",
+			Config: InnerConfigurationType2{
+				CC: "c2",
+				EE: "e2",
+			},
+		}
+		expected2 := gin.H{
+			"type": "type2",
+			"aa":   "a2",
+			"bb":   "b2",
+			"cc":   "c2",
+			"ee":   "e2",
+		}
+		got2, err := ParametrizedConfigurationMarshalYAML(config2, available)
+		if err != nil {
+			t.Fatalf("ParametrizedConfigurationMarshalYAML() error:\n%+v", err)
+		}
+		if diff := Diff(got2, expected2); diff != "" {
+			t.Fatalf("ParametrizedConfigurationMarshalYAML() (-got, +want):\n%s", diff)
+		}
+
+		config3 := OuterConfiguration{
+			AA: "a3",
+			BB: "b3",
+			Config: struct {
+				FF string
+			}{},
+		}
+		if _, err := ParametrizedConfigurationMarshalYAML(config3, available); err == nil {
+			t.Fatal("ParametrizedConfigurationMarshalYAML() did not error")
+		}
+	})
+}

common/helpers/reflect.go

@@ -9,7 +9,7 @@ import "reflect"
 // interface, else it returns the value unmodified.
 func ElemOrIdentity(value reflect.Value) reflect.Value {
 	if value.Kind() == reflect.Interface {
-		return value.Elem()
+		value = value.Elem()
 	}
 	return value
 }

inlet/flow/config.go

@@ -4,14 +4,6 @@
 package flow
 
 import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"reflect"
-	"strings"
-
-	"github.com/gin-gonic/gin"
-	"github.com/mitchellh/mapstructure"
 	"golang.org/x/time/rate"
 
 	"akvorado/common/helpers"
@@ -53,127 +45,14 @@ type InputConfiguration struct {
 	Config input.Configuration
 }
 
-// ConfigurationUnmarshallerHook will help decode the Configuration
-// structure by selecting the appropriate concrete type for
-// input.Connfiguration, depending on the type contained in the
-// source.
-func ConfigurationUnmarshallerHook() mapstructure.DecodeHookFunc {
-	return func(from, to reflect.Value) (interface{}, error) {
-		if to.Type() != reflect.TypeOf(InputConfiguration{}) {
-			return from.Interface(), nil
-		}
-		configField := to.FieldByName("Config")
-		fromConfig := reflect.MakeMap(reflect.TypeOf(gin.H{}))
-
-		// Find "type" key in map to get input type. Keep
-		// "decoder" as is. Move everything else in "config".
-		var inputType string
-		if from.Kind() != reflect.Map {
-			return nil, errors.New("input configuration should be a map")
-		}
-		mapKeys := from.MapKeys()
-	outer:
-		for _, key := range mapKeys {
-			var keyStr string
-			// YAML may unmarshal keys to interfaces
-			if helpers.ElemOrIdentity(key).Kind() == reflect.String {
-				keyStr = helpers.ElemOrIdentity(key).String()
-			} else {
-				continue
-			}
-			switch strings.ToLower(keyStr) {
-			case "type":
-				inputTypeVal := helpers.ElemOrIdentity(from.MapIndex(key))
-				if inputTypeVal.Kind() != reflect.String {
-					return nil, fmt.Errorf("type should be a string not %s", inputTypeVal.Kind())
-				}
-				inputType = strings.ToLower(inputTypeVal.String())
-				from.SetMapIndex(key, reflect.Value{})
-			case "config":
-				return nil, errors.New("input configuration should not have a config key")
-			default:
-				t := to.Type()
-				for i := 0; i < t.NumField(); i++ {
-					if helpers.MapStructureMatchName(keyStr, t.Field(i).Name) {
-						// Don't touch
-						continue outer
-					}
-				}
-				fromConfig.SetMapIndex(reflect.ValueOf(keyStr), from.MapIndex(key))
-				from.SetMapIndex(key, reflect.Value{})
-			}
-		}
-		from.SetMapIndex(reflect.ValueOf("config"), fromConfig)
-
-		if !configField.IsNil() && inputType == "" {
-			// Get current type.
-			currentType := configField.Elem().Type().Elem()
-			for k, v := range inputs {
-				if reflect.TypeOf(v()).Elem() == currentType {
-					inputType = k
-					break
-				}
-			}
-		}
-		if inputType == "" {
-			return nil, errors.New("input configuration has no type")
-		}
-
-		// Get the appropriate input.Configuration for the string
-		input, ok := inputs[inputType]
-		if !ok {
-			return nil, fmt.Errorf("%q is not a known input type", inputType)
-		}
-
-		// Alter config with a copy of the concrete type
-		defaultV := input()
-		original := reflect.Indirect(reflect.ValueOf(defaultV))
-		if !configField.IsNil() && configField.Elem().Type().Elem() == reflect.TypeOf(defaultV).Elem() {
-			// Use the value we already have instead of default.
-			original = reflect.Indirect(configField.Elem())
-		}
-		copy := reflect.New(original.Type())
-		copy.Elem().Set(reflect.ValueOf(original.Interface()))
-		configField.Set(copy)
-
-		// Resume decoding
-		return from.Interface(), nil
-	}
-}
-
 // MarshalYAML undoes ConfigurationUnmarshallerHook().
 func (ic InputConfiguration) MarshalYAML() (interface{}, error) {
-	var typeStr string
-	for k, v := range inputs {
-		if reflect.TypeOf(v()).Elem() == reflect.TypeOf(ic.Config).Elem() {
-			typeStr = k
-			break
-		}
-	}
-	if typeStr == "" {
-		return nil, errors.New("unable to guess input configuration type")
-	}
-	result := gin.H{"type": typeStr}
-	outerConfigStruct := reflect.ValueOf(ic)
-	for i, field := range reflect.VisibleFields(outerConfigStruct.Type()) {
-		if field.Name != "Config" {
-			result[strings.ToLower(field.Name)] = outerConfigStruct.Field(i).Interface()
-		}
-	}
-	innerConfigStruct := reflect.ValueOf(ic.Config).Elem()
-	for i, field := range reflect.VisibleFields(innerConfigStruct.Type()) {
-		result[strings.ToLower(field.Name)] = innerConfigStruct.Field(i).Interface()
-	}
-	return result, nil
+	return helpers.ParametrizedConfigurationMarshalYAML(ic, inputs)
 }
 
 // MarshalJSON undoes ConfigurationUnmarshallerHook().
 func (ic InputConfiguration) MarshalJSON() ([]byte, error) {
-	result, err := ic.MarshalYAML()
-	if err != nil {
-		return nil, err
-	}
-	return json.Marshal(result)
+	return helpers.ParametrizedConfigurationMarshalJSON(ic, inputs)
 }
 
 var inputs = map[string](func() input.Configuration){
@@ -182,5 +61,6 @@ var inputs = map[string](func() input.Configuration){
 }
 
 func init() {
-	helpers.RegisterMapstructureUnmarshallerHook(ConfigurationUnmarshallerHook())
+	helpers.RegisterMapstructureUnmarshallerHook(
+		helpers.ParametrizedConfigurationUnmarshallerHook(InputConfiguration{}, inputs))
 }