From 1db159fd1fec4c585ccbc5ce1a64a10ccec003fb Mon Sep 17 00:00:00 2001 From: Vincent Bernat Date: Mon, 21 Mar 2022 11:23:42 +0100 Subject: [PATCH] cmd: ability to override settings with environment variables --- cmd/serve.go | 28 +++++++++++++++++++- cmd/serve_test.go | 61 ++++++++++++++++++++++++++++++++++++++++++- docs/configuration.md | 27 +++++++++++++++++-- 3 files changed, 112 insertions(+), 4 deletions(-) diff --git a/cmd/serve.go b/cmd/serve.go index a6876d99..b21cf839 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" netHTTP "net/http" + "os" "runtime" "strings" @@ -94,15 +95,40 @@ and exports them to Kafka.`, DecodeHook: mapstructure.ComposeDecodeHookFunc( mapstructure.TextUnmarshallerHookFunc(), mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), ), }) if err != nil { return fmt.Errorf("unable to create configuration decoder: %w", err) } - if decoder.Decode(rawConfig); err != nil { + if err := decoder.Decode(rawConfig); err != nil { return fmt.Errorf("unable to parse configuration: %w", err) } + // Override with environment variables + for _, keyval := range os.Environ() { + kv := strings.SplitN(keyval, "=", 2) + if len(kv) != 2 { + continue + } + kk := strings.Split(kv[0], "_") + if kk[0] != "AKVORADO" || len(kk) < 2 { + continue + } + // From AKVORADO_SQUID_PURPLE_QUIRK=47, we + // build a map "squid -> purple -> quirk -> 47" + var rawConfig interface{} + rawConfig = kv[1] + for i := len(kk) - 1; i > 0; i-- { + rawConfig = map[string]interface{}{ + kk[i]: rawConfig, + } + } + if err := decoder.Decode(rawConfig); err != nil { + return fmt.Errorf("unable to parse override %q: %w", kv[0], err) + } + } + // Dump configuration if requested if ServeOptions.dumpConfiguration { output, err := yaml.Marshal(config) diff --git a/cmd/serve_test.go b/cmd/serve_test.go index 78f14a03..47713e85 100644 --- a/cmd/serve_test.go +++ b/cmd/serve_test.go @@ -58,7 +58,66 @@ core: cmd.ServeOptionsReset() err := root.Execute() if err != nil { - t.Errorf("`serve -D -C` error:\n%+v", err) + t.Fatalf("`serve -D -C` error:\n%+v", err) + } + var got cmd.ServeConfiguration + if err := yaml.Unmarshal(buf.Bytes(), &got); err != nil { + t.Fatalf("Unmarshal() error:\n%+v", err) + } + if diff := helpers.Diff(got, want); diff != "" { + t.Errorf("`serve -D -C` (-got, +want):\n%s", diff) + } +} + +func TestServeEnvOverride(t *testing.T) { + // Configuration file + want := cmd.DefaultServeConfiguration + want.HTTP.Listen = "127.0.0.1:8000" + want.Flow.Listen = "0.0.0.0:2055" + want.Flow.Workers = 3 + want.SNMP.Workers = 2 + want.SNMP.CacheDuration = 1*time.Hour + 30*time.Minute + want.SNMP.DefaultCommunity = "privateer" + want.Kafka.Topic = "netflow" + want.Kafka.Version = kafka.Version(sarama.V2_8_1_0) + want.Kafka.CompressionCodec = kafka.CompressionCodec(sarama.CompressionZSTD) + want.Kafka.Brokers = []string{"127.0.0.1:9092", "127.0.0.2:9092"} + want.Core.Workers = 3 + config := `--- +http: + listen: 127.0.0.1:8000 +flow: + listen: 0.0.0.0:2055 + workers: 2 +snmp: + workers: 2 + cache-duration: 2h +kafka: + topic: netflow + compression-codec: zstd + version: 2.8.1 +core: + workers: 3 +` + configFile := filepath.Join(t.TempDir(), "akvorado.yaml") + ioutil.WriteFile(configFile, []byte(config), 0644) + + // Environment + os.Setenv("AKVORADO_FLOW_WORKERS", "3") + os.Setenv("AKVORADO_SNMP_CACHEDURATION", "1h30m") + os.Setenv("AKVORADO_SNMP_DEFAULTCOMMUNITY", "privateer") + os.Setenv("AKVORADO_KAFKA_BROKERS", "127.0.0.1:9092,127.0.0.2:9092") + + // Start serves with it + root := cmd.RootCmd + buf := new(bytes.Buffer) + root.SetOut(buf) + root.SetErr(os.Stderr) + root.SetArgs([]string{"serve", "-D", "-C", "--config", configFile}) + cmd.ServeOptionsReset() + err := root.Execute() + if err != nil { + t.Fatalf("`serve -D -C` error:\n%+v", err) } var got cmd.ServeConfiguration if err := yaml.Unmarshal(buf.Bytes(), &got); err != nil { diff --git a/docs/configuration.md b/docs/configuration.md index d87723d6..8301e6c3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -13,9 +13,32 @@ configured through a different section: - `clickhouse`: [Clickhouse helper](#clickhouse) - `core`: [Core](#core) -You can get the default configuration with `./akvorado --dump --check`. +You can get the default configuration with `./akvorado --dump +--check`. Durations can be written in seconds or using strings like +`10h20m`. -Durations can be written in seconds or using strings like `10h20m`. +It is also possible to override configuration settings using +environment variables. You need to remove any `-` from key names and +use `_` to handle nesting. Then, put `AKVORADO_` as a prefix. For +example, let's consider the following configuration file: + +```yaml +kafka: + topic: test-topic + topic-configuration: + num-partitions: 1 + brokers: + - 192.0.2.1:9092 + - 192.0.2.2:9092 +``` + +It can be translated to: + +```sh +AKVORADO_KAFKA_TOPIC=test-topic +AKVORADO_KAFKA_TOPICCONFIGURATION_NUMPARTITIONS=1 +AKVORADO_KAFKA_BROKERS=192.0.2.1:9092,192.0.2.2:9092 +``` ## Reporting