cmd: accept an !include tag for YAML files

.github/workflows/ci.yml

@@ -201,7 +201,7 @@ jobs:
           tar zcvf docker-compose-quickstart.tar.gz \
             docker-compose.yml docker-compose-demo.yml .env \
             orchestrator/clickhouse/data/docker-entrypoint.sh \
-            akvorado.yaml
+            akvorado*.yaml
 
       # Publish release
       - name: Publish release

akvorado-console.yaml

@@ -0,0 +1,16 @@
+---
+http:
+  cache:
+    type: redis
+    server: redis:6379
+database:
+  saved-filters:
+    # These are prepopulated filters you can select in a drop-down
+    # menu. Users can add more filters interactively.
+    - description: "From Netflix"
+      content: >-
+        InIfBoundary = external AND SrcAS = AS2906
+    - description: "From GAFAM"
+      content: >-
+        InIfBoundary = external AND
+        SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075)

akvorado-demo.yaml

@@ -0,0 +1,347 @@
+---
+.demo-exporter-flows:
+  - &http-src
+    src-port: [80, 443]
+    dst-port: 0
+    protocol: tcp
+    size: 1300
+  - &http-dst
+    src-port: 0
+    dst-port: [80, 443]
+    protocol: tcp
+    size: 1300
+  - &quic-src
+    src-port: 443
+    dst-port: 0
+    protocol: udp
+    size: 1200
+  - &ssh-src
+    src-port: 22
+    dst-port: 0
+    protocol: tcp
+    size: 200
+  - &ssh-dst
+    src-port: 0
+    dst-port: 22
+    protocol: tcp
+    size: 300
+  - &to-v4-customers
+    dst-net: 192.0.2.0/24
+    dst-as: 64501
+  - &to-v6-customers
+    dst-net: 2a01:db8:cafe:1::/64
+    dst-as: 64501
+  - &to-v4-servers
+    dst-net: 203.0.113.0/24
+    dst-as: 64501
+  - &to-v6-servers
+    dst-net: 2a01:db8:cafe:2::/64
+    dst-as: 64501
+  - &from-v4-google
+    src-net: 216.58.206.0/24
+    src-as: 15169
+  - &from-v6-google
+    src-net: 2a00:1450:4007:807::2000/124
+    src-as: 15169
+  - &from-v4-facebook
+    src-net: 179.60.192.0/24
+    src-as: 32934
+  - &from-v6-facebook
+    src-net: 2a03:2880:f130:83:face:b00c:0::/112
+    src-as: 32934
+  - &from-v4-netflix
+    src-net: 198.38.120.0/23
+    src-as: 2906
+  - &from-v6-netflix
+    src-net: 2a00:86c0:115:115::/112
+    src-as: 2906
+  - &from-v4-akamai
+    src-net: 23.33.27.0/24
+    src-as: 20940
+  - &from-v6-akamai
+    src-net: 2a02:26f0:9100:28:0:17c0::/112
+    src-as: 20940
+  - &from-v4-amazon
+    src-net: 52.84.175.0/24
+    src-as: 16509
+  - &from-v6-amazon
+    src-net: 2600:9000:218d:4a00:15:74db::/112
+    src-as: 16509
+  - &from-v4-fastly
+    src-net: 199.232.178.0/29
+    src-as: 54113
+  - &from-v6-fastly
+    src-net: 2a04:4e42:1d::/126
+    src-as: 54113
+  - &from-v4-twitch
+    src-net: 52.223.202.128/27
+    src-as: 46489
+  - &from-v4-renater
+    src-net: 138.231.0.0/16
+    src-as: 2269
+  - &from-v4-random
+    src-net: 92.0.0.0/8
+    src-as: [12322, 3215, 3303, 15557, 3320, 13335, 6185, 202818, 60068, 16276, 8075, 32590]
+  - &from-v6-random
+    src-net: 2a01:cb00::/32
+    src-as: [12322, 3215, 3303, 15557, 3320, 13335, 6185, 202818, 60068, 16276, 8075, 32590]
+
+"":
+  - snmp:
+      name: th2-edge1.example.com
+      interfaces:
+        10: "Transit: Telia"
+        11: "IX: AMSIX"
+        20: "core"
+        21: "core"
+      listen: 0.0.0.0:161
+    bmp: &bmp
+      target: akvorado-inlet:10179
+      routes:
+        - prefixes: 192.0.2.0/24,2a01:db8:cafe:1::/64
+          aspath: 64501
+          communities: 65401:10,65401:12
+          large-communities: 65401:100:200,65401:100:201
+        - prefixes: 203.0.113.0/24,2a01:db8:cafe:2::/64
+          aspath: 65401
+          communities: 65401:10,65401:13
+          large-communities: 65401:100:200,65401:100:213
+        - prefixes: 216.58.206.0/24,2a00:1450:4007:807::2000/124
+          aspath: 174,1299,15169
+          communities: 174:22004,174:21100
+        - prefixes: 179.60.192.0/24,2a03:2880:f130:83:face:b00c:0::/112
+          aspath: 1299,1299,32934
+          communities: 1299:30000,1299:30220
+        - prefixes: 198.38.120.0/23,2a00:86c0:115:115::/112
+          aspath: 5511,1299,1299,32934
+          communities: 1299:30000,1299:30310
+        - prefixes: 23.33.27.0/24,2a02:26f0:9100:28:0:17c0::/112
+          aspath: 174,174,174,20940
+          communities: 174:22002,174:21200
+        - prefixes: 52.84.175.0/24,2600:9000:218d:4a00:15:74db::/112
+          aspath: 16509
+        - prefixes: 199.232.178.0/29,2a04:4e42:1d::/126
+          aspath: 1299,54113
+          communities: 1299:35000,1299:35200
+        - prefixes: 52.223.202.128/27
+          aspath: 16509,46489
+        - prefixes: 138.231.0.0/16
+          aspath: 1299,174,2269,2269
+          communities: 1299:30000,1299:30400
+        - prefixes: 0.0.0.0/0
+          aspath: 174
+        - prefixes: ::/0
+          aspath: 1299
+    flows: &flows1
+      samplingrate: 50000
+      target: akvorado-inlet:2055
+      flows:
+        # Google
+        - per-second: 1
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 16h
+          multiplier: 3
+          reverse-direction-ratio: 0.1
+          <<: [*from-v4-google, *to-v4-customers, *http-src]
+        - per-second: 0.5
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 16h
+          multiplier: 5
+          reverse-direction-ratio: 0.1
+          <<: [*from-v4-google, *to-v4-customers, *quic-src]
+        - per-second: 1.4
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 21h
+          multiplier: 3
+          reverse-direction-ratio: 0.1
+          <<: [*from-v6-google, *to-v6-customers, *http-src]
+        - per-second: 0.8
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 21h
+          multiplier: 5
+          reverse-direction-ratio: 0.1
+          <<: [*from-v6-google, *to-v6-customers, *quic-src]
+        # Facebook
+        - per-second: 1.1
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 16h
+          multiplier: 3
+          reverse-direction-ratio: 0.2
+          <<: [*from-v4-facebook, *to-v4-customers, *http-src]
+        - per-second: 0.2
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 16h
+          multiplier: 3
+          reverse-direction-ratio: 0.2
+          <<: [*from-v4-facebook, *to-v4-customers, *quic-src]
+        - per-second: 1.8
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 18h
+          multiplier: 3
+          reverse-direction-ratio: 0.2
+          <<: [*from-v6-facebook, *to-v6-customers, *http-src]
+        - per-second: 0.2
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 20h
+          multiplier: 3
+          reverse-direction-ratio: 0.2
+          <<: [*from-v6-facebook, *to-v6-customers, *quic-src]
+        # Netflix
+        - per-second: 0.2
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 22h
+          multiplier: 20
+          reverse-direction-ratio: 0.1
+          <<: [*from-v4-netflix, *to-v4-customers, *http-src]
+        - per-second: 0.7
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 22h
+          multiplier: 20
+          reverse-direction-ratio: 0.1
+          <<: [*from-v6-netflix, *to-v6-customers, *http-src]
+        # Twitch
+        - per-second: 0.12
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 21h
+          multiplier: 17
+          reverse-direction-ratio: 0.4
+          <<: [*from-v4-twitch, *to-v4-customers, *http-src]
+        # Akamai
+        - per-second: 0.14
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 18h
+          multiplier: 1.3
+          reverse-direction-ratio: 0.1
+          <<: [*from-v4-akamai, *to-v4-customers, *http-src]
+        - per-second: 0.8
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 18h
+          multiplier: 1.3
+          reverse-direction-ratio: 0.1
+          <<: [*from-v6-akamai, *to-v6-customers, *http-src]
+        # Fastly
+        - per-second: 0.4
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 15h
+          multiplier: 1.3
+          reverse-direction-ratio: 0.1
+          <<: [*from-v4-fastly, *to-v4-customers, *http-src]
+        - per-second: 0.7
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 14h
+          multiplier: 1.3
+          reverse-direction-ratio: 0.1
+          <<: [*from-v6-fastly, *to-v6-customers, *http-src]
+        # Amazon
+        - per-second: 0.3
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 18h
+          multiplier: 1.3
+          reverse-direction-ratio: 0.15
+          <<: [*from-v4-amazon, *to-v4-customers, *http-src]
+        - per-second: 0.1
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 18h
+          multiplier: 1.3
+          reverse-direction-ratio: 0.15
+          <<: [*from-v6-amazon, *to-v6-customers, *http-src]
+
+        # Random SSH
+        - per-second: 0.1
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 15h
+          multiplier: 1.2
+          reverse-direction-ratio: 0.5
+          <<: [*from-v4-renater, *to-v4-customers, *ssh-src]
+        # Servers
+        - per-second: 0.1
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 15h
+          multiplier: 1.2
+          reverse-direction-ratio: 0.2
+          <<: [*from-v4-renater, *to-v4-servers, *ssh-dst]
+        - per-second: 0.2
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 15h
+          multiplier: 1.2
+          reverse-direction-ratio: 0.15
+          <<: [*from-v4-random, *to-v4-servers, *http-dst]
+        - per-second: 0.2
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 15h
+          multiplier: 1.2
+          reverse-direction-ratio: 0.1
+          <<: [*from-v6-random, *to-v6-servers, *http-dst]
+
+        # Noise
+        - &random-flow
+          per-second: 1
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 20h
+          multiplier: 1
+          protocol: [tcp, udp]
+          srcport: [80, 443, 22, 25461, 8080, 4500, 993, 8801]
+          reverse-direction-ratio: 0.25
+          <<: [*from-v4-random, *to-v4-customers]
+        - <<: [*from-v6-random, *to-v6-customers, *random-flow]
+  - snmp:
+      name: th2-edge2.example.com
+      interfaces:
+        10: "Transit: Cogent"
+        11: "IX: DECIX"
+        20: "core"
+        21: "core"
+      listen: 0.0.0.0:161
+    bmp:
+      <<: *bmp
+    flows:
+      <<: *flows1
+      seed: 100
+  - snmp:
+      name: dc3-edge1.example.com
+      interfaces:
+        10: "Transit: Tata"
+        11: "Transit: Lumen"
+        20: "core"
+        21: "core"
+      listen: 0.0.0.0:161
+    bmp:
+      <<: *bmp
+    flows:
+      <<: *flows1
+      seed: 200
+  - snmp:
+      name: dc5-edge2.example.com
+      interfaces:
+        10: "IX: FranceIX"
+        11: "Transit: Cogent"
+        20: "core"
+        21: "core"
+      listen: 0.0.0.0:161
+    bmp:
+      <<: *bmp
+    flows:
+      <<: *flows1
+      seed: 300

akvorado-inlet.yaml

@@ -0,0 +1,44 @@
+---
+kafka:
+  compression-codec: zstd
+geoip:
+  optional: true
+  # When running on Docker, these paths are inside the container.
+  # Check docker-compose.yml for details.
+  asn-database: /usr/share/GeoIP/GeoLite2-ASN.mmdb
+  geo-database: /usr/share/GeoIP/GeoLite2-Country.mmdb
+snmp:
+  workers: 10
+  communities:
+    ::/0: public
+flow:
+  inputs:
+    - type: udp
+      decoder: netflow
+      listen: 0.0.0.0:2055
+      workers: 6
+      receive-buffer: 10485760
+    - type: udp
+      decoder: sflow
+      listen: 0.0.0.0:6343
+      workers: 6
+      receive-buffer: 10485760
+core:
+  workers: 6
+  exporter-classifiers:
+    # This is an example. This should be customized depending on how
+    # your exporters are named.
+    - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1")
+    - ClassifyRegion("europe")
+    - ClassifyTenant("acme")
+    - ClassifyRole("edge")
+  interface-classifiers:
+    # This is an example. This must be customized depending on the
+    # descriptions of your interfaces. In the following, we assume
+    # external interfaces are named "Transit: Cogent" Or "IX:
+    # FranceIX".
+    - |
+      ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") &&
+      ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") &&
+      ClassifyExternal()
+    - ClassifyInternal()

akvorado.yaml

@@ -3,6 +3,7 @@
 # You can get all default values with `akvorado orchestrator /dev/null
 # --dump --check` or `docker-compose run akvorado-orchestrator
 # orchestrator /dev/null --dump --check`.
+
 kafka:
   topic: flows
   version: 3.3.1
@@ -56,414 +57,8 @@ clickhouse:
     #     .prefixes[] |
     #     { prefix: (.ipv4Prefix // .ipv6Prefix), tenant: "google-cloud", region: .scope }
 
-inlet:
+inlet: !include "akvorado-inlet.yaml"
-  kafka:
+console: !include "akvorado-console.yaml"
-    compression-codec: zstd
-  geoip:
-    optional: true
-    # When running on Docker, these paths are inside the container.
-    # Check docker-compose.yml for details.
-    asn-database: /usr/share/GeoIP/GeoLite2-ASN.mmdb
-    geo-database: /usr/share/GeoIP/GeoLite2-Country.mmdb
-  snmp:
-    workers: 10
-    communities:
-      ::/0: public
-  flow:
-    inputs:
-      - type: udp
-        decoder: netflow
-        listen: 0.0.0.0:2055
-        workers: 6
-        receive-buffer: 10485760
-      - type: udp
-        decoder: sflow
-        listen: 0.0.0.0:6343
-        workers: 6
-        receive-buffer: 10485760
-  core:
-    workers: 6
-    exporter-classifiers:
-      # This is an example. This should be customized depending on how
-      # your exporters are named.
-      - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1")
-      - ClassifyRegion("europe")
-      - ClassifyTenant("acme")
-      - ClassifyRole("edge")
-    interface-classifiers:
-      # This is an example. This must be customized depending on the
-      # descriptions of your interfaces. In the following, we assume
-      # external interfaces are named "Transit: Cogent" Or "IX:
-      # FranceIX".
-      - |
-        ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") &&
-        ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") &&
-        ClassifyExternal()
-      - ClassifyInternal()
 
-console:
+# Remove the following line if you don't want to get demo data
-  http:
+demo-exporter: !include "akvorado-demo.yaml"
-    cache:
-      type: redis
-      server: redis:6379
-  database:
-    saved-filters:
-      # These are prepopulated filters you can select in a drop-down
-      # menu. Users can add more filters interactively.
-      - description: "From Netflix"
-        content: >-
-          InIfBoundary = external AND SrcAS = AS2906
-      - description: "From GAFAM"
-        content: >-
-          InIfBoundary = external AND
-          SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075)
-
-# The remaining of this configuration file should be removed if you
-# don't want to get demo data.
-
-.demo-exporter-flows:
-  - &http-src
-    src-port: [80, 443]
-    dst-port: 0
-    protocol: tcp
-    size: 1300
-  - &http-dst
-    src-port: 0
-    dst-port: [80, 443]
-    protocol: tcp
-    size: 1300
-  - &quic-src
-    src-port: 443
-    dst-port: 0
-    protocol: udp
-    size: 1200
-  - &ssh-src
-    src-port: 22
-    dst-port: 0
-    protocol: tcp
-    size: 200
-  - &ssh-dst
-    src-port: 0
-    dst-port: 22
-    protocol: tcp
-    size: 300
-  - &to-v4-customers
-    dst-net: 192.0.2.0/24
-    dst-as: 64501
-  - &to-v6-customers
-    dst-net: 2a01:db8:cafe:1::/64
-    dst-as: 64501
-  - &to-v4-servers
-    dst-net: 203.0.113.0/24
-    dst-as: 64501
-  - &to-v6-servers
-    dst-net: 2a01:db8:cafe:2::/64
-    dst-as: 64501
-  - &from-v4-google
-    src-net: 216.58.206.0/24
-    src-as: 15169
-  - &from-v6-google
-    src-net: 2a00:1450:4007:807::2000/124
-    src-as: 15169
-  - &from-v4-facebook
-    src-net: 179.60.192.0/24
-    src-as: 32934
-  - &from-v6-facebook
-    src-net: 2a03:2880:f130:83:face:b00c:0::/112
-    src-as: 32934
-  - &from-v4-netflix
-    src-net: 198.38.120.0/23
-    src-as: 2906
-  - &from-v6-netflix
-    src-net: 2a00:86c0:115:115::/112
-    src-as: 2906
-  - &from-v4-akamai
-    src-net: 23.33.27.0/24
-    src-as: 20940
-  - &from-v6-akamai
-    src-net: 2a02:26f0:9100:28:0:17c0::/112
-    src-as: 20940
-  - &from-v4-amazon
-    src-net: 52.84.175.0/24
-    src-as: 16509
-  - &from-v6-amazon
-    src-net: 2600:9000:218d:4a00:15:74db::/112
-    src-as: 16509
-  - &from-v4-fastly
-    src-net: 199.232.178.0/29
-    src-as: 54113
-  - &from-v6-fastly
-    src-net: 2a04:4e42:1d::/126
-    src-as: 54113
-  - &from-v4-twitch
-    src-net: 52.223.202.128/27
-    src-as: 46489
-  - &from-v4-renater
-    src-net: 138.231.0.0/16
-    src-as: 2269
-  - &from-v4-random
-    src-net: 92.0.0.0/8
-    src-as: [12322, 3215, 3303, 15557, 3320, 13335, 6185, 202818, 60068, 16276, 8075, 32590]
-  - &from-v6-random
-    src-net: 2a01:cb00::/32
-    src-as: [12322, 3215, 3303, 15557, 3320, 13335, 6185, 202818, 60068, 16276, 8075, 32590]
-
-demo-exporter:
-  - snmp:
-      name: th2-edge1.example.com
-      interfaces:
-        10: "Transit: Telia"
-        11: "IX: AMSIX"
-        20: "core"
-        21: "core"
-      listen: 0.0.0.0:161
-    bmp: &bmp
-      target: akvorado-inlet:10179
-      routes:
-        - prefixes: 192.0.2.0/24,2a01:db8:cafe:1::/64
-          aspath: 64501
-          communities: 65401:10,65401:12
-          large-communities: 65401:100:200,65401:100:201
-        - prefixes: 203.0.113.0/24,2a01:db8:cafe:2::/64
-          aspath: 65401
-          communities: 65401:10,65401:13
-          large-communities: 65401:100:200,65401:100:213
-        - prefixes: 216.58.206.0/24,2a00:1450:4007:807::2000/124
-          aspath: 174,1299,15169
-          communities: 174:22004,174:21100
-        - prefixes: 179.60.192.0/24,2a03:2880:f130:83:face:b00c:0::/112
-          aspath: 1299,1299,32934
-          communities: 1299:30000,1299:30220
-        - prefixes: 198.38.120.0/23,2a00:86c0:115:115::/112
-          aspath: 5511,1299,1299,32934
-          communities: 1299:30000,1299:30310
-        - prefixes: 23.33.27.0/24,2a02:26f0:9100:28:0:17c0::/112
-          aspath: 174,174,174,20940
-          communities: 174:22002,174:21200
-        - prefixes: 52.84.175.0/24,2600:9000:218d:4a00:15:74db::/112
-          aspath: 16509
-        - prefixes: 199.232.178.0/29,2a04:4e42:1d::/126
-          aspath: 1299,54113
-          communities: 1299:35000,1299:35200
-        - prefixes: 52.223.202.128/27
-          aspath: 16509,46489
-        - prefixes: 138.231.0.0/16
-          aspath: 1299,174,2269,2269
-          communities: 1299:30000,1299:30400
-        - prefixes: 0.0.0.0/0
-          aspath: 174
-        - prefixes: ::/0
-          aspath: 1299
-    flows: &flows1
-      samplingrate: 50000
-      target: akvorado-inlet:2055
-      flows:
-        # Google
-        - per-second: 1
-          in-if-index: 10
-          out-if-index: [20, 21]
-          peak-hour: 16h
-          multiplier: 3
-          reverse-direction-ratio: 0.1
-          <<: [*from-v4-google, *to-v4-customers, *http-src]
-        - per-second: 0.5
-          in-if-index: 10
-          out-if-index: [20, 21]
-          peak-hour: 16h
-          multiplier: 5
-          reverse-direction-ratio: 0.1
-          <<: [*from-v4-google, *to-v4-customers, *quic-src]
-        - per-second: 1.4
-          in-if-index: 10
-          out-if-index: [20, 21]
-          peak-hour: 21h
-          multiplier: 3
-          reverse-direction-ratio: 0.1
-          <<: [*from-v6-google, *to-v6-customers, *http-src]
-        - per-second: 0.8
-          in-if-index: 10
-          out-if-index: [20, 21]
-          peak-hour: 21h
-          multiplier: 5
-          reverse-direction-ratio: 0.1
-          <<: [*from-v6-google, *to-v6-customers, *quic-src]
-        # Facebook
-        - per-second: 1.1
-          in-if-index: [10, 11]
-          out-if-index: [20, 21]
-          peak-hour: 16h
-          multiplier: 3
-          reverse-direction-ratio: 0.2
-          <<: [*from-v4-facebook, *to-v4-customers, *http-src]
-        - per-second: 0.2
-          in-if-index: [10, 11]
-          out-if-index: [20, 21]
-          peak-hour: 16h
-          multiplier: 3
-          reverse-direction-ratio: 0.2
-          <<: [*from-v4-facebook, *to-v4-customers, *quic-src]
-        - per-second: 1.8
-          in-if-index: [10, 11]
-          out-if-index: [20, 21]
-          peak-hour: 18h
-          multiplier: 3
-          reverse-direction-ratio: 0.2
-          <<: [*from-v6-facebook, *to-v6-customers, *http-src]
-        - per-second: 0.2
-          in-if-index: [10, 11]
-          out-if-index: [20, 21]
-          peak-hour: 20h
-          multiplier: 3
-          reverse-direction-ratio: 0.2
-          <<: [*from-v6-facebook, *to-v6-customers, *quic-src]
-        # Netflix
-        - per-second: 0.2
-          in-if-index: 10
-          out-if-index: [20, 21]
-          peak-hour: 22h
-          multiplier: 20
-          reverse-direction-ratio: 0.1
-          <<: [*from-v4-netflix, *to-v4-customers, *http-src]
-        - per-second: 0.7
-          in-if-index: 10
-          out-if-index: [20, 21]
-          peak-hour: 22h
-          multiplier: 20
-          reverse-direction-ratio: 0.1
-          <<: [*from-v6-netflix, *to-v6-customers, *http-src]
-        # Twitch
-        - per-second: 0.12
-          in-if-index: 10
-          out-if-index: [20, 21]
-          peak-hour: 21h
-          multiplier: 17
-          reverse-direction-ratio: 0.4
-          <<: [*from-v4-twitch, *to-v4-customers, *http-src]
-        # Akamai
-        - per-second: 0.14
-          in-if-index: [10, 11]
-          out-if-index: [20, 21]
-          peak-hour: 18h
-          multiplier: 1.3
-          reverse-direction-ratio: 0.1
-          <<: [*from-v4-akamai, *to-v4-customers, *http-src]
-        - per-second: 0.8
-          in-if-index: [10, 11]
-          out-if-index: [20, 21]
-          peak-hour: 18h
-          multiplier: 1.3
-          reverse-direction-ratio: 0.1
-          <<: [*from-v6-akamai, *to-v6-customers, *http-src]
-        # Fastly
-        - per-second: 0.4
-          in-if-index: [10, 11]
-          out-if-index: [20, 21]
-          peak-hour: 15h
-          multiplier: 1.3
-          reverse-direction-ratio: 0.1
-          <<: [*from-v4-fastly, *to-v4-customers, *http-src]
-        - per-second: 0.7
-          in-if-index: [10, 11]
-          out-if-index: [20, 21]
-          peak-hour: 14h
-          multiplier: 1.3
-          reverse-direction-ratio: 0.1
-          <<: [*from-v6-fastly, *to-v6-customers, *http-src]
-        # Amazon
-        - per-second: 0.3
-          in-if-index: [10, 11]
-          out-if-index: [20, 21]
-          peak-hour: 18h
-          multiplier: 1.3
-          reverse-direction-ratio: 0.15
-          <<: [*from-v4-amazon, *to-v4-customers, *http-src]
-        - per-second: 0.1
-          in-if-index: 10
-          out-if-index: [20, 21]
-          peak-hour: 18h
-          multiplier: 1.3
-          reverse-direction-ratio: 0.15
-          <<: [*from-v6-amazon, *to-v6-customers, *http-src]
-
-        # Random SSH
-        - per-second: 0.1
-          in-if-index: 10
-          out-if-index: [20, 21]
-          peak-hour: 15h
-          multiplier: 1.2
-          reverse-direction-ratio: 0.5
-          <<: [*from-v4-renater, *to-v4-customers, *ssh-src]
-        # Servers
-        - per-second: 0.1
-          in-if-index: 10
-          out-if-index: [20, 21]
-          peak-hour: 15h
-          multiplier: 1.2
-          reverse-direction-ratio: 0.2
-          <<: [*from-v4-renater, *to-v4-servers, *ssh-dst]
-        - per-second: 0.2
-          in-if-index: 10
-          out-if-index: [20, 21]
-          peak-hour: 15h
-          multiplier: 1.2
-          reverse-direction-ratio: 0.15
-          <<: [*from-v4-random, *to-v4-servers, *http-dst]
-        - per-second: 0.2
-          in-if-index: 10
-          out-if-index: [20, 21]
-          peak-hour: 15h
-          multiplier: 1.2
-          reverse-direction-ratio: 0.1
-          <<: [*from-v6-random, *to-v6-servers, *http-dst]
-
-        # Noise
-        - &random-flow
-          per-second: 1
-          in-if-index: 10
-          out-if-index: [20, 21]
-          peak-hour: 20h
-          multiplier: 1
-          protocol: [tcp, udp]
-          srcport: [80, 443, 22, 25461, 8080, 4500, 993, 8801]
-          reverse-direction-ratio: 0.25
-          <<: [*from-v4-random, *to-v4-customers]
-        - <<: [*from-v6-random, *to-v6-customers, *random-flow]
-  - snmp:
-      name: th2-edge2.example.com
-      interfaces:
-        10: "Transit: Cogent"
-        11: "IX: DECIX"
-        20: "core"
-        21: "core"
-      listen: 0.0.0.0:161
-    bmp:
-      <<: *bmp
-    flows:
-      <<: *flows1
-      seed: 100
-  - snmp:
-      name: dc3-edge1.example.com
-      interfaces:
-        10: "Transit: Tata"
-        11: "Transit: Lumen"
-        20: "core"
-        21: "core"
-      listen: 0.0.0.0:161
-    bmp:
-      <<: *bmp
-    flows:
-      <<: *flows1
-      seed: 200
-  - snmp:
-      name: dc5-edge2.example.com
-      interfaces:
-        10: "IX: FranceIX"
-        11: "Transit: Cogent"
-        20: "core"
-        21: "core"
-      listen: 0.0.0.0:161
-    bmp:
-      <<: *bmp
-    flows:
-      <<: *flows1
-      seed: 300

cmd/config.go

@@ -6,11 +6,11 @@ package cmd
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"reflect"
 	"sort"
 	"strconv"
@@ -19,7 +19,8 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/go-playground/validator/v10"
 	"github.com/mitchellh/mapstructure"
-	"gopkg.in/yaml.v3"
+
+	"akvorado/common/helpers/yaml"
 
 	"akvorado/common/helpers"
 )
@@ -66,11 +67,15 @@ func (c ConfigRelatedOptions) Parse(out io.Writer, component string, config inte
 				return fmt.Errorf("unable to parse YAML configuration file: %w", err)
 			}
 		} else {
-			input, err := ioutil.ReadFile(cfgFile)
+			cfgFile, err := filepath.EvalSymlinks(cfgFile)
 			if err != nil {
-				return fmt.Errorf("unable to read configuration file: %w", err)
+				return fmt.Errorf("cannot follow symlink: %w", err)
 			}
-			if err := yaml.Unmarshal(input, &rawConfig); err != nil {
+			dirname, filename := filepath.Split(cfgFile)
+			if dirname == "" {
+				dirname = "."
+			}
+			if err := yaml.UnmarshalWithInclude(os.DirFS(dirname), filename, &rawConfig); err != nil {
 				return fmt.Errorf("unable to parse YAML configuration file: %w", err)
 			}
 		}

cmd/config_test.go

@@ -16,7 +16,8 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
-	"gopkg.in/yaml.v3"
+
+	"akvorado/common/helpers/yaml"
 
 	"akvorado/cmd"
 	"akvorado/common/helpers"

cmd/orchestrator_test.go

@@ -15,7 +15,7 @@ import (
 	"akvorado/common/helpers"
 	"akvorado/common/reporter"
 
-	"gopkg.in/yaml.v3"
+	"akvorado/common/helpers/yaml"
 )
 
 func TestOrchestratorStart(t *testing.T) {

common/helpers/subnetmap_test.go

@@ -9,7 +9,8 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/mitchellh/mapstructure"
-	"gopkg.in/yaml.v3"
+
+	"akvorado/common/helpers/yaml"
 
 	"akvorado/common/helpers"
 )

common/helpers/tests_config.go

@@ -10,7 +10,8 @@ import (
 	"testing"
 
 	"github.com/mitchellh/mapstructure"
-	"gopkg.in/yaml.v3"
+
+	"akvorado/common/helpers/yaml"
 )
 
 // ConfigurationDecodeCases describes a test case for configuration

common/helpers/yaml/marshal.go

@@ -0,0 +1,13 @@
+// SPDX-FileCopyrightText: 2023 Free Mobile
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package yaml
+
+import "gopkg.in/yaml.v3"
+
+// Marshal serializes the value provided into a YAML document. The structure of
+// the generated document will reflect the structure of the value itself. Maps
+// and pointers (to struct, string, int, etc) are accepted as the in value.
+func Marshal(in interface{}) (out []byte, err error) {
+	return yaml.Marshal(in)
+}

common/helpers/yaml/testdata/1.yaml

@@ -0,0 +1,2 @@
+---
+name: "1.yaml"

common/helpers/yaml/testdata/2.yaml

@@ -0,0 +1,2 @@
+---
+name: "2.yaml"

common/helpers/yaml/testdata/base.yaml

@@ -0,0 +1,6 @@
+---
+file1: !include 1.yaml
+file2: !include 2.yaml
+nested: !include nested.yaml
+list1: !include list1.yaml
+list2: !include list2.yaml

common/helpers/yaml/testdata/list1.yaml

@@ -0,0 +1,6 @@
+---
+# We cannot include a list, so we use a map with an empty key
+"":
+  - el1
+  - el2
+  - el3

common/helpers/yaml/testdata/list2.yaml

@@ -0,0 +1,11 @@
+---
+
+.hidden:
+  - &squid
+    protocol: tcp
+    size: 1300
+
+"":
+  - *squid
+  - el2
+  - el3

common/helpers/yaml/testdata/nested.yaml

@@ -0,0 +1,2 @@
+---
+file1: !include 1.yaml

common/helpers/yaml/unmarshal.go

@@ -0,0 +1,80 @@
+// SPDX-FileCopyrightText: 2023 Free Mobile
+// SPDX-License-Identifier: AGPL-3.0-only
+
+// Package yaml implements YAML support for the Go language. It adds the ability
+// to use the "!include" tag.
+package yaml
+
+import (
+	"fmt"
+	"io/fs"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+// Unmarshal decodes the first document found within the in byte slice and
+// assigns decoded values into the out value.
+func Unmarshal(in []byte, out interface{}) (err error) {
+	return yaml.Unmarshal(in, out)
+}
+
+// UnmarshalWithInclude decodes the first document found within the in byte
+// slice and assigns decoded values into the out value. It also accepts the
+// "!include" tag to include additional files contained in the provided fs.
+func UnmarshalWithInclude(fsys fs.FS, input string, out interface{}) (err error) {
+	var outNode yaml.Node
+	in, err := fs.ReadFile(fsys, input)
+	if err != nil {
+		return fmt.Errorf("cannot read %s: %w", input, err)
+	}
+	if err := Unmarshal(in, &outNode); err != nil {
+		return fmt.Errorf("in %s: %w", input, err)
+	}
+
+	if outNode.Kind == yaml.DocumentNode {
+		outNode = *outNode.Content[0]
+	}
+	if outNode.Kind == yaml.MappingNode {
+		// Remove hidden entries (prefixed with ".")
+		for i := 0; i < len(outNode.Content)-1; {
+			key := outNode.Content[i]
+			if key.Kind == yaml.ScalarNode && key.Tag == "!!str" && strings.HasPrefix(key.Value, ".") {
+				outNode.Content = outNode.Content[2:]
+			} else {
+				i += 2
+			}
+		}
+		// If we only have a 1-entry map whose key is empty, use the value
+		if len(outNode.Content) == 2 {
+			key := outNode.Content[0]
+			if key.Kind == yaml.ScalarNode && key.Tag == "!!str" && key.Value == "" {
+				outNode = *outNode.Content[1]
+			}
+		}
+	}
+
+	// Walk the content nodes and replace them with the file they refer to.
+	todo := []*yaml.Node{&outNode}
+	for len(todo) > 0 {
+		current := todo[0]
+		todo = todo[1:]
+		if current.Tag != "!include" {
+			todo = append(todo, current.Content...)
+			continue
+		}
+		if current.Alias != nil {
+			return fmt.Errorf("at line %d of %s, no alias is allowed for !include", current.Line, input)
+		}
+		if len(current.Content) > 0 {
+			return fmt.Errorf("at line %d of %s, no content is allowed for !include", current.Line, input)
+		}
+		var outNode yaml.Node
+		if err := UnmarshalWithInclude(fsys, current.Value, &outNode); err != nil {
+			return fmt.Errorf("at line %d of %s: %w", current.Line, input, err)
+		}
+		*current = outNode
+	}
+
+	return outNode.Decode(out)
+}

common/helpers/yaml/unmarshal_test.go

@@ -0,0 +1,37 @@
+// SPDX-FileCopyrightText: 2023 Free Mobile
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package yaml_test
+
+import (
+	"os"
+	"testing"
+
+	"akvorado/common/helpers"
+	"akvorado/common/helpers/yaml"
+
+	"github.com/gin-gonic/gin"
+)
+
+func TestUnmarshalWithIn(t *testing.T) {
+	fsys := os.DirFS("testdata")
+	var got interface{}
+	if err := yaml.UnmarshalWithInclude(fsys, "base.yaml", &got); err != nil {
+		t.Fatalf("UnmarshalWithInclude() error:\n%+v", err)
+	}
+	expected := gin.H{
+		"file1": gin.H{"name": "1.yaml"},
+		"file2": gin.H{"name": "2.yaml"},
+		"nested": gin.H{
+			"file1": gin.H{"name": "1.yaml"},
+		},
+		"list1": []string{"el1", "el2", "el3"},
+		"list2": []interface{}{gin.H{
+			"protocol": "tcp",
+			"size":     1300,
+		}, "el2", "el3"},
+	}
+	if diff := helpers.Diff(got, expected); diff != "" {
+		t.Fatalf("UnmarshalWithInclude() (-got, +want):\n%s", diff)
+	}
+}

console/data/docs/00-intro.md

@@ -28,7 +28,7 @@ Once running, *Akvorado* web interface should be running on port 8081. A few
 synthetic flows are generated in the background. To disable them:
 
 1. Remove `:docker-compose-demo.yml` from `.env`,
-2. Remove the associated configuration at the end of `akvorado.yaml`, and
+2. Comment the last line of `akvorado.yaml`, and
 3. Run `docker-compose up -d --remove-orphans`.
 
 If you want to send you own flows, the inlet is accepting both NetFlow

console/data/docs/99-changelog.md

@@ -11,6 +11,10 @@ identified with a specific icon:
 - 🩹: bug fix
 - 🌱: miscellaneous change
 
+## Unreleased
+
+- 🌱 *common*: accept an `!include` tag to include other YAML files in `akvorado.yaml`
+
 ## 1.7.1 - 2023-01-27
 
 This is an important bugfix release. `DstNet*` values were classified using the

inlet/flow/config_test.go

@@ -8,7 +8,8 @@ import (
 	"testing"
 
 	"github.com/gin-gonic/gin"
-	"gopkg.in/yaml.v3"
+
+	"akvorado/common/helpers/yaml"
 
 	"akvorado/common/helpers"
 	"akvorado/inlet/flow/input/file"