From cdbae1c6432b1de7e8713160c7bcae50b37cccf7 Mon Sep 17 00:00:00 2001 From: Vincent Bernat Date: Sun, 12 Feb 2023 14:40:56 +0100 Subject: [PATCH] Revert "cmd: accept an !include tag for YAML files" This reverts commit b8698bc060f08c69d69f39a5336bd9e2ea5bb0d8. This change is not complete, it is missing adaptations in docker-compose.yml. --- .github/workflows/ci.yml | 2 +- akvorado-console.yaml | 16 - akvorado-demo.yaml | 347 ------------------- akvorado-inlet.yaml | 44 --- akvorado.yaml | 415 ++++++++++++++++++++++- cmd/config.go | 15 +- cmd/config_test.go | 3 +- cmd/orchestrator_test.go | 2 +- common/helpers/subnetmap_test.go | 3 +- common/helpers/tests_config.go | 3 +- common/helpers/yaml/marshal.go | 13 - common/helpers/yaml/testdata/1.yaml | 2 - common/helpers/yaml/testdata/2.yaml | 2 - common/helpers/yaml/testdata/base.yaml | 6 - common/helpers/yaml/testdata/list1.yaml | 6 - common/helpers/yaml/testdata/list2.yaml | 11 - common/helpers/yaml/testdata/nested.yaml | 2 - common/helpers/yaml/unmarshal.go | 80 ----- common/helpers/yaml/unmarshal_test.go | 37 -- console/data/docs/00-intro.md | 2 +- console/data/docs/99-changelog.md | 1 - inlet/flow/config_test.go | 3 +- 22 files changed, 422 insertions(+), 593 deletions(-) delete mode 100644 akvorado-console.yaml delete mode 100644 akvorado-demo.yaml delete mode 100644 akvorado-inlet.yaml delete mode 100644 common/helpers/yaml/marshal.go delete mode 100644 common/helpers/yaml/testdata/1.yaml delete mode 100644 common/helpers/yaml/testdata/2.yaml delete mode 100644 common/helpers/yaml/testdata/base.yaml delete mode 100644 common/helpers/yaml/testdata/list1.yaml delete mode 100644 common/helpers/yaml/testdata/list2.yaml delete mode 100644 common/helpers/yaml/testdata/nested.yaml delete mode 100644 common/helpers/yaml/unmarshal.go delete mode 100644 common/helpers/yaml/unmarshal_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98c15a21..740cf8fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -201,7 +201,7 @@ jobs: tar zcvf docker-compose-quickstart.tar.gz \ docker-compose.yml docker-compose-demo.yml .env \ orchestrator/clickhouse/data/docker-entrypoint.sh \ - akvorado*.yaml + akvorado.yaml # Publish release - name: Publish release diff --git a/akvorado-console.yaml b/akvorado-console.yaml deleted file mode 100644 index 78187578..00000000 --- a/akvorado-console.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -http: - cache: - type: redis - server: redis:6379 -database: - saved-filters: - # These are prepopulated filters you can select in a drop-down - # menu. Users can add more filters interactively. - - description: "From Netflix" - content: >- - InIfBoundary = external AND SrcAS = AS2906 - - description: "From GAFAM" - content: >- - InIfBoundary = external AND - SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075) diff --git a/akvorado-demo.yaml b/akvorado-demo.yaml deleted file mode 100644 index 9531b5e4..00000000 --- a/akvorado-demo.yaml +++ /dev/null @@ -1,347 +0,0 @@ ---- -.demo-exporter-flows: - - &http-src - src-port: [80, 443] - dst-port: 0 - protocol: tcp - size: 1300 - - &http-dst - src-port: 0 - dst-port: [80, 443] - protocol: tcp - size: 1300 - - &quic-src - src-port: 443 - dst-port: 0 - protocol: udp - size: 1200 - - &ssh-src - src-port: 22 - dst-port: 0 - protocol: tcp - size: 200 - - &ssh-dst - src-port: 0 - dst-port: 22 - protocol: tcp - size: 300 - - &to-v4-customers - dst-net: 192.0.2.0/24 - dst-as: 64501 - - &to-v6-customers - dst-net: 2a01:db8:cafe:1::/64 - dst-as: 64501 - - &to-v4-servers - dst-net: 203.0.113.0/24 - dst-as: 64501 - - &to-v6-servers - dst-net: 2a01:db8:cafe:2::/64 - dst-as: 64501 - - &from-v4-google - src-net: 216.58.206.0/24 - src-as: 15169 - - &from-v6-google - src-net: 2a00:1450:4007:807::2000/124 - src-as: 15169 - - &from-v4-facebook - src-net: 179.60.192.0/24 - src-as: 32934 - - &from-v6-facebook - src-net: 2a03:2880:f130:83:face:b00c:0::/112 - src-as: 32934 - - &from-v4-netflix - src-net: 198.38.120.0/23 - src-as: 2906 - - &from-v6-netflix - src-net: 2a00:86c0:115:115::/112 - src-as: 2906 - - &from-v4-akamai - src-net: 23.33.27.0/24 - src-as: 20940 - - &from-v6-akamai - src-net: 2a02:26f0:9100:28:0:17c0::/112 - src-as: 20940 - - &from-v4-amazon - src-net: 52.84.175.0/24 - src-as: 16509 - - &from-v6-amazon - src-net: 2600:9000:218d:4a00:15:74db::/112 - src-as: 16509 - - &from-v4-fastly - src-net: 199.232.178.0/29 - src-as: 54113 - - &from-v6-fastly - src-net: 2a04:4e42:1d::/126 - src-as: 54113 - - &from-v4-twitch - src-net: 52.223.202.128/27 - src-as: 46489 - - &from-v4-renater - src-net: 138.231.0.0/16 - src-as: 2269 - - &from-v4-random - src-net: 92.0.0.0/8 - src-as: [12322, 3215, 3303, 15557, 3320, 13335, 6185, 202818, 60068, 16276, 8075, 32590] - - &from-v6-random - src-net: 2a01:cb00::/32 - src-as: [12322, 3215, 3303, 15557, 3320, 13335, 6185, 202818, 60068, 16276, 8075, 32590] - -"": - - snmp: - name: th2-edge1.example.com - interfaces: - 10: "Transit: Telia" - 11: "IX: AMSIX" - 20: "core" - 21: "core" - listen: 0.0.0.0:161 - bmp: &bmp - target: akvorado-inlet:10179 - routes: - - prefixes: 192.0.2.0/24,2a01:db8:cafe:1::/64 - aspath: 64501 - communities: 65401:10,65401:12 - large-communities: 65401:100:200,65401:100:201 - - prefixes: 203.0.113.0/24,2a01:db8:cafe:2::/64 - aspath: 65401 - communities: 65401:10,65401:13 - large-communities: 65401:100:200,65401:100:213 - - prefixes: 216.58.206.0/24,2a00:1450:4007:807::2000/124 - aspath: 174,1299,15169 - communities: 174:22004,174:21100 - - prefixes: 179.60.192.0/24,2a03:2880:f130:83:face:b00c:0::/112 - aspath: 1299,1299,32934 - communities: 1299:30000,1299:30220 - - prefixes: 198.38.120.0/23,2a00:86c0:115:115::/112 - aspath: 5511,1299,1299,32934 - communities: 1299:30000,1299:30310 - - prefixes: 23.33.27.0/24,2a02:26f0:9100:28:0:17c0::/112 - aspath: 174,174,174,20940 - communities: 174:22002,174:21200 - - prefixes: 52.84.175.0/24,2600:9000:218d:4a00:15:74db::/112 - aspath: 16509 - - prefixes: 199.232.178.0/29,2a04:4e42:1d::/126 - aspath: 1299,54113 - communities: 1299:35000,1299:35200 - - prefixes: 52.223.202.128/27 - aspath: 16509,46489 - - prefixes: 138.231.0.0/16 - aspath: 1299,174,2269,2269 - communities: 1299:30000,1299:30400 - - prefixes: 0.0.0.0/0 - aspath: 174 - - prefixes: ::/0 - aspath: 1299 - flows: &flows1 - samplingrate: 50000 - target: akvorado-inlet:2055 - flows: - # Google - - per-second: 1 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 16h - multiplier: 3 - reverse-direction-ratio: 0.1 - <<: [*from-v4-google, *to-v4-customers, *http-src] - - per-second: 0.5 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 16h - multiplier: 5 - reverse-direction-ratio: 0.1 - <<: [*from-v4-google, *to-v4-customers, *quic-src] - - per-second: 1.4 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 21h - multiplier: 3 - reverse-direction-ratio: 0.1 - <<: [*from-v6-google, *to-v6-customers, *http-src] - - per-second: 0.8 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 21h - multiplier: 5 - reverse-direction-ratio: 0.1 - <<: [*from-v6-google, *to-v6-customers, *quic-src] - # Facebook - - per-second: 1.1 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 16h - multiplier: 3 - reverse-direction-ratio: 0.2 - <<: [*from-v4-facebook, *to-v4-customers, *http-src] - - per-second: 0.2 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 16h - multiplier: 3 - reverse-direction-ratio: 0.2 - <<: [*from-v4-facebook, *to-v4-customers, *quic-src] - - per-second: 1.8 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 18h - multiplier: 3 - reverse-direction-ratio: 0.2 - <<: [*from-v6-facebook, *to-v6-customers, *http-src] - - per-second: 0.2 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 20h - multiplier: 3 - reverse-direction-ratio: 0.2 - <<: [*from-v6-facebook, *to-v6-customers, *quic-src] - # Netflix - - per-second: 0.2 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 22h - multiplier: 20 - reverse-direction-ratio: 0.1 - <<: [*from-v4-netflix, *to-v4-customers, *http-src] - - per-second: 0.7 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 22h - multiplier: 20 - reverse-direction-ratio: 0.1 - <<: [*from-v6-netflix, *to-v6-customers, *http-src] - # Twitch - - per-second: 0.12 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 21h - multiplier: 17 - reverse-direction-ratio: 0.4 - <<: [*from-v4-twitch, *to-v4-customers, *http-src] - # Akamai - - per-second: 0.14 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 18h - multiplier: 1.3 - reverse-direction-ratio: 0.1 - <<: [*from-v4-akamai, *to-v4-customers, *http-src] - - per-second: 0.8 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 18h - multiplier: 1.3 - reverse-direction-ratio: 0.1 - <<: [*from-v6-akamai, *to-v6-customers, *http-src] - # Fastly - - per-second: 0.4 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 15h - multiplier: 1.3 - reverse-direction-ratio: 0.1 - <<: [*from-v4-fastly, *to-v4-customers, *http-src] - - per-second: 0.7 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 14h - multiplier: 1.3 - reverse-direction-ratio: 0.1 - <<: [*from-v6-fastly, *to-v6-customers, *http-src] - # Amazon - - per-second: 0.3 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 18h - multiplier: 1.3 - reverse-direction-ratio: 0.15 - <<: [*from-v4-amazon, *to-v4-customers, *http-src] - - per-second: 0.1 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 18h - multiplier: 1.3 - reverse-direction-ratio: 0.15 - <<: [*from-v6-amazon, *to-v6-customers, *http-src] - - # Random SSH - - per-second: 0.1 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 15h - multiplier: 1.2 - reverse-direction-ratio: 0.5 - <<: [*from-v4-renater, *to-v4-customers, *ssh-src] - # Servers - - per-second: 0.1 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 15h - multiplier: 1.2 - reverse-direction-ratio: 0.2 - <<: [*from-v4-renater, *to-v4-servers, *ssh-dst] - - per-second: 0.2 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 15h - multiplier: 1.2 - reverse-direction-ratio: 0.15 - <<: [*from-v4-random, *to-v4-servers, *http-dst] - - per-second: 0.2 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 15h - multiplier: 1.2 - reverse-direction-ratio: 0.1 - <<: [*from-v6-random, *to-v6-servers, *http-dst] - - # Noise - - &random-flow - per-second: 1 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 20h - multiplier: 1 - protocol: [tcp, udp] - srcport: [80, 443, 22, 25461, 8080, 4500, 993, 8801] - reverse-direction-ratio: 0.25 - <<: [*from-v4-random, *to-v4-customers] - - <<: [*from-v6-random, *to-v6-customers, *random-flow] - - snmp: - name: th2-edge2.example.com - interfaces: - 10: "Transit: Cogent" - 11: "IX: DECIX" - 20: "core" - 21: "core" - listen: 0.0.0.0:161 - bmp: - <<: *bmp - flows: - <<: *flows1 - seed: 100 - - snmp: - name: dc3-edge1.example.com - interfaces: - 10: "Transit: Tata" - 11: "Transit: Lumen" - 20: "core" - 21: "core" - listen: 0.0.0.0:161 - bmp: - <<: *bmp - flows: - <<: *flows1 - seed: 200 - - snmp: - name: dc5-edge2.example.com - interfaces: - 10: "IX: FranceIX" - 11: "Transit: Cogent" - 20: "core" - 21: "core" - listen: 0.0.0.0:161 - bmp: - <<: *bmp - flows: - <<: *flows1 - seed: 300 diff --git a/akvorado-inlet.yaml b/akvorado-inlet.yaml deleted file mode 100644 index 8ce5527c..00000000 --- a/akvorado-inlet.yaml +++ /dev/null @@ -1,44 +0,0 @@ ---- -kafka: - compression-codec: zstd -geoip: - optional: true - # When running on Docker, these paths are inside the container. - # Check docker-compose.yml for details. - asn-database: /usr/share/GeoIP/GeoLite2-ASN.mmdb - geo-database: /usr/share/GeoIP/GeoLite2-Country.mmdb -snmp: - workers: 10 - communities: - ::/0: public -flow: - inputs: - - type: udp - decoder: netflow - listen: 0.0.0.0:2055 - workers: 6 - receive-buffer: 10485760 - - type: udp - decoder: sflow - listen: 0.0.0.0:6343 - workers: 6 - receive-buffer: 10485760 -core: - workers: 6 - exporter-classifiers: - # This is an example. This should be customized depending on how - # your exporters are named. - - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") - - ClassifyRegion("europe") - - ClassifyTenant("acme") - - ClassifyRole("edge") - interface-classifiers: - # This is an example. This must be customized depending on the - # descriptions of your interfaces. In the following, we assume - # external interfaces are named "Transit: Cogent" Or "IX: - # FranceIX". - - | - ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && - ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && - ClassifyExternal() - - ClassifyInternal() diff --git a/akvorado.yaml b/akvorado.yaml index ba5e49c8..b1635126 100644 --- a/akvorado.yaml +++ b/akvorado.yaml @@ -3,7 +3,6 @@ # You can get all default values with `akvorado orchestrator /dev/null # --dump --check` or `docker-compose run akvorado-orchestrator # orchestrator /dev/null --dump --check`. - kafka: topic: flows version: 3.3.1 @@ -57,8 +56,414 @@ clickhouse: # .prefixes[] | # { prefix: (.ipv4Prefix // .ipv6Prefix), tenant: "google-cloud", region: .scope } -inlet: !include "akvorado-inlet.yaml" -console: !include "akvorado-console.yaml" +inlet: + kafka: + compression-codec: zstd + geoip: + optional: true + # When running on Docker, these paths are inside the container. + # Check docker-compose.yml for details. + asn-database: /usr/share/GeoIP/GeoLite2-ASN.mmdb + geo-database: /usr/share/GeoIP/GeoLite2-Country.mmdb + snmp: + workers: 10 + communities: + ::/0: public + flow: + inputs: + - type: udp + decoder: netflow + listen: 0.0.0.0:2055 + workers: 6 + receive-buffer: 10485760 + - type: udp + decoder: sflow + listen: 0.0.0.0:6343 + workers: 6 + receive-buffer: 10485760 + core: + workers: 6 + exporter-classifiers: + # This is an example. This should be customized depending on how + # your exporters are named. + - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") + - ClassifyRegion("europe") + - ClassifyTenant("acme") + - ClassifyRole("edge") + interface-classifiers: + # This is an example. This must be customized depending on the + # descriptions of your interfaces. In the following, we assume + # external interfaces are named "Transit: Cogent" Or "IX: + # FranceIX". + - | + ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && + ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && + ClassifyExternal() + - ClassifyInternal() -# Remove the following line if you don't want to get demo data -demo-exporter: !include "akvorado-demo.yaml" +console: + http: + cache: + type: redis + server: redis:6379 + database: + saved-filters: + # These are prepopulated filters you can select in a drop-down + # menu. Users can add more filters interactively. + - description: "From Netflix" + content: >- + InIfBoundary = external AND SrcAS = AS2906 + - description: "From GAFAM" + content: >- + InIfBoundary = external AND + SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075) + +# The remaining of this configuration file should be removed if you +# don't want to get demo data. + +.demo-exporter-flows: + - &http-src + src-port: [80, 443] + dst-port: 0 + protocol: tcp + size: 1300 + - &http-dst + src-port: 0 + dst-port: [80, 443] + protocol: tcp + size: 1300 + - &quic-src + src-port: 443 + dst-port: 0 + protocol: udp + size: 1200 + - &ssh-src + src-port: 22 + dst-port: 0 + protocol: tcp + size: 200 + - &ssh-dst + src-port: 0 + dst-port: 22 + protocol: tcp + size: 300 + - &to-v4-customers + dst-net: 192.0.2.0/24 + dst-as: 64501 + - &to-v6-customers + dst-net: 2a01:db8:cafe:1::/64 + dst-as: 64501 + - &to-v4-servers + dst-net: 203.0.113.0/24 + dst-as: 64501 + - &to-v6-servers + dst-net: 2a01:db8:cafe:2::/64 + dst-as: 64501 + - &from-v4-google + src-net: 216.58.206.0/24 + src-as: 15169 + - &from-v6-google + src-net: 2a00:1450:4007:807::2000/124 + src-as: 15169 + - &from-v4-facebook + src-net: 179.60.192.0/24 + src-as: 32934 + - &from-v6-facebook + src-net: 2a03:2880:f130:83:face:b00c:0::/112 + src-as: 32934 + - &from-v4-netflix + src-net: 198.38.120.0/23 + src-as: 2906 + - &from-v6-netflix + src-net: 2a00:86c0:115:115::/112 + src-as: 2906 + - &from-v4-akamai + src-net: 23.33.27.0/24 + src-as: 20940 + - &from-v6-akamai + src-net: 2a02:26f0:9100:28:0:17c0::/112 + src-as: 20940 + - &from-v4-amazon + src-net: 52.84.175.0/24 + src-as: 16509 + - &from-v6-amazon + src-net: 2600:9000:218d:4a00:15:74db::/112 + src-as: 16509 + - &from-v4-fastly + src-net: 199.232.178.0/29 + src-as: 54113 + - &from-v6-fastly + src-net: 2a04:4e42:1d::/126 + src-as: 54113 + - &from-v4-twitch + src-net: 52.223.202.128/27 + src-as: 46489 + - &from-v4-renater + src-net: 138.231.0.0/16 + src-as: 2269 + - &from-v4-random + src-net: 92.0.0.0/8 + src-as: [12322, 3215, 3303, 15557, 3320, 13335, 6185, 202818, 60068, 16276, 8075, 32590] + - &from-v6-random + src-net: 2a01:cb00::/32 + src-as: [12322, 3215, 3303, 15557, 3320, 13335, 6185, 202818, 60068, 16276, 8075, 32590] + +demo-exporter: + - snmp: + name: th2-edge1.example.com + interfaces: + 10: "Transit: Telia" + 11: "IX: AMSIX" + 20: "core" + 21: "core" + listen: 0.0.0.0:161 + bmp: &bmp + target: akvorado-inlet:10179 + routes: + - prefixes: 192.0.2.0/24,2a01:db8:cafe:1::/64 + aspath: 64501 + communities: 65401:10,65401:12 + large-communities: 65401:100:200,65401:100:201 + - prefixes: 203.0.113.0/24,2a01:db8:cafe:2::/64 + aspath: 65401 + communities: 65401:10,65401:13 + large-communities: 65401:100:200,65401:100:213 + - prefixes: 216.58.206.0/24,2a00:1450:4007:807::2000/124 + aspath: 174,1299,15169 + communities: 174:22004,174:21100 + - prefixes: 179.60.192.0/24,2a03:2880:f130:83:face:b00c:0::/112 + aspath: 1299,1299,32934 + communities: 1299:30000,1299:30220 + - prefixes: 198.38.120.0/23,2a00:86c0:115:115::/112 + aspath: 5511,1299,1299,32934 + communities: 1299:30000,1299:30310 + - prefixes: 23.33.27.0/24,2a02:26f0:9100:28:0:17c0::/112 + aspath: 174,174,174,20940 + communities: 174:22002,174:21200 + - prefixes: 52.84.175.0/24,2600:9000:218d:4a00:15:74db::/112 + aspath: 16509 + - prefixes: 199.232.178.0/29,2a04:4e42:1d::/126 + aspath: 1299,54113 + communities: 1299:35000,1299:35200 + - prefixes: 52.223.202.128/27 + aspath: 16509,46489 + - prefixes: 138.231.0.0/16 + aspath: 1299,174,2269,2269 + communities: 1299:30000,1299:30400 + - prefixes: 0.0.0.0/0 + aspath: 174 + - prefixes: ::/0 + aspath: 1299 + flows: &flows1 + samplingrate: 50000 + target: akvorado-inlet:2055 + flows: + # Google + - per-second: 1 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 16h + multiplier: 3 + reverse-direction-ratio: 0.1 + <<: [*from-v4-google, *to-v4-customers, *http-src] + - per-second: 0.5 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 16h + multiplier: 5 + reverse-direction-ratio: 0.1 + <<: [*from-v4-google, *to-v4-customers, *quic-src] + - per-second: 1.4 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 21h + multiplier: 3 + reverse-direction-ratio: 0.1 + <<: [*from-v6-google, *to-v6-customers, *http-src] + - per-second: 0.8 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 21h + multiplier: 5 + reverse-direction-ratio: 0.1 + <<: [*from-v6-google, *to-v6-customers, *quic-src] + # Facebook + - per-second: 1.1 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 16h + multiplier: 3 + reverse-direction-ratio: 0.2 + <<: [*from-v4-facebook, *to-v4-customers, *http-src] + - per-second: 0.2 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 16h + multiplier: 3 + reverse-direction-ratio: 0.2 + <<: [*from-v4-facebook, *to-v4-customers, *quic-src] + - per-second: 1.8 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 18h + multiplier: 3 + reverse-direction-ratio: 0.2 + <<: [*from-v6-facebook, *to-v6-customers, *http-src] + - per-second: 0.2 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 20h + multiplier: 3 + reverse-direction-ratio: 0.2 + <<: [*from-v6-facebook, *to-v6-customers, *quic-src] + # Netflix + - per-second: 0.2 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 22h + multiplier: 20 + reverse-direction-ratio: 0.1 + <<: [*from-v4-netflix, *to-v4-customers, *http-src] + - per-second: 0.7 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 22h + multiplier: 20 + reverse-direction-ratio: 0.1 + <<: [*from-v6-netflix, *to-v6-customers, *http-src] + # Twitch + - per-second: 0.12 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 21h + multiplier: 17 + reverse-direction-ratio: 0.4 + <<: [*from-v4-twitch, *to-v4-customers, *http-src] + # Akamai + - per-second: 0.14 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 18h + multiplier: 1.3 + reverse-direction-ratio: 0.1 + <<: [*from-v4-akamai, *to-v4-customers, *http-src] + - per-second: 0.8 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 18h + multiplier: 1.3 + reverse-direction-ratio: 0.1 + <<: [*from-v6-akamai, *to-v6-customers, *http-src] + # Fastly + - per-second: 0.4 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 15h + multiplier: 1.3 + reverse-direction-ratio: 0.1 + <<: [*from-v4-fastly, *to-v4-customers, *http-src] + - per-second: 0.7 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 14h + multiplier: 1.3 + reverse-direction-ratio: 0.1 + <<: [*from-v6-fastly, *to-v6-customers, *http-src] + # Amazon + - per-second: 0.3 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 18h + multiplier: 1.3 + reverse-direction-ratio: 0.15 + <<: [*from-v4-amazon, *to-v4-customers, *http-src] + - per-second: 0.1 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 18h + multiplier: 1.3 + reverse-direction-ratio: 0.15 + <<: [*from-v6-amazon, *to-v6-customers, *http-src] + + # Random SSH + - per-second: 0.1 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 15h + multiplier: 1.2 + reverse-direction-ratio: 0.5 + <<: [*from-v4-renater, *to-v4-customers, *ssh-src] + # Servers + - per-second: 0.1 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 15h + multiplier: 1.2 + reverse-direction-ratio: 0.2 + <<: [*from-v4-renater, *to-v4-servers, *ssh-dst] + - per-second: 0.2 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 15h + multiplier: 1.2 + reverse-direction-ratio: 0.15 + <<: [*from-v4-random, *to-v4-servers, *http-dst] + - per-second: 0.2 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 15h + multiplier: 1.2 + reverse-direction-ratio: 0.1 + <<: [*from-v6-random, *to-v6-servers, *http-dst] + + # Noise + - &random-flow + per-second: 1 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 20h + multiplier: 1 + protocol: [tcp, udp] + srcport: [80, 443, 22, 25461, 8080, 4500, 993, 8801] + reverse-direction-ratio: 0.25 + <<: [*from-v4-random, *to-v4-customers] + - <<: [*from-v6-random, *to-v6-customers, *random-flow] + - snmp: + name: th2-edge2.example.com + interfaces: + 10: "Transit: Cogent" + 11: "IX: DECIX" + 20: "core" + 21: "core" + listen: 0.0.0.0:161 + bmp: + <<: *bmp + flows: + <<: *flows1 + seed: 100 + - snmp: + name: dc3-edge1.example.com + interfaces: + 10: "Transit: Tata" + 11: "Transit: Lumen" + 20: "core" + 21: "core" + listen: 0.0.0.0:161 + bmp: + <<: *bmp + flows: + <<: *flows1 + seed: 200 + - snmp: + name: dc5-edge2.example.com + interfaces: + 10: "IX: FranceIX" + 11: "Transit: Cogent" + 20: "core" + 21: "core" + listen: 0.0.0.0:161 + bmp: + <<: *bmp + flows: + <<: *flows1 + seed: 300 diff --git a/cmd/config.go b/cmd/config.go index 8f5310d1..0e9f3734 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -6,11 +6,11 @@ package cmd import ( "fmt" "io" + "io/ioutil" "mime" "net/http" "net/url" "os" - "path/filepath" "reflect" "sort" "strconv" @@ -19,8 +19,7 @@ import ( "github.com/gin-gonic/gin" "github.com/go-playground/validator/v10" "github.com/mitchellh/mapstructure" - - "akvorado/common/helpers/yaml" + "gopkg.in/yaml.v3" "akvorado/common/helpers" ) @@ -67,15 +66,11 @@ func (c ConfigRelatedOptions) Parse(out io.Writer, component string, config inte return fmt.Errorf("unable to parse YAML configuration file: %w", err) } } else { - cfgFile, err := filepath.EvalSymlinks(cfgFile) + input, err := ioutil.ReadFile(cfgFile) if err != nil { - return fmt.Errorf("cannot follow symlink: %w", err) + return fmt.Errorf("unable to read configuration file: %w", err) } - dirname, filename := filepath.Split(cfgFile) - if dirname == "" { - dirname = "." - } - if err := yaml.UnmarshalWithInclude(os.DirFS(dirname), filename, &rawConfig); err != nil { + if err := yaml.Unmarshal(input, &rawConfig); err != nil { return fmt.Errorf("unable to parse YAML configuration file: %w", err) } } diff --git a/cmd/config_test.go b/cmd/config_test.go index 09ccbd32..c7a4c31a 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -16,8 +16,7 @@ import ( "time" "github.com/gin-gonic/gin" - - "akvorado/common/helpers/yaml" + "gopkg.in/yaml.v3" "akvorado/cmd" "akvorado/common/helpers" diff --git a/cmd/orchestrator_test.go b/cmd/orchestrator_test.go index 4d3d847b..72ad1be9 100644 --- a/cmd/orchestrator_test.go +++ b/cmd/orchestrator_test.go @@ -15,7 +15,7 @@ import ( "akvorado/common/helpers" "akvorado/common/reporter" - "akvorado/common/helpers/yaml" + "gopkg.in/yaml.v3" ) func TestOrchestratorStart(t *testing.T) { diff --git a/common/helpers/subnetmap_test.go b/common/helpers/subnetmap_test.go index 69ec8dca..7c6982ff 100644 --- a/common/helpers/subnetmap_test.go +++ b/common/helpers/subnetmap_test.go @@ -9,8 +9,7 @@ import ( "github.com/gin-gonic/gin" "github.com/mitchellh/mapstructure" - - "akvorado/common/helpers/yaml" + "gopkg.in/yaml.v3" "akvorado/common/helpers" ) diff --git a/common/helpers/tests_config.go b/common/helpers/tests_config.go index a73dbf26..40930bde 100644 --- a/common/helpers/tests_config.go +++ b/common/helpers/tests_config.go @@ -10,8 +10,7 @@ import ( "testing" "github.com/mitchellh/mapstructure" - - "akvorado/common/helpers/yaml" + "gopkg.in/yaml.v3" ) // ConfigurationDecodeCases describes a test case for configuration diff --git a/common/helpers/yaml/marshal.go b/common/helpers/yaml/marshal.go deleted file mode 100644 index 53dfbcd5..00000000 --- a/common/helpers/yaml/marshal.go +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-FileCopyrightText: 2023 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -package yaml - -import "gopkg.in/yaml.v3" - -// Marshal serializes the value provided into a YAML document. The structure of -// the generated document will reflect the structure of the value itself. Maps -// and pointers (to struct, string, int, etc) are accepted as the in value. -func Marshal(in interface{}) (out []byte, err error) { - return yaml.Marshal(in) -} diff --git a/common/helpers/yaml/testdata/1.yaml b/common/helpers/yaml/testdata/1.yaml deleted file mode 100644 index 03189f81..00000000 --- a/common/helpers/yaml/testdata/1.yaml +++ /dev/null @@ -1,2 +0,0 @@ ---- -name: "1.yaml" diff --git a/common/helpers/yaml/testdata/2.yaml b/common/helpers/yaml/testdata/2.yaml deleted file mode 100644 index c9f52d00..00000000 --- a/common/helpers/yaml/testdata/2.yaml +++ /dev/null @@ -1,2 +0,0 @@ ---- -name: "2.yaml" diff --git a/common/helpers/yaml/testdata/base.yaml b/common/helpers/yaml/testdata/base.yaml deleted file mode 100644 index 98c06afa..00000000 --- a/common/helpers/yaml/testdata/base.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -file1: !include 1.yaml -file2: !include 2.yaml -nested: !include nested.yaml -list1: !include list1.yaml -list2: !include list2.yaml diff --git a/common/helpers/yaml/testdata/list1.yaml b/common/helpers/yaml/testdata/list1.yaml deleted file mode 100644 index a1850dab..00000000 --- a/common/helpers/yaml/testdata/list1.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# We cannot include a list, so we use a map with an empty key -"": - - el1 - - el2 - - el3 diff --git a/common/helpers/yaml/testdata/list2.yaml b/common/helpers/yaml/testdata/list2.yaml deleted file mode 100644 index a596a513..00000000 --- a/common/helpers/yaml/testdata/list2.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- - -.hidden: - - &squid - protocol: tcp - size: 1300 - -"": - - *squid - - el2 - - el3 diff --git a/common/helpers/yaml/testdata/nested.yaml b/common/helpers/yaml/testdata/nested.yaml deleted file mode 100644 index 577cd0e1..00000000 --- a/common/helpers/yaml/testdata/nested.yaml +++ /dev/null @@ -1,2 +0,0 @@ ---- -file1: !include 1.yaml diff --git a/common/helpers/yaml/unmarshal.go b/common/helpers/yaml/unmarshal.go deleted file mode 100644 index 6e03b80c..00000000 --- a/common/helpers/yaml/unmarshal.go +++ /dev/null @@ -1,80 +0,0 @@ -// SPDX-FileCopyrightText: 2023 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -// Package yaml implements YAML support for the Go language. It adds the ability -// to use the "!include" tag. -package yaml - -import ( - "fmt" - "io/fs" - "strings" - - "gopkg.in/yaml.v3" -) - -// Unmarshal decodes the first document found within the in byte slice and -// assigns decoded values into the out value. -func Unmarshal(in []byte, out interface{}) (err error) { - return yaml.Unmarshal(in, out) -} - -// UnmarshalWithInclude decodes the first document found within the in byte -// slice and assigns decoded values into the out value. It also accepts the -// "!include" tag to include additional files contained in the provided fs. -func UnmarshalWithInclude(fsys fs.FS, input string, out interface{}) (err error) { - var outNode yaml.Node - in, err := fs.ReadFile(fsys, input) - if err != nil { - return fmt.Errorf("cannot read %s: %w", input, err) - } - if err := Unmarshal(in, &outNode); err != nil { - return fmt.Errorf("in %s: %w", input, err) - } - - if outNode.Kind == yaml.DocumentNode { - outNode = *outNode.Content[0] - } - if outNode.Kind == yaml.MappingNode { - // Remove hidden entries (prefixed with ".") - for i := 0; i < len(outNode.Content)-1; { - key := outNode.Content[i] - if key.Kind == yaml.ScalarNode && key.Tag == "!!str" && strings.HasPrefix(key.Value, ".") { - outNode.Content = outNode.Content[2:] - } else { - i += 2 - } - } - // If we only have a 1-entry map whose key is empty, use the value - if len(outNode.Content) == 2 { - key := outNode.Content[0] - if key.Kind == yaml.ScalarNode && key.Tag == "!!str" && key.Value == "" { - outNode = *outNode.Content[1] - } - } - } - - // Walk the content nodes and replace them with the file they refer to. - todo := []*yaml.Node{&outNode} - for len(todo) > 0 { - current := todo[0] - todo = todo[1:] - if current.Tag != "!include" { - todo = append(todo, current.Content...) - continue - } - if current.Alias != nil { - return fmt.Errorf("at line %d of %s, no alias is allowed for !include", current.Line, input) - } - if len(current.Content) > 0 { - return fmt.Errorf("at line %d of %s, no content is allowed for !include", current.Line, input) - } - var outNode yaml.Node - if err := UnmarshalWithInclude(fsys, current.Value, &outNode); err != nil { - return fmt.Errorf("at line %d of %s: %w", current.Line, input, err) - } - *current = outNode - } - - return outNode.Decode(out) -} diff --git a/common/helpers/yaml/unmarshal_test.go b/common/helpers/yaml/unmarshal_test.go deleted file mode 100644 index 3b0cea90..00000000 --- a/common/helpers/yaml/unmarshal_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-FileCopyrightText: 2023 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -package yaml_test - -import ( - "os" - "testing" - - "akvorado/common/helpers" - "akvorado/common/helpers/yaml" - - "github.com/gin-gonic/gin" -) - -func TestUnmarshalWithIn(t *testing.T) { - fsys := os.DirFS("testdata") - var got interface{} - if err := yaml.UnmarshalWithInclude(fsys, "base.yaml", &got); err != nil { - t.Fatalf("UnmarshalWithInclude() error:\n%+v", err) - } - expected := gin.H{ - "file1": gin.H{"name": "1.yaml"}, - "file2": gin.H{"name": "2.yaml"}, - "nested": gin.H{ - "file1": gin.H{"name": "1.yaml"}, - }, - "list1": []string{"el1", "el2", "el3"}, - "list2": []interface{}{gin.H{ - "protocol": "tcp", - "size": 1300, - }, "el2", "el3"}, - } - if diff := helpers.Diff(got, expected); diff != "" { - t.Fatalf("UnmarshalWithInclude() (-got, +want):\n%s", diff) - } -} diff --git a/console/data/docs/00-intro.md b/console/data/docs/00-intro.md index 59cddbe9..1a6eeeb5 100644 --- a/console/data/docs/00-intro.md +++ b/console/data/docs/00-intro.md @@ -28,7 +28,7 @@ Once running, *Akvorado* web interface should be running on port 8081. A few synthetic flows are generated in the background. To disable them: 1. Remove `:docker-compose-demo.yml` from `.env`, -2. Comment the last line of `akvorado.yaml`, and +2. Remove the associated configuration at the end of `akvorado.yaml`, and 3. Run `docker-compose up -d --remove-orphans`. If you want to send you own flows, the inlet is accepting both NetFlow diff --git a/console/data/docs/99-changelog.md b/console/data/docs/99-changelog.md index e3f8374f..6e27d79b 100644 --- a/console/data/docs/99-changelog.md +++ b/console/data/docs/99-changelog.md @@ -24,7 +24,6 @@ for a few columns. - 🩹 *console*: fix calculation of the L2 overhead when selecting L2 bps - 🩹 *console*: fix behavior of dimension limit field when empty - 🌱 *console*: accept `IN` and `NOTIN` operators for `ExporterAddr`, `SrcAddr`, `DstAddr`, `SrcAddrNAT`, `DstAddrNAT` -- 🌱 *common*: accept an `!include` tag to include other YAML files in `akvorado.yaml` - 🌱 *inlet*: optimize to reduce the number of queries to the system clock - 🌱 *orchestrator*: reduce storage for `InIfDescription`, `OutIfDescription`, `SrcAddr`, `DstAddr`, `Bytes`, and `Packets` diff --git a/inlet/flow/config_test.go b/inlet/flow/config_test.go index 88856dd2..3f6cfe6c 100644 --- a/inlet/flow/config_test.go +++ b/inlet/flow/config_test.go @@ -8,8 +8,7 @@ import ( "testing" "github.com/gin-gonic/gin" - - "akvorado/common/helpers/yaml" + "gopkg.in/yaml.v3" "akvorado/common/helpers" "akvorado/inlet/flow/input/file"