diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8627371e..216086a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -201,7 +201,7 @@ jobs: tar zcvf docker-compose-quickstart.tar.gz \ docker-compose.yml docker-compose-demo.yml .env \ orchestrator/clickhouse/data/docker-entrypoint.sh \ - akvorado.yaml + akvorado*.yaml # Publish release - name: Publish release diff --git a/akvorado-console.yaml b/akvorado-console.yaml new file mode 100644 index 00000000..78187578 --- /dev/null +++ b/akvorado-console.yaml @@ -0,0 +1,16 @@ +--- +http: + cache: + type: redis + server: redis:6379 +database: + saved-filters: + # These are prepopulated filters you can select in a drop-down + # menu. Users can add more filters interactively. + - description: "From Netflix" + content: >- + InIfBoundary = external AND SrcAS = AS2906 + - description: "From GAFAM" + content: >- + InIfBoundary = external AND + SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075) diff --git a/akvorado-demo.yaml b/akvorado-demo.yaml new file mode 100644 index 00000000..9531b5e4 --- /dev/null +++ b/akvorado-demo.yaml @@ -0,0 +1,347 @@ +--- +.demo-exporter-flows: + - &http-src + src-port: [80, 443] + dst-port: 0 + protocol: tcp + size: 1300 + - &http-dst + src-port: 0 + dst-port: [80, 443] + protocol: tcp + size: 1300 + - &quic-src + src-port: 443 + dst-port: 0 + protocol: udp + size: 1200 + - &ssh-src + src-port: 22 + dst-port: 0 + protocol: tcp + size: 200 + - &ssh-dst + src-port: 0 + dst-port: 22 + protocol: tcp + size: 300 + - &to-v4-customers + dst-net: 192.0.2.0/24 + dst-as: 64501 + - &to-v6-customers + dst-net: 2a01:db8:cafe:1::/64 + dst-as: 64501 + - &to-v4-servers + dst-net: 203.0.113.0/24 + dst-as: 64501 + - &to-v6-servers + dst-net: 2a01:db8:cafe:2::/64 + dst-as: 64501 + - &from-v4-google + src-net: 216.58.206.0/24 + src-as: 15169 + - &from-v6-google + src-net: 2a00:1450:4007:807::2000/124 + src-as: 15169 + - &from-v4-facebook + src-net: 179.60.192.0/24 + src-as: 32934 + - &from-v6-facebook + src-net: 2a03:2880:f130:83:face:b00c:0::/112 + src-as: 32934 + - &from-v4-netflix + src-net: 198.38.120.0/23 + src-as: 2906 + - &from-v6-netflix + src-net: 2a00:86c0:115:115::/112 + src-as: 2906 + - &from-v4-akamai + src-net: 23.33.27.0/24 + src-as: 20940 + - &from-v6-akamai + src-net: 2a02:26f0:9100:28:0:17c0::/112 + src-as: 20940 + - &from-v4-amazon + src-net: 52.84.175.0/24 + src-as: 16509 + - &from-v6-amazon + src-net: 2600:9000:218d:4a00:15:74db::/112 + src-as: 16509 + - &from-v4-fastly + src-net: 199.232.178.0/29 + src-as: 54113 + - &from-v6-fastly + src-net: 2a04:4e42:1d::/126 + src-as: 54113 + - &from-v4-twitch + src-net: 52.223.202.128/27 + src-as: 46489 + - &from-v4-renater + src-net: 138.231.0.0/16 + src-as: 2269 + - &from-v4-random + src-net: 92.0.0.0/8 + src-as: [12322, 3215, 3303, 15557, 3320, 13335, 6185, 202818, 60068, 16276, 8075, 32590] + - &from-v6-random + src-net: 2a01:cb00::/32 + src-as: [12322, 3215, 3303, 15557, 3320, 13335, 6185, 202818, 60068, 16276, 8075, 32590] + +"": + - snmp: + name: th2-edge1.example.com + interfaces: + 10: "Transit: Telia" + 11: "IX: AMSIX" + 20: "core" + 21: "core" + listen: 0.0.0.0:161 + bmp: &bmp + target: akvorado-inlet:10179 + routes: + - prefixes: 192.0.2.0/24,2a01:db8:cafe:1::/64 + aspath: 64501 + communities: 65401:10,65401:12 + large-communities: 65401:100:200,65401:100:201 + - prefixes: 203.0.113.0/24,2a01:db8:cafe:2::/64 + aspath: 65401 + communities: 65401:10,65401:13 + large-communities: 65401:100:200,65401:100:213 + - prefixes: 216.58.206.0/24,2a00:1450:4007:807::2000/124 + aspath: 174,1299,15169 + communities: 174:22004,174:21100 + - prefixes: 179.60.192.0/24,2a03:2880:f130:83:face:b00c:0::/112 + aspath: 1299,1299,32934 + communities: 1299:30000,1299:30220 + - prefixes: 198.38.120.0/23,2a00:86c0:115:115::/112 + aspath: 5511,1299,1299,32934 + communities: 1299:30000,1299:30310 + - prefixes: 23.33.27.0/24,2a02:26f0:9100:28:0:17c0::/112 + aspath: 174,174,174,20940 + communities: 174:22002,174:21200 + - prefixes: 52.84.175.0/24,2600:9000:218d:4a00:15:74db::/112 + aspath: 16509 + - prefixes: 199.232.178.0/29,2a04:4e42:1d::/126 + aspath: 1299,54113 + communities: 1299:35000,1299:35200 + - prefixes: 52.223.202.128/27 + aspath: 16509,46489 + - prefixes: 138.231.0.0/16 + aspath: 1299,174,2269,2269 + communities: 1299:30000,1299:30400 + - prefixes: 0.0.0.0/0 + aspath: 174 + - prefixes: ::/0 + aspath: 1299 + flows: &flows1 + samplingrate: 50000 + target: akvorado-inlet:2055 + flows: + # Google + - per-second: 1 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 16h + multiplier: 3 + reverse-direction-ratio: 0.1 + <<: [*from-v4-google, *to-v4-customers, *http-src] + - per-second: 0.5 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 16h + multiplier: 5 + reverse-direction-ratio: 0.1 + <<: [*from-v4-google, *to-v4-customers, *quic-src] + - per-second: 1.4 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 21h + multiplier: 3 + reverse-direction-ratio: 0.1 + <<: [*from-v6-google, *to-v6-customers, *http-src] + - per-second: 0.8 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 21h + multiplier: 5 + reverse-direction-ratio: 0.1 + <<: [*from-v6-google, *to-v6-customers, *quic-src] + # Facebook + - per-second: 1.1 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 16h + multiplier: 3 + reverse-direction-ratio: 0.2 + <<: [*from-v4-facebook, *to-v4-customers, *http-src] + - per-second: 0.2 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 16h + multiplier: 3 + reverse-direction-ratio: 0.2 + <<: [*from-v4-facebook, *to-v4-customers, *quic-src] + - per-second: 1.8 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 18h + multiplier: 3 + reverse-direction-ratio: 0.2 + <<: [*from-v6-facebook, *to-v6-customers, *http-src] + - per-second: 0.2 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 20h + multiplier: 3 + reverse-direction-ratio: 0.2 + <<: [*from-v6-facebook, *to-v6-customers, *quic-src] + # Netflix + - per-second: 0.2 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 22h + multiplier: 20 + reverse-direction-ratio: 0.1 + <<: [*from-v4-netflix, *to-v4-customers, *http-src] + - per-second: 0.7 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 22h + multiplier: 20 + reverse-direction-ratio: 0.1 + <<: [*from-v6-netflix, *to-v6-customers, *http-src] + # Twitch + - per-second: 0.12 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 21h + multiplier: 17 + reverse-direction-ratio: 0.4 + <<: [*from-v4-twitch, *to-v4-customers, *http-src] + # Akamai + - per-second: 0.14 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 18h + multiplier: 1.3 + reverse-direction-ratio: 0.1 + <<: [*from-v4-akamai, *to-v4-customers, *http-src] + - per-second: 0.8 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 18h + multiplier: 1.3 + reverse-direction-ratio: 0.1 + <<: [*from-v6-akamai, *to-v6-customers, *http-src] + # Fastly + - per-second: 0.4 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 15h + multiplier: 1.3 + reverse-direction-ratio: 0.1 + <<: [*from-v4-fastly, *to-v4-customers, *http-src] + - per-second: 0.7 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 14h + multiplier: 1.3 + reverse-direction-ratio: 0.1 + <<: [*from-v6-fastly, *to-v6-customers, *http-src] + # Amazon + - per-second: 0.3 + in-if-index: [10, 11] + out-if-index: [20, 21] + peak-hour: 18h + multiplier: 1.3 + reverse-direction-ratio: 0.15 + <<: [*from-v4-amazon, *to-v4-customers, *http-src] + - per-second: 0.1 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 18h + multiplier: 1.3 + reverse-direction-ratio: 0.15 + <<: [*from-v6-amazon, *to-v6-customers, *http-src] + + # Random SSH + - per-second: 0.1 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 15h + multiplier: 1.2 + reverse-direction-ratio: 0.5 + <<: [*from-v4-renater, *to-v4-customers, *ssh-src] + # Servers + - per-second: 0.1 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 15h + multiplier: 1.2 + reverse-direction-ratio: 0.2 + <<: [*from-v4-renater, *to-v4-servers, *ssh-dst] + - per-second: 0.2 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 15h + multiplier: 1.2 + reverse-direction-ratio: 0.15 + <<: [*from-v4-random, *to-v4-servers, *http-dst] + - per-second: 0.2 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 15h + multiplier: 1.2 + reverse-direction-ratio: 0.1 + <<: [*from-v6-random, *to-v6-servers, *http-dst] + + # Noise + - &random-flow + per-second: 1 + in-if-index: 10 + out-if-index: [20, 21] + peak-hour: 20h + multiplier: 1 + protocol: [tcp, udp] + srcport: [80, 443, 22, 25461, 8080, 4500, 993, 8801] + reverse-direction-ratio: 0.25 + <<: [*from-v4-random, *to-v4-customers] + - <<: [*from-v6-random, *to-v6-customers, *random-flow] + - snmp: + name: th2-edge2.example.com + interfaces: + 10: "Transit: Cogent" + 11: "IX: DECIX" + 20: "core" + 21: "core" + listen: 0.0.0.0:161 + bmp: + <<: *bmp + flows: + <<: *flows1 + seed: 100 + - snmp: + name: dc3-edge1.example.com + interfaces: + 10: "Transit: Tata" + 11: "Transit: Lumen" + 20: "core" + 21: "core" + listen: 0.0.0.0:161 + bmp: + <<: *bmp + flows: + <<: *flows1 + seed: 200 + - snmp: + name: dc5-edge2.example.com + interfaces: + 10: "IX: FranceIX" + 11: "Transit: Cogent" + 20: "core" + 21: "core" + listen: 0.0.0.0:161 + bmp: + <<: *bmp + flows: + <<: *flows1 + seed: 300 diff --git a/akvorado-inlet.yaml b/akvorado-inlet.yaml new file mode 100644 index 00000000..8ce5527c --- /dev/null +++ b/akvorado-inlet.yaml @@ -0,0 +1,44 @@ +--- +kafka: + compression-codec: zstd +geoip: + optional: true + # When running on Docker, these paths are inside the container. + # Check docker-compose.yml for details. + asn-database: /usr/share/GeoIP/GeoLite2-ASN.mmdb + geo-database: /usr/share/GeoIP/GeoLite2-Country.mmdb +snmp: + workers: 10 + communities: + ::/0: public +flow: + inputs: + - type: udp + decoder: netflow + listen: 0.0.0.0:2055 + workers: 6 + receive-buffer: 10485760 + - type: udp + decoder: sflow + listen: 0.0.0.0:6343 + workers: 6 + receive-buffer: 10485760 +core: + workers: 6 + exporter-classifiers: + # This is an example. This should be customized depending on how + # your exporters are named. + - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") + - ClassifyRegion("europe") + - ClassifyTenant("acme") + - ClassifyRole("edge") + interface-classifiers: + # This is an example. This must be customized depending on the + # descriptions of your interfaces. In the following, we assume + # external interfaces are named "Transit: Cogent" Or "IX: + # FranceIX". + - | + ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && + ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && + ClassifyExternal() + - ClassifyInternal() diff --git a/akvorado.yaml b/akvorado.yaml index b1635126..ba5e49c8 100644 --- a/akvorado.yaml +++ b/akvorado.yaml @@ -3,6 +3,7 @@ # You can get all default values with `akvorado orchestrator /dev/null # --dump --check` or `docker-compose run akvorado-orchestrator # orchestrator /dev/null --dump --check`. + kafka: topic: flows version: 3.3.1 @@ -56,414 +57,8 @@ clickhouse: # .prefixes[] | # { prefix: (.ipv4Prefix // .ipv6Prefix), tenant: "google-cloud", region: .scope } -inlet: - kafka: - compression-codec: zstd - geoip: - optional: true - # When running on Docker, these paths are inside the container. - # Check docker-compose.yml for details. - asn-database: /usr/share/GeoIP/GeoLite2-ASN.mmdb - geo-database: /usr/share/GeoIP/GeoLite2-Country.mmdb - snmp: - workers: 10 - communities: - ::/0: public - flow: - inputs: - - type: udp - decoder: netflow - listen: 0.0.0.0:2055 - workers: 6 - receive-buffer: 10485760 - - type: udp - decoder: sflow - listen: 0.0.0.0:6343 - workers: 6 - receive-buffer: 10485760 - core: - workers: 6 - exporter-classifiers: - # This is an example. This should be customized depending on how - # your exporters are named. - - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") - - ClassifyRegion("europe") - - ClassifyTenant("acme") - - ClassifyRole("edge") - interface-classifiers: - # This is an example. This must be customized depending on the - # descriptions of your interfaces. In the following, we assume - # external interfaces are named "Transit: Cogent" Or "IX: - # FranceIX". - - | - ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && - ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && - ClassifyExternal() - - ClassifyInternal() +inlet: !include "akvorado-inlet.yaml" +console: !include "akvorado-console.yaml" -console: - http: - cache: - type: redis - server: redis:6379 - database: - saved-filters: - # These are prepopulated filters you can select in a drop-down - # menu. Users can add more filters interactively. - - description: "From Netflix" - content: >- - InIfBoundary = external AND SrcAS = AS2906 - - description: "From GAFAM" - content: >- - InIfBoundary = external AND - SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075) - -# The remaining of this configuration file should be removed if you -# don't want to get demo data. - -.demo-exporter-flows: - - &http-src - src-port: [80, 443] - dst-port: 0 - protocol: tcp - size: 1300 - - &http-dst - src-port: 0 - dst-port: [80, 443] - protocol: tcp - size: 1300 - - &quic-src - src-port: 443 - dst-port: 0 - protocol: udp - size: 1200 - - &ssh-src - src-port: 22 - dst-port: 0 - protocol: tcp - size: 200 - - &ssh-dst - src-port: 0 - dst-port: 22 - protocol: tcp - size: 300 - - &to-v4-customers - dst-net: 192.0.2.0/24 - dst-as: 64501 - - &to-v6-customers - dst-net: 2a01:db8:cafe:1::/64 - dst-as: 64501 - - &to-v4-servers - dst-net: 203.0.113.0/24 - dst-as: 64501 - - &to-v6-servers - dst-net: 2a01:db8:cafe:2::/64 - dst-as: 64501 - - &from-v4-google - src-net: 216.58.206.0/24 - src-as: 15169 - - &from-v6-google - src-net: 2a00:1450:4007:807::2000/124 - src-as: 15169 - - &from-v4-facebook - src-net: 179.60.192.0/24 - src-as: 32934 - - &from-v6-facebook - src-net: 2a03:2880:f130:83:face:b00c:0::/112 - src-as: 32934 - - &from-v4-netflix - src-net: 198.38.120.0/23 - src-as: 2906 - - &from-v6-netflix - src-net: 2a00:86c0:115:115::/112 - src-as: 2906 - - &from-v4-akamai - src-net: 23.33.27.0/24 - src-as: 20940 - - &from-v6-akamai - src-net: 2a02:26f0:9100:28:0:17c0::/112 - src-as: 20940 - - &from-v4-amazon - src-net: 52.84.175.0/24 - src-as: 16509 - - &from-v6-amazon - src-net: 2600:9000:218d:4a00:15:74db::/112 - src-as: 16509 - - &from-v4-fastly - src-net: 199.232.178.0/29 - src-as: 54113 - - &from-v6-fastly - src-net: 2a04:4e42:1d::/126 - src-as: 54113 - - &from-v4-twitch - src-net: 52.223.202.128/27 - src-as: 46489 - - &from-v4-renater - src-net: 138.231.0.0/16 - src-as: 2269 - - &from-v4-random - src-net: 92.0.0.0/8 - src-as: [12322, 3215, 3303, 15557, 3320, 13335, 6185, 202818, 60068, 16276, 8075, 32590] - - &from-v6-random - src-net: 2a01:cb00::/32 - src-as: [12322, 3215, 3303, 15557, 3320, 13335, 6185, 202818, 60068, 16276, 8075, 32590] - -demo-exporter: - - snmp: - name: th2-edge1.example.com - interfaces: - 10: "Transit: Telia" - 11: "IX: AMSIX" - 20: "core" - 21: "core" - listen: 0.0.0.0:161 - bmp: &bmp - target: akvorado-inlet:10179 - routes: - - prefixes: 192.0.2.0/24,2a01:db8:cafe:1::/64 - aspath: 64501 - communities: 65401:10,65401:12 - large-communities: 65401:100:200,65401:100:201 - - prefixes: 203.0.113.0/24,2a01:db8:cafe:2::/64 - aspath: 65401 - communities: 65401:10,65401:13 - large-communities: 65401:100:200,65401:100:213 - - prefixes: 216.58.206.0/24,2a00:1450:4007:807::2000/124 - aspath: 174,1299,15169 - communities: 174:22004,174:21100 - - prefixes: 179.60.192.0/24,2a03:2880:f130:83:face:b00c:0::/112 - aspath: 1299,1299,32934 - communities: 1299:30000,1299:30220 - - prefixes: 198.38.120.0/23,2a00:86c0:115:115::/112 - aspath: 5511,1299,1299,32934 - communities: 1299:30000,1299:30310 - - prefixes: 23.33.27.0/24,2a02:26f0:9100:28:0:17c0::/112 - aspath: 174,174,174,20940 - communities: 174:22002,174:21200 - - prefixes: 52.84.175.0/24,2600:9000:218d:4a00:15:74db::/112 - aspath: 16509 - - prefixes: 199.232.178.0/29,2a04:4e42:1d::/126 - aspath: 1299,54113 - communities: 1299:35000,1299:35200 - - prefixes: 52.223.202.128/27 - aspath: 16509,46489 - - prefixes: 138.231.0.0/16 - aspath: 1299,174,2269,2269 - communities: 1299:30000,1299:30400 - - prefixes: 0.0.0.0/0 - aspath: 174 - - prefixes: ::/0 - aspath: 1299 - flows: &flows1 - samplingrate: 50000 - target: akvorado-inlet:2055 - flows: - # Google - - per-second: 1 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 16h - multiplier: 3 - reverse-direction-ratio: 0.1 - <<: [*from-v4-google, *to-v4-customers, *http-src] - - per-second: 0.5 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 16h - multiplier: 5 - reverse-direction-ratio: 0.1 - <<: [*from-v4-google, *to-v4-customers, *quic-src] - - per-second: 1.4 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 21h - multiplier: 3 - reverse-direction-ratio: 0.1 - <<: [*from-v6-google, *to-v6-customers, *http-src] - - per-second: 0.8 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 21h - multiplier: 5 - reverse-direction-ratio: 0.1 - <<: [*from-v6-google, *to-v6-customers, *quic-src] - # Facebook - - per-second: 1.1 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 16h - multiplier: 3 - reverse-direction-ratio: 0.2 - <<: [*from-v4-facebook, *to-v4-customers, *http-src] - - per-second: 0.2 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 16h - multiplier: 3 - reverse-direction-ratio: 0.2 - <<: [*from-v4-facebook, *to-v4-customers, *quic-src] - - per-second: 1.8 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 18h - multiplier: 3 - reverse-direction-ratio: 0.2 - <<: [*from-v6-facebook, *to-v6-customers, *http-src] - - per-second: 0.2 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 20h - multiplier: 3 - reverse-direction-ratio: 0.2 - <<: [*from-v6-facebook, *to-v6-customers, *quic-src] - # Netflix - - per-second: 0.2 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 22h - multiplier: 20 - reverse-direction-ratio: 0.1 - <<: [*from-v4-netflix, *to-v4-customers, *http-src] - - per-second: 0.7 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 22h - multiplier: 20 - reverse-direction-ratio: 0.1 - <<: [*from-v6-netflix, *to-v6-customers, *http-src] - # Twitch - - per-second: 0.12 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 21h - multiplier: 17 - reverse-direction-ratio: 0.4 - <<: [*from-v4-twitch, *to-v4-customers, *http-src] - # Akamai - - per-second: 0.14 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 18h - multiplier: 1.3 - reverse-direction-ratio: 0.1 - <<: [*from-v4-akamai, *to-v4-customers, *http-src] - - per-second: 0.8 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 18h - multiplier: 1.3 - reverse-direction-ratio: 0.1 - <<: [*from-v6-akamai, *to-v6-customers, *http-src] - # Fastly - - per-second: 0.4 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 15h - multiplier: 1.3 - reverse-direction-ratio: 0.1 - <<: [*from-v4-fastly, *to-v4-customers, *http-src] - - per-second: 0.7 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 14h - multiplier: 1.3 - reverse-direction-ratio: 0.1 - <<: [*from-v6-fastly, *to-v6-customers, *http-src] - # Amazon - - per-second: 0.3 - in-if-index: [10, 11] - out-if-index: [20, 21] - peak-hour: 18h - multiplier: 1.3 - reverse-direction-ratio: 0.15 - <<: [*from-v4-amazon, *to-v4-customers, *http-src] - - per-second: 0.1 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 18h - multiplier: 1.3 - reverse-direction-ratio: 0.15 - <<: [*from-v6-amazon, *to-v6-customers, *http-src] - - # Random SSH - - per-second: 0.1 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 15h - multiplier: 1.2 - reverse-direction-ratio: 0.5 - <<: [*from-v4-renater, *to-v4-customers, *ssh-src] - # Servers - - per-second: 0.1 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 15h - multiplier: 1.2 - reverse-direction-ratio: 0.2 - <<: [*from-v4-renater, *to-v4-servers, *ssh-dst] - - per-second: 0.2 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 15h - multiplier: 1.2 - reverse-direction-ratio: 0.15 - <<: [*from-v4-random, *to-v4-servers, *http-dst] - - per-second: 0.2 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 15h - multiplier: 1.2 - reverse-direction-ratio: 0.1 - <<: [*from-v6-random, *to-v6-servers, *http-dst] - - # Noise - - &random-flow - per-second: 1 - in-if-index: 10 - out-if-index: [20, 21] - peak-hour: 20h - multiplier: 1 - protocol: [tcp, udp] - srcport: [80, 443, 22, 25461, 8080, 4500, 993, 8801] - reverse-direction-ratio: 0.25 - <<: [*from-v4-random, *to-v4-customers] - - <<: [*from-v6-random, *to-v6-customers, *random-flow] - - snmp: - name: th2-edge2.example.com - interfaces: - 10: "Transit: Cogent" - 11: "IX: DECIX" - 20: "core" - 21: "core" - listen: 0.0.0.0:161 - bmp: - <<: *bmp - flows: - <<: *flows1 - seed: 100 - - snmp: - name: dc3-edge1.example.com - interfaces: - 10: "Transit: Tata" - 11: "Transit: Lumen" - 20: "core" - 21: "core" - listen: 0.0.0.0:161 - bmp: - <<: *bmp - flows: - <<: *flows1 - seed: 200 - - snmp: - name: dc5-edge2.example.com - interfaces: - 10: "IX: FranceIX" - 11: "Transit: Cogent" - 20: "core" - 21: "core" - listen: 0.0.0.0:161 - bmp: - <<: *bmp - flows: - <<: *flows1 - seed: 300 +# Remove the following line if you don't want to get demo data +demo-exporter: !include "akvorado-demo.yaml" diff --git a/cmd/config.go b/cmd/config.go index 0e9f3734..8f5310d1 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -6,11 +6,11 @@ package cmd import ( "fmt" "io" - "io/ioutil" "mime" "net/http" "net/url" "os" + "path/filepath" "reflect" "sort" "strconv" @@ -19,7 +19,8 @@ import ( "github.com/gin-gonic/gin" "github.com/go-playground/validator/v10" "github.com/mitchellh/mapstructure" - "gopkg.in/yaml.v3" + + "akvorado/common/helpers/yaml" "akvorado/common/helpers" ) @@ -66,11 +67,15 @@ func (c ConfigRelatedOptions) Parse(out io.Writer, component string, config inte return fmt.Errorf("unable to parse YAML configuration file: %w", err) } } else { - input, err := ioutil.ReadFile(cfgFile) + cfgFile, err := filepath.EvalSymlinks(cfgFile) if err != nil { - return fmt.Errorf("unable to read configuration file: %w", err) + return fmt.Errorf("cannot follow symlink: %w", err) } - if err := yaml.Unmarshal(input, &rawConfig); err != nil { + dirname, filename := filepath.Split(cfgFile) + if dirname == "" { + dirname = "." + } + if err := yaml.UnmarshalWithInclude(os.DirFS(dirname), filename, &rawConfig); err != nil { return fmt.Errorf("unable to parse YAML configuration file: %w", err) } } diff --git a/cmd/config_test.go b/cmd/config_test.go index a4625eb3..2aff12ac 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -16,7 +16,8 @@ import ( "time" "github.com/gin-gonic/gin" - "gopkg.in/yaml.v3" + + "akvorado/common/helpers/yaml" "akvorado/cmd" "akvorado/common/helpers" diff --git a/cmd/orchestrator_test.go b/cmd/orchestrator_test.go index 58a57c0e..10b3eda2 100644 --- a/cmd/orchestrator_test.go +++ b/cmd/orchestrator_test.go @@ -15,7 +15,7 @@ import ( "akvorado/common/helpers" "akvorado/common/reporter" - "gopkg.in/yaml.v3" + "akvorado/common/helpers/yaml" ) func TestOrchestratorStart(t *testing.T) { diff --git a/common/helpers/subnetmap_test.go b/common/helpers/subnetmap_test.go index 7c6982ff..69ec8dca 100644 --- a/common/helpers/subnetmap_test.go +++ b/common/helpers/subnetmap_test.go @@ -9,7 +9,8 @@ import ( "github.com/gin-gonic/gin" "github.com/mitchellh/mapstructure" - "gopkg.in/yaml.v3" + + "akvorado/common/helpers/yaml" "akvorado/common/helpers" ) diff --git a/common/helpers/tests_config.go b/common/helpers/tests_config.go index 40930bde..a73dbf26 100644 --- a/common/helpers/tests_config.go +++ b/common/helpers/tests_config.go @@ -10,7 +10,8 @@ import ( "testing" "github.com/mitchellh/mapstructure" - "gopkg.in/yaml.v3" + + "akvorado/common/helpers/yaml" ) // ConfigurationDecodeCases describes a test case for configuration diff --git a/common/helpers/yaml/marshal.go b/common/helpers/yaml/marshal.go new file mode 100644 index 00000000..53dfbcd5 --- /dev/null +++ b/common/helpers/yaml/marshal.go @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2023 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package yaml + +import "gopkg.in/yaml.v3" + +// Marshal serializes the value provided into a YAML document. The structure of +// the generated document will reflect the structure of the value itself. Maps +// and pointers (to struct, string, int, etc) are accepted as the in value. +func Marshal(in interface{}) (out []byte, err error) { + return yaml.Marshal(in) +} diff --git a/common/helpers/yaml/testdata/1.yaml b/common/helpers/yaml/testdata/1.yaml new file mode 100644 index 00000000..03189f81 --- /dev/null +++ b/common/helpers/yaml/testdata/1.yaml @@ -0,0 +1,2 @@ +--- +name: "1.yaml" diff --git a/common/helpers/yaml/testdata/2.yaml b/common/helpers/yaml/testdata/2.yaml new file mode 100644 index 00000000..c9f52d00 --- /dev/null +++ b/common/helpers/yaml/testdata/2.yaml @@ -0,0 +1,2 @@ +--- +name: "2.yaml" diff --git a/common/helpers/yaml/testdata/base.yaml b/common/helpers/yaml/testdata/base.yaml new file mode 100644 index 00000000..98c06afa --- /dev/null +++ b/common/helpers/yaml/testdata/base.yaml @@ -0,0 +1,6 @@ +--- +file1: !include 1.yaml +file2: !include 2.yaml +nested: !include nested.yaml +list1: !include list1.yaml +list2: !include list2.yaml diff --git a/common/helpers/yaml/testdata/list1.yaml b/common/helpers/yaml/testdata/list1.yaml new file mode 100644 index 00000000..a1850dab --- /dev/null +++ b/common/helpers/yaml/testdata/list1.yaml @@ -0,0 +1,6 @@ +--- +# We cannot include a list, so we use a map with an empty key +"": + - el1 + - el2 + - el3 diff --git a/common/helpers/yaml/testdata/list2.yaml b/common/helpers/yaml/testdata/list2.yaml new file mode 100644 index 00000000..a596a513 --- /dev/null +++ b/common/helpers/yaml/testdata/list2.yaml @@ -0,0 +1,11 @@ +--- + +.hidden: + - &squid + protocol: tcp + size: 1300 + +"": + - *squid + - el2 + - el3 diff --git a/common/helpers/yaml/testdata/nested.yaml b/common/helpers/yaml/testdata/nested.yaml new file mode 100644 index 00000000..577cd0e1 --- /dev/null +++ b/common/helpers/yaml/testdata/nested.yaml @@ -0,0 +1,2 @@ +--- +file1: !include 1.yaml diff --git a/common/helpers/yaml/unmarshal.go b/common/helpers/yaml/unmarshal.go new file mode 100644 index 00000000..6e03b80c --- /dev/null +++ b/common/helpers/yaml/unmarshal.go @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: 2023 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +// Package yaml implements YAML support for the Go language. It adds the ability +// to use the "!include" tag. +package yaml + +import ( + "fmt" + "io/fs" + "strings" + + "gopkg.in/yaml.v3" +) + +// Unmarshal decodes the first document found within the in byte slice and +// assigns decoded values into the out value. +func Unmarshal(in []byte, out interface{}) (err error) { + return yaml.Unmarshal(in, out) +} + +// UnmarshalWithInclude decodes the first document found within the in byte +// slice and assigns decoded values into the out value. It also accepts the +// "!include" tag to include additional files contained in the provided fs. +func UnmarshalWithInclude(fsys fs.FS, input string, out interface{}) (err error) { + var outNode yaml.Node + in, err := fs.ReadFile(fsys, input) + if err != nil { + return fmt.Errorf("cannot read %s: %w", input, err) + } + if err := Unmarshal(in, &outNode); err != nil { + return fmt.Errorf("in %s: %w", input, err) + } + + if outNode.Kind == yaml.DocumentNode { + outNode = *outNode.Content[0] + } + if outNode.Kind == yaml.MappingNode { + // Remove hidden entries (prefixed with ".") + for i := 0; i < len(outNode.Content)-1; { + key := outNode.Content[i] + if key.Kind == yaml.ScalarNode && key.Tag == "!!str" && strings.HasPrefix(key.Value, ".") { + outNode.Content = outNode.Content[2:] + } else { + i += 2 + } + } + // If we only have a 1-entry map whose key is empty, use the value + if len(outNode.Content) == 2 { + key := outNode.Content[0] + if key.Kind == yaml.ScalarNode && key.Tag == "!!str" && key.Value == "" { + outNode = *outNode.Content[1] + } + } + } + + // Walk the content nodes and replace them with the file they refer to. + todo := []*yaml.Node{&outNode} + for len(todo) > 0 { + current := todo[0] + todo = todo[1:] + if current.Tag != "!include" { + todo = append(todo, current.Content...) + continue + } + if current.Alias != nil { + return fmt.Errorf("at line %d of %s, no alias is allowed for !include", current.Line, input) + } + if len(current.Content) > 0 { + return fmt.Errorf("at line %d of %s, no content is allowed for !include", current.Line, input) + } + var outNode yaml.Node + if err := UnmarshalWithInclude(fsys, current.Value, &outNode); err != nil { + return fmt.Errorf("at line %d of %s: %w", current.Line, input, err) + } + *current = outNode + } + + return outNode.Decode(out) +} diff --git a/common/helpers/yaml/unmarshal_test.go b/common/helpers/yaml/unmarshal_test.go new file mode 100644 index 00000000..3b0cea90 --- /dev/null +++ b/common/helpers/yaml/unmarshal_test.go @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: 2023 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package yaml_test + +import ( + "os" + "testing" + + "akvorado/common/helpers" + "akvorado/common/helpers/yaml" + + "github.com/gin-gonic/gin" +) + +func TestUnmarshalWithIn(t *testing.T) { + fsys := os.DirFS("testdata") + var got interface{} + if err := yaml.UnmarshalWithInclude(fsys, "base.yaml", &got); err != nil { + t.Fatalf("UnmarshalWithInclude() error:\n%+v", err) + } + expected := gin.H{ + "file1": gin.H{"name": "1.yaml"}, + "file2": gin.H{"name": "2.yaml"}, + "nested": gin.H{ + "file1": gin.H{"name": "1.yaml"}, + }, + "list1": []string{"el1", "el2", "el3"}, + "list2": []interface{}{gin.H{ + "protocol": "tcp", + "size": 1300, + }, "el2", "el3"}, + } + if diff := helpers.Diff(got, expected); diff != "" { + t.Fatalf("UnmarshalWithInclude() (-got, +want):\n%s", diff) + } +} diff --git a/console/data/docs/00-intro.md b/console/data/docs/00-intro.md index 1a6eeeb5..59cddbe9 100644 --- a/console/data/docs/00-intro.md +++ b/console/data/docs/00-intro.md @@ -28,7 +28,7 @@ Once running, *Akvorado* web interface should be running on port 8081. A few synthetic flows are generated in the background. To disable them: 1. Remove `:docker-compose-demo.yml` from `.env`, -2. Remove the associated configuration at the end of `akvorado.yaml`, and +2. Comment the last line of `akvorado.yaml`, and 3. Run `docker-compose up -d --remove-orphans`. If you want to send you own flows, the inlet is accepting both NetFlow diff --git a/console/data/docs/99-changelog.md b/console/data/docs/99-changelog.md index b96f6734..14cc0c69 100644 --- a/console/data/docs/99-changelog.md +++ b/console/data/docs/99-changelog.md @@ -11,6 +11,10 @@ identified with a specific icon: - 🩹: bug fix - 🌱: miscellaneous change +## Unreleased + +- 🌱 *common*: accept an `!include` tag to include other YAML files in `akvorado.yaml` + ## 1.7.1 - 2023-01-27 This is an important bugfix release. `DstNet*` values were classified using the diff --git a/inlet/flow/config_test.go b/inlet/flow/config_test.go index 3f6cfe6c..88856dd2 100644 --- a/inlet/flow/config_test.go +++ b/inlet/flow/config_test.go @@ -8,7 +8,8 @@ import ( "testing" "github.com/gin-gonic/gin" - "gopkg.in/yaml.v3" + + "akvorado/common/helpers/yaml" "akvorado/common/helpers" "akvorado/inlet/flow/input/file"