From ac68c5970e2c8eb1444483814e6a7b1f5c1956e3 Mon Sep 17 00:00:00 2001 From: Vincent Bernat Date: Tue, 17 Dec 2024 06:31:10 +0100 Subject: [PATCH] inlet: split inlet into new inlet and outlet This change split the inlet component into a simpler inlet and a new outlet component. The new inlet component receive flows and put them in Kafka, unparsed. The outlet component takes them from Kafka and resume the processing from here (flow parsing, enrichment) and puts them in ClickHouse. The main goal is to ensure the inlet does a minimal work to not be late when processing packets (and restart faster). It also brings some simplification as the number of knobs to tune everything is reduced: for inlet, we only need to tune the queue size for UDP, the number of workers and a few Kafka parameters; for outlet, we need to tune a few Kafka parameters, the number of workers and a few ClickHouse parameters. The outlet component features a simple Kafka input component. The core component becomes just a callback function. There is also a new ClickHouse component to push data to ClickHouse using the low-level ch-go library with batch inserts. This processing has an impact on the internal representation of a FlowMessage. Previously, it was tailored to dynamically build the protobuf message to be put in Kafka. Now, it builds the batch request to be sent to ClickHouse. This makes the FlowMessage structure hides the content of the next batch request and therefore, it should be reused. This also changes the way we decode flows as they don't output FlowMessage anymore, they reuse one that is provided to each worker. The ClickHouse tables are slightly updated. Instead of using Kafka engine, the Null engine is used instead. Fix #1122 --- .github/workflows/ci.yml | 8 +- .gitignore | 3 +- .gitlab-ci.yml | 2 +- Makefile | 49 +- bin/protoc-gen-go | 2 + cmd/inlet.go | 182 +----- cmd/orchestrator.go | 320 +++++---- cmd/outlet.go | 304 +++++++++ cmd/outlet_test.go | 31 + .../bmp-to-routing/expected.yaml | 6 +- .../core-ignore-asn-from-flows/expected.yaml | 2 +- .../configurations/empty/expected.yaml | 2 +- .../configurations/gnmi/expected.yaml | 2 +- .../schema-customdict/expected.yaml | 4 +- .../schema-enable-disable/expected.yaml | 2 +- .../configurations/shipped/expected.yaml | 2 +- .../snmp-communities-migration/expected.yaml | 2 +- .../snmp-credentials-migration/expected.yaml | 2 +- .../configurations/snmp-ports/expected.yaml | 2 +- .../snmp-to-metadata/expected.yaml | 2 +- common/clickhousedb/config.go | 18 + common/helpers/ether.go | 15 + common/pb/rawflow.go | 78 +++ common/pb/rawflow.proto | 26 + common/schema/clickhouse.go | 307 ++++++++- common/schema/clickhouse_test.go | 609 ++++++++++++++++++ common/schema/definition.go | 107 +-- common/schema/definition_test.go | 11 - common/schema/insert_test.go | 193 ++++++ common/schema/message.go | 110 ++++ common/schema/protobuf.go | 262 -------- common/schema/protobuf_test.go | 242 ------- common/schema/root_test.go | 23 - common/schema/tests.go | 83 --- common/schema/types.go | 77 +-- common/schema/types_test.go | 17 - config/akvorado.yaml | 1 + config/demo.yaml | 2 +- config/inlet.yaml | 26 - config/outlet.yaml | 28 + console/data/docs/01-install.md | 8 +- console/data/docs/03-usage.md | 1 - console/data/docs/99-changelog.md | 12 + demoexporter/flows/nfdata_test.go | 172 ++--- docker/docker-compose-demo.yml | 2 + docker/docker-compose.yml | 33 +- docker/prometheus.yml | 2 +- flake.nix | 2 + go.mod | 1 + inlet/core/root.go | 167 ----- inlet/core/root_test.go | 363 ----------- inlet/flow/config.go | 20 +- inlet/flow/config_test.go | 45 +- inlet/flow/decoder.go | 66 -- inlet/flow/decoder/config.go | 19 - inlet/flow/decoder/sflow/root_test.go | 543 ---------------- inlet/flow/decoder/tests.go | 36 -- inlet/flow/decoder_test.go | 92 --- inlet/flow/input/file/config.go | 3 + inlet/flow/input/file/root.go | 55 +- inlet/flow/input/file/root_test.go | 68 +- inlet/flow/input/root.go | 12 +- inlet/flow/input/udp/root.go | 72 +-- inlet/flow/input/udp/root_test.go | 142 ++-- inlet/flow/rate.go | 57 -- inlet/flow/root.go | 115 +--- inlet/flow/root_test.go | 147 ++--- inlet/flow/schemas_test.go | 29 - inlet/flow/tests.go | 49 -- inlet/kafka/config.go | 2 +- inlet/kafka/functional_test.go | 6 +- inlet/kafka/root.go | 10 +- inlet/kafka/root_test.go | 13 +- inlet/kafka/tests.go | 2 - orchestrator/clickhouse/config.go | 25 +- orchestrator/clickhouse/config_test.go | 1 - orchestrator/clickhouse/migrations.go | 15 +- orchestrator/clickhouse/migrations_helpers.go | 127 +--- orchestrator/clickhouse/migrations_test.go | 32 +- .../testdata/states/002-cluster.csv | 21 + .../clickhouse/testdata/states/011.csv | 4 +- .../clickhouse/testdata/states/012.csv | 17 + orchestrator/kafka/functional_test.go | 5 +- orchestrator/kafka/root.go | 8 +- orchestrator/root.go | 2 + outlet/clickhouse/config.go | 24 + outlet/clickhouse/example_test.go | 140 ++++ outlet/clickhouse/functional_test.go | 211 ++++++ outlet/clickhouse/metrics.go | 48 ++ outlet/clickhouse/root.go | 44 ++ outlet/clickhouse/root_test.go | 54 ++ outlet/clickhouse/tests.go | 51 ++ outlet/clickhouse/worker.go | 146 +++++ {inlet => outlet}/core/classifier.go | 0 {inlet => outlet}/core/classifier_test.go | 0 {inlet => outlet}/core/config.go | 4 +- {inlet => outlet}/core/config_test.go | 0 {inlet => outlet}/core/enricher.go | 71 +- {inlet => outlet}/core/enricher_test.go | 210 +++--- {inlet => outlet}/core/http.go | 22 +- {inlet => outlet}/core/metrics.go | 15 + outlet/core/release.go | 8 + outlet/core/root.go | 106 +++ outlet/core/root_test.go | 387 +++++++++++ outlet/core/worker.go | 102 +++ outlet/flow/decoder.go | 83 +++ outlet/flow/decoder/gob/root.go | 104 +++ outlet/flow/decoder/gob/root_test.go | 102 +++ {inlet => outlet}/flow/decoder/helpers.go | 55 +- .../flow/decoder/helpers_test.go | 8 +- .../flow/decoder/netflow/decode.go | 151 +++-- .../flow/decoder/netflow/root.go | 84 ++- .../flow/decoder/netflow/root_test.go | 407 +++++++----- .../netflow/testdata/data+templates.pcap | Bin .../flow/decoder/netflow/testdata/data.pcap | Bin .../netflow/testdata/datalink-data.pcap | Bin .../netflow/testdata/datalink-template.pcap | Bin .../decoder/netflow/testdata/icmp-data.pcap | Bin .../netflow/testdata/icmp-template.pcap | Bin .../flow/decoder/netflow/testdata/mpls.pcap | Bin .../testdata/multiplesamplingrates-data.pcap | Bin .../multiplesamplingrates-options-data.pcap | Bin ...ultiplesamplingrates-options-template.pcap | Bin .../multiplesamplingrates-template.pcap | Bin .../flow/decoder/netflow/testdata/nat.pcap | Bin .../flow/decoder/netflow/testdata/nfv5.pcap | Bin .../netflow/testdata/options-data.pcap | Bin .../netflow/testdata/options-template.pcap | Bin .../netflow/testdata/samplingrate-data.pcap | Bin .../testdata/samplingrate-template.pcap | Bin .../decoder/netflow/testdata/template.pcap | Bin {inlet => outlet}/flow/decoder/root.go | 22 +- .../flow/decoder/sflow/decode.go | 67 +- {inlet => outlet}/flow/decoder/sflow/root.go | 21 +- outlet/flow/decoder/sflow/root_test.go | 554 ++++++++++++++++ .../decoder/sflow/testdata/data-1140.pcap | Bin .../testdata/data-discard-interface.pcap | Bin .../decoder/sflow/testdata/data-icmpv4.pcap | Bin .../decoder/sflow/testdata/data-icmpv6.pcap | Bin .../sflow/testdata/data-local-interface.pcap | Bin .../testdata/data-multiple-interfaces.pcap | Bin .../decoder/sflow/testdata/data-qinq.pcap | Bin .../testdata/data-sflow-expanded-sample.pcap | Bin .../sflow/testdata/data-sflow-ipv4-data.pcap | Bin .../sflow/testdata/data-sflow-raw-ipv4.pcap | Bin .../flow/decoder/testdata/mpls-ipv4.pcap | Bin .../flow/decoder/testdata/vlan-ipv6.pcap | Bin outlet/flow/decoder_test.go | 334 ++++++++++ outlet/flow/root.go | 67 ++ outlet/flow/tests.go | 16 + outlet/kafka/config.go | 42 ++ outlet/kafka/config_test.go | 16 + outlet/kafka/consumer.go | 115 ++++ outlet/kafka/functional_test.go | 141 ++++ outlet/kafka/metrics.go | 55 ++ outlet/kafka/root.go | 165 +++++ outlet/kafka/root_test.go | 106 +++ outlet/kafka/tests.go | 50 ++ {inlet => outlet}/metadata/cache.go | 2 +- {inlet => outlet}/metadata/cache_test.go | 10 +- {inlet => outlet}/metadata/config.go | 8 +- {inlet => outlet}/metadata/config_test.go | 0 .../metadata/provider/gnmi/collector.go | 2 +- .../metadata/provider/gnmi/collector_test.go | 2 +- .../metadata/provider/gnmi/config.go | 2 +- .../metadata/provider/gnmi/config_test.go | 0 .../metadata/provider/gnmi/metrics.go | 0 .../metadata/provider/gnmi/root.go | 2 +- .../metadata/provider/gnmi/srlinux_test.go | 6 +- .../metadata/provider/gnmi/subscribe.go | 0 .../metadata/provider/gnmi/utils.go | 0 .../metadata/provider/gnmi/utils_test.go | 0 {inlet => outlet}/metadata/provider/root.go | 0 .../metadata/provider/snmp/config.go | 2 +- .../metadata/provider/snmp/config_test.go | 0 .../metadata/provider/snmp/poller.go | 2 +- .../metadata/provider/snmp/poller_test.go | 4 +- .../metadata/provider/snmp/root.go | 2 +- .../metadata/provider/snmp/tests.go | 0 .../metadata/provider/static/config.go | 2 +- .../metadata/provider/static/config_test.go | 2 +- .../metadata/provider/static/root.go | 2 +- .../metadata/provider/static/root_test.go | 2 +- .../metadata/provider/static/source.go | 2 +- .../metadata/provider/static/source_test.go | 2 +- .../metadata/provider/static/tests.go | 0 {inlet => outlet}/metadata/root.go | 15 +- {inlet => outlet}/metadata/root_test.go | 10 +- {inlet => outlet}/metadata/tests.go | 2 +- {inlet => outlet}/routing/config.go | 6 +- {inlet => outlet}/routing/config_test.go | 0 {inlet => outlet}/routing/metrics.go | 0 .../routing/provider/bioris/config.go | 2 +- .../routing/provider/bioris/config_test.go | 0 .../routing/provider/bioris/metrics.go | 0 .../routing/provider/bioris/root.go | 6 +- .../routing/provider/bioris/root_test.go | 6 +- .../routing/provider/bmp/config.go | 2 +- .../routing/provider/bmp/config_test.go | 0 .../routing/provider/bmp/events.go | 0 .../routing/provider/bmp/hash.go | 0 .../routing/provider/bmp/lookup.go | 2 +- .../routing/provider/bmp/metrics.go | 0 {inlet => outlet}/routing/provider/bmp/rd.go | 0 .../routing/provider/bmp/rd_test.go | 2 +- .../routing/provider/bmp/release.go | 0 .../routing/provider/bmp/remove.go | 5 +- {inlet => outlet}/routing/provider/bmp/rib.go | 0 .../routing/provider/bmp/rib_test.go | 0 .../routing/provider/bmp/root.go | 4 +- .../routing/provider/bmp/root_test.go | 60 +- .../routing/provider/bmp/serve.go | 0 .../provider/bmp/testdata/bmp-eor.pcap | Bin .../provider/bmp/testdata/bmp-init.pcap | Bin .../provider/bmp/testdata/bmp-l3vpn.pcap | Bin .../provider/bmp/testdata/bmp-peer-down.pcap | Bin .../provider/bmp/testdata/bmp-peer-up.pcap | Bin .../provider/bmp/testdata/bmp-peers-up.pcap | Bin .../bmp/testdata/bmp-reach-addpath.pcap | Bin .../testdata/bmp-reach-unknown-family.pcap | Bin .../provider/bmp/testdata/bmp-reach-vpls.pcap | Bin .../provider/bmp/testdata/bmp-reach.pcap | Bin .../provider/bmp/testdata/bmp-terminate.pcap | Bin .../provider/bmp/testdata/bmp-unreach.pcap | Bin .../routing/provider/bmp/tests.go | 2 +- .../routing/provider/bmp/utils.go | 0 .../routing/provider/bmp/utils_test.go | 0 {inlet => outlet}/routing/provider/root.go | 0 {inlet => outlet}/routing/root.go | 2 +- {inlet => outlet}/routing/root_test.go | 0 {inlet => outlet}/routing/tests.go | 2 +- 231 files changed, 6488 insertions(+), 3891 deletions(-) create mode 100755 bin/protoc-gen-go create mode 100644 cmd/outlet.go create mode 100644 cmd/outlet_test.go create mode 100644 common/pb/rawflow.go create mode 100644 common/pb/rawflow.proto create mode 100644 common/schema/clickhouse_test.go create mode 100644 common/schema/insert_test.go create mode 100644 common/schema/message.go delete mode 100644 common/schema/protobuf.go delete mode 100644 common/schema/protobuf_test.go delete mode 100644 common/schema/types_test.go create mode 100644 config/outlet.yaml delete mode 100644 inlet/core/root.go delete mode 100644 inlet/core/root_test.go delete mode 100644 inlet/flow/decoder.go delete mode 100644 inlet/flow/decoder/config.go delete mode 100644 inlet/flow/decoder/sflow/root_test.go delete mode 100644 inlet/flow/decoder/tests.go delete mode 100644 inlet/flow/decoder_test.go delete mode 100644 inlet/flow/rate.go delete mode 100644 inlet/flow/schemas_test.go delete mode 100644 inlet/flow/tests.go create mode 100644 orchestrator/clickhouse/testdata/states/002-cluster.csv create mode 100644 orchestrator/clickhouse/testdata/states/012.csv create mode 100644 outlet/clickhouse/config.go create mode 100644 outlet/clickhouse/example_test.go create mode 100644 outlet/clickhouse/functional_test.go create mode 100644 outlet/clickhouse/metrics.go create mode 100644 outlet/clickhouse/root.go create mode 100644 outlet/clickhouse/root_test.go create mode 100644 outlet/clickhouse/tests.go create mode 100644 outlet/clickhouse/worker.go rename {inlet => outlet}/core/classifier.go (100%) rename {inlet => outlet}/core/classifier_test.go (100%) rename {inlet => outlet}/core/config.go (98%) rename {inlet => outlet}/core/config_test.go (100%) rename {inlet => outlet}/core/enricher.go (78%) rename {inlet => outlet}/core/enricher_test.go (84%) rename {inlet => outlet}/core/http.go (69%) rename {inlet => outlet}/core/metrics.go (83%) create mode 100644 outlet/core/release.go create mode 100644 outlet/core/root.go create mode 100644 outlet/core/root_test.go create mode 100644 outlet/core/worker.go create mode 100644 outlet/flow/decoder.go create mode 100644 outlet/flow/decoder/gob/root.go create mode 100644 outlet/flow/decoder/gob/root_test.go rename {inlet => outlet}/flow/decoder/helpers.go (72%) rename {inlet => outlet}/flow/decoder/helpers_test.go (92%) rename {inlet => outlet}/flow/decoder/netflow/decode.go (66%) rename {inlet => outlet}/flow/decoder/netflow/root.go (80%) rename {inlet => outlet}/flow/decoder/netflow/root_test.go (56%) rename {inlet => outlet}/flow/decoder/netflow/testdata/data+templates.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/data.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/datalink-data.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/datalink-template.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/icmp-data.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/icmp-template.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/mpls.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/multiplesamplingrates-data.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/multiplesamplingrates-options-data.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/multiplesamplingrates-options-template.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/multiplesamplingrates-template.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/nat.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/nfv5.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/options-data.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/options-template.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/samplingrate-data.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/samplingrate-template.pcap (100%) rename {inlet => outlet}/flow/decoder/netflow/testdata/template.pcap (100%) rename {inlet => outlet}/flow/decoder/root.go (54%) rename {inlet => outlet}/flow/decoder/sflow/decode.go (64%) rename {inlet => outlet}/flow/decoder/sflow/root.go (90%) create mode 100644 outlet/flow/decoder/sflow/root_test.go rename {inlet => outlet}/flow/decoder/sflow/testdata/data-1140.pcap (100%) rename {inlet => outlet}/flow/decoder/sflow/testdata/data-discard-interface.pcap (100%) rename {inlet => outlet}/flow/decoder/sflow/testdata/data-icmpv4.pcap (100%) rename {inlet => outlet}/flow/decoder/sflow/testdata/data-icmpv6.pcap (100%) rename {inlet => outlet}/flow/decoder/sflow/testdata/data-local-interface.pcap (100%) rename {inlet => outlet}/flow/decoder/sflow/testdata/data-multiple-interfaces.pcap (100%) rename {inlet => outlet}/flow/decoder/sflow/testdata/data-qinq.pcap (100%) rename {inlet => outlet}/flow/decoder/sflow/testdata/data-sflow-expanded-sample.pcap (100%) rename {inlet => outlet}/flow/decoder/sflow/testdata/data-sflow-ipv4-data.pcap (100%) rename {inlet => outlet}/flow/decoder/sflow/testdata/data-sflow-raw-ipv4.pcap (100%) rename {inlet => outlet}/flow/decoder/testdata/mpls-ipv4.pcap (100%) rename {inlet => outlet}/flow/decoder/testdata/vlan-ipv6.pcap (100%) create mode 100644 outlet/flow/decoder_test.go create mode 100644 outlet/flow/root.go create mode 100644 outlet/flow/tests.go create mode 100644 outlet/kafka/config.go create mode 100644 outlet/kafka/config_test.go create mode 100644 outlet/kafka/consumer.go create mode 100644 outlet/kafka/functional_test.go create mode 100644 outlet/kafka/metrics.go create mode 100644 outlet/kafka/root.go create mode 100644 outlet/kafka/root_test.go create mode 100644 outlet/kafka/tests.go rename {inlet => outlet}/metadata/cache.go (98%) rename {inlet => outlet}/metadata/cache_test.go (97%) rename {inlet => outlet}/metadata/config.go (93%) rename {inlet => outlet}/metadata/config_test.go (100%) rename {inlet => outlet}/metadata/provider/gnmi/collector.go (99%) rename {inlet => outlet}/metadata/provider/gnmi/collector_test.go (98%) rename {inlet => outlet}/metadata/provider/gnmi/config.go (99%) rename {inlet => outlet}/metadata/provider/gnmi/config_test.go (100%) rename {inlet => outlet}/metadata/provider/gnmi/metrics.go (100%) rename {inlet => outlet}/metadata/provider/gnmi/root.go (98%) rename {inlet => outlet}/metadata/provider/gnmi/srlinux_test.go (99%) rename {inlet => outlet}/metadata/provider/gnmi/subscribe.go (100%) rename {inlet => outlet}/metadata/provider/gnmi/utils.go (100%) rename {inlet => outlet}/metadata/provider/gnmi/utils_test.go (100%) rename {inlet => outlet}/metadata/provider/root.go (100%) rename {inlet => outlet}/metadata/provider/snmp/config.go (99%) rename {inlet => outlet}/metadata/provider/snmp/config_test.go (100%) rename {inlet => outlet}/metadata/provider/snmp/poller.go (99%) rename {inlet => outlet}/metadata/provider/snmp/poller_test.go (98%) rename {inlet => outlet}/metadata/provider/snmp/root.go (98%) rename {inlet => outlet}/metadata/provider/snmp/tests.go (100%) rename {inlet => outlet}/metadata/provider/static/config.go (98%) rename {inlet => outlet}/metadata/provider/static/config_test.go (98%) rename {inlet => outlet}/metadata/provider/static/root.go (98%) rename {inlet => outlet}/metadata/provider/static/root_test.go (99%) rename {inlet => outlet}/metadata/provider/static/source.go (98%) rename {inlet => outlet}/metadata/provider/static/source_test.go (99%) rename {inlet => outlet}/metadata/provider/static/tests.go (100%) rename {inlet => outlet}/metadata/root.go (97%) rename {inlet => outlet}/metadata/root_test.go (97%) rename {inlet => outlet}/metadata/tests.go (98%) rename {inlet => outlet}/routing/config.go (90%) rename {inlet => outlet}/routing/config_test.go (100%) rename {inlet => outlet}/routing/metrics.go (100%) rename {inlet => outlet}/routing/provider/bioris/config.go (97%) rename {inlet => outlet}/routing/provider/bioris/config_test.go (100%) rename {inlet => outlet}/routing/provider/bioris/metrics.go (100%) rename {inlet => outlet}/routing/provider/bioris/root.go (98%) rename {inlet => outlet}/routing/provider/bioris/root_test.go (98%) rename {inlet => outlet}/routing/provider/bmp/config.go (98%) rename {inlet => outlet}/routing/provider/bmp/config_test.go (100%) rename {inlet => outlet}/routing/provider/bmp/events.go (100%) rename {inlet => outlet}/routing/provider/bmp/hash.go (100%) rename {inlet => outlet}/routing/provider/bmp/lookup.go (98%) rename {inlet => outlet}/routing/provider/bmp/metrics.go (100%) rename {inlet => outlet}/routing/provider/bmp/rd.go (100%) rename {inlet => outlet}/routing/provider/bmp/rd_test.go (98%) rename {inlet => outlet}/routing/provider/bmp/release.go (100%) rename {inlet => outlet}/routing/provider/bmp/remove.go (96%) rename {inlet => outlet}/routing/provider/bmp/rib.go (100%) rename {inlet => outlet}/routing/provider/bmp/rib_test.go (100%) rename {inlet => outlet}/routing/provider/bmp/root.go (97%) rename {inlet => outlet}/routing/provider/bmp/root_test.go (95%) rename {inlet => outlet}/routing/provider/bmp/serve.go (100%) rename {inlet => outlet}/routing/provider/bmp/testdata/bmp-eor.pcap (100%) rename {inlet => outlet}/routing/provider/bmp/testdata/bmp-init.pcap (100%) rename {inlet => outlet}/routing/provider/bmp/testdata/bmp-l3vpn.pcap (100%) rename {inlet => outlet}/routing/provider/bmp/testdata/bmp-peer-down.pcap (100%) rename {inlet => outlet}/routing/provider/bmp/testdata/bmp-peer-up.pcap (100%) rename {inlet => outlet}/routing/provider/bmp/testdata/bmp-peers-up.pcap (100%) rename {inlet => outlet}/routing/provider/bmp/testdata/bmp-reach-addpath.pcap (100%) rename {inlet => outlet}/routing/provider/bmp/testdata/bmp-reach-unknown-family.pcap (100%) rename {inlet => outlet}/routing/provider/bmp/testdata/bmp-reach-vpls.pcap (100%) rename {inlet => outlet}/routing/provider/bmp/testdata/bmp-reach.pcap (100%) rename {inlet => outlet}/routing/provider/bmp/testdata/bmp-terminate.pcap (100%) rename {inlet => outlet}/routing/provider/bmp/testdata/bmp-unreach.pcap (100%) rename {inlet => outlet}/routing/provider/bmp/tests.go (99%) rename {inlet => outlet}/routing/provider/bmp/utils.go (100%) rename {inlet => outlet}/routing/provider/bmp/utils_test.go (100%) rename {inlet => outlet}/routing/provider/root.go (100%) rename {inlet => outlet}/routing/root.go (98%) rename {inlet => outlet}/routing/root_test.go (100%) rename {inlet => outlet}/routing/tests.go (95%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 653d764d..81a13219 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,10 +47,8 @@ jobs: run: docker compose -f docker/docker-compose-dev.yml up --wait --wait-timeout 60 --quiet-pull - name: Setup uses: ./.github/actions/setup - - # Install dependencies - name: Install dependencies - run: sudo apt-get install -qqy shared-mime-info curl + run: sudo apt-get install -qqy shared-mime-info curl protobuf-compiler # Build and test - name: Build @@ -95,6 +93,8 @@ jobs: persist-credentials: false - name: Setup uses: ./.github/actions/setup + - name: Install dependencies + run: brew install protobuf # Build and test - name: Build @@ -157,6 +157,8 @@ jobs: uses: ./.github/actions/setup with: go-version: ${{ matrix.go-version }} + - name: Install dependencies + run: sudo apt-get install -qqy protobuf-compiler - name: Build run: make && ./bin/akvorado version - uses: actions/cache/save@v4 diff --git a/.gitignore b/.gitignore index deec8781..d1c35200 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -/bin/ +/bin/akvorado /test/ /orchestrator/clickhouse/data/asns.csv /orchestrator/clickhouse/data/tcp.csv @@ -7,6 +7,7 @@ /common/schema/definition_gen.go mock_*.go *_enumer.go +*.pb.go /console/data/frontend/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 182f41c0..54f77e8e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,7 +23,7 @@ run tests: # past but this is a slight burden to maintain in addition to GitHub CI. # Check commit ceaa6ebf8ef6 for the last version supporting functional # tests. - - time apk add --no-cache git make gcc musl-dev shared-mime-info npm curl + - time apk add --no-cache git make gcc musl-dev shared-mime-info npm curl protoc - export GOMODCACHE=$PWD/.go-cache - npm config --user set cache $PWD/.npm-cache - time go mod download diff --git a/Makefile b/Makefile index 45db462b..d3ea18f2 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,6 @@ DATE ?= $(shell date +%FT%T%z) VERSION ?= $(shell git describe --tags --always --dirty --match=v* 2> /dev/null || \ cat .version 2> /dev/null || echo v0) PKGS = $(or $(PKG),$(shell env GO111MODULE=on $(GO) list ./...)) -BIN = bin GO = go NPM = npm @@ -19,18 +18,18 @@ M = $(shell if [ "$$(tput colors 2> /dev/null || echo 0)" -ge 8 ]; then printf " GENERATED_JS = \ console/frontend/node_modules GENERATED_GO = \ + common/pb/rawflow.pb.go \ common/schema/definition_gen.go \ orchestrator/clickhouse/data/asns.csv \ orchestrator/clickhouse/data/protocols.csv \ orchestrator/clickhouse/data/tcp.csv \ orchestrator/clickhouse/data/udp.csv \ console/filter/parser.go \ - inlet/core/asnprovider_enumer.go \ - inlet/core/netprovider_enumer.go \ - inlet/flow/decoder/timestampsource_enumer.go \ - inlet/metadata/provider/snmp/authprotocol_enumer.go \ - inlet/metadata/provider/snmp/privprotocol_enumer.go \ - inlet/metadata/provider/gnmi/ifspeedpathunit_enumer.go \ + outlet/core/asnprovider_enumer.go \ + outlet/core/netprovider_enumer.go \ + outlet/metadata/provider/snmp/authprotocol_enumer.go \ + outlet/metadata/provider/snmp/privprotocol_enumer.go \ + outlet/metadata/provider/gnmi/ifspeedpathunit_enumer.go \ console/homepagetopwidget_enumer.go \ common/kafka/saslmechanism_enumer.go GENERATED_TEST_GO = \ @@ -42,20 +41,17 @@ GENERATED = \ console/data/frontend .PHONY: all -all: fmt lint $(GENERATED) | $(BIN) ; $(info $(M) building executable…) @ ## Build program binary +all: fmt lint $(GENERATED) ; $(info $(M) building executable…) @ ## Build program binary $Q $(GO) build \ -tags release \ -ldflags '-X $(MODULE)/common/helpers.AkvoradoVersion=$(VERSION)' \ - -o $(BIN)/$(basename $(MODULE)) main.go + -o bin/$(basename $(MODULE)) main.go .PHONY: all_js all_js: .fmt-js~ .lint-js~ $(GENERATED_JS) console/data/frontend # Tools -$(BIN): - @mkdir -p $@ - ENUMER = go tool enumer GOCOV = go tool gocov GOCOVXML = go tool gocov-xml @@ -63,6 +59,8 @@ GOIMPORTS = go tool goimports GOTESTSUM = go tool gotestsum MOCKGEN = go tool mockgen PIGEON = go tool pigeon +PROTOC = protoc +PROTOC_GEN_GO = bin/protoc-gen-go REVIVE = go tool revive WWHRD = go tool wwhrd @@ -70,6 +68,9 @@ WWHRD = go tool wwhrd .DELETE_ON_ERROR: +common/pb/rawflow.pb.go: common/pb/rawflow.proto ; $(info $(M) compiling protocol buffers definition…) + $Q $(PROTOC) -I=. --plugin=$(PROTOC_GEN_GO) --go_out=. --go_opt=module=$(MODULE) $< + common/clickhousedb/mocks/mock_driver.go: go.mod ; $(info $(M) generate mocks for ClickHouse driver…) $Q $(MOCKGEN) -package mocks -build_constraint "!release" -destination $@ \ github.com/ClickHouse/clickhouse-go/v2/lib/driver Conn,Row,Rows,ColumnType @@ -81,18 +82,16 @@ conntrackfixer/mocks/mock_conntrackfixer.go: go.mod ; $(info $(M) generate mocks touch $@ ; \ fi -inlet/core/asnprovider_enumer.go: go.mod inlet/core/config.go ; $(info $(M) generate enums for ASNProvider…) - $Q $(ENUMER) -type=ASNProvider -text -transform=kebab -trimprefix=ASNProvider inlet/core/config.go -inlet/core/netprovider_enumer.go: go.mod inlet/core/config.go ; $(info $(M) generate enums for NetProvider…) - $Q $(ENUMER) -type=NetProvider -text -transform=kebab -trimprefix=NetProvider inlet/core/config.go -inlet/flow/decoder/timestampsource_enumer.go: go.mod inlet/flow/decoder/config.go ; $(info $(M) generate enums for TimestampSource…) - $Q $(ENUMER) -type=TimestampSource -text -transform=kebab -trimprefix=TimestampSource inlet/flow/decoder/config.go -inlet/metadata/provider/snmp/authprotocol_enumer.go: go.mod inlet/metadata/provider/snmp/config.go ; $(info $(M) generate enums for AuthProtocol…) - $Q $(ENUMER) -type=AuthProtocol -text -transform=kebab -trimprefix=AuthProtocol inlet/metadata/provider/snmp/config.go -inlet/metadata/provider/snmp/privprotocol_enumer.go: go.mod inlet/metadata/provider/snmp/config.go ; $(info $(M) generate enums for PrivProtocol…) - $Q $(ENUMER) -type=PrivProtocol -text -transform=kebab -trimprefix=PrivProtocol inlet/metadata/provider/snmp/config.go -inlet/metadata/provider/gnmi/ifspeedpathunit_enumer.go: go.mod inlet/metadata/provider/gnmi/config.go ; $(info $(M) generate enums for IfSpeedPathUnit…) - $Q $(ENUMER) -type=IfSpeedPathUnit -text -transform=kebab -trimprefix=Speed inlet/metadata/provider/gnmi/config.go +outlet/core/asnprovider_enumer.go: go.mod outlet/core/config.go ; $(info $(M) generate enums for ASNProvider…) + $Q $(ENUMER) -type=ASNProvider -text -transform=kebab -trimprefix=ASNProvider outlet/core/config.go +outlet/core/netprovider_enumer.go: go.mod outlet/core/config.go ; $(info $(M) generate enums for NetProvider…) + $Q $(ENUMER) -type=NetProvider -text -transform=kebab -trimprefix=NetProvider outlet/core/config.go +outlet/metadata/provider/snmp/authprotocol_enumer.go: go.mod outlet/metadata/provider/snmp/config.go ; $(info $(M) generate enums for AuthProtocol…) + $Q $(ENUMER) -type=AuthProtocol -text -transform=kebab -trimprefix=AuthProtocol outlet/metadata/provider/snmp/config.go +outlet/metadata/provider/snmp/privprotocol_enumer.go: go.mod outlet/metadata/provider/snmp/config.go ; $(info $(M) generate enums for PrivProtocol…) + $Q $(ENUMER) -type=PrivProtocol -text -transform=kebab -trimprefix=PrivProtocol outlet/metadata/provider/snmp/config.go +outlet/metadata/provider/gnmi/ifspeedpathunit_enumer.go: go.mod outlet/metadata/provider/gnmi/config.go ; $(info $(M) generate enums for IfSpeedPathUnit…) + $Q $(ENUMER) -type=IfSpeedPathUnit -text -transform=kebab -trimprefix=Speed outlet/metadata/provider/gnmi/config.go console/homepagetopwidget_enumer.go: go.mod console/config.go ; $(info $(M) generate enums for HomepageTopWidget…) $Q $(ENUMER) -type=HomepageTopWidget -text -json -transform=kebab -trimprefix=HomepageTopWidget console/config.go common/kafka/saslmechanism_enumer.go: go.mod common/kafka/config.go ; $(info $(M) generate enums for SASLMechanism…) @@ -226,7 +225,7 @@ licensecheck: console/frontend/node_modules ; $(info $(M) check dependency licen .PHONY: clean clean: ; $(info $(M) cleaning…) @ ## Cleanup everything - @rm -rf test $(GENERATED) inlet/flow/decoder/flow-*.pb.go *~ bin + @rm -rf test $(GENERATED) inlet/flow/decoder/flow-*.pb.go *~ bin/akvorado .PHONY: help help: diff --git a/bin/protoc-gen-go b/bin/protoc-gen-go new file mode 100755 index 00000000..90ca6f62 --- /dev/null +++ b/bin/protoc-gen-go @@ -0,0 +1,2 @@ +#!/bin/sh +go tool protoc-gen-go "$@" diff --git a/cmd/inlet.go b/cmd/inlet.go index 0feba71a..9a25588c 100644 --- a/cmd/inlet.go +++ b/cmd/inlet.go @@ -5,24 +5,14 @@ package cmd import ( "fmt" - "reflect" - "github.com/gin-gonic/gin" - "github.com/go-viper/mapstructure/v2" "github.com/spf13/cobra" "akvorado/common/daemon" - "akvorado/common/helpers" "akvorado/common/httpserver" "akvorado/common/reporter" - "akvorado/common/schema" - "akvorado/inlet/core" "akvorado/inlet/flow" "akvorado/inlet/kafka" - "akvorado/inlet/metadata" - "akvorado/inlet/metadata/provider/snmp" - "akvorado/inlet/routing" - "akvorado/inlet/routing/provider/bmp" ) // InletConfiguration represents the configuration file for the inlet command. @@ -30,11 +20,7 @@ type InletConfiguration struct { Reporting reporter.Configuration HTTP httpserver.Configuration Flow flow.Configuration - Metadata metadata.Configuration - Routing routing.Configuration Kafka kafka.Configuration - Core core.Configuration - Schema schema.Configuration } // Reset resets the configuration for the inlet command to its default value. @@ -43,14 +29,8 @@ func (c *InletConfiguration) Reset() { HTTP: httpserver.DefaultConfiguration(), Reporting: reporter.DefaultConfiguration(), Flow: flow.DefaultConfiguration(), - Metadata: metadata.DefaultConfiguration(), - Routing: routing.DefaultConfiguration(), Kafka: kafka.DefaultConfiguration(), - Core: core.DefaultConfiguration(), - Schema: schema.DefaultConfiguration(), } - c.Metadata.Providers = []metadata.ProviderConfiguration{{Config: snmp.DefaultConfiguration()}} - c.Routing.Provider.Config = bmp.DefaultConfiguration() } type inletOptions struct { @@ -66,7 +46,7 @@ var inletCmd = &cobra.Command{ Use: "inlet", Short: "Start Akvorado's inlet service", Long: `Akvorado is a Netflow/IPFIX collector. The inlet service handles flow ingestion, -enrichment and export to Kafka.`, +and export to Kafka.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { config := InletConfiguration{} @@ -103,48 +83,19 @@ func inletStart(r *reporter.Reporter, config InletConfiguration, checkOnly bool) if err != nil { return fmt.Errorf("unable to initialize http component: %w", err) } - schemaComponent, err := schema.New(config.Schema) - if err != nil { - return fmt.Errorf("unable to initialize schema component: %w", err) - } - flowComponent, err := flow.New(r, config.Flow, flow.Dependencies{ - Daemon: daemonComponent, - HTTP: httpComponent, - Schema: schemaComponent, - }) - if err != nil { - return fmt.Errorf("unable to initialize flow component: %w", err) - } - metadataComponent, err := metadata.New(r, config.Metadata, metadata.Dependencies{ - Daemon: daemonComponent, - }) - if err != nil { - return fmt.Errorf("unable to initialize metadata component: %w", err) - } - routingComponent, err := routing.New(r, config.Routing, routing.Dependencies{ - Daemon: daemonComponent, - }) - if err != nil { - return fmt.Errorf("unable to initialize routing component: %w", err) - } kafkaComponent, err := kafka.New(r, config.Kafka, kafka.Dependencies{ Daemon: daemonComponent, - Schema: schemaComponent, }) if err != nil { return fmt.Errorf("unable to initialize Kafka component: %w", err) } - coreComponent, err := core.New(r, config.Core, core.Dependencies{ - Daemon: daemonComponent, - Flow: flowComponent, - Metadata: metadataComponent, - Routing: routingComponent, - Kafka: kafkaComponent, - HTTP: httpComponent, - Schema: schemaComponent, + flowComponent, err := flow.New(r, config.Flow, flow.Dependencies{ + Daemon: daemonComponent, + HTTP: httpComponent, + Kafka: kafkaComponent, }) if err != nil { - return fmt.Errorf("unable to initialize core component: %w", err) + return fmt.Errorf("unable to initialize flow component: %w", err) } // Expose some information and metrics @@ -159,129 +110,8 @@ func inletStart(r *reporter.Reporter, config InletConfiguration, checkOnly bool) // Start all the components. components := []interface{}{ httpComponent, - metadataComponent, - routingComponent, kafkaComponent, - coreComponent, flowComponent, } return StartStopComponents(r, daemonComponent, components) } - -// InletConfigurationUnmarshallerHook renames SNMP configuration to metadata and -// BMP configuration to routing. -func InletConfigurationUnmarshallerHook() mapstructure.DecodeHookFunc { - return func(from, to reflect.Value) (interface{}, error) { - if from.Kind() != reflect.Map || from.IsNil() || to.Type() != reflect.TypeOf(InletConfiguration{}) { - return from.Interface(), nil - } - - // snmp → metadata - { - var snmpKey, metadataKey *reflect.Value - fromKeys := from.MapKeys() - for i, k := range fromKeys { - k = helpers.ElemOrIdentity(k) - if k.Kind() != reflect.String { - return from.Interface(), nil - } - if helpers.MapStructureMatchName(k.String(), "Snmp") { - snmpKey = &fromKeys[i] - } else if helpers.MapStructureMatchName(k.String(), "Metadata") { - metadataKey = &fromKeys[i] - } - } - if snmpKey != nil { - if metadataKey != nil { - return nil, fmt.Errorf("cannot have both %q and %q", snmpKey.String(), metadataKey.String()) - } - - // Build the metadata configuration - providerValue := gin.H{} - metadataValue := gin.H{} - // Dispatch values from snmp key into metadata - snmpMap := helpers.ElemOrIdentity(from.MapIndex(*snmpKey)) - snmpKeys := snmpMap.MapKeys() - outerSNMP: - for i, k := range snmpKeys { - k = helpers.ElemOrIdentity(k) - if k.Kind() != reflect.String { - continue - } - if helpers.MapStructureMatchName(k.String(), "PollerCoalesce") { - metadataValue["MaxBatchRequests"] = snmpMap.MapIndex(snmpKeys[i]).Interface() - continue - } - metadataConfig := reflect.TypeOf(metadata.Configuration{}) - for j := range metadataConfig.NumField() { - if helpers.MapStructureMatchName(k.String(), metadataConfig.Field(j).Name) { - metadataValue[k.String()] = snmpMap.MapIndex(snmpKeys[i]).Interface() - continue outerSNMP - } - } - providerValue[k.String()] = snmpMap.MapIndex(snmpKeys[i]).Interface() - } - - providerValue["type"] = "snmp" - metadataValue["provider"] = providerValue - from.SetMapIndex(reflect.ValueOf("metadata"), reflect.ValueOf(metadataValue)) - from.SetMapIndex(*snmpKey, reflect.Value{}) - } - } - - // bmp → routing - { - var bmpKey, routingKey *reflect.Value - fromKeys := from.MapKeys() - for i, k := range fromKeys { - k = helpers.ElemOrIdentity(k) - if k.Kind() != reflect.String { - return from.Interface(), nil - } - if helpers.MapStructureMatchName(k.String(), "Bmp") { - bmpKey = &fromKeys[i] - } else if helpers.MapStructureMatchName(k.String(), "Routing") { - routingKey = &fromKeys[i] - } - } - if bmpKey != nil { - if routingKey != nil { - return nil, fmt.Errorf("cannot have both %q and %q", bmpKey.String(), routingKey.String()) - } - - // Build the routing configuration - providerValue := gin.H{} - routingValue := gin.H{} - // Dispatch values from bmp key into routing - bmpMap := helpers.ElemOrIdentity(from.MapIndex(*bmpKey)) - bmpKeys := bmpMap.MapKeys() - outerBMP: - for i, k := range bmpKeys { - k = helpers.ElemOrIdentity(k) - if k.Kind() != reflect.String { - continue - } - routingConfig := reflect.TypeOf(routing.Configuration{}) - for j := range routingConfig.NumField() { - if helpers.MapStructureMatchName(k.String(), routingConfig.Field(j).Name) { - routingValue[k.String()] = bmpMap.MapIndex(bmpKeys[i]).Interface() - continue outerBMP - } - } - providerValue[k.String()] = bmpMap.MapIndex(bmpKeys[i]).Interface() - } - - providerValue["type"] = "bmp" - routingValue["provider"] = providerValue - from.SetMapIndex(reflect.ValueOf("routing"), reflect.ValueOf(routingValue)) - from.SetMapIndex(*bmpKey, reflect.Value{}) - } - } - - return from.Interface(), nil - } -} - -func init() { - helpers.RegisterMapstructureUnmarshallerHook(InletConfigurationUnmarshallerHook()) -} diff --git a/cmd/orchestrator.go b/cmd/orchestrator.go index 218972f0..120af630 100644 --- a/cmd/orchestrator.go +++ b/cmd/orchestrator.go @@ -37,6 +37,7 @@ type OrchestratorConfiguration struct { Schema schema.Configuration // Other service configurations Inlet []InletConfiguration `validate:"dive"` + Outlet []OutletConfiguration `validate:"dive"` Console []ConsoleConfiguration `validate:"dive"` DemoExporter []DemoExporterConfiguration `validate:"dive"` } @@ -45,6 +46,8 @@ type OrchestratorConfiguration struct { func (c *OrchestratorConfiguration) Reset() { inletConfiguration := InletConfiguration{} inletConfiguration.Reset() + outletConfiguration := OutletConfiguration{} + outletConfiguration.Reset() consoleConfiguration := ConsoleConfiguration{} consoleConfiguration.Reset() *c = OrchestratorConfiguration{ @@ -58,6 +61,7 @@ func (c *OrchestratorConfiguration) Reset() { Schema: schema.DefaultConfiguration(), // Other service configurations Inlet: []InletConfiguration{inletConfiguration}, + Outlet: []OutletConfiguration{outletConfiguration}, Console: []ConsoleConfiguration{consoleConfiguration}, DemoExporter: []DemoExporterConfiguration{}, } @@ -83,14 +87,19 @@ components and centralizes configuration of the various other components.`, OrchestratorOptions.Path = args[0] OrchestratorOptions.BeforeDump = func(metadata mapstructure.Metadata) { // Override some parts of the configuration - if !slices.Contains(metadata.Keys, "ClickHouse.Kafka.Brokers[0]") { - config.ClickHouse.Kafka.Configuration = config.Kafka.Configuration - } for idx := range config.Inlet { if !slices.Contains(metadata.Keys, fmt.Sprintf("Inlet[%d].Kafka.Brokers[0]", idx)) { config.Inlet[idx].Kafka.Configuration = config.Kafka.Configuration } - config.Inlet[idx].Schema = config.Schema + } + for idx := range config.Outlet { + if !slices.Contains(metadata.Keys, fmt.Sprintf("Outlet[%d].ClickHouse.Servers[0]", idx)) { + config.Outlet[idx].ClickHouseDB = config.ClickHouseDB + } + if !slices.Contains(metadata.Keys, fmt.Sprintf("Outlet[%d].Kafka.Brokers[0]", idx)) { + config.Outlet[idx].Kafka.Configuration = config.Kafka.Configuration + } + config.Outlet[idx].Schema = config.Schema } for idx := range config.Console { if !slices.Contains(metadata.Keys, fmt.Sprintf("Console[%d].ClickHouse.Servers[0]", idx)) { @@ -144,14 +153,12 @@ func orchestratorStart(r *reporter.Reporter, config OrchestratorConfiguration, c if err != nil { return fmt.Errorf("unable to initialize ClickHouse component: %w", err) } - geoipComponent, err := geoip.New(r, config.GeoIP, geoip.Dependencies{ Daemon: daemonComponent, }) if err != nil { return fmt.Errorf("unable to initialize GeoIP component: %w", err) } - clickhouseComponent, err := clickhouse.New(r, config.ClickHouse, clickhouse.Dependencies{ Daemon: daemonComponent, HTTP: httpComponent, @@ -171,6 +178,9 @@ func orchestratorStart(r *reporter.Reporter, config OrchestratorConfiguration, c for idx := range config.Inlet { orchestratorComponent.RegisterConfiguration(orchestrator.InletService, config.Inlet[idx]) } + for idx := range config.Outlet { + orchestratorComponent.RegisterConfiguration(orchestrator.OutletService, config.Outlet[idx]) + } for idx := range config.Console { orchestratorComponent.RegisterConfiguration(orchestrator.ConsoleService, config.Console[idx]) } @@ -188,7 +198,7 @@ func orchestratorStart(r *reporter.Reporter, config OrchestratorConfiguration, c } // Start all the components. - components := []interface{}{ + components := []any{ geoipComponent, httpComponent, clickhouseDBComponent, @@ -198,136 +208,224 @@ func orchestratorStart(r *reporter.Reporter, config OrchestratorConfiguration, c return StartStopComponents(r, daemonComponent, components) } -// OrchestratorConfigurationUnmarshallerHook migrates GeoIP configuration from inlet -// component to clickhouse component and ClickHouse database configuration from -// clickhouse component to clickhousedb component. -func OrchestratorConfigurationUnmarshallerHook() mapstructure.DecodeHookFunc { - return func(from, to reflect.Value) (interface{}, error) { +// orchestratorGeoIPMigrationHook migrates GeoIP configuration from inlet +// component to clickhouse component +func orchestratorGeoIPMigrationHook() mapstructure.DecodeHookFunc { + return func(from, to reflect.Value) (any, error) { if from.Kind() != reflect.Map || from.IsNil() || to.Type() != reflect.TypeOf(OrchestratorConfiguration{}) { return from.Interface(), nil } - inletgeoip: - // inlet/geoip → geoip - for { - var ( - inletKey, geoIPKey, inletGeoIPValue *reflect.Value - ) + var ( + inletKey, geoIPKey, inletGeoIPValue *reflect.Value + ) - fromKeys := from.MapKeys() - for i, k := range fromKeys { + fromKeys := from.MapKeys() + for i, k := range fromKeys { + k = helpers.ElemOrIdentity(k) + if k.Kind() != reflect.String { + return from.Interface(), nil + } + if helpers.MapStructureMatchName(k.String(), "Inlet") { + inletKey = &fromKeys[i] + } else if helpers.MapStructureMatchName(k.String(), "GeoIP") { + geoIPKey = &fromKeys[i] + } + } + if inletKey == nil { + return from.Interface(), nil + } + + // Take the first geoip configuration and delete the others + inletConfigs := helpers.ElemOrIdentity(from.MapIndex(*inletKey)) + if inletConfigs.Kind() != reflect.Slice { + inletConfigs = reflect.ValueOf([]any{inletConfigs.Interface()}) + } + for i := range inletConfigs.Len() { + fromInlet := helpers.ElemOrIdentity(inletConfigs.Index(i)) + if fromInlet.Kind() != reflect.Map { + return from.Interface(), nil + } + fromInletKeys := fromInlet.MapKeys() + for _, k := range fromInletKeys { k = helpers.ElemOrIdentity(k) if k.Kind() != reflect.String { - break inletgeoip + return from.Interface(), nil } - if helpers.MapStructureMatchName(k.String(), "Inlet") { - inletKey = &fromKeys[i] - } else if helpers.MapStructureMatchName(k.String(), "GeoIP") { - geoIPKey = &fromKeys[i] + if helpers.MapStructureMatchName(k.String(), "GeoIP") { + if inletGeoIPValue == nil { + v := fromInlet.MapIndex(k) + inletGeoIPValue = &v + } } } - if inletKey == nil { - break inletgeoip + } + if inletGeoIPValue == nil { + return from.Interface(), nil + } + if geoIPKey != nil { + return nil, errors.New("cannot have both \"GeoIP\" in inlet and clickhouse configuration") + } + + from.SetMapIndex(reflect.ValueOf("geoip"), *inletGeoIPValue) + for i := range inletConfigs.Len() { + fromInlet := helpers.ElemOrIdentity(inletConfigs.Index(i)) + fromInletKeys := fromInlet.MapKeys() + for _, k := range fromInletKeys { + k = helpers.ElemOrIdentity(k) + if helpers.MapStructureMatchName(k.String(), "GeoIP") { + fromInlet.SetMapIndex(k, reflect.Value{}) + } + } + } + + return from.Interface(), nil + } +} + +// orchestratorClickHouseMigrationHook migrates ClickHouse database +// configuration from clickhouse component to clickhousedb component +func orchestratorClickHouseMigrationHook() mapstructure.DecodeHookFunc { + return func(from, to reflect.Value) (any, error) { + if from.Kind() != reflect.Map || from.IsNil() || to.Type() != reflect.TypeOf(OrchestratorConfiguration{}) { + return from.Interface(), nil + } + + var clickhouseKey, clickhouseDBKey *reflect.Value + fromKeys := from.MapKeys() + for i, k := range fromKeys { + k = helpers.ElemOrIdentity(k) + if k.Kind() != reflect.String { + continue + } + if helpers.MapStructureMatchName(k.String(), "ClickHouse") { + clickhouseKey = &fromKeys[i] + } else if helpers.MapStructureMatchName(k.String(), "ClickHouseDB") { + clickhouseDBKey = &fromKeys[i] + } + } + + if clickhouseKey != nil { + var clickhouseDB reflect.Value + if clickhouseDBKey != nil { + clickhouseDB = helpers.ElemOrIdentity(from.MapIndex(*clickhouseDBKey)) + } else { + clickhouseDB = reflect.ValueOf(gin.H{}) } - // Take the first geoip configuration and delete the others + clickhouse := helpers.ElemOrIdentity(from.MapIndex(*clickhouseKey)) + if clickhouse.Kind() == reflect.Map { + clickhouseKeys := clickhouse.MapKeys() + // Fields to migrate from clickhouse to clickhousedb + fieldsToMigrate := []string{ + "Servers", "Cluster", "Database", "Username", "Password", + "MaxOpenConns", "DialTimeout", "TLS", + } + found := false + for _, k := range clickhouseKeys { + k = helpers.ElemOrIdentity(k) + if k.Kind() != reflect.String { + continue + } + for _, field := range fieldsToMigrate { + if helpers.MapStructureMatchName(k.String(), field) { + if clickhouseDBKey != nil { + return nil, errors.New("cannot have both \"ClickHouseDB\" and ClickHouse database settings in \"ClickHouse\"") + } + clickhouseDB.SetMapIndex(k, helpers.ElemOrIdentity(clickhouse.MapIndex(k))) + clickhouse.SetMapIndex(k, reflect.Value{}) + found = true + break + } + } + } + if clickhouseDBKey == nil && found { + from.SetMapIndex(reflect.ValueOf("clickhousedb"), clickhouseDB) + } + } + } + + return from.Interface(), nil + } +} + +// orchestratorInletToOutletMigrationHook migrates inlet configuration to outlet +// configuration. This only works if there is no outlet configuration and if +// there is only one inlet configuration. +func orchestratorInletToOutletMigrationHook() mapstructure.DecodeHookFunc { + return func(from, to reflect.Value) (any, error) { + if from.Kind() != reflect.Map || from.IsNil() || to.Type() != reflect.TypeOf(OrchestratorConfiguration{}) { + return from.Interface(), nil + } + + // inlet fields (Metadata, Routing, Core, Schema) → outlet + var inletKey, outletKey *reflect.Value + fromKeys := from.MapKeys() + for i, k := range fromKeys { + k = helpers.ElemOrIdentity(k) + if k.Kind() != reflect.String { + continue + } + if helpers.MapStructureMatchName(k.String(), "Inlet") { + inletKey = &fromKeys[i] + } else if helpers.MapStructureMatchName(k.String(), "Outlet") { + outletKey = &fromKeys[i] + } + } + + if inletKey != nil { inletConfigs := helpers.ElemOrIdentity(from.MapIndex(*inletKey)) if inletConfigs.Kind() != reflect.Slice { - inletConfigs = reflect.ValueOf([]interface{}{inletConfigs.Interface()}) + inletConfigs = reflect.ValueOf([]any{inletConfigs.Interface()}) } + + // Fields to migrate from inlet to outlet + fieldsToMigrate := []string{ + // Current keys + "Metadata", "Routing", "Core", "Schema", + // Older keys (which will be migrated) + "BMP", "SNMP", + } + + // Process each inlet configuration for i := range inletConfigs.Len() { fromInlet := helpers.ElemOrIdentity(inletConfigs.Index(i)) if fromInlet.Kind() != reflect.Map { - break inletgeoip + continue } + modified := false + toOutlet := reflect.ValueOf(gin.H{}) + + // Migrate fields from inlet to outlet fromInletKeys := fromInlet.MapKeys() for _, k := range fromInletKeys { k = helpers.ElemOrIdentity(k) if k.Kind() != reflect.String { - break inletgeoip + continue } - if helpers.MapStructureMatchName(k.String(), "GeoIP") { - if inletGeoIPValue == nil { - v := fromInlet.MapIndex(k) - inletGeoIPValue = &v - } - } - } - } - if inletGeoIPValue == nil { - break inletgeoip - } - if geoIPKey != nil { - return nil, errors.New("cannot have both \"GeoIP\" in inlet and clickhouse configuration") - } - - from.SetMapIndex(reflect.ValueOf("geoip"), *inletGeoIPValue) - for i := range inletConfigs.Len() { - fromInlet := helpers.ElemOrIdentity(inletConfigs.Index(i)) - fromInletKeys := fromInlet.MapKeys() - for _, k := range fromInletKeys { - k = helpers.ElemOrIdentity(k) - if helpers.MapStructureMatchName(k.String(), "GeoIP") { - fromInlet.SetMapIndex(k, reflect.Value{}) - } - } - } - break - } - - { - // clickhouse database fields → clickhousedb - var clickhouseKey, clickhouseDBKey *reflect.Value - fromKeys := from.MapKeys() - for i, k := range fromKeys { - k = helpers.ElemOrIdentity(k) - if k.Kind() != reflect.String { - continue - } - if helpers.MapStructureMatchName(k.String(), "ClickHouse") { - clickhouseKey = &fromKeys[i] - } else if helpers.MapStructureMatchName(k.String(), "ClickHouseDB") { - clickhouseDBKey = &fromKeys[i] - } - } - - if clickhouseKey != nil { - var clickhouseDB reflect.Value - if clickhouseDBKey != nil { - clickhouseDB = helpers.ElemOrIdentity(from.MapIndex(*clickhouseDBKey)) - } else { - clickhouseDB = reflect.ValueOf(gin.H{}) - } - - clickhouse := helpers.ElemOrIdentity(from.MapIndex(*clickhouseKey)) - if clickhouse.Kind() == reflect.Map { - clickhouseKeys := clickhouse.MapKeys() - // Fields to migrate from clickhouse to clickhousedb - fieldsToMigrate := []string{ - "Servers", "Cluster", "Database", "Username", "Password", - "MaxOpenConns", "DialTimeout", "TLS", - } - found := false - for _, k := range clickhouseKeys { - k = helpers.ElemOrIdentity(k) - if k.Kind() != reflect.String { + for _, field := range fieldsToMigrate { + if !helpers.MapStructureMatchName(k.String(), field) { continue } - for _, field := range fieldsToMigrate { - if helpers.MapStructureMatchName(k.String(), field) { - if clickhouseDBKey != nil { - return nil, errors.New("cannot have both \"ClickHouseDB\" and ClickHouse database settings in \"ClickHouse\"") - } - clickhouseDB.SetMapIndex(k, helpers.ElemOrIdentity(clickhouse.MapIndex(k))) - clickhouse.SetMapIndex(k, reflect.Value{}) - found = true - break - } + // We can only do a migration if we have no existing + // outlet configuration AND only one inlet configuration. + if outletKey != nil { + return nil, fmt.Errorf("cannot have both \"inlet\" configuration with %q field and \"outlet\" configuration", field) } + if inletConfigs.Len() > 1 { + return nil, fmt.Errorf("cannot migrate %q from %q to %q as there are several inlet configurations", field, "inlet", "outlet") + } + toOutlet.SetMapIndex(k, helpers.ElemOrIdentity(fromInlet.MapIndex(k))) + fromInlet.SetMapIndex(k, reflect.Value{}) + modified = true + break } - if clickhouseDBKey == nil && found { - from.SetMapIndex(reflect.ValueOf("clickhousedb"), clickhouseDB) - } + } + + if modified { + // We know there is no existing outlet configuration. + outletConfigs := reflect.ValueOf([]any{toOutlet}) + from.SetMapIndex(reflect.ValueOf("outlet"), outletConfigs) } } } @@ -337,5 +435,7 @@ func OrchestratorConfigurationUnmarshallerHook() mapstructure.DecodeHookFunc { } func init() { - helpers.RegisterMapstructureUnmarshallerHook(OrchestratorConfigurationUnmarshallerHook()) + helpers.RegisterMapstructureUnmarshallerHook(orchestratorGeoIPMigrationHook()) + helpers.RegisterMapstructureUnmarshallerHook(orchestratorClickHouseMigrationHook()) + helpers.RegisterMapstructureUnmarshallerHook(orchestratorInletToOutletMigrationHook()) } diff --git a/cmd/outlet.go b/cmd/outlet.go new file mode 100644 index 00000000..4c6777f8 --- /dev/null +++ b/cmd/outlet.go @@ -0,0 +1,304 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package cmd + +import ( + "fmt" + "reflect" + + "github.com/gin-gonic/gin" + "github.com/go-viper/mapstructure/v2" + "github.com/spf13/cobra" + + "akvorado/common/clickhousedb" + "akvorado/common/daemon" + "akvorado/common/helpers" + "akvorado/common/httpserver" + "akvorado/common/reporter" + "akvorado/common/schema" + "akvorado/outlet/clickhouse" + "akvorado/outlet/core" + "akvorado/outlet/flow" + "akvorado/outlet/kafka" + "akvorado/outlet/metadata" + "akvorado/outlet/metadata/provider/snmp" + "akvorado/outlet/routing" + "akvorado/outlet/routing/provider/bmp" +) + +// OutletConfiguration represents the configuration file for the outlet command. +type OutletConfiguration struct { + Reporting reporter.Configuration + HTTP httpserver.Configuration + Metadata metadata.Configuration + Routing routing.Configuration + Kafka kafka.Configuration + ClickHouseDB clickhousedb.Configuration + ClickHouse clickhouse.Configuration + Core core.Configuration + Schema schema.Configuration +} + +// Reset resets the configuration for the outlet command to its default value. +func (c *OutletConfiguration) Reset() { + *c = OutletConfiguration{ + HTTP: httpserver.DefaultConfiguration(), + Reporting: reporter.DefaultConfiguration(), + Metadata: metadata.DefaultConfiguration(), + Routing: routing.DefaultConfiguration(), + Kafka: kafka.DefaultConfiguration(), + ClickHouseDB: clickhousedb.DefaultConfiguration(), + ClickHouse: clickhouse.DefaultConfiguration(), + Core: core.DefaultConfiguration(), + Schema: schema.DefaultConfiguration(), + } + c.Metadata.Providers = []metadata.ProviderConfiguration{{Config: snmp.DefaultConfiguration()}} + c.Routing.Provider.Config = bmp.DefaultConfiguration() +} + +type outletOptions struct { + ConfigRelatedOptions + CheckMode bool +} + +// OutletOptions stores the command-line option values for the outlet +// command. +var OutletOptions outletOptions + +var outletCmd = &cobra.Command{ + Use: "outlet", + Short: "Start Akvorado's outlet service", + Long: `Akvorado is a Netflow/IPFIX collector. The outlet service handles flow ingestion, +enrichment and export to Kafka.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + config := OutletConfiguration{} + OutletOptions.Path = args[0] + if err := OutletOptions.Parse(cmd.OutOrStdout(), "outlet", &config); err != nil { + return err + } + + r, err := reporter.New(config.Reporting) + if err != nil { + return fmt.Errorf("unable to initialize reporter: %w", err) + } + return outletStart(r, config, OutletOptions.CheckMode) + }, +} + +func init() { + RootCmd.AddCommand(outletCmd) + outletCmd.Flags().BoolVarP(&OutletOptions.ConfigRelatedOptions.Dump, "dump", "D", false, + "Dump configuration before starting") + outletCmd.Flags().BoolVarP(&OutletOptions.CheckMode, "check", "C", false, + "Check configuration, but does not start") +} + +func outletStart(r *reporter.Reporter, config OutletConfiguration, checkOnly bool) error { + // Initialize the various components + daemonComponent, err := daemon.New(r) + if err != nil { + return fmt.Errorf("unable to initialize daemon component: %w", err) + } + httpComponent, err := httpserver.New(r, config.HTTP, httpserver.Dependencies{ + Daemon: daemonComponent, + }) + if err != nil { + return fmt.Errorf("unable to initialize http component: %w", err) + } + schemaComponent, err := schema.New(config.Schema) + if err != nil { + return fmt.Errorf("unable to initialize schema component: %w", err) + } + flowComponent, err := flow.New(r, flow.Dependencies{ + Schema: schemaComponent, + }) + if err != nil { + return fmt.Errorf("unable to initialize flow component: %w", err) + } + metadataComponent, err := metadata.New(r, config.Metadata, metadata.Dependencies{ + Daemon: daemonComponent, + }) + if err != nil { + return fmt.Errorf("unable to initialize metadata component: %w", err) + } + routingComponent, err := routing.New(r, config.Routing, routing.Dependencies{ + Daemon: daemonComponent, + }) + if err != nil { + return fmt.Errorf("unable to initialize routing component: %w", err) + } + kafkaComponent, err := kafka.New(r, config.Kafka, kafka.Dependencies{ + Daemon: daemonComponent, + }) + if err != nil { + return fmt.Errorf("unable to initialize Kafka component: %w", err) + } + clickhouseDBComponent, err := clickhousedb.New(r, config.ClickHouseDB, clickhousedb.Dependencies{ + Daemon: daemonComponent, + }) + if err != nil { + return fmt.Errorf("unable to initialize ClickHouse component: %w", err) + } + clickhouseComponent, err := clickhouse.New(r, config.ClickHouse, clickhouse.Dependencies{ + ClickHouse: clickhouseDBComponent, + Schema: schemaComponent, + }) + if err != nil { + return fmt.Errorf("unable to initialize outlet ClickHouse component: %w", err) + } + coreComponent, err := core.New(r, config.Core, core.Dependencies{ + Daemon: daemonComponent, + Flow: flowComponent, + Metadata: metadataComponent, + Routing: routingComponent, + Kafka: kafkaComponent, + ClickHouse: clickhouseComponent, + HTTP: httpComponent, + Schema: schemaComponent, + }) + if err != nil { + return fmt.Errorf("unable to initialize core component: %w", err) + } + + // Expose some information and metrics + addCommonHTTPHandlers(r, "outlet", httpComponent) + versionMetrics(r) + + // If we only asked for a check, stop here. + if checkOnly { + return nil + } + + // Start all the components. + components := []any{ + httpComponent, + clickhouseDBComponent, + clickhouseComponent, + flowComponent, + metadataComponent, + routingComponent, + kafkaComponent, + coreComponent, + } + return StartStopComponents(r, daemonComponent, components) +} + +// OutletConfigurationUnmarshallerHook renames SNMP configuration to metadata and +// BMP configuration to routing. +func OutletConfigurationUnmarshallerHook() mapstructure.DecodeHookFunc { + return func(from, to reflect.Value) (interface{}, error) { + if from.Kind() != reflect.Map || from.IsNil() || to.Type() != reflect.TypeOf(OutletConfiguration{}) { + return from.Interface(), nil + } + + // snmp → metadata + { + var snmpKey, metadataKey *reflect.Value + fromKeys := from.MapKeys() + for i, k := range fromKeys { + k = helpers.ElemOrIdentity(k) + if k.Kind() != reflect.String { + return from.Interface(), nil + } + if helpers.MapStructureMatchName(k.String(), "Snmp") { + snmpKey = &fromKeys[i] + } else if helpers.MapStructureMatchName(k.String(), "Metadata") { + metadataKey = &fromKeys[i] + } + } + if snmpKey != nil { + if metadataKey != nil { + return nil, fmt.Errorf("cannot have both %q and %q", snmpKey.String(), metadataKey.String()) + } + + // Build the metadata configuration + providerValue := gin.H{} + metadataValue := gin.H{} + // Dispatch values from snmp key into metadata + snmpMap := helpers.ElemOrIdentity(from.MapIndex(*snmpKey)) + snmpKeys := snmpMap.MapKeys() + outerSNMP: + for i, k := range snmpKeys { + k = helpers.ElemOrIdentity(k) + if k.Kind() != reflect.String { + continue + } + if helpers.MapStructureMatchName(k.String(), "PollerCoalesce") { + metadataValue["MaxBatchRequests"] = snmpMap.MapIndex(snmpKeys[i]).Interface() + continue + } + metadataConfig := reflect.TypeOf(metadata.Configuration{}) + for j := range metadataConfig.NumField() { + if helpers.MapStructureMatchName(k.String(), metadataConfig.Field(j).Name) { + metadataValue[k.String()] = snmpMap.MapIndex(snmpKeys[i]).Interface() + continue outerSNMP + } + } + providerValue[k.String()] = snmpMap.MapIndex(snmpKeys[i]).Interface() + } + + providerValue["type"] = "snmp" + metadataValue["provider"] = providerValue + from.SetMapIndex(reflect.ValueOf("metadata"), reflect.ValueOf(metadataValue)) + from.SetMapIndex(*snmpKey, reflect.Value{}) + } + } + + // bmp → routing + { + var bmpKey, routingKey *reflect.Value + fromKeys := from.MapKeys() + for i, k := range fromKeys { + k = helpers.ElemOrIdentity(k) + if k.Kind() != reflect.String { + return from.Interface(), nil + } + if helpers.MapStructureMatchName(k.String(), "Bmp") { + bmpKey = &fromKeys[i] + } else if helpers.MapStructureMatchName(k.String(), "Routing") { + routingKey = &fromKeys[i] + } + } + if bmpKey != nil { + if routingKey != nil { + return nil, fmt.Errorf("cannot have both %q and %q", bmpKey.String(), routingKey.String()) + } + + // Build the routing configuration + providerValue := gin.H{} + routingValue := gin.H{} + // Dispatch values from bmp key into routing + bmpMap := helpers.ElemOrIdentity(from.MapIndex(*bmpKey)) + bmpKeys := bmpMap.MapKeys() + outerBMP: + for i, k := range bmpKeys { + k = helpers.ElemOrIdentity(k) + if k.Kind() != reflect.String { + continue + } + routingConfig := reflect.TypeOf(routing.Configuration{}) + for j := range routingConfig.NumField() { + if helpers.MapStructureMatchName(k.String(), routingConfig.Field(j).Name) { + routingValue[k.String()] = bmpMap.MapIndex(bmpKeys[i]).Interface() + continue outerBMP + } + } + providerValue[k.String()] = bmpMap.MapIndex(bmpKeys[i]).Interface() + } + + providerValue["type"] = "bmp" + routingValue["provider"] = providerValue + from.SetMapIndex(reflect.ValueOf("routing"), reflect.ValueOf(routingValue)) + from.SetMapIndex(*bmpKey, reflect.Value{}) + } + } + + return from.Interface(), nil + } +} + +func init() { + helpers.RegisterMapstructureUnmarshallerHook(OutletConfigurationUnmarshallerHook()) +} diff --git a/cmd/outlet_test.go b/cmd/outlet_test.go new file mode 100644 index 00000000..0c4db8ec --- /dev/null +++ b/cmd/outlet_test.go @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package cmd + +import ( + "bytes" + "testing" + + "akvorado/common/reporter" +) + +func TestOutletStart(t *testing.T) { + r := reporter.NewMock(t) + config := OutletConfiguration{} + config.Reset() + if err := outletStart(r, config, true); err != nil { + t.Fatalf("outletStart() error:\n%+v", err) + } +} + +func TestOutlet(t *testing.T) { + root := RootCmd + buf := new(bytes.Buffer) + root.SetOut(buf) + root.SetArgs([]string{"outlet", "--check", "/dev/null"}) + err := root.Execute() + if err != nil { + t.Errorf("`outlet` error:\n%+v", err) + } +} diff --git a/cmd/testdata/configurations/bmp-to-routing/expected.yaml b/cmd/testdata/configurations/bmp-to-routing/expected.yaml index b2c6b4f0..1f9104ad 100644 --- a/cmd/testdata/configurations/bmp-to-routing/expected.yaml +++ b/cmd/testdata/configurations/bmp-to-routing/expected.yaml @@ -1,6 +1,6 @@ --- paths: - inlet.0.routing: + outlet.0.routing: provider: type: bmp listen: 127.0.0.1:1179 @@ -13,8 +13,8 @@ paths: ribpeerremovalmaxqueue: 10000 ribpeerremovalmaxtime: 100ms ribpeerremovalsleepinterval: 500ms - inlet.0.core.asnproviders: + outlet.0.core.asnproviders: - flow - routing - inlet.0.core.netproviders: + outlet.0.core.netproviders: - routing diff --git a/cmd/testdata/configurations/core-ignore-asn-from-flows/expected.yaml b/cmd/testdata/configurations/core-ignore-asn-from-flows/expected.yaml index 96116455..2d9bbee1 100644 --- a/cmd/testdata/configurations/core-ignore-asn-from-flows/expected.yaml +++ b/cmd/testdata/configurations/core-ignore-asn-from-flows/expected.yaml @@ -1,5 +1,5 @@ --- paths: - inlet.0.core.asnproviders: + outlet.0.core.asnproviders: - routing - geo-ip diff --git a/cmd/testdata/configurations/empty/expected.yaml b/cmd/testdata/configurations/empty/expected.yaml index 1c061be5..f90d97d5 100644 --- a/cmd/testdata/configurations/empty/expected.yaml +++ b/cmd/testdata/configurations/empty/expected.yaml @@ -4,5 +4,5 @@ paths: - 127.0.0.1:9092 inlet.0.kafka.brokers: - 127.0.0.1:9092 - clickhouse.kafka.brokers: + outlet.0.kafka.brokers: - 127.0.0.1:9092 diff --git a/cmd/testdata/configurations/gnmi/expected.yaml b/cmd/testdata/configurations/gnmi/expected.yaml index dbc9480b..9e622e79 100644 --- a/cmd/testdata/configurations/gnmi/expected.yaml +++ b/cmd/testdata/configurations/gnmi/expected.yaml @@ -1,6 +1,6 @@ --- paths: - inlet.0.metadata.providers: + outlet.0.metadata.providers: - type: gnmi timeout: "1s" minimalrefreshinterval: "1m0s" diff --git a/cmd/testdata/configurations/schema-customdict/expected.yaml b/cmd/testdata/configurations/schema-customdict/expected.yaml index 8156a2b0..50c7fda2 100644 --- a/cmd/testdata/configurations/schema-customdict/expected.yaml +++ b/cmd/testdata/configurations/schema-customdict/expected.yaml @@ -1,6 +1,6 @@ --- paths: - inlet.0.schema: + outlet.0.schema: customdictionaries: test: source: test.csv @@ -45,4 +45,4 @@ paths: enabled: [] materialize: [] maintableonly: [] - notmaintableonly: [] \ No newline at end of file + notmaintableonly: [] diff --git a/cmd/testdata/configurations/schema-enable-disable/expected.yaml b/cmd/testdata/configurations/schema-enable-disable/expected.yaml index a65e9854..2078f23a 100644 --- a/cmd/testdata/configurations/schema-enable-disable/expected.yaml +++ b/cmd/testdata/configurations/schema-enable-disable/expected.yaml @@ -1,6 +1,6 @@ --- paths: - inlet.0.schema: + outlet.0.schema: customdictionaries: {} disabled: - SrcCountry diff --git a/cmd/testdata/configurations/shipped/expected.yaml b/cmd/testdata/configurations/shipped/expected.yaml index 551dce85..27621cce 100644 --- a/cmd/testdata/configurations/shipped/expected.yaml +++ b/cmd/testdata/configurations/shipped/expected.yaml @@ -45,7 +45,7 @@ paths: - kafka:9092 inlet.0.kafka.brokers: - kafka:9092 - clickhouse.kafka.brokers: + outlet.0.kafka.brokers: - kafka:9092 console.0.clickhouse.servers: - clickhouse:9000 diff --git a/cmd/testdata/configurations/snmp-communities-migration/expected.yaml b/cmd/testdata/configurations/snmp-communities-migration/expected.yaml index acedeaef..4be37535 100644 --- a/cmd/testdata/configurations/snmp-communities-migration/expected.yaml +++ b/cmd/testdata/configurations/snmp-communities-migration/expected.yaml @@ -1,6 +1,6 @@ --- paths: - inlet.0.metadata.providers: + outlet.0.metadata.providers: - type: snmp pollerretries: 1 pollertimeout: 1s diff --git a/cmd/testdata/configurations/snmp-credentials-migration/expected.yaml b/cmd/testdata/configurations/snmp-credentials-migration/expected.yaml index 3c68496e..4dcecf2f 100644 --- a/cmd/testdata/configurations/snmp-credentials-migration/expected.yaml +++ b/cmd/testdata/configurations/snmp-credentials-migration/expected.yaml @@ -1,6 +1,6 @@ --- paths: - inlet.0.metadata.providers: + outlet.0.metadata.providers: - type: snmp pollerretries: 1 pollertimeout: 1s diff --git a/cmd/testdata/configurations/snmp-ports/expected.yaml b/cmd/testdata/configurations/snmp-ports/expected.yaml index e6c8b042..bfaf2e2b 100644 --- a/cmd/testdata/configurations/snmp-ports/expected.yaml +++ b/cmd/testdata/configurations/snmp-ports/expected.yaml @@ -1,4 +1,4 @@ --- paths: - inlet.0.metadata.providers.0.ports: + outlet.0.metadata.providers.0.ports: ::/0: 1611 diff --git a/cmd/testdata/configurations/snmp-to-metadata/expected.yaml b/cmd/testdata/configurations/snmp-to-metadata/expected.yaml index bf4dd4ca..df2b4ece 100644 --- a/cmd/testdata/configurations/snmp-to-metadata/expected.yaml +++ b/cmd/testdata/configurations/snmp-to-metadata/expected.yaml @@ -1,6 +1,6 @@ --- paths: - inlet.0.metadata: + outlet.0.metadata: workers: 10 maxbatchrequests: 20 cacheduration: 30m0s diff --git a/common/clickhousedb/config.go b/common/clickhousedb/config.go index 34922dab..04fa78fa 100644 --- a/common/clickhousedb/config.go +++ b/common/clickhousedb/config.go @@ -6,6 +6,8 @@ package clickhousedb import ( "time" + "github.com/ClickHouse/ch-go" + "akvorado/common/helpers" ) @@ -55,3 +57,19 @@ func (c *Component) ClusterName() string { func (c *Component) DatabaseName() string { return c.config.Database } + +// ChGoOptions returns options suitable to use with ch-go and the list of +// available servers. +func (c *Component) ChGoOptions() (ch.Options, []string) { + tlsConfig, _ := c.config.TLS.MakeTLSConfig() + return ch.Options{ + Address: c.config.Servers[0], + Database: c.config.Database, + User: c.config.Username, + Password: c.config.Password, + Compression: ch.CompressionLZ4, + ClientName: "akvorado", + DialTimeout: c.config.DialTimeout, + TLS: tlsConfig, + }, c.config.Servers +} diff --git a/common/helpers/ether.go b/common/helpers/ether.go index 11c22e19..3fa19453 100644 --- a/common/helpers/ether.go +++ b/common/helpers/ether.go @@ -3,9 +3,24 @@ package helpers +import "net" + const ( // ETypeIPv4 is the ether type for IPv4 ETypeIPv4 = 0x800 // ETypeIPv6 is the ether type for IPv6 ETypeIPv6 = 0x86dd ) + +// MACToUint64 converts a MAC address to an uint64 +func MACToUint64(mac net.HardwareAddr) uint64 { + if len(mac) != 6 { + return 0 + } + return uint64(mac[0])<<40 | + uint64(mac[1])<<32 | + uint64(mac[2])<<24 | + uint64(mac[3])<<16 | + uint64(mac[4])<<8 | + uint64(mac[5]) +} diff --git a/common/pb/rawflow.go b/common/pb/rawflow.go new file mode 100644 index 00000000..b322598a --- /dev/null +++ b/common/pb/rawflow.go @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: 2024 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +// Package pb contains the definition of RawFlow, the protobuf-based +// structure to exchange flows between the inlet and the outlet. +package pb + +import ( + "errors" + "fmt" + + "akvorado/common/helpers/bimap" +) + +// Version is the version of the schema. On incompatible changes, this should be +// bumped. +var Version = 5 + +var decoderMap = bimap.New(map[RawFlow_Decoder]string{ + RawFlow_DECODER_NETFLOW: "netflow", + RawFlow_DECODER_SFLOW: "sflow", +}) + +// MarshalText turns a decoder to text +func (d RawFlow_Decoder) MarshalText() ([]byte, error) { + got, ok := decoderMap.LoadValue(d) + if ok { + return []byte(got), nil + } + return nil, errors.New("unknown decoder") +} + +// UnmarshalText provides a decoder from text +func (d *RawFlow_Decoder) UnmarshalText(input []byte) error { + if len(input) == 0 { + *d = RawFlow_DECODER_UNSPECIFIED + return nil + } + got, ok := decoderMap.LoadKey(string(input)) + if ok { + *d = got + return nil + } + return errors.New("unknown decoder") +} + +var tsMap = bimap.New(map[RawFlow_TimestampSource]string{ + RawFlow_TS_INPUT: "input", // this is the default value + RawFlow_TS_NETFLOW_FIRST_SWITCHED: "netflow-first-switched", + RawFlow_TS_NETFLOW_PACKET: "netflow-packet", +}) + +// MarshalText turns a timestamp source to text +func (ts RawFlow_TimestampSource) MarshalText() ([]byte, error) { + got, ok := tsMap.LoadValue(ts) + if ok { + return []byte(got), nil + } + return nil, errors.New("unknown timestamp source") +} + +// UnmarshalText provides a timestamp source from text +func (ts *RawFlow_TimestampSource) UnmarshalText(input []byte) error { + if len(input) == 0 { + *ts = RawFlow_TS_INPUT + return nil + } + if string(input) == "udp" { + *ts = RawFlow_TS_INPUT + return nil + } + got, ok := tsMap.LoadKey(string(input)) + if ok { + *ts = got + return nil + } + return fmt.Errorf("unknown timestamp source %q", string(input)) +} diff --git a/common/pb/rawflow.proto b/common/pb/rawflow.proto new file mode 100644 index 00000000..6737482e --- /dev/null +++ b/common/pb/rawflow.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; +package input; +option go_package = "akvorado/common/pb"; + +// RawFlow is an undecoded flow with some options. +message RawFlow { + uint64 time_received = 1; // when the flow was received + bytes payload = 2; // payload of the flow + bytes source_address = 3; // source IPv6 address + bool use_source_address = 4; // use source address as exporter address + + // Decoding options + enum Decoder { + DECODER_UNSPECIFIED = 0; + DECODER_NETFLOW = 1; + DECODER_SFLOW = 2; + DECODER_GOB = 3; + } + enum TimestampSource { + TS_INPUT = 0; + TS_NETFLOW_PACKET = 1; + TS_NETFLOW_FIRST_SWITCHED = 2; + } + Decoder decoder = 5; + TimestampSource timestamp_source = 6; +} diff --git a/common/schema/clickhouse.go b/common/schema/clickhouse.go index 2844cc98..1dcd428d 100644 --- a/common/schema/clickhouse.go +++ b/common/schema/clickhouse.go @@ -4,9 +4,15 @@ package schema import ( + "encoding/base32" "fmt" + "hash/fnv" + "net/netip" "slices" "strings" + "time" + + "github.com/ClickHouse/ch-go/proto" ) // ClickHouseDefinition turns a column into a declaration for ClickHouse @@ -21,6 +27,33 @@ func (column Column) ClickHouseDefinition() string { return strings.Join(result, " ") } +// newProtoColumn turns a column into its proto.Column definition +func (column Column) newProtoColumn() proto.Column { + if strings.HasPrefix(column.ClickHouseType, "Enum8(") { + // Enum8 is a special case. We do not want to use ColAuto as it comes + // with a performance penalty due to conversion between key values. + return new(proto.ColEnum8) + } + + col := &proto.ColAuto{} + err := col.Infer(proto.ColumnType(column.ClickHouseType)) + if err != nil { + panic(fmt.Sprintf("unhandled ClickHouse type %q", column.ClickHouseType)) + } + return col.Data +} + +// wrapProtoColumn optionally wraps the proto.Column for use in proto.Input +func (column Column) wrapProtoColumn(in proto.Column) proto.Column { + if strings.HasPrefix(column.ClickHouseType, "Enum8(") { + // Enum8 is a special case. See above. + ddl := column.ClickHouseType[6 : len(column.ClickHouseType)-1] + return proto.Wrap(in, ddl) + } + + return in +} + // ClickHouseTableOption is an option to alter the values returned by ClickHouseCreateTable() and ClickHouseSelectColumns(). type ClickHouseTableOption int @@ -29,18 +62,12 @@ const ( ClickHouseSkipMainOnlyColumns ClickHouseTableOption = iota // ClickHouseSkipGeneratedColumns skips the columns with a GenerateFrom value ClickHouseSkipGeneratedColumns - // ClickHouseSkipTransformColumns skips the columns with a TransformFrom value - ClickHouseSkipTransformColumns // ClickHouseSkipAliasedColumns skips the columns with a Alias value ClickHouseSkipAliasedColumns // ClickHouseSkipTimeReceived skips the time received column ClickHouseSkipTimeReceived - // ClickHouseUseTransformFromType uses the type from TransformFrom if any - ClickHouseUseTransformFromType // ClickHouseSubstituteGenerates changes the column name to use the default generated value ClickHouseSubstituteGenerates - // ClickHouseSubstituteTransforms changes the column name to use the transformed value - ClickHouseSubstituteTransforms ) // ClickHouseCreateTable returns the columns for the CREATE TABLE clause in ClickHouse. @@ -72,27 +99,12 @@ func (schema Schema) clickhouseIterate(fn func(Column), options ...ClickHouseTab if slices.Contains(options, ClickHouseSkipGeneratedColumns) && column.ClickHouseGenerateFrom != "" && !column.ClickHouseSelfGenerated { continue } - if slices.Contains(options, ClickHouseSkipTransformColumns) && column.ClickHouseTransformFrom != nil { - continue - } if slices.Contains(options, ClickHouseSkipAliasedColumns) && column.ClickHouseAlias != "" { continue } - if slices.Contains(options, ClickHouseUseTransformFromType) && column.ClickHouseTransformFrom != nil { - for _, ocol := range column.ClickHouseTransformFrom { - // We assume we only need to use name/type - column.Name = ocol.Name - column.ClickHouseType = ocol.ClickHouseType - fn(column) - } - continue - } if slices.Contains(options, ClickHouseSubstituteGenerates) && column.ClickHouseGenerateFrom != "" { column.Name = fmt.Sprintf("%s AS %s", column.ClickHouseGenerateFrom, column.Name) } - if slices.Contains(options, ClickHouseSubstituteTransforms) && column.ClickHouseTransformFrom != nil { - column.Name = fmt.Sprintf("%s AS %s", column.ClickHouseTransformTo, column.Name) - } fn(column) } } @@ -119,3 +131,256 @@ func (schema Schema) ClickHousePrimaryKeys() []string { } return cols } + +// ClickHouseHash returns an hash of the inpt table in ClickHouse +func (schema Schema) ClickHouseHash() string { + hash := fnv.New128() + create := schema.ClickHouseCreateTable(ClickHouseSkipGeneratedColumns, ClickHouseSkipAliasedColumns) + hash.Write([]byte(create)) + hashString := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash.Sum(nil)) + return fmt.Sprintf("%sv5", hashString) +} + +// AppendDateTime adds a DateTime value to the provided column +func (bf *FlowMessage) AppendDateTime(columnKey ColumnKey, value uint32) { + col := bf.batch.columns[columnKey] + if value == 0 || col == nil || bf.batch.columnSet.Test(uint(columnKey)) { + return + } + bf.batch.columnSet.Set(uint(columnKey)) + col.(*proto.ColDateTime).AppendRaw(proto.DateTime(value)) + bf.appendDebug(columnKey, value) +} + +// AppendUint adds an UInt64/32/16/8 or Enum8 value to the provided column +func (bf *FlowMessage) AppendUint(columnKey ColumnKey, value uint64) { + col := bf.batch.columns[columnKey] + if value == 0 || col == nil || bf.batch.columnSet.Test(uint(columnKey)) { + return + } + switch col := col.(type) { + case *proto.ColUInt64: + col.Append(value) + case *proto.ColUInt32: + col.Append(uint32(value)) + case *proto.ColUInt16: + col.Append(uint16(value)) + case *proto.ColUInt8: + col.Append(uint8(value)) + case *proto.ColEnum8: + col.Append(proto.Enum8(value)) + default: + panic(fmt.Sprintf("unhandled uint type %q", col.Type())) + } + bf.batch.columnSet.Set(uint(columnKey)) + bf.appendDebug(columnKey, value) +} + +// AppendString adds a String value to the provided column +func (bf *FlowMessage) AppendString(columnKey ColumnKey, value string) { + col := bf.batch.columns[columnKey] + if value == "" || col == nil || bf.batch.columnSet.Test(uint(columnKey)) { + return + } + switch col := col.(type) { + case *proto.ColLowCardinality[string]: + col.Append(value) + default: + panic(fmt.Sprintf("unhandled string type %q", col.Type())) + } + bf.batch.columnSet.Set(uint(columnKey)) + bf.appendDebug(columnKey, value) +} + +// AppendIPv6 adds an IPv6 value to the provided column +func (bf *FlowMessage) AppendIPv6(columnKey ColumnKey, value netip.Addr) { + col := bf.batch.columns[columnKey] + if !value.IsValid() || col == nil || bf.batch.columnSet.Test(uint(columnKey)) { + return + } + switch col := col.(type) { + case *proto.ColIPv6: + col.Append(value.As16()) + case *proto.ColLowCardinality[proto.IPv6]: + col.Append(value.As16()) + default: + panic(fmt.Sprintf("unhandled string type %q", col.Type())) + } + bf.batch.columnSet.Set(uint(columnKey)) + bf.appendDebug(columnKey, value) +} + +// AppendArrayUInt32 adds an Array(UInt32) value to the provided column +func (bf *FlowMessage) AppendArrayUInt32(columnKey ColumnKey, value []uint32) { + col := bf.batch.columns[columnKey] + if len(value) == 0 || col == nil || bf.batch.columnSet.Test(uint(columnKey)) { + return + } + bf.batch.columnSet.Set(uint(columnKey)) + col.(*proto.ColArr[uint32]).Append(value) + bf.appendDebug(columnKey, value) +} + +// AppendArrayUInt128 adds an Array(UInt128) value to the provided column +func (bf *FlowMessage) AppendArrayUInt128(columnKey ColumnKey, value []UInt128) { + col := bf.batch.columns[columnKey] + if len(value) == 0 || col == nil || bf.batch.columnSet.Test(uint(columnKey)) { + return + } + bf.batch.columnSet.Set(uint(columnKey)) + col.(*proto.ColArr[proto.UInt128]).Append(value) + bf.appendDebug(columnKey, value) +} + +func (bf *FlowMessage) appendDebug(columnKey ColumnKey, value any) { + if !debug { + return + } + if bf.OtherColumns == nil { + bf.OtherColumns = make(map[ColumnKey]any) + } + bf.OtherColumns[columnKey] = value +} + +// check executes some sanity checks when in debug mode. It should be called +// only after finalization. +func (bf *FlowMessage) check() { + if !debug { + return + } + if debug { + // Check that all columns have the right amount of rows + for idx, col := range bf.batch.columns { + if col == nil { + continue + } + if col.Rows() != bf.batch.rowCount { + panic(fmt.Sprintf("row %s has a count of %d instead of %d", ColumnKey(idx), col.Rows(), bf.batch.rowCount)) + } + } + } +} + +// appendDefaultValue appends a default/zero value to the given column. +func (bf *FlowMessage) appendDefaultValues() { + for idx, col := range bf.batch.columns { + // Skip unpopulated columns + if col == nil { + continue + } + // Or columns already set + if bf.batch.columnSet.Test(uint(idx)) { + continue + } + // Put the default value depending on the real type + switch col := col.(type) { + case *proto.ColUInt64: + col.Append(0) + case *proto.ColUInt32: + col.Append(0) + case *proto.ColUInt16: + col.Append(0) + case *proto.ColUInt8: + col.Append(0) + case *proto.ColIPv6: + col.Append([16]byte{}) + case *proto.ColDateTime: + col.Append(time.Unix(0, 0)) + case *proto.ColEnum8: + col.Append(0) + case *proto.ColLowCardinality[string]: + col.Append("") + case *proto.ColLowCardinality[proto.IPv6]: + col.Append(proto.IPv6{}) + case *proto.ColArr[uint32]: + col.Append([]uint32{}) + case *proto.ColArr[proto.UInt128]: + col.Append([]proto.UInt128{}) + default: + panic(fmt.Sprintf("unhandled ClickHouse type %q", col.Type())) + } + } +} + +// Undo reverts the current changes. This should revert the various Append() functions. +func (bf *FlowMessage) Undo() { + for idx, col := range bf.batch.columns { + if col == nil { + continue + } + if !bf.batch.columnSet.Test(uint(idx)) { + continue + } + switch col := col.(type) { + case *proto.ColUInt64: + *col = (*col)[:len(*col)-1] + case *proto.ColUInt32: + *col = (*col)[:len(*col)-1] + case *proto.ColUInt16: + *col = (*col)[:len(*col)-1] + case *proto.ColUInt8: + *col = (*col)[:len(*col)-1] + case *proto.ColIPv6: + *col = (*col)[:len(*col)-1] + case *proto.ColDateTime: + col.Data = col.Data[:len(col.Data)-1] + case *proto.ColEnum8: + *col = (*col)[:len(*col)-1] + case *proto.ColLowCardinality[string]: + col.Values = col.Values[:len(col.Values)-1] + case *proto.ColLowCardinality[proto.IPv6]: + col.Values = col.Values[:len(col.Values)-1] + case *proto.ColArr[uint32]: + l := len(col.Offsets) + if l > 0 { + start := uint64(0) + if l > 1 { + start = col.Offsets[l-2] + } + data := col.Data.(*proto.ColUInt32) + *data = (*data)[:start] + col.Data = data + col.Offsets = col.Offsets[:l-1] + } + case *proto.ColArr[proto.UInt128]: + l := len(col.Offsets) + if l > 0 { + start := uint64(0) + if l > 1 { + start = col.Offsets[l-2] + } + data := col.Data.(*proto.ColUInt128) + *data = (*data)[:start] + col.Data = data + col.Offsets = col.Offsets[:l-1] + } + default: + panic(fmt.Sprintf("unhandled ClickHouse type %q", col.Type())) + } + } + bf.reset() +} + +// Finalize finalizes the current FlowMessage. It can then be reused for the +// next one. It is crucial to always call Finalize, otherwise the batch could be +// faulty. +func (bf *FlowMessage) Finalize() { + bf.AppendDateTime(ColumnTimeReceived, bf.TimeReceived) + bf.AppendUint(ColumnSamplingRate, bf.SamplingRate) + bf.AppendIPv6(ColumnExporterAddress, bf.ExporterAddress) + bf.AppendUint(ColumnSrcAS, uint64(bf.SrcAS)) + bf.AppendUint(ColumnDstAS, uint64(bf.DstAS)) + bf.AppendUint(ColumnSrcNetMask, uint64(bf.SrcNetMask)) + bf.AppendUint(ColumnDstNetMask, uint64(bf.DstNetMask)) + bf.AppendIPv6(ColumnSrcAddr, bf.SrcAddr) + bf.AppendIPv6(ColumnDstAddr, bf.DstAddr) + bf.AppendIPv6(ColumnNextHop, bf.NextHop) + if !bf.schema.IsDisabled(ColumnGroupL2) { + bf.AppendUint(ColumnSrcVlan, uint64(bf.SrcVlan)) + bf.AppendUint(ColumnDstVlan, uint64(bf.DstVlan)) + } + bf.batch.rowCount++ + bf.appendDefaultValues() + bf.reset() + bf.check() +} diff --git a/common/schema/clickhouse_test.go b/common/schema/clickhouse_test.go new file mode 100644 index 00000000..cee66359 --- /dev/null +++ b/common/schema/clickhouse_test.go @@ -0,0 +1,609 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package schema + +import ( + "net/netip" + "slices" + "testing" + + "akvorado/common/helpers" + + "github.com/ClickHouse/ch-go/proto" +) + +func TestAppendDefault(t *testing.T) { + c := NewMock(t).EnableAllColumns() + bf := c.NewFlowMessage() + bf.Finalize() + if bf.batch.rowCount != 1 { + t.Errorf("rowCount should be 1, not %d", bf.batch.rowCount) + } + if bf.batch.columnSet.Any() { + t.Error("columnSet should be empty after finalize") + } + for idx, col := range bf.batch.columns { + if col == nil { + continue + } + if col.Rows() != 1 { + t.Errorf("column %q should be length 1", ColumnKey(idx)) + } + } +} + +func TestAppendBasics(t *testing.T) { + c := NewMock(t) + bf := c.NewFlowMessage() + + // Test basic append + bf.AppendDateTime(ColumnTimeReceived, 1000) + bf.AppendUint(ColumnSamplingRate, 20000) + bf.AppendUint(ColumnDstAS, 65000) + + // Test zero value (should not append) + bf.AppendUint(ColumnSrcAS, 0) + + // Test duplicate append + bf.AppendUint(ColumnPackets, 100) + bf.AppendUint(ColumnPackets, 200) + + expected := map[ColumnKey]any{ + ColumnTimeReceived: 1000, + ColumnSamplingRate: 20000, + ColumnDstAS: 65000, + ColumnPackets: 100, + } + got := bf.OtherColumns + if diff := helpers.Diff(got, expected); diff != "" { + t.Errorf("Append() (-got, +want):\n%s", diff) + } + + bf.Finalize() + for idx, col := range bf.batch.columns { + if col == nil { + continue + } + if col.Rows() != 1 { + t.Errorf("column %q should be length 1", ColumnKey(idx)) + } + } +} + +func TestAppendWithDisabledColumns(t *testing.T) { + c := NewMock(t) + bf := c.NewFlowMessage() + + // Try to append to a disabled column (L2 group is disabled by default in mock) + bf.AppendUint(ColumnSrcVlan, 100) + bf.Finalize() +} + +func TestAppendArrayUInt32Columns(t *testing.T) { + c := NewMock(t) + bf := c.NewFlowMessage() + bf.AppendArrayUInt32(ColumnDstASPath, []uint32{65400, 65500, 65001}) + bf.Finalize() + bf.AppendArrayUInt32(ColumnDstASPath, []uint32{65403, 65503, 65003}) + bf.Finalize() + + // Verify column has data + got := bf.batch.columns[ColumnDstASPath].(*proto.ColArr[uint32]) + expected := proto.ColArr[uint32]{ + Offsets: proto.ColUInt64{3, 6}, + Data: &proto.ColUInt32{65400, 65500, 65001, 65403, 65503, 65003}, + } + if diff := helpers.Diff(got, expected); diff != "" { + t.Errorf("AppendArrayUInt32 (-got, +want):\n%s", diff) + } +} + +func TestAppendArrayUInt128Columns(t *testing.T) { + c := NewMock(t) + bf := c.NewFlowMessage() + bf.AppendArrayUInt128(ColumnDstLargeCommunities, []UInt128{ + { + High: (65401 << 32) + 100, + Low: 200, + }, + { + High: (65401 << 32) + 100, + Low: 201, + }, + }) + bf.Finalize() + + got := bf.batch.columns[ColumnDstLargeCommunities].(*proto.ColArr[proto.UInt128]) + expected := proto.ColArr[proto.UInt128]{ + Offsets: proto.ColUInt64{2}, + Data: &proto.ColUInt128{ + {High: (65401 << 32) + 100, Low: 200}, + {High: (65401 << 32) + 100, Low: 201}, + }, + } + if diff := helpers.Diff(got, expected); diff != "" { + t.Errorf("AppendArrayUInt128 (-got, +want):\n%s", diff) + } +} + +func TestUndoUInt64(t *testing.T) { + c := NewMock(t) + bf := c.NewFlowMessage() + + // Add two values + bf.AppendUint(ColumnBytes, 100) + bf.AppendUint(ColumnPackets, 200) + + // Check we have the expected initial state + bytesCol := bf.batch.columns[ColumnBytes].(*proto.ColUInt64) + packetsCol := bf.batch.columns[ColumnPackets].(*proto.ColUInt64) + + expectedBytes := proto.ColUInt64{100} + expectedPackets := proto.ColUInt64{200} + + if diff := helpers.Diff(bytesCol, expectedBytes); diff != "" { + t.Errorf("Initial bytes column state (-got, +want):\n%s", diff) + } + if diff := helpers.Diff(packetsCol, expectedPackets); diff != "" { + t.Errorf("Initial packets column state (-got, +want):\n%s", diff) + } + + // Undo should remove the last appended values + bf.Undo() + + expectedBytesAfter := proto.ColUInt64{} + expectedPacketsAfter := proto.ColUInt64{} + + if diff := helpers.Diff(bytesCol, expectedBytesAfter); diff != "" { + t.Errorf("Bytes column after undo (-got, +want):\n%s", diff) + } + if diff := helpers.Diff(packetsCol, expectedPacketsAfter); diff != "" { + t.Errorf("Packets column after undo (-got, +want):\n%s", diff) + } +} + +func TestUndoUInt32(t *testing.T) { + c := NewMock(t) + bf := c.NewFlowMessage() + + // Add two values + bf.AppendUint(ColumnSrcAS, 65001) + bf.AppendUint(ColumnDstAS, 65002) + + // Check we have the expected initial state + srcCol := bf.batch.columns[ColumnSrcAS].(*proto.ColUInt32) + dstCol := bf.batch.columns[ColumnDstAS].(*proto.ColUInt32) + + expectedSrc := proto.ColUInt32{65001} + expectedDst := proto.ColUInt32{65002} + + if diff := helpers.Diff(srcCol, expectedSrc); diff != "" { + t.Errorf("Initial SrcAS column state (-got, +want):\n%s", diff) + } + if diff := helpers.Diff(dstCol, expectedDst); diff != "" { + t.Errorf("Initial DstAS column state (-got, +want):\n%s", diff) + } + + // Undo should remove the last appended values + bf.Undo() + + expectedSrcAfter := proto.ColUInt32{} + expectedDstAfter := proto.ColUInt32{} + + if diff := helpers.Diff(srcCol, expectedSrcAfter); diff != "" { + t.Errorf("SrcAS column after undo (-got, +want):\n%s", diff) + } + if diff := helpers.Diff(dstCol, expectedDstAfter); diff != "" { + t.Errorf("DstAS column after undo (-got, +want):\n%s", diff) + } +} + +func TestUndoUInt16(t *testing.T) { + c := NewMock(t) + bf := c.NewFlowMessage() + + // Add two values + bf.AppendUint(ColumnSrcPort, 80) + bf.AppendUint(ColumnDstPort, 443) + + // Check we have the expected initial state + srcCol := bf.batch.columns[ColumnSrcPort].(*proto.ColUInt16) + dstCol := bf.batch.columns[ColumnDstPort].(*proto.ColUInt16) + + expectedSrc := proto.ColUInt16{80} + expectedDst := proto.ColUInt16{443} + + if diff := helpers.Diff(srcCol, expectedSrc); diff != "" { + t.Errorf("Initial SrcPort column state (-got, +want):\n%s", diff) + } + if diff := helpers.Diff(dstCol, expectedDst); diff != "" { + t.Errorf("Initial DstPort column state (-got, +want):\n%s", diff) + } + + // Undo should remove the last appended values + bf.Undo() + + expectedSrcAfter := proto.ColUInt16{} + expectedDstAfter := proto.ColUInt16{} + + if diff := helpers.Diff(srcCol, expectedSrcAfter); diff != "" { + t.Errorf("SrcPort column after undo (-got, +want):\n%s", diff) + } + if diff := helpers.Diff(dstCol, expectedDstAfter); diff != "" { + t.Errorf("DstPort column after undo (-got, +want):\n%s", diff) + } +} + +func TestUndoUInt8(t *testing.T) { + c := NewMock(t) + bf := c.NewFlowMessage() + + // Add value + bf.AppendUint(ColumnSrcNetMask, 6) + + // Check we have the expected initial state + col := bf.batch.columns[ColumnSrcNetMask].(*proto.ColUInt8) + expected := proto.ColUInt8{6} + + if diff := helpers.Diff(col, expected); diff != "" { + t.Errorf("Initial Proto column state (-got, +want):\n%s", diff) + } + + // Undo should remove the last appended value + bf.Undo() + + expectedAfter := proto.ColUInt8{} + + if diff := helpers.Diff(col, expectedAfter); diff != "" { + t.Errorf("Proto column after undo (-got, +want):\n%s", diff) + } +} + +func TestUndoIPv6(t *testing.T) { + c := NewMock(t) + bf := c.NewFlowMessage() + + // Add IPv6 values + srcAddr := netip.MustParseAddr("2001:db8::1") + dstAddr := netip.MustParseAddr("2001:db8::2") + + bf.AppendIPv6(ColumnSrcAddr, srcAddr) + bf.AppendIPv6(ColumnDstAddr, dstAddr) + + // Check we have the expected initial state + srcCol := bf.batch.columns[ColumnSrcAddr].(*proto.ColIPv6) + dstCol := bf.batch.columns[ColumnDstAddr].(*proto.ColIPv6) + + expectedSrc := proto.ColIPv6{srcAddr.As16()} + expectedDst := proto.ColIPv6{dstAddr.As16()} + + if diff := helpers.Diff(srcCol, expectedSrc); diff != "" { + t.Errorf("Initial SrcAddr column state (-got, +want):\n%s", diff) + } + if diff := helpers.Diff(dstCol, expectedDst); diff != "" { + t.Errorf("Initial DstAddr column state (-got, +want):\n%s", diff) + } + + // Undo should remove the values + bf.Undo() + + expectedSrcAfter := proto.ColIPv6{} + expectedDstAfter := proto.ColIPv6{} + + if diff := helpers.Diff(srcCol, expectedSrcAfter); diff != "" { + t.Errorf("SrcAddr column after undo (-got, +want):\n%s", diff) + } + if diff := helpers.Diff(dstCol, expectedDstAfter); diff != "" { + t.Errorf("DstAddr column after undo (-got, +want):\n%s", diff) + } +} + +func TestUndoDateTime(t *testing.T) { + c := NewMock(t) + bf := c.NewFlowMessage() + + // Add DateTime value + bf.AppendDateTime(ColumnTimeReceived, 1000) + + // Check we have the expected initial state + col := bf.batch.columns[ColumnTimeReceived].(*proto.ColDateTime) + expected := proto.ColDateTime{Data: []proto.DateTime{1000}} + + if diff := helpers.Diff(col, expected); diff != "" { + t.Errorf("Initial TimeReceived column state (-got, +want):\n%s", diff) + } + + // Undo should remove the value + bf.Undo() + + expectedAfter := proto.ColDateTime{Data: []proto.DateTime{}} + + if diff := helpers.Diff(col, expectedAfter); diff != "" { + t.Errorf("TimeReceived column after undo (-got, +want):\n%s", diff) + } +} + +func TestUndoEnum8(t *testing.T) { + c := NewMock(t) + bf := c.NewFlowMessage() + + // Add Enum8 value (using interface boundary enum) + bf.AppendUint(ColumnInIfBoundary, uint64(InterfaceBoundaryExternal)) + + // Check we have the expected initial state + col := bf.batch.columns[ColumnInIfBoundary].(*proto.ColEnum8) + expected := proto.ColEnum8{proto.Enum8(InterfaceBoundaryExternal)} + + if diff := helpers.Diff(col, expected); diff != "" { + t.Errorf("Initial InIfBoundary column state (-got, +want):\n%s", diff) + } + + // Undo should remove the value + bf.Undo() + + expectedAfter := proto.ColEnum8{} + + if diff := helpers.Diff(col, expectedAfter); diff != "" { + t.Errorf("InIfBoundary column after undo (-got, +want):\n%s", diff) + } +} + +func TestUndoLowCardinalityString(t *testing.T) { + c := NewMock(t) + bf := c.NewFlowMessage() + + // Add LowCardinality string values + bf.AppendString(ColumnExporterName, "router1") + bf.AppendString(ColumnExporterRole, "edge") + + // Check we have the expected initial state + nameCol := bf.batch.columns[ColumnExporterName].(*proto.ColLowCardinality[string]) + roleCol := bf.batch.columns[ColumnExporterRole].(*proto.ColLowCardinality[string]) + + expectedName := proto.ColLowCardinality[string]{Values: []string{"router1"}} + expectedRole := proto.ColLowCardinality[string]{Values: []string{"edge"}} + + if diff := helpers.Diff(nameCol, expectedName); diff != "" { + t.Errorf("Initial ExporterName column state (-got, +want):\n%s", diff) + } + if diff := helpers.Diff(roleCol, expectedRole); diff != "" { + t.Errorf("Initial ExporterRole column state (-got, +want):\n%s", diff) + } + + // Undo should remove the values + bf.Undo() + + expectedNameAfter := proto.ColLowCardinality[string]{Values: []string{}} + expectedRoleAfter := proto.ColLowCardinality[string]{Values: []string{}} + + if diff := helpers.Diff(nameCol, expectedNameAfter); diff != "" { + t.Errorf("ExporterName column after undo (-got, +want):\n%s", diff) + } + if diff := helpers.Diff(roleCol, expectedRoleAfter); diff != "" { + t.Errorf("ExporterRole column after undo (-got, +want):\n%s", diff) + } +} + +func TestUndoLowCardinalityIPv6(t *testing.T) { + c := NewMock(t) + bf := c.NewFlowMessage() + + // Add LowCardinality IPv6 value + addr := netip.MustParseAddr("2001:db8::1") + bf.AppendIPv6(ColumnExporterAddress, addr) + + // Check we have the expected initial state + col := bf.batch.columns[ColumnExporterAddress].(*proto.ColLowCardinality[proto.IPv6]) + expected := proto.ColLowCardinality[proto.IPv6]{Values: []proto.IPv6{addr.As16()}} + + if diff := helpers.Diff(col, expected); diff != "" { + t.Errorf("Initial ExporterAddress column state (-got, +want):\n%s", diff) + } + + // Undo should remove the value + bf.Undo() + + expectedAfter := proto.ColLowCardinality[proto.IPv6]{Values: []proto.IPv6{}} + + if diff := helpers.Diff(col, expectedAfter); diff != "" { + t.Errorf("ExporterAddress column after undo (-got, +want):\n%s", diff) + } +} + +func TestUndoArrayUInt32(t *testing.T) { + c := NewMock(t) + + t.Run("one value", func(t *testing.T) { + bf := c.NewFlowMessage() + bf.AppendArrayUInt32(ColumnDstASPath, []uint32{65001, 65002, 65003}) + + // Check we have the expected initial state + col := bf.batch.columns[ColumnDstASPath].(*proto.ColArr[uint32]) + expected := proto.ColArr[uint32]{ + Offsets: proto.ColUInt64{3}, + Data: &proto.ColUInt32{65001, 65002, 65003}, + } + + if diff := helpers.Diff(*col, expected); diff != "" { + t.Errorf("Initial DstASPath column state (-got, +want):\n%s", diff) + } + + // Undo should remove the array + bf.Undo() + + expectedAfter := proto.ColArr[uint32]{ + Offsets: proto.ColUInt64{}, + Data: &proto.ColUInt32{}, + } + + if diff := helpers.Diff(*col, expectedAfter); diff != "" { + t.Errorf("DstASPath column after undo (-got, +want):\n%s", diff) + } + }) + + t.Run("two values", func(t *testing.T) { + bf := c.NewFlowMessage() + bf.AppendArrayUInt32(ColumnDstASPath, []uint32{65001, 65002, 65003}) + bf.Finalize() + bf.AppendArrayUInt32(ColumnDstASPath, []uint32{65007, 65008}) + + // Check we have the expected initial state + col := bf.batch.columns[ColumnDstASPath].(*proto.ColArr[uint32]) + expected := proto.ColArr[uint32]{ + Offsets: proto.ColUInt64{3, 5}, + Data: &proto.ColUInt32{65001, 65002, 65003, 65007, 65008}, + } + + if diff := helpers.Diff(*col, expected); diff != "" { + t.Errorf("Initial DstASPath column state (-got, +want):\n%s", diff) + } + + // Undo should remove the last array + bf.Undo() + + expectedAfter := proto.ColArr[uint32]{ + Offsets: proto.ColUInt64{3}, + Data: &proto.ColUInt32{65001, 65002, 65003}, + } + + if diff := helpers.Diff(*col, expectedAfter); diff != "" { + t.Errorf("DstASPath column after undo (-got, +want):\n%s", diff) + } + }) + +} + +func TestUndoArrayUInt128(t *testing.T) { + c := NewMock(t) + + t.Run("one value", func(t *testing.T) { + bf := c.NewFlowMessage() + + // Add Array(UInt128) value + bf.AppendArrayUInt128(ColumnDstLargeCommunities, []UInt128{ + {High: (65401 << 32) + 100, Low: 200}, + {High: (65401 << 32) + 100, Low: 201}, + }) + + // Check we have the expected initial state + col := bf.batch.columns[ColumnDstLargeCommunities].(*proto.ColArr[proto.UInt128]) + expected := proto.ColArr[proto.UInt128]{ + Offsets: proto.ColUInt64{2}, + Data: &proto.ColUInt128{ + {High: (65401 << 32) + 100, Low: 200}, + {High: (65401 << 32) + 100, Low: 201}, + }, + } + + if diff := helpers.Diff(*col, expected); diff != "" { + t.Errorf("Initial DstLargeCommunities column state (-got, +want):\n%s", diff) + } + + // Undo should remove the array + bf.Undo() + + expectedAfter := proto.ColArr[proto.UInt128]{ + Offsets: proto.ColUInt64{}, + Data: &proto.ColUInt128{}, + } + + if diff := helpers.Diff(*col, expectedAfter); diff != "" { + t.Errorf("DstLargeCommunities column after undo (-got, +want):\n%s", diff) + } + }) + + t.Run("two values", func(t *testing.T) { + bf := c.NewFlowMessage() + + // Add first Array(UInt128) value + bf.AppendArrayUInt128(ColumnDstLargeCommunities, []UInt128{ + {High: (65401 << 32) + 100, Low: 200}, + {High: (65401 << 32) + 100, Low: 201}, + }) + bf.Finalize() + + // Add second Array(UInt128) value + bf.AppendArrayUInt128(ColumnDstLargeCommunities, []UInt128{ + {High: (65402 << 32) + 100, Low: 300}, + }) + + // Check we have the expected initial state + col := bf.batch.columns[ColumnDstLargeCommunities].(*proto.ColArr[proto.UInt128]) + expected := proto.ColArr[proto.UInt128]{ + Offsets: proto.ColUInt64{2, 3}, + Data: &proto.ColUInt128{ + {High: (65401 << 32) + 100, Low: 200}, + {High: (65401 << 32) + 100, Low: 201}, + {High: (65402 << 32) + 100, Low: 300}, + }, + } + + if diff := helpers.Diff(*col, expected); diff != "" { + t.Errorf("Initial DstLargeCommunities column state (-got, +want):\n%s", diff) + } + + // Undo should remove the last array + bf.Undo() + + expectedAfter := proto.ColArr[proto.UInt128]{ + Offsets: proto.ColUInt64{2}, + Data: &proto.ColUInt128{ + {High: (65401 << 32) + 100, Low: 200}, + {High: (65401 << 32) + 100, Low: 201}, + }, + } + + if diff := helpers.Diff(*col, expectedAfter); diff != "" { + t.Errorf("DstLargeCommunities column after undo (-got, +want):\n%s", diff) + } + }) +} + +func TestBuildProtoInput(t *testing.T) { + // Use a smaller version + exporterAddress := netip.MustParseAddr("::ffff:203.0.113.14") + c := NewMock(t) + bf := c.NewFlowMessage() + got := bf.ClickHouseProtoInput() + + bf.TimeReceived = 1000 + bf.SamplingRate = 20000 + bf.ExporterAddress = exporterAddress + bf.AppendUint(ColumnDstAS, 65000) + bf.AppendUint(ColumnBytes, 200) + bf.AppendUint(ColumnPackets, 300) + bf.Finalize() + + bf.Clear() + + bf.TimeReceived = 1002 + bf.ExporterAddress = exporterAddress + bf.AppendUint(ColumnSrcAS, 65000) + bf.AppendUint(ColumnBytes, 2000) + bf.AppendUint(ColumnPackets, 30) + bf.AppendUint(ColumnBytes, 300) // Duplicate! + bf.Finalize() + + bf.TimeReceived = 1003 + bf.ExporterAddress = exporterAddress + bf.AppendUint(ColumnSrcAS, 65001) + bf.AppendUint(ColumnBytes, 202) + bf.AppendUint(ColumnPackets, 3) + bf.Finalize() + + // Let's compare a subset + expected := proto.Input{ + {Name: "TimeReceived", Data: proto.ColDateTime{Data: []proto.DateTime{1002, 1003}}}, + {Name: "SrcAS", Data: proto.ColUInt32{65000, 65001}}, + {Name: "DstAS", Data: proto.ColUInt32{0, 0}}, + {Name: "Bytes", Data: proto.ColUInt64{2000, 202}}, + {Name: "Packets", Data: proto.ColUInt64{30, 3}}, + } + got = slices.DeleteFunc(got, func(col proto.InputColumn) bool { + return !slices.Contains([]string{"TimeReceived", "SrcAS", "DstAS", "Packets", "Bytes"}, col.Name) + }) + if diff := helpers.Diff(got, expected); diff != "" { + t.Fatalf("ClickHouseProtoInput() (-got, +want):\n%s", diff) + } +} diff --git a/common/schema/definition.go b/common/schema/definition.go index 7eed59af..afc419e6 100644 --- a/common/schema/definition.go +++ b/common/schema/definition.go @@ -12,8 +12,6 @@ import ( "akvorado/common/helpers/bimap" "github.com/bits-and-blooms/bitset" - "google.golang.org/protobuf/encoding/protowire" - "google.golang.org/protobuf/reflect/protoreflect" ) // InterfaceBoundary identifies wether the interface is facing inside or outside the network. @@ -133,9 +131,6 @@ const ( ColumnDst3rdAS ColumnDstCommunities ColumnDstLargeCommunities - ColumnDstLargeCommunitiesASN - ColumnDstLargeCommunitiesLocalData1 - ColumnDstLargeCommunitiesLocalData2 ColumnInIfName ColumnOutIfName ColumnInIfDescription @@ -212,7 +207,6 @@ func flows() Schema { ClickHouseType: "DateTime", ClickHouseCodec: "DoubleDelta, LZ4", ConsoleNotDimension: true, - ProtobufType: protoreflect.Uint64Kind, }, {Key: ColumnSamplingRate, NoDisable: true, ClickHouseType: "UInt64", ConsoleNotDimension: true}, {Key: ColumnExporterAddress, ParserType: "ip", ClickHouseType: "LowCardinality(IPv6)"}, @@ -385,25 +379,10 @@ END`, ClickHouseType: "Array(UInt32)", }, { - Key: ColumnDstLargeCommunities, - ClickHouseMainOnly: true, - ClickHouseType: "Array(UInt128)", - ClickHouseTransformFrom: []Column{ - { - Key: ColumnDstLargeCommunitiesASN, - ClickHouseType: "Array(UInt32)", - }, - { - Key: ColumnDstLargeCommunitiesLocalData1, - ClickHouseType: "Array(UInt32)", - }, - { - Key: ColumnDstLargeCommunitiesLocalData2, - ClickHouseType: "Array(UInt32)", - }, - }, - ClickHouseTransformTo: "arrayMap((asn, l1, l2) -> ((bitShiftLeft(CAST(asn, 'UInt128'), 64) + bitShiftLeft(CAST(l1, 'UInt128'), 32)) + CAST(l2, 'UInt128')), DstLargeCommunitiesASN, DstLargeCommunitiesLocalData1, DstLargeCommunitiesLocalData2)", - ConsoleNotDimension: true, + Key: ColumnDstLargeCommunities, + ClickHouseMainOnly: true, + ClickHouseType: "Array(UInt128)", + ConsoleNotDimension: true, }, {Key: ColumnInIfName, ParserType: "string", ClickHouseType: "LowCardinality(String)"}, {Key: ColumnInIfDescription, ParserType: "string", ClickHouseType: "LowCardinality(String)", ClickHouseNotSortingKey: true}, @@ -414,13 +393,6 @@ END`, Key: ColumnInIfBoundary, ClickHouseType: fmt.Sprintf("Enum8('undefined' = %d, 'external' = %d, 'internal' = %d)", InterfaceBoundaryUndefined, InterfaceBoundaryExternal, InterfaceBoundaryInternal), ClickHouseNotSortingKey: true, - ProtobufType: protoreflect.EnumKind, - ProtobufEnumName: "Boundary", - ProtobufEnum: map[int]string{ - int(InterfaceBoundaryUndefined): "UNDEFINED", - int(InterfaceBoundaryExternal): "EXTERNAL", - int(InterfaceBoundaryInternal): "INTERNAL", - }, }, {Key: ColumnEType, ClickHouseType: "UInt32"}, // TODO: UInt16 but hard to change, primary key {Key: ColumnProto, ClickHouseType: "UInt32"}, // TODO: UInt8 but hard to change, primary key @@ -574,9 +546,9 @@ END`, }.finalize() } -func (column *Column) shouldBeProto() bool { - return column.ClickHouseTransformFrom == nil && - (column.ClickHouseGenerateFrom == "" || column.ClickHouseSelfGenerated) && +// shouldProvideValue tells if we should send a value for this column to ClickHouse. +func (column *Column) shouldProvideValue() bool { + return (column.ClickHouseGenerateFrom == "" || column.ClickHouseSelfGenerated) && column.ClickHouseAlias == "" } @@ -592,30 +564,12 @@ func (schema Schema) finalize() Schema { column.Name = name } - // Also true name for columns in ClickHouseTransformFrom - for idx, ecolumn := range column.ClickHouseTransformFrom { - if ecolumn.Name == "" { - name, ok := columnNameMap.LoadValue(ecolumn.Key) - if !ok { - panic(fmt.Sprintf("missing name mapping for %d", ecolumn.Key)) - } - column.ClickHouseTransformFrom[idx].Name = name - } - } - // Non-main columns with an alias are NotSortingKey if !column.ClickHouseMainOnly && column.ClickHouseAlias != "" { column.ClickHouseNotSortingKey = true } - // Transform implicit dependencies - for idx := range column.ClickHouseTransformFrom { - deps := column.ClickHouseTransformFrom[idx].Depends - deps = append(deps, column.Key) - slices.Sort(deps) - column.ClickHouseTransformFrom[idx].Depends = slices.Compact(deps) - column.Depends = append(column.Depends, column.ClickHouseTransformFrom[idx].Key) - } + // Deduplicate dependencies slices.Sort(column.Depends) column.Depends = slices.Compact(column.Depends) @@ -639,7 +593,6 @@ func (schema Schema) finalize() Schema { panic(fmt.Sprintf("missing name mapping for %q", column.Name)) } column.ClickHouseAlias = strings.ReplaceAll(column.ClickHouseAlias, "Src", "Dst") - column.ClickHouseTransformFrom = slices.Clone(column.ClickHouseTransformFrom) ncolumns = append(ncolumns, column) } } else if strings.HasPrefix(column.Name, "InIf") { @@ -650,53 +603,12 @@ func (schema Schema) finalize() Schema { panic(fmt.Sprintf("missing name mapping for %q", column.Name)) } column.ClickHouseAlias = strings.ReplaceAll(column.ClickHouseAlias, "InIf", "OutIf") - column.ClickHouseTransformFrom = slices.Clone(column.ClickHouseTransformFrom) ncolumns = append(ncolumns, column) } } } schema.columns = ncolumns - // Set Protobuf index and type - protobufIndex := 1 - ncolumns = []Column{} - for _, column := range schema.columns { - pcolumns := []*Column{&column} - for idx := range column.ClickHouseTransformFrom { - pcolumns = append(pcolumns, &column.ClickHouseTransformFrom[idx]) - } - for _, column := range pcolumns { - if column.ProtobufIndex == 0 { - if !column.shouldBeProto() { - column.ProtobufIndex = -1 - continue - } - - column.ProtobufIndex = protowire.Number(protobufIndex) - protobufIndex++ - } - - if column.ProtobufType == 0 && - column.shouldBeProto() { - switch column.ClickHouseType { - case "String", "LowCardinality(String)", "FixedString(2)": - column.ProtobufType = protoreflect.StringKind - case "UInt64": - column.ProtobufType = protoreflect.Uint64Kind - case "UInt32", "UInt16", "UInt8": - column.ProtobufType = protoreflect.Uint32Kind - case "IPv6", "LowCardinality(IPv6)": - column.ProtobufType = protoreflect.BytesKind - case "Array(UInt32)": - column.ProtobufType = protoreflect.Uint32Kind - column.ProtobufRepeated = true - } - } - } - ncolumns = append(ncolumns, column) - } - schema.columns = ncolumns - // Build column index maxKey := ColumnTimeReceived for _, column := range schema.columns { @@ -707,9 +619,6 @@ func (schema Schema) finalize() Schema { schema.columnIndex = make([]*Column, maxKey+1) for i, column := range schema.columns { schema.columnIndex[column.Key] = &schema.columns[i] - for j, column := range column.ClickHouseTransformFrom { - schema.columnIndex[column.Key] = &schema.columns[i].ClickHouseTransformFrom[j] - } } // Update disabledGroups diff --git a/common/schema/definition_test.go b/common/schema/definition_test.go index ee45c42d..02eb8b45 100644 --- a/common/schema/definition_test.go +++ b/common/schema/definition_test.go @@ -22,17 +22,6 @@ func TestFlowsClickHouse(t *testing.T) { } } -func TestFlowsProtobuf(t *testing.T) { - c := NewMock(t) - for _, column := range c.Columns() { - if column.ProtobufIndex >= 0 { - if column.ProtobufType == 0 { - t.Errorf("column %s has not protobuf type", column.Name) - } - } - } -} - func TestColumnIndex(t *testing.T) { c := NewMock(t) for i := ColumnTimeReceived; i < ColumnLast; i++ { diff --git a/common/schema/insert_test.go b/common/schema/insert_test.go new file mode 100644 index 00000000..583f1a37 --- /dev/null +++ b/common/schema/insert_test.go @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: 2025 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package schema_test + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/netip" + "testing" + "time" + + "github.com/ClickHouse/ch-go" + "github.com/ClickHouse/clickhouse-go/v2" + + "akvorado/common/helpers" + "akvorado/common/schema" +) + +func TestInsertMemory(t *testing.T) { + c := schema.NewMock(t) + bf := c.NewFlowMessage() + exporterAddress := netip.MustParseAddr("::ffff:203.0.113.14") + + bf.TimeReceived = 1000 + bf.SamplingRate = 20000 + bf.ExporterAddress = exporterAddress + bf.AppendString(schema.ColumnExporterName, "router1.example.net") + bf.AppendUint(schema.ColumnSrcAS, 65000) + bf.AppendUint(schema.ColumnDstAS, 12322) + bf.AppendUint(schema.ColumnBytes, 20) + bf.AppendUint(schema.ColumnPackets, 3) + bf.AppendUint(schema.ColumnInIfBoundary, uint64(schema.InterfaceBoundaryInternal)) + bf.AppendUint(schema.ColumnOutIfBoundary, uint64(schema.InterfaceBoundaryExternal)) + bf.AppendUint(schema.ColumnInIfSpeed, 10000) + bf.AppendUint(schema.ColumnEType, helpers.ETypeIPv4) + bf.Finalize() + + bf.TimeReceived = 1001 + bf.SamplingRate = 20000 + bf.ExporterAddress = exporterAddress + bf.AppendString(schema.ColumnExporterName, "router1.example.net") + bf.AppendUint(schema.ColumnSrcAS, 12322) + bf.AppendUint(schema.ColumnDstAS, 65000) + bf.AppendUint(schema.ColumnBytes, 200) + bf.AppendUint(schema.ColumnPackets, 3) + bf.AppendUint(schema.ColumnInIfBoundary, uint64(schema.InterfaceBoundaryExternal)) + bf.AppendUint(schema.ColumnOutIfSpeed, 10000) + bf.AppendUint(schema.ColumnEType, helpers.ETypeIPv4) + bf.AppendArrayUInt32(schema.ColumnDstASPath, []uint32{65400, 65500, 65001}) + bf.AppendArrayUInt128(schema.ColumnDstLargeCommunities, []schema.UInt128{ + { + High: 65401, + Low: (100 << 32) + 200, + }, + { + High: 65401, + Low: (100 << 32) + 201, + }, + }) + bf.Finalize() + + server := helpers.CheckExternalService(t, "ClickHouse", []string{"clickhouse:9000", "127.0.0.1:9000"}) + ctx := t.Context() + + conn, err := ch.Dial(ctx, ch.Options{ + Address: server, + DialTimeout: 100 * time.Millisecond, + Settings: []ch.Setting{ + {Key: "allow_suspicious_low_cardinality_types", Value: "1"}, + }, + }) + if err != nil { + t.Fatalf("Dial() error:\n%+v", err) + } + + // Create the table + q := fmt.Sprintf( + `CREATE OR REPLACE TABLE test_table_insert (%s) ENGINE = Memory`, + c.ClickHouseCreateTable(schema.ClickHouseSkipAliasedColumns, schema.ClickHouseSkipGeneratedColumns), + ) + t.Logf("Query: %s", q) + if err := conn.Do(ctx, ch.Query{ + Body: q, + }); err != nil { + t.Fatalf("Do() error:\n%+v", err) + } + + // Insert + input := bf.ClickHouseProtoInput() + if err := conn.Do(ctx, ch.Query{ + Body: input.Into("test_table_insert"), + Input: input, + OnInput: func(ctx context.Context) error { + bf.Clear() + // No more data to send! + return io.EOF + }, + }); err != nil { + t.Fatalf("Do() error:\n%+v", err) + } + + // Check the result (with the full-featured client) + { + conn, err := clickhouse.Open(&clickhouse.Options{ + Addr: []string{server}, + DialTimeout: 100 * time.Millisecond, + }) + if err != nil { + t.Fatalf("clickhouse.Open() error:\n%+v", err) + } + // Use formatRow to get JSON representation + rows, err := conn.Query(ctx, "SELECT formatRow('JSONEachRow', *) FROM test_table_insert ORDER BY TimeReceived") + if err != nil { + t.Fatalf("clickhouse.Query() error:\n%+v", err) + } + + var got []map[string]any + for rows.Next() { + var jsonRow string + if err := rows.Scan(&jsonRow); err != nil { + t.Fatalf("rows.Scan() error:\n%+v", err) + } + + var row map[string]any + if err := json.Unmarshal([]byte(jsonRow), &row); err != nil { + t.Fatalf("json.Unmarshal() error:\n%+v", err) + } + + // Remove fields with default values + for k, v := range row { + switch val := v.(type) { + case string: + if val == "" || val == "::" { + delete(row, k) + } + case float64: + if val == 0 { + delete(row, k) + } + case []any: + if len(val) == 0 { + delete(row, k) + } + } + } + got = append(got, row) + } + rows.Close() + + expected := []map[string]any{ + { + "TimeReceived": "1970-01-01 00:16:40", + "SamplingRate": "20000", + "ExporterAddress": "::ffff:203.0.113.14", + "ExporterName": "router1.example.net", + "SrcAS": 65000, + "DstAS": 12322, + "Bytes": "20", + "Packets": "3", + "InIfBoundary": "internal", + "OutIfBoundary": "external", + "InIfSpeed": 10000, + "EType": helpers.ETypeIPv4, + }, { + "TimeReceived": "1970-01-01 00:16:41", + "SamplingRate": "20000", + "ExporterAddress": "::ffff:203.0.113.14", + "ExporterName": "router1.example.net", + "SrcAS": 12322, + "DstAS": 65000, + "Bytes": "200", + "Packets": "3", + "InIfBoundary": "external", + "OutIfBoundary": "undefined", + "OutIfSpeed": 10000, + "EType": helpers.ETypeIPv4, + "DstASPath": []uint32{65400, 65500, 65001}, + "DstLargeCommunities": []string{ + "1206435509165107881967816", // 65401:100:200 + "1206435509165107881967817", // 65401:100:201 + }, + }, + } + + if diff := helpers.Diff(got, expected); diff != "" { + t.Errorf("Insert (-got, +want):\n%s", diff) + } + } + +} diff --git a/common/schema/message.go b/common/schema/message.go new file mode 100644 index 00000000..2a9fb814 --- /dev/null +++ b/common/schema/message.go @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: 2025 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package schema + +import ( + "net/netip" + + "github.com/ClickHouse/ch-go/proto" + "github.com/bits-and-blooms/bitset" +) + +// FlowMessage is the abstract representation of a flow through various subsystems. +type FlowMessage struct { + TimeReceived uint32 + SamplingRate uint64 + + // For exporter classifier + ExporterAddress netip.Addr + + // For interface classifier + InIf uint32 + OutIf uint32 + SrcVlan uint16 + DstVlan uint16 + + // For routing component + SrcAddr netip.Addr + DstAddr netip.Addr + NextHop netip.Addr + + // Core component may override them + SrcAS uint32 + DstAS uint32 + SrcNetMask uint8 + DstNetMask uint8 + + // Only for tests + OtherColumns map[ColumnKey]any + + batch clickhouseBatch + schema *Schema +} + +// clickhouseBatch stores columns for efficient streaming. It is embedded +// inside a FlowMessage. +type clickhouseBatch struct { + columns []proto.Column // Indexed by ColumnKey + columnSet bitset.BitSet // Track which columns have been set + rowCount int // Number of rows accumulated + input proto.Input // Input including all columns to stream to ClickHouse +} + +// reset resets a flow message. All public fields are set to 0, +// but the current ClickHouse batch is left untouched. +func (bf *FlowMessage) reset() { + *bf = FlowMessage{ + batch: bf.batch, + schema: bf.schema, + } + bf.batch.columnSet.ClearAll() +} + +// Clear clears all column data. +func (bf *FlowMessage) Clear() { + bf.reset() + bf.batch.input.Reset() + bf.batch.rowCount = 0 +} + +// ClickHouseProtoInput returns the proto.Input that can be used to stream results +// to ClickHouse. +func (bf *FlowMessage) ClickHouseProtoInput() proto.Input { + return bf.batch.input +} + +// NewFlowMessage creates a new FlowMessage for the given schema with ClickHouse batch initialized. +func (schema *Schema) NewFlowMessage() *FlowMessage { + bf := &FlowMessage{ + schema: schema, + } + + maxKey := ColumnKey(0) + for _, column := range bf.schema.columns { + if column.Key > maxKey { + maxKey = column.Key + } + } + + bf.batch.columns = make([]proto.Column, maxKey+1) + bf.batch.columnSet = *bitset.New(uint(maxKey + 1)) + bf.batch.rowCount = 0 + + for _, column := range bf.schema.columns { + if !column.Disabled && column.shouldProvideValue() { + bf.batch.columns[column.Key] = column.newProtoColumn() + bf.batch.input = append(bf.batch.input, proto.InputColumn{ + Name: column.Name, + Data: column.wrapProtoColumn(bf.batch.columns[column.Key]), + }) + } + } + + return bf +} + +// FlowCount return the number of flows batched +func (bf *FlowMessage) FlowCount() int { + return bf.batch.rowCount +} diff --git a/common/schema/protobuf.go b/common/schema/protobuf.go deleted file mode 100644 index 9e2e9613..00000000 --- a/common/schema/protobuf.go +++ /dev/null @@ -1,262 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -package schema - -import ( - "encoding/base32" - "fmt" - "hash/fnv" - "net/netip" - "slices" - "strings" - - "github.com/bits-and-blooms/bitset" - "google.golang.org/protobuf/encoding/protowire" - "google.golang.org/protobuf/reflect/protoreflect" -) - -// ProtobufMessageHash returns the name of the protobuf definition. -func (schema Schema) ProtobufMessageHash() string { - name, _ := schema.protobufMessageHashAndDefinition() - return name -} - -// ProtobufDefinition returns the protobuf definition. -func (schema Schema) ProtobufDefinition() string { - _, definition := schema.protobufMessageHashAndDefinition() - return definition -} - -// protobufMessageHashAndDefinition returns the name of the protobuf definition -// along with the protobuf definition itself (.proto file). -func (schema Schema) protobufMessageHashAndDefinition() (string, string) { - lines := []string{} - enums := map[string]string{} - - hash := fnv.New128() - for _, column := range schema.Columns() { - for _, column := range append([]Column{column}, column.ClickHouseTransformFrom...) { - if column.ProtobufIndex < 0 { - continue - } - - t := column.ProtobufType.String() - - // Enum definition - if column.ProtobufType == protoreflect.EnumKind { - if _, ok := enums[column.ProtobufEnumName]; !ok { - definition := []string{} - keys := []int{} - for key := range column.ProtobufEnum { - keys = append(keys, key) - } - slices.Sort(keys) - for _, key := range keys { - definition = append(definition, fmt.Sprintf("%s = %d;", column.ProtobufEnum[key], key)) - } - enums[column.ProtobufEnumName] = fmt.Sprintf("enum %s { %s }", - column.ProtobufEnumName, - strings.Join(definition, " ")) - } - t = column.ProtobufEnumName - } - - // Column definition - if column.ProtobufRepeated { - t = fmt.Sprintf("repeated %s", t) - } - line := fmt.Sprintf("%s %s = %d;", - t, - column.Name, - column.ProtobufIndex, - ) - lines = append(lines, line) - hash.Write([]byte(line)) - } - } - - enumDefinitions := []string{} - for _, v := range enums { - enumDefinitions = append(enumDefinitions, v) - hash.Write([]byte(v)) - } - hashString := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash.Sum(nil)) - - return hashString, fmt.Sprintf(` -syntax = "proto3"; - -message FlowMessagev%s { - %s - - %s -} -`, hashString, strings.Join(enumDefinitions, "\n "), strings.Join(lines, "\n ")) -} - -// ProtobufMarshal transforms a basic flow into protobuf bytes. The provided flow should -// not be reused afterwards. -func (schema *Schema) ProtobufMarshal(bf *FlowMessage) []byte { - schema.ProtobufAppendVarint(bf, ColumnTimeReceived, bf.TimeReceived) - schema.ProtobufAppendVarint(bf, ColumnSamplingRate, uint64(bf.SamplingRate)) - schema.ProtobufAppendIP(bf, ColumnExporterAddress, bf.ExporterAddress) - schema.ProtobufAppendVarint(bf, ColumnSrcAS, uint64(bf.SrcAS)) - schema.ProtobufAppendVarint(bf, ColumnDstAS, uint64(bf.DstAS)) - schema.ProtobufAppendVarint(bf, ColumnSrcNetMask, uint64(bf.SrcNetMask)) - schema.ProtobufAppendVarint(bf, ColumnDstNetMask, uint64(bf.DstNetMask)) - schema.ProtobufAppendIP(bf, ColumnSrcAddr, bf.SrcAddr) - schema.ProtobufAppendIP(bf, ColumnDstAddr, bf.DstAddr) - schema.ProtobufAppendIP(bf, ColumnNextHop, bf.NextHop) - if !schema.IsDisabled(ColumnGroupL2) { - schema.ProtobufAppendVarint(bf, ColumnSrcVlan, uint64(bf.SrcVlan)) - schema.ProtobufAppendVarint(bf, ColumnDstVlan, uint64(bf.DstVlan)) - } - - // Add length and move it as a prefix - end := len(bf.protobuf) - payloadLen := end - maxSizeVarint - bf.protobuf = protowire.AppendVarint(bf.protobuf, uint64(payloadLen)) - sizeLen := len(bf.protobuf) - end - result := bf.protobuf[maxSizeVarint-sizeLen : end] - copy(result, bf.protobuf[end:end+sizeLen]) - bf.protobuf = result - - return result -} - -// ProtobufAppendVarint append a varint to the protobuf representation of a flow. -func (schema *Schema) ProtobufAppendVarint(bf *FlowMessage, columnKey ColumnKey, value uint64) { - // Check if value is 0 to avoid a lookup. - if value > 0 { - schema.ProtobufAppendVarintForce(bf, columnKey, value) - } -} - -// ProtobufAppendVarintForce append a varint to the protobuf representation of a flow, even if it is a 0-value. -func (schema *Schema) ProtobufAppendVarintForce(bf *FlowMessage, columnKey ColumnKey, value uint64) { - column, _ := schema.LookupColumnByKey(columnKey) - column.ProtobufAppendVarintForce(bf, value) -} - -// ProtobufAppendVarint append a varint to the protobuf representation of a flow. -func (column *Column) ProtobufAppendVarint(bf *FlowMessage, value uint64) { - if value > 0 { - column.ProtobufAppendVarintForce(bf, value) - } -} - -// ProtobufAppendVarintForce append a varint to the protobuf representation of a flow, even when 0. -func (column *Column) ProtobufAppendVarintForce(bf *FlowMessage, value uint64) { - bf.init() - if column.protobufCanAppend(bf) { - bf.protobuf = protowire.AppendTag(bf.protobuf, column.ProtobufIndex, protowire.VarintType) - bf.protobuf = protowire.AppendVarint(bf.protobuf, value) - bf.protobufSet.Set(uint(column.ProtobufIndex)) - if debug { - column.appendDebug(bf, value) - } - } -} - -func (column Column) protobufCanAppend(bf *FlowMessage) bool { - return column.ProtobufIndex > 0 && - !column.Disabled && - (column.ProtobufRepeated || !bf.protobufSet.Test(uint(column.ProtobufIndex))) -} - -// ProtobufAppendBytes append a slice of bytes to the protobuf representation -// of a flow. -func (schema *Schema) ProtobufAppendBytes(bf *FlowMessage, columnKey ColumnKey, value []byte) { - if len(value) > 0 { - schema.ProtobufAppendBytesForce(bf, columnKey, value) - } -} - -// ProtobufAppendBytesForce append a slice of bytes to the protobuf representation -// of a flow, even when empty -func (schema *Schema) ProtobufAppendBytesForce(bf *FlowMessage, columnKey ColumnKey, value []byte) { - column, _ := schema.LookupColumnByKey(columnKey) - column.ProtobufAppendBytesForce(bf, value) -} - -// ProtobufAppendBytes append a slice of bytes to the protobuf representation -// of a flow. -func (column *Column) ProtobufAppendBytes(bf *FlowMessage, value []byte) { - if len(value) > 0 { - column.ProtobufAppendBytesForce(bf, value) - } -} - -// ProtobufAppendBytesForce append a slice of bytes to the protobuf representation -// of a flow, even when empty -func (column *Column) ProtobufAppendBytesForce(bf *FlowMessage, value []byte) { - bf.init() - if column.protobufCanAppend(bf) { - bf.protobuf = protowire.AppendTag(bf.protobuf, column.ProtobufIndex, protowire.BytesType) - bf.protobuf = protowire.AppendBytes(bf.protobuf, value) - bf.protobufSet.Set(uint(column.ProtobufIndex)) - if debug { - column.appendDebug(bf, value) - } - } -} - -// ProtobufAppendIP append an IP to the protobuf representation -// of a flow. -func (schema *Schema) ProtobufAppendIP(bf *FlowMessage, columnKey ColumnKey, value netip.Addr) { - if value.IsValid() { - column, _ := schema.LookupColumnByKey(columnKey) - column.ProtobufAppendIPForce(bf, value) - } -} - -// ProtobufAppendIP append an IP to the protobuf representation -// of a flow. -func (column *Column) ProtobufAppendIP(bf *FlowMessage, value netip.Addr) { - if value.IsValid() { - column.ProtobufAppendIPForce(bf, value) - } -} - -// ProtobufAppendIPForce append an IP to the protobuf representation -// of a flow, even when not valid -func (column *Column) ProtobufAppendIPForce(bf *FlowMessage, value netip.Addr) { - bf.init() - if column.protobufCanAppend(bf) { - v := value.As16() - bf.protobuf = protowire.AppendTag(bf.protobuf, column.ProtobufIndex, protowire.BytesType) - bf.protobuf = protowire.AppendBytes(bf.protobuf, v[:]) - bf.protobufSet.Set(uint(column.ProtobufIndex)) - if debug { - column.appendDebug(bf, value) - } - } -} - -func (column *Column) appendDebug(bf *FlowMessage, value interface{}) { - if bf.ProtobufDebug == nil { - bf.ProtobufDebug = make(map[ColumnKey]interface{}) - } - if column.ProtobufRepeated { - if current, ok := bf.ProtobufDebug[column.Key]; ok { - bf.ProtobufDebug[column.Key] = append(current.([]interface{}), value) - } else { - bf.ProtobufDebug[column.Key] = []interface{}{value} - } - } else { - bf.ProtobufDebug[column.Key] = value - } -} - -// Bytes returns protobuf bytes. The flow should have been processed by -// `ProtobufMarshal` first. -func (bf *FlowMessage) Bytes() []byte { - return bf.protobuf -} - -func (bf *FlowMessage) init() { - if bf.protobuf == nil { - bf.protobuf = make([]byte, maxSizeVarint, 500) - bf.protobufSet = *bitset.New(uint(ColumnLast)) - } -} diff --git a/common/schema/protobuf_test.go b/common/schema/protobuf_test.go deleted file mode 100644 index a68a6dc5..00000000 --- a/common/schema/protobuf_test.go +++ /dev/null @@ -1,242 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -package schema - -import ( - "fmt" - "net/netip" - "strings" - "testing" - - "akvorado/common/helpers" - - "google.golang.org/protobuf/encoding/protowire" - "google.golang.org/protobuf/reflect/protoreflect" -) - -func TestProtobufDefinition(t *testing.T) { - // Use a smaller version - flows := Schema{ - columns: []Column{ - { - Key: ColumnTimeReceived, - ClickHouseType: "DateTime", - ProtobufType: protoreflect.Uint64Kind, - }, - {Key: ColumnSamplingRate, ClickHouseType: "UInt64"}, - {Key: ColumnExporterAddress, ClickHouseType: "LowCardinality(IPv6)"}, - {Key: ColumnExporterName, ClickHouseType: "LowCardinality(String)"}, - { - Key: ColumnSrcAddr, - ClickHouseType: "IPv6", - }, - { - Key: ColumnSrcNetMask, - ClickHouseType: "UInt8", - }, - { - Key: ColumnSrcNetPrefix, - ClickHouseType: "String", - ClickHouseAlias: `something`, - }, - {Key: ColumnSrcAS, ClickHouseType: "UInt32"}, - { - Key: ColumnSrcNetName, - ClickHouseType: "LowCardinality(String)", - ClickHouseGenerateFrom: fmt.Sprintf("dictGetOrDefault('%s', 'name', SrcAddr, '')", DictionaryNetworks), - }, - { - Key: ColumnDstASPath, - ClickHouseType: "Array(UInt32)", - }, - { - Key: ColumnDstLargeCommunities, - ClickHouseType: "Array(UInt128)", - ClickHouseTransformFrom: []Column{ - {Key: ColumnDstLargeCommunitiesASN, ClickHouseType: "Array(UInt32)"}, - {Key: ColumnDstLargeCommunitiesLocalData1, ClickHouseType: "Array(UInt32)"}, - {Key: ColumnDstLargeCommunitiesLocalData2, ClickHouseType: "Array(UInt32)"}, - }, - ClickHouseTransformTo: "something", - }, - {Key: ColumnInIfName, ClickHouseType: "LowCardinality(String)"}, - { - Key: ColumnInIfBoundary, - ClickHouseType: "Enum8('undefined' = 0, 'external' = 1, 'internal' = 2)", - ClickHouseNotSortingKey: true, - ProtobufType: protoreflect.EnumKind, - ProtobufEnumName: "Boundary", - ProtobufEnum: map[int]string{ - 0: "UNDEFINED", - 1: "EXTERNAL", - 2: "INTERNAL", - }, - }, - {Key: ColumnBytes, ClickHouseType: "UInt64"}, - }, - }.finalize() - - got := flows.ProtobufDefinition() - expected := ` -syntax = "proto3"; - -message FlowMessagev5WRSGBXQDXZSUHZQE6QEHLI5JM { - enum Boundary { UNDEFINED = 0; EXTERNAL = 1; INTERNAL = 2; } - - uint64 TimeReceived = 1; - uint64 SamplingRate = 2; - bytes ExporterAddress = 3; - string ExporterName = 4; - bytes SrcAddr = 5; - bytes DstAddr = 6; - uint32 SrcNetMask = 7; - uint32 DstNetMask = 8; - uint32 SrcAS = 9; - uint32 DstAS = 10; - repeated uint32 DstASPath = 11; - repeated uint32 DstLargeCommunitiesASN = 12; - repeated uint32 DstLargeCommunitiesLocalData1 = 13; - repeated uint32 DstLargeCommunitiesLocalData2 = 14; - string InIfName = 15; - string OutIfName = 16; - Boundary InIfBoundary = 17; - Boundary OutIfBoundary = 18; - uint64 Bytes = 19; -} -` - if diff := helpers.Diff(strings.Split(got, "\n"), strings.Split(expected, "\n")); diff != "" { - t.Fatalf("ProtobufDefinition() (-got, +want): %s", diff) - } -} - -func TestProtobufMarshal(t *testing.T) { - c := NewMock(t) - exporterAddress := netip.MustParseAddr("::ffff:203.0.113.14") - bf := &FlowMessage{} - bf.TimeReceived = 1000 - bf.SamplingRate = 20000 - bf.ExporterAddress = exporterAddress - c.ProtobufAppendVarint(bf, ColumnDstAS, 65000) - c.ProtobufAppendVarint(bf, ColumnBytes, 200) - c.ProtobufAppendVarint(bf, ColumnPackets, 300) - c.ProtobufAppendVarint(bf, ColumnBytes, 300) // duplicate! - - got := c.ProtobufMarshal(bf) - - size, n := protowire.ConsumeVarint(got) - if uint64(len(got)-n) != size { - t.Fatalf("ProtobufMarshal() produced an incorrect size: %d + %d != %d", size, n, len(got)) - } - - // text schema definition for reference - // syntax = "proto3"; - - // message FlowMessagevLAABIGYMRYZPTGOYIIFZNYDEQM { - // enum Boundary { UNDEFINED = 0; EXTERNAL = 1; INTERNAL = 2; } - - // uint64 TimeReceived = 1; - // uint64 SamplingRate = 2; - // bytes ExporterAddress = 3; - // string ExporterName = 4; - // string ExporterGroup = 5; - // string ExporterRole = 6; - // string ExporterSite = 7; - // string ExporterRegion = 8; - // string ExporterTenant = 9; - // bytes SrcAddr = 10; - // bytes DstAddr = 11; - // uint32 SrcNetMask = 12; - // uint32 DstNetMask = 13; - // uint32 SrcAS = 14; - // uint32 DstAS = 15; - // repeated uint32 DstASPath = 18; - // repeated uint32 DstCommunities = 19; - // repeated uint32 DstLargeCommunitiesASN = 20; - // repeated uint32 DstLargeCommunitiesLocalData1 = 21; - // repeated uint32 DstLargeCommunitiesLocalData2 = 22; - // string InIfName = 23; - // string OutIfName = 24; - // string InIfDescription = 25; - // string OutIfDescription = 26; - // uint32 InIfSpeed = 27; - // uint32 OutIfSpeed = 28; - // string InIfConnectivity = 29; - // string OutIfConnectivity = 30; - // string InIfProvider = 31; - // string OutIfProvider = 32; - // Boundary InIfBoundary = 33; - // Boundary OutIfBoundary = 34; - // uint32 EType = 35; - // uint32 Proto = 36; - // uint32 SrcPort = 37; - // uint32 DstPort = 38; - // uint64 Bytes = 39; - // uint64 Packets = 40; - // uint32 ForwardingStatus = 41; - // } - // to check: https://protobuf-decoder.netlify.app/ - t.Run("compare as bytes", func(t *testing.T) { - expected := []byte{ - // DstAS - // 15: 65000 - 0x78, 0xe8, 0xfb, 0x03, - // Bytes - // 39: 200 - 0xb8, 0x02, 0xc8, 0x01, - // Packet - // 40: 300 - 0xc0, 0x02, 0xac, 0x02, - // TimeReceived - // 1: 1000 - 0x08, 0xe8, 0x07, - // SamplingRate - // 2: 20000 - 0x10, 0xa0, 0x9c, 0x01, - // ExporterAddress - // 3: ::ffff:203.0.113.14 - 0x1a, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xff, 0xcb, 0x0, 0x71, 0xe, - } - if diff := helpers.Diff(got[n:], expected); diff != "" { - t.Logf("got: %v", got) - - t.Fatalf("ProtobufMarshal() (-got, +want):\n%s", diff) - } - }) - - t.Run("compare as protobuf message", func(t *testing.T) { - got := c.ProtobufDecode(t, got) - expected := FlowMessage{ - TimeReceived: 1000, - SamplingRate: 20000, - ExporterAddress: exporterAddress, - DstAS: 65000, - ProtobufDebug: map[ColumnKey]interface{}{ - ColumnBytes: 200, - ColumnPackets: 300, - }, - } - if diff := helpers.Diff(got, expected); diff != "" { - t.Fatalf("ProtobufDecode() (-got, +want):\n%s", diff) - } - }) -} - -func BenchmarkProtobufMarshal(b *testing.B) { - c := NewMock(b) - exporterAddress := netip.MustParseAddr("::ffff:203.0.113.14") - DisableDebug(b) - for b.Loop() { - bf := &FlowMessage{ - TimeReceived: 1000, - SamplingRate: 20000, - ExporterAddress: exporterAddress, - } - c.ProtobufAppendVarint(bf, ColumnDstAS, 65000) - c.ProtobufAppendVarint(bf, ColumnBytes, 200) - c.ProtobufAppendVarint(bf, ColumnPackets, 300) - c.ProtobufAppendVarint(bf, ColumnBytes, 300) // duplicate! - c.ProtobufAppendVarint(bf, ColumnSrcVlan, 1600) // disabled! - c.ProtobufMarshal(bf) - } -} diff --git a/common/schema/root_test.go b/common/schema/root_test.go index 393f9aef..fb7b02cb 100644 --- a/common/schema/root_test.go +++ b/common/schema/root_test.go @@ -38,29 +38,6 @@ func TestDisableForbiddenColumns(t *testing.T) { if _, err := schema.New(config); err == nil { t.Fatal("New() did not error") } - - config = schema.DefaultConfiguration() - config.Disabled = []schema.ColumnKey{schema.ColumnDstLargeCommunitiesASN} - if _, err := schema.New(config); err == nil { - t.Fatal("New() did not error") - } - - config = schema.DefaultConfiguration() - config.Disabled = []schema.ColumnKey{schema.ColumnDstLargeCommunities} - if _, err := schema.New(config); err == nil { - t.Fatal("New() did not error") - } - - config = schema.DefaultConfiguration() - config.Disabled = []schema.ColumnKey{ - schema.ColumnDstLargeCommunities, - schema.ColumnDstLargeCommunitiesASN, - schema.ColumnDstLargeCommunitiesLocalData1, - schema.ColumnDstLargeCommunitiesLocalData2, - } - if _, err := schema.New(config); err != nil { - t.Fatalf("New() error:\n%+v", err) - } } func TestCustomDictionaries(t *testing.T) { diff --git a/common/schema/tests.go b/common/schema/tests.go index 3e4e2aef..1520910c 100644 --- a/common/schema/tests.go +++ b/common/schema/tests.go @@ -7,17 +7,10 @@ package schema import ( "fmt" - "net/netip" "reflect" - "strings" "testing" "akvorado/common/helpers" - - "github.com/jhump/protoreflect/desc" - "github.com/jhump/protoreflect/desc/protoparse" - "github.com/jhump/protoreflect/dynamic" - "google.golang.org/protobuf/encoding/protowire" ) var debug = true @@ -41,82 +34,6 @@ func NewMock(t testing.TB) *Component { return c } -// ProtobufDecode decodes the provided protobuf message. -func (schema *Schema) ProtobufDecode(t *testing.T, input []byte) *FlowMessage { - t.Helper() - parser := protoparse.Parser{ - Accessor: protoparse.FileContentsFromMap(map[string]string{ - "flow.proto": schema.ProtobufDefinition(), - }), - } - descs, err := parser.ParseFiles("flow.proto") - if err != nil { - t.Fatalf("ParseFiles(%q) error:\n%+v", "flow.proto", err) - } - - var descriptor *desc.MessageDescriptor - for _, msg := range descs[0].GetMessageTypes() { - if strings.HasPrefix(msg.GetName(), "FlowMessagev") { - descriptor = msg - break - } - } - if descriptor == nil { - t.Fatal("cannot find message descriptor") - } - - message := dynamic.NewMessage(descriptor) - size, n := protowire.ConsumeVarint(input) - if len(input)-n != int(size) { - t.Fatalf("bad length for protobuf message: %d - %d != %d", len(input), n, size) - } - if err := message.Unmarshal(input[n:]); err != nil { - t.Fatalf("Unmarshal() error:\n%+v", err) - } - textVersion, _ := message.MarshalTextIndent() - t.Logf("Unmarshal():\n%s", textVersion) - - flow := FlowMessage{ - ProtobufDebug: map[ColumnKey]interface{}{}, - } - for _, field := range message.GetKnownFields() { - k := int(field.GetNumber()) - name := field.GetName() - switch name { - case "TimeReceived": - flow.TimeReceived = message.GetFieldByNumber(k).(uint64) - case "SamplingRate": - flow.SamplingRate = uint32(message.GetFieldByNumber(k).(uint64)) - case "ExporterAddress": - ip, _ := netip.AddrFromSlice(message.GetFieldByNumber(k).([]byte)) - flow.ExporterAddress = ip - case "SrcAddr": - ip, _ := netip.AddrFromSlice(message.GetFieldByNumber(k).([]byte)) - flow.SrcAddr = ip - case "DstAddr": - ip, _ := netip.AddrFromSlice(message.GetFieldByNumber(k).([]byte)) - flow.DstAddr = ip - case "SrcAS": - flow.SrcAS = uint32(message.GetFieldByNumber(k).(uint32)) - case "DstAS": - flow.DstAS = uint32(message.GetFieldByNumber(k).(uint32)) - default: - column, ok := schema.LookupColumnByName(name) - if !ok { - break - } - key := column.Key - value := message.GetFieldByNumber(k) - if reflect.ValueOf(value).IsZero() { - break - } - flow.ProtobufDebug[key] = value - } - } - - return &flow -} - // EnableAllColumns enable all columns and returns itself. func (schema *Component) EnableAllColumns() *Component { for i := range schema.columns { diff --git a/common/schema/types.go b/common/schema/types.go index 9dba1416..33633d4b 100644 --- a/common/schema/types.go +++ b/common/schema/types.go @@ -4,11 +4,8 @@ package schema import ( - "net/netip" - + "github.com/ClickHouse/ch-go/proto" "github.com/bits-and-blooms/bitset" - "google.golang.org/protobuf/encoding/protowire" - "google.golang.org/protobuf/reflect/protoreflect" ) // Schema is the data schema. @@ -42,18 +39,18 @@ type Column struct { // instead of being retrieved from the protobuf. `TransformFrom' and // `TransformTo' work in pairs. The first one is the set of column in the // raw table while the second one is how to transform it for the main table. - ClickHouseType string - ClickHouseMaterializedType string - ClickHouseCodec string - ClickHouseAlias string - ClickHouseNotSortingKey bool - ClickHouseGenerateFrom string - ClickHouseTransformFrom []Column - ClickHouseTransformTo string - ClickHouseMainOnly bool - // ClickHouseSelfGenerated identifies a column as being formatted using itself as source - ClickHouseSelfGenerated bool - + ClickHouseType string // ClickHouse type for the column + ClickHouseMaterializedType string // ClickHouse type when we request materialization + ClickHouseCodec string // Compression codec + ClickHouseAlias string // Alias expression + // ClickHouseNotSortingKey is to be used for columns whose content is + // derived from another column. Like Exporter* all derive from + // ExporterAddress. + ClickHouseNotSortingKey bool + // ClickHouseGenerateFrom computes the content of the column using another column + ClickHouseGenerateFrom string + ClickHouseMainOnly bool // Only include this column in the main table + ClickHouseSelfGenerated bool // Generated (partly) from its own value // ClickHouseMaterialized indicates that the column was materialized (and is not by default) ClickHouseMaterialized bool @@ -61,55 +58,13 @@ type Column struct { // truncatable when used as a dimension. ConsoleNotDimension bool ConsoleTruncateIP bool - - // For protobuf. The index is automatically derived from the position, - // unless specified. Use -1 to not include the column into the protobuf - // schema. - ProtobufIndex protowire.Number - ProtobufType protoreflect.Kind // Uint64Kind, Uint32Kind, BytesKind, StringKind, EnumKind - ProtobufEnum map[int]string - ProtobufEnumName string - ProtobufRepeated bool } // ColumnKey is the name of a column -type ColumnKey int +type ColumnKey uint // ColumnGroup represents a group of columns type ColumnGroup uint -// FlowMessage is the abstract representation of a flow through various subsystems. -type FlowMessage struct { - TimeReceived uint64 - SamplingRate uint32 - - // For exporter classifier - ExporterAddress netip.Addr - - // For interface classifier - InIf uint32 - OutIf uint32 - SrcVlan uint16 - DstVlan uint16 - - // For geolocation or BMP - SrcAddr netip.Addr - DstAddr netip.Addr - NextHop netip.Addr - - // Core component may override them - SrcAS uint32 - DstAS uint32 - GotASPath bool - GotCommunities bool - - SrcNetMask uint8 - DstNetMask uint8 - - // protobuf is the protobuf representation for the information not contained above. - protobuf []byte - protobufSet bitset.BitSet - ProtobufDebug map[ColumnKey]interface{} `json:"-"` // for testing purpose -} - -const maxSizeVarint = 10 // protowire.SizeVarint(^uint64(0)) +// UInt128 is an unsigned 128-bit number +type UInt128 = proto.UInt128 diff --git a/common/schema/types_test.go b/common/schema/types_test.go deleted file mode 100644 index 5fb97d93..00000000 --- a/common/schema/types_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -package schema - -import ( - "testing" - - "google.golang.org/protobuf/encoding/protowire" -) - -func TestMaxSizeVarint(t *testing.T) { - got := protowire.SizeVarint(^uint64(0)) - if got != maxSizeVarint { - t.Fatalf("maximum size for varint is %d, not %d", got, maxSizeVarint) - } -} diff --git a/config/akvorado.yaml b/config/akvorado.yaml index 46d98c8c..46d23ee8 100644 --- a/config/akvorado.yaml +++ b/config/akvorado.yaml @@ -76,6 +76,7 @@ clickhouse: # { prefix: (.ipv4Prefix // .ipv6Prefix), tenant: "google-cloud", region: .scope } inlet: !include "inlet.yaml" +outlet: !include "outlet.yaml" console: !include "console.yaml" # Remove the following line if you don't want to get demo data diff --git a/config/demo.yaml b/config/demo.yaml index 16d96c26..6ac428a2 100644 --- a/config/demo.yaml +++ b/config/demo.yaml @@ -96,7 +96,7 @@ 21: "core" listen: :161 bmp: &bmp - target: akvorado-inlet:10179 + target: akvorado-outlet:10179 routes: - prefixes: 192.0.2.0/24,2a01:db8:cafe:1::/64 aspath: 64501 diff --git a/config/inlet.yaml b/config/inlet.yaml index 41790c5f..7e8886f8 100644 --- a/config/inlet.yaml +++ b/config/inlet.yaml @@ -1,13 +1,6 @@ --- kafka: compression-codec: zstd -metadata: - workers: 10 - provider: - type: snmp - credentials: - ::/0: - communities: public flow: inputs: - type: udp @@ -20,22 +13,3 @@ flow: listen: :6343 workers: 6 receive-buffer: 10485760 -core: - workers: 6 - exporter-classifiers: - # This is an example. This should be customized depending on how - # your exporters are named. - - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") - - ClassifyRegion("europe") - - ClassifyTenant("acme") - - ClassifyRole("edge") - interface-classifiers: - # This is an example. This must be customized depending on the - # descriptions of your interfaces. In the following, we assume - # external interfaces are named "Transit: Cogent" Or "IX: - # FranceIX". - - | - ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && - ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && - ClassifyExternal() - - ClassifyInternal() diff --git a/config/outlet.yaml b/config/outlet.yaml new file mode 100644 index 00000000..dd85a7c9 --- /dev/null +++ b/config/outlet.yaml @@ -0,0 +1,28 @@ +--- +metadata: + workers: 10 + provider: + type: snmp + credentials: + ::/0: + communities: public +kafka: + workers: 6 +core: + exporter-classifiers: + # This is an example. This should be customized depending on how + # your exporters are named. + - ClassifySiteRegex(Exporter.Name, "^([^-]+)-", "$1") + - ClassifyRegion("europe") + - ClassifyTenant("acme") + - ClassifyRole("edge") + interface-classifiers: + # This is an example. This must be customized depending on the + # descriptions of your interfaces. In the following, we assume + # external interfaces are named "Transit: Cogent" Or "IX: + # FranceIX". + - | + ClassifyConnectivityRegex(Interface.Description, "^(?i)(transit|pni|ppni|ix):? ", "$1") && + ClassifyProviderRegex(Interface.Description, "^\\S+?\\s(\\S+)", "$1") && + ClassifyExternal() + - ClassifyInternal() diff --git a/console/data/docs/01-install.md b/console/data/docs/01-install.md index 6d650eac..73bdb74c 100644 --- a/console/data/docs/01-install.md +++ b/console/data/docs/01-install.md @@ -32,12 +32,12 @@ Currently, only a pre-built binary for Linux x86-64 is provided. ## Compilation from source -You need a proper installation of [Go](https://go.dev/doc/install) (1.24+), and -[NodeJS](https://nodejs.org/en/download/) (20+) with NPM (6+). For example, on -Debian: +You need a proper installation of [Go](https://go.dev/doc/install) (1.24+), +[NodeJS](https://nodejs.org/en/download/) (20+) with NPM (6+), and +[protoc](https://protobuf.dev/installation/). For example, on Debian: ```console -# apt install golang nodejs npm +# apt install golang nodejs npm protobuf-compiler # go version go version go1.24.1 linux/amd64 # node --version diff --git a/console/data/docs/03-usage.md b/console/data/docs/03-usage.md index cd74732a..ef1c05bd 100644 --- a/console/data/docs/03-usage.md +++ b/console/data/docs/03-usage.md @@ -50,7 +50,6 @@ process flows. The following endpoints are exposed by the HTTP component embedded into the service: - `/api/v0/inlet/flows`: stream the received flows -- `/api/v0/inlet/schemas.proto`: protobuf schema ## Orchestrator service diff --git a/console/data/docs/99-changelog.md b/console/data/docs/99-changelog.md index 1e836311..6ff30fbe 100644 --- a/console/data/docs/99-changelog.md +++ b/console/data/docs/99-changelog.md @@ -13,6 +13,15 @@ identified with a specific icon: ## Unreleased +This release introduce a new component: the outlet. Previously, ClickHouse was +fetching data directly from Kafka. However, this required to push the protobuf +schema using an out-of-band method. This makes cloud deployments more complex. +The inlet now pushes incoming raw flows to Kafka without decoding them. The +outlet takes them, decode them, enriches them, and push them to ClickHouse. This +also reduces the likeliness to lose packets. This change should be transparent +on most setups but you are encouraged to review the new proposed configuration +in the [quickstart tarball][] and update your own configuration. + As it seems a good time as any, Zookeeper is removed from the `docker compose` setup (except when using ClickHouse cluster mode). Kafka is now using the KRaft mode. You can follow the [migration documentation][], but is easier to loose a @@ -25,6 +34,8 @@ bit of data and reset the Kafka container: # docker compose up -d ``` +- 💥 *outlet*: new service +- 💥 *inlet*: flow rate limiting feature has been removed - 💥 *docker*: switch Kafka to KRaft mode - 🩹 *console*: fix deletion of saved filters - 🩹 *console*: fix intermittent failure when requesting previous period @@ -37,6 +48,7 @@ bit of data and reset the Kafka container: - 🌱 *inlet*: improve performance of classifiers [migration documentation]: https://github.com/bitnami/containers/blob/main/bitnami/kafka/README.md#migrating-from-zookeeper-mode-to-kraft-mode +[quickstart tarball]: https://github.com/akvorado/akvorado/releases/latest/download/docker-compose-quickstart.tar.gz ## 1.11.5 - 2025-05-11 diff --git a/demoexporter/flows/nfdata_test.go b/demoexporter/flows/nfdata_test.go index 804b7ff8..7f00256b 100644 --- a/demoexporter/flows/nfdata_test.go +++ b/demoexporter/flows/nfdata_test.go @@ -11,15 +11,18 @@ import ( "time" "akvorado/common/helpers" + "akvorado/common/pb" "akvorado/common/reporter" "akvorado/common/schema" - "akvorado/inlet/flow/decoder" - "akvorado/inlet/flow/decoder/netflow" + "akvorado/outlet/flow/decoder" + "akvorado/outlet/flow/decoder/netflow" ) func TestGetNetflowData(t *testing.T) { r := reporter.NewMock(t) - nfdecoder := netflow.New(r, decoder.Dependencies{Schema: schema.NewMock(t)}, decoder.Option{TimestampSource: decoder.TimestampSourceUDP}) + sch := schema.NewMock(t) + bf := sch.NewFlowMessage() + nfdecoder := netflow.New(r, decoder.Dependencies{Schema: sch}) ch := getNetflowTemplates( context.Background(), @@ -27,11 +30,22 @@ func TestGetNetflowData(t *testing.T) { 30000, time.Date(2022, 3, 15, 14, 33, 0, 0, time.UTC), time.Date(2022, 3, 15, 15, 33, 0, 0, time.UTC)) - got := []interface{}{} + got := []*schema.FlowMessage{} + finalize := func() { + bf.TimeReceived = 0 + // Keep a copy of the current flow message + clone := *bf + got = append(got, &clone) + // And clear the flow message + bf.Clear() + } + for payload := range ch { - got = append(got, nfdecoder.Decode(decoder.RawFlow{ - Payload: payload, Source: net.ParseIP("127.0.0.1"), - })) + if _, err := nfdecoder.Decode(decoder.RawFlow{ + Payload: payload, Source: netip.MustParseAddr("::ffff:127.0.0.1"), + }, decoder.Option{TimestampSource: pb.RawFlow_TS_INPUT}, bf, finalize); err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } } ch = getNetflowData( @@ -97,90 +111,76 @@ func TestGetNetflowData(t *testing.T) { time.Date(2022, 3, 15, 14, 33, 0, 0, time.UTC), time.Date(2022, 3, 15, 16, 33, 0, 0, time.UTC)) for payload := range ch { - got = append(got, nfdecoder.Decode(decoder.RawFlow{ - Payload: payload, Source: net.ParseIP("127.0.0.1"), - })) + if _, err := nfdecoder.Decode(decoder.RawFlow{ + Payload: payload, Source: netip.MustParseAddr("::ffff:127.0.0.1"), + }, decoder.Option{TimestampSource: pb.RawFlow_TS_INPUT}, bf, finalize); err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } } - expected := []interface{}{ - []interface{}{}, // templates - []interface{}{ - &schema.FlowMessage{ - SamplingRate: 30000, - ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), - SrcAddr: netip.MustParseAddr("::ffff:192.0.2.206"), - DstAddr: netip.MustParseAddr("::ffff:203.0.113.165"), - InIf: 10, - OutIf: 20, - SrcAS: 65201, - DstAS: 65202, - SrcNetMask: 24, - DstNetMask: 23, - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 1500, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv4, - schema.ColumnProto: 6, - schema.ColumnSrcPort: 443, - schema.ColumnDstPort: 34974, - schema.ColumnForwardingStatus: 64, - }, - }, - &schema.FlowMessage{ - SamplingRate: 30000, - ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), - SrcAddr: netip.MustParseAddr("::ffff:192.0.2.236"), - DstAddr: netip.MustParseAddr("::ffff:203.0.113.67"), - InIf: 10, - OutIf: 20, - SrcAS: 65201, - DstAS: 65202, - SrcNetMask: 24, - DstNetMask: 24, - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 1339, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv4, - schema.ColumnProto: 6, - schema.ColumnSrcPort: 443, - schema.ColumnDstPort: 33199, - schema.ColumnForwardingStatus: 64, - }, + expected := []*schema.FlowMessage{ + { + SamplingRate: 30000, + ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), + SrcAddr: netip.MustParseAddr("::ffff:192.0.2.206"), + DstAddr: netip.MustParseAddr("::ffff:203.0.113.165"), + InIf: 10, + OutIf: 20, + SrcAS: 65201, + DstAS: 65202, + SrcNetMask: 24, + DstNetMask: 23, + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 1500, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv4, + schema.ColumnProto: 6, + schema.ColumnSrcPort: 443, + schema.ColumnDstPort: 34974, + schema.ColumnForwardingStatus: 64, }, }, - []interface{}{ - &schema.FlowMessage{ - SamplingRate: 30000, - ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), - SrcAddr: netip.MustParseAddr("2001:db8::1"), - DstAddr: netip.MustParseAddr("2001:db8:2:0:cea5:d643:ec43:3772"), - InIf: 20, - OutIf: 10, - SrcAS: 65201, - DstAS: 65202, - SrcNetMask: 48, - DstNetMask: 48, - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 1300, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv6, - schema.ColumnProto: 6, - schema.ColumnSrcPort: 33179, - schema.ColumnDstPort: 443, - schema.ColumnForwardingStatus: 64, - }, + { + SamplingRate: 30000, + ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), + SrcAddr: netip.MustParseAddr("::ffff:192.0.2.236"), + DstAddr: netip.MustParseAddr("::ffff:203.0.113.67"), + InIf: 10, + OutIf: 20, + SrcAS: 65201, + DstAS: 65202, + SrcNetMask: 24, + DstNetMask: 24, + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 1339, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv4, + schema.ColumnProto: 6, + schema.ColumnSrcPort: 443, + schema.ColumnDstPort: 33199, + schema.ColumnForwardingStatus: 64, + }, + }, + { + SamplingRate: 30000, + ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), + SrcAddr: netip.MustParseAddr("2001:db8::1"), + DstAddr: netip.MustParseAddr("2001:db8:2:0:cea5:d643:ec43:3772"), + InIf: 20, + OutIf: 10, + SrcAS: 65201, + DstAS: 65202, + SrcNetMask: 48, + DstNetMask: 48, + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 1300, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv6, + schema.ColumnProto: 6, + schema.ColumnSrcPort: 33179, + schema.ColumnDstPort: 443, + schema.ColumnForwardingStatus: 64, }, }, - } - for idx1 := range got { - if got[idx1] == nil { - continue - } - switch g := got[idx1].(type) { - case []*schema.FlowMessage: - for idx2 := range g { - g[idx2].TimeReceived = 0 - } - } } if diff := helpers.Diff(got, expected); diff != "" { diff --git a/docker/docker-compose-demo.yml b/docker/docker-compose-demo.yml index adbe8eea..ea3ffe49 100644 --- a/docker/docker-compose-demo.yml +++ b/docker/docker-compose-demo.yml @@ -9,6 +9,8 @@ services: depends_on: akvorado-inlet: condition: service_healthy + akvorado-outlet: + condition: service_healthy akvorado-exporter1: <<: *exporter command: demo-exporter http://akvorado-orchestrator:8080#1 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 7e804e8a..ecd3b9a7 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -105,7 +105,6 @@ services: ports: - 2055:2055/udp - 6343:6343/udp - - 10179:10179/tcp restart: unless-stopped depends_on: akvorado-orchestrator: @@ -115,8 +114,6 @@ services: command: inlet http://akvorado-orchestrator:8080 volumes: - akvorado-run:/run/akvorado - environment: - - AKVORADO_CFG_INLET_METADATA_CACHEPERSISTFILE=/run/akvorado/metadata.cache labels: - traefik.enable=true # Disable access logging of /api/v0/inlet/metrics @@ -129,6 +126,36 @@ services: - traefik.http.routers.akvorado-inlet.rule=PathPrefix(`/api/v0/inlet`) - traefik.http.services.akvorado-inlet.loadbalancer.server.port=8080 - akvorado.conntrack.fix=true + akvorado-outlet: + extends: + file: versions.yml + service: akvorado + ports: + - 10179:10179/tcp + restart: unless-stopped + depends_on: + akvorado-orchestrator: + condition: service_healthy + kafka: + condition: service_healthy + clickhouse: + condition: service_healthy + command: outlet http://akvorado-orchestrator:8080 + volumes: + - akvorado-run:/run/akvorado + environment: + - AKVORADO_CFG_OUTLET_METADATA_CACHEPERSISTFILE=/run/akvorado/metadata.cache + labels: + - traefik.enable=true + # Disable access logging of /api/v0/outlet/metrics + - traefik.http.routers.akvorado-outlet-metrics.entrypoints=private + - traefik.http.routers.akvorado-outlet-metrics.rule=PathPrefix(`/api/v0/outlet/metrics`) + - traefik.http.routers.akvorado-outlet-metrics.service=akvorado-outlet + - traefik.http.routers.akvorado-outlet-metrics.observability.accesslogs=false + # Everything else is exposed to private entrypoing in /api/v0/outlet + - traefik.http.routers.akvorado-outlet.entrypoints=private + - traefik.http.routers.akvorado-outlet.rule=PathPrefix(`/api/v0/outlet`) + - traefik.http.services.akvorado-outlet.loadbalancer.server.port=8080 akvorado-conntrack-fixer: extends: file: versions.yml diff --git a/docker/prometheus.yml b/docker/prometheus.yml index e25fa6f8..9ac77705 100644 --- a/docker/prometheus.yml +++ b/docker/prometheus.yml @@ -60,7 +60,7 @@ scrape_configs: - com.docker.compose.project=akvorado relabel_configs: - source_labels: [__meta_docker_container_label_com_docker_compose_service] - regex: akvorado-(inlet|orchestrator|console) + regex: akvorado-(inlet|outlet|orchestrator|console) action: keep - source_labels: [__meta_docker_port_private] regex: 8080 diff --git a/flake.nix b/flake.nix index eefbb790..0c02df6b 100644 --- a/flake.nix +++ b/flake.nix @@ -55,6 +55,7 @@ find . -print0 | xargs -0 touch -d @0 make all \ + PROTOC=${pkgs.protobuf}/bin/protoc \ ASNS_URL=${asn2org}/asns.csv \ SERVICES_URL=${ianaServiceNames} ''; @@ -110,6 +111,7 @@ nodejs pkgs.git pkgs.curl + pkgs.protobuf ]; }; }); diff --git a/go.mod b/go.mod index 0b860f50..fcedd121 100644 --- a/go.mod +++ b/go.mod @@ -218,6 +218,7 @@ tool ( github.com/mna/pigeon go.uber.org/mock/mockgen golang.org/x/tools/cmd/goimports + google.golang.org/protobuf/cmd/protoc-gen-go gotest.tools/gotestsum ) diff --git a/inlet/core/root.go b/inlet/core/root.go deleted file mode 100644 index 67b92a6d..00000000 --- a/inlet/core/root.go +++ /dev/null @@ -1,167 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -// Package core plumbs all the other components together. -package core - -import ( - "fmt" - "sync/atomic" - "time" - - "gopkg.in/tomb.v2" - - "akvorado/common/daemon" - "akvorado/common/helpers/cache" - "akvorado/common/httpserver" - "akvorado/common/reporter" - "akvorado/common/schema" - "akvorado/inlet/flow" - "akvorado/inlet/kafka" - "akvorado/inlet/metadata" - "akvorado/inlet/routing" -) - -// Component represents the HTTP compomenent. -type Component struct { - r *reporter.Reporter - d *Dependencies - t tomb.Tomb - config Configuration - - metrics metrics - - healthy chan reporter.ChannelHealthcheckFunc - httpFlowClients uint32 // for dumping flows - httpFlowChannel chan *schema.FlowMessage - httpFlowFlushDelay time.Duration - - classifierExporterCache *cache.Cache[exporterInfo, exporterClassification] - classifierInterfaceCache *cache.Cache[exporterAndInterfaceInfo, interfaceClassification] - classifierErrLogger reporter.Logger -} - -// Dependencies define the dependencies of the HTTP component. -type Dependencies struct { - Daemon daemon.Component - Flow *flow.Component - Metadata *metadata.Component - Routing *routing.Component - Kafka *kafka.Component - HTTP *httpserver.Component - Schema *schema.Component -} - -// New creates a new core component. -func New(r *reporter.Reporter, configuration Configuration, dependencies Dependencies) (*Component, error) { - c := Component{ - r: r, - d: &dependencies, - config: configuration, - - healthy: make(chan reporter.ChannelHealthcheckFunc), - httpFlowClients: 0, - httpFlowChannel: make(chan *schema.FlowMessage, 10), - httpFlowFlushDelay: time.Second, - - classifierExporterCache: cache.New[exporterInfo, exporterClassification](), - classifierInterfaceCache: cache.New[exporterAndInterfaceInfo, interfaceClassification](), - classifierErrLogger: r.Sample(reporter.BurstSampler(10*time.Second, 3)), - } - c.d.Daemon.Track(&c.t, "inlet/core") - c.initMetrics() - return &c, nil -} - -// Start starts the core component. -func (c *Component) Start() error { - c.r.Info().Msg("starting core component") - for i := range c.config.Workers { - workerID := i - c.t.Go(func() error { - return c.runWorker(workerID) - }) - } - - // Classifier cache expiration - c.t.Go(func() error { - for { - select { - case <-c.t.Dying(): - return nil - case <-time.After(c.config.ClassifierCacheDuration): - before := time.Now().Add(-c.config.ClassifierCacheDuration) - c.classifierExporterCache.DeleteLastAccessedBefore(before) - c.classifierInterfaceCache.DeleteLastAccessedBefore(before) - } - } - }) - - c.r.RegisterHealthcheck("core", c.channelHealthcheck()) - c.d.HTTP.GinRouter.GET("/api/v0/inlet/flows", c.FlowsHTTPHandler) - return nil -} - -// runWorker starts a worker. -func (c *Component) runWorker(workerID int) error { - c.r.Debug().Int("worker", workerID).Msg("starting core worker") - - for { - select { - case <-c.t.Dying(): - c.r.Debug().Int("worker", workerID).Msg("stopping core worker") - return nil - case cb, ok := <-c.healthy: - if ok { - cb(reporter.HealthcheckOK, fmt.Sprintf("worker %d ok", workerID)) - } - case flow := <-c.d.Flow.Flows(): - if flow == nil { - c.r.Info().Int("worker", workerID).Msg("no more flow available, stopping") - return nil - } - - exporter := flow.ExporterAddress.Unmap().String() - c.metrics.flowsReceived.WithLabelValues(exporter).Inc() - - // Enrichment - ip := flow.ExporterAddress - if skip := c.enrichFlow(ip, exporter, flow); skip { - continue - } - - // Serialize flow to Protobuf - buf := c.d.Schema.ProtobufMarshal(flow) - - // Forward to Kafka. This could block and buf is now owned by the - // Kafka subsystem! - c.metrics.flowsForwarded.WithLabelValues(exporter).Inc() - c.d.Kafka.Send(exporter, buf) - - // If we have HTTP clients, send to them too - if atomic.LoadUint32(&c.httpFlowClients) > 0 { - select { - case c.httpFlowChannel <- flow: // OK - default: // Overflow, best effort and ignore - } - } - - } - } -} - -// Stop stops the core component. -func (c *Component) Stop() error { - defer func() { - close(c.httpFlowChannel) - close(c.healthy) - c.r.Info().Msg("core component stopped") - }() - c.r.Info().Msg("stopping core component") - c.t.Kill(nil) - return c.t.Wait() -} - -func (c *Component) channelHealthcheck() reporter.HealthcheckFunc { - return reporter.ChannelHealthcheck(c.t.Context(nil), c.healthy) -} diff --git a/inlet/core/root_test.go b/inlet/core/root_test.go deleted file mode 100644 index 8e899020..00000000 --- a/inlet/core/root_test.go +++ /dev/null @@ -1,363 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -package core - -import ( - "bufio" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/netip" - "strings" - "testing" - "time" - - "github.com/IBM/sarama" - "github.com/gin-gonic/gin" - - "akvorado/common/daemon" - "akvorado/common/helpers" - "akvorado/common/httpserver" - "akvorado/common/reporter" - "akvorado/common/schema" - "akvorado/inlet/flow" - "akvorado/inlet/kafka" - "akvorado/inlet/metadata" - "akvorado/inlet/routing" -) - -func TestCore(t *testing.T) { - r := reporter.NewMock(t) - - // Prepare all components. - daemonComponent := daemon.NewMock(t) - metadataComponent := metadata.NewMock(t, r, metadata.DefaultConfiguration(), - metadata.Dependencies{Daemon: daemonComponent}) - flowComponent := flow.NewMock(t, r, flow.DefaultConfiguration()) - kafkaComponent, kafkaProducer := kafka.NewMock(t, r, kafka.DefaultConfiguration()) - httpComponent := httpserver.NewMock(t, r) - routingComponent := routing.NewMock(t, r) - routingComponent.PopulateRIB(t) - - // Instantiate and start core - sch := schema.NewMock(t) - c, err := New(r, DefaultConfiguration(), Dependencies{ - Daemon: daemonComponent, - Flow: flowComponent, - Metadata: metadataComponent, - Kafka: kafkaComponent, - HTTP: httpComponent, - Routing: routingComponent, - Schema: sch, - }) - if err != nil { - t.Fatalf("New() error:\n%+v", err) - } - helpers.StartStop(t, c) - - flowMessage := func(exporter string, in, out uint32) *schema.FlowMessage { - msg := &schema.FlowMessage{ - TimeReceived: 200, - SamplingRate: 1000, - ExporterAddress: netip.MustParseAddr(exporter), - InIf: in, - OutIf: out, - SrcAddr: netip.MustParseAddr("67.43.156.77"), - DstAddr: netip.MustParseAddr("2.125.160.216"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 6765, - schema.ColumnPackets: 4, - schema.ColumnEType: 0x800, - schema.ColumnProto: 6, - schema.ColumnSrcPort: 8534, - schema.ColumnDstPort: 80, - }, - } - for k, v := range msg.ProtobufDebug { - vi := v.(int) - sch.ProtobufAppendVarint(msg, k, uint64(vi)) - } - return msg - } - - expectedFlowMessage := func(exporter string, in, out uint32) *schema.FlowMessage { - expected := flowMessage(exporter, in, out) - expected.SrcAS = 0 // no geoip enrich anymore - expected.DstAS = 0 // no geoip enrich anymore - expected.InIf = 0 // not serialized - expected.OutIf = 0 // not serialized - expected.ExporterAddress = netip.AddrFrom16(expected.ExporterAddress.As16()) - expected.SrcAddr = netip.AddrFrom16(expected.SrcAddr.As16()) - expected.DstAddr = netip.AddrFrom16(expected.DstAddr.As16()) - expected.ProtobufDebug[schema.ColumnInIfName] = fmt.Sprintf("Gi0/0/%d", in) - expected.ProtobufDebug[schema.ColumnOutIfName] = fmt.Sprintf("Gi0/0/%d", out) - expected.ProtobufDebug[schema.ColumnInIfDescription] = fmt.Sprintf("Interface %d", in) - expected.ProtobufDebug[schema.ColumnOutIfDescription] = fmt.Sprintf("Interface %d", out) - expected.ProtobufDebug[schema.ColumnInIfSpeed] = 1000 - expected.ProtobufDebug[schema.ColumnOutIfSpeed] = 1000 - expected.ProtobufDebug[schema.ColumnExporterName] = strings.ReplaceAll(exporter, ".", "_") - return expected - } - - t.Run("kafka", func(t *testing.T) { - // Inject several messages with a cache miss from the SNMP - // component for each of them. No message sent to Kafka. - flowComponent.Inject(flowMessage("192.0.2.142", 434, 677)) - flowComponent.Inject(flowMessage("192.0.2.143", 434, 677)) - flowComponent.Inject(flowMessage("192.0.2.143", 437, 677)) - flowComponent.Inject(flowMessage("192.0.2.143", 434, 679)) - - time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_core_", "-flows_processing_") - expectedMetrics := map[string]string{ - `classifier_exporter_cache_size_items`: "0", - `classifier_interface_cache_size_items`: "0", - `flows_errors_total{error="SNMP cache miss",exporter="192.0.2.142"}`: "1", - `flows_errors_total{error="SNMP cache miss",exporter="192.0.2.143"}`: "3", - `received_flows_total{exporter="192.0.2.142"}`: "1", - `received_flows_total{exporter="192.0.2.143"}`: "3", - `flows_http_clients`: "0", - } - if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { - t.Fatalf("Metrics (-got, +want):\n%s", diff) - } - - // Inject again the messages, this time, we will get a cache hit! - kafkaProducer.ExpectInputAndSucceed() - flowComponent.Inject(flowMessage("192.0.2.142", 434, 677)) - kafkaProducer.ExpectInputAndSucceed() - flowComponent.Inject(flowMessage("192.0.2.143", 437, 679)) - - time.Sleep(20 * time.Millisecond) - gotMetrics = r.GetMetrics("akvorado_inlet_core_", "classifier_", "-flows_processing_", "flows_", "received_", "forwarded_") - expectedMetrics = map[string]string{ - `classifier_exporter_cache_size_items`: "0", - `classifier_interface_cache_size_items`: "0", - `flows_errors_total{error="SNMP cache miss",exporter="192.0.2.142"}`: "1", - `flows_errors_total{error="SNMP cache miss",exporter="192.0.2.143"}`: "3", - `received_flows_total{exporter="192.0.2.142"}`: "2", - `received_flows_total{exporter="192.0.2.143"}`: "4", - `forwarded_flows_total{exporter="192.0.2.142"}`: "1", - `forwarded_flows_total{exporter="192.0.2.143"}`: "1", - `flows_http_clients`: "0", - } - if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { - t.Fatalf("Metrics (-got, +want):\n%s", diff) - } - - // Now, check we get the message we expect - input := flowMessage("192.0.2.142", 434, 677) - received := make(chan bool) - kafkaProducer.ExpectInputWithMessageCheckerFunctionAndSucceed(func(msg *sarama.ProducerMessage) error { - defer close(received) - expectedTopic := fmt.Sprintf("flows-%s", sch.ProtobufMessageHash()) - if msg.Topic != expectedTopic { - t.Errorf("Kafka message topic (-got, +want):\n-%s\n+%s", msg.Topic, expectedTopic) - } - - b, err := msg.Value.Encode() - if err != nil { - t.Fatalf("Kafka message encoding error:\n%+v", err) - } - got := sch.ProtobufDecode(t, b) - expected := expectedFlowMessage("192.0.2.142", 434, 677) - if diff := helpers.Diff(&got, expected); diff != "" { - t.Errorf("Kafka message (-got, +want):\n%s", diff) - } - - return nil - }) - flowComponent.Inject(input) - select { - case <-received: - case <-time.After(time.Second): - t.Fatal("Kafka message not received") - } - - // Try to inject a message with missing sampling rate - input = flowMessage("192.0.2.142", 434, 677) - input.SamplingRate = 0 - flowComponent.Inject(input) - time.Sleep(20 * time.Millisecond) - gotMetrics = r.GetMetrics("akvorado_inlet_core_", "classifier_", "-flows_processing_", "flows_", "forwarded_", "received_") - expectedMetrics = map[string]string{ - `classifier_exporter_cache_size_items`: "0", - `classifier_interface_cache_size_items`: "0", - `flows_errors_total{error="SNMP cache miss",exporter="192.0.2.142"}`: "1", - `flows_errors_total{error="SNMP cache miss",exporter="192.0.2.143"}`: "3", - `flows_errors_total{error="sampling rate missing",exporter="192.0.2.142"}`: "1", - `received_flows_total{exporter="192.0.2.142"}`: "4", - `received_flows_total{exporter="192.0.2.143"}`: "4", - `forwarded_flows_total{exporter="192.0.2.142"}`: "2", - `forwarded_flows_total{exporter="192.0.2.143"}`: "1", - `flows_http_clients`: "0", - } - if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { - t.Fatalf("Metrics (-got, +want):\n%s", diff) - } - }) - - // Test the healthcheck function - t.Run("healthcheck", func(t *testing.T) { - got := r.RunHealthchecks(context.Background()) - if diff := helpers.Diff(got.Details["core"], reporter.HealthcheckResult{ - Status: reporter.HealthcheckOK, - Reason: "worker 0 ok", - }); diff != "" { - t.Fatalf("runHealthcheck() (-got, +want):\n%s", diff) - } - }) - - // Test HTTP flow clients (JSON) - t.Run("http flows", func(t *testing.T) { - c.httpFlowFlushDelay = 20 * time.Millisecond - - resp, err := http.Get(fmt.Sprintf("http://%s/api/v0/inlet/flows", c.d.HTTP.LocalAddr())) - if err != nil { - t.Fatalf("GET /api/v0/inlet/flows:\n%+v", err) - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - t.Fatalf("GET /api/v0/inlet/flows status code %d", resp.StatusCode) - } - - // Metrics should tell we have a client - gotMetrics := r.GetMetrics("akvorado_inlet_core_", "flows_http_clients", "-flows_processing_") - expectedMetrics := map[string]string{ - `flows_http_clients`: "1", - } - if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { - t.Fatalf("Metrics (-got, +want):\n%s", diff) - } - - // Produce some flows - for range 12 { - kafkaProducer.ExpectInputAndSucceed() - flowComponent.Inject(flowMessage("192.0.2.142", 434, 677)) - } - - // Decode some of them - reader := bufio.NewReader(resp.Body) - decoder := json.NewDecoder(reader) - for range 10 { - var got gin.H - if err := decoder.Decode(&got); err != nil { - t.Fatalf("GET /api/v0/inlet/flows error while reading body:\n%+v", err) - } - expected := gin.H{ - "TimeReceived": 200, - "SamplingRate": 1000, - "ExporterAddress": "192.0.2.142", - "SrcAddr": "67.43.156.77", - "DstAddr": "2.125.160.216", - "SrcAS": 0, // no geoip enrich anymore - "InIf": 434, - "OutIf": 677, - - "NextHop": "", - "SrcNetMask": 0, - "DstNetMask": 0, - "SrcVlan": 0, - "DstVlan": 0, - "GotASPath": false, - "GotCommunities": false, - "DstAS": 0, - } - if diff := helpers.Diff(got, expected); diff != "" { - t.Fatalf("GET /api/v0/inlet/flows (-got, +want):\n%s", diff) - } - } - }) - - // Test HTTP flow clients with a limit - time.Sleep(10 * time.Millisecond) - t.Run("http flows with limit", func(t *testing.T) { - resp, err := http.Get(fmt.Sprintf("http://%s/api/v0/inlet/flows?limit=4", c.d.HTTP.LocalAddr())) - if err != nil { - t.Fatalf("GET /api/v0/inlet/flows:\n%+v", err) - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - t.Fatalf("GET /api/v0/inlet/flows status code %d", resp.StatusCode) - } - - // Metrics should tell we have a client - gotMetrics := r.GetMetrics("akvorado_inlet_core_", "flows_http_clients") - expectedMetrics := map[string]string{ - `flows_http_clients`: "1", - } - if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { - t.Fatalf("Metrics (-got, +want):\n%s", diff) - } - - // Produce some flows - for range 12 { - kafkaProducer.ExpectInputAndSucceed() - flowComponent.Inject(flowMessage("192.0.2.142", 434, 677)) - } - - // Check we got only 4 - reader := bufio.NewReader(resp.Body) - count := 0 - for { - _, err := reader.ReadString('\n') - if err == io.EOF { - t.Log("EOF") - break - } - if err != nil { - t.Fatalf("GET /api/v0/inlet/flows error while reading:\n%+v", err) - } - count++ - if count > 4 { - break - } - } - if count != 4 { - t.Fatalf("GET /api/v0/inlet/flows got less than 4 flows (%d)", count) - } - }) - - // Test HTTP flow clients using protobuf - time.Sleep(10 * time.Millisecond) - t.Run("http flows with protobuf", func(t *testing.T) { - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s/api/v0/inlet/flows?limit=1", c.d.HTTP.LocalAddr()), nil) - if err != nil { - t.Fatalf("http.NewRequest() error:\n%+v", err) - } - req.Header.Set("accept", "application/x-protobuf") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("GET /api/v0/inlet/flows:\n%+v", err) - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - t.Fatalf("GET /api/v0/inlet/flows status code %d", resp.StatusCode) - } - - // Produce some flows - for range 12 { - kafkaProducer.ExpectInputAndSucceed() - flowComponent.Inject(flowMessage("192.0.2.142", 434, 677)) - } - - // Get the resulting flow - reader := bufio.NewReader(resp.Body) - got, err := io.ReadAll(reader) - if err != nil { - t.Fatalf("GET /api/v0/inlet/flows error while reading:\n%+v", err) - } - t.Logf("got %v", got) - - // Decode - sch := schema.NewMock(t) - decoded := sch.ProtobufDecode(t, got) - expected := expectedFlowMessage("192.0.2.142", 434, 677) - if diff := helpers.Diff(decoded, expected); diff != "" { - t.Errorf("HTTP message (-got, +want):\n%s", diff) - } - }) -} diff --git a/inlet/flow/config.go b/inlet/flow/config.go index ecbe89e6..eed0f0ba 100644 --- a/inlet/flow/config.go +++ b/inlet/flow/config.go @@ -4,10 +4,8 @@ package flow import ( - "golang.org/x/time/rate" - "akvorado/common/helpers" - "akvorado/inlet/flow/decoder" + "akvorado/common/pb" "akvorado/inlet/flow/input" "akvorado/inlet/flow/input/file" "akvorado/inlet/flow/input/udp" @@ -17,21 +15,18 @@ import ( type Configuration struct { // Inputs define a list of input modules to enable Inputs []InputConfiguration `validate:"dive"` - // RateLimit defines a rate limit on the number of flows per - // second. The limit is per-exporter. - RateLimit rate.Limit `validate:"isdefault|min=100"` } // DefaultConfiguration represents the default configuration for the flow component func DefaultConfiguration() Configuration { return Configuration{ Inputs: []InputConfiguration{{ - TimestampSource: decoder.TimestampSourceUDP, - Decoder: "netflow", + TimestampSource: pb.RawFlow_TS_INPUT, + Decoder: pb.RawFlow_DECODER_NETFLOW, Config: udp.DefaultConfiguration(), }, { - TimestampSource: decoder.TimestampSourceUDP, - Decoder: "sflow", + TimestampSource: pb.RawFlow_TS_INPUT, + Decoder: pb.RawFlow_DECODER_SFLOW, Config: udp.DefaultConfiguration(), }}, } @@ -40,12 +35,12 @@ func DefaultConfiguration() Configuration { // InputConfiguration represents the configuration for an input. type InputConfiguration struct { // Decoder is the decoder to associate to the input. - Decoder string + Decoder pb.RawFlow_Decoder `validate:"required"` // UseSrcAddrForExporterAddr replaces the exporter address by the transport // source address. UseSrcAddrForExporterAddr bool // TimestampSource identify the source to use to timestamp the flows - TimestampSource decoder.TimestampSource + TimestampSource pb.RawFlow_TimestampSource // Config is the actual configuration of the input. Config input.Configuration } @@ -61,6 +56,7 @@ var inputs = map[string](func() input.Configuration){ } func init() { + helpers.RegisterMapstructureDeprecatedFields[Configuration]("RateLimit") helpers.RegisterMapstructureUnmarshallerHook( helpers.ParametrizedConfigurationUnmarshallerHook(InputConfiguration{}, inputs)) } diff --git a/inlet/flow/config_test.go b/inlet/flow/config_test.go index c6fbc560..987f062d 100644 --- a/inlet/flow/config_test.go +++ b/inlet/flow/config_test.go @@ -10,9 +10,9 @@ import ( "github.com/gin-gonic/gin" "akvorado/common/helpers/yaml" + "akvorado/common/pb" "akvorado/common/helpers" - "akvorado/inlet/flow/decoder" "akvorado/inlet/flow/input/file" "akvorado/inlet/flow/input/udp" ) @@ -42,7 +42,7 @@ func TestDecodeConfiguration(t *testing.T) { }, Expected: Configuration{ Inputs: []InputConfiguration{{ - Decoder: "netflow", + Decoder: pb.RawFlow_DECODER_NETFLOW, Config: &udp.Configuration{ Workers: 3, QueueSize: 100000, @@ -50,7 +50,7 @@ func TestDecodeConfiguration(t *testing.T) { }, UseSrcAddrForExporterAddr: true, }, { - Decoder: "sflow", + Decoder: pb.RawFlow_DECODER_SFLOW, Config: &udp.Configuration{ Workers: 3, QueueSize: 100000, @@ -64,10 +64,10 @@ func TestDecodeConfiguration(t *testing.T) { Initial: func() interface{} { return Configuration{ Inputs: []InputConfiguration{{ - Decoder: "netflow", + Decoder: pb.RawFlow_DECODER_NETFLOW, Config: udp.DefaultConfiguration(), }, { - Decoder: "sflow", + Decoder: pb.RawFlow_DECODER_SFLOW, Config: udp.DefaultConfiguration(), }}, } @@ -91,14 +91,14 @@ func TestDecodeConfiguration(t *testing.T) { }, Expected: Configuration{ Inputs: []InputConfiguration{{ - Decoder: "netflow", + Decoder: pb.RawFlow_DECODER_NETFLOW, Config: &udp.Configuration{ Workers: 3, QueueSize: 100000, Listen: "192.0.2.1:2055", }, }, { - Decoder: "sflow", + Decoder: pb.RawFlow_DECODER_SFLOW, Config: &udp.Configuration{ Workers: 3, QueueSize: 100000, @@ -111,7 +111,7 @@ func TestDecodeConfiguration(t *testing.T) { Initial: func() interface{} { return Configuration{ Inputs: []InputConfiguration{{ - Decoder: "netflow", + Decoder: pb.RawFlow_DECODER_NETFLOW, Config: udp.DefaultConfiguration(), }}, } @@ -128,7 +128,7 @@ func TestDecodeConfiguration(t *testing.T) { }, Expected: Configuration{ Inputs: []InputConfiguration{{ - Decoder: "netflow", + Decoder: pb.RawFlow_DECODER_NETFLOW, Config: &file.Configuration{ Paths: []string{"file1", "file2"}, }, @@ -139,8 +139,8 @@ func TestDecodeConfiguration(t *testing.T) { Initial: func() interface{} { return Configuration{ Inputs: []InputConfiguration{{ - Decoder: "netflow", - TimestampSource: decoder.TimestampSourceUDP, + Decoder: pb.RawFlow_DECODER_NETFLOW, + TimestampSource: pb.RawFlow_TS_INPUT, Config: &udp.Configuration{ Workers: 2, QueueSize: 100, @@ -160,7 +160,7 @@ func TestDecodeConfiguration(t *testing.T) { }, Expected: Configuration{ Inputs: []InputConfiguration{{ - Decoder: "netflow", + Decoder: pb.RawFlow_DECODER_NETFLOW, Config: &udp.Configuration{ Workers: 2, QueueSize: 100, @@ -197,7 +197,7 @@ func TestDecodeConfiguration(t *testing.T) { Initial: func() interface{} { return Configuration{ Inputs: []InputConfiguration{{ - Decoder: "netflow", + Decoder: pb.RawFlow_DECODER_NETFLOW, Config: &udp.Configuration{ Workers: 2, QueueSize: 100, @@ -218,8 +218,8 @@ func TestDecodeConfiguration(t *testing.T) { }, Expected: Configuration{ Inputs: []InputConfiguration{{ - Decoder: "netflow", - TimestampSource: decoder.TimestampSourceNetflowPacket, + Decoder: pb.RawFlow_DECODER_NETFLOW, + TimestampSource: pb.RawFlow_TS_NETFLOW_PACKET, Config: &udp.Configuration{ Workers: 2, QueueSize: 100, @@ -233,7 +233,7 @@ func TestDecodeConfiguration(t *testing.T) { Initial: func() interface{} { return Configuration{ Inputs: []InputConfiguration{{ - Decoder: "netflow", + Decoder: pb.RawFlow_DECODER_NETFLOW, Config: &udp.Configuration{ Workers: 2, QueueSize: 100, @@ -254,8 +254,8 @@ func TestDecodeConfiguration(t *testing.T) { }, Expected: Configuration{ Inputs: []InputConfiguration{{ - Decoder: "netflow", - TimestampSource: decoder.TimestampSourceNetflowFirstSwitched, + Decoder: pb.RawFlow_DECODER_NETFLOW, + TimestampSource: pb.RawFlow_TS_NETFLOW_FIRST_SWITCHED, Config: &udp.Configuration{ Workers: 2, QueueSize: 100, @@ -271,15 +271,15 @@ func TestMarshalYAML(t *testing.T) { cfg := Configuration{ Inputs: []InputConfiguration{ { - Decoder: "netflow", - TimestampSource: decoder.TimestampSourceNetflowFirstSwitched, + Decoder: pb.RawFlow_DECODER_NETFLOW, + TimestampSource: pb.RawFlow_TS_NETFLOW_FIRST_SWITCHED, Config: &udp.Configuration{ Listen: "192.0.2.11:2055", QueueSize: 1000, Workers: 3, }, }, { - Decoder: "sflow", + Decoder: pb.RawFlow_DECODER_SFLOW, Config: &udp.Configuration{ Listen: "192.0.2.11:6343", QueueSize: 1000, @@ -306,11 +306,10 @@ func TestMarshalYAML(t *testing.T) { listen: 192.0.2.11:6343 queuesize: 1000 receivebuffer: 0 - timestampsource: udp + timestampsource: input type: udp usesrcaddrforexporteraddr: true workers: 3 -ratelimit: 0 ` if diff := helpers.Diff(strings.Split(string(got), "\n"), strings.Split(expected, "\n")); diff != "" { t.Fatalf("Marshal() (-got, +want):\n%s", diff) diff --git a/inlet/flow/decoder.go b/inlet/flow/decoder.go deleted file mode 100644 index f23c9639..00000000 --- a/inlet/flow/decoder.go +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -package flow - -import ( - "net/netip" - - "akvorado/common/schema" - "akvorado/inlet/flow/decoder" - "akvorado/inlet/flow/decoder/netflow" - "akvorado/inlet/flow/decoder/sflow" -) - -type wrappedDecoder struct { - c *Component - orig decoder.Decoder - useSrcAddrForExporterAddr bool -} - -// Decode decodes a flow while keeping some stats. -func (wd *wrappedDecoder) Decode(in decoder.RawFlow) []*schema.FlowMessage { - defer func() { - if r := recover(); r != nil { - wd.c.metrics.decoderErrors.WithLabelValues(wd.orig.Name()). - Inc() - } - }() - decoded := wd.orig.Decode(in) - - if decoded == nil { - wd.c.metrics.decoderErrors.WithLabelValues(wd.orig.Name()). - Inc() - return nil - } - - if wd.useSrcAddrForExporterAddr { - exporterAddress, _ := netip.AddrFromSlice(in.Source.To16()) - for _, f := range decoded { - f.ExporterAddress = exporterAddress - } - } - - wd.c.metrics.decoderStats.WithLabelValues(wd.orig.Name()). - Inc() - return decoded -} - -// Name returns the name of the original decoder. -func (wd *wrappedDecoder) Name() string { - return wd.orig.Name() -} - -// wrapDecoder wraps the provided decoders to get statistics from it. -func (c *Component) wrapDecoder(d decoder.Decoder, useSrcAddrForExporterAddr bool) decoder.Decoder { - return &wrappedDecoder{ - c: c, - orig: d, - useSrcAddrForExporterAddr: useSrcAddrForExporterAddr, - } -} - -var decoders = map[string]decoder.NewDecoderFunc{ - "netflow": netflow.New, - "sflow": sflow.New, -} diff --git a/inlet/flow/decoder/config.go b/inlet/flow/decoder/config.go deleted file mode 100644 index 56bfc9e0..00000000 --- a/inlet/flow/decoder/config.go +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -package decoder - -// TimestampSource defines the method to use to extract the TimeReceived for the flows -type TimestampSource uint - -const ( - // TimestampSourceUDP tells the decoder to use the kernel time at which - // the UDP packet was received - TimestampSourceUDP TimestampSource = iota - // TimestampSourceNetflowPacket tells the decoder to use the timestamp - // from the router in the netflow packet - TimestampSourceNetflowPacket - // TimestampSourceNetflowFirstSwitched tells the decoder to use the timestamp - // from each flow "FIRST_SWITCHED" field - TimestampSourceNetflowFirstSwitched -) diff --git a/inlet/flow/decoder/sflow/root_test.go b/inlet/flow/decoder/sflow/root_test.go deleted file mode 100644 index 04144f5f..00000000 --- a/inlet/flow/decoder/sflow/root_test.go +++ /dev/null @@ -1,543 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Tchadel Icard -// SPDX-License-Identifier: AGPL-3.0-only - -package sflow - -import ( - "net" - "net/netip" - "path/filepath" - "testing" - - "akvorado/common/helpers" - "akvorado/common/reporter" - "akvorado/common/schema" - "akvorado/inlet/flow/decoder" -) - -func TestDecode(t *testing.T) { - r := reporter.NewMock(t) - sdecoder := New(r, decoder.Dependencies{Schema: schema.NewMock(t).EnableAllColumns()}, decoder.Option{}) - - // Send data - data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-1140.pcap")) - got := sdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) - if got == nil { - t.Fatalf("Decode() error on data") - } - expectedFlows := []*schema.FlowMessage{ - { - SamplingRate: 1024, - InIf: 27, - OutIf: 28, - SrcVlan: 100, - DstVlan: 100, - SrcAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:38"), - DstAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:39"), - ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 1500, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv6, - schema.ColumnProto: 6, - schema.ColumnSrcPort: 46026, - schema.ColumnDstPort: 22, - schema.ColumnSrcMAC: 40057391053392, - schema.ColumnDstMAC: 40057381862408, - schema.ColumnIPTTL: 64, - schema.ColumnIPTos: 0x8, - schema.ColumnIPv6FlowLabel: 0x68094, - schema.ColumnTCPFlags: 0x10, - }, - }, { - SamplingRate: 1024, - SrcAddr: netip.MustParseAddr("::ffff:104.26.8.24"), - DstAddr: netip.MustParseAddr("::ffff:45.90.161.46"), - ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), - NextHop: netip.MustParseAddr("::ffff:45.90.161.46"), - InIf: 49001, - OutIf: 25, - DstVlan: 100, - SrcAS: 13335, - DstAS: 39421, - SrcNetMask: 20, - DstNetMask: 27, - GotASPath: false, - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 421, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv4, - schema.ColumnProto: 6, - schema.ColumnSrcPort: 443, - schema.ColumnDstPort: 56876, - schema.ColumnSrcMAC: 216372595274807, - schema.ColumnDstMAC: 191421060163210, - schema.ColumnIPFragmentID: 0xa572, - schema.ColumnIPTTL: 59, - schema.ColumnTCPFlags: 0x18, - }, - }, { - SamplingRate: 1024, - SrcAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:38"), - DstAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:39"), - ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), - InIf: 27, - OutIf: 28, - SrcVlan: 100, - DstVlan: 100, - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 1500, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv6, - schema.ColumnProto: 6, - schema.ColumnSrcPort: 46026, - schema.ColumnDstPort: 22, - schema.ColumnSrcMAC: 40057391053392, - schema.ColumnDstMAC: 40057381862408, - schema.ColumnIPTTL: 64, - schema.ColumnIPTos: 0x8, - schema.ColumnIPv6FlowLabel: 0x68094, - schema.ColumnTCPFlags: 0x10, - }, - }, { - SamplingRate: 1024, - InIf: 28, - OutIf: 49001, - SrcVlan: 100, - SrcAS: 39421, - DstAS: 26615, - SrcAddr: netip.MustParseAddr("::ffff:45.90.161.148"), - DstAddr: netip.MustParseAddr("::ffff:191.87.91.27"), - ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), - NextHop: netip.MustParseAddr("::ffff:31.14.69.110"), - SrcNetMask: 27, - DstNetMask: 17, - GotASPath: true, - GotCommunities: true, - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 40, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv4, - schema.ColumnProto: 6, - schema.ColumnSrcPort: 55658, - schema.ColumnDstPort: 5555, - schema.ColumnSrcMAC: 138617863011056, - schema.ColumnDstMAC: 216372595274807, - schema.ColumnDstASPath: []uint32{203698, 6762, 26615}, - schema.ColumnDstCommunities: []uint64{2583495656, 2583495657, 4259880000, 4259880001, 4259900001}, - schema.ColumnIPFragmentID: 0xd431, - schema.ColumnIPTTL: 255, - schema.ColumnTCPFlags: 0x2, - }, - }, { - SamplingRate: 1024, - SrcAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:38"), - DstAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:39"), - ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), - InIf: 27, - OutIf: 28, - SrcVlan: 100, - DstVlan: 100, - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 1500, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv6, - schema.ColumnProto: 6, - schema.ColumnSrcPort: 46026, - schema.ColumnDstPort: 22, - schema.ColumnSrcMAC: 40057391053392, - schema.ColumnDstMAC: 40057381862408, - schema.ColumnIPTTL: 64, - schema.ColumnIPTos: 0x8, - schema.ColumnIPv6FlowLabel: 0x68094, - schema.ColumnTCPFlags: 0x10, - }, - }, - } - for _, f := range got { - f.TimeReceived = 0 - } - - if diff := helpers.Diff(got, expectedFlows); diff != "" { - t.Fatalf("Decode() (-got, +want):\n%s", diff) - } - gotMetrics := r.GetMetrics( - "akvorado_inlet_flow_decoder_sflow_", - "flows_total", - "sample_", - ) - expectedMetrics := map[string]string{ - `flows_total{agent="172.16.0.3",exporter="127.0.0.1",version="5"}`: "1", - `sample_records_sum{agent="172.16.0.3",exporter="127.0.0.1",type="FlowSample",version="5"}`: "14", - `sample_sum{agent="172.16.0.3",exporter="127.0.0.1",type="FlowSample",version="5"}`: "5", - } - if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { - t.Fatalf("Metrics after data (-got, +want):\n%s", diff) - } -} - -func TestDecodeInterface(t *testing.T) { - r := reporter.NewMock(t) - sdecoder := New(r, decoder.Dependencies{Schema: schema.NewMock(t)}, decoder.Option{}) - - t.Run("local interface", func(t *testing.T) { - // Send data - data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-local-interface.pcap")) - got := sdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) - if got == nil { - t.Fatalf("Decode() error on data") - } - expectedFlows := []*schema.FlowMessage{ - { - SamplingRate: 1024, - SrcAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:38"), - DstAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:39"), - ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), - InIf: 27, - OutIf: 0, // local interface - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 1500, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv6, - schema.ColumnProto: 6, - schema.ColumnSrcPort: 46026, - schema.ColumnDstPort: 22, - }, - }, - } - for _, f := range got { - f.TimeReceived = 0 - } - - if diff := helpers.Diff(got, expectedFlows); diff != "" { - t.Fatalf("Decode() (-got, +want):\n%s", diff) - } - }) - - t.Run("discard interface", func(t *testing.T) { - // Send data - data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-discard-interface.pcap")) - got := sdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) - if got == nil { - t.Fatalf("Decode() error on data") - } - expectedFlows := []*schema.FlowMessage{ - { - SamplingRate: 1024, - SrcAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:38"), - DstAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:39"), - ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), - InIf: 27, - OutIf: 0, // discard interface - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 1500, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv6, - schema.ColumnProto: 6, - schema.ColumnSrcPort: 46026, - schema.ColumnDstPort: 22, - schema.ColumnForwardingStatus: 128, - }, - }, - } - for _, f := range got { - f.TimeReceived = 0 - } - - if diff := helpers.Diff(got, expectedFlows); diff != "" { - t.Fatalf("Decode() (-got, +want):\n%s", diff) - } - }) - - t.Run("multiple interfaces", func(t *testing.T) { - // Send data - data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-multiple-interfaces.pcap")) - got := sdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) - if got == nil { - t.Fatalf("Decode() error on data") - } - expectedFlows := []*schema.FlowMessage{ - { - SamplingRate: 1024, - SrcAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:38"), - DstAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:39"), - ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), - InIf: 27, - OutIf: 0, // multiple interfaces - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 1500, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv6, - schema.ColumnProto: 6, - schema.ColumnSrcPort: 46026, - schema.ColumnDstPort: 22, - }, - }, - } - for _, f := range got { - f.TimeReceived = 0 - } - - if diff := helpers.Diff(got, expectedFlows); diff != "" { - t.Fatalf("Decode() (-got, +want):\n%s", diff) - } - }) -} - -func TestDecodeSamples(t *testing.T) { - r := reporter.NewMock(t) - sdecoder := New(r, decoder.Dependencies{Schema: schema.NewMock(t).EnableAllColumns()}, decoder.Option{}) - - t.Run("expanded flow sample", func(t *testing.T) { - // Send data - data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-sflow-expanded-sample.pcap")) - got := sdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) - if got == nil { - t.Fatalf("Decode() error on data") - } - expectedFlows := []*schema.FlowMessage{ - { - SamplingRate: 1000, - InIf: 29001, - OutIf: 1285816721, - SrcAddr: netip.MustParseAddr("::ffff:52.52.52.52"), - DstAddr: netip.MustParseAddr("::ffff:53.53.53.53"), - ExporterAddress: netip.MustParseAddr("::ffff:49.49.49.49"), - NextHop: netip.MustParseAddr("::ffff:54.54.54.54"), - SrcAS: 203476, - DstAS: 203361, - SrcVlan: 809, - GotASPath: true, - GotCommunities: true, - SrcNetMask: 32, - DstNetMask: 22, - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 104, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv4, - schema.ColumnProto: 6, - schema.ColumnSrcPort: 22, - schema.ColumnDstPort: 52237, - schema.ColumnDstASPath: []uint32{8218, 29605, 203361}, - schema.ColumnDstCommunities: []uint64{538574949, 1911619684, 1911669584, 1911671290}, - schema.ColumnTCPFlags: 0x18, - schema.ColumnIPFragmentID: 0xab4e, - schema.ColumnIPTTL: 61, - schema.ColumnIPTos: 0x8, - schema.ColumnSrcMAC: 0x948ed30a713b, - schema.ColumnDstMAC: 0x22421f4a9fcd, - }, - }, - } - for _, f := range got { - f.TimeReceived = 0 - } - - if diff := helpers.Diff(got, expectedFlows); diff != "" { - t.Fatalf("Decode() (-got, +want):\n%s", diff) - } - }) - - t.Run("flow sample with IPv4 data", func(t *testing.T) { - // Send data - data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-sflow-ipv4-data.pcap")) - got := sdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) - if got == nil { - t.Fatalf("Decode() error on data") - } - expectedFlows := []*schema.FlowMessage{ - { - SamplingRate: 256, - InIf: 0, - OutIf: 182, - DstVlan: 3001, - SrcAddr: netip.MustParseAddr("::ffff:50.50.50.50"), - DstAddr: netip.MustParseAddr("::ffff:51.51.51.51"), - ExporterAddress: netip.MustParseAddr("::ffff:49.49.49.49"), - GotASPath: false, - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 1344, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv4, - schema.ColumnProto: 17, - schema.ColumnSrcPort: 46622, - schema.ColumnDstPort: 58631, - schema.ColumnSrcMAC: 1094287164743, - schema.ColumnDstMAC: 1101091482116, - schema.ColumnIPFragmentID: 41647, - schema.ColumnIPTTL: 64, - }, - }, - } - for _, f := range got { - f.TimeReceived = 0 - } - - if diff := helpers.Diff(got, expectedFlows); diff != "" { - t.Fatalf("Decode() (-got, +want):\n%s", diff) - } - }) - - t.Run("flow sample with IPv4 raw packet", func(t *testing.T) { - data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-sflow-raw-ipv4.pcap")) - got := sdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) - if got == nil { - t.Fatalf("Decode() error on data") - } - expectedFlows := []*schema.FlowMessage{ - { - SamplingRate: 1, - InIf: 0, - OutIf: 2, - SrcAddr: netip.MustParseAddr("::ffff:69.58.92.107"), - DstAddr: netip.MustParseAddr("::ffff:92.222.186.1"), - ExporterAddress: netip.MustParseAddr("::ffff:172.19.64.116"), - GotASPath: false, - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 32, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv4, - schema.ColumnProto: 1, - schema.ColumnIPFragmentID: 4329, - schema.ColumnIPTTL: 64, - schema.ColumnIPTos: 8, - }, - }, { - SamplingRate: 1, - InIf: 0, - OutIf: 2, - SrcAddr: netip.MustParseAddr("::ffff:69.58.92.107"), - DstAddr: netip.MustParseAddr("::ffff:92.222.184.1"), - ExporterAddress: netip.MustParseAddr("::ffff:172.19.64.116"), - GotASPath: false, - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 32, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv4, - schema.ColumnProto: 1, - schema.ColumnIPFragmentID: 62945, - schema.ColumnIPTTL: 64, - schema.ColumnIPTos: 8, - }, - }, - } - for _, f := range got { - f.TimeReceived = 0 - } - - if diff := helpers.Diff(got, expectedFlows); diff != "" { - t.Fatalf("Decode() (-got, +want):\n%s", diff) - } - }) - - t.Run("flow sample with ICMPv4", func(t *testing.T) { - data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-icmpv4.pcap")) - got := sdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) - if got == nil { - t.Fatalf("Decode() error on data") - } - expectedFlows := []*schema.FlowMessage{ - { - SamplingRate: 1, - SrcAddr: netip.MustParseAddr("::ffff:203.0.113.4"), - DstAddr: netip.MustParseAddr("::ffff:203.0.113.5"), - ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), - GotASPath: false, - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 84, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv4, - schema.ColumnProto: 1, - schema.ColumnDstMAC: 0xd25b45ee5ecf, - schema.ColumnSrcMAC: 0xe2efc68f8cd4, - schema.ColumnICMPv4Type: 8, - // schema.ColumnICMPv4Code: 0, - schema.ColumnIPTTL: 64, - schema.ColumnIPFragmentID: 0x90c5, - }, - }, - } - for _, f := range got { - f.TimeReceived = 0 - } - - if diff := helpers.Diff(got, expectedFlows); diff != "" { - t.Fatalf("Decode() (-got, +want):\n%s", diff) - } - }) - - t.Run("flow sample with ICMPv6", func(t *testing.T) { - data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-icmpv6.pcap")) - got := sdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) - if got == nil { - t.Fatalf("Decode() error on data") - } - expectedFlows := []*schema.FlowMessage{ - { - SamplingRate: 1, - SrcAddr: netip.MustParseAddr("fe80::d05b:45ff:feee:5ecf"), - DstAddr: netip.MustParseAddr("2001:db8::"), - ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), - GotASPath: false, - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 72, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv6, - schema.ColumnProto: 58, - schema.ColumnSrcMAC: 0xd25b45ee5ecf, - schema.ColumnDstMAC: 0xe2efc68f8cd4, - schema.ColumnIPTTL: 255, - schema.ColumnICMPv6Type: 135, - // schema.ColumnICMPv6Code: 0, - }, - }, - } - for _, f := range got { - f.TimeReceived = 0 - } - - if diff := helpers.Diff(got, expectedFlows); diff != "" { - t.Fatalf("Decode() (-got, +want):\n%s", diff) - } - }) - - t.Run("flow sample with QinQ", func(t *testing.T) { - data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-qinq.pcap")) - got := sdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) - if got == nil { - t.Fatalf("Decode() error on data") - } - expectedFlows := []*schema.FlowMessage{ - { - SamplingRate: 4096, - InIf: 369098852, - OutIf: 369098851, - SrcVlan: 1493, - SrcAddr: netip.MustParseAddr("::ffff:49.49.49.2"), - DstAddr: netip.MustParseAddr("::ffff:49.49.49.109"), - ExporterAddress: netip.MustParseAddr("::ffff:172.17.128.58"), - GotASPath: false, - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 80, - schema.ColumnPackets: 1, - schema.ColumnEType: helpers.ETypeIPv4, - schema.ColumnProto: 6, - schema.ColumnSrcMAC: 0x4caea3520ff6, - schema.ColumnDstMAC: 0x000110621493, - schema.ColumnIPTTL: 62, - schema.ColumnIPFragmentID: 56159, - schema.ColumnTCPFlags: 16, - schema.ColumnSrcPort: 32017, - schema.ColumnDstPort: 443, - }, - }, - } - for _, f := range got { - f.TimeReceived = 0 - } - - if diff := helpers.Diff(got, expectedFlows); diff != "" { - t.Fatalf("Decode() (-got, +want):\n%s", diff) - } - }) -} diff --git a/inlet/flow/decoder/tests.go b/inlet/flow/decoder/tests.go deleted file mode 100644 index 488b887e..00000000 --- a/inlet/flow/decoder/tests.go +++ /dev/null @@ -1,36 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -//go:build !release - -package decoder - -import ( - "net/netip" - - "akvorado/common/schema" -) - -// DummyDecoder is a simple decoder producing flows from random data. -// The payload is copied in IfDescription -type DummyDecoder struct { - Schema *schema.Component -} - -// Decode returns uninteresting flow messages. -func (dc *DummyDecoder) Decode(in RawFlow) []*schema.FlowMessage { - exporterAddress, _ := netip.AddrFromSlice(in.Source.To16()) - f := &schema.FlowMessage{ - TimeReceived: uint64(in.TimeReceived.UTC().Unix()), - ExporterAddress: exporterAddress, - } - dc.Schema.ProtobufAppendVarint(f, schema.ColumnBytes, uint64(len(in.Payload))) - dc.Schema.ProtobufAppendVarint(f, schema.ColumnPackets, 1) - dc.Schema.ProtobufAppendBytes(f, schema.ColumnInIfDescription, in.Payload) - return []*schema.FlowMessage{f} -} - -// Name returns the original name. -func (dc *DummyDecoder) Name() string { - return "dummy" -} diff --git a/inlet/flow/decoder_test.go b/inlet/flow/decoder_test.go deleted file mode 100644 index 3e06444c..00000000 --- a/inlet/flow/decoder_test.go +++ /dev/null @@ -1,92 +0,0 @@ -// SPDX-FileCopyrightText: 2023 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -package flow - -import ( - "net" - "path/filepath" - "testing" - - "akvorado/common/helpers" - "akvorado/common/reporter" - "akvorado/common/schema" - "akvorado/inlet/flow/decoder" - "akvorado/inlet/flow/decoder/netflow" - "akvorado/inlet/flow/decoder/sflow" -) - -// The goal is to benchmark flow decoding + encoding to protobuf - -func BenchmarkDecodeEncodeNetflow(b *testing.B) { - schema.DisableDebug(b) - r := reporter.NewMock(b) - sch := schema.NewMock(b) - nfdecoder := netflow.New(r, decoder.Dependencies{Schema: sch}, decoder.Option{TimestampSource: decoder.TimestampSourceUDP}) - - template := helpers.ReadPcapL4(b, filepath.Join("decoder", "netflow", "testdata", "options-template.pcap")) - got := nfdecoder.Decode(decoder.RawFlow{Payload: template, Source: net.ParseIP("127.0.0.1")}) - if got == nil || len(got) != 0 { - b.Fatal("Decode() error on options template") - } - data := helpers.ReadPcapL4(b, filepath.Join("decoder", "netflow", "testdata", "options-data.pcap")) - got = nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) - if got == nil || len(got) != 0 { - b.Fatal("Decode() error on options data") - } - template = helpers.ReadPcapL4(b, filepath.Join("decoder", "netflow", "testdata", "template.pcap")) - got = nfdecoder.Decode(decoder.RawFlow{Payload: template, Source: net.ParseIP("127.0.0.1")}) - if got == nil || len(got) != 0 { - b.Fatal("Decode() error on template") - } - data = helpers.ReadPcapL4(b, filepath.Join("decoder", "netflow", "testdata", "data.pcap")) - - for _, withEncoding := range []bool{true, false} { - title := map[bool]string{ - true: "with encoding", - false: "without encoding", - }[withEncoding] - b.Run(title, func(b *testing.B) { - for b.Loop() { - got = nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) - if withEncoding { - for _, flow := range got { - sch.ProtobufMarshal(flow) - } - } - } - if got[0].ProtobufDebug != nil { - b.Fatal("debug is enabled") - } - }) - } -} - -func BenchmarkDecodeEncodeSflow(b *testing.B) { - schema.DisableDebug(b) - r := reporter.NewMock(b) - sch := schema.NewMock(b) - sdecoder := sflow.New(r, decoder.Dependencies{Schema: sch}, decoder.Option{TimestampSource: decoder.TimestampSourceUDP}) - data := helpers.ReadPcapL4(b, filepath.Join("decoder", "sflow", "testdata", "data-1140.pcap")) - - for _, withEncoding := range []bool{true, false} { - title := map[bool]string{ - true: "with encoding", - false: "without encoding", - }[withEncoding] - var got []*schema.FlowMessage - b.Run(title, func(b *testing.B) { - for b.Loop() { - got = sdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) - if withEncoding { - for _, flow := range got { - sch.ProtobufMarshal(flow) - } - } - } - if got[0].ProtobufDebug != nil { - b.Fatal("debug is enabled") - } - }) - } -} diff --git a/inlet/flow/input/file/config.go b/inlet/flow/input/file/config.go index 34449385..5cf95bbf 100644 --- a/inlet/flow/input/file/config.go +++ b/inlet/flow/input/file/config.go @@ -9,6 +9,9 @@ import "akvorado/inlet/flow/input" type Configuration struct { // Paths to use as input Paths []string `validate:"min=1,dive,required"` + // MaxFlows tell how many flows will be read before stopping production. 0 + // means to not stop. + MaxFlows uint } // DefaultConfiguration descrives the default configuration for file input. diff --git a/inlet/flow/input/file/root.go b/inlet/flow/input/file/root.go index c5ffb700..c3a433bc 100644 --- a/inlet/flow/input/file/root.go +++ b/inlet/flow/input/file/root.go @@ -13,9 +13,8 @@ import ( "gopkg.in/tomb.v2" "akvorado/common/daemon" + "akvorado/common/pb" "akvorado/common/reporter" - "akvorado/common/schema" - "akvorado/inlet/flow/decoder" "akvorado/inlet/flow/input" ) @@ -24,62 +23,66 @@ type Input struct { r *reporter.Reporter t tomb.Tomb config *Configuration - - ch chan []*schema.FlowMessage // channel to send flows to - decoder decoder.Decoder + send input.SendFunc } // New instantiate a new UDP listener from the provided configuration. -func (configuration *Configuration) New(r *reporter.Reporter, daemon daemon.Component, dec decoder.Decoder) (input.Input, error) { +func (configuration *Configuration) New(r *reporter.Reporter, daemon daemon.Component, send input.SendFunc) (input.Input, error) { if len(configuration.Paths) == 0 { return nil, errors.New("no paths provided for file input") } input := &Input{ - r: r, - config: configuration, - ch: make(chan []*schema.FlowMessage), - decoder: dec, + r: r, + config: configuration, + send: send, } daemon.Track(&input.t, "inlet/flow/input/file") return input, nil } -// Start starts reading the provided files to produce fake flows in a loop. -func (in *Input) Start() (<-chan []*schema.FlowMessage, error) { +// Start starts streaming files to produce flake flows in a loop. +func (in *Input) Start() error { in.r.Info().Msg("file input starting") in.t.Go(func() error { + count := uint(0) + payload := make([]byte, 9000) + flow := pb.RawFlow{} for idx := 0; true; idx++ { + if in.config.MaxFlows > 0 && count >= in.config.MaxFlows { + <-in.t.Dying() + return nil + } + path := in.config.Paths[idx%len(in.config.Paths)] data, err := os.ReadFile(path) if err != nil { in.r.Err(err).Str("path", path).Msg("unable to read path") return err } - flows := in.decoder.Decode(decoder.RawFlow{ - TimeReceived: time.Now(), - Payload: data, - Source: net.ParseIP("127.0.0.1"), - }) - if len(flows) == 0 { - continue - } + + // Mimic the way it works with UDP + n := copy(payload, data) + flow.Reset() + flow.TimeReceived = uint64(time.Now().Unix()) + flow.Payload = payload[:n] + flow.SourceAddress = net.ParseIP("127.0.0.1").To16() + + in.send("127.0.0.1", &flow) + count++ select { case <-in.t.Dying(): return nil - case in.ch <- flows: + default: } } return nil }) - return in.ch, nil + return nil } // Stop stops the UDP listeners func (in *Input) Stop() error { - defer func() { - close(in.ch) - in.r.Info().Msg("file input stopped") - }() + defer in.r.Info().Msg("file input stopped") in.t.Kill(nil) return in.t.Wait() } diff --git a/inlet/flow/input/file/root_test.go b/inlet/flow/input/file/root_test.go index 2b112404..3d2033b2 100644 --- a/inlet/flow/input/file/root_test.go +++ b/inlet/flow/input/file/root_test.go @@ -4,29 +4,62 @@ package file import ( + "net" "path" + "sync" "testing" "time" "akvorado/common/daemon" "akvorado/common/helpers" + "akvorado/common/pb" "akvorado/common/reporter" - "akvorado/common/schema" - "akvorado/inlet/flow/decoder" ) func TestFileInput(t *testing.T) { r := reporter.NewMock(t) configuration := DefaultConfiguration().(*Configuration) configuration.Paths = []string{path.Join("testdata", "file1.txt"), path.Join("testdata", "file2.txt")} - in, err := configuration.New(r, daemon.NewMock(t), &decoder.DummyDecoder{ - Schema: schema.NewMock(t), - }) + + done := make(chan bool) + expected := []pb.RawFlow{ + { + Payload: []byte("hello world!\n"), + SourceAddress: net.ParseIP("127.0.0.1").To16(), + }, { + Payload: []byte("bye bye\n"), + SourceAddress: net.ParseIP("127.0.0.1").To16(), + }, { + Payload: []byte("hello world!\n"), + SourceAddress: net.ParseIP("127.0.0.1").To16(), + }, + } + var mu sync.Mutex + got := []*pb.RawFlow{} + send := func(_ string, flow *pb.RawFlow) { + // Make a copy + payload := make([]byte, len(flow.Payload)) + copy(payload, flow.Payload) + newFlow := pb.RawFlow{ + TimeReceived: 0, + Payload: payload, + SourceAddress: flow.SourceAddress, + } + mu.Lock() + if len(got) < len(expected) { + got = append(got, &newFlow) + if len(got) == len(expected) { + close(done) + } + } + mu.Unlock() + } + + in, err := configuration.New(r, daemon.NewMock(t), send) if err != nil { t.Fatalf("New() error:\n%+v", err) } - ch, err := in.Start() - if err != nil { + if err := in.Start(); err != nil { t.Fatalf("Start() error:\n%+v", err) } defer func() { @@ -35,21 +68,12 @@ func TestFileInput(t *testing.T) { } }() - // Get it back - expected := []string{"hello world!\n", "bye bye\n", "hello world!\n"} - got := []string{} -out: - for range len(expected) { - select { - case got1 := <-ch: - for _, fl := range got1 { - got = append(got, string(fl.ProtobufDebug[schema.ColumnInIfDescription].([]byte))) - } - case <-time.After(50 * time.Millisecond): - break out + select { + case <-time.After(50 * time.Millisecond): + t.Fatal("timeout while waiting to receive flows") + case <-done: + if diff := helpers.Diff(got, expected); diff != "" { + t.Fatalf("Input data (-got, +want):\n%s", diff) } } - if diff := helpers.Diff(got, expected); diff != "" { - t.Fatalf("Input data (-got, +want):\n%s", diff) - } } diff --git a/inlet/flow/input/root.go b/inlet/flow/input/root.go index 02366623..466e5aee 100644 --- a/inlet/flow/input/root.go +++ b/inlet/flow/input/root.go @@ -6,21 +6,23 @@ package input import ( "akvorado/common/daemon" + "akvorado/common/pb" "akvorado/common/reporter" - "akvorado/common/schema" - "akvorado/inlet/flow/decoder" ) // Input is the interface any input should meet type Input interface { - // Start instructs an input to start producing flows on the returned channel. - Start() (<-chan []*schema.FlowMessage, error) + // Start instructs an input to start producing flows to be sent to Kafka component. + Start() error // Stop instructs the input to stop producing flows. Stop() error } +// SendFunc is a function to send a flow to Kafka +type SendFunc func(exporter string, flow *pb.RawFlow) + // Configuration defines the interface to instantiate an input module from its configuration. type Configuration interface { // New instantiates a new input from its configuration. - New(r *reporter.Reporter, daemon daemon.Component, dec decoder.Decoder) (Input, error) + New(r *reporter.Reporter, daemon daemon.Component, send SendFunc) (Input, error) } diff --git a/inlet/flow/input/udp/root.go b/inlet/flow/input/udp/root.go index 234c0543..d5bee048 100644 --- a/inlet/flow/input/udp/root.go +++ b/inlet/flow/input/udp/root.go @@ -15,9 +15,8 @@ import ( "gopkg.in/tomb.v2" "akvorado/common/daemon" + "akvorado/common/pb" "akvorado/common/reporter" - "akvorado/common/schema" - "akvorado/inlet/flow/decoder" "akvorado/inlet/flow/input" ) @@ -32,23 +31,19 @@ type Input struct { packets *reporter.CounterVec packetSizeSum *reporter.SummaryVec errors *reporter.CounterVec - outDrops *reporter.CounterVec inDrops *reporter.GaugeVec - decodedFlows *reporter.CounterVec } - address net.Addr // listening address, for testing purpoese - ch chan []*schema.FlowMessage // channel to send flows to - decoder decoder.Decoder // decoder to use + address net.Addr // listening address, for testing purpoese + send input.SendFunc // function to send to kafka } // New instantiate a new UDP listener from the provided configuration. -func (configuration *Configuration) New(r *reporter.Reporter, daemon daemon.Component, dec decoder.Decoder) (input.Input, error) { +func (configuration *Configuration) New(r *reporter.Reporter, daemon daemon.Component, send input.SendFunc) (input.Input, error) { input := &Input{ - r: r, - config: configuration, - ch: make(chan []*schema.FlowMessage, configuration.QueueSize), - decoder: dec, + r: r, + config: configuration, + send: send, } input.metrics.bytes = r.CounterVec( @@ -80,13 +75,6 @@ func (configuration *Configuration) New(r *reporter.Reporter, daemon daemon.Comp }, []string{"listener", "worker"}, ) - input.metrics.outDrops = r.CounterVec( - reporter.CounterOpts{ - Name: "out_dropped_packets_total", - Help: "Dropped packets due to internal queue full.", - }, - []string{"listener", "worker", "exporter"}, - ) input.metrics.inDrops = r.GaugeVec( reporter.GaugeOpts{ Name: "in_dropped_packets_total", @@ -94,20 +82,13 @@ func (configuration *Configuration) New(r *reporter.Reporter, daemon daemon.Comp }, []string{"listener", "worker"}, ) - input.metrics.decodedFlows = r.CounterVec( - reporter.CounterOpts{ - Name: "decoded_flows_total", - Help: "Number of flows decoded and written to the internal queue", - }, - []string{"listener", "worker", "exporter"}, - ) daemon.Track(&input.t, "inlet/flow/input/udp") return input, nil } // Start starts listening to the provided UDP socket and producing flows. -func (in *Input) Start() (<-chan []*schema.FlowMessage, error) { +func (in *Input) Start() error { in.r.Info().Str("listen", in.config.Listen).Msg("starting UDP input") // Listen to UDP port @@ -122,12 +103,12 @@ func (in *Input) Start() (<-chan []*schema.FlowMessage, error) { var err error listenAddr, err = net.ResolveUDPAddr("udp", in.config.Listen) if err != nil { - return nil, fmt.Errorf("unable to resolve %v: %w", in.config.Listen, err) + return fmt.Errorf("unable to resolve %v: %w", in.config.Listen, err) } } pconn, err := listenConfig.ListenPacket(in.t.Context(context.Background()), "udp", listenAddr.String()) if err != nil { - return nil, fmt.Errorf("unable to listen to %v: %w", listenAddr, err) + return fmt.Errorf("unable to listen to %v: %w", listenAddr, err) } udpConn := pconn.(*net.UDPConn) in.address = udpConn.LocalAddr() @@ -152,11 +133,13 @@ func (in *Input) Start() (<-chan []*schema.FlowMessage, error) { in.t.Go(func() error { payload := make([]byte, 9000) oob := make([]byte, oobLength) + flow := pb.RawFlow{} listen := in.config.Listen l := in.r.With(). Str("worker", worker). Str("listen", listen). Logger() + dying := in.t.Dying() errLogger := l.Sample(reporter.BurstSampler(time.Minute, 1)) for count := 0; ; count++ { n, oobn, _, source, err := conns[workerID].ReadMsgUDP(payload, oob) @@ -189,25 +172,17 @@ func (in *Input) Start() (<-chan []*schema.FlowMessage, error) { Inc() in.metrics.packetSizeSum.WithLabelValues(listen, worker, srcIP). Observe(float64(n)) - flows := in.decoder.Decode(decoder.RawFlow{ - TimeReceived: oobMsg.Received, - Payload: payload[:n], - Source: source.IP, - }) - if len(flows) == 0 { - continue - } + + flow.Reset() + flow.TimeReceived = uint64(oobMsg.Received.Unix()) + flow.Payload = payload[:n] + flow.SourceAddress = source.IP.To16() + in.send(srcIP, &flow) + select { - case <-in.t.Dying(): + case <-dying: return nil - case in.ch <- flows: - in.metrics.decodedFlows.WithLabelValues(listen, worker, srcIP). - Add(float64(len((flows)))) default: - errLogger.Warn().Msgf("dropping flow due to queue full (size %d)", - in.config.QueueSize) - in.metrics.outDrops.WithLabelValues(listen, worker, srcIP). - Inc() } } }) @@ -223,16 +198,13 @@ func (in *Input) Start() (<-chan []*schema.FlowMessage, error) { return nil }) - return in.ch, nil + return nil } // Stop stops the UDP listeners func (in *Input) Stop() error { l := in.r.With().Str("listen", in.config.Listen).Logger() - defer func() { - close(in.ch) - l.Info().Msg("UDP listener stopped") - }() + defer l.Info().Msg("UDP listener stopped") in.t.Kill(nil) return in.t.Wait() } diff --git a/inlet/flow/input/udp/root_test.go b/inlet/flow/input/udp/root_test.go index 4c6f3ad1..3c54047d 100644 --- a/inlet/flow/input/udp/root_test.go +++ b/inlet/flow/input/udp/root_test.go @@ -5,27 +5,61 @@ package udp import ( "net" - "net/netip" "testing" "time" "akvorado/common/daemon" "akvorado/common/helpers" + "akvorado/common/pb" "akvorado/common/reporter" - "akvorado/common/schema" - "akvorado/inlet/flow/decoder" ) func TestUDPInput(t *testing.T) { r := reporter.NewMock(t) configuration := DefaultConfiguration().(*Configuration) configuration.Listen = "127.0.0.1:0" - in, err := configuration.New(r, daemon.NewMock(t), &decoder.DummyDecoder{Schema: schema.NewMock(t)}) + + done := make(chan bool) + expected := &pb.RawFlow{ + SourceAddress: net.ParseIP("127.0.0.1").To16(), + Payload: []byte("hello world!"), + } + send := func(_ string, got *pb.RawFlow) { + expected.TimeReceived = got.TimeReceived + + delta := uint64(time.Now().UTC().Unix()) - got.TimeReceived + if delta > 1 { + t.Errorf("TimeReceived out of range: %d (now: %d)", got.TimeReceived, time.Now().UTC().Unix()) + } + if diff := helpers.Diff(got, expected); diff != "" { + t.Fatalf("Input data (-got, +want):\n%s", diff) + } + + // Check metrics + gotMetrics := r.GetMetrics("akvorado_inlet_flow_input_udp_") + expectedMetrics := map[string]string{ + `bytes_total{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0"}`: "12", + `packets_total{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0"}`: "1", + `in_dropped_packets_total{listener="127.0.0.1:0",worker="0"}`: "0", + `summary_size_bytes_count{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0"}`: "1", + `summary_size_bytes_sum{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0"}`: "12", + `summary_size_bytes{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0",quantile="0.5"}`: "12", + `summary_size_bytes{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0",quantile="0.9"}`: "12", + `summary_size_bytes{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0",quantile="0.99"}`: "12", + } + if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { + t.Fatalf("Input metrics (-got, +want):\n%s", diff) + } + + close(done) + + } + + in, err := configuration.New(r, daemon.NewMock(t), send) if err != nil { t.Fatalf("New() error:\n%+v", err) } - ch, err := in.Start() - if err != nil { + if err := in.Start(); err != nil { t.Fatalf("Start() error:\n%+v", err) } defer func() { @@ -46,103 +80,9 @@ func TestUDPInput(t *testing.T) { } // Get it back - var got []*schema.FlowMessage select { - case got = <-ch: - if len(got) == 0 { - t.Fatalf("empty decoded flows received") - } case <-time.After(20 * time.Millisecond): t.Fatal("no decoded flows received") - } - - delta := uint64(time.Now().UTC().Unix()) - got[0].TimeReceived - if delta > 1 { - t.Errorf("TimeReceived out of range: %d (now: %d)", got[0].TimeReceived, time.Now().UTC().Unix()) - } - expected := []*schema.FlowMessage{ - { - TimeReceived: got[0].TimeReceived, - ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnBytes: 12, - schema.ColumnPackets: 1, - schema.ColumnInIfDescription: []byte("hello world!"), - }, - }, - } - if diff := helpers.Diff(got, expected); diff != "" { - t.Fatalf("Input data (-got, +want):\n%s", diff) - } - - // Check metrics - gotMetrics := r.GetMetrics("akvorado_inlet_flow_input_udp_") - expectedMetrics := map[string]string{ - `bytes_total{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0"}`: "12", - `decoded_flows_total{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0"}`: "1", - `packets_total{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0"}`: "1", - `in_dropped_packets_total{listener="127.0.0.1:0",worker="0"}`: "0", - `summary_size_bytes_count{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0"}`: "1", - `summary_size_bytes_sum{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0"}`: "12", - `summary_size_bytes{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0",quantile="0.5"}`: "12", - `summary_size_bytes{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0",quantile="0.9"}`: "12", - `summary_size_bytes{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0",quantile="0.99"}`: "12", - } - if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { - t.Fatalf("Input metrics (-got, +want):\n%s", diff) - } -} - -func TestOverflow(t *testing.T) { - r := reporter.NewMock(t) - configuration := DefaultConfiguration().(*Configuration) - configuration.Listen = "127.0.0.1:0" - configuration.QueueSize = 1 - in, err := configuration.New(r, daemon.NewMock(t), &decoder.DummyDecoder{ - Schema: schema.NewMock(t), - }) - if err != nil { - t.Fatalf("New() error:\n%+v", err) - } - _, err = in.Start() - if err != nil { - t.Fatalf("Start() error:\n%+v", err) - } - defer func() { - if err := in.Stop(); err != nil { - t.Fatalf("Stop() error:\n%+v", err) - } - }() - - // Connect - conn, err := net.Dial("udp", in.(*Input).address.String()) - if err != nil { - t.Fatalf("Dial() error:\n%+v", err) - } - - // Send data - for range 10 { - if _, err := conn.Write([]byte("hello world!")); err != nil { - t.Fatalf("Write() error:\n%+v", err) - } - } - time.Sleep(20 * time.Millisecond) - - // Check metrics - gotMetrics := r.GetMetrics("akvorado_inlet_flow_input_udp_") - expectedMetrics := map[string]string{ - `bytes_total{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0"}`: "120", - `decoded_flows_total{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0"}`: "1", - `in_dropped_packets_total{listener="127.0.0.1:0",worker="0"}`: "0", - `out_dropped_packets_total{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0"}`: "9", - `packets_total{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0"}`: "10", - `summary_size_bytes_count{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0"}`: "10", - `summary_size_bytes_sum{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0"}`: "120", - `summary_size_bytes{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0",quantile="0.5"}`: "12", - `summary_size_bytes{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0",quantile="0.9"}`: "12", - `summary_size_bytes{exporter="127.0.0.1",listener="127.0.0.1:0",worker="0",quantile="0.99"}`: "12", - } - if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { - t.Fatalf("Input metrics (-got, +want):\n%s", diff) + case <-done: } } diff --git a/inlet/flow/rate.go b/inlet/flow/rate.go deleted file mode 100644 index cbc91547..00000000 --- a/inlet/flow/rate.go +++ /dev/null @@ -1,57 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -package flow - -import ( - "time" - - "akvorado/common/schema" - - "golang.org/x/time/rate" -) - -type limiter struct { - l *rate.Limiter - dropped uint64 // dropped during the current second - total uint64 // total during the current second - dropRate float64 // drop rate during the last second - currentTick time.Time -} - -// allowMessages tell if we can transmit the provided messages, -// depending on the rate limiter configuration. If yes, their sampling -// rate may be modified to match current drop rate. -func (c *Component) allowMessages(fmsgs []*schema.FlowMessage) bool { - count := len(fmsgs) - if c.config.RateLimit == 0 || count == 0 { - return true - } - exporter := fmsgs[0].ExporterAddress - exporterLimiter, ok := c.limiters[exporter] - if !ok { - exporterLimiter = &limiter{ - l: rate.NewLimiter(rate.Limit(c.config.RateLimit), int(c.config.RateLimit/10)), - } - c.limiters[exporter] = exporterLimiter - } - now := time.Now() - tick := now.Truncate(200 * time.Millisecond) // we use a 200-millisecond resolution - if exporterLimiter.currentTick.UnixMilli() != tick.UnixMilli() { - exporterLimiter.dropRate = float64(exporterLimiter.dropped) / float64(exporterLimiter.total) - exporterLimiter.dropped = 0 - exporterLimiter.total = 0 - exporterLimiter.currentTick = tick - } - exporterLimiter.total += uint64(count) - if !exporterLimiter.l.AllowN(now, count) { - exporterLimiter.dropped += uint64(count) - return false - } - if exporterLimiter.dropRate > 0 { - for _, flow := range fmsgs { - flow.SamplingRate *= uint32(1 / (1 - exporterLimiter.dropRate)) - } - } - return true -} diff --git a/inlet/flow/root.go b/inlet/flow/root.go index 82bc6b95..387e1307 100644 --- a/inlet/flow/root.go +++ b/inlet/flow/root.go @@ -1,23 +1,21 @@ // SPDX-FileCopyrightText: 2022 Free Mobile // SPDX-License-Identifier: AGPL-3.0-only -// Package flow handle incoming flows (currently Netflow v9 and IPFIX). +// Package flow handle incoming Netflow/IPFIX/sflow flows. package flow import ( "errors" - "fmt" - "net/http" - "net/netip" + "google.golang.org/protobuf/proto" "gopkg.in/tomb.v2" "akvorado/common/daemon" "akvorado/common/httpserver" + "akvorado/common/pb" "akvorado/common/reporter" - "akvorado/common/schema" - "akvorado/inlet/flow/decoder" "akvorado/inlet/flow/input" + "akvorado/inlet/kafka" ) // Component represents the flow component. @@ -27,17 +25,6 @@ type Component struct { t tomb.Tomb config Configuration - metrics struct { - decoderStats *reporter.CounterVec - decoderErrors *reporter.CounterVec - } - - // Channel for sending flows out of the package. - outgoingFlows chan *schema.FlowMessage - - // Per-exporter rate-limiters - limiters map[netip.Addr]*limiter - // Inputs inputs []input.Input } @@ -46,7 +33,7 @@ type Component struct { type Dependencies struct { Daemon daemon.Component HTTP *httpserver.Component - Schema *schema.Component + Kafka *kafka.Component } // New creates a new flow component. @@ -56,99 +43,50 @@ func New(r *reporter.Reporter, configuration Configuration, dependencies Depende } c := Component{ - r: r, - d: &dependencies, - config: configuration, - outgoingFlows: make(chan *schema.FlowMessage), - limiters: make(map[netip.Addr]*limiter), - inputs: make([]input.Input, len(configuration.Inputs)), - } - - // Initialize decoders (at most once each) - alreadyInitialized := map[string]decoder.Decoder{} - decs := make([]decoder.Decoder, len(configuration.Inputs)) - for idx, input := range c.config.Inputs { - dec, ok := alreadyInitialized[input.Decoder] - if ok { - decs[idx] = dec - continue - } - decoderfunc, ok := decoders[input.Decoder] - if !ok { - return nil, fmt.Errorf("unknown decoder %q", input.Decoder) - } - dec = decoderfunc(r, decoder.Dependencies{Schema: c.d.Schema}, decoder.Option{TimestampSource: input.TimestampSource}) - alreadyInitialized[input.Decoder] = dec - decs[idx] = c.wrapDecoder(dec, input.UseSrcAddrForExporterAddr) + r: r, + d: &dependencies, + config: configuration, + inputs: make([]input.Input, len(configuration.Inputs)), } // Initialize inputs for idx, input := range c.config.Inputs { var err error - c.inputs[idx], err = input.Config.New(r, c.d.Daemon, decs[idx]) + c.inputs[idx], err = input.Config.New(r, c.d.Daemon, c.Send(input)) if err != nil { return nil, err } } - // Metrics - c.metrics.decoderStats = c.r.CounterVec( - reporter.CounterOpts{ - Name: "decoder_flows_total", - Help: "Decoder processed count.", - }, - []string{"name"}, - ) - c.metrics.decoderErrors = c.r.CounterVec( - reporter.CounterOpts{ - Name: "decoder_errors_total", - Help: "Decoder processed error count.", - }, - []string{"name"}, - ) - c.d.Daemon.Track(&c.t, "inlet/flow") - c.d.HTTP.AddHandler("/api/v0/inlet/flow/schema.proto", - http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "text/plain") - w.Write([]byte(c.d.Schema.ProtobufDefinition())) - })) - return &c, nil } -// Flows returns a channel to receive flows. -func (c *Component) Flows() <-chan *schema.FlowMessage { - return c.outgoingFlows +// Send sends a raw flow to Kafka. +func (c *Component) Send(config InputConfiguration) input.SendFunc { + return func(exporter string, flow *pb.RawFlow) { + flow.TimestampSource = config.TimestampSource + flow.Decoder = config.Decoder + flow.UseSourceAddress = config.UseSrcAddrForExporterAddr + if bytes, err := proto.Marshal(flow); err == nil { + c.d.Kafka.Send(exporter, bytes) + } + } } // Start starts the flow component. func (c *Component) Start() error { for _, input := range c.inputs { - ch, err := input.Start() + err := input.Start() stopper := input.Stop if err != nil { return err } c.t.Go(func() error { - defer stopper() - for { - select { - case <-c.t.Dying(): - return nil - case fmsgs := <-ch: - if c.allowMessages(fmsgs) { - for _, fmsg := range fmsgs { - select { - case <-c.t.Dying(): - return nil - case c.outgoingFlows <- fmsg: - } - } - } - } - } + <-c.t.Dying() + stopper() + return nil }) } return nil @@ -156,10 +94,7 @@ func (c *Component) Start() error { // Stop stops the flow component func (c *Component) Stop() error { - defer func() { - close(c.outgoingFlows) - c.r.Info().Msg("flow component stopped") - }() + defer c.r.Info().Msg("flow component stopped") c.r.Info().Msg("stopping flow component") c.t.Kill(nil) return c.t.Wait() diff --git a/inlet/flow/root_test.go b/inlet/flow/root_test.go index 8e30e728..5623be95 100644 --- a/inlet/flow/root_test.go +++ b/inlet/flow/root_test.go @@ -4,122 +4,89 @@ package flow import ( + "bytes" "fmt" - "os" "path" "runtime" "testing" "time" + "akvorado/common/daemon" "akvorado/common/helpers" + "akvorado/common/httpserver" + "akvorado/common/pb" "akvorado/common/reporter" "akvorado/inlet/flow/input/file" + "akvorado/inlet/kafka" + + "github.com/IBM/sarama" ) func TestFlow(t *testing.T) { - var nominalRate int _, src, _, _ := runtime.Caller(0) - base := path.Join(path.Dir(src), "decoder", "netflow", "testdata") - outDir := t.TempDir() - outFiles := []string{} - for idx, f := range []string{ - "options-template.pcap", - "options-data.pcap", - "template.pcap", - "data.pcap", "data.pcap", "data.pcap", "data.pcap", - "data.pcap", "data.pcap", "data.pcap", "data.pcap", - "data.pcap", "data.pcap", "data.pcap", "data.pcap", - "data.pcap", "data.pcap", "data.pcap", "data.pcap", - } { - outFile := path.Join(outDir, fmt.Sprintf("data-%d", idx)) - err := os.WriteFile(outFile, helpers.ReadPcapL4(t, path.Join(base, f)), 0o666) - if err != nil { - t.Fatalf("WriteFile(%q) error:\n%+v", outFile, err) - } - outFiles = append(outFiles, outFile) + base := path.Join(path.Dir(src), "input", "file", "testdata") + paths := []string{ + path.Join(base, "file1.txt"), + path.Join(base, "file2.txt"), } inputs := []InputConfiguration{ { - Decoder: "netflow", Config: &file.Configuration{ - Paths: outFiles, + Paths: paths, + MaxFlows: 100, }, }, } - for retry := 2; retry >= 0; retry-- { - // Without rate limiting - { - r := reporter.NewMock(t) - config := DefaultConfiguration() - config.Inputs = inputs - c := NewMock(t, r, config) + r := reporter.NewMock(t) + config := DefaultConfiguration() + config.Inputs = inputs - // Receive flows - now := time.Now() - for range 1000 { - select { - case <-c.Flows(): - case <-time.After(100 * time.Millisecond): - t.Fatalf("no flow received") + producer, mockProducer := kafka.NewMock(t, r, kafka.DefaultConfiguration()) + done := make(chan bool) + for i := range 100 { + mockProducer.ExpectInputWithMessageCheckerFunctionAndSucceed(func(got *sarama.ProducerMessage) error { + if i == 99 { + defer close(done) + } + expected := sarama.ProducerMessage{ + Topic: fmt.Sprintf("flows-v%d", pb.Version), + Key: got.Key, + Value: got.Value, + Partition: got.Partition, + } + if diff := helpers.Diff(got, expected); diff != "" { + t.Fatalf("Send() (-got, +want):\n%s", diff) + } + val, _ := got.Value.Encode() + if i%2 == 0 { + if !bytes.Contains(val, []byte("hello world!")) { + t.Fatalf("Send() did not return %q", "hello world!") + } + } else { + if !bytes.Contains(val, []byte("bye bye")) { + t.Fatalf("Send() did not return %q", "bye bye") } } - elapsed := time.Now().Sub(now) - t.Logf("Elapsed time for 1000 messages is %s", elapsed) - nominalRate = int(1000 * (time.Second / elapsed)) - } + return nil + }) + } - // With rate limiting - if runtime.GOOS == "Linux" { - r := reporter.NewMock(t) - config := DefaultConfiguration() - config.RateLimit = 1000 - config.Inputs = inputs - c := NewMock(t, r, config) + c, err := New(r, config, Dependencies{ + Daemon: daemon.NewMock(t), + HTTP: httpserver.NewMock(t, r), + Kafka: producer, + }) + if err != nil { + t.Fatalf("New() error:\n%+v", err) + } + helpers.StartStop(t, c) - // Receive flows - twoSeconds := time.After(2 * time.Second) - count := 0 - outer1: - for { - select { - case <-c.Flows(): - count++ - case <-twoSeconds: - break outer1 - } - } - t.Logf("During the first two seconds, got %d flows", count) - - if count > 2200 || count < 2000 { - t.Fatalf("Got %d flows instead of 2100 (burst included)", count) - } - - if nominalRate == 0 { - return - } - select { - case flow := <-c.Flows(): - // This is hard to estimate the number of - // flows we should have got. We use the - // nominal rate but it was done with rate - // limiting disabled (so less code). - // Therefore, we are super conservative on the - // upper limit of the sampling rate. However, - // the lower limit should be OK. - t.Logf("Nominal rate was %d/second", nominalRate) - expectedRate := uint64(30000 / 1000 * nominalRate) - if flow.SamplingRate > uint32(1000*expectedRate/100) || flow.SamplingRate < uint32(70*expectedRate/100) { - if retry > 0 { - continue - } - t.Fatalf("Sampling rate is %d, expected %d", flow.SamplingRate, expectedRate) - } - case <-time.After(100 * time.Millisecond): - t.Fatalf("no flow received") - } - break - } + // Wait for flows + select { + case <-done: + case <-time.After(time.Second): + t.Fatalf("flows not received") } } diff --git a/inlet/flow/schemas_test.go b/inlet/flow/schemas_test.go deleted file mode 100644 index 7c2a0194..00000000 --- a/inlet/flow/schemas_test.go +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -package flow - -import ( - "testing" - - "akvorado/common/helpers" - "akvorado/common/reporter" -) - -func TestHTTPEndpoints(t *testing.T) { - r := reporter.NewMock(t) - c := NewMock(t, r, DefaultConfiguration()) - - cases := helpers.HTTPEndpointCases{ - { - URL: "/api/v0/inlet/flow/schema.proto", - ContentType: "text/plain", - FirstLines: []string{ - "", - `syntax = "proto3";`, - }, - }, - } - - helpers.TestHTTPEndpoints(t, c.d.HTTP.LocalAddr(), cases) -} diff --git a/inlet/flow/tests.go b/inlet/flow/tests.go deleted file mode 100644 index 86085b96..00000000 --- a/inlet/flow/tests.go +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Free Mobile -// SPDX-License-Identifier: AGPL-3.0-only - -//go:build !release - -package flow - -import ( - "testing" - - "akvorado/common/daemon" - "akvorado/common/helpers" - "akvorado/common/httpserver" - "akvorado/common/reporter" - "akvorado/common/schema" - "akvorado/inlet/flow/input/udp" -) - -// NewMock creates a new flow importer listening on a random port. It -// is autostarted. -func NewMock(t *testing.T, r *reporter.Reporter, config Configuration) *Component { - t.Helper() - if config.Inputs == nil { - config.Inputs = []InputConfiguration{ - { - Decoder: "netflow", - Config: &udp.Configuration{ - Listen: "127.0.0.1:0", - QueueSize: 10, - }, - }, - } - } - c, err := New(r, config, Dependencies{ - Daemon: daemon.NewMock(t), - HTTP: httpserver.NewMock(t, r), - Schema: schema.NewMock(t), - }) - if err != nil { - t.Fatalf("New() error:\n%+v", err) - } - helpers.StartStop(t, c) - return c -} - -// Inject inject the provided flow message, as if it was received. -func (c *Component) Inject(fmsg *schema.FlowMessage) { - c.outgoingFlows <- fmsg -} diff --git a/inlet/kafka/config.go b/inlet/kafka/config.go index 933ff434..d14df86b 100644 --- a/inlet/kafka/config.go +++ b/inlet/kafka/config.go @@ -34,7 +34,7 @@ func DefaultConfiguration() Configuration { Configuration: kafka.DefaultConfiguration(), FlushInterval: time.Second, FlushBytes: int(sarama.MaxRequestSize) - 1, - MaxMessageBytes: 1000000, + MaxMessageBytes: 1_000_000, CompressionCodec: CompressionCodec(sarama.CompressionNone), QueueSize: 32, } diff --git a/inlet/kafka/functional_test.go b/inlet/kafka/functional_test.go index eecbfe79..d175ce83 100644 --- a/inlet/kafka/functional_test.go +++ b/inlet/kafka/functional_test.go @@ -15,22 +15,22 @@ import ( "akvorado/common/daemon" "akvorado/common/helpers" "akvorado/common/kafka" + "akvorado/common/pb" "akvorado/common/reporter" - "akvorado/common/schema" ) func TestRealKafka(t *testing.T) { client, brokers := kafka.SetupKafkaBroker(t) topicName := fmt.Sprintf("test-topic-%d", rand.Int()) + expectedTopicName := fmt.Sprintf("%s-v%d", topicName, pb.Version) configuration := DefaultConfiguration() configuration.Topic = topicName configuration.Brokers = brokers configuration.Version = kafka.Version(sarama.V2_8_1_0) configuration.FlushInterval = 100 * time.Millisecond - expectedTopicName := fmt.Sprintf("%s-%s", topicName, schema.NewMock(t).ProtobufMessageHash()) r := reporter.NewMock(t) - c, err := New(r, configuration, Dependencies{Daemon: daemon.NewMock(t), Schema: schema.NewMock(t)}) + c, err := New(r, configuration, Dependencies{Daemon: daemon.NewMock(t)}) if err != nil { t.Fatalf("New() error:\n%+v", err) } diff --git a/inlet/kafka/root.go b/inlet/kafka/root.go index ea0643a4..f4feb60f 100644 --- a/inlet/kafka/root.go +++ b/inlet/kafka/root.go @@ -16,8 +16,8 @@ import ( "akvorado/common/daemon" "akvorado/common/kafka" + "akvorado/common/pb" "akvorado/common/reporter" - "akvorado/common/schema" ) // Component represents the Kafka exporter. @@ -27,8 +27,8 @@ type Component struct { t tomb.Tomb config Configuration - kafkaTopic string kafkaConfig *sarama.Config + kafkaTopic string kafkaProducer sarama.AsyncProducer createKafkaProducer func() (sarama.AsyncProducer, error) metrics metrics @@ -37,7 +37,6 @@ type Component struct { // Dependencies define the dependencies of the Kafka exporter. type Dependencies struct { Daemon daemon.Component - Schema *schema.Component } // New creates a new Kafka exporter component. @@ -66,7 +65,7 @@ func New(reporter *reporter.Reporter, configuration Configuration, dependencies config: configuration, kafkaConfig: kafkaConfig, - kafkaTopic: fmt.Sprintf("%s-%s", configuration.Topic, dependencies.Schema.ProtobufMessageHash()), + kafkaTopic: fmt.Sprintf("%s-v%d", configuration.Topic, pb.Version), } c.initMetrics() c.createKafkaProducer = func() (sarama.AsyncProducer, error) { @@ -95,9 +94,10 @@ func (c *Component) Start() error { c.t.Go(func() error { defer kafkaProducer.Close() errLogger := c.r.Sample(reporter.BurstSampler(10*time.Second, 3)) + dying := c.t.Dying() for { select { - case <-c.t.Dying(): + case <-dying: c.r.Debug().Msg("stop error logger") return nil case msg := <-kafkaProducer.Errors(): diff --git a/inlet/kafka/root_test.go b/inlet/kafka/root_test.go index a01639ce..e5b3274a 100644 --- a/inlet/kafka/root_test.go +++ b/inlet/kafka/root_test.go @@ -14,8 +14,8 @@ import ( "akvorado/common/daemon" "akvorado/common/helpers" + "akvorado/common/pb" "akvorado/common/reporter" - "akvorado/common/schema" ) func TestKafka(t *testing.T) { @@ -26,8 +26,9 @@ func TestKafka(t *testing.T) { received := make(chan bool) mockProducer.ExpectInputWithMessageCheckerFunctionAndSucceed(func(got *sarama.ProducerMessage) error { defer close(received) + topic := fmt.Sprintf("flows-v%d", pb.Version) expected := sarama.ProducerMessage{ - Topic: fmt.Sprintf("flows-%s", c.d.Schema.ProtobufMessageHash()), + Topic: topic, Key: got.Key, Value: sarama.ByteEncoder("hello world!"), Partition: got.Partition, @@ -51,9 +52,9 @@ func TestKafka(t *testing.T) { time.Sleep(10 * time.Millisecond) gotMetrics := r.GetMetrics("akvorado_inlet_kafka_") expectedMetrics := map[string]string{ - `sent_bytes_total{exporter="127.0.0.1"}`: "26", - fmt.Sprintf(`errors_total{error="kafka: Failed to produce message to topic flows-%s: noooo"}`, c.d.Schema.ProtobufMessageHash()): "1", - `sent_messages_total{exporter="127.0.0.1"}`: "2", + `sent_bytes_total{exporter="127.0.0.1"}`: "26", + `errors_total{error="kafka: Failed to produce message to topic flows-v5: noooo"}`: "1", + `sent_messages_total{exporter="127.0.0.1"}`: "2", } if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { t.Fatalf("Metrics (-got, +want):\n%s", diff) @@ -62,7 +63,7 @@ func TestKafka(t *testing.T) { func TestKafkaMetrics(t *testing.T) { r := reporter.NewMock(t) - c, err := New(r, DefaultConfiguration(), Dependencies{Daemon: daemon.NewMock(t), Schema: schema.NewMock(t)}) + c, err := New(r, DefaultConfiguration(), Dependencies{Daemon: daemon.NewMock(t)}) if err != nil { t.Fatalf("New() error:\n%+v", err) } diff --git a/inlet/kafka/tests.go b/inlet/kafka/tests.go index b4da2762..897fc723 100644 --- a/inlet/kafka/tests.go +++ b/inlet/kafka/tests.go @@ -14,7 +14,6 @@ import ( "akvorado/common/daemon" "akvorado/common/helpers" "akvorado/common/reporter" - "akvorado/common/schema" ) // NewMock creates a new Kafka component with a mocked Kafka. It will @@ -23,7 +22,6 @@ func NewMock(t *testing.T, reporter *reporter.Reporter, configuration Configurat t.Helper() c, err := New(reporter, configuration, Dependencies{ Daemon: daemon.NewMock(t), - Schema: schema.NewMock(t), }) if err != nil { t.Fatalf("New() error:\n%+v", err) diff --git a/orchestrator/clickhouse/config.go b/orchestrator/clickhouse/config.go index f2089a95..7c7f4780 100644 --- a/orchestrator/clickhouse/config.go +++ b/orchestrator/clickhouse/config.go @@ -10,7 +10,6 @@ import ( "akvorado/common/remotedatasourcefetcher" "akvorado/common/helpers" - "akvorado/common/kafka" "github.com/go-viper/mapstructure/v2" ) @@ -19,8 +18,6 @@ import ( type Configuration struct { // SkipMigrations tell if we should skip migrations. SkipMigrations bool - // Kafka describes Kafka-specific configuration - Kafka KafkaConfiguration // Resolutions describe the various resolutions to use to // store data and the associated TTLs. Resolutions []ResolutionConfiguration `validate:"min=1,dive"` @@ -67,26 +64,9 @@ type ResolutionConfiguration struct { TTL time.Duration `validate:"isdefault|min=1h"` } -// KafkaConfiguration describes Kafka-specific configuration -type KafkaConfiguration struct { - kafka.Configuration `mapstructure:",squash" yaml:"-,inline"` - // Consumers tell how many consumers to use to poll data from Kafka - Consumers int `validate:"min=1"` - // GroupName defines the Kafka consumers group used to poll data from topic, - // shared between all Consumers. - GroupName string - // EngineSettings allows one to set arbitrary settings for Kafka engine in - // ClickHouse. - EngineSettings []string -} - // DefaultConfiguration represents the default configuration for the ClickHouse configurator. func DefaultConfiguration() Configuration { return Configuration{ - Kafka: KafkaConfiguration{ - Consumers: 1, - GroupName: "clickhouse", - }, Resolutions: []ResolutionConfiguration{ {0, 15 * 24 * time.Hour}, // 15 days {time.Minute, 7 * 24 * time.Hour}, // 7 days @@ -141,6 +121,9 @@ func NetworkAttributesUnmarshallerHook() mapstructure.DecodeHookFunc { func init() { helpers.RegisterMapstructureUnmarshallerHook(helpers.SubnetMapUnmarshallerHook[NetworkAttributes]()) helpers.RegisterMapstructureUnmarshallerHook(NetworkAttributesUnmarshallerHook()) - helpers.RegisterMapstructureDeprecatedFields[Configuration]("SystemLogTTL", "PrometheusEndpoint") + helpers.RegisterMapstructureDeprecatedFields[Configuration]( + "SystemLogTTL", + "PrometheusEndpoint", + "Kafka") helpers.RegisterSubnetMapValidation[NetworkAttributes]() } diff --git a/orchestrator/clickhouse/config_test.go b/orchestrator/clickhouse/config_test.go index 431bfb04..dcf56199 100644 --- a/orchestrator/clickhouse/config_test.go +++ b/orchestrator/clickhouse/config_test.go @@ -88,7 +88,6 @@ func TestNetworkNamesUnmarshalHook(t *testing.T) { func TestDefaultConfiguration(t *testing.T) { config := DefaultConfiguration() - config.Kafka.Topic = "flow" if err := helpers.Validate.Struct(config); err != nil { t.Fatalf("validate.Struct() error:\n%+v", err) } diff --git a/orchestrator/clickhouse/migrations.go b/orchestrator/clickhouse/migrations.go index cf528489..2a3f302d 100644 --- a/orchestrator/clickhouse/migrations.go +++ b/orchestrator/clickhouse/migrations.go @@ -45,17 +45,12 @@ func (c *Component) migrateDatabase() error { } // Grab some information about the database - var threads uint8 var version string - row := c.d.ClickHouse.QueryRow(ctx, `SELECT getSetting('max_threads'), version()`) - if err := row.Scan(&threads, &version); err != nil { + row := c.d.ClickHouse.QueryRow(ctx, `SELECT version()`) + if err := row.Scan(&version); err != nil { c.r.Err(err).Msg("unable to parse database settings") return fmt.Errorf("unable to parse database settings: %w", err) } - if c.config.Kafka.Consumers > int(threads) { - c.r.Warn().Msgf("too many consumers requested, capping to %d", threads) - c.config.Kafka.Consumers = int(threads) - } if err := validateVersion(version); err != nil { return fmt.Errorf("incorrect ClickHouse version: %w", err) } @@ -162,12 +157,6 @@ func (c *Component) migrateDatabase() error { c.createExportersConsumerView, c.createRawFlowsTable, c.createRawFlowsConsumerView, - c.createRawFlowsErrors, - func(ctx context.Context) error { - return c.createDistributedTable(ctx, "flows_raw_errors") - }, - c.createRawFlowsErrorsConsumerView, - c.deleteOldRawFlowsErrorsView, ) if err != nil { return err diff --git a/orchestrator/clickhouse/migrations_helpers.go b/orchestrator/clickhouse/migrations_helpers.go index cbee3597..636c7355 100644 --- a/orchestrator/clickhouse/migrations_helpers.go +++ b/orchestrator/clickhouse/migrations_helpers.go @@ -281,36 +281,18 @@ CREATE MATERIALIZED VIEW exporters_consumer TO %s AS %s // createRawFlowsTable creates the raw flow table func (c *Component) createRawFlowsTable(ctx context.Context) error { - hash := c.d.Schema.ProtobufMessageHash() + hash := c.d.Schema.ClickHouseHash() tableName := fmt.Sprintf("flows_%s_raw", hash) - kafkaSettings := []string{ - fmt.Sprintf(`kafka_broker_list = %s`, - quoteString(strings.Join(c.config.Kafka.Brokers, ","))), - fmt.Sprintf(`kafka_topic_list = %s`, - quoteString(fmt.Sprintf("%s-%s", c.config.Kafka.Topic, hash))), - fmt.Sprintf(`kafka_group_name = %s`, quoteString(c.config.Kafka.GroupName)), - `kafka_format = 'Protobuf'`, - fmt.Sprintf(`kafka_schema = 'flow-%s.proto:FlowMessagev%s'`, hash, hash), - fmt.Sprintf(`kafka_num_consumers = %d`, c.config.Kafka.Consumers), - `kafka_thread_per_consumer = 1`, - `kafka_handle_error_mode = 'stream'`, - } - for _, setting := range c.config.Kafka.EngineSettings { - kafkaSettings = append(kafkaSettings, setting) - } - kafkaEngine := fmt.Sprintf("Kafka SETTINGS %s", strings.Join(kafkaSettings, ", ")) // Build CREATE query createQuery, err := stemplate( - `CREATE TABLE {{ .Database }}.{{ .Table }} ({{ .Schema }}) ENGINE = {{ .Engine }}`, + `CREATE TABLE {{ .Database }}.{{ .Table }} ({{ .Schema }}) ENGINE = Null`, gin.H{ "Database": c.d.ClickHouse.DatabaseName(), "Table": tableName, "Schema": c.d.Schema.ClickHouseCreateTable( schema.ClickHouseSkipGeneratedColumns, - schema.ClickHouseUseTransformFromType, schema.ClickHouseSkipAliasedColumns), - "Engine": kafkaEngine, }) if err != nil { return fmt.Errorf("cannot build query to create raw flows table: %w", err) @@ -328,7 +310,6 @@ func (c *Component) createRawFlowsTable(ctx context.Context) error { c.r.Info().Msg("create raw flows table") for _, table := range []string{ fmt.Sprintf("%s_consumer", tableName), - fmt.Sprintf("%s_errors", tableName), tableName, } { if err := c.d.ClickHouse.ExecOnCluster(ctx, fmt.Sprintf(`DROP TABLE IF EXISTS %s SYNC`, table)); err != nil { @@ -348,20 +329,19 @@ func (c *Component) createRawFlowsTable(ctx context.Context) error { var dictionaryNetworksLookupRegex = regexp.MustCompile(`\bc_(Src|Dst)Networks\[([[:lower:]]+)\]\B`) func (c *Component) createRawFlowsConsumerView(ctx context.Context) error { - tableName := fmt.Sprintf("flows_%s_raw", c.d.Schema.ProtobufMessageHash()) + tableName := fmt.Sprintf("flows_%s_raw", c.d.Schema.ClickHouseHash()) viewName := fmt.Sprintf("%s_consumer", tableName) // Build SELECT query args := gin.H{ "Columns": strings.Join(c.d.Schema.ClickHouseSelectColumns( schema.ClickHouseSubstituteGenerates, - schema.ClickHouseSubstituteTransforms, schema.ClickHouseSkipAliasedColumns), ", "), "Database": c.d.ClickHouse.DatabaseName(), "Table": tableName, } selectQuery, err := stemplate( - `SELECT {{ .Columns }} FROM {{ .Database }}.{{ .Table }} WHERE length(_error) = 0`, + `SELECT {{ .Columns }} FROM {{ .Database }}.{{ .Table }}`, args) if err != nil { return fmt.Errorf("cannot build select statement for raw flows consumer view: %w", err) @@ -445,105 +425,6 @@ func (c *Component) createRawFlowsConsumerView(ctx context.Context) error { return nil } -func (c *Component) createRawFlowsErrors(ctx context.Context) error { - name := c.localTable("flows_raw_errors") - createQuery, err := stemplate(`CREATE TABLE {{ .Database }}.{{ .Table }} -(`+"`timestamp`"+` DateTime, - `+"`topic`"+` LowCardinality(String), - `+"`partition`"+` UInt64, - `+"`offset`"+` UInt64, - `+"`raw`"+` String, - `+"`error`"+` String) -ENGINE = {{ .Engine }} -PARTITION BY toYYYYMMDDhhmmss(toStartOfHour(timestamp)) -ORDER BY (timestamp, topic, partition, offset) -TTL timestamp + toIntervalDay(1) -`, gin.H{ - "Table": name, - "Database": c.d.ClickHouse.DatabaseName(), - "Engine": c.mergeTreeEngine(name, ""), - }) - if err != nil { - return fmt.Errorf("cannot build query to create flow error table: %w", err) - } - if ok, err := c.tableAlreadyExists(ctx, name, "create_table_query", createQuery); err != nil { - return err - } else if ok { - c.r.Info().Msgf("table %s already exists, skip migration", name) - return errSkipStep - } - c.r.Info().Msgf("create table %s", name) - createOrReplaceQuery := strings.Replace(createQuery, "CREATE ", "CREATE OR REPLACE ", 1) - if err := c.d.ClickHouse.ExecOnCluster(ctx, createOrReplaceQuery); err != nil { - return fmt.Errorf("cannot create table %s: %w", name, err) - } - return nil -} - -func (c *Component) createRawFlowsErrorsConsumerView(ctx context.Context) error { - source := fmt.Sprintf("flows_%s_raw", c.d.Schema.ProtobufMessageHash()) - viewName := "flows_raw_errors_consumer" - - // Build SELECT query - selectQuery, err := stemplate(` -SELECT - now() AS timestamp, - _topic AS topic, - _partition AS partition, - _offset AS offset, - _raw_message AS raw, - _error AS error -FROM {{ .Database }}.{{ .Table }} -WHERE length(_error) > 0`, gin.H{ - "Database": c.d.ClickHouse.DatabaseName(), - "Table": source, - }) - if err != nil { - return fmt.Errorf("cannot build select statement for raw flows error: %w", err) - } - - // Check the existing one - if ok, err := c.tableAlreadyExists(ctx, viewName, "as_select", selectQuery); err != nil { - return err - } else if ok { - c.r.Info().Msg("raw flows errors view already exists, skip migration") - return errSkipStep - } - - // Drop and create - c.r.Info().Msg("create raw flows errors view") - if err := c.d.ClickHouse.ExecOnCluster(ctx, fmt.Sprintf(`DROP TABLE IF EXISTS %s SYNC`, viewName)); err != nil { - return fmt.Errorf("cannot drop table %s: %w", viewName, err) - } - if err := c.d.ClickHouse.ExecOnCluster(ctx, - fmt.Sprintf(`CREATE MATERIALIZED VIEW %s TO %s AS %s`, - viewName, c.distributedTable("flows_raw_errors"), selectQuery)); err != nil { - return fmt.Errorf("cannot create raw flows errors view: %w", err) - } - - return nil -} - -func (c *Component) deleteOldRawFlowsErrorsView(ctx context.Context) error { - tableName := fmt.Sprintf("flows_%s_raw", c.d.Schema.ProtobufMessageHash()) - viewName := fmt.Sprintf("%s_errors", tableName) - - // Check the existing one - if ok, err := c.tableAlreadyExists(ctx, viewName, "name", viewName); err != nil { - return err - } else if !ok { - c.r.Debug().Msg("old raw flows errors view does not exist, skip migration") - return errSkipStep - } - - // Drop - c.r.Info().Msg("delete old raw flows errors view") - if err := c.d.ClickHouse.ExecOnCluster(ctx, fmt.Sprintf(`DROP TABLE IF EXISTS %s SYNC`, viewName)); err != nil { - return fmt.Errorf("cannot drop table %s: %w", viewName, err) - } - return nil -} - func (c *Component) createOrUpdateFlowsTable(ctx context.Context, resolution ResolutionConfiguration) error { ctx = clickhouse.Context(ctx, clickhouse.WithSettings(clickhouse.Settings{ "allow_suspicious_low_cardinality_types": 1, diff --git a/orchestrator/clickhouse/migrations_test.go b/orchestrator/clickhouse/migrations_test.go index efe98c10..3ac238cd 100644 --- a/orchestrator/clickhouse/migrations_test.go +++ b/orchestrator/clickhouse/migrations_test.go @@ -20,7 +20,6 @@ import ( "akvorado/common/daemon" "akvorado/common/helpers" "akvorado/common/httpserver" - "akvorado/common/kafka" "akvorado/common/reporter" "akvorado/common/schema" "akvorado/orchestrator/geoip" @@ -116,15 +115,23 @@ func loadAllTables(t *testing.T, ch *clickhousedb.Component, sch *schema.Compone } func isOldTable(schema *schema.Component, table string) bool { - if strings.Contains(table, schema.ProtobufMessageHash()) { + if strings.Contains(table, schema.ClickHouseHash()) { return false } - if table == "flows_raw_errors" { - return false - } - if strings.HasSuffix(table, "_raw") || strings.HasSuffix(table, "_raw_consumer") || strings.HasSuffix(table, "_raw_errors") { + if strings.HasPrefix(table, "test_") { return true } + oldSuffixes := []string{ + "_raw", + "_raw_consumer", + "_raw_errors", "_raw_errors_local", + "_raw_errors_consumer", + } + for _, suffix := range oldSuffixes { + if strings.HasSuffix(table, suffix) { + return true + } + } return false } @@ -136,7 +143,6 @@ func startTestComponent(t *testing.T, r *reporter.Reporter, chComponent *clickho } configuration := DefaultConfiguration() configuration.OrchestratorURL = "http://127.0.0.1:0" - configuration.Kafka.Configuration = kafka.DefaultConfiguration() ch, err := New(r, configuration, Dependencies{ Daemon: daemon.NewMock(t), HTTP: httpserver.NewMock(t, r), @@ -234,7 +240,7 @@ WHERE database=currentDatabase() AND table NOT LIKE '.%'`) if err != nil { t.Fatalf("Query() error:\n%+v", err) } - hash := ch.d.Schema.ProtobufMessageHash() + hash := ch.d.Schema.ClickHouseHash() got := []string{} for rows.Next() { var table string @@ -263,9 +269,6 @@ WHERE database=currentDatabase() AND table NOT LIKE '.%'`) fmt.Sprintf("flows_%s_raw", hash), fmt.Sprintf("flows_%s_raw_consumer", hash), "flows_local", - "flows_raw_errors", - "flows_raw_errors_consumer", - "flows_raw_errors_local", schema.DictionaryICMP, schema.DictionaryNetworks, schema.DictionaryProtocols, @@ -360,9 +363,6 @@ func TestMigrationFromPreviousStates(t *testing.T) { schema.ColumnDstASPath, schema.ColumnDstCommunities, schema.ColumnDstLargeCommunities, - schema.ColumnDstLargeCommunitiesASN, - schema.ColumnDstLargeCommunitiesLocalData1, - schema.ColumnDstLargeCommunitiesLocalData2, } sch, err := schema.New(schConfig) if err != nil { @@ -442,7 +442,7 @@ AND name LIKE $3`, "flows", ch.d.ClickHouse.DatabaseName(), "%DimensionAttribute // Check if the rows were created in the consumer flows table rowConsumer := ch.d.ClickHouse.QueryRow( context.Background(), - fmt.Sprintf(`SHOW CREATE flows_%s_raw_consumer`, ch.d.Schema.ProtobufMessageHash())) + fmt.Sprintf(`SHOW CREATE flows_%s_raw_consumer`, ch.d.Schema.ClickHouseHash())) var existingConsumer string if err := rowConsumer.Scan(&existingConsumer); err != nil { t.Fatalf("Scan() error:\n%+v", err) @@ -517,7 +517,7 @@ AND name LIKE $3`, "flows", ch.d.ClickHouse.DatabaseName(), "%DimensionAttribute // Check if the rows were removed in the consumer flows table rowConsumer := ch.d.ClickHouse.QueryRow( context.Background(), - fmt.Sprintf(`SHOW CREATE flows_%s_raw_consumer`, ch.d.Schema.ProtobufMessageHash())) + fmt.Sprintf(`SHOW CREATE flows_%s_raw_consumer`, ch.d.Schema.ClickHouseHash())) var existingConsumer string if err := rowConsumer.Scan(&existingConsumer); err != nil { t.Fatalf("Scan() error:\n%+v", err) diff --git a/orchestrator/clickhouse/testdata/states/002-cluster.csv b/orchestrator/clickhouse/testdata/states/002-cluster.csv new file mode 100644 index 00000000..0f7204ef --- /dev/null +++ b/orchestrator/clickhouse/testdata/states/002-cluster.csv @@ -0,0 +1,21 @@ +asns,"CREATE DICTIONARY default.asns (`asn` UInt32 INJECTIVE, `name` String) PRIMARY KEY asn SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/asns.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" +icmp,"CREATE DICTIONARY default.icmp (`proto` UInt8, `type` UInt8, `code` UInt8, `name` String) PRIMARY KEY proto, type, code SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/icmp.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(COMPLEX_KEY_HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" +networks,"CREATE DICTIONARY default.networks (`network` String, `name` String, `role` String, `site` String, `region` String, `city` String, `state` String, `country` String, `tenant` String, `asn` UInt32) PRIMARY KEY network SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/networks.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(IP_TRIE()) SETTINGS(format_csv_allow_single_quotes = 0)" +protocols,"CREATE DICTIONARY default.protocols (`proto` UInt8 INJECTIVE, `name` String, `description` String) PRIMARY KEY proto SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/protocols.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" +tcp,"CREATE DICTIONARY default.tcp (`port` UInt16 INJECTIVE, `name` String) PRIMARY KEY port SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/tcp.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" +udp,"CREATE DICTIONARY default.udp (`port` UInt16 INJECTIVE, `name` String) PRIMARY KEY port SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/udp.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" +exporters,"CREATE TABLE default.exporters (`TimeReceived` DateTime, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `IfName` LowCardinality(String), `IfDescription` LowCardinality(String), `IfSpeed` UInt32, `IfConnectivity` LowCardinality(String), `IfProvider` LowCardinality(String), `IfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2)) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/shard-{shard}/exporters', 'replica-{replica}', TimeReceived) ORDER BY (ExporterAddress, IfName) TTL TimeReceived + toIntervalDay(1) SETTINGS index_granularity = 8192" +flows_1h0m0s_local,"CREATE TABLE default.flows_1h0m0s_local (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `PacketSize` UInt64 ALIAS intDiv(Bytes, Packets), `PacketSizeBucket` LowCardinality(String) ALIAS multiIf(PacketSize < 64, '0-63', PacketSize < 128, '64-127', PacketSize < 256, '128-255', PacketSize < 512, '256-511', PacketSize < 768, '512-767', PacketSize < 1024, '768-1023', PacketSize < 1280, '1024-1279', PacketSize < 1501, '1280-1500', PacketSize < 2048, '1501-2047', PacketSize < 3072, '2048-3071', PacketSize < 4096, '3072-4095', PacketSize < 8192, '4096-8191', PacketSize < 10240, '8192-10239', PacketSize < 16384, '10240-16383', PacketSize < 32768, '16384-32767', PacketSize < 65536, '32768-65535', '65536-Inf'), `ForwardingStatus` UInt32) ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/shard-{shard}/flows_1h0m0s_local', 'replica-{replica}', (Bytes, Packets)) PARTITION BY toYYYYMMDDhhmmss(toStartOfInterval(TimeReceived, toIntervalSecond(622080))) PRIMARY KEY (TimeReceived, ExporterAddress, EType, Proto, InIfName, SrcAS, ForwardingStatus, OutIfName, DstAS, SamplingRate) ORDER BY (TimeReceived, ExporterAddress, EType, Proto, InIfName, SrcAS, ForwardingStatus, OutIfName, DstAS, SamplingRate, SrcNetName, DstNetName, SrcNetRole, DstNetRole, SrcNetSite, DstNetSite, SrcNetRegion, DstNetRegion, SrcNetTenant, DstNetTenant, SrcCountry, DstCountry, SrcGeoCity, DstGeoCity, SrcGeoState, DstGeoState, Dst1stAS, Dst2ndAS, Dst3rdAS) TTL TimeReceived + toIntervalSecond(31104000) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1" +flows_1m0s_local,"CREATE TABLE default.flows_1m0s_local (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `PacketSize` UInt64 ALIAS intDiv(Bytes, Packets), `PacketSizeBucket` LowCardinality(String) ALIAS multiIf(PacketSize < 64, '0-63', PacketSize < 128, '64-127', PacketSize < 256, '128-255', PacketSize < 512, '256-511', PacketSize < 768, '512-767', PacketSize < 1024, '768-1023', PacketSize < 1280, '1024-1279', PacketSize < 1501, '1280-1500', PacketSize < 2048, '1501-2047', PacketSize < 3072, '2048-3071', PacketSize < 4096, '3072-4095', PacketSize < 8192, '4096-8191', PacketSize < 10240, '8192-10239', PacketSize < 16384, '10240-16383', PacketSize < 32768, '16384-32767', PacketSize < 65536, '32768-65535', '65536-Inf'), `ForwardingStatus` UInt32) ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/shard-{shard}/flows_1m0s_local', 'replica-{replica}', (Bytes, Packets)) PARTITION BY toYYYYMMDDhhmmss(toStartOfInterval(TimeReceived, toIntervalSecond(12096))) PRIMARY KEY (TimeReceived, ExporterAddress, EType, Proto, InIfName, SrcAS, ForwardingStatus, OutIfName, DstAS, SamplingRate) ORDER BY (TimeReceived, ExporterAddress, EType, Proto, InIfName, SrcAS, ForwardingStatus, OutIfName, DstAS, SamplingRate, SrcNetName, DstNetName, SrcNetRole, DstNetRole, SrcNetSite, DstNetSite, SrcNetRegion, DstNetRegion, SrcNetTenant, DstNetTenant, SrcCountry, DstCountry, SrcGeoCity, DstGeoCity, SrcGeoState, DstGeoState, Dst1stAS, Dst2ndAS, Dst3rdAS) TTL TimeReceived + toIntervalSecond(604800) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1" +flows_5m0s_local,"CREATE TABLE default.flows_5m0s_local (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `PacketSize` UInt64 ALIAS intDiv(Bytes, Packets), `PacketSizeBucket` LowCardinality(String) ALIAS multiIf(PacketSize < 64, '0-63', PacketSize < 128, '64-127', PacketSize < 256, '128-255', PacketSize < 512, '256-511', PacketSize < 768, '512-767', PacketSize < 1024, '768-1023', PacketSize < 1280, '1024-1279', PacketSize < 1501, '1280-1500', PacketSize < 2048, '1501-2047', PacketSize < 3072, '2048-3071', PacketSize < 4096, '3072-4095', PacketSize < 8192, '4096-8191', PacketSize < 10240, '8192-10239', PacketSize < 16384, '10240-16383', PacketSize < 32768, '16384-32767', PacketSize < 65536, '32768-65535', '65536-Inf'), `ForwardingStatus` UInt32) ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/shard-{shard}/flows_5m0s_local', 'replica-{replica}', (Bytes, Packets)) PARTITION BY toYYYYMMDDhhmmss(toStartOfInterval(TimeReceived, toIntervalSecond(155520))) PRIMARY KEY (TimeReceived, ExporterAddress, EType, Proto, InIfName, SrcAS, ForwardingStatus, OutIfName, DstAS, SamplingRate) ORDER BY (TimeReceived, ExporterAddress, EType, Proto, InIfName, SrcAS, ForwardingStatus, OutIfName, DstAS, SamplingRate, SrcNetName, DstNetName, SrcNetRole, DstNetRole, SrcNetSite, DstNetSite, SrcNetRegion, DstNetRegion, SrcNetTenant, DstNetTenant, SrcCountry, DstCountry, SrcGeoCity, DstGeoCity, SrcGeoState, DstGeoState, Dst1stAS, Dst2ndAS, Dst3rdAS) TTL TimeReceived + toIntervalSecond(7776000) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1" +flows_I6D3KDQCRUBCNCGF4BSOWTRMVIv5_raw,"CREATE TABLE default.flows_I6D3KDQCRUBCNCGF4BSOWTRMVIv5_raw (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAddr` IPv6 CODEC(ZSTD(1)), `DstAddr` IPv6 CODEC(ZSTD(1)), `SrcNetMask` UInt8, `DstNetMask` UInt8, `SrcAS` UInt32, `DstAS` UInt32, `DstASPath` Array(UInt32), `DstCommunities` Array(UInt32), `DstLargeCommunities` Array(UInt128), `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `SrcPort` UInt16, `DstPort` UInt16, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `ForwardingStatus` UInt32) ENGINE = Null" +flows_local,"CREATE TABLE default.flows_local (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAddr` IPv6 CODEC(ZSTD(1)), `DstAddr` IPv6 CODEC(ZSTD(1)), `SrcNetMask` UInt8, `DstNetMask` UInt8, `SrcNetPrefix` String ALIAS multiIf(EType = 2048, concat(replaceRegexpOne(CAST(IPv6CIDRToRange(SrcAddr, CAST(96 + SrcNetMask, 'UInt8')).1, 'String'), '^::ffff:', ''), '/', CAST(SrcNetMask, 'String')), EType = 34525, concat(CAST(IPv6CIDRToRange(SrcAddr, SrcNetMask).1, 'String'), '/', CAST(SrcNetMask, 'String')), ''), `DstNetPrefix` String ALIAS multiIf(EType = 2048, concat(replaceRegexpOne(CAST(IPv6CIDRToRange(DstAddr, CAST(96 + DstNetMask, 'UInt8')).1, 'String'), '^::ffff:', ''), '/', CAST(DstNetMask, 'String')), EType = 34525, concat(CAST(IPv6CIDRToRange(DstAddr, DstNetMask).1, 'String'), '/', CAST(DstNetMask, 'String')), ''), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `DstASPath` Array(UInt32), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `DstCommunities` Array(UInt32), `DstLargeCommunities` Array(UInt128), `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `SrcPort` UInt16, `DstPort` UInt16, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `PacketSize` UInt64 ALIAS intDiv(Bytes, Packets), `PacketSizeBucket` LowCardinality(String) ALIAS multiIf(PacketSize < 64, '0-63', PacketSize < 128, '64-127', PacketSize < 256, '128-255', PacketSize < 512, '256-511', PacketSize < 768, '512-767', PacketSize < 1024, '768-1023', PacketSize < 1280, '1024-1279', PacketSize < 1501, '1280-1500', PacketSize < 2048, '1501-2047', PacketSize < 3072, '2048-3071', PacketSize < 4096, '3072-4095', PacketSize < 8192, '4096-8191', PacketSize < 10240, '8192-10239', PacketSize < 16384, '10240-16383', PacketSize < 32768, '16384-32767', PacketSize < 65536, '32768-65535', '65536-Inf'), `ForwardingStatus` UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/shard-{shard}/flows_local', 'replica-{replica}') PARTITION BY toYYYYMMDDhhmmss(toStartOfInterval(TimeReceived, toIntervalSecond(25920))) ORDER BY (toStartOfFiveMinutes(TimeReceived), ExporterAddress, InIfName, OutIfName) TTL TimeReceived + toIntervalSecond(1296000) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1" +flows,"CREATE TABLE default.flows (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAddr` IPv6 CODEC(ZSTD(1)), `DstAddr` IPv6 CODEC(ZSTD(1)), `SrcNetMask` UInt8, `DstNetMask` UInt8, `SrcNetPrefix` String ALIAS multiIf(EType = 2048, concat(replaceRegexpOne(CAST(IPv6CIDRToRange(SrcAddr, CAST(96 + SrcNetMask, 'UInt8')).1, 'String'), '^::ffff:', ''), '/', CAST(SrcNetMask, 'String')), EType = 34525, concat(CAST(IPv6CIDRToRange(SrcAddr, SrcNetMask).1, 'String'), '/', CAST(SrcNetMask, 'String')), ''), `DstNetPrefix` String ALIAS multiIf(EType = 2048, concat(replaceRegexpOne(CAST(IPv6CIDRToRange(DstAddr, CAST(96 + DstNetMask, 'UInt8')).1, 'String'), '^::ffff:', ''), '/', CAST(DstNetMask, 'String')), EType = 34525, concat(CAST(IPv6CIDRToRange(DstAddr, DstNetMask).1, 'String'), '/', CAST(DstNetMask, 'String')), ''), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `DstASPath` Array(UInt32), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `DstCommunities` Array(UInt32), `DstLargeCommunities` Array(UInt128), `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `SrcPort` UInt16, `DstPort` UInt16, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `PacketSize` UInt64 ALIAS intDiv(Bytes, Packets), `PacketSizeBucket` LowCardinality(String) ALIAS multiIf(PacketSize < 64, '0-63', PacketSize < 128, '64-127', PacketSize < 256, '128-255', PacketSize < 512, '256-511', PacketSize < 768, '512-767', PacketSize < 1024, '768-1023', PacketSize < 1280, '1024-1279', PacketSize < 1501, '1280-1500', PacketSize < 2048, '1501-2047', PacketSize < 3072, '2048-3071', PacketSize < 4096, '3072-4095', PacketSize < 8192, '4096-8191', PacketSize < 10240, '8192-10239', PacketSize < 16384, '10240-16383', PacketSize < 32768, '16384-32767', PacketSize < 65536, '32768-65535', '65536-Inf'), `ForwardingStatus` UInt32) ENGINE = Distributed('akvorado', 'default', 'flows_local', rand())" +flows_1h0m0s,"CREATE TABLE default.flows_1h0m0s (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `PacketSize` UInt64 ALIAS intDiv(Bytes, Packets), `PacketSizeBucket` LowCardinality(String) ALIAS multiIf(PacketSize < 64, '0-63', PacketSize < 128, '64-127', PacketSize < 256, '128-255', PacketSize < 512, '256-511', PacketSize < 768, '512-767', PacketSize < 1024, '768-1023', PacketSize < 1280, '1024-1279', PacketSize < 1501, '1280-1500', PacketSize < 2048, '1501-2047', PacketSize < 3072, '2048-3071', PacketSize < 4096, '3072-4095', PacketSize < 8192, '4096-8191', PacketSize < 10240, '8192-10239', PacketSize < 16384, '10240-16383', PacketSize < 32768, '16384-32767', PacketSize < 65536, '32768-65535', '65536-Inf'), `ForwardingStatus` UInt32) ENGINE = Distributed('akvorado', 'default', 'flows_1h0m0s_local', rand())" +flows_1m0s,"CREATE TABLE default.flows_1m0s (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `PacketSize` UInt64 ALIAS intDiv(Bytes, Packets), `PacketSizeBucket` LowCardinality(String) ALIAS multiIf(PacketSize < 64, '0-63', PacketSize < 128, '64-127', PacketSize < 256, '128-255', PacketSize < 512, '256-511', PacketSize < 768, '512-767', PacketSize < 1024, '768-1023', PacketSize < 1280, '1024-1279', PacketSize < 1501, '1280-1500', PacketSize < 2048, '1501-2047', PacketSize < 3072, '2048-3071', PacketSize < 4096, '3072-4095', PacketSize < 8192, '4096-8191', PacketSize < 10240, '8192-10239', PacketSize < 16384, '10240-16383', PacketSize < 32768, '16384-32767', PacketSize < 65536, '32768-65535', '65536-Inf'), `ForwardingStatus` UInt32) ENGINE = Distributed('akvorado', 'default', 'flows_1m0s_local', rand())" +flows_5m0s,"CREATE TABLE default.flows_5m0s (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `PacketSize` UInt64 ALIAS intDiv(Bytes, Packets), `PacketSizeBucket` LowCardinality(String) ALIAS multiIf(PacketSize < 64, '0-63', PacketSize < 128, '64-127', PacketSize < 256, '128-255', PacketSize < 512, '256-511', PacketSize < 768, '512-767', PacketSize < 1024, '768-1023', PacketSize < 1280, '1024-1279', PacketSize < 1501, '1280-1500', PacketSize < 2048, '1501-2047', PacketSize < 3072, '2048-3071', PacketSize < 4096, '3072-4095', PacketSize < 8192, '4096-8191', PacketSize < 10240, '8192-10239', PacketSize < 16384, '10240-16383', PacketSize < 32768, '16384-32767', PacketSize < 65536, '32768-65535', '65536-Inf'), `ForwardingStatus` UInt32) ENGINE = Distributed('akvorado', 'default', 'flows_5m0s_local', rand())" +exporters_consumer,"CREATE MATERIALIZED VIEW default.exporters_consumer TO default.exporters (`TimeReceived` DateTime, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `IfName` String, `IfDescription` String, `IfSpeed` UInt32, `IfConnectivity` String, `IfProvider` String, `IfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2)) AS SELECT DISTINCT TimeReceived, ExporterAddress, ExporterName, ExporterGroup, ExporterRole, ExporterSite, ExporterRegion, ExporterTenant, [InIfName, OutIfName][num] AS IfName, [InIfDescription, OutIfDescription][num] AS IfDescription, [InIfSpeed, OutIfSpeed][num] AS IfSpeed, [InIfConnectivity, OutIfConnectivity][num] AS IfConnectivity, [InIfProvider, OutIfProvider][num] AS IfProvider, [InIfBoundary, OutIfBoundary][num] AS IfBoundary FROM default.flows ARRAY JOIN arrayEnumerate([1, 2]) AS num" +flows_1h0m0s_consumer,"CREATE MATERIALIZED VIEW default.flows_1h0m0s_consumer TO default.flows_1h0m0s_local (`TimeReceived` DateTime, `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64, `Packets` UInt64, `ForwardingStatus` UInt32) AS SELECT toStartOfInterval(TimeReceived, toIntervalSecond(3600)) AS TimeReceived, SamplingRate, ExporterAddress, ExporterName, ExporterGroup, ExporterRole, ExporterSite, ExporterRegion, ExporterTenant, SrcAS, DstAS, SrcNetName, DstNetName, SrcNetRole, DstNetRole, SrcNetSite, DstNetSite, SrcNetRegion, DstNetRegion, SrcNetTenant, DstNetTenant, SrcCountry, DstCountry, SrcGeoCity, DstGeoCity, SrcGeoState, DstGeoState, Dst1stAS, Dst2ndAS, Dst3rdAS, InIfName, OutIfName, InIfDescription, OutIfDescription, InIfSpeed, OutIfSpeed, InIfConnectivity, OutIfConnectivity, InIfProvider, OutIfProvider, InIfBoundary, OutIfBoundary, EType, Proto, Bytes, Packets, ForwardingStatus FROM default.flows_local" +flows_1m0s_consumer,"CREATE MATERIALIZED VIEW default.flows_1m0s_consumer TO default.flows_1m0s_local (`TimeReceived` DateTime, `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64, `Packets` UInt64, `ForwardingStatus` UInt32) AS SELECT toStartOfInterval(TimeReceived, toIntervalSecond(60)) AS TimeReceived, SamplingRate, ExporterAddress, ExporterName, ExporterGroup, ExporterRole, ExporterSite, ExporterRegion, ExporterTenant, SrcAS, DstAS, SrcNetName, DstNetName, SrcNetRole, DstNetRole, SrcNetSite, DstNetSite, SrcNetRegion, DstNetRegion, SrcNetTenant, DstNetTenant, SrcCountry, DstCountry, SrcGeoCity, DstGeoCity, SrcGeoState, DstGeoState, Dst1stAS, Dst2ndAS, Dst3rdAS, InIfName, OutIfName, InIfDescription, OutIfDescription, InIfSpeed, OutIfSpeed, InIfConnectivity, OutIfConnectivity, InIfProvider, OutIfProvider, InIfBoundary, OutIfBoundary, EType, Proto, Bytes, Packets, ForwardingStatus FROM default.flows_local" +flows_5m0s_consumer,"CREATE MATERIALIZED VIEW default.flows_5m0s_consumer TO default.flows_5m0s_local (`TimeReceived` DateTime, `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64, `Packets` UInt64, `ForwardingStatus` UInt32) AS SELECT toStartOfInterval(TimeReceived, toIntervalSecond(300)) AS TimeReceived, SamplingRate, ExporterAddress, ExporterName, ExporterGroup, ExporterRole, ExporterSite, ExporterRegion, ExporterTenant, SrcAS, DstAS, SrcNetName, DstNetName, SrcNetRole, DstNetRole, SrcNetSite, DstNetSite, SrcNetRegion, DstNetRegion, SrcNetTenant, DstNetTenant, SrcCountry, DstCountry, SrcGeoCity, DstGeoCity, SrcGeoState, DstGeoState, Dst1stAS, Dst2ndAS, Dst3rdAS, InIfName, OutIfName, InIfDescription, OutIfDescription, InIfSpeed, OutIfSpeed, InIfConnectivity, OutIfConnectivity, InIfProvider, OutIfProvider, InIfBoundary, OutIfBoundary, EType, Proto, Bytes, Packets, ForwardingStatus FROM default.flows_local" +flows_I6D3KDQCRUBCNCGF4BSOWTRMVIv5_raw_consumer,"CREATE MATERIALIZED VIEW default.flows_I6D3KDQCRUBCNCGF4BSOWTRMVIv5_raw_consumer TO default.flows (`TimeReceived` DateTime, `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAddr` IPv6, `DstAddr` IPv6, `SrcNetMask` UInt8, `DstNetMask` UInt8, `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` String, `DstNetName` String, `SrcNetRole` String, `DstNetRole` String, `SrcNetSite` String, `DstNetSite` String, `SrcNetRegion` String, `DstNetRegion` String, `SrcNetTenant` String, `DstNetTenant` String, `SrcCountry` String, `DstCountry` String, `SrcGeoCity` String, `DstGeoCity` String, `SrcGeoState` String, `DstGeoState` String, `DstASPath` Array(UInt32), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `DstCommunities` Array(UInt32), `DstLargeCommunities` Array(UInt128), `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `SrcPort` UInt16, `DstPort` UInt16, `Bytes` UInt64, `Packets` UInt64, `ForwardingStatus` UInt32) AS WITH arrayCompact(DstASPath) AS c_DstASPath, dictGet('default.networks', ('asn', 'name', 'role', 'site', 'region', 'tenant', 'country', 'city', 'state'), SrcAddr) AS c_SrcNetworks, dictGet('default.networks', ('asn', 'name', 'role', 'site', 'region', 'tenant', 'country', 'city', 'state'), DstAddr) AS c_DstNetworks SELECT TimeReceived, SamplingRate, ExporterAddress, ExporterName, ExporterGroup, ExporterRole, ExporterSite, ExporterRegion, ExporterTenant, SrcAddr, DstAddr, SrcNetMask, DstNetMask, if(SrcAS = 0, c_SrcNetworks.1, SrcAS) AS SrcAS, if(DstAS = 0, c_DstNetworks.1, DstAS) AS DstAS, c_SrcNetworks.2 AS SrcNetName, c_DstNetworks.2 AS DstNetName, c_SrcNetworks.3 AS SrcNetRole, c_DstNetworks.3 AS DstNetRole, c_SrcNetworks.4 AS SrcNetSite, c_DstNetworks.4 AS DstNetSite, c_SrcNetworks.5 AS SrcNetRegion, c_DstNetworks.5 AS DstNetRegion, c_SrcNetworks.6 AS SrcNetTenant, c_DstNetworks.6 AS DstNetTenant, c_SrcNetworks.7 AS SrcCountry, c_DstNetworks.7 AS DstCountry, c_SrcNetworks.8 AS SrcGeoCity, c_DstNetworks.8 AS DstGeoCity, c_SrcNetworks.9 AS SrcGeoState, c_DstNetworks.9 AS DstGeoState, DstASPath, c_DstASPath[1] AS Dst1stAS, c_DstASPath[2] AS Dst2ndAS, c_DstASPath[3] AS Dst3rdAS, DstCommunities, DstLargeCommunities, InIfName, OutIfName, InIfDescription, OutIfDescription, InIfSpeed, OutIfSpeed, InIfConnectivity, OutIfConnectivity, InIfProvider, OutIfProvider, InIfBoundary, OutIfBoundary, EType, Proto, SrcPort, DstPort, Bytes, Packets, ForwardingStatus FROM default.flows_I6D3KDQCRUBCNCGF4BSOWTRMVIv5_raw" diff --git a/orchestrator/clickhouse/testdata/states/011.csv b/orchestrator/clickhouse/testdata/states/011.csv index cb3c06ae..862748b2 100644 --- a/orchestrator/clickhouse/testdata/states/011.csv +++ b/orchestrator/clickhouse/testdata/states/011.csv @@ -2,6 +2,8 @@ asns,"CREATE DICTIONARY default.asns (`asn` UInt32 INJECTIVE, `name` String) PRI icmp,"CREATE DICTIONARY default.icmp (`proto` UInt8, `type` UInt8, `code` UInt8, `name` String) PRIMARY KEY proto, type, code SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/icmp.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(COMPLEX_KEY_HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" networks,"CREATE DICTIONARY default.networks (`network` String, `name` String, `role` String, `site` String, `region` String, `city` String, `state` String, `country` String, `tenant` String, `asn` UInt32) PRIMARY KEY network SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/networks.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(IP_TRIE()) SETTINGS(format_csv_allow_single_quotes = 0)" protocols,"CREATE DICTIONARY default.protocols (`proto` UInt8 INJECTIVE, `name` String, `description` String) PRIMARY KEY proto SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/protocols.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" +tcp,"CREATE DICTIONARY default.tcp (`port` UInt16 INJECTIVE, `name` String) PRIMARY KEY port SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/tcp.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" +udp,"CREATE DICTIONARY default.udp (`port` UInt16 INJECTIVE, `name` String) PRIMARY KEY port SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/udp.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" exporters,"CREATE TABLE default.exporters (`TimeReceived` DateTime, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `IfName` LowCardinality(String), `IfDescription` LowCardinality(String), `IfSpeed` UInt32, `IfConnectivity` LowCardinality(String), `IfProvider` LowCardinality(String), `IfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2)) ENGINE = ReplacingMergeTree(TimeReceived) ORDER BY (ExporterAddress, IfName) TTL TimeReceived + toIntervalDay(1) SETTINGS index_granularity = 8192" flows,"CREATE TABLE default.flows (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAddr` IPv6 CODEC(ZSTD(1)), `DstAddr` IPv6 CODEC(ZSTD(1)), `SrcNetMask` UInt8, `DstNetMask` UInt8, `SrcNetPrefix` String ALIAS multiIf(EType = 2048, concat(replaceRegexpOne(CAST(IPv6CIDRToRange(SrcAddr, CAST(96 + SrcNetMask, 'UInt8')).1, 'String'), '^::ffff:', ''), '/', CAST(SrcNetMask, 'String')), EType = 34525, concat(CAST(IPv6CIDRToRange(SrcAddr, SrcNetMask).1, 'String'), '/', CAST(SrcNetMask, 'String')), ''), `DstNetPrefix` String ALIAS multiIf(EType = 2048, concat(replaceRegexpOne(CAST(IPv6CIDRToRange(DstAddr, CAST(96 + DstNetMask, 'UInt8')).1, 'String'), '^::ffff:', ''), '/', CAST(DstNetMask, 'String')), EType = 34525, concat(CAST(IPv6CIDRToRange(DstAddr, DstNetMask).1, 'String'), '/', CAST(DstNetMask, 'String')), ''), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `DstASPath` Array(UInt32), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `DstCommunities` Array(UInt32), `DstLargeCommunities` Array(UInt128), `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `SrcPort` UInt16, `DstPort` UInt16, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `PacketSize` UInt64 ALIAS intDiv(Bytes, Packets), `PacketSizeBucket` LowCardinality(String) ALIAS multiIf(PacketSize < 64, '0-63', PacketSize < 128, '64-127', PacketSize < 256, '128-255', PacketSize < 512, '256-511', PacketSize < 768, '512-767', PacketSize < 1024, '768-1023', PacketSize < 1280, '1024-1279', PacketSize < 1501, '1280-1500', PacketSize < 2048, '1501-2047', PacketSize < 3072, '2048-3071', PacketSize < 4096, '3072-4095', PacketSize < 8192, '4096-8191', PacketSize < 10240, '8192-10239', PacketSize < 16384, '10240-16383', PacketSize < 32768, '16384-32767', PacketSize < 65536, '32768-65535', '65536-Inf'), `ForwardingStatus` UInt32) ENGINE = MergeTree PARTITION BY toYYYYMMDDhhmmss(toStartOfInterval(TimeReceived, toIntervalSecond(25920))) ORDER BY (toStartOfFiveMinutes(TimeReceived), ExporterAddress, InIfName, OutIfName) TTL TimeReceived + toIntervalSecond(1296000) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1" flows_1h0m0s,"CREATE TABLE default.flows_1h0m0s (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `PacketSize` UInt64 ALIAS intDiv(Bytes, Packets), `PacketSizeBucket` LowCardinality(String) ALIAS multiIf(PacketSize < 64, '0-63', PacketSize < 128, '64-127', PacketSize < 256, '128-255', PacketSize < 512, '256-511', PacketSize < 768, '512-767', PacketSize < 1024, '768-1023', PacketSize < 1280, '1024-1279', PacketSize < 1501, '1280-1500', PacketSize < 2048, '1501-2047', PacketSize < 3072, '2048-3071', PacketSize < 4096, '3072-4095', PacketSize < 8192, '4096-8191', PacketSize < 10240, '8192-10239', PacketSize < 16384, '10240-16383', PacketSize < 32768, '16384-32767', PacketSize < 65536, '32768-65535', '65536-Inf'), `ForwardingStatus` UInt32) ENGINE = SummingMergeTree((Bytes, Packets)) PARTITION BY toYYYYMMDDhhmmss(toStartOfInterval(TimeReceived, toIntervalSecond(622080))) PRIMARY KEY (TimeReceived, ExporterAddress, EType, Proto, InIfName, SrcAS, ForwardingStatus, OutIfName, DstAS, SamplingRate) ORDER BY (TimeReceived, ExporterAddress, EType, Proto, InIfName, SrcAS, ForwardingStatus, OutIfName, DstAS, SamplingRate, SrcNetName, DstNetName, SrcNetRole, DstNetRole, SrcNetSite, DstNetSite, SrcNetRegion, DstNetRegion, SrcNetTenant, DstNetTenant, SrcCountry, DstCountry, SrcGeoCity, DstGeoCity, SrcGeoState, DstGeoState, Dst1stAS, Dst2ndAS, Dst3rdAS) TTL TimeReceived + toIntervalSecond(31104000) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1" @@ -15,5 +17,3 @@ flows_1m0s_consumer,"CREATE MATERIALIZED VIEW default.flows_1m0s_consumer TO def flows_5m0s_consumer,"CREATE MATERIALIZED VIEW default.flows_5m0s_consumer TO default.flows_5m0s (`TimeReceived` DateTime, `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64, `Packets` UInt64, `ForwardingStatus` UInt32) AS SELECT toStartOfInterval(TimeReceived, toIntervalSecond(300)) AS TimeReceived, SamplingRate, ExporterAddress, ExporterName, ExporterGroup, ExporterRole, ExporterSite, ExporterRegion, ExporterTenant, SrcAS, DstAS, SrcNetName, DstNetName, SrcNetRole, DstNetRole, SrcNetSite, DstNetSite, SrcNetRegion, DstNetRegion, SrcNetTenant, DstNetTenant, SrcCountry, DstCountry, SrcGeoCity, DstGeoCity, SrcGeoState, DstGeoState, Dst1stAS, Dst2ndAS, Dst3rdAS, InIfName, OutIfName, InIfDescription, OutIfDescription, InIfSpeed, OutIfSpeed, InIfConnectivity, OutIfConnectivity, InIfProvider, OutIfProvider, InIfBoundary, OutIfBoundary, EType, Proto, Bytes, Packets, ForwardingStatus FROM default.flows" flows_LAABIGYMRYZPTGOYIIFZNYDEQM_raw_consumer,"CREATE MATERIALIZED VIEW default.flows_LAABIGYMRYZPTGOYIIFZNYDEQM_raw_consumer TO default.flows (`TimeReceived` DateTime, `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAddr` IPv6, `DstAddr` IPv6, `SrcNetMask` UInt8, `DstNetMask` UInt8, `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` String, `DstNetName` String, `SrcNetRole` String, `DstNetRole` String, `SrcNetSite` String, `DstNetSite` String, `SrcNetRegion` String, `DstNetRegion` String, `SrcNetTenant` String, `DstNetTenant` String, `SrcCountry` String, `DstCountry` String, `SrcGeoCity` String, `DstGeoCity` String, `SrcGeoState` String, `DstGeoState` String, `DstASPath` Array(UInt32), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `DstCommunities` Array(UInt32), `DstLargeCommunities` Array(UInt128), `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `SrcPort` UInt16, `DstPort` UInt16, `Bytes` UInt64, `Packets` UInt64, `ForwardingStatus` UInt32) AS WITH arrayCompact(DstASPath) AS c_DstASPath, dictGet('default.networks', ('asn', 'name', 'role', 'site', 'region', 'tenant', 'country', 'city', 'state'), SrcAddr) AS c_SrcNetworks, dictGet('default.networks', ('asn', 'name', 'role', 'site', 'region', 'tenant', 'country', 'city', 'state'), DstAddr) AS c_DstNetworks SELECT TimeReceived, SamplingRate, ExporterAddress, ExporterName, ExporterGroup, ExporterRole, ExporterSite, ExporterRegion, ExporterTenant, SrcAddr, DstAddr, SrcNetMask, DstNetMask, if(SrcAS = 0, c_SrcNetworks.1, SrcAS) AS SrcAS, if(DstAS = 0, c_DstNetworks.1, DstAS) AS DstAS, c_SrcNetworks.2 AS SrcNetName, c_DstNetworks.2 AS DstNetName, c_SrcNetworks.3 AS SrcNetRole, c_DstNetworks.3 AS DstNetRole, c_SrcNetworks.4 AS SrcNetSite, c_DstNetworks.4 AS DstNetSite, c_SrcNetworks.5 AS SrcNetRegion, c_DstNetworks.5 AS DstNetRegion, c_SrcNetworks.6 AS SrcNetTenant, c_DstNetworks.6 AS DstNetTenant, c_SrcNetworks.7 AS SrcCountry, c_DstNetworks.7 AS DstCountry, c_SrcNetworks.8 AS SrcGeoCity, c_DstNetworks.8 AS DstGeoCity, c_SrcNetworks.9 AS SrcGeoState, c_DstNetworks.9 AS DstGeoState, DstASPath, c_DstASPath[1] AS Dst1stAS, c_DstASPath[2] AS Dst2ndAS, c_DstASPath[3] AS Dst3rdAS, DstCommunities, arrayMap((asn, l1, l2) -> ((bitShiftLeft(CAST(asn, 'UInt128'), 64) + bitShiftLeft(CAST(l1, 'UInt128'), 32)) + CAST(l2, 'UInt128')), DstLargeCommunitiesASN, DstLargeCommunitiesLocalData1, DstLargeCommunitiesLocalData2) AS DstLargeCommunities, InIfName, OutIfName, InIfDescription, OutIfDescription, InIfSpeed, OutIfSpeed, InIfConnectivity, OutIfConnectivity, InIfProvider, OutIfProvider, InIfBoundary, OutIfBoundary, EType, Proto, SrcPort, DstPort, Bytes, Packets, ForwardingStatus FROM default.flows_LAABIGYMRYZPTGOYIIFZNYDEQM_raw WHERE length(_error) = 0" flows_raw_errors_consumer,"CREATE MATERIALIZED VIEW default.flows_raw_errors_consumer TO default.flows_raw_errors (`timestamp` DateTime, `topic` LowCardinality(String), `partition` UInt64, `offset` UInt64, `raw` String, `error` String) AS SELECT now() AS timestamp, _topic AS topic, _partition AS partition, _offset AS offset, _raw_message AS raw, _error AS error FROM default.flows_LAABIGYMRYZPTGOYIIFZNYDEQM_raw WHERE length(_error) > 0" -tcp,"CREATE DICTIONARY default.tcp (`port` UInt16 INJECTIVE, `name` String) PRIMARY KEY port SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/tcp.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" -udp,"CREATE DICTIONARY default.udp (`port` UInt16 INJECTIVE, `name` String) PRIMARY KEY port SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/udp.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" diff --git a/orchestrator/clickhouse/testdata/states/012.csv b/orchestrator/clickhouse/testdata/states/012.csv new file mode 100644 index 00000000..23f817d7 --- /dev/null +++ b/orchestrator/clickhouse/testdata/states/012.csv @@ -0,0 +1,17 @@ +asns,"CREATE DICTIONARY default.asns (`asn` UInt32 INJECTIVE, `name` String) PRIMARY KEY asn SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/asns.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" +icmp,"CREATE DICTIONARY default.icmp (`proto` UInt8, `type` UInt8, `code` UInt8, `name` String) PRIMARY KEY proto, type, code SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/icmp.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(COMPLEX_KEY_HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" +networks,"CREATE DICTIONARY default.networks (`network` String, `name` String, `role` String, `site` String, `region` String, `city` String, `state` String, `country` String, `tenant` String, `asn` UInt32) PRIMARY KEY network SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/networks.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(IP_TRIE()) SETTINGS(format_csv_allow_single_quotes = 0)" +protocols,"CREATE DICTIONARY default.protocols (`proto` UInt8 INJECTIVE, `name` String, `description` String) PRIMARY KEY proto SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/protocols.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" +tcp,"CREATE DICTIONARY default.tcp (`port` UInt16 INJECTIVE, `name` String) PRIMARY KEY port SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/tcp.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" +udp,"CREATE DICTIONARY default.udp (`port` UInt16 INJECTIVE, `name` String) PRIMARY KEY port SOURCE(HTTP(URL 'http://127.0.0.1:0/api/v0/orchestrator/clickhouse/udp.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)" +exporters,"CREATE TABLE default.exporters (`TimeReceived` DateTime, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `IfName` LowCardinality(String), `IfDescription` LowCardinality(String), `IfSpeed` UInt32, `IfConnectivity` LowCardinality(String), `IfProvider` LowCardinality(String), `IfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2)) ENGINE = ReplacingMergeTree(TimeReceived) ORDER BY (ExporterAddress, IfName) TTL TimeReceived + toIntervalDay(1) SETTINGS index_granularity = 8192" +flows,"CREATE TABLE default.flows (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAddr` IPv6 CODEC(ZSTD(1)), `DstAddr` IPv6 CODEC(ZSTD(1)), `SrcNetMask` UInt8, `DstNetMask` UInt8, `SrcNetPrefix` String ALIAS multiIf(EType = 2048, concat(replaceRegexpOne(CAST(IPv6CIDRToRange(SrcAddr, CAST(96 + SrcNetMask, 'UInt8')).1, 'String'), '^::ffff:', ''), '/', CAST(SrcNetMask, 'String')), EType = 34525, concat(CAST(IPv6CIDRToRange(SrcAddr, SrcNetMask).1, 'String'), '/', CAST(SrcNetMask, 'String')), ''), `DstNetPrefix` String ALIAS multiIf(EType = 2048, concat(replaceRegexpOne(CAST(IPv6CIDRToRange(DstAddr, CAST(96 + DstNetMask, 'UInt8')).1, 'String'), '^::ffff:', ''), '/', CAST(DstNetMask, 'String')), EType = 34525, concat(CAST(IPv6CIDRToRange(DstAddr, DstNetMask).1, 'String'), '/', CAST(DstNetMask, 'String')), ''), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `DstASPath` Array(UInt32), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `DstCommunities` Array(UInt32), `DstLargeCommunities` Array(UInt128), `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `SrcPort` UInt16, `DstPort` UInt16, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `PacketSize` UInt64 ALIAS intDiv(Bytes, Packets), `PacketSizeBucket` LowCardinality(String) ALIAS multiIf(PacketSize < 64, '0-63', PacketSize < 128, '64-127', PacketSize < 256, '128-255', PacketSize < 512, '256-511', PacketSize < 768, '512-767', PacketSize < 1024, '768-1023', PacketSize < 1280, '1024-1279', PacketSize < 1501, '1280-1500', PacketSize < 2048, '1501-2047', PacketSize < 3072, '2048-3071', PacketSize < 4096, '3072-4095', PacketSize < 8192, '4096-8191', PacketSize < 10240, '8192-10239', PacketSize < 16384, '10240-16383', PacketSize < 32768, '16384-32767', PacketSize < 65536, '32768-65535', '65536-Inf'), `ForwardingStatus` UInt32) ENGINE = MergeTree PARTITION BY toYYYYMMDDhhmmss(toStartOfInterval(TimeReceived, toIntervalSecond(25920))) ORDER BY (toStartOfFiveMinutes(TimeReceived), ExporterAddress, InIfName, OutIfName) TTL TimeReceived + toIntervalSecond(1296000) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1" +flows_1h0m0s,"CREATE TABLE default.flows_1h0m0s (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `PacketSize` UInt64 ALIAS intDiv(Bytes, Packets), `PacketSizeBucket` LowCardinality(String) ALIAS multiIf(PacketSize < 64, '0-63', PacketSize < 128, '64-127', PacketSize < 256, '128-255', PacketSize < 512, '256-511', PacketSize < 768, '512-767', PacketSize < 1024, '768-1023', PacketSize < 1280, '1024-1279', PacketSize < 1501, '1280-1500', PacketSize < 2048, '1501-2047', PacketSize < 3072, '2048-3071', PacketSize < 4096, '3072-4095', PacketSize < 8192, '4096-8191', PacketSize < 10240, '8192-10239', PacketSize < 16384, '10240-16383', PacketSize < 32768, '16384-32767', PacketSize < 65536, '32768-65535', '65536-Inf'), `ForwardingStatus` UInt32) ENGINE = SummingMergeTree((Bytes, Packets)) PARTITION BY toYYYYMMDDhhmmss(toStartOfInterval(TimeReceived, toIntervalSecond(622080))) PRIMARY KEY (TimeReceived, ExporterAddress, EType, Proto, InIfName, SrcAS, ForwardingStatus, OutIfName, DstAS, SamplingRate) ORDER BY (TimeReceived, ExporterAddress, EType, Proto, InIfName, SrcAS, ForwardingStatus, OutIfName, DstAS, SamplingRate, SrcNetName, DstNetName, SrcNetRole, DstNetRole, SrcNetSite, DstNetSite, SrcNetRegion, DstNetRegion, SrcNetTenant, DstNetTenant, SrcCountry, DstCountry, SrcGeoCity, DstGeoCity, SrcGeoState, DstGeoState, Dst1stAS, Dst2ndAS, Dst3rdAS) TTL TimeReceived + toIntervalSecond(31104000) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1" +flows_1m0s,"CREATE TABLE default.flows_1m0s (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `PacketSize` UInt64 ALIAS intDiv(Bytes, Packets), `PacketSizeBucket` LowCardinality(String) ALIAS multiIf(PacketSize < 64, '0-63', PacketSize < 128, '64-127', PacketSize < 256, '128-255', PacketSize < 512, '256-511', PacketSize < 768, '512-767', PacketSize < 1024, '768-1023', PacketSize < 1280, '1024-1279', PacketSize < 1501, '1280-1500', PacketSize < 2048, '1501-2047', PacketSize < 3072, '2048-3071', PacketSize < 4096, '3072-4095', PacketSize < 8192, '4096-8191', PacketSize < 10240, '8192-10239', PacketSize < 16384, '10240-16383', PacketSize < 32768, '16384-32767', PacketSize < 65536, '32768-65535', '65536-Inf'), `ForwardingStatus` UInt32) ENGINE = SummingMergeTree((Bytes, Packets)) PARTITION BY toYYYYMMDDhhmmss(toStartOfInterval(TimeReceived, toIntervalSecond(12096))) PRIMARY KEY (TimeReceived, ExporterAddress, EType, Proto, InIfName, SrcAS, ForwardingStatus, OutIfName, DstAS, SamplingRate) ORDER BY (TimeReceived, ExporterAddress, EType, Proto, InIfName, SrcAS, ForwardingStatus, OutIfName, DstAS, SamplingRate, SrcNetName, DstNetName, SrcNetRole, DstNetRole, SrcNetSite, DstNetSite, SrcNetRegion, DstNetRegion, SrcNetTenant, DstNetTenant, SrcCountry, DstCountry, SrcGeoCity, DstGeoCity, SrcGeoState, DstGeoState, Dst1stAS, Dst2ndAS, Dst3rdAS) TTL TimeReceived + toIntervalSecond(604800) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1" +flows_5m0s,"CREATE TABLE default.flows_5m0s (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `PacketSize` UInt64 ALIAS intDiv(Bytes, Packets), `PacketSizeBucket` LowCardinality(String) ALIAS multiIf(PacketSize < 64, '0-63', PacketSize < 128, '64-127', PacketSize < 256, '128-255', PacketSize < 512, '256-511', PacketSize < 768, '512-767', PacketSize < 1024, '768-1023', PacketSize < 1280, '1024-1279', PacketSize < 1501, '1280-1500', PacketSize < 2048, '1501-2047', PacketSize < 3072, '2048-3071', PacketSize < 4096, '3072-4095', PacketSize < 8192, '4096-8191', PacketSize < 10240, '8192-10239', PacketSize < 16384, '10240-16383', PacketSize < 32768, '16384-32767', PacketSize < 65536, '32768-65535', '65536-Inf'), `ForwardingStatus` UInt32) ENGINE = SummingMergeTree((Bytes, Packets)) PARTITION BY toYYYYMMDDhhmmss(toStartOfInterval(TimeReceived, toIntervalSecond(155520))) PRIMARY KEY (TimeReceived, ExporterAddress, EType, Proto, InIfName, SrcAS, ForwardingStatus, OutIfName, DstAS, SamplingRate) ORDER BY (TimeReceived, ExporterAddress, EType, Proto, InIfName, SrcAS, ForwardingStatus, OutIfName, DstAS, SamplingRate, SrcNetName, DstNetName, SrcNetRole, DstNetRole, SrcNetSite, DstNetSite, SrcNetRegion, DstNetRegion, SrcNetTenant, DstNetTenant, SrcCountry, DstCountry, SrcGeoCity, DstGeoCity, SrcGeoState, DstGeoState, Dst1stAS, Dst2ndAS, Dst3rdAS) TTL TimeReceived + toIntervalSecond(7776000) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1" +flows_I6D3KDQCRUBCNCGF4BSOWTRMVIv5_raw,"CREATE TABLE default.flows_I6D3KDQCRUBCNCGF4BSOWTRMVIv5_raw (`TimeReceived` DateTime CODEC(DoubleDelta, LZ4), `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAddr` IPv6 CODEC(ZSTD(1)), `DstAddr` IPv6 CODEC(ZSTD(1)), `SrcNetMask` UInt8, `DstNetMask` UInt8, `SrcAS` UInt32, `DstAS` UInt32, `DstASPath` Array(UInt32), `DstCommunities` Array(UInt32), `DstLargeCommunities` Array(UInt128), `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `SrcPort` UInt16, `DstPort` UInt16, `Bytes` UInt64 CODEC(T64, LZ4), `Packets` UInt64 CODEC(T64, LZ4), `ForwardingStatus` UInt32) ENGINE = Null" +exporters_consumer,"CREATE MATERIALIZED VIEW default.exporters_consumer TO default.exporters (`TimeReceived` DateTime, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `IfName` String, `IfDescription` String, `IfSpeed` UInt32, `IfConnectivity` String, `IfProvider` String, `IfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2)) AS SELECT DISTINCT TimeReceived, ExporterAddress, ExporterName, ExporterGroup, ExporterRole, ExporterSite, ExporterRegion, ExporterTenant, [InIfName, OutIfName][num] AS IfName, [InIfDescription, OutIfDescription][num] AS IfDescription, [InIfSpeed, OutIfSpeed][num] AS IfSpeed, [InIfConnectivity, OutIfConnectivity][num] AS IfConnectivity, [InIfProvider, OutIfProvider][num] AS IfProvider, [InIfBoundary, OutIfBoundary][num] AS IfBoundary FROM default.flows ARRAY JOIN arrayEnumerate([1, 2]) AS num" +flows_1h0m0s_consumer,"CREATE MATERIALIZED VIEW default.flows_1h0m0s_consumer TO default.flows_1h0m0s (`TimeReceived` DateTime, `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64, `Packets` UInt64, `ForwardingStatus` UInt32) AS SELECT toStartOfInterval(TimeReceived, toIntervalSecond(3600)) AS TimeReceived, SamplingRate, ExporterAddress, ExporterName, ExporterGroup, ExporterRole, ExporterSite, ExporterRegion, ExporterTenant, SrcAS, DstAS, SrcNetName, DstNetName, SrcNetRole, DstNetRole, SrcNetSite, DstNetSite, SrcNetRegion, DstNetRegion, SrcNetTenant, DstNetTenant, SrcCountry, DstCountry, SrcGeoCity, DstGeoCity, SrcGeoState, DstGeoState, Dst1stAS, Dst2ndAS, Dst3rdAS, InIfName, OutIfName, InIfDescription, OutIfDescription, InIfSpeed, OutIfSpeed, InIfConnectivity, OutIfConnectivity, InIfProvider, OutIfProvider, InIfBoundary, OutIfBoundary, EType, Proto, Bytes, Packets, ForwardingStatus FROM default.flows" +flows_1m0s_consumer,"CREATE MATERIALIZED VIEW default.flows_1m0s_consumer TO default.flows_1m0s (`TimeReceived` DateTime, `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64, `Packets` UInt64, `ForwardingStatus` UInt32) AS SELECT toStartOfInterval(TimeReceived, toIntervalSecond(60)) AS TimeReceived, SamplingRate, ExporterAddress, ExporterName, ExporterGroup, ExporterRole, ExporterSite, ExporterRegion, ExporterTenant, SrcAS, DstAS, SrcNetName, DstNetName, SrcNetRole, DstNetRole, SrcNetSite, DstNetSite, SrcNetRegion, DstNetRegion, SrcNetTenant, DstNetTenant, SrcCountry, DstCountry, SrcGeoCity, DstGeoCity, SrcGeoState, DstGeoState, Dst1stAS, Dst2ndAS, Dst3rdAS, InIfName, OutIfName, InIfDescription, OutIfDescription, InIfSpeed, OutIfSpeed, InIfConnectivity, OutIfConnectivity, InIfProvider, OutIfProvider, InIfBoundary, OutIfBoundary, EType, Proto, Bytes, Packets, ForwardingStatus FROM default.flows" +flows_5m0s_consumer,"CREATE MATERIALIZED VIEW default.flows_5m0s_consumer TO default.flows_5m0s (`TimeReceived` DateTime, `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` LowCardinality(String), `DstNetName` LowCardinality(String), `SrcNetRole` LowCardinality(String), `DstNetRole` LowCardinality(String), `SrcNetSite` LowCardinality(String), `DstNetSite` LowCardinality(String), `SrcNetRegion` LowCardinality(String), `DstNetRegion` LowCardinality(String), `SrcNetTenant` LowCardinality(String), `DstNetTenant` LowCardinality(String), `SrcCountry` FixedString(2), `DstCountry` FixedString(2), `SrcGeoCity` LowCardinality(String), `DstGeoCity` LowCardinality(String), `SrcGeoState` LowCardinality(String), `DstGeoState` LowCardinality(String), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `Bytes` UInt64, `Packets` UInt64, `ForwardingStatus` UInt32) AS SELECT toStartOfInterval(TimeReceived, toIntervalSecond(300)) AS TimeReceived, SamplingRate, ExporterAddress, ExporterName, ExporterGroup, ExporterRole, ExporterSite, ExporterRegion, ExporterTenant, SrcAS, DstAS, SrcNetName, DstNetName, SrcNetRole, DstNetRole, SrcNetSite, DstNetSite, SrcNetRegion, DstNetRegion, SrcNetTenant, DstNetTenant, SrcCountry, DstCountry, SrcGeoCity, DstGeoCity, SrcGeoState, DstGeoState, Dst1stAS, Dst2ndAS, Dst3rdAS, InIfName, OutIfName, InIfDescription, OutIfDescription, InIfSpeed, OutIfSpeed, InIfConnectivity, OutIfConnectivity, InIfProvider, OutIfProvider, InIfBoundary, OutIfBoundary, EType, Proto, Bytes, Packets, ForwardingStatus FROM default.flows" +flows_I6D3KDQCRUBCNCGF4BSOWTRMVIv5_raw_consumer,"CREATE MATERIALIZED VIEW default.flows_I6D3KDQCRUBCNCGF4BSOWTRMVIv5_raw_consumer TO default.flows (`TimeReceived` DateTime, `SamplingRate` UInt64, `ExporterAddress` LowCardinality(IPv6), `ExporterName` LowCardinality(String), `ExporterGroup` LowCardinality(String), `ExporterRole` LowCardinality(String), `ExporterSite` LowCardinality(String), `ExporterRegion` LowCardinality(String), `ExporterTenant` LowCardinality(String), `SrcAddr` IPv6, `DstAddr` IPv6, `SrcNetMask` UInt8, `DstNetMask` UInt8, `SrcAS` UInt32, `DstAS` UInt32, `SrcNetName` String, `DstNetName` String, `SrcNetRole` String, `DstNetRole` String, `SrcNetSite` String, `DstNetSite` String, `SrcNetRegion` String, `DstNetRegion` String, `SrcNetTenant` String, `DstNetTenant` String, `SrcCountry` String, `DstCountry` String, `SrcGeoCity` String, `DstGeoCity` String, `SrcGeoState` String, `DstGeoState` String, `DstASPath` Array(UInt32), `Dst1stAS` UInt32, `Dst2ndAS` UInt32, `Dst3rdAS` UInt32, `DstCommunities` Array(UInt32), `DstLargeCommunities` Array(UInt128), `InIfName` LowCardinality(String), `OutIfName` LowCardinality(String), `InIfDescription` LowCardinality(String), `OutIfDescription` LowCardinality(String), `InIfSpeed` UInt32, `OutIfSpeed` UInt32, `InIfConnectivity` LowCardinality(String), `OutIfConnectivity` LowCardinality(String), `InIfProvider` LowCardinality(String), `OutIfProvider` LowCardinality(String), `InIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `OutIfBoundary` Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), `EType` UInt32, `Proto` UInt32, `SrcPort` UInt16, `DstPort` UInt16, `Bytes` UInt64, `Packets` UInt64, `ForwardingStatus` UInt32) AS WITH arrayCompact(DstASPath) AS c_DstASPath, dictGet('default.networks', ('asn', 'name', 'role', 'site', 'region', 'tenant', 'country', 'city', 'state'), SrcAddr) AS c_SrcNetworks, dictGet('default.networks', ('asn', 'name', 'role', 'site', 'region', 'tenant', 'country', 'city', 'state'), DstAddr) AS c_DstNetworks SELECT TimeReceived, SamplingRate, ExporterAddress, ExporterName, ExporterGroup, ExporterRole, ExporterSite, ExporterRegion, ExporterTenant, SrcAddr, DstAddr, SrcNetMask, DstNetMask, if(SrcAS = 0, c_SrcNetworks.1, SrcAS) AS SrcAS, if(DstAS = 0, c_DstNetworks.1, DstAS) AS DstAS, c_SrcNetworks.2 AS SrcNetName, c_DstNetworks.2 AS DstNetName, c_SrcNetworks.3 AS SrcNetRole, c_DstNetworks.3 AS DstNetRole, c_SrcNetworks.4 AS SrcNetSite, c_DstNetworks.4 AS DstNetSite, c_SrcNetworks.5 AS SrcNetRegion, c_DstNetworks.5 AS DstNetRegion, c_SrcNetworks.6 AS SrcNetTenant, c_DstNetworks.6 AS DstNetTenant, c_SrcNetworks.7 AS SrcCountry, c_DstNetworks.7 AS DstCountry, c_SrcNetworks.8 AS SrcGeoCity, c_DstNetworks.8 AS DstGeoCity, c_SrcNetworks.9 AS SrcGeoState, c_DstNetworks.9 AS DstGeoState, DstASPath, c_DstASPath[1] AS Dst1stAS, c_DstASPath[2] AS Dst2ndAS, c_DstASPath[3] AS Dst3rdAS, DstCommunities, DstLargeCommunities, InIfName, OutIfName, InIfDescription, OutIfDescription, InIfSpeed, OutIfSpeed, InIfConnectivity, OutIfConnectivity, InIfProvider, OutIfProvider, InIfBoundary, OutIfBoundary, EType, Proto, SrcPort, DstPort, Bytes, Packets, ForwardingStatus FROM default.flows_I6D3KDQCRUBCNCGF4BSOWTRMVIv5_raw" diff --git a/orchestrator/kafka/functional_test.go b/orchestrator/kafka/functional_test.go index 0588d776..b7ea84ca 100644 --- a/orchestrator/kafka/functional_test.go +++ b/orchestrator/kafka/functional_test.go @@ -12,6 +12,7 @@ import ( "akvorado/common/helpers" "akvorado/common/kafka" + "akvorado/common/pb" "akvorado/common/reporter" "akvorado/common/schema" ) @@ -24,7 +25,7 @@ func TestTopicCreation(t *testing.T) { segmentBytes := "107374184" segmentBytes2 := "10737184" cleanupPolicy := "delete" - expectedTopicName := fmt.Sprintf("%s-%s", topicName, schema.NewMock(t).ProtobufMessageHash()) + expectedTopicName := fmt.Sprintf("%s-v%d", topicName, pb.Version) cases := []struct { Name string @@ -103,7 +104,7 @@ func TestTopicMorePartitions(t *testing.T) { client, brokers := kafka.SetupKafkaBroker(t) topicName := fmt.Sprintf("test-topic-%d", rand.Int()) - expectedTopicName := fmt.Sprintf("%s-%s", topicName, schema.NewMock(t).ProtobufMessageHash()) + expectedTopicName := fmt.Sprintf("%s-v%d", topicName, pb.Version) configuration := DefaultConfiguration() configuration.Topic = topicName diff --git a/orchestrator/kafka/root.go b/orchestrator/kafka/root.go index d6264e34..5d9a8c2a 100644 --- a/orchestrator/kafka/root.go +++ b/orchestrator/kafka/root.go @@ -11,6 +11,7 @@ import ( "github.com/IBM/sarama" "akvorado/common/kafka" + "akvorado/common/pb" "akvorado/common/reporter" "akvorado/common/schema" ) @@ -40,14 +41,15 @@ func New(r *reporter.Reporter, config Configuration, dependencies Dependencies) return nil, fmt.Errorf("cannot validate Kafka configuration: %w", err) } - return &Component{ + c := Component{ r: r, d: dependencies, config: config, kafkaConfig: kafkaConfig, - kafkaTopic: fmt.Sprintf("%s-%s", config.Topic, dependencies.Schema.ProtobufMessageHash()), - }, nil + kafkaTopic: fmt.Sprintf("%s-v%d", config.Topic, pb.Version), + } + return &c, nil } // Start starts Kafka configuration. diff --git a/orchestrator/root.go b/orchestrator/root.go index 90dd67aa..7a6b6cc1 100644 --- a/orchestrator/root.go +++ b/orchestrator/root.go @@ -32,6 +32,8 @@ type ServiceType string var ( // InletService represents the inlet service type InletService ServiceType = "inlet" + // OutletService represents the outlet service type + OutletService ServiceType = "outlet" // OrchestratorService represents the orchestrator service type OrchestratorService ServiceType = "orchestrator" // ConsoleService represents the console service type diff --git a/outlet/clickhouse/config.go b/outlet/clickhouse/config.go new file mode 100644 index 00000000..6bdb7c56 --- /dev/null +++ b/outlet/clickhouse/config.go @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: 2025 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package clickhouse + +import ( + "time" +) + +// Configuration describes the configuration for the ClickHouse exporter. +type Configuration struct { + // MaximumBatchSize is the maximum number of rows to send to ClickHouse in one batch. + MaximumBatchSize uint `validate:"min=1"` + // MaximumWaitTime is the maximum number of seconds to wait before sending the current batch. + MaximumWaitTime time.Duration `validate:"min=100ms"` +} + +// DefaultConfiguration represents the default configuration for the ClickHouse exporter. +func DefaultConfiguration() Configuration { + return Configuration{ + MaximumBatchSize: 5000, + MaximumWaitTime: time.Second, + } +} diff --git a/outlet/clickhouse/example_test.go b/outlet/clickhouse/example_test.go new file mode 100644 index 00000000..437034af --- /dev/null +++ b/outlet/clickhouse/example_test.go @@ -0,0 +1,140 @@ +// SPDX-FileCopyrightText: 2016-2023 ClickHouse, Inc. +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileComment: This is basically a copy of https://github.com/ClickHouse/ch-go/blob/main/examples/insert/main.go + +package clickhouse_test + +import ( + "context" + "io" + "testing" + "time" + + "github.com/ClickHouse/ch-go" + "github.com/ClickHouse/ch-go/proto" + + "akvorado/common/helpers" +) + +func TestInsertMemory(t *testing.T) { + server := helpers.CheckExternalService(t, "ClickHouse", []string{"clickhouse:9000", "127.0.0.1:9000"}) + ctx := context.Background() + + conn, err := ch.Dial(ctx, ch.Options{ + Address: server, + DialTimeout: 100 * time.Millisecond, + }) + if err != nil { + t.Fatalf("Dial() error:\n%+v", err) + } + + if err := conn.Do(ctx, ch.Query{ + Body: `CREATE OR REPLACE TABLE test_table_insert +( + ts DateTime64(9), + severity_text Enum8('INFO'=1, 'DEBUG'=2), + severity_number UInt8, + service_name LowCardinality(String), + body String, + name String, + arr Array(String) +) ENGINE = Memory`, + }); err != nil { + t.Fatalf("Do() error:\n%+v", err) + } + + // Define all columns of table. + var ( + body proto.ColStr + name proto.ColStr + sevText proto.ColEnum + sevNumber proto.ColUInt8 + + // or new(proto.ColStr).LowCardinality() + serviceName = proto.NewLowCardinality(new(proto.ColStr)) + ts = new(proto.ColDateTime64).WithPrecision(proto.PrecisionNano) // DateTime64(9) + arr = new(proto.ColStr).Array() // Array(String) + now = time.Date(2010, 1, 1, 10, 22, 33, 345678, time.UTC) + ) + + input := proto.Input{ + {Name: "ts", Data: ts}, + {Name: "severity_text", Data: &sevText}, + {Name: "severity_number", Data: &sevNumber}, + {Name: "service_name", Data: serviceName}, + {Name: "body", Data: &body}, + {Name: "name", Data: &name}, + {Name: "arr", Data: arr}, + } + + t.Run("one block", func(t *testing.T) { + // Append 10 rows to initial data block. + for range 10 { + body.AppendBytes([]byte("Hello")) + ts.Append(now) + name.Append("name") + sevText.Append("INFO") + sevNumber.Append(10) + arr.Append([]string{"foo", "bar", "baz"}) + serviceName.Append("service") + } + + // Insert single data block. + if err := conn.Do(ctx, ch.Query{ + Body: "INSERT INTO test_table_insert VALUES", + Input: input, + }); err != nil { + t.Fatalf("Do() error:\n%+v", err) + } + }) + + t.Run("streaming", func(t *testing.T) { + // Stream data to ClickHouse server in multiple data blocks. + var blocks int + if err := conn.Do(ctx, ch.Query{ + Body: input.Into("test_table_insert"), // helper that generates INSERT INTO query with all columns + Input: input, + + // OnInput is called to prepare Input data before encoding and sending + // to ClickHouse server. + OnInput: func(ctx context.Context) error { + // On OnInput call, you should fill the input data. + // + // NB: You should reset the input columns, they are + // not reset automatically. + // + // That is, we are re-using the same input columns and + // if we will return nil without doing anything, data will be + // just duplicated. + + input.Reset() // calls "Reset" on each column + + if blocks >= 10 { + // Stop streaming. + // + // This will also write tailing input data if any, + // but we just reset the input, so it is currently blank. + return io.EOF + } + + // Append new values: + for range 10 { + body.AppendBytes([]byte("Hello")) + ts.Append(now) + name.Append("name") + sevText.Append("DEBUG") + sevNumber.Append(10) + arr.Append([]string{"foo", "bar", "baz"}) + serviceName.Append("service") + } + + // Data will be encoded and sent to ClickHouse server after returning nil. + // The Do method will return error if any. + blocks++ + return nil + }, + }); err != nil { + t.Fatalf("Do() error:\n%+v", err) + } + }) +} diff --git a/outlet/clickhouse/functional_test.go b/outlet/clickhouse/functional_test.go new file mode 100644 index 00000000..8cd722e4 --- /dev/null +++ b/outlet/clickhouse/functional_test.go @@ -0,0 +1,211 @@ +// SPDX-FileCopyrightText: 2025 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package clickhouse_test + +import ( + "context" + "fmt" + "testing" + "time" + + clickhousego "github.com/ClickHouse/clickhouse-go/v2" + + "akvorado/common/clickhousedb" + "akvorado/common/daemon" + "akvorado/common/helpers" + "akvorado/common/reporter" + "akvorado/common/schema" + "akvorado/outlet/clickhouse" +) + +func TestInsert(t *testing.T) { + server := helpers.CheckExternalService(t, "ClickHouse", []string{"clickhouse:9000", "127.0.0.1:9000"}) + r := reporter.NewMock(t) + sch := schema.NewMock(t) + bf := sch.NewFlowMessage() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + ctx = clickhousego.Context(ctx, clickhousego.WithSettings(clickhousego.Settings{ + "allow_suspicious_low_cardinality_types": 1, + })) + + // Create components + dbConf := clickhousedb.DefaultConfiguration() + dbConf.Servers = []string{server} + dbConf.DialTimeout = 100 * time.Millisecond + chdb, err := clickhousedb.New(r, dbConf, clickhousedb.Dependencies{ + Daemon: daemon.NewMock(t), + }) + if err != nil { + t.Fatalf("clickhousedb.New() error:\n%+v", err) + } + helpers.StartStop(t, chdb) + conf := clickhouse.DefaultConfiguration() + conf.MaximumBatchSize = 10 + conf.MaximumWaitTime = time.Second + ch, err := clickhouse.New(r, conf, clickhouse.Dependencies{ + ClickHouse: chdb, + Schema: sch, + }) + if err != nil { + t.Fatalf("clickhouse.New() error:\n%+v", err) + } + helpers.StartStop(t, ch) + + // Create table + tableName := fmt.Sprintf("flows_%s_raw", sch.ClickHouseHash()) + err = chdb.Exec(ctx, fmt.Sprintf("CREATE OR REPLACE TABLE %s (%s) ENGINE = Memory", tableName, + sch.ClickHouseCreateTable( + schema.ClickHouseSkipGeneratedColumns, + schema.ClickHouseSkipAliasedColumns))) + if err != nil { + t.Fatalf("chdb.Exec() error:\n%+v", err) + } + // Drop any left-over consumer (from orchestrator tests). Otherwise, we get an error like this: + // Bad URI syntax: bad or invalid port number: 0 + err = chdb.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s_consumer", tableName)) + if err != nil { + t.Fatalf("chdb.Exec() error:\n%+v", err) + } + + // Expected records + type result struct { + TimeReceived time.Time + SrcAS uint32 + DstAS uint32 + ExporterName string + EType uint32 + } + expected := []result{} + + // Create one worker and send some values + w := ch.NewWorker(1, bf) + for i := range 23 { + i = i + 1 + // 1: first batch (max time) + // 2 to 11: second batch (max batch) + // 12 to 15: third batch (max time) + // 16 to 23: third batch (last one) + bf.TimeReceived = uint32(100 + i) + bf.SrcAS = uint32(65400 + i) + bf.DstAS = uint32(65500 + i) + bf.AppendString(schema.ColumnExporterName, fmt.Sprintf("exporter-%d", i)) + bf.AppendString(schema.ColumnExporterName, "emptyness") + bf.AppendUint(schema.ColumnEType, helpers.ETypeIPv6) + expected = append(expected, result{ + TimeReceived: time.Unix(int64(bf.TimeReceived), 0).UTC(), + SrcAS: bf.SrcAS, + DstAS: bf.DstAS, + ExporterName: fmt.Sprintf("exporter-%d", i), + EType: helpers.ETypeIPv6, + }) + if i == 15 { + time.Sleep(time.Second) + } + w.FinalizeAndSend(ctx) + if i == 23 { + w.Flush(ctx) + } + + // Check metrics + gotMetrics := r.GetMetrics("akvorado_outlet_clickhouse_", "-insert_time", "-wait_time") + var expectedMetrics map[string]string + if i < 11 { + expectedMetrics = map[string]string{ + "batches_total": "1", + "flows_total": "1", + } + } else if i < 15 { + expectedMetrics = map[string]string{ + "batches_total": "2", + "flows_total": "11", + } + } else if i < 23 { + expectedMetrics = map[string]string{ + "batches_total": "3", + "flows_total": "15", + } + } else { + expectedMetrics = map[string]string{ + "batches_total": "4", + "flows_total": "23", + } + } + if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { + t.Errorf("Metrics, iteration %d, (-got, +want):\n%s", i, diff) + } + + // Check if we have anything inserted in the table + var results []result + err := chdb.Select(ctx, &results, + fmt.Sprintf("SELECT TimeReceived, SrcAS, DstAS, ExporterName, EType FROM %s ORDER BY TimeReceived ASC", tableName)) + if err != nil { + t.Fatalf("chdb.Select() error:\n%+v", err) + } + reallyExpected := expected + if i < 11 { + reallyExpected = expected[:min(len(expected), 1)] + } else if i < 15 { + reallyExpected = expected[:min(len(expected), 11)] + } else if i < 23 { + reallyExpected = expected[:min(len(expected), 15)] + } + if diff := helpers.Diff(results, reallyExpected); diff != "" { + t.Fatalf("chdb.Select(), iteration %d, (-got, +want):\n%s", i, diff) + } + } +} + +func TestMultipleServers(t *testing.T) { + servers := []string{ + helpers.CheckExternalService(t, "ClickHouse", []string{"clickhouse:9000", "127.0.0.1:9000"}), + } + for range 100 { + servers = append(servers, "127.0.0.1:0") + } + for range 10 { + r := reporter.NewMock(t) + sch := schema.NewMock(t) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + ctx = clickhousego.Context(ctx, clickhousego.WithSettings(clickhousego.Settings{ + "allow_suspicious_low_cardinality_types": 1, + })) + + // Create components + dbConf := clickhousedb.DefaultConfiguration() + dbConf.Servers = servers + dbConf.DialTimeout = 100 * time.Millisecond + chdb, err := clickhousedb.New(r, dbConf, clickhousedb.Dependencies{ + Daemon: daemon.NewMock(t), + }) + if err != nil { + t.Fatalf("clickhousedb.New() error:\n%+v", err) + } + helpers.StartStop(t, chdb) + conf := clickhouse.DefaultConfiguration() + conf.MaximumBatchSize = 10 + ch, err := clickhouse.New(r, conf, clickhouse.Dependencies{ + ClickHouse: chdb, + Schema: sch, + }) + if err != nil { + t.Fatalf("clickhouse.New() error:\n%+v", err) + } + helpers.StartStop(t, ch) + + // Trigger an empty send + bf := sch.NewFlowMessage() + w := ch.NewWorker(1, bf) + w.Flush(ctx) + + // Check metrics + gotMetrics := r.GetMetrics("akvorado_outlet_clickhouse_", "errors_total") + if gotMetrics[`errors_total{error="connect"}`] == "0" { + continue + } + return + } + t.Fatalf("w.Flush(): cannot trigger connect error") +} diff --git a/outlet/clickhouse/metrics.go b/outlet/clickhouse/metrics.go new file mode 100644 index 00000000..a9bcfc24 --- /dev/null +++ b/outlet/clickhouse/metrics.go @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: 2025 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package clickhouse + +import "akvorado/common/reporter" + +type metrics struct { + batches reporter.Counter + flows reporter.Counter + waitTime reporter.Histogram + insertTime reporter.Histogram + errors *reporter.CounterVec +} + +func (c *realComponent) initMetrics() { + c.metrics.batches = c.r.Counter( + reporter.CounterOpts{ + Name: "batches_total", + Help: "Number of batches of flows sent to ClickHouse", + }, + ) + c.metrics.flows = c.r.Counter( + reporter.CounterOpts{ + Name: "flows_total", + Help: "Number of flows sent to ClickHouse", + }, + ) + c.metrics.waitTime = c.r.Histogram( + reporter.HistogramOpts{ + Name: "wait_time_seconds", + Help: "Time spent waiting before sending a batch to ClickHouse", + }, + ) + c.metrics.insertTime = c.r.Histogram( + reporter.HistogramOpts{ + Name: "insert_time_seconds", + Help: "Time spent inserting data to ClickHouse", + }, + ) + c.metrics.errors = c.r.CounterVec( + reporter.CounterOpts{ + Name: "errors_total", + Help: "Errors while inserting into ClickHouse", + }, + []string{"error"}, + ) +} diff --git a/outlet/clickhouse/root.go b/outlet/clickhouse/root.go new file mode 100644 index 00000000..beeb6c78 --- /dev/null +++ b/outlet/clickhouse/root.go @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: 2025 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +// Package clickhouse handles flow exports to ClickHouse. This component is +// "inert" and does not track its spawned workers. It is the responsability of +// the dependent component to flush data before shutting down. +package clickhouse + +import ( + "akvorado/common/clickhousedb" + "akvorado/common/reporter" + "akvorado/common/schema" +) + +// Component is the interface for the ClickHouse exporter component. +type Component interface { + NewWorker(int, *schema.FlowMessage) Worker +} + +// realComponent implements the ClickHouse exporter +type realComponent struct { + r *reporter.Reporter + d *Dependencies + config Configuration + + metrics metrics +} + +// Dependencies defines the dependencies of the ClickHouse exporter +type Dependencies struct { + ClickHouse *clickhousedb.Component + Schema *schema.Component +} + +// New creates a new core component. +func New(r *reporter.Reporter, configuration Configuration, dependencies Dependencies) (Component, error) { + c := realComponent{ + r: r, + d: &dependencies, + config: configuration, + } + c.initMetrics() + return &c, nil +} diff --git a/outlet/clickhouse/root_test.go b/outlet/clickhouse/root_test.go new file mode 100644 index 00000000..4593d75a --- /dev/null +++ b/outlet/clickhouse/root_test.go @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: 2025 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package clickhouse_test + +import ( + "fmt" + "sync" + "testing" + + "akvorado/common/helpers" + "akvorado/common/schema" + "akvorado/outlet/clickhouse" +) + +func TestMock(t *testing.T) { + sch := schema.NewMock(t) + bf := sch.NewFlowMessage() + + var messages []*schema.FlowMessage + var messagesMutex sync.Mutex + ch := clickhouse.NewMock(t, func(msg *schema.FlowMessage) { + messagesMutex.Lock() + defer messagesMutex.Unlock() + messages = append(messages, msg) + }) + helpers.StartStop(t, ch) + + expected := []*schema.FlowMessage{} + w := ch.NewWorker(1, bf) + for i := range 20 { + i = i + 1 // 1 to 20 + bf.TimeReceived = uint32(100 + i) + bf.SrcAS = uint32(65400 + i) + bf.DstAS = uint32(65500 + i) + bf.AppendString(schema.ColumnExporterName, fmt.Sprintf("exporter-%d", i)) + expected = append(expected, &schema.FlowMessage{ + TimeReceived: bf.TimeReceived, + SrcAS: bf.SrcAS, + DstAS: bf.DstAS, + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnExporterName: fmt.Sprintf("exporter-%d", i), + }, + }) + w.FinalizeAndSend(t.Context()) + + // Check if we have anything inserted in the table + messagesMutex.Lock() + if diff := helpers.Diff(messages, expected); diff != "" { + t.Fatalf("Mock(), iteration %d, (-got, +want):\n%s", i, diff) + } + messagesMutex.Unlock() + } +} diff --git a/outlet/clickhouse/tests.go b/outlet/clickhouse/tests.go new file mode 100644 index 00000000..52a3d565 --- /dev/null +++ b/outlet/clickhouse/tests.go @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: 2025 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +//go:build !release + +package clickhouse + +import ( + "context" + "testing" + + "akvorado/common/schema" +) + +// mockComponent is a mock version of the ClickHouse exporter. +type mockComponent struct { + callback func(*schema.FlowMessage) +} + +// NewMock creates a new mock exporter that calls the provided callback function with each received flow message. +func NewMock(_ *testing.T, callback func(*schema.FlowMessage)) Component { + return &mockComponent{ + callback: callback, + } +} + +// NewWorker creates a new mock worker. +func (c *mockComponent) NewWorker(_ int, bf *schema.FlowMessage) Worker { + return &mockWorker{ + c: c, + bf: bf, + } +} + +// mockWorker is a mock version of the ClickHouse worker. +type mockWorker struct { + c *mockComponent + bf *schema.FlowMessage +} + +// FinalizeAndSend always "send" the current flows. +func (w *mockWorker) FinalizeAndSend(ctx context.Context) { + w.Flush(ctx) +} + +// Send will record the sent flows for testing purpose. +func (w *mockWorker) Flush(_ context.Context) { + clone := *w.bf + w.c.callback(&clone) + w.bf.Clear() // Clear instead of finalizing +} diff --git a/outlet/clickhouse/worker.go b/outlet/clickhouse/worker.go new file mode 100644 index 00000000..c543a1c8 --- /dev/null +++ b/outlet/clickhouse/worker.go @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: 2025 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package clickhouse + +import ( + "context" + "fmt" + "math/rand" + "time" + + "github.com/ClickHouse/ch-go" + "github.com/cenkalti/backoff/v4" + + "akvorado/common/reporter" + "akvorado/common/schema" +) + +// Worker represents a worker sending to ClickHouse. It is synchronous (no +// goroutines) and most functions are bound to a context. +type Worker interface { + FinalizeAndSend(context.Context) + Flush(context.Context) +} + +// realWorker is a working implementation of Worker. +type realWorker struct { + c *realComponent + bf *schema.FlowMessage + last time.Time + logger reporter.Logger + + conn *ch.Client + servers []string + options ch.Options +} + +// NewWorker creates a new worker to push data to ClickHouse. +func (c *realComponent) NewWorker(i int, bf *schema.FlowMessage) Worker { + opts, servers := c.d.ClickHouse.ChGoOptions() + w := realWorker{ + c: c, + bf: bf, + logger: c.r.With().Int("worker", i).Logger(), + + servers: servers, + options: opts, + } + return &w +} + +// FinalizeAndSend sends data to ClickHouse after finalizing if we have a full batch or exceeded the maximum wait time. +func (w *realWorker) FinalizeAndSend(ctx context.Context) { + w.bf.Finalize() + now := time.Now() + if w.bf.FlowCount() >= int(w.c.config.MaximumBatchSize) || w.last.Add(w.c.config.MaximumWaitTime).Before(now) { + // Record wait time since last send + if !w.last.IsZero() { + waitTime := now.Sub(w.last) + w.c.metrics.waitTime.Observe(waitTime.Seconds()) + } + w.Flush(ctx) + w.last = now + } +} + +// Flush sends remaining data to ClickHouse without an additional condition. It +// should be called before shutting down to flush remaining data. Otherwise, +// FinalizeAndSend() should be used instead. +func (w *realWorker) Flush(ctx context.Context) { + // We try to send as long as possible. The only exit condition is an + // expiration of the context. + b := backoff.NewExponentialBackOff() + b.MaxElapsedTime = 0 + b.MaxInterval = 30 * time.Second + b.InitialInterval = 20 * time.Millisecond + backoff.Retry(func() error { + // Connect or reconnect if connection is broken. + if err := w.connect(ctx); err != nil { + w.logger.Err(err).Msg("cannot connect to ClickHouse") + return err + } + + // Send to ClickHouse in flows_XXXXX_raw. + start := time.Now() + if err := w.conn.Do(ctx, ch.Query{ + Body: w.bf.ClickHouseProtoInput().Into(fmt.Sprintf("flows_%s_raw", w.c.d.Schema.ClickHouseHash())), + Input: w.bf.ClickHouseProtoInput(), + }); err != nil { + w.logger.Err(err).Msg("cannot send batch to ClickHouse") + w.c.metrics.errors.WithLabelValues("send").Inc() + return err + } + pushDuration := time.Since(start) + w.c.metrics.insertTime.Observe(pushDuration.Seconds()) + w.c.metrics.batches.Inc() + w.c.metrics.flows.Add(float64(w.bf.FlowCount())) + + // Clear batch + w.bf.Clear() + return nil + }, backoff.WithContext(b, ctx)) +} + +// connect establishes or reestablish the connection to ClickHouse. +func (w *realWorker) connect(ctx context.Context) error { + // If connection exists and is healthy, reuse it + if w.conn != nil { + if err := w.conn.Ping(ctx); err == nil { + return nil + } + // Connection is unhealthy, close it + w.conn.Close() + w.conn = nil + } + + // Try each server until one connects successfully + var lastErr error + for _, idx := range rand.Perm(len(w.servers)) { + w.options.Address = w.servers[idx] + conn, err := ch.Dial(ctx, w.options) + if err != nil { + w.logger.Err(err).Str("server", w.options.Address).Msg("failed to connect to ClickHouse server") + w.c.metrics.errors.WithLabelValues("connect").Inc() + lastErr = err + continue + } + + // Test the connection + if err := conn.Ping(ctx); err != nil { + w.logger.Err(err).Str("server", w.options.Address).Msg("ClickHouse server ping failed") + w.c.metrics.errors.WithLabelValues("ping").Inc() + conn.Close() + conn = nil + lastErr = err + continue + } + + // Success + w.conn = conn + w.logger.Info().Str("server", w.options.Address).Msg("connected to ClickHouse server") + return nil + } + + return lastErr +} diff --git a/inlet/core/classifier.go b/outlet/core/classifier.go similarity index 100% rename from inlet/core/classifier.go rename to outlet/core/classifier.go diff --git a/inlet/core/classifier_test.go b/outlet/core/classifier_test.go similarity index 100% rename from inlet/core/classifier_test.go rename to outlet/core/classifier_test.go diff --git a/inlet/core/config.go b/outlet/core/config.go similarity index 98% rename from inlet/core/config.go rename to outlet/core/config.go index d857e8b8..172289a4 100644 --- a/inlet/core/config.go +++ b/outlet/core/config.go @@ -16,8 +16,6 @@ import ( // Configuration describes the configuration for the core component. type Configuration struct { - // Number of workers for the core component - Workers int `validate:"min=1"` // ExporterClassifiers defines rules for exporter classification ExporterClassifiers []ExporterClassifierRule // InterfaceClassifiers defines rules for interface classification @@ -39,7 +37,6 @@ type Configuration struct { // DefaultConfiguration represents the default configuration for the core component. func DefaultConfiguration() Configuration { return Configuration{ - Workers: 1, ExporterClassifiers: []ExporterClassifierRule{}, InterfaceClassifiers: []InterfaceClassifierRule{}, ClassifierCacheDuration: 5 * time.Minute, @@ -154,4 +151,5 @@ func init() { helpers.RegisterMapstructureUnmarshallerHook(ASNProviderUnmarshallerHook()) helpers.RegisterMapstructureUnmarshallerHook(NetProviderUnmarshallerHook()) helpers.RegisterMapstructureUnmarshallerHook(helpers.SubnetMapUnmarshallerHook[uint]()) + helpers.RegisterMapstructureDeprecatedFields[Configuration]("Workers") } diff --git a/inlet/core/config_test.go b/outlet/core/config_test.go similarity index 100% rename from inlet/core/config_test.go rename to outlet/core/config_test.go diff --git a/inlet/core/enricher.go b/outlet/core/enricher.go similarity index 78% rename from inlet/core/enricher.go rename to outlet/core/enricher.go index fb0bc2c6..d0a85a07 100644 --- a/inlet/core/enricher.go +++ b/outlet/core/enricher.go @@ -19,7 +19,7 @@ type exporterAndInterfaceInfo struct { } // enrichFlow adds more data to a flow. -func (c *Component) enrichFlow(exporterIP netip.Addr, exporterStr string, flow *schema.FlowMessage) (skip bool) { +func (w *worker) enrichFlow(exporterIP netip.Addr, exporterStr string) (skip bool) { var flowExporterName string var flowInIfName, flowInIfDescription, flowOutIfName, flowOutIfDescription string var flowInIfSpeed, flowOutIfSpeed, flowInIfIndex, flowOutIfIndex uint32 @@ -30,6 +30,9 @@ func (c *Component) enrichFlow(exporterIP netip.Addr, exporterStr string, flow * inIfClassification := interfaceClassification{} outIfClassification := interfaceClassification{} + flow := w.bf + c := w.c + if flow.InIf != 0 { answer, ok := c.d.Metadata.Lookup(t, exporterIP, uint(flow.InIf)) if !ok { @@ -87,11 +90,11 @@ func (c *Component) enrichFlow(exporterIP netip.Addr, exporterStr string, flow * } if samplingRate, ok := c.config.OverrideSamplingRate.Lookup(exporterIP); ok && samplingRate > 0 { - flow.SamplingRate = uint32(samplingRate) + flow.SamplingRate = uint64(samplingRate) } if flow.SamplingRate == 0 { if samplingRate, ok := c.config.DefaultSamplingRate.Lookup(exporterIP); ok && samplingRate > 0 { - flow.SamplingRate = uint32(samplingRate) + flow.SamplingRate = uint64(samplingRate) } else { c.metrics.flowsErrors.WithLabelValues(exporterStr, "sampling rate missing").Inc() skip = true @@ -128,28 +131,22 @@ func (c *Component) enrichFlow(exporterIP netip.Addr, exporterStr string, flow * // set asns according to user config flow.SrcAS = c.getASNumber(flow.SrcAS, sourceRouting.ASN) flow.DstAS = c.getASNumber(flow.DstAS, destRouting.ASN) - if !flow.GotCommunities { - for _, comm := range destRouting.Communities { - c.d.Schema.ProtobufAppendVarint(flow, schema.ColumnDstCommunities, uint64(comm)) + flow.AppendArrayUInt32(schema.ColumnDstCommunities, destRouting.Communities) + flow.AppendArrayUInt32(schema.ColumnDstASPath, destRouting.ASPath) + if len(destRouting.LargeCommunities) > 0 { + communities := make([]schema.UInt128, len(destRouting.LargeCommunities)) + for i, comm := range destRouting.LargeCommunities { + communities[i] = schema.UInt128{ + High: uint64(comm.ASN), + Low: (uint64(comm.LocalData1) << 32) + uint64(comm.LocalData2), + } } - } - if !flow.GotASPath { - for _, asn := range destRouting.ASPath { - c.d.Schema.ProtobufAppendVarint(flow, schema.ColumnDstASPath, uint64(asn)) - } - } - for _, comm := range destRouting.LargeCommunities { - c.d.Schema.ProtobufAppendVarintForce(flow, - schema.ColumnDstLargeCommunitiesASN, uint64(comm.ASN)) - c.d.Schema.ProtobufAppendVarintForce(flow, - schema.ColumnDstLargeCommunitiesLocalData1, uint64(comm.LocalData1)) - c.d.Schema.ProtobufAppendVarintForce(flow, - schema.ColumnDstLargeCommunitiesLocalData2, uint64(comm.LocalData2)) + flow.AppendArrayUInt128(schema.ColumnDstLargeCommunities, communities) } - c.d.Schema.ProtobufAppendBytes(flow, schema.ColumnExporterName, []byte(flowExporterName)) - c.d.Schema.ProtobufAppendVarint(flow, schema.ColumnInIfSpeed, uint64(flowInIfSpeed)) - c.d.Schema.ProtobufAppendVarint(flow, schema.ColumnOutIfSpeed, uint64(flowOutIfSpeed)) + flow.AppendString(schema.ColumnExporterName, flowExporterName) + flow.AppendUint(schema.ColumnInIfSpeed, uint64(flowInIfSpeed)) + flow.AppendUint(schema.ColumnOutIfSpeed, uint64(flowOutIfSpeed)) return } @@ -219,11 +216,11 @@ func (c *Component) writeExporter(flow *schema.FlowMessage, classification expor if classification.Reject { return false } - c.d.Schema.ProtobufAppendBytes(flow, schema.ColumnExporterGroup, []byte(classification.Group)) - c.d.Schema.ProtobufAppendBytes(flow, schema.ColumnExporterRole, []byte(classification.Role)) - c.d.Schema.ProtobufAppendBytes(flow, schema.ColumnExporterSite, []byte(classification.Site)) - c.d.Schema.ProtobufAppendBytes(flow, schema.ColumnExporterRegion, []byte(classification.Region)) - c.d.Schema.ProtobufAppendBytes(flow, schema.ColumnExporterTenant, []byte(classification.Tenant)) + flow.AppendString(schema.ColumnExporterGroup, classification.Group) + flow.AppendString(schema.ColumnExporterRole, classification.Role) + flow.AppendString(schema.ColumnExporterSite, classification.Site) + flow.AppendString(schema.ColumnExporterRegion, classification.Region) + flow.AppendString(schema.ColumnExporterTenant, classification.Tenant) return true } @@ -264,17 +261,17 @@ func (c *Component) writeInterface(flow *schema.FlowMessage, classification inte return false } if directionIn { - c.d.Schema.ProtobufAppendBytes(flow, schema.ColumnInIfName, []byte(classification.Name)) - c.d.Schema.ProtobufAppendBytes(flow, schema.ColumnInIfDescription, []byte(classification.Description)) - c.d.Schema.ProtobufAppendBytes(flow, schema.ColumnInIfConnectivity, []byte(classification.Connectivity)) - c.d.Schema.ProtobufAppendBytes(flow, schema.ColumnInIfProvider, []byte(classification.Provider)) - c.d.Schema.ProtobufAppendVarint(flow, schema.ColumnInIfBoundary, uint64(classification.Boundary)) + flow.AppendString(schema.ColumnInIfName, classification.Name) + flow.AppendString(schema.ColumnInIfDescription, classification.Description) + flow.AppendString(schema.ColumnInIfConnectivity, classification.Connectivity) + flow.AppendString(schema.ColumnInIfProvider, classification.Provider) + flow.AppendUint(schema.ColumnInIfBoundary, uint64(classification.Boundary)) } else { - c.d.Schema.ProtobufAppendBytes(flow, schema.ColumnOutIfName, []byte(classification.Name)) - c.d.Schema.ProtobufAppendBytes(flow, schema.ColumnOutIfDescription, []byte(classification.Description)) - c.d.Schema.ProtobufAppendBytes(flow, schema.ColumnOutIfConnectivity, []byte(classification.Connectivity)) - c.d.Schema.ProtobufAppendBytes(flow, schema.ColumnOutIfProvider, []byte(classification.Provider)) - c.d.Schema.ProtobufAppendVarint(flow, schema.ColumnOutIfBoundary, uint64(classification.Boundary)) + flow.AppendString(schema.ColumnOutIfName, classification.Name) + flow.AppendString(schema.ColumnOutIfDescription, classification.Description) + flow.AppendString(schema.ColumnOutIfConnectivity, classification.Connectivity) + flow.AppendString(schema.ColumnOutIfProvider, classification.Provider) + flow.AppendUint(schema.ColumnOutIfBoundary, uint64(classification.Boundary)) } return true } diff --git a/inlet/core/enricher_test.go b/outlet/core/enricher_test.go similarity index 84% rename from inlet/core/enricher_test.go rename to outlet/core/enricher_test.go index b8507684..4476581d 100644 --- a/inlet/core/enricher_test.go +++ b/outlet/core/enricher_test.go @@ -4,24 +4,29 @@ package core import ( + "bytes" + "encoding/gob" "fmt" "net/netip" + "sync" "testing" "time" - "github.com/IBM/sarama" "github.com/gin-gonic/gin" "github.com/go-viper/mapstructure/v2" + "google.golang.org/protobuf/proto" "akvorado/common/daemon" "akvorado/common/helpers" "akvorado/common/httpserver" + "akvorado/common/pb" "akvorado/common/reporter" "akvorado/common/schema" - "akvorado/inlet/flow" - "akvorado/inlet/kafka" - "akvorado/inlet/metadata" - "akvorado/inlet/routing" + "akvorado/outlet/clickhouse" + "akvorado/outlet/flow" + "akvorado/outlet/kafka" + "akvorado/outlet/metadata" + "akvorado/outlet/routing" ) func TestEnrich(t *testing.T) { @@ -44,8 +49,10 @@ func TestEnrich(t *testing.T) { }, OutputFlow: &schema.FlowMessage{ SamplingRate: 1000, + InIf: 100, + OutIf: 200, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnExporterName: "192_0_2_142", schema.ColumnInIfName: "Gi0/0/100", schema.ColumnOutIfName: "Gi0/0/200", @@ -72,8 +79,10 @@ func TestEnrich(t *testing.T) { }, OutputFlow: &schema.FlowMessage{ SamplingRate: 500, + InIf: 100, + OutIf: 200, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnExporterName: "192_0_2_142", schema.ColumnInIfName: "Gi0/0/100", schema.ColumnOutIfName: "Gi0/0/200", @@ -95,8 +104,10 @@ func TestEnrich(t *testing.T) { }, OutputFlow: &schema.FlowMessage{ SamplingRate: 500, + InIf: 100, + OutIf: 200, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnExporterName: "192_0_2_142", schema.ColumnInIfName: "Gi0/0/100", schema.ColumnOutIfName: "Gi0/0/200", @@ -122,8 +133,10 @@ func TestEnrich(t *testing.T) { }, OutputFlow: &schema.FlowMessage{ SamplingRate: 500, + InIf: 100, + OutIf: 200, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnExporterName: "192_0_2_142", schema.ColumnInIfName: "Gi0/0/100", schema.ColumnOutIfName: "Gi0/0/200", @@ -152,8 +165,10 @@ func TestEnrich(t *testing.T) { }, OutputFlow: &schema.FlowMessage{ SamplingRate: 1000, + InIf: 100, + OutIf: 200, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnExporterName: "192_0_2_142", schema.ColumnExporterRegion: "asia", schema.ColumnExporterTenant: "alfred", @@ -184,8 +199,10 @@ func TestEnrich(t *testing.T) { }, OutputFlow: &schema.FlowMessage{ SamplingRate: 1000, + InIf: 100, + OutIf: 200, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnExporterName: "192_0_2_142", schema.ColumnExporterTenant: "alfred", schema.ColumnInIfName: "Gi0/0/100", @@ -246,8 +263,10 @@ func TestEnrich(t *testing.T) { }, OutputFlow: &schema.FlowMessage{ SamplingRate: 1000, + InIf: 100, + OutIf: 200, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnExporterName: "192_0_2_142", schema.ColumnInIfProvider: "index1", schema.ColumnOutIfProvider: "index2", @@ -277,8 +296,10 @@ func TestEnrich(t *testing.T) { }, OutputFlow: &schema.FlowMessage{ SamplingRate: 1000, + InIf: 100, + OutIf: 200, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnExporterName: "192_0_2_142", schema.ColumnInIfName: "eth100", schema.ColumnOutIfName: "Gi0/0/200", @@ -300,15 +321,19 @@ func TestEnrich(t *testing.T) { SamplingRate: 1000, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), InIf: 100, - SrcVlan: 10, OutIf: 200, + SrcVlan: 10, DstVlan: 300, } }, OutputFlow: &schema.FlowMessage{ SamplingRate: 1000, + InIf: 100, + OutIf: 200, + SrcVlan: 10, + DstVlan: 300, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnExporterName: "192_0_2_142", schema.ColumnInIfName: "Gi0/0/100", schema.ColumnOutIfName: "Gi0/0/200.300", @@ -340,8 +365,10 @@ ClassifyProviderRegex(Interface.Description, "^Transit: ([^ ]+)", "$1")`, }, OutputFlow: &schema.FlowMessage{ SamplingRate: 1000, + InIf: 100, + OutIf: 200, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnExporterName: "192_0_2_142", schema.ColumnInIfName: "Gi0/0/100", schema.ColumnOutIfName: "Gi0/0/200", @@ -371,8 +398,10 @@ ClassifyProviderRegex(Interface.Description, "^Transit: ([^ ]+)", "$1")`, }, OutputFlow: &schema.FlowMessage{ SamplingRate: 1000, + InIf: 100, + OutIf: 200, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnExporterName: "192_0_2_142", schema.ColumnInIfName: "Gi0/0/100", schema.ColumnOutIfName: "Gi0/0/200", @@ -402,8 +431,10 @@ ClassifyProviderRegex(Interface.Description, "^Transit: ([^ ]+)", "$1")`, }, OutputFlow: &schema.FlowMessage{ SamplingRate: 1000, + InIf: 100, + OutIf: 200, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnExporterName: "192_0_2_142", schema.ColumnInIfName: "Gi0/0/100", schema.ColumnOutIfName: "Gi0/0/200", @@ -434,8 +465,10 @@ ClassifyProviderRegex(Interface.Description, "^Transit: ([^ ]+)", "$1")`, }, OutputFlow: &schema.FlowMessage{ SamplingRate: 1000, + InIf: 100, + OutIf: 200, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnExporterName: "192_0_2_142", schema.ColumnInIfName: "Gi0/0/100", schema.ColumnOutIfName: "Gi0/0/200", @@ -471,8 +504,10 @@ ClassifyProviderRegex(Interface.Description, "^Transit: ([^ ]+)", "$1")`, }, OutputFlow: &schema.FlowMessage{ SamplingRate: 1000, + InIf: 1010, + OutIf: 2010, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnExporterName: "192_0_2_142", schema.ColumnExporterGroup: "metadata group", schema.ColumnExporterRegion: "metadata region", @@ -509,26 +544,28 @@ ClassifyProviderRegex(Interface.Description, "^Transit: ([^ ]+)", "$1")`, }, OutputFlow: &schema.FlowMessage{ SamplingRate: 1000, + InIf: 100, + OutIf: 200, ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.142"), SrcAddr: netip.MustParseAddr("::ffff:192.0.2.142"), DstAddr: netip.MustParseAddr("::ffff:192.0.2.10"), SrcAS: 1299, DstAS: 174, - ProtobufDebug: map[schema.ColumnKey]interface{}{ - schema.ColumnExporterName: "192_0_2_142", - schema.ColumnInIfName: "Gi0/0/100", - schema.ColumnOutIfName: "Gi0/0/200", - schema.ColumnInIfDescription: "Interface 100", - schema.ColumnOutIfDescription: "Interface 200", - schema.ColumnInIfSpeed: 1000, - schema.ColumnOutIfSpeed: 1000, - schema.ColumnDstASPath: []uint32{64200, 1299, 174}, - schema.ColumnDstCommunities: []uint32{100, 200, 400}, - schema.ColumnDstLargeCommunitiesASN: []int32{64200}, - schema.ColumnDstLargeCommunitiesLocalData1: []int32{2}, - schema.ColumnDstLargeCommunitiesLocalData2: []int32{3}, - schema.ColumnSrcNetMask: 27, - schema.ColumnDstNetMask: 27, + SrcNetMask: 27, + DstNetMask: 27, + OtherColumns: map[schema.ColumnKey]interface{}{ + schema.ColumnExporterName: "192_0_2_142", + schema.ColumnInIfName: "Gi0/0/100", + schema.ColumnOutIfName: "Gi0/0/200", + schema.ColumnInIfDescription: "Interface 100", + schema.ColumnOutIfDescription: "Interface 200", + schema.ColumnInIfSpeed: 1000, + schema.ColumnOutIfSpeed: 1000, + schema.ColumnDstASPath: []uint32{64200, 1299, 174}, + schema.ColumnDstCommunities: []uint32{100, 200, 400}, + schema.ColumnDstLargeCommunities: []schema.UInt128{ + {High: 64200, Low: (uint64(2) << 32) + uint64(3)}, + }, }, }, }, @@ -541,11 +578,21 @@ ClassifyProviderRegex(Interface.Description, "^Transit: ([^ ]+)", "$1")`, daemonComponent := daemon.NewMock(t) metadataComponent := metadata.NewMock(t, r, metadata.DefaultConfiguration(), metadata.Dependencies{Daemon: daemonComponent}) - flowComponent := flow.NewMock(t, r, flow.DefaultConfiguration()) - kafkaComponent, kafkaProducer := kafka.NewMock(t, r, kafka.DefaultConfiguration()) + flowComponent, err := flow.New(r, flow.Dependencies{Schema: schema.NewMock(t)}) + if err != nil { + t.Fatalf("flow.New() error:\n%+v", err) + } httpComponent := httpserver.NewMock(t, r) routingComponent := routing.NewMock(t, r) routingComponent.PopulateRIB(t) + kafkaComponent, incoming := kafka.NewMock(t, kafka.DefaultConfiguration()) + var clickhouseMessages []*schema.FlowMessage + var clickhouseMessagesMutex sync.Mutex + clickhouseComponent := clickhouse.NewMock(t, func(msg *schema.FlowMessage) { + clickhouseMessagesMutex.Lock() + defer clickhouseMessagesMutex.Unlock() + clickhouseMessages = append(clickhouseMessages, msg) + }) // Prepare a configuration configuration := DefaultConfiguration() @@ -559,55 +606,70 @@ ClassifyProviderRegex(Interface.Description, "^Transit: ([^ ]+)", "$1")`, // Instantiate and start core c, err := New(r, configuration, Dependencies{ - Daemon: daemonComponent, - Flow: flowComponent, - Metadata: metadataComponent, - Kafka: kafkaComponent, - HTTP: httpComponent, - Routing: routingComponent, - Schema: schema.NewMock(t), + Daemon: daemonComponent, + Flow: flowComponent, + Metadata: metadataComponent, + Kafka: kafkaComponent, + ClickHouse: clickhouseComponent, + HTTP: httpComponent, + Routing: routingComponent, + Schema: schema.NewMock(t), }) if err != nil { t.Fatalf("New() error:\n%+v", err) } - helpers.StartStop(t, c) - // Inject twice since otherwise, we get a cache miss - received := make(chan bool) - if tc.OutputFlow != nil { - kafkaProducer.ExpectInputWithMessageCheckerFunctionAndSucceed( - func(msg *sarama.ProducerMessage) error { - defer close(received) - b, err := msg.Value.Encode() - if err != nil { - t.Fatalf("Kafka message encoding error:\n%+v", err) - } - t.Logf("Raw message: %v", b) - got := c.d.Schema.ProtobufDecode(t, b) - if diff := helpers.Diff(&got, tc.OutputFlow); diff != "" { - t.Errorf("Classifier (-got, +want):\n%s", diff) - } - return nil - }) + helpers.StartStop(t, c) + clickhouseMessagesMutex.Lock() + clickhouseMessages = clickhouseMessages[:0] + clickhouseMessagesMutex.Unlock() + + inputFlow := tc.InputFlow() + var buf bytes.Buffer + encoder := gob.NewEncoder(&buf) + if err := encoder.Encode(inputFlow); err != nil { + t.Fatalf("gob.Encode() error: %v", err) } - // Else, we should not get a message, but that's not possible to test. - flowComponent.Inject(tc.InputFlow()) - time.Sleep(50 * time.Millisecond) // Needed to let poller does its job - flowComponent.Inject(tc.InputFlow()) - if tc.OutputFlow != nil { - select { - case <-received: - case <-time.After(1 * time.Second): - t.Fatal("Kafka message not received") + + rawFlow := &pb.RawFlow{ + TimeReceived: uint64(time.Now().Unix()), + Payload: buf.Bytes(), + SourceAddress: inputFlow.ExporterAddress.AsSlice(), + UseSourceAddress: false, + Decoder: pb.RawFlow_DECODER_GOB, + TimestampSource: pb.RawFlow_TS_INPUT, + } + + data, err := proto.Marshal(rawFlow) + if err != nil { + t.Fatalf("proto.Marshal() error: %v", err) + } + + // Test twice to check cache behavior + incoming <- data + time.Sleep(100 * time.Millisecond) + incoming <- data + time.Sleep(100 * time.Millisecond) + + clickhouseMessagesMutex.Lock() + clickhouseMessagesLen := len(clickhouseMessages) + var lastMessage *schema.FlowMessage + if clickhouseMessagesLen > 0 { + lastMessage = clickhouseMessages[clickhouseMessagesLen-1] + } + clickhouseMessagesMutex.Unlock() + + if tc.OutputFlow != nil && clickhouseMessagesLen > 0 { + if diff := helpers.Diff(lastMessage, tc.OutputFlow); diff != "" { + t.Errorf("Enriched flow differs (-got, +want):\n%s", diff) } - } else { - time.Sleep(100 * time.Millisecond) } - gotMetrics := r.GetMetrics("akvorado_inlet_core_", "-processing_", "flows_", "received_", "forwarded_") + gotMetrics := r.GetMetrics("akvorado_outlet_core_", "-processing_", "flows_", "received_", "forwarded_") expectedMetrics := map[string]string{ `flows_errors_total{error="SNMP cache miss",exporter="192.0.2.142"}`: "1", `flows_http_clients`: "0", `received_flows_total{exporter="192.0.2.142"}`: "2", + `received_raw_flows_total`: "2", } if tc.OutputFlow != nil { expectedMetrics[`forwarded_flows_total{exporter="192.0.2.142"}`] = "1" diff --git a/inlet/core/http.go b/outlet/core/http.go similarity index 69% rename from inlet/core/http.go rename to outlet/core/http.go index 875e65d7..576eeab7 100644 --- a/inlet/core/http.go +++ b/outlet/core/http.go @@ -18,7 +18,7 @@ type flowsParameters struct { } // FlowsHTTPHandler streams a JSON copy of all flows just after -// sending them to Kafka. Under load, some flows may not be sent. This +// sending them to ClickHouse. Under load, some flows may not be sent. This // is intended for debug only. func (c *Component) FlowsHTTPHandler(gc *gin.Context) { var params flowsParameters @@ -27,7 +27,7 @@ func (c *Component) FlowsHTTPHandler(gc *gin.Context) { gc.JSON(http.StatusBadRequest, gin.H{"message": helpers.Capitalize(err.Error())}) return } - format := gc.NegotiateFormat("application/json", "application/x-protobuf") + dying := c.t.Dying() atomic.AddUint32(&c.httpFlowClients, 1) defer atomic.AddUint32(&c.httpFlowClients, ^uint32(0)) @@ -40,23 +40,15 @@ func (c *Component) FlowsHTTPHandler(gc *gin.Context) { for { select { - case <-c.t.Dying(): + case <-dying: return case <-gc.Request.Context().Done(): return case msg := <-c.httpFlowChannel: - switch format { - case "application/json": - if params.Limit == 1 { - gc.IndentedJSON(http.StatusOK, msg) - } else { - gc.JSON(http.StatusOK, msg) - gc.Writer.Write([]byte("\n")) - } - case "application/x-protobuf": - gc.Set("Content-Type", format) - gc.Writer.Write(msg.Bytes()) - } + gc.Header("Content-Type", "application/json") + gc.Status(http.StatusOK) + gc.Writer.Write(msg) + gc.Writer.Write([]byte("\n")) count++ if params.Limit > 0 && count == params.Limit { diff --git a/inlet/core/metrics.go b/outlet/core/metrics.go similarity index 83% rename from inlet/core/metrics.go rename to outlet/core/metrics.go index 17547016..25baa6ad 100644 --- a/inlet/core/metrics.go +++ b/outlet/core/metrics.go @@ -10,6 +10,8 @@ import ( ) type metrics struct { + rawFlowsReceived reporter.Counter + rawFlowsErrors *reporter.CounterVec flowsReceived *reporter.CounterVec flowsForwarded *reporter.CounterVec flowsErrors *reporter.CounterVec @@ -21,6 +23,19 @@ type metrics struct { } func (c *Component) initMetrics() { + c.metrics.rawFlowsReceived = c.r.Counter( + reporter.CounterOpts{ + Name: "received_raw_flows_total", + Help: "Number of incoming raw flows (proto).", + }, + ) + c.metrics.rawFlowsErrors = c.r.CounterVec( + reporter.CounterOpts{ + Name: "raw_flows_errors_total", + Help: "Number of raw flows with errors.", + }, + []string{"error"}, + ) c.metrics.flowsReceived = c.r.CounterVec( reporter.CounterOpts{ Name: "received_flows_total", diff --git a/outlet/core/release.go b/outlet/core/release.go new file mode 100644 index 00000000..6a425ee6 --- /dev/null +++ b/outlet/core/release.go @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: 2025 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +//go:build release + +package core + +const debug = false diff --git a/outlet/core/root.go b/outlet/core/root.go new file mode 100644 index 00000000..be72a124 --- /dev/null +++ b/outlet/core/root.go @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +// Package core plumbs all the other components together. +package core + +import ( + "time" + + "gopkg.in/tomb.v2" + + "akvorado/common/daemon" + "akvorado/common/helpers/cache" + "akvorado/common/httpserver" + "akvorado/common/reporter" + "akvorado/common/schema" + "akvorado/outlet/clickhouse" + "akvorado/outlet/flow" + "akvorado/outlet/kafka" + "akvorado/outlet/metadata" + "akvorado/outlet/routing" +) + +// Component represents the HTTP compomenent. +type Component struct { + r *reporter.Reporter + d *Dependencies + t tomb.Tomb + config Configuration + + metrics metrics + + httpFlowClients uint32 // for dumping flows + httpFlowChannel chan []byte + httpFlowFlushDelay time.Duration + + classifierExporterCache *cache.Cache[exporterInfo, exporterClassification] + classifierInterfaceCache *cache.Cache[exporterAndInterfaceInfo, interfaceClassification] + classifierErrLogger reporter.Logger +} + +// Dependencies define the dependencies of the HTTP component. +type Dependencies struct { + Daemon daemon.Component + Flow *flow.Component + Metadata *metadata.Component + Routing *routing.Component + Kafka kafka.Component + ClickHouse clickhouse.Component + HTTP *httpserver.Component + Schema *schema.Component +} + +// New creates a new core component. +func New(r *reporter.Reporter, configuration Configuration, dependencies Dependencies) (*Component, error) { + c := Component{ + r: r, + d: &dependencies, + config: configuration, + + httpFlowClients: 0, + httpFlowChannel: make(chan []byte, 10), + httpFlowFlushDelay: time.Second, + + classifierExporterCache: cache.New[exporterInfo, exporterClassification](), + classifierInterfaceCache: cache.New[exporterAndInterfaceInfo, interfaceClassification](), + classifierErrLogger: r.Sample(reporter.BurstSampler(10*time.Second, 3)), + } + c.d.Daemon.Track(&c.t, "outlet/core") + c.initMetrics() + return &c, nil +} + +// Start starts the core component. +func (c *Component) Start() error { + c.r.Info().Msg("starting core component") + c.d.Kafka.StartWorkers(c.newWorker) + + // Classifier cache expiration + c.t.Go(func() error { + for { + select { + case <-c.t.Dying(): + return nil + case <-time.After(c.config.ClassifierCacheDuration): + before := time.Now().Add(-c.config.ClassifierCacheDuration) + c.classifierExporterCache.DeleteLastAccessedBefore(before) + c.classifierInterfaceCache.DeleteLastAccessedBefore(before) + } + } + }) + + c.d.HTTP.GinRouter.GET("/api/v0/outlet/flows", c.FlowsHTTPHandler) + return nil +} + +// Stop stops the core component. +func (c *Component) Stop() error { + defer func() { + close(c.httpFlowChannel) + c.r.Info().Msg("core component stopped") + }() + c.r.Info().Msg("stopping core component") + c.t.Kill(nil) + return c.t.Wait() +} diff --git a/outlet/core/root_test.go b/outlet/core/root_test.go new file mode 100644 index 00000000..79c2aa5d --- /dev/null +++ b/outlet/core/root_test.go @@ -0,0 +1,387 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package core + +import ( + "bufio" + "bytes" + "encoding/gob" + "encoding/json" + "fmt" + "io" + "net/http" + "net/netip" + "strings" + "sync" + "testing" + "time" + + "github.com/gin-gonic/gin" + "google.golang.org/protobuf/proto" + + "akvorado/common/daemon" + "akvorado/common/helpers" + "akvorado/common/httpserver" + "akvorado/common/pb" + "akvorado/common/reporter" + "akvorado/common/schema" + "akvorado/outlet/clickhouse" + "akvorado/outlet/flow" + "akvorado/outlet/kafka" + "akvorado/outlet/metadata" + "akvorado/outlet/routing" +) + +func TestCore(t *testing.T) { + r := reporter.NewMock(t) + + // Prepare all components. + daemonComponent := daemon.NewMock(t) + metadataComponent := metadata.NewMock(t, r, metadata.DefaultConfiguration(), + metadata.Dependencies{Daemon: daemonComponent}) + flowComponent, err := flow.New(r, flow.Dependencies{Schema: schema.NewMock(t)}) + if err != nil { + t.Fatalf("flow.New() error:\n%+v", err) + } + httpComponent := httpserver.NewMock(t, r) + routingComponent := routing.NewMock(t, r) + routingComponent.PopulateRIB(t) + kafkaComponent, incoming := kafka.NewMock(t, kafka.DefaultConfiguration()) + var clickhouseMessages []*schema.FlowMessage + var clickhouseMessagesMutex sync.Mutex + clickhouseComponent := clickhouse.NewMock(t, func(msg *schema.FlowMessage) { + clickhouseMessagesMutex.Lock() + defer clickhouseMessagesMutex.Unlock() + clickhouseMessages = append(clickhouseMessages, msg) + }) + + // Instantiate and start core + sch := schema.NewMock(t) + c, err := New(r, DefaultConfiguration(), Dependencies{ + Daemon: daemonComponent, + Flow: flowComponent, + Metadata: metadataComponent, + Kafka: kafkaComponent, + ClickHouse: clickhouseComponent, + HTTP: httpComponent, + Routing: routingComponent, + Schema: sch, + }) + if err != nil { + t.Fatalf("New() error:\n%+v", err) + } + helpers.StartStop(t, c) + + flowMessage := func(exporter string, in, out uint32) *schema.FlowMessage { + msg := &schema.FlowMessage{ + TimeReceived: 200, + SamplingRate: 1000, + ExporterAddress: netip.AddrFrom16(netip.MustParseAddr(exporter).As16()), + InIf: in, + OutIf: out, + SrcAddr: netip.MustParseAddr("::ffff:67.43.156.77"), + DstAddr: netip.MustParseAddr("::ffff:2.125.160.216"), + OtherColumns: map[schema.ColumnKey]interface{}{ + schema.ColumnBytes: 6765, + schema.ColumnPackets: 4, + schema.ColumnEType: 0x800, + schema.ColumnProto: 6, + schema.ColumnSrcPort: 8534, + schema.ColumnDstPort: 80, + }, + } + return msg + } + + expectedFlowMessage := func(exporter string, in, out uint32) *schema.FlowMessage { + expected := flowMessage(exporter, in, out) + expected.SrcAS = 0 // no geoip enrich anymore + expected.DstAS = 0 // no geoip enrich anymore + expected.OtherColumns[schema.ColumnInIfName] = fmt.Sprintf("Gi0/0/%d", in) + expected.OtherColumns[schema.ColumnOutIfName] = fmt.Sprintf("Gi0/0/%d", out) + expected.OtherColumns[schema.ColumnInIfDescription] = fmt.Sprintf("Interface %d", in) + expected.OtherColumns[schema.ColumnOutIfDescription] = fmt.Sprintf("Interface %d", out) + expected.OtherColumns[schema.ColumnInIfSpeed] = 1000 + expected.OtherColumns[schema.ColumnOutIfSpeed] = 1000 + expected.OtherColumns[schema.ColumnExporterName] = strings.ReplaceAll(exporter, ".", "_") + return expected + } + + // Helper function to inject flows using the new mechanism + injectFlow := func(flow *schema.FlowMessage) { + t.Helper() + var buf bytes.Buffer + encoder := gob.NewEncoder(&buf) + if err := encoder.Encode(flow); err != nil { + t.Fatalf("gob.Encode() error: %v", err) + } + + rawFlow := &pb.RawFlow{ + TimeReceived: uint64(time.Now().Unix()), + Payload: buf.Bytes(), + SourceAddress: flow.ExporterAddress.AsSlice(), + UseSourceAddress: false, + Decoder: pb.RawFlow_DECODER_GOB, + TimestampSource: pb.RawFlow_TS_INPUT, + } + + data, err := proto.Marshal(rawFlow) + if err != nil { + t.Fatalf("proto.Marshal() error: %v", err) + } + + // Send to kafka mock's incoming channel + incoming <- data + } + + t.Run("core", func(t *testing.T) { + clickhouseMessagesMutex.Lock() + clickhouseMessages = clickhouseMessages[:0] + clickhouseMessagesMutex.Unlock() + + // Inject several messages with a cache miss from the SNMP component. + injectFlow(flowMessage("192.0.2.142", 434, 677)) + injectFlow(flowMessage("192.0.2.143", 434, 677)) + injectFlow(flowMessage("192.0.2.143", 437, 677)) + injectFlow(flowMessage("192.0.2.143", 434, 679)) + time.Sleep(20 * time.Millisecond) + + gotMetrics := r.GetMetrics("akvorado_outlet_core_", "-flows_processing_") + expectedMetrics := map[string]string{ + `classifier_exporter_cache_size_items`: "0", + `classifier_interface_cache_size_items`: "0", + `flows_errors_total{error="SNMP cache miss",exporter="192.0.2.142"}`: "1", + `flows_errors_total{error="SNMP cache miss",exporter="192.0.2.143"}`: "3", + `received_flows_total{exporter="192.0.2.142"}`: "1", + `received_flows_total{exporter="192.0.2.143"}`: "3", + `received_raw_flows_total`: "4", + `flows_http_clients`: "0", + } + if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { + t.Fatalf("Metrics (-got, +want):\n%s", diff) + } + + // Inject again the messages, this time, we will get a cache hit! + injectFlow(flowMessage("192.0.2.142", 434, 677)) + injectFlow(flowMessage("192.0.2.143", 437, 679)) + time.Sleep(20 * time.Millisecond) + + // Should have 2 more flows in clickhouseMessages now + clickhouseMessagesMutex.Lock() + clickhouseMessagesLen := len(clickhouseMessages) + clickhouseMessagesMutex.Unlock() + if clickhouseMessagesLen < 2 { + t.Fatalf("Expected at least 2 flows in clickhouseMessages, got %d", clickhouseMessagesLen) + } + + time.Sleep(20 * time.Millisecond) + gotMetrics = r.GetMetrics("akvorado_outlet_core_", "classifier_", "-flows_processing_", "flows_", "received_", "forwarded_") + expectedMetrics = map[string]string{ + `classifier_exporter_cache_size_items`: "0", + `classifier_interface_cache_size_items`: "0", + `flows_errors_total{error="SNMP cache miss",exporter="192.0.2.142"}`: "1", + `flows_errors_total{error="SNMP cache miss",exporter="192.0.2.143"}`: "3", + `received_flows_total{exporter="192.0.2.142"}`: "2", + `received_flows_total{exporter="192.0.2.143"}`: "4", + `received_raw_flows_total`: "6", + `forwarded_flows_total{exporter="192.0.2.142"}`: "1", + `forwarded_flows_total{exporter="192.0.2.143"}`: "1", + `flows_http_clients`: "0", + } + if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { + t.Fatalf("Metrics (-got, +want):\n%s", diff) + } + + // Now, check we get the message we expect + clickhouseMessagesMutex.Lock() + clickhouseMessages = clickhouseMessages[:0] + clickhouseMessagesMutex.Unlock() + input := flowMessage("192.0.2.142", 434, 677) + injectFlow(input) + time.Sleep(20 * time.Millisecond) + + // Check the flow was stored in clickhouseMessages + expected := []*schema.FlowMessage{expectedFlowMessage("192.0.2.142", 434, 677)} + clickhouseMessagesMutex.Lock() + clickhouseMessagesCopy := make([]*schema.FlowMessage, len(clickhouseMessages)) + copy(clickhouseMessagesCopy, clickhouseMessages) + clickhouseMessagesMutex.Unlock() + if diff := helpers.Diff(clickhouseMessagesCopy, expected); diff != "" { + t.Fatalf("Flow message (-got, +want):\n%s", diff) + } + + // Try to inject a message with missing sampling rate + input = flowMessage("192.0.2.142", 434, 677) + input.SamplingRate = 0 + injectFlow(input) + time.Sleep(20 * time.Millisecond) + + gotMetrics = r.GetMetrics("akvorado_outlet_core_", "classifier_", "-flows_processing_", "flows_", "forwarded_", "received_") + expectedMetrics = map[string]string{ + `classifier_exporter_cache_size_items`: "0", + `classifier_interface_cache_size_items`: "0", + `flows_errors_total{error="SNMP cache miss",exporter="192.0.2.142"}`: "1", + `flows_errors_total{error="SNMP cache miss",exporter="192.0.2.143"}`: "3", + `flows_errors_total{error="sampling rate missing",exporter="192.0.2.142"}`: "1", + `received_flows_total{exporter="192.0.2.142"}`: "4", + `received_flows_total{exporter="192.0.2.143"}`: "4", + `forwarded_flows_total{exporter="192.0.2.142"}`: "2", + `forwarded_flows_total{exporter="192.0.2.143"}`: "1", + `flows_http_clients`: "0", + `received_raw_flows_total`: "8", + } + if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { + t.Fatalf("Metrics (-got, +want):\n%s", diff) + } + }) + + // Test HTTP flow clients (JSON) + t.Run("http flows", func(t *testing.T) { + c.httpFlowFlushDelay = 20 * time.Millisecond + + resp, err := http.Get(fmt.Sprintf("http://%s/api/v0/outlet/flows", c.d.HTTP.LocalAddr())) + if err != nil { + t.Fatalf("GET /api/v0/outlet/flows:\n%+v", err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("GET /api/v0/outlet/flows status code %d", resp.StatusCode) + } + + // Metrics should tell we have a client + gotMetrics := r.GetMetrics("akvorado_outlet_core_", "flows_http_clients", "-flows_processing_") + expectedMetrics := map[string]string{ + `flows_http_clients`: "1", + } + if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { + t.Fatalf("Metrics (-got, +want):\n%s", diff) + } + + // Produce some flows + clickhouseMessagesMutex.Lock() + clickhouseMessages = clickhouseMessages[:0] + clickhouseMessagesMutex.Unlock() + for range 12 { + injectFlow(flowMessage("192.0.2.142", 434, 677)) + } + + // Wait for flows to be processed + time.Sleep(100 * time.Millisecond) + + // Should have 12 flows in clickhouseMessages + clickhouseMessagesMutex.Lock() + clickhouseMessagesLen := len(clickhouseMessages) + clickhouseMessagesMutex.Unlock() + if clickhouseMessagesLen != 12 { + t.Fatalf("Expected 12 flows in clickhouseMessages, got %d", clickhouseMessagesLen) + } + + // Decode some of them + reader := bufio.NewReader(resp.Body) + decoder := json.NewDecoder(reader) + for range 10 { + var got gin.H + if err := decoder.Decode(&got); err != nil { + t.Fatalf("GET /api/v0/outlet/flows error while reading body:\n%+v", err) + } + expected := gin.H{ + "TimeReceived": 200, + "SamplingRate": 1000, + "ExporterAddress": "::ffff:192.0.2.142", + "SrcAddr": "::ffff:67.43.156.77", + "DstAddr": "::ffff:2.125.160.216", + "SrcAS": 0, // no geoip enrich anymore + "InIf": 434, + "OutIf": 677, + "NextHop": "", + "SrcNetMask": 0, + "DstNetMask": 0, + "SrcVlan": 0, + "DstVlan": 0, + "DstAS": 0, + "OtherColumns": gin.H{ + "ExporterName": "192_0_2_142", + "InIfName": "Gi0/0/434", + "OutIfName": "Gi0/0/677", + "InIfDescription": "Interface 434", + "OutIfDescription": "Interface 677", + "InIfSpeed": 1000, + "OutIfSpeed": 1000, + "Bytes": 6765, + "Packets": 4, + "SrcPort": 8534, + "DstPort": 80, + "EType": 2048, + "Proto": 6, + }, + } + if diff := helpers.Diff(got, expected); diff != "" { + t.Fatalf("GET /api/v0/outlet/flows (-got, +want):\n%s", diff) + } + } + }) + + // Test HTTP flow clients with a limit + time.Sleep(10 * time.Millisecond) + t.Run("http flows with limit", func(t *testing.T) { + resp, err := http.Get(fmt.Sprintf("http://%s/api/v0/outlet/flows?limit=4", c.d.HTTP.LocalAddr())) + if err != nil { + t.Fatalf("GET /api/v0/outlet/flows:\n%+v", err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("GET /api/v0/outlet/flows status code %d", resp.StatusCode) + } + + // Metrics should tell we have a client + gotMetrics := r.GetMetrics("akvorado_outlet_core_", "flows_http_clients") + expectedMetrics := map[string]string{ + `flows_http_clients`: "1", + } + if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { + t.Fatalf("Metrics (-got, +want):\n%s", diff) + } + + // Produce some flows + clickhouseMessagesMutex.Lock() + clickhouseMessages = clickhouseMessages[:0] + clickhouseMessagesMutex.Unlock() + for range 12 { + injectFlow(flowMessage("192.0.2.142", 434, 677)) + } + + // Wait for flows to be processed + time.Sleep(100 * time.Millisecond) + + // Should have 12 flows in clickhouseMessages + clickhouseMessagesMutex.Lock() + clickhouseMessagesLen := len(clickhouseMessages) + clickhouseMessagesMutex.Unlock() + if clickhouseMessagesLen != 12 { + t.Fatalf("Expected 12 flows in clickhouseMessages, got %d", clickhouseMessagesLen) + } + + // Check we got only 4 + reader := bufio.NewReader(resp.Body) + count := 0 + for { + _, err := reader.ReadString('\n') + if err == io.EOF { + t.Log("EOF") + break + } + if err != nil { + t.Fatalf("GET /api/v0/outlet/flows error while reading:\n%+v", err) + } + count++ + if count > 4 { + break + } + } + if count != 4 { + t.Fatalf("GET /api/v0/outlet/flows got less than 4 flows (%d)", count) + } + }) + +} diff --git a/outlet/core/worker.go b/outlet/core/worker.go new file mode 100644 index 00000000..7063ad9a --- /dev/null +++ b/outlet/core/worker.go @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: 2025 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package core + +import ( + "context" + "encoding/json" + "fmt" + "sync/atomic" + "time" + + "akvorado/common/pb" + "akvorado/common/reporter" + "akvorado/common/schema" + "akvorado/outlet/clickhouse" + "akvorado/outlet/kafka" + + "google.golang.org/protobuf/proto" +) + +// worker represents a worker processing incoming flows. +type worker struct { + c *Component + l reporter.Logger + cw clickhouse.Worker + bf *schema.FlowMessage +} + +// newWorker instantiates a new worker and returns a callback function to +// process an incoming flow and a function to call on shutdown. +func (c *Component) newWorker(i int) (kafka.ReceiveFunc, kafka.ShutdownFunc) { + bf := c.d.Schema.NewFlowMessage() + w := worker{ + c: c, + l: c.r.With().Int("worker", i).Logger(), + bf: bf, + cw: c.d.ClickHouse.NewWorker(i, bf), + } + return w.processIncomingFlow, w.shutdown +} + +// shutdown shutdowns the worker, flushing any remaining data. +func (w *worker) shutdown() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + w.cw.Flush(ctx) +} + +// processIncomingFlow processes one incoming flow from Kafka. +func (w *worker) processIncomingFlow(ctx context.Context, data []byte) error { + // Do nothing if we are shutting down + if !w.c.t.Alive() { + return kafka.ErrStopProcessing + } + + // Raw flaw decoding: fatal + w.c.metrics.rawFlowsReceived.Inc() + var rawflow pb.RawFlow + if err := proto.Unmarshal(data, &rawflow); err != nil { + w.c.metrics.rawFlowsErrors.WithLabelValues("cannot decode protobuf") + return fmt.Errorf("cannot decode raw flow: %w", err) + } + + // Porcess each decoded flow + finalize := func() { + // Accounting + exporter := w.bf.ExporterAddress.Unmap().String() + w.c.metrics.flowsReceived.WithLabelValues(exporter).Inc() + + // Enrichment: not fatal + ip := w.bf.ExporterAddress + if skip := w.enrichFlow(ip, exporter); skip { + w.bf.Undo() + return + } + + // If we have HTTP clients, send to them too + if atomic.LoadUint32(&w.c.httpFlowClients) > 0 { + if jsonBytes, err := json.Marshal(w.bf); err == nil { + select { + case w.c.httpFlowChannel <- jsonBytes: // OK + default: // Overflow, best effort and ignore + } + } + } + + // Finalize and forward to ClickHouse + w.c.metrics.flowsForwarded.WithLabelValues(exporter).Inc() + w.cw.FinalizeAndSend(ctx) + } + + // Flow decoding: not fatal + err := w.c.d.Flow.Decode(&rawflow, w.bf, finalize) + if err != nil { + w.c.metrics.rawFlowsErrors.WithLabelValues("cannot decode payload") + return nil + } + + return nil + +} diff --git a/outlet/flow/decoder.go b/outlet/flow/decoder.go new file mode 100644 index 00000000..2f8cc575 --- /dev/null +++ b/outlet/flow/decoder.go @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package flow + +import ( + "errors" + "fmt" + "net/netip" + "runtime/debug" + "time" + + "akvorado/common/pb" + "akvorado/common/schema" + "akvorado/outlet/flow/decoder" + "akvorado/outlet/flow/decoder/netflow" + "akvorado/outlet/flow/decoder/sflow" +) + +// Decode decodes a raw flow from protobuf into flow messages. +func (c *Component) Decode(rawFlow *pb.RawFlow, bf *schema.FlowMessage, finalize decoder.FinalizeFlowFunc) error { + // Get decoder directly by type + dec, ok := c.decoders[rawFlow.Decoder] + if !ok { + return fmt.Errorf("decoder %v not available", rawFlow.Decoder) + } + + // Convert pb.RawFlow to decoder.RawFlow + sourceIP, ok := netip.AddrFromSlice(rawFlow.SourceAddress) + if !ok { + return errors.New("missing source address") + } + + decoderInput := decoder.RawFlow{ + TimeReceived: time.Unix(int64(rawFlow.TimeReceived), 0), + Payload: rawFlow.Payload, + Source: sourceIP, + } + + // Decode the flow + options := decoder.Option{ + TimestampSource: rawFlow.TimestampSource, + } + + if err := c.decodeWithMetrics(dec, decoderInput, options, bf, func() { + if rawFlow.UseSourceAddress { + bf.ExporterAddress = sourceIP + } + finalize() + }); err != nil { + return fmt.Errorf("failed to decode flow: %w", err) + } + + return nil +} + +// decodeWithMetrics wraps decoder calls with metrics tracking. +func (c *Component) decodeWithMetrics(dec decoder.Decoder, input decoder.RawFlow, options decoder.Option, bf *schema.FlowMessage, finalize decoder.FinalizeFlowFunc) error { + defer func() { + if r := recover(); r != nil { + c.errLogger.Error(). + Str("decoder", dec.Name()). + Str("panic", fmt.Sprint(r)). + Str("stack", string(debug.Stack())). + Msg("panic while decoding") + c.metrics.decoderErrors.WithLabelValues(dec.Name()).Inc() + } + }() + + n, err := dec.Decode(input, options, bf, finalize) + if err != nil { + c.metrics.decoderErrors.WithLabelValues(dec.Name()).Inc() + return err + } + c.metrics.decoderStats.WithLabelValues(dec.Name()).Add(float64(n)) + + return nil +} + +var availableDecoders = map[pb.RawFlow_Decoder]decoder.NewDecoderFunc{ + pb.RawFlow_DECODER_NETFLOW: netflow.New, + pb.RawFlow_DECODER_SFLOW: sflow.New, +} diff --git a/outlet/flow/decoder/gob/root.go b/outlet/flow/decoder/gob/root.go new file mode 100644 index 00000000..2f4e42db --- /dev/null +++ b/outlet/flow/decoder/gob/root.go @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +//go:build !release + +// Package gob handles gob decoding for testing purposes. +package gob + +import ( + "bytes" + "encoding/gob" + "net/netip" + "reflect" + + "akvorado/common/reporter" + "akvorado/common/schema" + "akvorado/outlet/flow/decoder" +) + +func init() { + // Register types that may appear in OtherColumns for gob encoding/decoding + gob.Register(schema.InterfaceBoundary(0)) +} + +// Decoder contains the state for the gob decoder. +type Decoder struct { + r *reporter.Reporter + d decoder.Dependencies +} + +// New creates a new gob decoder. +func New(r *reporter.Reporter, dependencies decoder.Dependencies) decoder.Decoder { + return &Decoder{ + r: r, + d: dependencies, + } +} + +// Name returns the decoder name. +func (d *Decoder) Name() string { + return "gob" +} + +// Decode decodes a gob-encoded FlowMessage. +func (d *Decoder) Decode(in decoder.RawFlow, _ decoder.Option, bf *schema.FlowMessage, finalize decoder.FinalizeFlowFunc) (int, error) { + var decoded schema.FlowMessage + + buf := bytes.NewReader(in.Payload) + decoder := gob.NewDecoder(buf) + + if err := decoder.Decode(&decoded); err != nil { + return 0, err + } + + // We need to "replay" the decoded flow. We use reflection for this. + decodedValue := reflect.ValueOf(&decoded).Elem() + bfValue := reflect.ValueOf(bf).Elem() + decodedType := decodedValue.Type() + + // Copy all public fields except OtherColumns + for i := 0; i < decodedValue.NumField(); i++ { + field := decodedType.Field(i) + if !field.IsExported() || field.Name == "OtherColumns" { + continue + } + + sourceField := decodedValue.Field(i) + targetField := bfValue.FieldByName(field.Name) + if targetField.IsValid() && targetField.CanSet() { + targetField.Set(sourceField) + } + } + + // Handle OtherColumns + if decoded.OtherColumns != nil { + for columnKey, value := range decoded.OtherColumns { + switch v := value.(type) { + case uint64: + bf.AppendUint(columnKey, v) + case uint32: + bf.AppendUint(columnKey, uint64(v)) + case uint16: + bf.AppendUint(columnKey, uint64(v)) + case uint8: + bf.AppendUint(columnKey, uint64(v)) + case int: + bf.AppendUint(columnKey, uint64(v)) + case string: + bf.AppendString(columnKey, v) + case netip.Addr: + bf.AppendIPv6(columnKey, v) + case schema.InterfaceBoundary: + bf.AppendUint(columnKey, uint64(v)) + case []uint32: + bf.AppendArrayUInt32(columnKey, v) + case []schema.UInt128: + bf.AppendArrayUInt128(columnKey, v) + } + } + } + + finalize() + return 1, nil +} diff --git a/outlet/flow/decoder/gob/root_test.go b/outlet/flow/decoder/gob/root_test.go new file mode 100644 index 00000000..5b56ba59 --- /dev/null +++ b/outlet/flow/decoder/gob/root_test.go @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +//go:build !release + +package gob + +import ( + "bytes" + "encoding/gob" + "net/netip" + "testing" + "time" + + "akvorado/common/helpers" + "akvorado/common/reporter" + "akvorado/common/schema" + "akvorado/outlet/flow/decoder" +) + +func TestGobDecoder(t *testing.T) { + r := reporter.NewMock(t) + sch := schema.NewMock(t) + d := New(r, decoder.Dependencies{Schema: sch}) + bf := sch.NewFlowMessage() + got := []*schema.FlowMessage{} + finalize := func() { + // Keep a copy of the current flow message + clone := *bf + got = append(got, &clone) + // And clear the flow message + bf.Clear() + } + + // Create a test FlowMessage + originalFlow := &schema.FlowMessage{ + TimeReceived: uint32(time.Now().Unix()), + SamplingRate: 1000, + ExporterAddress: netip.MustParseAddr("::ffff:192.0.2.1"), + InIf: 10, + OutIf: 20, + SrcAddr: netip.MustParseAddr("::ffff:192.0.2.100"), + DstAddr: netip.MustParseAddr("::ffff:192.0.2.200"), + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: uint64(1024), + schema.ColumnPackets: 10, + schema.ColumnInIfBoundary: schema.InterfaceBoundaryExternal, + schema.ColumnExporterName: "hello", + schema.ColumnDstNetMask: uint32(8), + schema.ColumnDstPort: uint16(80), + schema.ColumnDstASPath: []uint32{65000, 65001, 65002}, + }, + } + + // Encode to gob + var buf bytes.Buffer + encoder := gob.NewEncoder(&buf) + if err := encoder.Encode(originalFlow); err != nil { + t.Fatalf("gob.Encode() error: %v", err) + } + + // Test decoding + rawFlow := decoder.RawFlow{ + TimeReceived: time.Now(), + Payload: buf.Bytes(), + Source: netip.MustParseAddr("::ffff:192.0.2.1"), + } + + nb, err := d.Decode(rawFlow, decoder.Option{}, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } + if nb != 1 { + t.Errorf("Decode() returned %d instead of 1", nb) + } + + // Compare the decoded flow with the original + if diff := helpers.Diff(got, []*schema.FlowMessage{originalFlow}); diff != "" { + t.Errorf("decoded flow differs (-got, +want):\n%s", diff) + } +} + +func TestGobDecoderInvalidPayload(t *testing.T) { + r := reporter.NewMock(t) + sch := schema.NewMock(t) + d := New(r, decoder.Dependencies{Schema: sch}) + bf := sch.NewFlowMessage() + + rawFlow := decoder.RawFlow{ + TimeReceived: time.Now(), + Payload: []byte("invalid gob data"), + Source: netip.MustParseAddr("::ffff:192.0.2.1"), + } + + nb, err := d.Decode(rawFlow, decoder.Option{}, bf, func() {}) + if err == nil { + t.Errorf("expected error for invalid payload") + } + if nb != 0 { + t.Errorf("Decode() returned %d instead of 0", nb) + } +} diff --git a/inlet/flow/decoder/helpers.go b/outlet/flow/decoder/helpers.go similarity index 72% rename from inlet/flow/decoder/helpers.go rename to outlet/flow/decoder/helpers.go index 99281886..c12c68ba 100644 --- a/inlet/flow/decoder/helpers.go +++ b/outlet/flow/decoder/helpers.go @@ -18,18 +18,18 @@ func ParseIPv4(sch *schema.Component, bf *schema.FlowMessage, data []byte) uint6 if len(data) < 20 { return 0 } - sch.ProtobufAppendVarint(bf, schema.ColumnEType, helpers.ETypeIPv4) + bf.AppendUint(schema.ColumnEType, helpers.ETypeIPv4) l3length = uint64(binary.BigEndian.Uint16(data[2:4])) bf.SrcAddr = DecodeIP(data[12:16]) bf.DstAddr = DecodeIP(data[16:20]) proto = data[9] fragoffset := binary.BigEndian.Uint16(data[6:8]) & 0x1fff if !sch.IsDisabled(schema.ColumnGroupL3L4) { - sch.ProtobufAppendVarint(bf, schema.ColumnIPTos, uint64(data[1])) - sch.ProtobufAppendVarint(bf, schema.ColumnIPTTL, uint64(data[8])) - sch.ProtobufAppendVarint(bf, schema.ColumnIPFragmentID, + bf.AppendUint(schema.ColumnIPTos, uint64(data[1])) + bf.AppendUint(schema.ColumnIPTTL, uint64(data[8])) + bf.AppendUint(schema.ColumnIPFragmentID, uint64(binary.BigEndian.Uint16(data[4:6]))) - sch.ProtobufAppendVarint(bf, schema.ColumnIPFragmentOffset, + bf.AppendUint(schema.ColumnIPFragmentOffset, uint64(fragoffset)) } ihl := int((data[0] & 0xf) * 4) @@ -38,7 +38,7 @@ func ParseIPv4(sch *schema.Component, bf *schema.FlowMessage, data []byte) uint6 } else { data = data[:0] } - sch.ProtobufAppendVarint(bf, schema.ColumnProto, uint64(proto)) + bf.AppendUint(schema.ColumnProto, uint64(proto)) if fragoffset == 0 { ParseL4(sch, bf, data, proto) } @@ -53,21 +53,21 @@ func ParseIPv6(sch *schema.Component, bf *schema.FlowMessage, data []byte) uint6 return 0 } l3length = uint64(binary.BigEndian.Uint16(data[4:6])) + 40 - sch.ProtobufAppendVarint(bf, schema.ColumnEType, helpers.ETypeIPv6) + bf.AppendUint(schema.ColumnEType, helpers.ETypeIPv6) bf.SrcAddr = DecodeIP(data[8:24]) bf.DstAddr = DecodeIP(data[24:40]) proto = data[6] - sch.ProtobufAppendVarint(bf, schema.ColumnProto, uint64(proto)) + bf.AppendUint(schema.ColumnProto, uint64(proto)) if !sch.IsDisabled(schema.ColumnGroupL3L4) { - sch.ProtobufAppendVarint(bf, schema.ColumnIPTos, + bf.AppendUint(schema.ColumnIPTos, uint64(binary.BigEndian.Uint16(data[0:2])&0xff0>>4)) - sch.ProtobufAppendVarint(bf, schema.ColumnIPTTL, uint64(data[7])) - sch.ProtobufAppendVarint(bf, schema.ColumnIPv6FlowLabel, + bf.AppendUint(schema.ColumnIPTTL, uint64(data[7])) + bf.AppendUint(schema.ColumnIPv6FlowLabel, uint64(binary.BigEndian.Uint32(data[0:4])&0xfffff)) // TODO fragmentID/fragmentOffset are in a separate header } data = data[40:] - sch.ProtobufAppendVarint(bf, schema.ColumnProto, uint64(proto)) + bf.AppendUint(schema.ColumnProto, uint64(proto)) ParseL4(sch, bf, data, proto) return l3length } @@ -77,9 +77,9 @@ func ParseL4(sch *schema.Component, bf *schema.FlowMessage, data []byte, proto u if proto == 6 || proto == 17 { // UDP or TCP if len(data) > 4 { - sch.ProtobufAppendVarint(bf, schema.ColumnSrcPort, + bf.AppendUint(schema.ColumnSrcPort, uint64(binary.BigEndian.Uint16(data[0:2]))) - sch.ProtobufAppendVarint(bf, schema.ColumnDstPort, + bf.AppendUint(schema.ColumnDstPort, uint64(binary.BigEndian.Uint16(data[2:4]))) } } @@ -87,23 +87,23 @@ func ParseL4(sch *schema.Component, bf *schema.FlowMessage, data []byte, proto u if proto == 6 { // TCP if len(data) > 13 { - sch.ProtobufAppendVarint(bf, schema.ColumnTCPFlags, + bf.AppendUint(schema.ColumnTCPFlags, uint64(data[13])) } } else if proto == 1 { // ICMPv4 if len(data) > 2 { - sch.ProtobufAppendVarint(bf, schema.ColumnICMPv4Type, + bf.AppendUint(schema.ColumnICMPv4Type, uint64(data[0])) - sch.ProtobufAppendVarint(bf, schema.ColumnICMPv4Code, + bf.AppendUint(schema.ColumnICMPv4Code, uint64(data[1])) } } else if proto == 58 { // ICMPv6 if len(data) > 2 { - sch.ProtobufAppendVarint(bf, schema.ColumnICMPv6Type, + bf.AppendUint(schema.ColumnICMPv6Type, uint64(data[0])) - sch.ProtobufAppendVarint(bf, schema.ColumnICMPv6Code, + bf.AppendUint(schema.ColumnICMPv6Code, uint64(data[1])) } } @@ -116,9 +116,9 @@ func ParseEthernet(sch *schema.Component, bf *schema.FlowMessage, data []byte) u return 0 } if !sch.IsDisabled(schema.ColumnGroupL2) { - sch.ProtobufAppendVarint(bf, schema.ColumnDstMAC, + bf.AppendUint(schema.ColumnDstMAC, binary.BigEndian.Uint64([]byte{0, 0, data[0], data[1], data[2], data[3], data[4], data[5]})) - sch.ProtobufAppendVarint(bf, schema.ColumnSrcMAC, + bf.AppendUint(schema.ColumnSrcMAC, binary.BigEndian.Uint64([]byte{0, 0, data[6], data[7], data[8], data[9], data[10], data[11]})) } etherType := data[12:14] @@ -140,6 +140,7 @@ func ParseEthernet(sch *schema.Component, bf *schema.FlowMessage, data []byte) u } if etherType[0] == 0x88 && etherType[1] == 0x47 { // MPLS + mplsLabels := make([]uint32, 0, 5) for { if len(data) < 5 { return 0 @@ -147,18 +148,22 @@ func ParseEthernet(sch *schema.Component, bf *schema.FlowMessage, data []byte) u label := binary.BigEndian.Uint32(append([]byte{0}, data[:3]...)) >> 4 bottom := data[2] & 1 data = data[4:] - sch.ProtobufAppendVarint(bf, schema.ColumnMPLSLabels, uint64(label)) + mplsLabels = append(mplsLabels, label) if bottom == 1 || label <= 15 { - if data[0]&0xf0>>4 == 4 { + switch data[0] & 0xf0 >> 4 { + case 4: etherType = []byte{0x8, 0x0} - } else if data[0]&0xf0>>4 == 6 { + case 6: etherType = []byte{0x86, 0xdd} - } else { + default: return 0 } break } } + if len(mplsLabels) > 0 { + bf.AppendArrayUInt32(schema.ColumnMPLSLabels, mplsLabels) + } } if etherType[0] == 0x8 && etherType[1] == 0x0 { return ParseIPv4(sch, bf, data) diff --git a/inlet/flow/decoder/helpers_test.go b/outlet/flow/decoder/helpers_test.go similarity index 92% rename from inlet/flow/decoder/helpers_test.go rename to outlet/flow/decoder/helpers_test.go index a12733ac..1b57803f 100644 --- a/inlet/flow/decoder/helpers_test.go +++ b/outlet/flow/decoder/helpers_test.go @@ -15,7 +15,7 @@ import ( func TestDecodeMPLSAndIPv4(t *testing.T) { sch := schema.NewMock(t).EnableAllColumns() pcap := helpers.ReadPcapL2(t, filepath.Join("testdata", "mpls-ipv4.pcap")) - bf := &schema.FlowMessage{} + bf := sch.NewFlowMessage() l := ParseEthernet(sch, bf, pcap) if l != 40 { t.Errorf("ParseEthernet() returned %d, expected 40", l) @@ -23,7 +23,7 @@ func TestDecodeMPLSAndIPv4(t *testing.T) { expected := schema.FlowMessage{ SrcAddr: netip.MustParseAddr("::ffff:10.31.0.1"), DstAddr: netip.MustParseAddr("::ffff:10.34.0.1"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnEType: helpers.ETypeIPv4, schema.ColumnProto: 6, schema.ColumnSrcPort: 11001, @@ -45,7 +45,7 @@ func TestDecodeMPLSAndIPv4(t *testing.T) { func TestDecodeVLANAndIPv6(t *testing.T) { sch := schema.NewMock(t).EnableAllColumns() pcap := helpers.ReadPcapL2(t, filepath.Join("testdata", "vlan-ipv6.pcap")) - bf := &schema.FlowMessage{} + bf := sch.NewFlowMessage() l := ParseEthernet(sch, bf, pcap) if l != 179 { t.Errorf("ParseEthernet() returned %d, expected 179", l) @@ -54,7 +54,7 @@ func TestDecodeVLANAndIPv6(t *testing.T) { SrcVlan: 100, SrcAddr: netip.MustParseAddr("2402:f000:1:8e01::5555"), DstAddr: netip.MustParseAddr("2607:fcd0:100:2300::b108:2a6b"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]interface{}{ schema.ColumnEType: helpers.ETypeIPv6, schema.ColumnProto: 4, schema.ColumnIPTTL: 246, diff --git a/inlet/flow/decoder/netflow/decode.go b/outlet/flow/decoder/netflow/decode.go similarity index 66% rename from inlet/flow/decoder/netflow/decode.go rename to outlet/flow/decoder/netflow/decode.go index 28286b56..dfa64d5f 100644 --- a/inlet/flow/decoder/netflow/decode.go +++ b/outlet/flow/decoder/netflow/decode.go @@ -9,8 +9,9 @@ import ( "net/netip" "akvorado/common/helpers" + "akvorado/common/pb" "akvorado/common/schema" - "akvorado/inlet/flow/decoder" + "akvorado/outlet/flow/decoder" "github.com/netsampler/goflow2/v2/decoders/netflow" "github.com/netsampler/goflow2/v2/decoders/netflowlegacy" @@ -21,46 +22,39 @@ import ( // values in the sub-range of 1-127 are compatible with field types used by // NetFlow version 9 [RFC3954]." -func (nd *Decoder) decodeNFv5(packet *netflowlegacy.PacketNetFlowV5, ts, sysUptime uint64) []*schema.FlowMessage { - flowMessageSet := []*schema.FlowMessage{} - +func (nd *Decoder) decodeNFv5(packet *netflowlegacy.PacketNetFlowV5, ts, sysUptime uint64, options decoder.Option, bf *schema.FlowMessage, finalize decoder.FinalizeFlowFunc) { for _, record := range packet.Records { - bf := &schema.FlowMessage{ - SamplingRate: uint32(packet.SamplingInterval), - InIf: uint32(record.Input), - OutIf: uint32(record.Output), - SrcAddr: decodeIPFromUint32(uint32(record.SrcAddr)), - DstAddr: decodeIPFromUint32(uint32(record.DstAddr)), - NextHop: decodeIPFromUint32(uint32(record.NextHop)), - SrcNetMask: record.SrcMask, - DstNetMask: record.DstMask, - SrcAS: uint32(record.SrcAS), - DstAS: uint32(record.DstAS), - } - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnBytes, uint64(record.DOctets)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnPackets, uint64(record.DPkts)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnEType, helpers.ETypeIPv4) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnProto, uint64(record.Proto)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnSrcPort, uint64(record.SrcPort)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnDstPort, uint64(record.DstPort)) + bf.SamplingRate = uint64(packet.SamplingInterval) + bf.InIf = uint32(record.Input) + bf.OutIf = uint32(record.Output) + bf.SrcAddr = decodeIPFromUint32(uint32(record.SrcAddr)) + bf.DstAddr = decodeIPFromUint32(uint32(record.DstAddr)) + bf.NextHop = decodeIPFromUint32(uint32(record.NextHop)) + bf.SrcNetMask = record.SrcMask + bf.DstNetMask = record.DstMask + bf.SrcAS = uint32(record.SrcAS) + bf.DstAS = uint32(record.DstAS) + bf.AppendUint(schema.ColumnBytes, uint64(record.DOctets)) + bf.AppendUint(schema.ColumnPackets, uint64(record.DPkts)) + bf.AppendUint(schema.ColumnEType, helpers.ETypeIPv4) + bf.AppendUint(schema.ColumnProto, uint64(record.Proto)) + bf.AppendUint(schema.ColumnSrcPort, uint64(record.SrcPort)) + bf.AppendUint(schema.ColumnDstPort, uint64(record.DstPort)) if !nd.d.Schema.IsDisabled(schema.ColumnGroupL3L4) { - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnIPTos, uint64(record.Tos)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnTCPFlags, uint64(record.TCPFlags)) + bf.AppendUint(schema.ColumnIPTos, uint64(record.Tos)) + bf.AppendUint(schema.ColumnTCPFlags, uint64(record.TCPFlags)) } - if nd.useTsFromFirstSwitched { - bf.TimeReceived = ts - sysUptime + uint64(record.First) + if options.TimestampSource == pb.RawFlow_TS_NETFLOW_FIRST_SWITCHED { + bf.TimeReceived = uint32(ts - sysUptime + uint64(record.First)) } if bf.SamplingRate == 0 { bf.SamplingRate = 1 } - flowMessageSet = append(flowMessageSet, bf) + finalize() } - return flowMessageSet } -func (nd *Decoder) decodeNFv9IPFIX(version uint16, obsDomainID uint32, flowSets []interface{}, samplingRateSys *samplingRateSystem, ts, sysUptime uint64) []*schema.FlowMessage { - flowMessageSet := []*schema.FlowMessage{} - +func (nd *Decoder) decodeNFv9IPFIX(version uint16, obsDomainID uint32, flowSets []interface{}, samplingRateSys *samplingRateSystem, ts, sysUptime uint64, options decoder.Option, bf *schema.FlowMessage, finalize decoder.FinalizeFlowFunc) { // Look for sampling rate in option data flowsets for _, flowSet := range flowSets { switch tFlowSet := flowSet.(type) { @@ -96,22 +90,18 @@ func (nd *Decoder) decodeNFv9IPFIX(version uint16, obsDomainID uint32, flowSets } case netflow.DataFlowSet: for _, record := range tFlowSet.Records { - flow := nd.decodeRecord(version, obsDomainID, samplingRateSys, record.Values, ts, sysUptime) - if flow != nil { - flowMessageSet = append(flowMessageSet, flow) - } + nd.decodeRecord(version, obsDomainID, samplingRateSys, record.Values, ts, sysUptime, options, bf) + finalize() } } } - - return flowMessageSet } -func (nd *Decoder) decodeRecord(version uint16, obsDomainID uint32, samplingRateSys *samplingRateSystem, fields []netflow.DataField, ts, sysUptime uint64) *schema.FlowMessage { +func (nd *Decoder) decodeRecord(version uint16, obsDomainID uint32, samplingRateSys *samplingRateSystem, fields []netflow.DataField, ts, sysUptime uint64, options decoder.Option, bf *schema.FlowMessage) { var etype, dstPort, srcPort uint16 var proto, icmpType, icmpCode uint8 var foundIcmpTypeCode bool - bf := &schema.FlowMessage{} + mplsLabels := make([]uint32, 0, 5) dataLinkFrameSectionIdx := -1 for idx, field := range fields { v, ok := field.Value.([]byte) @@ -122,13 +112,13 @@ func (nd *Decoder) decodeRecord(version uint16, obsDomainID uint32, samplingRate switch field.Type { // Statistics case netflow.IPFIX_FIELD_octetDeltaCount, netflow.IPFIX_FIELD_postOctetDeltaCount, netflow.IPFIX_FIELD_initiatorOctets, netflow.IPFIX_FIELD_responderOctets: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnBytes, decodeUNumber(v)) + bf.AppendUint(schema.ColumnBytes, decodeUNumber(v)) case netflow.IPFIX_FIELD_packetDeltaCount, netflow.IPFIX_FIELD_postPacketDeltaCount: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnPackets, decodeUNumber(v)) + bf.AppendUint(schema.ColumnPackets, decodeUNumber(v)) case netflow.IPFIX_FIELD_samplingInterval, netflow.IPFIX_FIELD_samplerRandomInterval: - bf.SamplingRate = uint32(decodeUNumber(v)) + bf.SamplingRate = decodeUNumber(v) case netflow.IPFIX_FIELD_samplerId, netflow.IPFIX_FIELD_selectorId: - bf.SamplingRate = samplingRateSys.GetSamplingRate(version, obsDomainID, decodeUNumber(v)) + bf.SamplingRate = uint64(samplingRateSys.GetSamplingRate(version, obsDomainID, decodeUNumber(v))) // L3 case netflow.IPFIX_FIELD_sourceIPv4Address: @@ -153,13 +143,13 @@ func (nd *Decoder) decodeRecord(version uint16, obsDomainID uint32, samplingRate // L4 case netflow.IPFIX_FIELD_sourceTransportPort: srcPort = uint16(decodeUNumber(v)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnSrcPort, uint64(srcPort)) + bf.AppendUint(schema.ColumnSrcPort, uint64(srcPort)) case netflow.IPFIX_FIELD_destinationTransportPort: dstPort = uint16(decodeUNumber(v)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnDstPort, uint64(dstPort)) + bf.AppendUint(schema.ColumnDstPort, uint64(dstPort)) case netflow.IPFIX_FIELD_protocolIdentifier: proto = uint8(decodeUNumber(v)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnProto, uint64(proto)) + bf.AppendUint(schema.ColumnProto, uint64(proto)) // Network case netflow.IPFIX_FIELD_bgpSourceAsNumber: @@ -181,24 +171,27 @@ func (nd *Decoder) decodeRecord(version uint16, obsDomainID uint32, samplingRate // MPLS case netflow.IPFIX_FIELD_mplsTopLabelStackSection, netflow.IPFIX_FIELD_mplsLabelStackSection2, netflow.IPFIX_FIELD_mplsLabelStackSection3, netflow.IPFIX_FIELD_mplsLabelStackSection4, netflow.IPFIX_FIELD_mplsLabelStackSection5, netflow.IPFIX_FIELD_mplsLabelStackSection6, netflow.IPFIX_FIELD_mplsLabelStackSection7, netflow.IPFIX_FIELD_mplsLabelStackSection8, netflow.IPFIX_FIELD_mplsLabelStackSection9, netflow.IPFIX_FIELD_mplsLabelStackSection10: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnMPLSLabels, decodeUNumber(v)>>4) + uv := decodeUNumber(v) >> 4 + if uv > 0 { + mplsLabels = append(mplsLabels, uint32(uv)) + } // Remaining case netflow.IPFIX_FIELD_forwardingStatus: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnForwardingStatus, decodeUNumber(v)) + bf.AppendUint(schema.ColumnForwardingStatus, decodeUNumber(v)) default: - if nd.useTsFromFirstSwitched { + if options.TimestampSource == pb.RawFlow_TS_NETFLOW_FIRST_SWITCHED { switch field.Type { case netflow.NFV9_FIELD_FIRST_SWITCHED: - bf.TimeReceived = ts - sysUptime + decodeUNumber(v) + bf.TimeReceived = uint32(ts - sysUptime + decodeUNumber(v)) case netflow.IPFIX_FIELD_flowStartSeconds: - bf.TimeReceived = decodeUNumber(v) + bf.TimeReceived = uint32(decodeUNumber(v)) case netflow.IPFIX_FIELD_flowStartMilliseconds: - bf.TimeReceived = decodeUNumber(v) / 1000 + bf.TimeReceived = uint32(decodeUNumber(v) / 1000) case netflow.IPFIX_FIELD_flowStartMicroseconds: - bf.TimeReceived = decodeUNumber(v) / 1_000_000 + bf.TimeReceived = uint32(decodeUNumber(v) / 1_000_000) case netflow.IPFIX_FIELD_flowStartNanoseconds: - bf.TimeReceived = ts + decodeUNumber(v)/1_000_000_000 + bf.TimeReceived = uint32(ts + decodeUNumber(v)/1_000_000_000) } } @@ -206,13 +199,13 @@ func (nd *Decoder) decodeRecord(version uint16, obsDomainID uint32, samplingRate // NAT switch field.Type { case netflow.IPFIX_FIELD_postNATSourceIPv4Address: - nd.d.Schema.ProtobufAppendIP(bf, schema.ColumnSrcAddrNAT, decodeIPFromBytes(v)) + bf.AppendIPv6(schema.ColumnSrcAddrNAT, decodeIPFromBytes(v)) case netflow.IPFIX_FIELD_postNATDestinationIPv4Address: - nd.d.Schema.ProtobufAppendIP(bf, schema.ColumnDstAddrNAT, decodeIPFromBytes(v)) + bf.AppendIPv6(schema.ColumnDstAddrNAT, decodeIPFromBytes(v)) case netflow.IPFIX_FIELD_postNAPTSourceTransportPort: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnSrcPortNAT, decodeUNumber(v)) + bf.AppendUint(schema.ColumnSrcPortNAT, decodeUNumber(v)) case netflow.IPFIX_FIELD_postNAPTDestinationTransportPort: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnDstPortNAT, decodeUNumber(v)) + bf.AppendUint(schema.ColumnDstPortNAT, decodeUNumber(v)) } } @@ -224,13 +217,13 @@ func (nd *Decoder) decodeRecord(version uint16, obsDomainID uint32, samplingRate case netflow.IPFIX_FIELD_postVlanId: bf.DstVlan = uint16(decodeUNumber(v)) case netflow.IPFIX_FIELD_sourceMacAddress: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnSrcMAC, decodeUNumber(v)) + bf.AppendUint(schema.ColumnSrcMAC, decodeUNumber(v)) case netflow.IPFIX_FIELD_destinationMacAddress: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnDstMAC, decodeUNumber(v)) + bf.AppendUint(schema.ColumnDstMAC, decodeUNumber(v)) case netflow.IPFIX_FIELD_postSourceMacAddress: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnSrcMAC, decodeUNumber(v)) + bf.AppendUint(schema.ColumnSrcMAC, decodeUNumber(v)) case netflow.IPFIX_FIELD_postDestinationMacAddress: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnDstMAC, decodeUNumber(v)) + bf.AppendUint(schema.ColumnDstMAC, decodeUNumber(v)) } } @@ -238,17 +231,17 @@ func (nd *Decoder) decodeRecord(version uint16, obsDomainID uint32, samplingRate // Misc L3/L4 fields switch field.Type { case netflow.IPFIX_FIELD_minimumTTL: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnIPTTL, decodeUNumber(v)) + bf.AppendUint(schema.ColumnIPTTL, decodeUNumber(v)) case netflow.IPFIX_FIELD_ipClassOfService: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnIPTos, decodeUNumber(v)) + bf.AppendUint(schema.ColumnIPTos, decodeUNumber(v)) case netflow.IPFIX_FIELD_flowLabelIPv6: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnIPv6FlowLabel, decodeUNumber(v)) + bf.AppendUint(schema.ColumnIPv6FlowLabel, decodeUNumber(v)) case netflow.IPFIX_FIELD_tcpControlBits: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnTCPFlags, decodeUNumber(v)) + bf.AppendUint(schema.ColumnTCPFlags, decodeUNumber(v)) case netflow.IPFIX_FIELD_fragmentIdentification: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnIPFragmentID, decodeUNumber(v)) + bf.AppendUint(schema.ColumnIPFragmentID, decodeUNumber(v)) case netflow.IPFIX_FIELD_fragmentOffset: - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnIPFragmentOffset, decodeUNumber(v)) + bf.AppendUint(schema.ColumnIPFragmentOffset, decodeUNumber(v)) // ICMP case netflow.IPFIX_FIELD_icmpTypeCodeIPv4, netflow.IPFIX_FIELD_icmpTypeCodeIPv6: @@ -269,8 +262,8 @@ func (nd *Decoder) decodeRecord(version uint16, obsDomainID uint32, samplingRate if dataLinkFrameSectionIdx >= 0 { data := fields[dataLinkFrameSectionIdx].Value.([]byte) if l3Length := decoder.ParseEthernet(nd.d.Schema, bf, data); l3Length > 0 { - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnBytes, l3Length) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnPackets, 1) + bf.AppendUint(schema.ColumnBytes, l3Length) + bf.AppendUint(schema.ColumnPackets, 1) } } if !nd.d.Schema.IsDisabled(schema.ColumnGroupL3L4) && (proto == 1 || proto == 58) { @@ -290,18 +283,20 @@ func (nd *Decoder) decodeRecord(version uint16, obsDomainID uint32, samplingRate } } if proto == 1 { - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnICMPv4Type, uint64(icmpType)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnICMPv4Code, uint64(icmpCode)) + bf.AppendUint(schema.ColumnICMPv4Type, uint64(icmpType)) + bf.AppendUint(schema.ColumnICMPv4Code, uint64(icmpCode)) } else { - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnICMPv6Type, uint64(icmpType)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnICMPv6Code, uint64(icmpCode)) + bf.AppendUint(schema.ColumnICMPv6Type, uint64(icmpType)) + bf.AppendUint(schema.ColumnICMPv6Code, uint64(icmpCode)) } } - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnEType, uint64(etype)) - if bf.SamplingRate == 0 { - bf.SamplingRate = samplingRateSys.GetSamplingRate(version, obsDomainID, 0) + bf.AppendUint(schema.ColumnEType, uint64(etype)) + if len(mplsLabels) > 0 { + bf.AppendArrayUInt32(schema.ColumnMPLSLabels, mplsLabels) + } + if bf.SamplingRate == 0 { + bf.SamplingRate = uint64(samplingRateSys.GetSamplingRate(version, obsDomainID, 0)) } - return bf } func decodeUNumber(b []byte) uint64 { diff --git a/inlet/flow/decoder/netflow/root.go b/outlet/flow/decoder/netflow/root.go similarity index 80% rename from inlet/flow/decoder/netflow/root.go rename to outlet/flow/decoder/netflow/root.go index aa15ef13..5bc3daa0 100644 --- a/inlet/flow/decoder/netflow/root.go +++ b/outlet/flow/decoder/netflow/root.go @@ -8,7 +8,7 @@ import ( "bytes" "encoding/binary" "errors" - "net/netip" + "fmt" "strconv" "sync" "time" @@ -16,9 +16,10 @@ import ( "github.com/netsampler/goflow2/v2/decoders/netflow" "github.com/netsampler/goflow2/v2/decoders/netflowlegacy" + "akvorado/common/pb" "akvorado/common/reporter" "akvorado/common/schema" - "akvorado/inlet/flow/decoder" + "akvorado/outlet/flow/decoder" ) // Decoder contains the state for the Netflow v9 decoder. @@ -39,20 +40,16 @@ type Decoder struct { setStatsSum *reporter.CounterVec templatesStats *reporter.CounterVec } - useTsFromNetflowsPacket bool - useTsFromFirstSwitched bool } // New instantiates a new netflow decoder. -func New(r *reporter.Reporter, dependencies decoder.Dependencies, option decoder.Option) decoder.Decoder { +func New(r *reporter.Reporter, dependencies decoder.Dependencies) decoder.Decoder { nd := &Decoder{ - r: r, - d: dependencies, - errLogger: r.Sample(reporter.BurstSampler(30*time.Second, 3)), - templates: map[string]*templateSystem{}, - sampling: map[string]*samplingRateSystem{}, - useTsFromNetflowsPacket: option.TimestampSource == decoder.TimestampSourceNetflowPacket, - useTsFromFirstSwitched: option.TimestampSource == decoder.TimestampSourceNetflowFirstSwitched, + r: r, + d: dependencies, + errLogger: r.Sample(reporter.BurstSampler(30*time.Second, 3)), + templates: map[string]*templateSystem{}, + sampling: map[string]*samplingRateSystem{}, } nd.metrics.errors = nd.r.CounterVec( @@ -169,9 +166,9 @@ func (s *samplingRateSystem) SetSamplingRate(version uint16, obsDomainID uint32, } // Decode decodes a Netflow payload. -func (nd *Decoder) Decode(in decoder.RawFlow) []*schema.FlowMessage { +func (nd *Decoder) Decode(in decoder.RawFlow, options decoder.Option, bf *schema.FlowMessage, finalize decoder.FinalizeFlowFunc) (int, error) { if len(in.Payload) < 2 { - return nil + return 0, errors.New("payload too small") } key := in.Source.String() nd.systemsLock.RLock() @@ -198,15 +195,21 @@ func (nd *Decoder) Decode(in decoder.RawFlow) []*schema.FlowMessage { } var ( - sysUptime uint64 - versionStr string - flowSets []interface{} - obsDomainID uint32 - flowMessageSet []*schema.FlowMessage + sysUptime uint64 + versionStr string + flowSets []any + obsDomainID uint32 ) version := binary.BigEndian.Uint16(in.Payload[:2]) buf := bytes.NewBuffer(in.Payload[2:]) - ts := uint64(in.TimeReceived.UTC().Unix()) + ts := uint64(in.TimeReceived.UTC().Unix()) // may be altered later + finalize2 := func() { + if bf.TimeReceived == 0 { + bf.TimeReceived = uint32(ts) + } + bf.ExporterAddress = in.Source + finalize() + } switch version { case 5: @@ -214,61 +217,63 @@ func (nd *Decoder) Decode(in decoder.RawFlow) []*schema.FlowMessage { if err := netflowlegacy.DecodeMessage(buf, &packetNFv5); err != nil { nd.metrics.errors.WithLabelValues(key, "NetFlow v5 decoding error").Inc() nd.errLogger.Err(err).Str("exporter", key).Msg("error while decoding NetFlow v5") - return nil + return 0, fmt.Errorf("NetFlow v5 decoding error: %w", err) } versionStr = "5" nd.metrics.setStatsSum.WithLabelValues(key, versionStr, "PDU").Inc() nd.metrics.setRecordsStatsSum.WithLabelValues(key, versionStr, "PDU"). Add(float64(len(packetNFv5.Records))) - if nd.useTsFromNetflowsPacket || nd.useTsFromFirstSwitched { + if options.TimestampSource == pb.RawFlow_TS_NETFLOW_PACKET || options.TimestampSource == pb.RawFlow_TS_NETFLOW_FIRST_SWITCHED { ts = uint64(packetNFv5.UnixSecs) sysUptime = uint64(packetNFv5.SysUptime) } - flowMessageSet = nd.decodeNFv5(&packetNFv5, ts, sysUptime) + nd.decodeNFv5(&packetNFv5, ts, sysUptime, options, bf, finalize2) case 9: var packetNFv9 netflow.NFv9Packet if err := netflow.DecodeMessageNetFlow(buf, templates, &packetNFv9); err != nil { if !errors.Is(err, netflow.ErrorTemplateNotFound) { nd.errLogger.Err(err).Str("exporter", key).Msg("error while decoding NetFlow v9") nd.metrics.errors.WithLabelValues(key, "NetFlow v9 decoding error").Inc() - } else { - nd.errLogger.Debug().Str("exporter", key).Msg("template not received yet") + return 0, fmt.Errorf("NetFlow v9 decoding error: %w", err) } - return nil + nd.errLogger.Debug().Str("exporter", key).Msg("template not received yet") + return 0, nil } versionStr = "9" flowSets = packetNFv9.FlowSets obsDomainID = packetNFv9.SourceId - if nd.useTsFromNetflowsPacket || nd.useTsFromFirstSwitched { + if options.TimestampSource == pb.RawFlow_TS_NETFLOW_PACKET || options.TimestampSource == pb.RawFlow_TS_NETFLOW_FIRST_SWITCHED { ts = uint64(packetNFv9.UnixSeconds) sysUptime = uint64(packetNFv9.SystemUptime) } - flowMessageSet = nd.decodeNFv9IPFIX(version, obsDomainID, flowSets, sampling, ts, sysUptime) + nd.decodeNFv9IPFIX(version, obsDomainID, flowSets, sampling, ts, sysUptime, options, bf, finalize2) case 10: var packetIPFIX netflow.IPFIXPacket if err := netflow.DecodeMessageIPFIX(buf, templates, &packetIPFIX); err != nil { if !errors.Is(err, netflow.ErrorTemplateNotFound) { nd.errLogger.Err(err).Str("exporter", key).Msg("error while decoding IPFIX") nd.metrics.errors.WithLabelValues(key, "IPFIX decoding error").Inc() - } else { - nd.errLogger.Debug().Str("exporter", key).Msg("template not received yet") + return 0, fmt.Errorf("NetFlow v9 decoding error: %w", err) } - return nil + nd.errLogger.Debug().Str("exporter", key).Msg("template not received yet") + return 0, nil } versionStr = "10" flowSets = packetIPFIX.FlowSets obsDomainID = packetIPFIX.ObservationDomainId - if nd.useTsFromNetflowsPacket { + if options.TimestampSource == pb.RawFlow_TS_NETFLOW_PACKET { ts = uint64(packetIPFIX.ExportTime) } - flowMessageSet = nd.decodeNFv9IPFIX(version, obsDomainID, flowSets, sampling, ts, sysUptime) + nd.decodeNFv9IPFIX(version, obsDomainID, flowSets, sampling, ts, sysUptime, options, bf, finalize2) default: + nd.errLogger.Warn().Str("exporter", key).Msg("unknown NetFlow version") nd.metrics.stats.WithLabelValues(key, "unknown"). Inc() - return nil + return 0, errors.New("unkown NetFlow version") } nd.metrics.stats.WithLabelValues(key, versionStr).Inc() + nb := 0 for _, fs := range flowSets { switch fsConv := fs.(type) { case netflow.TemplateFlowSet: @@ -296,18 +301,11 @@ func (nd *Decoder) Decode(in decoder.RawFlow) []*schema.FlowMessage { Inc() nd.metrics.setRecordsStatsSum.WithLabelValues(key, versionStr, "DataFlowSet"). Add(float64(len(fsConv.Records))) + nb += len(fsConv.Records) } } - exporterAddress, _ := netip.AddrFromSlice(in.Source.To16()) - for _, fmsg := range flowMessageSet { - if fmsg.TimeReceived == 0 { - fmsg.TimeReceived = ts - } - fmsg.ExporterAddress = exporterAddress - } - - return flowMessageSet + return nb, nil } // Name returns the name of the decoder. diff --git a/inlet/flow/decoder/netflow/root_test.go b/outlet/flow/decoder/netflow/root_test.go similarity index 56% rename from inlet/flow/decoder/netflow/root_test.go rename to outlet/flow/decoder/netflow/root_test.go index a8fe6e2d..62cdc814 100644 --- a/inlet/flow/decoder/netflow/root_test.go +++ b/outlet/flow/decoder/netflow/root_test.go @@ -5,38 +5,60 @@ package netflow import ( "fmt" - "net" "net/netip" "path/filepath" "testing" "akvorado/common/helpers" + "akvorado/common/pb" "akvorado/common/reporter" "akvorado/common/schema" - "akvorado/inlet/flow/decoder" + "akvorado/outlet/flow/decoder" ) -func TestDecode(t *testing.T) { +func setup(t *testing.T, clearTs bool) (*reporter.Reporter, decoder.Decoder, *schema.FlowMessage, *[]*schema.FlowMessage, decoder.FinalizeFlowFunc) { + t.Helper() r := reporter.NewMock(t) - nfdecoder := New(r, decoder.Dependencies{Schema: schema.NewMock(t).EnableAllColumns()}, decoder.Option{TimestampSource: decoder.TimestampSourceUDP}) + sch := schema.NewMock(t).EnableAllColumns() + nfdecoder := New(r, decoder.Dependencies{Schema: sch}) + bf := sch.NewFlowMessage() + got := []*schema.FlowMessage{} + finalize := func() { + if clearTs { + bf.TimeReceived = 0 + } + // Keep a copy of the current flow message + clone := *bf + got = append(got, &clone) + // And clear the flow message + bf.Clear() + } + return r, nfdecoder, bf, &got, finalize +} + +func TestDecode(t *testing.T) { + r, nfdecoder, bf, got, finalize := setup(t, true) + options := decoder.Option{TimestampSource: pb.RawFlow_TS_INPUT} // Send an option template template := helpers.ReadPcapL4(t, filepath.Join("testdata", "options-template.pcap")) - got := nfdecoder.Decode(decoder.RawFlow{Payload: template, Source: net.ParseIP("127.0.0.1")}) - if got == nil { - t.Fatalf("Decode() error on options template") + _, err := nfdecoder.Decode( + decoder.RawFlow{Payload: template, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error on options template:\n%+v", err) } - if len(got) != 0 { - t.Fatalf("Decode() on options template got flows") + if len(*got) != 0 { + t.Fatalf("Decode() on options template got flows:\n%+v", *got) } // Check metrics - gotMetrics := r.GetMetrics("akvorado_inlet_flow_decoder_netflow_") + gotMetrics := r.GetMetrics("akvorado_outlet_flow_decoder_netflow_") expectedMetrics := map[string]string{ - `flows_total{exporter="127.0.0.1",version="9"}`: "1", - `flowset_records_sum{exporter="127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", - `flowset_sum{exporter="127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", - `templates_total{exporter="127.0.0.1",obs_domain_id="0",template_id="257",type="options_template",version="9"}`: "1", + `flows_total{exporter="::ffff:127.0.0.1",version="9"}`: "1", + `flowset_records_sum{exporter="::ffff:127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", + `flowset_sum{exporter="::ffff:127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", + `templates_total{exporter="::ffff:127.0.0.1",obs_domain_id="0",template_id="257",type="options_template",version="9"}`: "1", } if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { t.Fatalf("Metrics after template (-got, +want):\n%s", diff) @@ -44,23 +66,25 @@ func TestDecode(t *testing.T) { // Send option data data := helpers.ReadPcapL4(t, filepath.Join("testdata", "options-data.pcap")) - got = nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) - if got == nil { - t.Fatalf("Decode() error on options data") + _, err = nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error on options data:\n%+v", err) } - if len(got) != 0 { + if len(*got) != 0 { t.Fatalf("Decode() on options data got flows") } // Check metrics - gotMetrics = r.GetMetrics("akvorado_inlet_flow_decoder_netflow_") + gotMetrics = r.GetMetrics("akvorado_outlet_flow_decoder_netflow_") expectedMetrics = map[string]string{ - `flows_total{exporter="127.0.0.1",version="9"}`: "2", - `flowset_records_sum{exporter="127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", - `flowset_records_sum{exporter="127.0.0.1",type="OptionsDataFlowSet",version="9"}`: "4", - `flowset_sum{exporter="127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", - `flowset_sum{exporter="127.0.0.1",type="OptionsDataFlowSet",version="9"}`: "1", - `templates_total{exporter="127.0.0.1",obs_domain_id="0",template_id="257",type="options_template",version="9"}`: "1", + `flows_total{exporter="::ffff:127.0.0.1",version="9"}`: "2", + `flowset_records_sum{exporter="::ffff:127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", + `flowset_records_sum{exporter="::ffff:127.0.0.1",type="OptionsDataFlowSet",version="9"}`: "4", + `flowset_sum{exporter="::ffff:127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", + `flowset_sum{exporter="::ffff:127.0.0.1",type="OptionsDataFlowSet",version="9"}`: "1", + `templates_total{exporter="::ffff:127.0.0.1",obs_domain_id="0",template_id="257",type="options_template",version="9"}`: "1", } if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { t.Fatalf("Metrics after template (-got, +want):\n%s", diff) @@ -68,26 +92,28 @@ func TestDecode(t *testing.T) { // Send a regular template template = helpers.ReadPcapL4(t, filepath.Join("testdata", "template.pcap")) - got = nfdecoder.Decode(decoder.RawFlow{Payload: template, Source: net.ParseIP("127.0.0.1")}) - if got == nil { - t.Fatalf("Decode() error on template") + _, err = nfdecoder.Decode( + decoder.RawFlow{Payload: template, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error on template:\n%+v", err) } - if len(got) != 0 { + if len(*got) != 0 { t.Fatalf("Decode() on template got flows") } // Check metrics - gotMetrics = r.GetMetrics("akvorado_inlet_flow_decoder_netflow_") + gotMetrics = r.GetMetrics("akvorado_outlet_flow_decoder_netflow_") expectedMetrics = map[string]string{ - `flows_total{exporter="127.0.0.1",version="9"}`: "3", - `flowset_records_sum{exporter="127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", - `flowset_records_sum{exporter="127.0.0.1",type="OptionsDataFlowSet",version="9"}`: "4", - `flowset_records_sum{exporter="127.0.0.1",type="TemplateFlowSet",version="9"}`: "1", - `flowset_sum{exporter="127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", - `flowset_sum{exporter="127.0.0.1",type="OptionsDataFlowSet",version="9"}`: "1", - `flowset_sum{exporter="127.0.0.1",type="TemplateFlowSet",version="9"}`: "1", - `templates_total{exporter="127.0.0.1",obs_domain_id="0",template_id="257",type="options_template",version="9"}`: "1", - `templates_total{exporter="127.0.0.1",obs_domain_id="0",template_id="260",type="template",version="9"}`: "1", + `flows_total{exporter="::ffff:127.0.0.1",version="9"}`: "3", + `flowset_records_sum{exporter="::ffff:127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", + `flowset_records_sum{exporter="::ffff:127.0.0.1",type="OptionsDataFlowSet",version="9"}`: "4", + `flowset_records_sum{exporter="::ffff:127.0.0.1",type="TemplateFlowSet",version="9"}`: "1", + `flowset_sum{exporter="::ffff:127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", + `flowset_sum{exporter="::ffff:127.0.0.1",type="OptionsDataFlowSet",version="9"}`: "1", + `flowset_sum{exporter="::ffff:127.0.0.1",type="TemplateFlowSet",version="9"}`: "1", + `templates_total{exporter="::ffff:127.0.0.1",obs_domain_id="0",template_id="257",type="options_template",version="9"}`: "1", + `templates_total{exporter="::ffff:127.0.0.1",obs_domain_id="0",template_id="260",type="template",version="9"}`: "1", } if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { t.Fatalf("Metrics after template (-got, +want):\n%s", diff) @@ -95,9 +121,11 @@ func TestDecode(t *testing.T) { // Send data data = helpers.ReadPcapL4(t, filepath.Join("testdata", "data.pcap")) - got = nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) - if got == nil { - t.Fatalf("Decode() error on data") + _, err = nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error on data:\n%+v", err) } expectedFlows := []*schema.FlowMessage{ { @@ -110,7 +138,7 @@ func TestDecode(t *testing.T) { OutIf: 450, SrcNetMask: 24, DstNetMask: 14, - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnBytes: 1500, schema.ColumnPackets: 1, schema.ColumnEType: helpers.ETypeIPv4, @@ -130,7 +158,7 @@ func TestDecode(t *testing.T) { NextHop: netip.MustParseAddr("::ffff:194.149.174.71"), SrcNetMask: 24, DstNetMask: 14, - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnBytes: 1500, schema.ColumnPackets: 1, schema.ColumnEType: helpers.ETypeIPv4, @@ -150,7 +178,7 @@ func TestDecode(t *testing.T) { NextHop: netip.MustParseAddr("::ffff:252.223.0.0"), SrcNetMask: 20, DstNetMask: 18, - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnBytes: 1400, schema.ColumnPackets: 1, schema.ColumnEType: helpers.ETypeIPv4, @@ -170,7 +198,7 @@ func TestDecode(t *testing.T) { OutIf: 451, SrcNetMask: 16, DstNetMask: 14, - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnBytes: 1448, schema.ColumnPackets: 1, schema.ColumnEType: helpers.ETypeIPv4, @@ -182,31 +210,28 @@ func TestDecode(t *testing.T) { }, }, } - for _, f := range got { - f.TimeReceived = 0 - } if diff := helpers.Diff(got, expectedFlows); diff != "" { t.Fatalf("Decode() (-got, +want):\n%s", diff) } gotMetrics = r.GetMetrics( - "akvorado_inlet_flow_decoder_netflow_", + "akvorado_outlet_flow_decoder_netflow_", "flows_total", "flowset_", "templates_", ) expectedMetrics = map[string]string{ - `flows_total{exporter="127.0.0.1",version="9"}`: "4", - `flowset_records_sum{exporter="127.0.0.1",type="DataFlowSet",version="9"}`: "4", - `flowset_records_sum{exporter="127.0.0.1",type="OptionsDataFlowSet",version="9"}`: "4", - `flowset_records_sum{exporter="127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", - `flowset_records_sum{exporter="127.0.0.1",type="TemplateFlowSet",version="9"}`: "1", - `flowset_sum{exporter="127.0.0.1",type="DataFlowSet",version="9"}`: "1", - `flowset_sum{exporter="127.0.0.1",type="OptionsDataFlowSet",version="9"}`: "1", - `flowset_sum{exporter="127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", - `flowset_sum{exporter="127.0.0.1",type="TemplateFlowSet",version="9"}`: "1", - `templates_total{exporter="127.0.0.1",obs_domain_id="0",template_id="257",type="options_template",version="9"}`: "1", - `templates_total{exporter="127.0.0.1",obs_domain_id="0",template_id="260",type="template",version="9"}`: "1", + `flows_total{exporter="::ffff:127.0.0.1",version="9"}`: "4", + `flowset_records_sum{exporter="::ffff:127.0.0.1",type="DataFlowSet",version="9"}`: "4", + `flowset_records_sum{exporter="::ffff:127.0.0.1",type="OptionsDataFlowSet",version="9"}`: "4", + `flowset_records_sum{exporter="::ffff:127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", + `flowset_records_sum{exporter="::ffff:127.0.0.1",type="TemplateFlowSet",version="9"}`: "1", + `flowset_sum{exporter="::ffff:127.0.0.1",type="DataFlowSet",version="9"}`: "1", + `flowset_sum{exporter="::ffff:127.0.0.1",type="OptionsDataFlowSet",version="9"}`: "1", + `flowset_sum{exporter="::ffff:127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", + `flowset_sum{exporter="::ffff:127.0.0.1",type="TemplateFlowSet",version="9"}`: "1", + `templates_total{exporter="::ffff:127.0.0.1",obs_domain_id="0",template_id="257",type="options_template",version="9"}`: "1", + `templates_total{exporter="::ffff:127.0.0.1",obs_domain_id="0",template_id="260",type="template",version="9"}`: "1", } if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { t.Fatalf("Metrics after data (-got, +want):\n%s", diff) @@ -214,22 +239,24 @@ func TestDecode(t *testing.T) { } func TestTemplatesMixedWithData(t *testing.T) { - r := reporter.NewMock(t) - nfdecoder := New(r, decoder.Dependencies{Schema: schema.NewMock(t)}, decoder.Option{TimestampSource: decoder.TimestampSourceUDP}) + r, nfdecoder, bf, _, finalize := setup(t, true) + options := decoder.Option{TimestampSource: pb.RawFlow_TS_INPUT} // Send packet with both data and templates template := helpers.ReadPcapL4(t, filepath.Join("testdata", "data+templates.pcap")) - nfdecoder.Decode(decoder.RawFlow{Payload: template, Source: net.ParseIP("127.0.0.1")}) + nfdecoder.Decode( + decoder.RawFlow{Payload: template, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) // We don't really care about the data, but we should have accepted the // templates. Check the stats. gotMetrics := r.GetMetrics( - "akvorado_inlet_flow_decoder_netflow_", + "akvorado_outlet_flow_decoder_netflow_", "templates_", ) expectedMetrics := map[string]string{ - `templates_total{exporter="127.0.0.1",obs_domain_id="17170432",template_id="256",type="options_template",version="9"}`: "1", - `templates_total{exporter="127.0.0.1",obs_domain_id="17170432",template_id="257",type="template",version="9"}`: "1", + `templates_total{exporter="::ffff:127.0.0.1",obs_domain_id="17170432",template_id="256",type="options_template",version="9"}`: "1", + `templates_total{exporter="::ffff:127.0.0.1",obs_domain_id="17170432",template_id="257",type="template",version="9"}`: "1", } if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { t.Fatalf("Metrics after data (-got, +want):\n%s", diff) @@ -237,13 +264,24 @@ func TestTemplatesMixedWithData(t *testing.T) { } func TestDecodeSamplingRate(t *testing.T) { - r := reporter.NewMock(t) - nfdecoder := New(r, decoder.Dependencies{Schema: schema.NewMock(t).EnableAllColumns()}, decoder.Option{TimestampSource: decoder.TimestampSourceUDP}) + _, nfdecoder, bf, got, finalize := setup(t, true) + options := decoder.Option{TimestampSource: pb.RawFlow_TS_INPUT} data := helpers.ReadPcapL4(t, filepath.Join("testdata", "samplingrate-template.pcap")) - got := nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) + _, err := nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } + data = helpers.ReadPcapL4(t, filepath.Join("testdata", "samplingrate-data.pcap")) - got = append(got, nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")})...) + _, err = nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } expectedFlows := []*schema.FlowMessage{ { @@ -254,7 +292,7 @@ func TestDecodeSamplingRate(t *testing.T) { InIf: 13, SrcVlan: 701, NextHop: netip.MustParseAddr("::ffff:0.0.0.0"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnPackets: 1, schema.ColumnBytes: 160, schema.ColumnProto: 6, @@ -265,27 +303,43 @@ func TestDecodeSamplingRate(t *testing.T) { }, } - for _, f := range got { - f.TimeReceived = 0 - } - - if diff := helpers.Diff(got[:1], expectedFlows); diff != "" { + if diff := helpers.Diff((*got)[:1], expectedFlows); diff != "" { t.Fatalf("Decode() (-got, +want):\n%s", diff) } } func TestDecodeMultipleSamplingRates(t *testing.T) { - r := reporter.NewMock(t) - nfdecoder := New(r, decoder.Dependencies{Schema: schema.NewMock(t).EnableAllColumns()}, decoder.Option{TimestampSource: decoder.TimestampSourceUDP}) + _, nfdecoder, bf, got, finalize := setup(t, true) + options := decoder.Option{TimestampSource: pb.RawFlow_TS_INPUT} data := helpers.ReadPcapL4(t, filepath.Join("testdata", "multiplesamplingrates-options-template.pcap")) - got := nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) + _, err := nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } data = helpers.ReadPcapL4(t, filepath.Join("testdata", "multiplesamplingrates-options-data.pcap")) - got = nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) + _, err = nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } data = helpers.ReadPcapL4(t, filepath.Join("testdata", "multiplesamplingrates-template.pcap")) - got = nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) + _, err = nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } data = helpers.ReadPcapL4(t, filepath.Join("testdata", "multiplesamplingrates-data.pcap")) - got = nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) + _, err = nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } expectedFlows := []*schema.FlowMessage{ { @@ -298,7 +352,7 @@ func TestDecodeMultipleSamplingRates(t *testing.T) { DstNetMask: 56, InIf: 97, OutIf: 6, - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnPackets: 18, schema.ColumnBytes: 1348, schema.ColumnProto: 6, @@ -322,7 +376,7 @@ func TestDecodeMultipleSamplingRates(t *testing.T) { DstNetMask: 48, InIf: 103, OutIf: 6, - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnPackets: 4, schema.ColumnBytes: 579, schema.ColumnProto: 17, @@ -337,30 +391,36 @@ func TestDecodeMultipleSamplingRates(t *testing.T) { }, } - for _, f := range got { - f.TimeReceived = 0 - } - - if diff := helpers.Diff(got[:2], expectedFlows); diff != "" { + if diff := helpers.Diff((*got)[:2], expectedFlows); diff != "" { t.Fatalf("Decode() (-got, +want):\n%s", diff) } } func TestDecodeICMP(t *testing.T) { - r := reporter.NewMock(t) - nfdecoder := New(r, decoder.Dependencies{Schema: schema.NewMock(t).EnableAllColumns()}, decoder.Option{TimestampSource: decoder.TimestampSourceUDP}) + _, nfdecoder, bf, got, finalize := setup(t, true) + options := decoder.Option{TimestampSource: pb.RawFlow_TS_INPUT} data := helpers.ReadPcapL4(t, filepath.Join("testdata", "icmp-template.pcap")) - got := nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) + _, err := nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } data = helpers.ReadPcapL4(t, filepath.Join("testdata", "icmp-data.pcap")) - got = append(got, nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")})...) + _, err = nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } expectedFlows := []*schema.FlowMessage{ { ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), SrcAddr: netip.MustParseAddr("2001:db8::"), DstAddr: netip.MustParseAddr("2001:db8::1"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnBytes: 104, schema.ColumnDstPort: 32768, schema.ColumnEType: 34525, @@ -373,7 +433,7 @@ func TestDecodeICMP(t *testing.T) { ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), SrcAddr: netip.MustParseAddr("2001:db8::1"), DstAddr: netip.MustParseAddr("2001:db8::"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnBytes: 104, schema.ColumnDstPort: 33024, schema.ColumnEType: 34525, @@ -386,7 +446,7 @@ func TestDecodeICMP(t *testing.T) { ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), SrcAddr: netip.MustParseAddr("::ffff:203.0.113.4"), DstAddr: netip.MustParseAddr("::ffff:203.0.113.5"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnBytes: 84, schema.ColumnDstPort: 2048, schema.ColumnEType: 2048, @@ -399,7 +459,7 @@ func TestDecodeICMP(t *testing.T) { ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), SrcAddr: netip.MustParseAddr("::ffff:203.0.113.5"), DstAddr: netip.MustParseAddr("::ffff:203.0.113.4"), - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnBytes: 84, schema.ColumnEType: 2048, schema.ColumnPackets: 1, @@ -408,9 +468,6 @@ func TestDecodeICMP(t *testing.T) { }, }, } - for _, f := range got { - f.TimeReceived = 0 - } if diff := helpers.Diff(got, expectedFlows); diff != "" { t.Fatalf("Decode() (-got, +want):\n%s", diff) @@ -419,13 +476,23 @@ func TestDecodeICMP(t *testing.T) { } func TestDecodeDataLink(t *testing.T) { - r := reporter.NewMock(t) - nfdecoder := New(r, decoder.Dependencies{Schema: schema.NewMock(t).EnableAllColumns()}, decoder.Option{TimestampSource: decoder.TimestampSourceUDP}) + _, nfdecoder, bf, got, finalize := setup(t, true) + options := decoder.Option{TimestampSource: pb.RawFlow_TS_INPUT} data := helpers.ReadPcapL4(t, filepath.Join("testdata", "datalink-template.pcap")) - got := nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) + _, err := nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } data = helpers.ReadPcapL4(t, filepath.Join("testdata", "datalink-data.pcap")) - got = append(got, nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")})...) + _, err = nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } expectedFlows := []*schema.FlowMessage{ { @@ -435,7 +502,7 @@ func TestDecodeDataLink(t *testing.T) { SrcVlan: 231, InIf: 582, OutIf: 0, - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnBytes: 96, schema.ColumnSrcPort: 55501, schema.ColumnDstPort: 11777, @@ -449,9 +516,6 @@ func TestDecodeDataLink(t *testing.T) { }, }, } - for _, f := range got { - f.TimeReceived = 0 - } if diff := helpers.Diff(got, expectedFlows); diff != "" { t.Fatalf("Decode() (-got, +want):\n%s", diff) @@ -459,12 +523,16 @@ func TestDecodeDataLink(t *testing.T) { } func TestDecodeWithoutTemplate(t *testing.T) { - r := reporter.NewMock(t) - nfdecoder := New(r, - decoder.Dependencies{Schema: schema.NewMock(t).EnableAllColumns()}, - decoder.Option{TimestampSource: decoder.TimestampSourceUDP}) + _, nfdecoder, bf, got, finalize := setup(t, true) + options := decoder.Option{TimestampSource: pb.RawFlow_TS_INPUT} + data := helpers.ReadPcapL4(t, filepath.Join("testdata", "datalink-data.pcap")) - got := nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) + _, err := nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } expectedFlows := []*schema.FlowMessage{} if diff := helpers.Diff(got, expectedFlows); diff != "" { @@ -473,11 +541,16 @@ func TestDecodeWithoutTemplate(t *testing.T) { } func TestDecodeMPLS(t *testing.T) { - r := reporter.NewMock(t) - nfdecoder := New(r, decoder.Dependencies{Schema: schema.NewMock(t).EnableAllColumns()}, decoder.Option{TimestampSource: decoder.TimestampSourceUDP}) + _, nfdecoder, bf, got, finalize := setup(t, true) + options := decoder.Option{TimestampSource: pb.RawFlow_TS_INPUT} data := helpers.ReadPcapL4(t, filepath.Join("testdata", "mpls.pcap")) - got := nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) + _, err := nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } expectedFlows := []*schema.FlowMessage{ { @@ -487,7 +560,7 @@ func TestDecodeMPLS(t *testing.T) { NextHop: netip.MustParseAddr("::ffff:0.0.0.0"), SamplingRate: 10, OutIf: 16, - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnBytes: 89, schema.ColumnPackets: 1, schema.ColumnEType: helpers.ETypeIPv6, @@ -505,7 +578,7 @@ func TestDecodeMPLS(t *testing.T) { NextHop: netip.MustParseAddr("::ffff:0.0.0.0"), SamplingRate: 10, OutIf: 17, - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnBytes: 890, schema.ColumnPackets: 10, schema.ColumnEType: helpers.ETypeIPv6, @@ -518,9 +591,6 @@ func TestDecodeMPLS(t *testing.T) { }, }, } - for _, f := range got { - f.TimeReceived = 0 - } if diff := helpers.Diff(got, expectedFlows); diff != "" { t.Fatalf("Decode() (-got, +want):\n%s", diff) @@ -528,21 +598,24 @@ func TestDecodeMPLS(t *testing.T) { } func TestDecodeNFv5(t *testing.T) { - for _, tsSource := range []decoder.TimestampSource{ - decoder.TimestampSourceNetflowPacket, - decoder.TimestampSourceNetflowFirstSwitched, + for _, tsSource := range []pb.RawFlow_TimestampSource{ + pb.RawFlow_TS_NETFLOW_PACKET, + pb.RawFlow_TS_NETFLOW_FIRST_SWITCHED, } { t.Run(fmt.Sprintf("%s", tsSource), func(t *testing.T) { - r := reporter.NewMock(t) - nfdecoder := New(r, - decoder.Dependencies{Schema: schema.NewMock(t).EnableAllColumns()}, - decoder.Option{TimestampSource: tsSource}) + _, nfdecoder, bf, got, finalize := setup(t, false) + options := decoder.Option{TimestampSource: tsSource} data := helpers.ReadPcapL4(t, filepath.Join("testdata", "nfv5.pcap")) - got := nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) + _, err := nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } - ts := uint64(1680626679) - if tsSource == decoder.TimestampSourceNetflowFirstSwitched { + ts := uint32(1680626679) + if tsSource == pb.RawFlow_TS_NETFLOW_FIRST_SWITCHED { ts = 1680611679 } @@ -560,7 +633,7 @@ func TestDecodeNFv5(t *testing.T) { DstAS: 10101, SrcNetMask: 19, DstNetMask: 24, - ProtobufDebug: map[schema.ColumnKey]interface{}{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnBytes: 133, schema.ColumnPackets: 1, schema.ColumnEType: helpers.ETypeIPv4, @@ -572,7 +645,7 @@ func TestDecodeNFv5(t *testing.T) { }, } - if diff := helpers.Diff(got[:1], expectedFlows); diff != "" { + if diff := helpers.Diff((*got)[:1], expectedFlows); diff != "" { t.Errorf("Decode() (-got, +want):\n%s", diff) } }) @@ -580,24 +653,34 @@ func TestDecodeNFv5(t *testing.T) { } func TestDecodeTimestampFromNetflowPacket(t *testing.T) { - r := reporter.NewMock(t) - nfdecoder := New(r, decoder.Dependencies{Schema: schema.NewMock(t).EnableAllColumns()}, decoder.Option{TimestampSource: decoder.TimestampSourceNetflowPacket}) + _, nfdecoder, bf, got, finalize := setup(t, false) + options := decoder.Option{TimestampSource: pb.RawFlow_TS_NETFLOW_PACKET} data := helpers.ReadPcapL4(t, filepath.Join("testdata", "template.pcap")) - got := nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) + _, err := nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } data = helpers.ReadPcapL4(t, filepath.Join("testdata", "data.pcap")) - got = append(got, nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")})...) + _, err = nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } // 4 flows in capture // all share the same timestamp with TimestampSourceNetflowPacket - expectedTs := []uint64{ + expectedTs := []uint32{ 1647285928, 1647285928, 1647285928, 1647285928, } - for i, flow := range got { + for i, flow := range *got { if flow.TimeReceived != expectedTs[i] { t.Errorf("Decode() (-got, +want):\n-%d, +%d", flow.TimeReceived, expectedTs[i]) } @@ -605,25 +688,35 @@ func TestDecodeTimestampFromNetflowPacket(t *testing.T) { } func TestDecodeTimestampFromFirstSwitched(t *testing.T) { - r := reporter.NewMock(t) - nfdecoder := New(r, decoder.Dependencies{Schema: schema.NewMock(t).EnableAllColumns()}, decoder.Option{TimestampSource: decoder.TimestampSourceNetflowFirstSwitched}) + _, nfdecoder, bf, got, finalize := setup(t, false) + options := decoder.Option{TimestampSource: pb.RawFlow_TS_NETFLOW_FIRST_SWITCHED} data := helpers.ReadPcapL4(t, filepath.Join("testdata", "template.pcap")) - got := nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) + _, err := nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } data = helpers.ReadPcapL4(t, filepath.Join("testdata", "data.pcap")) - got = append(got, nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")})...) + _, err = nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } // 4 flows in capture - var sysUptime uint64 = 944951609 - var packetTs uint64 = 1647285928 - expectedFirstSwitched := []uint64{ + var sysUptime uint32 = 944951609 + var packetTs uint32 = 1647285928 + expectedFirstSwitched := []uint32{ 944948659, 944948659, 944948660, 944948661, } - for i, flow := range got { + for i, flow := range *got { if val := packetTs - sysUptime + expectedFirstSwitched[i]; flow.TimeReceived != val { t.Errorf("Decode() (-got, +want):\n-%d, +%d", flow.TimeReceived, val) } @@ -631,20 +724,25 @@ func TestDecodeTimestampFromFirstSwitched(t *testing.T) { } func TestDecodeNAT(t *testing.T) { - r := reporter.NewMock(t) - nfdecoder := New(r, decoder.Dependencies{Schema: schema.NewMock(t).EnableAllColumns()}, decoder.Option{TimestampSource: decoder.TimestampSourceUDP}) + _, nfdecoder, bf, got, finalize := setup(t, true) + options := decoder.Option{TimestampSource: pb.RawFlow_TS_INPUT} // The following PCAP is a NAT event, there is no sampling rate, no bytes, // no packets. We can't do much with it. data := helpers.ReadPcapL4(t, filepath.Join("testdata", "nat.pcap")) - got := nfdecoder.Decode(decoder.RawFlow{Payload: data, Source: net.ParseIP("127.0.0.1")}) + _, err := nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } expectedFlows := []*schema.FlowMessage{ { ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), SrcAddr: netip.MustParseAddr("::ffff:172.16.100.198"), DstAddr: netip.MustParseAddr("::ffff:10.89.87.1"), - ProtobufDebug: map[schema.ColumnKey]any{ + OtherColumns: map[schema.ColumnKey]any{ schema.ColumnSrcPort: 35303, schema.ColumnDstPort: 53, schema.ColumnSrcAddrNAT: netip.MustParseAddr("::ffff:10.143.52.29"), @@ -656,11 +754,8 @@ func TestDecodeNAT(t *testing.T) { }, }, } - for _, f := range got { - f.TimeReceived = 0 - } - if diff := helpers.Diff(got[:1], expectedFlows); diff != "" { + if diff := helpers.Diff((*got)[:1], expectedFlows); diff != "" { t.Fatalf("Decode() (-got, +want):\n%s", diff) } } diff --git a/inlet/flow/decoder/netflow/testdata/data+templates.pcap b/outlet/flow/decoder/netflow/testdata/data+templates.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/data+templates.pcap rename to outlet/flow/decoder/netflow/testdata/data+templates.pcap diff --git a/inlet/flow/decoder/netflow/testdata/data.pcap b/outlet/flow/decoder/netflow/testdata/data.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/data.pcap rename to outlet/flow/decoder/netflow/testdata/data.pcap diff --git a/inlet/flow/decoder/netflow/testdata/datalink-data.pcap b/outlet/flow/decoder/netflow/testdata/datalink-data.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/datalink-data.pcap rename to outlet/flow/decoder/netflow/testdata/datalink-data.pcap diff --git a/inlet/flow/decoder/netflow/testdata/datalink-template.pcap b/outlet/flow/decoder/netflow/testdata/datalink-template.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/datalink-template.pcap rename to outlet/flow/decoder/netflow/testdata/datalink-template.pcap diff --git a/inlet/flow/decoder/netflow/testdata/icmp-data.pcap b/outlet/flow/decoder/netflow/testdata/icmp-data.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/icmp-data.pcap rename to outlet/flow/decoder/netflow/testdata/icmp-data.pcap diff --git a/inlet/flow/decoder/netflow/testdata/icmp-template.pcap b/outlet/flow/decoder/netflow/testdata/icmp-template.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/icmp-template.pcap rename to outlet/flow/decoder/netflow/testdata/icmp-template.pcap diff --git a/inlet/flow/decoder/netflow/testdata/mpls.pcap b/outlet/flow/decoder/netflow/testdata/mpls.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/mpls.pcap rename to outlet/flow/decoder/netflow/testdata/mpls.pcap diff --git a/inlet/flow/decoder/netflow/testdata/multiplesamplingrates-data.pcap b/outlet/flow/decoder/netflow/testdata/multiplesamplingrates-data.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/multiplesamplingrates-data.pcap rename to outlet/flow/decoder/netflow/testdata/multiplesamplingrates-data.pcap diff --git a/inlet/flow/decoder/netflow/testdata/multiplesamplingrates-options-data.pcap b/outlet/flow/decoder/netflow/testdata/multiplesamplingrates-options-data.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/multiplesamplingrates-options-data.pcap rename to outlet/flow/decoder/netflow/testdata/multiplesamplingrates-options-data.pcap diff --git a/inlet/flow/decoder/netflow/testdata/multiplesamplingrates-options-template.pcap b/outlet/flow/decoder/netflow/testdata/multiplesamplingrates-options-template.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/multiplesamplingrates-options-template.pcap rename to outlet/flow/decoder/netflow/testdata/multiplesamplingrates-options-template.pcap diff --git a/inlet/flow/decoder/netflow/testdata/multiplesamplingrates-template.pcap b/outlet/flow/decoder/netflow/testdata/multiplesamplingrates-template.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/multiplesamplingrates-template.pcap rename to outlet/flow/decoder/netflow/testdata/multiplesamplingrates-template.pcap diff --git a/inlet/flow/decoder/netflow/testdata/nat.pcap b/outlet/flow/decoder/netflow/testdata/nat.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/nat.pcap rename to outlet/flow/decoder/netflow/testdata/nat.pcap diff --git a/inlet/flow/decoder/netflow/testdata/nfv5.pcap b/outlet/flow/decoder/netflow/testdata/nfv5.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/nfv5.pcap rename to outlet/flow/decoder/netflow/testdata/nfv5.pcap diff --git a/inlet/flow/decoder/netflow/testdata/options-data.pcap b/outlet/flow/decoder/netflow/testdata/options-data.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/options-data.pcap rename to outlet/flow/decoder/netflow/testdata/options-data.pcap diff --git a/inlet/flow/decoder/netflow/testdata/options-template.pcap b/outlet/flow/decoder/netflow/testdata/options-template.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/options-template.pcap rename to outlet/flow/decoder/netflow/testdata/options-template.pcap diff --git a/inlet/flow/decoder/netflow/testdata/samplingrate-data.pcap b/outlet/flow/decoder/netflow/testdata/samplingrate-data.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/samplingrate-data.pcap rename to outlet/flow/decoder/netflow/testdata/samplingrate-data.pcap diff --git a/inlet/flow/decoder/netflow/testdata/samplingrate-template.pcap b/outlet/flow/decoder/netflow/testdata/samplingrate-template.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/samplingrate-template.pcap rename to outlet/flow/decoder/netflow/testdata/samplingrate-template.pcap diff --git a/inlet/flow/decoder/netflow/testdata/template.pcap b/outlet/flow/decoder/netflow/testdata/template.pcap similarity index 100% rename from inlet/flow/decoder/netflow/testdata/template.pcap rename to outlet/flow/decoder/netflow/testdata/template.pcap diff --git a/inlet/flow/decoder/root.go b/outlet/flow/decoder/root.go similarity index 54% rename from inlet/flow/decoder/root.go rename to outlet/flow/decoder/root.go index 3d62d902..a4d4db5c 100644 --- a/inlet/flow/decoder/root.go +++ b/outlet/flow/decoder/root.go @@ -6,19 +6,21 @@ package decoder import ( - "net" + "net/netip" "time" + "akvorado/common/pb" "akvorado/common/reporter" "akvorado/common/schema" ) // Decoder is the interface each decoder should implement. type Decoder interface { - // Decoder takes a raw flow and returns a - // slice of flow messages. Returning nil means there was an - // error during decoding. - Decode(in RawFlow) []*schema.FlowMessage + // Decoder takes a raw flow and options. It should enqueue new flows in the + // provided flow message. When a flow is enqueted, it will call the finalize + // function. It is important to not set an error once the flow is being + // built (as there is no rollback possible). + Decode(in RawFlow, options Option, bf *schema.FlowMessage, finalize FinalizeFlowFunc) (int, error) // Name returns the decoder name Name() string @@ -27,7 +29,7 @@ type Decoder interface { // Option specifies option to influence the behaviour of the decoder type Option struct { // TimestampSource is a selector for how to set the TimeReceived. - TimestampSource TimestampSource + TimestampSource pb.RawFlow_TimestampSource } // Dependencies are the dependencies for the decoder @@ -39,8 +41,12 @@ type Dependencies struct { type RawFlow struct { TimeReceived time.Time Payload []byte - Source net.IP + Source netip.Addr } // NewDecoderFunc is the signature of a function to instantiate a decoder. -type NewDecoderFunc func(*reporter.Reporter, Dependencies, Option) Decoder +type NewDecoderFunc func(*reporter.Reporter, Dependencies) Decoder + +// FinalizeFlowFunc is the signature of a function to finalize a flow. The +// caller has a reference to the flow message he provided. +type FinalizeFlowFunc func() diff --git a/inlet/flow/decoder/sflow/decode.go b/outlet/flow/decoder/sflow/decode.go similarity index 64% rename from inlet/flow/decoder/sflow/decode.go rename to outlet/flow/decoder/sflow/decode.go index e1d91324..812cf2f3 100644 --- a/inlet/flow/decoder/sflow/decode.go +++ b/outlet/flow/decoder/sflow/decode.go @@ -5,35 +5,35 @@ package sflow import ( + "net" + "akvorado/common/helpers" "akvorado/common/schema" - "akvorado/inlet/flow/decoder" + "akvorado/outlet/flow/decoder" "github.com/netsampler/goflow2/v2/decoders/sflow" ) -func (nd *Decoder) decode(packet sflow.Packet) []*schema.FlowMessage { - flowMessageSet := []*schema.FlowMessage{} - +func (nd *Decoder) decode(packet sflow.Packet, bf *schema.FlowMessage, finalize decoder.FinalizeFlowFunc) error { for _, flowSample := range packet.Samples { var records []sflow.FlowRecord - bf := &schema.FlowMessage{} forwardingStatus := 0 switch flowSample := flowSample.(type) { case sflow.FlowSample: records = flowSample.Records - bf.SamplingRate = flowSample.SamplingRate + bf.SamplingRate = uint64(flowSample.SamplingRate) bf.InIf = flowSample.Input bf.OutIf = flowSample.Output - if bf.OutIf&interfaceOutMask == interfaceOutDiscard { + switch bf.OutIf & interfaceOutMask { + case interfaceOutDiscard: bf.OutIf = 0 forwardingStatus = 128 - } else if bf.OutIf&interfaceOutMask == interfaceOutMultiple { + case interfaceOutMultiple: bf.OutIf = 0 } case sflow.ExpandedFlowSample: records = flowSample.Records - bf.SamplingRate = flowSample.SamplingRate + bf.SamplingRate = uint64(flowSample.SamplingRate) bf.InIf = flowSample.InputIfValue bf.OutIf = flowSample.OutputIfValue } @@ -46,8 +46,8 @@ func (nd *Decoder) decode(packet sflow.Packet) []*schema.FlowMessage { } bf.ExporterAddress = decoder.DecodeIP(packet.AgentIP) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnPackets, 1) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnForwardingStatus, uint64(forwardingStatus)) + bf.AppendUint(schema.ColumnPackets, 1) + bf.AppendUint(schema.ColumnForwardingStatus, uint64(forwardingStatus)) // Optimization: avoid parsing sampled header if we have everything already parsed hasSampledIPv4 := false @@ -84,20 +84,20 @@ func (nd *Decoder) decode(packet sflow.Packet) []*schema.FlowMessage { bf.SrcAddr = decoder.DecodeIP(recordData.SrcIP) bf.DstAddr = decoder.DecodeIP(recordData.DstIP) l3length = uint64(recordData.Length) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnProto, uint64(recordData.Protocol)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnSrcPort, uint64(recordData.SrcPort)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnDstPort, uint64(recordData.DstPort)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnEType, helpers.ETypeIPv4) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnIPTos, uint64(recordData.Tos)) + bf.AppendUint(schema.ColumnProto, uint64(recordData.Protocol)) + bf.AppendUint(schema.ColumnSrcPort, uint64(recordData.SrcPort)) + bf.AppendUint(schema.ColumnDstPort, uint64(recordData.DstPort)) + bf.AppendUint(schema.ColumnEType, helpers.ETypeIPv4) + bf.AppendUint(schema.ColumnIPTos, uint64(recordData.Tos)) case sflow.SampledIPv6: bf.SrcAddr = decoder.DecodeIP(recordData.SrcIP) bf.DstAddr = decoder.DecodeIP(recordData.DstIP) l3length = uint64(recordData.Length) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnProto, uint64(recordData.Protocol)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnSrcPort, uint64(recordData.SrcPort)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnDstPort, uint64(recordData.DstPort)) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnEType, helpers.ETypeIPv6) - nd.d.Schema.ProtobufAppendVarint(bf, schema.ColumnIPTos, uint64(recordData.Priority)) + bf.AppendUint(schema.ColumnProto, uint64(recordData.Protocol)) + bf.AppendUint(schema.ColumnSrcPort, uint64(recordData.SrcPort)) + bf.AppendUint(schema.ColumnDstPort, uint64(recordData.DstPort)) + bf.AppendUint(schema.ColumnEType, helpers.ETypeIPv6) + bf.AppendUint(schema.ColumnIPTos, uint64(recordData.Priority)) case sflow.SampledEthernet: if l3length == 0 { // That's the best we can guess. sFlow says: For a layer 2 @@ -107,8 +107,8 @@ func (nd *Decoder) decode(packet sflow.Packet) []*schema.FlowMessage { l3length = uint64(recordData.Length) - 16 } if !nd.d.Schema.IsDisabled(schema.ColumnGroupL2) { - nd.d.Schema.ProtobufAppendBytes(bf, schema.ColumnSrcMAC, recordData.SrcMac) - nd.d.Schema.ProtobufAppendBytes(bf, schema.ColumnDstMAC, recordData.DstMac) + bf.AppendUint(schema.ColumnSrcMAC, helpers.MACToUint64(net.HardwareAddr(recordData.SrcMac))) + bf.AppendUint(schema.ColumnDstMAC, helpers.MACToUint64(net.HardwareAddr(recordData.DstMac))) } case sflow.ExtendedSwitch: if !nd.d.Schema.IsDisabled(schema.ColumnGroupL2) { @@ -132,31 +132,22 @@ func (nd *Decoder) decode(packet sflow.Packet) []*schema.FlowMessage { } if len(recordData.ASPath) > 0 { bf.DstAS = recordData.ASPath[len(recordData.ASPath)-1] - if column, _ := nd.d.Schema.LookupColumnByKey(schema.ColumnDstASPath); !column.Disabled { - for _, asn := range recordData.ASPath { - column.ProtobufAppendVarint(bf, uint64(asn)) - } - } - bf.GotASPath = true + bf.AppendArrayUInt32(schema.ColumnDstASPath, recordData.ASPath) } if len(recordData.Communities) > 0 { - if column, _ := nd.d.Schema.LookupColumnByKey(schema.ColumnDstCommunities); !column.Disabled { - for _, comm := range recordData.Communities { - column.ProtobufAppendVarint(bf, uint64(comm)) - } - } - bf.GotCommunities = true + bf.AppendArrayUInt32(schema.ColumnDstCommunities, recordData.Communities) } } } if l3length > 0 { - nd.d.Schema.ProtobufAppendVarintForce(bf, schema.ColumnBytes, l3length) + bf.AppendUint(schema.ColumnBytes, l3length) } - flowMessageSet = append(flowMessageSet, bf) + + finalize() } - return flowMessageSet + return nil } func (nd *Decoder) parseSampledHeader(bf *schema.FlowMessage, header *sflow.SampledHeader) uint64 { diff --git a/inlet/flow/decoder/sflow/root.go b/outlet/flow/decoder/sflow/root.go similarity index 90% rename from inlet/flow/decoder/sflow/root.go rename to outlet/flow/decoder/sflow/root.go index 0798fc27..73782339 100644 --- a/inlet/flow/decoder/sflow/root.go +++ b/outlet/flow/decoder/sflow/root.go @@ -6,6 +6,7 @@ package sflow import ( "bytes" + "fmt" "net" "time" @@ -13,7 +14,7 @@ import ( "akvorado/common/reporter" "akvorado/common/schema" - "akvorado/inlet/flow/decoder" + "akvorado/outlet/flow/decoder" ) const ( @@ -43,7 +44,7 @@ type Decoder struct { } // New instantiates a new sFlow decoder. -func New(r *reporter.Reporter, dependencies decoder.Dependencies, _ decoder.Option) decoder.Decoder { +func New(r *reporter.Reporter, dependencies decoder.Dependencies) decoder.Decoder { nd := &Decoder{ r: r, d: dependencies, @@ -83,16 +84,16 @@ func New(r *reporter.Reporter, dependencies decoder.Dependencies, _ decoder.Opti } // Decode decodes an sFlow payload. -func (nd *Decoder) Decode(in decoder.RawFlow) []*schema.FlowMessage { +func (nd *Decoder) Decode(in decoder.RawFlow, _ decoder.Option, bf *schema.FlowMessage, finalize decoder.FinalizeFlowFunc) (int, error) { buf := bytes.NewBuffer(in.Payload) key := in.Source.String() - ts := uint64(in.TimeReceived.UTC().Unix()) + var packet sflow.Packet if err := sflow.DecodeMessageVersion(buf, &packet); err != nil { nd.metrics.errors.WithLabelValues(key, "sFlow decoding error").Inc() nd.errLogger.Err(err).Str("exporter", key).Msg("error while decoding sFlow") - return nil + return 0, fmt.Errorf("error while decoding sFlow: %w", err) } // Update some stats @@ -120,12 +121,10 @@ func (nd *Decoder) Decode(in decoder.RawFlow) []*schema.FlowMessage { } } - flowMessageSet := nd.decode(packet) - for _, fmsg := range flowMessageSet { - fmsg.TimeReceived = ts - } - - return flowMessageSet + return len(samples), nd.decode(packet, bf, func() { + bf.TimeReceived = uint32(ts) + finalize() + }) } // Name returns the name of the decoder. diff --git a/outlet/flow/decoder/sflow/root_test.go b/outlet/flow/decoder/sflow/root_test.go new file mode 100644 index 00000000..c0695e2f --- /dev/null +++ b/outlet/flow/decoder/sflow/root_test.go @@ -0,0 +1,554 @@ +// SPDX-FileCopyrightText: 2022 Tchadel Icard +// SPDX-License-Identifier: AGPL-3.0-only + +package sflow + +import ( + "net/netip" + "path/filepath" + "testing" + + "akvorado/common/helpers" + "akvorado/common/reporter" + "akvorado/common/schema" + "akvorado/outlet/flow/decoder" +) + +func TestDecode(t *testing.T) { + r := reporter.NewMock(t) + sch := schema.NewMock(t).EnableAllColumns() + sdecoder := New(r, decoder.Dependencies{Schema: sch}) + options := decoder.Option{} + bf := sch.NewFlowMessage() + got := []*schema.FlowMessage{} + finalize := func() { + bf.TimeReceived = 0 + // Keep a copy of the current flow message + clone := *bf + got = append(got, &clone) + // And clear the flow message + bf.Clear() + } + + // Send data + t.Run("basic", func(t *testing.T) { + got = got[:0] + data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-1140.pcap")) + _, err := sdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } + expectedFlows := []*schema.FlowMessage{ + { + SamplingRate: 1024, + InIf: 27, + OutIf: 28, + SrcVlan: 100, + DstVlan: 100, + SrcAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:38"), + DstAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:39"), + ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 1500, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv6, + schema.ColumnProto: 6, + schema.ColumnSrcPort: 46026, + schema.ColumnDstPort: 22, + schema.ColumnSrcMAC: 40057391053392, + schema.ColumnDstMAC: 40057381862408, + schema.ColumnIPTTL: 64, + schema.ColumnIPTos: 0x8, + schema.ColumnIPv6FlowLabel: 0x68094, + schema.ColumnTCPFlags: 0x10, + }, + }, { + SamplingRate: 1024, + SrcAddr: netip.MustParseAddr("::ffff:104.26.8.24"), + DstAddr: netip.MustParseAddr("::ffff:45.90.161.46"), + ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), + NextHop: netip.MustParseAddr("::ffff:45.90.161.46"), + InIf: 49001, + OutIf: 25, + DstVlan: 100, + SrcAS: 13335, + DstAS: 39421, + SrcNetMask: 20, + DstNetMask: 27, + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 421, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv4, + schema.ColumnProto: 6, + schema.ColumnSrcPort: 443, + schema.ColumnDstPort: 56876, + schema.ColumnSrcMAC: 216372595274807, + schema.ColumnDstMAC: 191421060163210, + schema.ColumnIPFragmentID: 0xa572, + schema.ColumnIPTTL: 59, + schema.ColumnTCPFlags: 0x18, + }, + }, { + SamplingRate: 1024, + SrcAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:38"), + DstAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:39"), + ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), + InIf: 27, + OutIf: 28, + SrcVlan: 100, + DstVlan: 100, + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 1500, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv6, + schema.ColumnProto: 6, + schema.ColumnSrcPort: 46026, + schema.ColumnDstPort: 22, + schema.ColumnSrcMAC: 40057391053392, + schema.ColumnDstMAC: 40057381862408, + schema.ColumnIPTTL: 64, + schema.ColumnIPTos: 0x8, + schema.ColumnIPv6FlowLabel: 0x68094, + schema.ColumnTCPFlags: 0x10, + }, + }, { + SamplingRate: 1024, + InIf: 28, + OutIf: 49001, + SrcVlan: 100, + SrcAS: 39421, + DstAS: 26615, + SrcAddr: netip.MustParseAddr("::ffff:45.90.161.148"), + DstAddr: netip.MustParseAddr("::ffff:191.87.91.27"), + ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), + NextHop: netip.MustParseAddr("::ffff:31.14.69.110"), + SrcNetMask: 27, + DstNetMask: 17, + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 40, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv4, + schema.ColumnProto: 6, + schema.ColumnSrcPort: 55658, + schema.ColumnDstPort: 5555, + schema.ColumnSrcMAC: 138617863011056, + schema.ColumnDstMAC: 216372595274807, + schema.ColumnDstASPath: []uint32{203698, 6762, 26615}, + schema.ColumnDstCommunities: []uint64{2583495656, 2583495657, 4259880000, 4259880001, 4259900001}, + schema.ColumnIPFragmentID: 0xd431, + schema.ColumnIPTTL: 255, + schema.ColumnTCPFlags: 0x2, + }, + }, { + SamplingRate: 1024, + SrcAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:38"), + DstAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:39"), + ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), + InIf: 27, + OutIf: 28, + SrcVlan: 100, + DstVlan: 100, + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 1500, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv6, + schema.ColumnProto: 6, + schema.ColumnSrcPort: 46026, + schema.ColumnDstPort: 22, + schema.ColumnSrcMAC: 40057391053392, + schema.ColumnDstMAC: 40057381862408, + schema.ColumnIPTTL: 64, + schema.ColumnIPTos: 0x8, + schema.ColumnIPv6FlowLabel: 0x68094, + schema.ColumnTCPFlags: 0x10, + }, + }, + } + + if diff := helpers.Diff(got, expectedFlows); diff != "" { + t.Fatalf("Decode() (-got, +want):\n%s", diff) + } + gotMetrics := r.GetMetrics( + "akvorado_outlet_flow_decoder_sflow_", + "flows_total", + "sample_", + ) + expectedMetrics := map[string]string{ + `flows_total{agent="172.16.0.3",exporter="::ffff:127.0.0.1",version="5"}`: "1", + `sample_records_sum{agent="172.16.0.3",exporter="::ffff:127.0.0.1",type="FlowSample",version="5"}`: "14", + `sample_sum{agent="172.16.0.3",exporter="::ffff:127.0.0.1",type="FlowSample",version="5"}`: "5", + } + if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { + t.Fatalf("Metrics after data (-got, +want):\n%s", diff) + } + }) + + t.Run("local interface", func(t *testing.T) { + got = got[:0] + data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-local-interface.pcap")) + _, err := sdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } + expectedFlows := []*schema.FlowMessage{ + { + SamplingRate: 1024, + SrcAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:38"), + DstAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:39"), + ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), + InIf: 27, + OutIf: 0, // local interface + SrcVlan: 100, + DstVlan: 100, + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 1500, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv6, + schema.ColumnProto: 6, + schema.ColumnSrcPort: 46026, + schema.ColumnDstPort: 22, + schema.ColumnSrcMAC: 40057391053392, + schema.ColumnDstMAC: 40057381862408, + schema.ColumnTCPFlags: 16, + schema.ColumnIPv6FlowLabel: 426132, + schema.ColumnIPTTL: 64, + schema.ColumnIPTos: 8, + }, + }, + } + + if diff := helpers.Diff(got, expectedFlows); diff != "" { + t.Fatalf("Decode() (-got, +want):\n%s", diff) + } + }) + + t.Run("discard interface", func(t *testing.T) { + got = got[:0] + data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-discard-interface.pcap")) + _, err := sdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } + expectedFlows := []*schema.FlowMessage{ + { + SamplingRate: 1024, + SrcAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:38"), + DstAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:39"), + ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), + InIf: 27, + OutIf: 0, // discard interface + SrcVlan: 100, + DstVlan: 100, + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 1500, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv6, + schema.ColumnProto: 6, + schema.ColumnSrcPort: 46026, + schema.ColumnDstPort: 22, + schema.ColumnForwardingStatus: 128, + schema.ColumnSrcMAC: 40057391053392, + schema.ColumnDstMAC: 40057381862408, + schema.ColumnTCPFlags: 16, + schema.ColumnIPv6FlowLabel: 426132, + schema.ColumnIPTTL: 64, + schema.ColumnIPTos: 8, + }, + }, + } + + if diff := helpers.Diff(got, expectedFlows); diff != "" { + t.Fatalf("Decode() (-got, +want):\n%s", diff) + } + }) + + t.Run("multiple interfaces", func(t *testing.T) { + got = got[:0] + data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-multiple-interfaces.pcap")) + _, err := sdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } + expectedFlows := []*schema.FlowMessage{ + { + SamplingRate: 1024, + SrcAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:38"), + DstAddr: netip.MustParseAddr("2a0c:8880:2:0:185:21:130:39"), + ExporterAddress: netip.MustParseAddr("::ffff:172.16.0.3"), + InIf: 27, + OutIf: 0, // multiple interfaces + SrcVlan: 100, + DstVlan: 100, + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 1500, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv6, + schema.ColumnProto: 6, + schema.ColumnSrcPort: 46026, + schema.ColumnDstPort: 22, + schema.ColumnSrcMAC: 40057391053392, + schema.ColumnDstMAC: 40057381862408, + schema.ColumnTCPFlags: 16, + schema.ColumnIPv6FlowLabel: 426132, + schema.ColumnIPTTL: 64, + schema.ColumnIPTos: 8, + }, + }, + } + + if diff := helpers.Diff(got, expectedFlows); diff != "" { + t.Fatalf("Decode() (-got, +want):\n%s", diff) + } + }) + + t.Run("expanded flow sample", func(t *testing.T) { + got = got[:0] + data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-sflow-expanded-sample.pcap")) + _, err := sdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } + expectedFlows := []*schema.FlowMessage{ + { + SamplingRate: 1000, + InIf: 29001, + OutIf: 1285816721, + SrcAddr: netip.MustParseAddr("::ffff:52.52.52.52"), + DstAddr: netip.MustParseAddr("::ffff:53.53.53.53"), + ExporterAddress: netip.MustParseAddr("::ffff:49.49.49.49"), + NextHop: netip.MustParseAddr("::ffff:54.54.54.54"), + SrcAS: 203476, + DstAS: 203361, + SrcVlan: 809, + SrcNetMask: 32, + DstNetMask: 22, + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 104, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv4, + schema.ColumnProto: 6, + schema.ColumnSrcPort: 22, + schema.ColumnDstPort: 52237, + schema.ColumnDstASPath: []uint32{8218, 29605, 203361}, + schema.ColumnDstCommunities: []uint64{538574949, 1911619684, 1911669584, 1911671290}, + schema.ColumnTCPFlags: 0x18, + schema.ColumnIPFragmentID: 0xab4e, + schema.ColumnIPTTL: 61, + schema.ColumnIPTos: 0x8, + schema.ColumnSrcMAC: 0x948ed30a713b, + schema.ColumnDstMAC: 0x22421f4a9fcd, + }, + }, + } + + if diff := helpers.Diff(got, expectedFlows); diff != "" { + t.Fatalf("Decode() (-got, +want):\n%s", diff) + } + }) + + t.Run("flow sample with IPv4 data", func(t *testing.T) { + got = got[:0] + data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-sflow-ipv4-data.pcap")) + _, err := sdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } + expectedFlows := []*schema.FlowMessage{ + { + SamplingRate: 256, + InIf: 0, + OutIf: 182, + DstVlan: 3001, + SrcAddr: netip.MustParseAddr("::ffff:50.50.50.50"), + DstAddr: netip.MustParseAddr("::ffff:51.51.51.51"), + ExporterAddress: netip.MustParseAddr("::ffff:49.49.49.49"), + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 1344, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv4, + schema.ColumnProto: 17, + schema.ColumnSrcPort: 46622, + schema.ColumnDstPort: 58631, + schema.ColumnSrcMAC: 1094287164743, + schema.ColumnDstMAC: 1101091482116, + schema.ColumnIPFragmentID: 41647, + schema.ColumnIPTTL: 64, + }, + }, + } + + if diff := helpers.Diff(got, expectedFlows); diff != "" { + t.Fatalf("Decode() (-got, +want):\n%s", diff) + } + }) + + t.Run("flow sample with IPv4 raw packet", func(t *testing.T) { + got = got[:0] + data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-sflow-raw-ipv4.pcap")) + _, err := sdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } + expectedFlows := []*schema.FlowMessage{ + { + SamplingRate: 1, + InIf: 0, + OutIf: 2, + SrcAddr: netip.MustParseAddr("::ffff:69.58.92.107"), + DstAddr: netip.MustParseAddr("::ffff:92.222.186.1"), + ExporterAddress: netip.MustParseAddr("::ffff:172.19.64.116"), + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 32, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv4, + schema.ColumnProto: 1, + schema.ColumnIPFragmentID: 4329, + schema.ColumnIPTTL: 64, + schema.ColumnIPTos: 8, + }, + }, { + SamplingRate: 1, + InIf: 0, + OutIf: 2, + SrcAddr: netip.MustParseAddr("::ffff:69.58.92.107"), + DstAddr: netip.MustParseAddr("::ffff:92.222.184.1"), + ExporterAddress: netip.MustParseAddr("::ffff:172.19.64.116"), + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 32, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv4, + schema.ColumnProto: 1, + schema.ColumnIPFragmentID: 62945, + schema.ColumnIPTTL: 64, + schema.ColumnIPTos: 8, + }, + }, + } + + if diff := helpers.Diff(got, expectedFlows); diff != "" { + t.Fatalf("Decode() (-got, +want):\n%s", diff) + } + }) + + t.Run("flow sample with ICMPv4", func(t *testing.T) { + got = got[:0] + data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-icmpv4.pcap")) + _, err := sdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } + expectedFlows := []*schema.FlowMessage{ + { + SamplingRate: 1, + SrcAddr: netip.MustParseAddr("::ffff:203.0.113.4"), + DstAddr: netip.MustParseAddr("::ffff:203.0.113.5"), + ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 84, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv4, + schema.ColumnProto: 1, + schema.ColumnDstMAC: 0xd25b45ee5ecf, + schema.ColumnSrcMAC: 0xe2efc68f8cd4, + schema.ColumnICMPv4Type: 8, + // schema.ColumnICMPv4Code: 0, + schema.ColumnIPTTL: 64, + schema.ColumnIPFragmentID: 0x90c5, + }, + }, + } + + if diff := helpers.Diff(got, expectedFlows); diff != "" { + t.Fatalf("Decode() (-got, +want):\n%s", diff) + } + }) + + t.Run("flow sample with ICMPv6", func(t *testing.T) { + got = got[:0] + data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-icmpv6.pcap")) + _, err := sdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } + expectedFlows := []*schema.FlowMessage{ + { + SamplingRate: 1, + SrcAddr: netip.MustParseAddr("fe80::d05b:45ff:feee:5ecf"), + DstAddr: netip.MustParseAddr("2001:db8::"), + ExporterAddress: netip.MustParseAddr("::ffff:127.0.0.1"), + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 72, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv6, + schema.ColumnProto: 58, + schema.ColumnSrcMAC: 0xd25b45ee5ecf, + schema.ColumnDstMAC: 0xe2efc68f8cd4, + schema.ColumnIPTTL: 255, + schema.ColumnICMPv6Type: 135, + // schema.ColumnICMPv6Code: 0, + }, + }, + } + + if diff := helpers.Diff(got, expectedFlows); diff != "" { + t.Fatalf("Decode() (-got, +want):\n%s", diff) + } + }) + + t.Run("flow sample with QinQ", func(t *testing.T) { + got = got[:0] + data := helpers.ReadPcapL4(t, filepath.Join("testdata", "data-qinq.pcap")) + _, err := sdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + t.Fatalf("Decode() error:\n%+v", err) + } + expectedFlows := []*schema.FlowMessage{ + { + SamplingRate: 4096, + InIf: 369098852, + OutIf: 369098851, + SrcVlan: 1493, + SrcAddr: netip.MustParseAddr("::ffff:49.49.49.2"), + DstAddr: netip.MustParseAddr("::ffff:49.49.49.109"), + ExporterAddress: netip.MustParseAddr("::ffff:172.17.128.58"), + OtherColumns: map[schema.ColumnKey]any{ + schema.ColumnBytes: 80, + schema.ColumnPackets: 1, + schema.ColumnEType: helpers.ETypeIPv4, + schema.ColumnProto: 6, + schema.ColumnSrcMAC: 0x4caea3520ff6, + schema.ColumnDstMAC: 0x000110621493, + schema.ColumnIPTTL: 62, + schema.ColumnIPFragmentID: 56159, + schema.ColumnTCPFlags: 16, + schema.ColumnSrcPort: 32017, + schema.ColumnDstPort: 443, + }, + }, + } + + if diff := helpers.Diff(got, expectedFlows); diff != "" { + t.Fatalf("Decode() (-got, +want):\n%s", diff) + } + }) +} diff --git a/inlet/flow/decoder/sflow/testdata/data-1140.pcap b/outlet/flow/decoder/sflow/testdata/data-1140.pcap similarity index 100% rename from inlet/flow/decoder/sflow/testdata/data-1140.pcap rename to outlet/flow/decoder/sflow/testdata/data-1140.pcap diff --git a/inlet/flow/decoder/sflow/testdata/data-discard-interface.pcap b/outlet/flow/decoder/sflow/testdata/data-discard-interface.pcap similarity index 100% rename from inlet/flow/decoder/sflow/testdata/data-discard-interface.pcap rename to outlet/flow/decoder/sflow/testdata/data-discard-interface.pcap diff --git a/inlet/flow/decoder/sflow/testdata/data-icmpv4.pcap b/outlet/flow/decoder/sflow/testdata/data-icmpv4.pcap similarity index 100% rename from inlet/flow/decoder/sflow/testdata/data-icmpv4.pcap rename to outlet/flow/decoder/sflow/testdata/data-icmpv4.pcap diff --git a/inlet/flow/decoder/sflow/testdata/data-icmpv6.pcap b/outlet/flow/decoder/sflow/testdata/data-icmpv6.pcap similarity index 100% rename from inlet/flow/decoder/sflow/testdata/data-icmpv6.pcap rename to outlet/flow/decoder/sflow/testdata/data-icmpv6.pcap diff --git a/inlet/flow/decoder/sflow/testdata/data-local-interface.pcap b/outlet/flow/decoder/sflow/testdata/data-local-interface.pcap similarity index 100% rename from inlet/flow/decoder/sflow/testdata/data-local-interface.pcap rename to outlet/flow/decoder/sflow/testdata/data-local-interface.pcap diff --git a/inlet/flow/decoder/sflow/testdata/data-multiple-interfaces.pcap b/outlet/flow/decoder/sflow/testdata/data-multiple-interfaces.pcap similarity index 100% rename from inlet/flow/decoder/sflow/testdata/data-multiple-interfaces.pcap rename to outlet/flow/decoder/sflow/testdata/data-multiple-interfaces.pcap diff --git a/inlet/flow/decoder/sflow/testdata/data-qinq.pcap b/outlet/flow/decoder/sflow/testdata/data-qinq.pcap similarity index 100% rename from inlet/flow/decoder/sflow/testdata/data-qinq.pcap rename to outlet/flow/decoder/sflow/testdata/data-qinq.pcap diff --git a/inlet/flow/decoder/sflow/testdata/data-sflow-expanded-sample.pcap b/outlet/flow/decoder/sflow/testdata/data-sflow-expanded-sample.pcap similarity index 100% rename from inlet/flow/decoder/sflow/testdata/data-sflow-expanded-sample.pcap rename to outlet/flow/decoder/sflow/testdata/data-sflow-expanded-sample.pcap diff --git a/inlet/flow/decoder/sflow/testdata/data-sflow-ipv4-data.pcap b/outlet/flow/decoder/sflow/testdata/data-sflow-ipv4-data.pcap similarity index 100% rename from inlet/flow/decoder/sflow/testdata/data-sflow-ipv4-data.pcap rename to outlet/flow/decoder/sflow/testdata/data-sflow-ipv4-data.pcap diff --git a/inlet/flow/decoder/sflow/testdata/data-sflow-raw-ipv4.pcap b/outlet/flow/decoder/sflow/testdata/data-sflow-raw-ipv4.pcap similarity index 100% rename from inlet/flow/decoder/sflow/testdata/data-sflow-raw-ipv4.pcap rename to outlet/flow/decoder/sflow/testdata/data-sflow-raw-ipv4.pcap diff --git a/inlet/flow/decoder/testdata/mpls-ipv4.pcap b/outlet/flow/decoder/testdata/mpls-ipv4.pcap similarity index 100% rename from inlet/flow/decoder/testdata/mpls-ipv4.pcap rename to outlet/flow/decoder/testdata/mpls-ipv4.pcap diff --git a/inlet/flow/decoder/testdata/vlan-ipv6.pcap b/outlet/flow/decoder/testdata/vlan-ipv6.pcap similarity index 100% rename from inlet/flow/decoder/testdata/vlan-ipv6.pcap rename to outlet/flow/decoder/testdata/vlan-ipv6.pcap diff --git a/outlet/flow/decoder_test.go b/outlet/flow/decoder_test.go new file mode 100644 index 00000000..bf4114b4 --- /dev/null +++ b/outlet/flow/decoder_test.go @@ -0,0 +1,334 @@ +// SPDX-FileCopyrightText: 2023 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package flow + +import ( + "net" + "net/netip" + "path" + "path/filepath" + "runtime" + "testing" + "time" + + "akvorado/common/helpers" + "akvorado/common/pb" + "akvorado/common/reporter" + "akvorado/common/schema" + "akvorado/outlet/flow/decoder" + "akvorado/outlet/flow/decoder/netflow" + "akvorado/outlet/flow/decoder/sflow" +) + +func TestFlowDecode(t *testing.T) { + r := reporter.NewMock(t) + sch := schema.NewMock(t) + c, err := New(r, Dependencies{Schema: sch}) + if err != nil { + t.Fatalf("New() error:\n%+v", err) + } + bf := sch.NewFlowMessage() + got := []*schema.FlowMessage{} + finalize := func() { + bf.TimeReceived = 0 + // Keep a copy of the current flow message + clone := *bf + got = append(got, &clone) + bf.Finalize() + } + + // Get test data path + _, src, _, _ := runtime.Caller(0) + base := path.Join(path.Dir(src), "decoder", "netflow", "testdata") + + // Test Netflow decoding + t.Run("netflow", func(t *testing.T) { + // Load template first + templateData := helpers.ReadPcapL4(t, path.Join(base, "options-template.pcap")) + templateRawFlow := &pb.RawFlow{ + TimeReceived: uint64(time.Now().UnixNano()), + Payload: templateData, + SourceAddress: net.ParseIP("127.0.0.1").To16(), + UseSourceAddress: false, + Decoder: pb.RawFlow_DECODER_NETFLOW, + TimestampSource: pb.RawFlow_TS_INPUT, + } + + // Decode template (should return empty slice for templates) + err := c.Decode(templateRawFlow, bf, finalize) + if err != nil { + t.Fatalf("Decode() template error:\n%+v", err) + } + if len(got) != 0 { + t.Logf("Template decode returned %d flows (expected 0)", len(got)) + } + + // Load options data + optionsData := helpers.ReadPcapL4(t, path.Join(base, "options-data.pcap")) + optionsRawFlow := &pb.RawFlow{ + TimeReceived: uint64(time.Now().UnixNano()), + Payload: optionsData, + SourceAddress: net.ParseIP("127.0.0.1").To16(), + UseSourceAddress: false, + Decoder: pb.RawFlow_DECODER_NETFLOW, + TimestampSource: pb.RawFlow_TS_INPUT, + } + + // Decode options data + err = c.Decode(optionsRawFlow, bf, finalize) + if err != nil { + t.Fatalf("Decode() options data error:\n%+v", err) + } + if len(got) != 0 { + t.Logf("Options data decode returned %d flows (expected 0)", len(got)) + } + + // Load template for actual data + dataTemplateData := helpers.ReadPcapL4(t, path.Join(base, "template.pcap")) + dataTemplateRawFlow := &pb.RawFlow{ + TimeReceived: uint64(time.Now().UnixNano()), + Payload: dataTemplateData, + SourceAddress: net.ParseIP("127.0.0.1").To16(), + UseSourceAddress: false, + Decoder: pb.RawFlow_DECODER_NETFLOW, + TimestampSource: pb.RawFlow_TS_INPUT, + } + + // Decode data template + err = c.Decode(dataTemplateRawFlow, bf, finalize) + if err != nil { + t.Fatalf("Decode() data template error:\n%+v", err) + } + if len(got) != 0 { + t.Logf("Data template decode returned %d flows (expected 0)", len(got)) + } + + // Load actual flow data + flowData := helpers.ReadPcapL4(t, path.Join(base, "data.pcap")) + flowRawFlow := &pb.RawFlow{ + TimeReceived: uint64(time.Now().UnixNano()), + Payload: flowData, + SourceAddress: net.ParseIP("127.0.0.1").To16(), + UseSourceAddress: false, + Decoder: pb.RawFlow_DECODER_NETFLOW, + TimestampSource: pb.RawFlow_TS_INPUT, + } + + // Decode actual flow data + err = c.Decode(flowRawFlow, bf, finalize) + if err != nil { + t.Fatalf("Decode() flow data error:\n%+v", err) + } + if len(got) == 0 { + t.Fatalf("Decode() returned no flows") + } + + t.Logf("Successfully decoded %d flows", len(got)) + + // Test with UseSourceAddress = true + got = got[:0] + flowRawFlow.UseSourceAddress = true + err = c.Decode(flowRawFlow, bf, finalize) + if err != nil { + t.Fatalf("Decode() with UseSourceAddress error:\n%+v", err) + } + if len(got) == 0 { + t.Fatalf("Decode() with UseSourceAddress returned no flows") + } + + // Verify exporter address was overridden (should be IPv4-mapped IPv6) + expectedAddr := "::ffff:127.0.0.1" + for _, flow := range got { + if flow.ExporterAddress.String() != expectedAddr { + t.Errorf("Expected exporter address %s, got %s", expectedAddr, flow.ExporterAddress.String()) + } + } + + gotMetrics := r.GetMetrics("akvorado_outlet_flow_decoder_") + expectedMetrics := map[string]string{ + `flows_total{name="netflow"}`: "8", + `netflow_packets_total{exporter="::ffff:127.0.0.1",version="9"}`: "5", + `netflow_records_total{exporter="::ffff:127.0.0.1",type="DataFlowSet",version="9"}`: "8", + `netflow_records_total{exporter="::ffff:127.0.0.1",type="OptionsDataFlowSet",version="9"}`: "4", + `netflow_records_total{exporter="::ffff:127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", + `netflow_records_total{exporter="::ffff:127.0.0.1",type="TemplateFlowSet",version="9"}`: "1", + `netflow_sets_total{exporter="::ffff:127.0.0.1",type="DataFlowSet",version="9"}`: "2", + `netflow_sets_total{exporter="::ffff:127.0.0.1",type="OptionsDataFlowSet",version="9"}`: "1", + `netflow_sets_total{exporter="::ffff:127.0.0.1",type="OptionsTemplateFlowSet",version="9"}`: "1", + `netflow_sets_total{exporter="::ffff:127.0.0.1",type="TemplateFlowSet",version="9"}`: "1", + `netflow_templates_total{exporter="::ffff:127.0.0.1",obs_domain_id="0",template_id="257",type="options_template",version="9"}`: "1", + `netflow_templates_total{exporter="::ffff:127.0.0.1",obs_domain_id="0",template_id="260",type="template",version="9"}`: "1", + } + if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { + t.Fatalf("Metrics (-got, +want):\n%s", diff) + } + }) + + // Test sflow decoding + t.Run("sflow", func(t *testing.T) { + got = got[:0] + sflowBase := path.Join(path.Dir(src), "decoder", "sflow", "testdata") + flowData := helpers.ReadPcapL4(t, path.Join(sflowBase, "data-1140.pcap")) + flowRawFlow := &pb.RawFlow{ + TimeReceived: uint64(time.Now().UnixNano()), + Payload: flowData, + SourceAddress: net.ParseIP("127.0.0.1").To16(), + UseSourceAddress: false, + Decoder: pb.RawFlow_DECODER_SFLOW, + TimestampSource: pb.RawFlow_TS_INPUT, + } + + err := c.Decode(flowRawFlow, bf, finalize) + if err != nil { + t.Fatalf("Decode() sflow error:\n%+v", err) + } + if len(got) == 0 { + t.Fatalf("Decode() sflow returned no flows") + } + + gotMetrics := r.GetMetrics("akvorado_outlet_flow_decoder_", "flows_total", "sflow_") + expectedMetrics := map[string]string{ + `flows_total{name="netflow"}`: "8", + `flows_total{name="sflow"}`: "5", + `sflow_flows_total{agent="172.16.0.3",exporter="::ffff:127.0.0.1",version="5"}`: "1", + `sflow_sample_records_sum{agent="172.16.0.3",exporter="::ffff:127.0.0.1",type="FlowSample",version="5"}`: "14", + `sflow_sample_sum{agent="172.16.0.3",exporter="::ffff:127.0.0.1",type="FlowSample",version="5"}`: "5", + } + if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { + t.Fatalf("Metrics (-got, +want):\n%s", diff) + } + + t.Logf("Successfully decoded %d sflow flows", len(got)) + }) + + // Test error cases + t.Run("errors", func(t *testing.T) { + // Unknown decoder + got = got[:0] + rawFlow := &pb.RawFlow{ + TimeReceived: uint64(time.Now().UnixNano()), + Payload: []byte("test"), + SourceAddress: net.ParseIP("127.0.0.1").To16(), + UseSourceAddress: false, + Decoder: pb.RawFlow_DECODER_UNSPECIFIED, + TimestampSource: pb.RawFlow_TS_INPUT, + } + + err := c.Decode(rawFlow, bf, finalize) + if err == nil { + t.Fatal("Expected error for unknown decoder") + } + + // Missing source address + rawFlow.Decoder = pb.RawFlow_DECODER_NETFLOW + rawFlow.SourceAddress = nil + err = c.Decode(rawFlow, bf, finalize) + if err == nil { + t.Fatal("Expected error for missing source address") + } + + // Invalid payload for Netflow + rawFlow.Decoder = pb.RawFlow_DECODER_NETFLOW + rawFlow.SourceAddress = net.ParseIP("127.0.0.1").To16() + rawFlow.Payload = []byte("invalid") + err = c.Decode(rawFlow, bf, finalize) + if err == nil { + t.Fatal("Expected error for invalid payload") + } + // Invalid payload for Netflow v5 + rawFlow.Payload = []byte{0, 5, 11, 12, 13, 14} + err = c.Decode(rawFlow, bf, finalize) + if err == nil { + t.Fatal("Expected error for invalid payload") + } + // Invalid payload for Netflow v9 + rawFlow.Payload = []byte{0, 9, 11, 12, 13, 14} + err = c.Decode(rawFlow, bf, finalize) + if err == nil { + t.Fatal("Expected error for invalid payload") + } + // Invalid payload for IPFIX + rawFlow.Payload = []byte{0, 10, 11, 12, 13, 14} + err = c.Decode(rawFlow, bf, finalize) + if err == nil { + t.Fatal("Expected error for invalid payload") + } + // Invalid payload for sFlow + rawFlow.Decoder = pb.RawFlow_DECODER_SFLOW + err = c.Decode(rawFlow, bf, finalize) + if err == nil { + t.Fatal("Expected error for invalid payload") + } + + gotMetrics := r.GetMetrics("akvorado_outlet_flow_decoder_", "errors", "netflow_errors", "sflow_errors") + expectedMetrics := map[string]string{ + `errors_total{name="netflow"}`: "4", + `errors_total{name="sflow"}`: "1", + `netflow_errors_total{error="IPFIX decoding error",exporter="::ffff:127.0.0.1"}`: "1", + `netflow_errors_total{error="NetFlow v5 decoding error",exporter="::ffff:127.0.0.1"}`: "1", + `netflow_errors_total{error="NetFlow v9 decoding error",exporter="::ffff:127.0.0.1"}`: "1", + `sflow_errors_total{error="sFlow decoding error",exporter="::ffff:127.0.0.1"}`: "1", + } + if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { + t.Fatalf("Metrics (-got, +want):\n%s", diff) + } + + }) +} + +func BenchmarkDecodeNetflow(b *testing.B) { + schema.DisableDebug(b) + r := reporter.NewMock(b) + sch := schema.NewMock(b) + bf := sch.NewFlowMessage() + finalize := func() {} + nfdecoder := netflow.New(r, decoder.Dependencies{Schema: sch}) + options := decoder.Option{TimestampSource: pb.RawFlow_TS_INPUT} + + template := helpers.ReadPcapL4(b, filepath.Join("decoder", "netflow", "testdata", "options-template.pcap")) + _, err := nfdecoder.Decode( + decoder.RawFlow{Payload: template, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + b.Fatalf("Decode() error on options template:\n%+v", err) + } + data := helpers.ReadPcapL4(b, filepath.Join("decoder", "netflow", "testdata", "options-data.pcap")) + _, err = nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + b.Fatalf("Decode() error on options data:\n%+v", err) + } + template = helpers.ReadPcapL4(b, filepath.Join("decoder", "netflow", "testdata", "template.pcap")) + _, err = nfdecoder.Decode( + decoder.RawFlow{Payload: template, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + if err != nil { + b.Fatalf("Decode() error on template:\n%+v", err) + } + data = helpers.ReadPcapL4(b, filepath.Join("decoder", "netflow", "testdata", "data.pcap")) + + for b.Loop() { + nfdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + } +} + +func BenchmarkDecodeSflow(b *testing.B) { + schema.DisableDebug(b) + r := reporter.NewMock(b) + sch := schema.NewMock(b) + bf := sch.NewFlowMessage() + finalize := func() {} + sdecoder := sflow.New(r, decoder.Dependencies{Schema: sch}) + options := decoder.Option{TimestampSource: pb.RawFlow_TS_INPUT} + data := helpers.ReadPcapL4(b, filepath.Join("decoder", "sflow", "testdata", "data-1140.pcap")) + + for b.Loop() { + sdecoder.Decode( + decoder.RawFlow{Payload: data, Source: netip.MustParseAddr("::ffff:127.0.0.1")}, + options, bf, finalize) + } +} diff --git a/outlet/flow/root.go b/outlet/flow/root.go new file mode 100644 index 00000000..a8aec9a1 --- /dev/null +++ b/outlet/flow/root.go @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +// Package flow handles flow decoding from protobuf messages. +package flow + +import ( + "time" + + "akvorado/common/pb" + "akvorado/common/reporter" + "akvorado/common/schema" + "akvorado/outlet/flow/decoder" +) + +// Component represents the flow decoder component. +type Component struct { + r *reporter.Reporter + d *Dependencies + errLogger reporter.Logger + + metrics struct { + decoderStats *reporter.CounterVec + decoderErrors *reporter.CounterVec + } + + // Available decoders + decoders map[pb.RawFlow_Decoder]decoder.Decoder +} + +// Dependencies are the dependencies of the flow component. +type Dependencies struct { + Schema *schema.Component +} + +// New creates a new flow component. +func New(r *reporter.Reporter, dependencies Dependencies) (*Component, error) { + c := Component{ + r: r, + d: &dependencies, + errLogger: r.Sample(reporter.BurstSampler(30*time.Second, 3)), + decoders: make(map[pb.RawFlow_Decoder]decoder.Decoder), + } + + // Initialize available decoders + for decoderType, decoderFunc := range availableDecoders { + c.decoders[decoderType] = decoderFunc(r, decoder.Dependencies{Schema: c.d.Schema}) + } + + // Metrics + c.metrics.decoderStats = c.r.CounterVec( + reporter.CounterOpts{ + Name: "decoder_flows_total", + Help: "Decoder processed count.", + }, + []string{"name"}, + ) + c.metrics.decoderErrors = c.r.CounterVec( + reporter.CounterOpts{ + Name: "decoder_errors_total", + Help: "Decoder processed error count.", + }, + []string{"name"}, + ) + + return &c, nil +} diff --git a/outlet/flow/tests.go b/outlet/flow/tests.go new file mode 100644 index 00000000..ad3f1911 --- /dev/null +++ b/outlet/flow/tests.go @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +//go:build !release + +package flow + +import ( + "akvorado/common/pb" + "akvorado/outlet/flow/decoder/gob" +) + +func init() { + // Add gob decoder for testing purposes only + availableDecoders[pb.RawFlow_DECODER_GOB] = gob.New +} diff --git a/outlet/kafka/config.go b/outlet/kafka/config.go new file mode 100644 index 00000000..32e8f136 --- /dev/null +++ b/outlet/kafka/config.go @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package kafka + +import ( + "time" + + "akvorado/common/kafka" +) + +// Configuration describes the configuration for the Kafka exporter. +type Configuration struct { + kafka.Configuration `mapstructure:",squash" yaml:"-,inline"` + // Workers define the number of workers to read messages from Kafka. + Workers int `validate:"min=1"` + // ConsumerGroup is the name of the consumer group to use + ConsumerGroup string `validate:"min=1,ascii"` + // MaxMessageBytes is the maximum permitted size of a message. Should be set + // equal or smaller than broker's `message.max.bytes`. + MaxMessageBytes int32 `validate:"min=1"` + // FetchMinBytes is the minimum number of bytes to wait before fetching a message. + FetchMinBytes int32 `validate:"min=1"` + // FetchMaxWaitTime is the minimum duration to wait to get at least the + // minimum number of bytes. + FetchMaxWaitTime time.Duration `validate:"min=100ms"` + // QueueSize defines the size of the channel used to receive from Kafka. + QueueSize int `validate:"min=1"` +} + +// DefaultConfiguration represents the default configuration for the Kafka exporter. +func DefaultConfiguration() Configuration { + return Configuration{ + Configuration: kafka.DefaultConfiguration(), + Workers: 1, + ConsumerGroup: "akvorado-outlet", + MaxMessageBytes: 1_000_000, + FetchMinBytes: 1_000_000, + FetchMaxWaitTime: time.Second, + QueueSize: 32, + } +} diff --git a/outlet/kafka/config_test.go b/outlet/kafka/config_test.go new file mode 100644 index 00000000..df7bef30 --- /dev/null +++ b/outlet/kafka/config_test.go @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package kafka + +import ( + "testing" + + "akvorado/common/helpers" +) + +func TestDefaultConfiguration(t *testing.T) { + if err := helpers.Validate.Struct(DefaultConfiguration()); err != nil { + t.Fatalf("validate.Struct() error:\n%+v", err) + } +} diff --git a/outlet/kafka/consumer.go b/outlet/kafka/consumer.go new file mode 100644 index 00000000..ef0d5884 --- /dev/null +++ b/outlet/kafka/consumer.go @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: 2024 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package kafka + +import ( + "context" + "errors" + "fmt" + "strconv" + "sync" + + "github.com/IBM/sarama" + "github.com/rs/zerolog" + + "akvorado/common/reporter" +) + +// ErrStopProcessing should be returned as an error when we need to stop processing more flows. +var ErrStopProcessing = errors.New("stop processing further flows") + +// Consumer is a Sarama consumer group consumer and should process flow +// messages. +type Consumer struct { + r *reporter.Reporter + l zerolog.Logger + + healthy chan reporter.ChannelHealthcheckFunc + metrics metrics + worker int + callback ReceiveFunc + mu sync.Mutex +} + +// ReceiveFunc is a function that will be called with each received messages. +type ReceiveFunc func(context.Context, []byte) error + +// ShutdownFunc is a function that will be called on shutdown of the consumer. +type ShutdownFunc func() + +// WorkerBuilderFunc returns a function to be called with each received messages and +// a function to be called when shutting down. +type WorkerBuilderFunc func(int) (ReceiveFunc, ShutdownFunc) + +// NewConsumer creates a new consumer. +func (c *realComponent) NewConsumer(worker int, callback ReceiveFunc) *Consumer { + return &Consumer{ + r: c.r, + l: c.r.With().Int("worker", worker).Logger(), + + healthy: c.healthy, + worker: worker, + metrics: c.metrics, + callback: callback, + } +} + +// Setup is called at the beginning of a new consumer session, before +// ConsumeClaim. +func (c *Consumer) Setup(sarama.ConsumerGroupSession) error { + c.l.Debug().Msg("start consumer group") + return nil +} + +// Cleanup is called once all ConsumeClaim goroutines have exited +func (c *Consumer) Cleanup(sarama.ConsumerGroupSession) error { + c.l.Debug().Msg("stop consumer group") + return nil +} + +// ConsumeClaim should process the incoming claims. +func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + l := c.l.With(). + Str("topic", claim.Topic()). + Int32("partition", claim.Partition()). + Int64("offset", claim.InitialOffset()).Logger() + l.Debug().Msg("process new consumer group claim") + worker := strconv.Itoa(c.worker) + c.metrics.claimsReceived.WithLabelValues(worker).Inc() + messagesReceived := c.metrics.messagesReceived.WithLabelValues(worker) + bytesReceived := c.metrics.bytesReceived.WithLabelValues(worker) + ctx := session.Context() + + for { + select { + case cb, ok := <-c.healthy: + if ok { + cb(reporter.HealthcheckOK, fmt.Sprintf("worker %d ok", c.worker)) + } + case message, ok := <-claim.Messages(): + if !ok { + return nil + } + messagesReceived.Inc() + bytesReceived.Add(float64(len(message.Value))) + + // ConsumeClaim can be called from multiple goroutines. We want each + // worker/consumer to not invoke callbacks concurrently. + c.mu.Lock() + if err := c.callback(ctx, message.Value); err == ErrStopProcessing { + c.mu.Unlock() + return nil + } else if err != nil { + c.mu.Unlock() + c.metrics.errorsReceived.WithLabelValues(worker).Inc() + l.Err(err).Msg("unable to handle incoming message") + return fmt.Errorf("unable to handle incoming message: %w", err) + } + c.mu.Unlock() + session.MarkMessage(message, "") + case <-ctx.Done(): + return nil + } + } +} diff --git a/outlet/kafka/functional_test.go b/outlet/kafka/functional_test.go new file mode 100644 index 00000000..641e643f --- /dev/null +++ b/outlet/kafka/functional_test.go @@ -0,0 +1,141 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package kafka + +import ( + "context" + "fmt" + "math/rand" + "testing" + "time" + + "github.com/IBM/sarama" + + "akvorado/common/daemon" + "akvorado/common/helpers" + "akvorado/common/kafka" + "akvorado/common/pb" + "akvorado/common/reporter" +) + +func TestRealKafka(t *testing.T) { + client, brokers := kafka.SetupKafkaBroker(t) + + // Create the topic + topicName := fmt.Sprintf("test-topic2-%d", rand.Int()) + expectedTopicName := fmt.Sprintf("%s-v%d", topicName, pb.Version) + admin, err := sarama.NewClusterAdminFromClient(client) + if err != nil { + t.Fatalf("NewClusterAdminFromClient() error:\n%+v", err) + } + defer admin.Close() + topicDetail := &sarama.TopicDetail{ + NumPartitions: 1, + ReplicationFactor: 1, + } + err = admin.CreateTopic(expectedTopicName, topicDetail, false) + if err != nil { + t.Fatalf("CreateTopic() error:\n%+v", err) + } + + // Create a producer + producer, err := sarama.NewSyncProducerFromClient(client) + if err != nil { + t.Fatalf("NewSyncProducerFromClient() error:\n%+v", err) + } + defer producer.Close() + + // Callback + got := []string{} + expected := []string{"hello", "hello 2", "hello 3"} + gotAll := make(chan bool) + callback := func(_ context.Context, message []byte) error { + got = append(got, string(message)) + if len(got) == len(expected) { + close(gotAll) + } + return nil + } + + // Start the component + configuration := DefaultConfiguration() + configuration.Topic = topicName + configuration.Brokers = brokers + configuration.Version = kafka.Version(sarama.V2_8_1_0) + configuration.FetchMaxWaitTime = 100 * time.Millisecond + r := reporter.NewMock(t) + c, err := New(r, configuration, Dependencies{Daemon: daemon.NewMock(t)}) + if err != nil { + t.Fatalf("New() error:\n%+v", err) + } + if err := c.(*realComponent).Start(); err != nil { + t.Fatalf("Start() error:\n%+v", err) + } + shutdownCalled := false + c.StartWorkers(func(_ int) (ReceiveFunc, ShutdownFunc) { return callback, func() { shutdownCalled = true } }) + + // Wait for a claim to be processed. Due to rebalance, it could take more than 3 seconds. + timeout := time.After(10 * time.Second) + for { + gotMetrics := r.GetMetrics("akvorado_outlet_kafka_") + if gotMetrics[`received_claims_total{worker="0"}`] == "1" { + break + } + select { + case <-timeout: + t.Fatal("No claim received") + case <-time.After(20 * time.Millisecond): + } + } + + // Send messages + for _, value := range expected { + msg := &sarama.ProducerMessage{ + Topic: expectedTopicName, + Value: sarama.StringEncoder(value), + } + if _, _, err := producer.SendMessage(msg); err != nil { + t.Fatalf("SendMessage() error:\n%+v", err) + } + } + + // Wait for them + select { + case <-time.After(5 * time.Second): + t.Fatal("Too long to get messages") + case <-gotAll: + } + + if diff := helpers.Diff(got, expected); diff != "" { + t.Errorf("Didn't received the expected messages (-got, +want):\n%s", diff) + } + + gotMetrics := r.GetMetrics("akvorado_outlet_kafka_", "received_") + expectedMetrics := map[string]string{ + `received_bytes_total{worker="0"}`: "19", + `received_claims_total{worker="0"}`: "1", + `received_messages_total{worker="0"}`: "3", + } + if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { + t.Errorf("Metrics (-got, +want):\n%s", diff) + } + + { + // Test the healthcheck function + got := r.RunHealthchecks(context.Background()) + if diff := helpers.Diff(got.Details["kafka"], reporter.HealthcheckResult{ + Status: reporter.HealthcheckOK, + Reason: "worker 0 ok", + }); diff != "" { + t.Fatalf("runHealthcheck() (-got, +want):\n%s", diff) + } + } + + if err := c.Stop(); err != nil { + t.Fatalf("Stop() error:\n%+v", err) + } + if !shutdownCalled { + t.Fatal("Stop() didn't call shutdown function") + } +} diff --git a/outlet/kafka/metrics.go b/outlet/kafka/metrics.go new file mode 100644 index 00000000..13eec300 --- /dev/null +++ b/outlet/kafka/metrics.go @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package kafka + +import ( + "akvorado/common/kafka" + "akvorado/common/reporter" +) + +type metrics struct { + c *realComponent + + messagesReceived *reporter.CounterVec + claimsReceived *reporter.CounterVec + bytesReceived *reporter.CounterVec + errorsReceived *reporter.CounterVec + + kafkaMetrics kafka.Metrics +} + +func (c *realComponent) initMetrics() { + c.metrics.c = c + + c.metrics.messagesReceived = c.r.CounterVec( + reporter.CounterOpts{ + Name: "received_messages_total", + Help: "Number of messages received for a given worker.", + }, + []string{"worker"}, + ) + c.metrics.claimsReceived = c.r.CounterVec( + reporter.CounterOpts{ + Name: "received_claims_total", + Help: "Number of claims received for a given worker.", + }, + []string{"worker"}, + ) + c.metrics.bytesReceived = c.r.CounterVec( + reporter.CounterOpts{ + Name: "received_bytes_total", + Help: "Number of bytes received for a given worker.", + }, + []string{"worker"}, + ) + c.metrics.errorsReceived = c.r.CounterVec( + reporter.CounterOpts{ + Name: "received_errors_total", + Help: "Number of errors while handling received messages for a given worker.", + }, + []string{"worker"}, + ) + + c.metrics.kafkaMetrics.Init(c.r, c.kafkaConfig.MetricRegistry) +} diff --git a/outlet/kafka/root.go b/outlet/kafka/root.go new file mode 100644 index 00000000..7e467c38 --- /dev/null +++ b/outlet/kafka/root.go @@ -0,0 +1,165 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +// Package kafka handles flow imports from Kafka. +package kafka + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/IBM/sarama" + "gopkg.in/tomb.v2" + + "akvorado/common/daemon" + "akvorado/common/kafka" + "akvorado/common/pb" + "akvorado/common/reporter" +) + +// Component is the interface a Kafka consumer should implement. +type Component interface { + StartWorkers(WorkerBuilderFunc) error + Stop() error +} + +// realComponent implements the Kafka consumer. +type realComponent struct { + r *reporter.Reporter + d *Dependencies + t tomb.Tomb + config Configuration + + kafkaConfig *sarama.Config + kafkaTopic string + + healthy chan reporter.ChannelHealthcheckFunc + clients []sarama.Client + metrics metrics +} + +// Dependencies define the dependencies of the Kafka exporter. +type Dependencies struct { + Daemon daemon.Component +} + +// New creates a new Kafka exporter component. +func New(r *reporter.Reporter, configuration Configuration, dependencies Dependencies) (Component, error) { + // Build Kafka configuration + kafkaConfig, err := kafka.NewConfig(configuration.Configuration) + if err != nil { + return nil, err + } + kafkaConfig.Consumer.Fetch.Max = configuration.MaxMessageBytes + kafkaConfig.Consumer.Fetch.Min = configuration.FetchMinBytes + kafkaConfig.Consumer.MaxWaitTime = configuration.FetchMaxWaitTime + kafkaConfig.Consumer.Group.Rebalance.GroupStrategies = []sarama.BalanceStrategy{ + sarama.NewBalanceStrategyRoundRobin(), + } + // kafkaConfig.Consumer.Offsets.AutoCommit.Enable = false + kafkaConfig.Consumer.Offsets.Initial = sarama.OffsetNewest + kafkaConfig.Metadata.RefreshFrequency = time.Minute + kafkaConfig.Metadata.AllowAutoTopicCreation = false + kafkaConfig.ChannelBufferSize = configuration.QueueSize + if err := kafkaConfig.Validate(); err != nil { + return nil, fmt.Errorf("cannot validate Kafka configuration: %w", err) + } + + c := realComponent{ + r: r, + d: &dependencies, + config: configuration, + + healthy: make(chan reporter.ChannelHealthcheckFunc), + kafkaConfig: kafkaConfig, + kafkaTopic: fmt.Sprintf("%s-v%d", configuration.Topic, pb.Version), + } + c.initMetrics() + c.r.RegisterHealthcheck("kafka", c.channelHealthcheck()) + c.d.Daemon.Track(&c.t, "outlet/kafka") + return &c, nil +} + +// Start starts the Kafka component. +func (c *realComponent) Start() error { + c.r.Info().Msg("starting Kafka component") + kafka.GlobalKafkaLogger.Register(c.r) + // Start the clients + for i := range c.config.Workers { + logger := c.r.With().Int("worker", i).Logger() + logger.Debug().Msg("starting") + client, err := sarama.NewClient(c.config.Brokers, c.kafkaConfig) + if err != nil { + logger.Err(err). + Int("worker", i). + Str("brokers", strings.Join(c.config.Brokers, ",")). + Msg("unable to create new client") + return fmt.Errorf("unable to create Kafka client: %w", err) + } + c.clients = append(c.clients, client) + } + return nil +} + +// StartWorkers will start the workers. This should only be called once. +func (c *realComponent) StartWorkers(workerBuilder WorkerBuilderFunc) error { + ctx := c.t.Context(context.Background()) + topics := []string{c.kafkaTopic} + for i := range c.config.Workers { + callback, shutdown := workerBuilder(i) + c.t.Go(func() error { + logger := c.r.With(). + Int("worker", i). + Logger() + client, err := sarama.NewConsumerGroupFromClient(c.config.ConsumerGroup, c.clients[i]) + if err != nil { + logger.Err(err). + Int("worker", i). + Str("brokers", strings.Join(c.config.Brokers, ",")). + Msg("unable to create group consumer") + return fmt.Errorf("unable to create Kafka group consumer: %w", err) + } + defer client.Close() + consumer := c.NewConsumer(i, callback) + defer shutdown() + for { + if err := client.Consume(ctx, topics, consumer); err != nil { + if errors.Is(err, sarama.ErrClosedConsumerGroup) { + return nil + } + if errors.Is(err, context.Canceled) { + return nil + } + logger.Err(err). + Int("worker", i). + Msg("cannot get message from consumer") + return fmt.Errorf("cannot get message from consumer: %w", err) + } + } + }) + } + return nil +} + +// Stop stops the Kafka component +func (c *realComponent) Stop() error { + defer func() { + c.kafkaConfig.MetricRegistry.UnregisterAll() + kafka.GlobalKafkaLogger.Unregister() + close(c.healthy) + for _, client := range c.clients { + client.Close() + } + c.r.Info().Msg("Kafka component stopped") + }() + c.r.Info().Msg("stopping Kafka component") + c.t.Kill(nil) + return c.t.Wait() +} + +func (c *realComponent) channelHealthcheck() reporter.HealthcheckFunc { + return reporter.ChannelHealthcheck(c.t.Context(nil), c.healthy) +} diff --git a/outlet/kafka/root_test.go b/outlet/kafka/root_test.go new file mode 100644 index 00000000..979ea918 --- /dev/null +++ b/outlet/kafka/root_test.go @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: 2022 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package kafka + +import ( + "context" + "testing" + "time" + + gometrics "github.com/rcrowley/go-metrics" + + "akvorado/common/daemon" + "akvorado/common/helpers" + "akvorado/common/reporter" +) + +func TestMock(t *testing.T) { + c, incoming := NewMock(t, DefaultConfiguration()) + + got := []string{} + expected := []string{"hello1", "hello2", "hello3"} + gotAll := make(chan bool) + shutdownCalled := false + callback := func(_ context.Context, message []byte) error { + got = append(got, string(message)) + if len(got) == len(expected) { + close(gotAll) + } + return nil + } + c.StartWorkers( + func(_ int) (ReceiveFunc, ShutdownFunc) { + return callback, func() { shutdownCalled = true } + }, + ) + + // Produce messages and wait for them + for _, msg := range expected { + incoming <- []byte(msg) + } + select { + case <-time.After(time.Second): + t.Fatal("Too long to get messages") + case <-gotAll: + } + + if diff := helpers.Diff(got, expected); diff != "" { + t.Errorf("Didn't received the expected messages (-got, +want):\n%s", diff) + } + + c.Stop() + if !shutdownCalled { + t.Error("Stop() should have triggered shutdown function") + } +} + +func TestKafkaMetrics(t *testing.T) { + r := reporter.NewMock(t) + c, err := New(r, DefaultConfiguration(), Dependencies{Daemon: daemon.NewMock(t)}) + if err != nil { + t.Fatalf("New() error:\n%+v", err) + } + kafkaConfig := c.(*realComponent).kafkaConfig + + // Manually put some metrics + gometrics.GetOrRegisterMeter("consumer-fetch-rate", kafkaConfig.MetricRegistry). + Mark(30) + gometrics.GetOrRegisterHistogram("consumer-batch-size", kafkaConfig.MetricRegistry, + gometrics.NewExpDecaySample(10, 1)). + Update(100) + gometrics.GetOrRegisterHistogram("consumer-fetch-response-size", kafkaConfig.MetricRegistry, + gometrics.NewExpDecaySample(10, 1)). + Update(200) + gometrics.GetOrRegisterCounter("consumer-group-join-total-akvorado", kafkaConfig.MetricRegistry). + Inc(20) + gometrics.GetOrRegisterCounter("consumer-group-join-failed-akvorado", kafkaConfig.MetricRegistry). + Inc(1) + gometrics.GetOrRegisterCounter("consumer-group-sync-total-akvorado", kafkaConfig.MetricRegistry). + Inc(4) + gometrics.GetOrRegisterCounter("consumer-group-sync-failed-akvorado", kafkaConfig.MetricRegistry). + Inc(1) + + gotMetrics := r.GetMetrics("akvorado_outlet_kafka_", "-consumer_fetch_rate") + expectedMetrics := map[string]string{ + `consumer_batch_messages_bucket{le="+Inf"}`: "1", + `consumer_batch_messages_bucket{le="0.5"}`: "100", + `consumer_batch_messages_bucket{le="0.9"}`: "100", + `consumer_batch_messages_bucket{le="0.99"}`: "100", + `consumer_batch_messages_count`: "1", + `consumer_batch_messages_sum`: "100", + `consumer_fetch_bytes_bucket{le="+Inf"}`: "1", + `consumer_fetch_bytes_bucket{le="0.5"}`: "200", + `consumer_fetch_bytes_bucket{le="0.9"}`: "200", + `consumer_fetch_bytes_bucket{le="0.99"}`: "200", + `consumer_fetch_bytes_count`: "1", + `consumer_fetch_bytes_sum`: "200", + `consumer_group_join_total{group="akvorado"}`: "20", + `consumer_group_join_failed_total{group="akvorado"}`: "1", + `consumer_group_sync_total{group="akvorado"}`: "4", + `consumer_group_sync_failed_total{group="akvorado"}`: "1", + } + if diff := helpers.Diff(gotMetrics, expectedMetrics); diff != "" { + t.Fatalf("Metrics (-got, +want):\n%s", diff) + } +} diff --git a/outlet/kafka/tests.go b/outlet/kafka/tests.go new file mode 100644 index 00000000..a86706e1 --- /dev/null +++ b/outlet/kafka/tests.go @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: 2024 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +//go:build !release + +package kafka + +import ( + "context" + "testing" +) + +type mockComponent struct { + config Configuration + incoming chan []byte +} + +// NewMock instantiates a fake Kafka consumer that will produce messages sent on +// the returned channel. +func NewMock(_ *testing.T, config Configuration) (Component, chan<- []byte) { + c := mockComponent{ + config: config, + incoming: make(chan []byte), + } + return &c, c.incoming +} + +// StartWorkers start a set of workers to produce received messages. +func (c *mockComponent) StartWorkers(workerBuilder WorkerBuilderFunc) error { + for i := range c.config.Workers { + callback, shutdown := workerBuilder(i) + defer shutdown() + go func() { + for { + message, ok := <-c.incoming + if !ok { + return + } + callback(context.Background(), message) + } + }() + } + return nil +} + +// Stop stops the mock component. +func (c *mockComponent) Stop() error { + close(c.incoming) + return nil +} diff --git a/inlet/metadata/cache.go b/outlet/metadata/cache.go similarity index 98% rename from inlet/metadata/cache.go rename to outlet/metadata/cache.go index 0f283db3..97503592 100644 --- a/inlet/metadata/cache.go +++ b/outlet/metadata/cache.go @@ -9,7 +9,7 @@ import ( "akvorado/common/helpers/cache" "akvorado/common/reporter" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" ) // Interface describes an interface. diff --git a/inlet/metadata/cache_test.go b/outlet/metadata/cache_test.go similarity index 97% rename from inlet/metadata/cache_test.go rename to outlet/metadata/cache_test.go index fc26bc1c..78a2d855 100644 --- a/inlet/metadata/cache_test.go +++ b/outlet/metadata/cache_test.go @@ -21,7 +21,7 @@ import ( "akvorado/common/helpers" "akvorado/common/reporter" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" ) func setupTestCache(t *testing.T) (*reporter.Reporter, *metadataCache) { @@ -58,7 +58,7 @@ func TestGetEmpty(t *testing.T) { r, sc := setupTestCache(t) expectCacheLookup(t, sc, "127.0.0.1", 676, provider.Answer{}) - gotMetrics := r.GetMetrics("akvorado_inlet_metadata_cache_") + gotMetrics := r.GetMetrics("akvorado_outlet_metadata_cache_") expectedMetrics := map[string]string{ `expired_entries_total`: "0", `hits_total`: "0", @@ -89,7 +89,7 @@ func TestSimpleLookup(t *testing.T) { expectCacheLookup(t, sc, "127.0.0.1", 787, provider.Answer{}) expectCacheLookup(t, sc, "127.0.0.2", 676, provider.Answer{}) - gotMetrics := r.GetMetrics("akvorado_inlet_metadata_cache_") + gotMetrics := r.GetMetrics("akvorado_outlet_metadata_cache_") expectedMetrics := map[string]string{ `expired_entries_total`: "0", `hits_total`: "1", @@ -173,7 +173,7 @@ func TestExpire(t *testing.T) { Exporter: provider.Exporter{Name: "localhost"}, Interface: provider.Interface{Name: "Gi0/0/0/1", Description: "Transit"}}) - gotMetrics := r.GetMetrics("akvorado_inlet_metadata_cache_") + gotMetrics := r.GetMetrics("akvorado_outlet_metadata_cache_") expectedMetrics := map[string]string{ `expired_entries_total`: "3", `hits_total`: "7", @@ -464,7 +464,7 @@ func TestConcurrentOperations(t *testing.T) { wg.Wait() for remaining := 20; remaining >= 0; remaining-- { - gotMetrics := r.GetMetrics("akvorado_inlet_metadata_cache_") + gotMetrics := r.GetMetrics("akvorado_outlet_metadata_cache_") hits, err := strconv.ParseFloat(gotMetrics["hits_total"], 64) if err != nil { t.Fatalf("strconv.ParseFloat() error:\n%+v", err) diff --git a/inlet/metadata/config.go b/outlet/metadata/config.go similarity index 93% rename from inlet/metadata/config.go rename to outlet/metadata/config.go index ff4e07e5..27690290 100644 --- a/inlet/metadata/config.go +++ b/outlet/metadata/config.go @@ -7,10 +7,10 @@ import ( "time" "akvorado/common/helpers" - "akvorado/inlet/metadata/provider" - "akvorado/inlet/metadata/provider/gnmi" - "akvorado/inlet/metadata/provider/snmp" - "akvorado/inlet/metadata/provider/static" + "akvorado/outlet/metadata/provider" + "akvorado/outlet/metadata/provider/gnmi" + "akvorado/outlet/metadata/provider/snmp" + "akvorado/outlet/metadata/provider/static" ) // Configuration describes the configuration for the metadata client diff --git a/inlet/metadata/config_test.go b/outlet/metadata/config_test.go similarity index 100% rename from inlet/metadata/config_test.go rename to outlet/metadata/config_test.go diff --git a/inlet/metadata/provider/gnmi/collector.go b/outlet/metadata/provider/gnmi/collector.go similarity index 99% rename from inlet/metadata/provider/gnmi/collector.go rename to outlet/metadata/provider/gnmi/collector.go index e3a4700a..c3dac814 100644 --- a/inlet/metadata/provider/gnmi/collector.go +++ b/outlet/metadata/provider/gnmi/collector.go @@ -12,7 +12,7 @@ import ( "strings" "time" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" "github.com/cenkalti/backoff/v4" "github.com/openconfig/gnmic/pkg/api" diff --git a/inlet/metadata/provider/gnmi/collector_test.go b/outlet/metadata/provider/gnmi/collector_test.go similarity index 98% rename from inlet/metadata/provider/gnmi/collector_test.go rename to outlet/metadata/provider/gnmi/collector_test.go index 395156ea..e765971f 100644 --- a/inlet/metadata/provider/gnmi/collector_test.go +++ b/outlet/metadata/provider/gnmi/collector_test.go @@ -7,7 +7,7 @@ import ( "testing" "akvorado/common/helpers" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" ) func TestStateUpdate(t *testing.T) { diff --git a/inlet/metadata/provider/gnmi/config.go b/outlet/metadata/provider/gnmi/config.go similarity index 99% rename from inlet/metadata/provider/gnmi/config.go rename to outlet/metadata/provider/gnmi/config.go index 05749826..dc67c624 100644 --- a/inlet/metadata/provider/gnmi/config.go +++ b/outlet/metadata/provider/gnmi/config.go @@ -9,7 +9,7 @@ import ( "time" "akvorado/common/helpers" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" "github.com/go-viper/mapstructure/v2" ) diff --git a/inlet/metadata/provider/gnmi/config_test.go b/outlet/metadata/provider/gnmi/config_test.go similarity index 100% rename from inlet/metadata/provider/gnmi/config_test.go rename to outlet/metadata/provider/gnmi/config_test.go diff --git a/inlet/metadata/provider/gnmi/metrics.go b/outlet/metadata/provider/gnmi/metrics.go similarity index 100% rename from inlet/metadata/provider/gnmi/metrics.go rename to outlet/metadata/provider/gnmi/metrics.go diff --git a/inlet/metadata/provider/gnmi/root.go b/outlet/metadata/provider/gnmi/root.go similarity index 98% rename from inlet/metadata/provider/gnmi/root.go rename to outlet/metadata/provider/gnmi/root.go index 6f0178e0..bf7622db 100644 --- a/inlet/metadata/provider/gnmi/root.go +++ b/outlet/metadata/provider/gnmi/root.go @@ -10,7 +10,7 @@ import ( "sync" "akvorado/common/reporter" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" ) // Provider represents the gNMI provider. diff --git a/inlet/metadata/provider/gnmi/srlinux_test.go b/outlet/metadata/provider/gnmi/srlinux_test.go similarity index 99% rename from inlet/metadata/provider/gnmi/srlinux_test.go rename to outlet/metadata/provider/gnmi/srlinux_test.go index e69464bc..eae653f8 100644 --- a/inlet/metadata/provider/gnmi/srlinux_test.go +++ b/outlet/metadata/provider/gnmi/srlinux_test.go @@ -17,7 +17,7 @@ import ( "akvorado/common/helpers" "akvorado/common/reporter" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" "github.com/openconfig/gnmi/proto/gnmi" "github.com/openconfig/gnmic/pkg/api" @@ -781,7 +781,7 @@ commit now t.Fatalf("Query() (-got, +want):\n%s", diff) } - gotMetrics := r.GetMetrics("akvorado_inlet_metadata_provider_gnmi_", "-collector_seconds") + gotMetrics := r.GetMetrics("akvorado_outlet_metadata_provider_gnmi_", "-collector_seconds") expectedMetrics := map[string]string{ `collector_count`: "1", `collector_ready_info{exporter="127.0.0.1"}`: "1", @@ -830,7 +830,7 @@ commit now t.Fatalf("Query() (-got, +want):\n%s", diff) } - gotMetrics = r.GetMetrics("akvorado_inlet_metadata_provider_gnmi_", "-collector_seconds") + gotMetrics = r.GetMetrics("akvorado_outlet_metadata_provider_gnmi_", "-collector_seconds") expectedMetrics = map[string]string{ `collector_count`: "1", `collector_ready_info{exporter="127.0.0.1"}`: "1", diff --git a/inlet/metadata/provider/gnmi/subscribe.go b/outlet/metadata/provider/gnmi/subscribe.go similarity index 100% rename from inlet/metadata/provider/gnmi/subscribe.go rename to outlet/metadata/provider/gnmi/subscribe.go diff --git a/inlet/metadata/provider/gnmi/utils.go b/outlet/metadata/provider/gnmi/utils.go similarity index 100% rename from inlet/metadata/provider/gnmi/utils.go rename to outlet/metadata/provider/gnmi/utils.go diff --git a/inlet/metadata/provider/gnmi/utils_test.go b/outlet/metadata/provider/gnmi/utils_test.go similarity index 100% rename from inlet/metadata/provider/gnmi/utils_test.go rename to outlet/metadata/provider/gnmi/utils_test.go diff --git a/inlet/metadata/provider/root.go b/outlet/metadata/provider/root.go similarity index 100% rename from inlet/metadata/provider/root.go rename to outlet/metadata/provider/root.go diff --git a/inlet/metadata/provider/snmp/config.go b/outlet/metadata/provider/snmp/config.go similarity index 99% rename from inlet/metadata/provider/snmp/config.go rename to outlet/metadata/provider/snmp/config.go index c2df3185..d6576358 100644 --- a/inlet/metadata/provider/snmp/config.go +++ b/outlet/metadata/provider/snmp/config.go @@ -13,7 +13,7 @@ import ( "github.com/gosnmp/gosnmp" "akvorado/common/helpers" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" ) // Configuration describes the configuration for the SNMP client diff --git a/inlet/metadata/provider/snmp/config_test.go b/outlet/metadata/provider/snmp/config_test.go similarity index 100% rename from inlet/metadata/provider/snmp/config_test.go rename to outlet/metadata/provider/snmp/config_test.go diff --git a/inlet/metadata/provider/snmp/poller.go b/outlet/metadata/provider/snmp/poller.go similarity index 99% rename from inlet/metadata/provider/snmp/poller.go rename to outlet/metadata/provider/snmp/poller.go index b43b2ca0..965466db 100644 --- a/inlet/metadata/provider/snmp/poller.go +++ b/outlet/metadata/provider/snmp/poller.go @@ -14,7 +14,7 @@ import ( "github.com/gosnmp/gosnmp" "akvorado/common/reporter" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" ) // Poll polls the SNMP provider for the requested interface indexes. diff --git a/inlet/metadata/provider/snmp/poller_test.go b/outlet/metadata/provider/snmp/poller_test.go similarity index 98% rename from inlet/metadata/provider/snmp/poller_test.go rename to outlet/metadata/provider/snmp/poller_test.go index 61d5f970..8ee382a0 100644 --- a/inlet/metadata/provider/snmp/poller_test.go +++ b/outlet/metadata/provider/snmp/poller_test.go @@ -17,7 +17,7 @@ import ( "akvorado/common/helpers" "akvorado/common/reporter" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" ) func TestPoller(t *testing.T) { @@ -291,7 +291,7 @@ func TestPoller(t *testing.T) { t.Fatalf("Poll() (-got, +want):\n%s", diff) } - gotMetrics := r.GetMetrics("akvorado_inlet_metadata_provider_snmp_poller_", "error_", "pending_", "success_") + gotMetrics := r.GetMetrics("akvorado_outlet_metadata_provider_snmp_poller_", "error_", "pending_", "success_") expectedMetrics := map[string]string{ fmt.Sprintf(`error_requests_total{error="ifalias missing",exporter="%s"}`, exporterStr): "2", // 643+644 fmt.Sprintf(`error_requests_total{error="ifdescr missing",exporter="%s"}`, exporterStr): "1", // 644 diff --git a/inlet/metadata/provider/snmp/root.go b/outlet/metadata/provider/snmp/root.go similarity index 98% rename from inlet/metadata/provider/snmp/root.go rename to outlet/metadata/provider/snmp/root.go index 49d5b491..3dab793c 100644 --- a/inlet/metadata/provider/snmp/root.go +++ b/outlet/metadata/provider/snmp/root.go @@ -12,7 +12,7 @@ import ( "time" "akvorado/common/reporter" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" ) // Provider represents the SNMP provider. diff --git a/inlet/metadata/provider/snmp/tests.go b/outlet/metadata/provider/snmp/tests.go similarity index 100% rename from inlet/metadata/provider/snmp/tests.go rename to outlet/metadata/provider/snmp/tests.go diff --git a/inlet/metadata/provider/static/config.go b/outlet/metadata/provider/static/config.go similarity index 98% rename from inlet/metadata/provider/static/config.go rename to outlet/metadata/provider/static/config.go index 5c45e487..2e4ceb95 100644 --- a/inlet/metadata/provider/static/config.go +++ b/outlet/metadata/provider/static/config.go @@ -8,7 +8,7 @@ import ( "akvorado/common/helpers" "akvorado/common/remotedatasourcefetcher" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" ) // Configuration describes the configuration for the static provider diff --git a/inlet/metadata/provider/static/config_test.go b/outlet/metadata/provider/static/config_test.go similarity index 98% rename from inlet/metadata/provider/static/config_test.go rename to outlet/metadata/provider/static/config_test.go index 37edb43a..e8944673 100644 --- a/inlet/metadata/provider/static/config_test.go +++ b/outlet/metadata/provider/static/config_test.go @@ -9,7 +9,7 @@ import ( "akvorado/common/helpers" "akvorado/common/remotedatasourcefetcher" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" ) func TestValidation(t *testing.T) { diff --git a/inlet/metadata/provider/static/root.go b/outlet/metadata/provider/static/root.go similarity index 98% rename from inlet/metadata/provider/static/root.go rename to outlet/metadata/provider/static/root.go index 74d2a61a..100131d1 100644 --- a/inlet/metadata/provider/static/root.go +++ b/outlet/metadata/provider/static/root.go @@ -9,7 +9,7 @@ import ( "akvorado/common/helpers" "akvorado/common/remotedatasourcefetcher" "akvorado/common/reporter" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" "context" "fmt" diff --git a/inlet/metadata/provider/static/root_test.go b/outlet/metadata/provider/static/root_test.go similarity index 99% rename from inlet/metadata/provider/static/root_test.go rename to outlet/metadata/provider/static/root_test.go index 02c30db6..6caadcc6 100644 --- a/inlet/metadata/provider/static/root_test.go +++ b/outlet/metadata/provider/static/root_test.go @@ -11,7 +11,7 @@ import ( "akvorado/common/helpers" "akvorado/common/reporter" "akvorado/common/schema" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" ) func TestStaticProvider(t *testing.T) { diff --git a/inlet/metadata/provider/static/source.go b/outlet/metadata/provider/static/source.go similarity index 98% rename from inlet/metadata/provider/static/source.go rename to outlet/metadata/provider/static/source.go index 7aaa0ba1..f7cde167 100644 --- a/inlet/metadata/provider/static/source.go +++ b/outlet/metadata/provider/static/source.go @@ -8,7 +8,7 @@ import ( "akvorado/common/helpers" "akvorado/common/remotedatasourcefetcher" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" ) type exporterInfo struct { diff --git a/inlet/metadata/provider/static/source_test.go b/outlet/metadata/provider/static/source_test.go similarity index 99% rename from inlet/metadata/provider/static/source_test.go rename to outlet/metadata/provider/static/source_test.go index 302ebe4a..f5050a50 100644 --- a/inlet/metadata/provider/static/source_test.go +++ b/outlet/metadata/provider/static/source_test.go @@ -15,7 +15,7 @@ import ( "akvorado/common/helpers" "akvorado/common/remotedatasourcefetcher" "akvorado/common/reporter" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" ) func TestInitStaticExporters(t *testing.T) { diff --git a/inlet/metadata/provider/static/tests.go b/outlet/metadata/provider/static/tests.go similarity index 100% rename from inlet/metadata/provider/static/tests.go rename to outlet/metadata/provider/static/tests.go diff --git a/inlet/metadata/root.go b/outlet/metadata/root.go similarity index 97% rename from inlet/metadata/root.go rename to outlet/metadata/root.go index 739619c0..656d3023 100644 --- a/inlet/metadata/root.go +++ b/outlet/metadata/root.go @@ -20,7 +20,7 @@ import ( "akvorado/common/daemon" "akvorado/common/reporter" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" ) // Component represents the metadata compomenent. @@ -82,7 +82,7 @@ func New(r *reporter.Reporter, configuration Configuration, dependencies Depende providerBreakerLoggers: make(map[netip.Addr]reporter.Logger), providers: make([]provider.Provider, 0, 1), } - c.d.Daemon.Track(&c.t, "inlet/metadata") + c.d.Daemon.Track(&c.t, "outlet/metadata") // Initialize providers for _, p := range c.config.Providers { @@ -164,9 +164,10 @@ func (c *Component) Start() error { healthyDispatcher := make(chan reporter.ChannelHealthcheckFunc) c.r.RegisterHealthcheck("metadata/dispatcher", reporter.ChannelHealthcheck(c.t.Context(nil), healthyDispatcher)) c.t.Go(func() error { + dying := c.t.Dying() for { select { - case <-c.t.Dying(): + case <-dying: c.r.Debug().Msg("stopping metadata dispatcher") return nil case cb, ok := <-healthyDispatcher: @@ -189,9 +190,10 @@ func (c *Component) Start() error { workerIDStr := strconv.Itoa(i) c.t.Go(func() error { c.r.Debug().Str("worker", workerIDStr).Msg("starting metadata provider") + dying := c.t.Dying() for { select { - case <-c.t.Dying(): + case <-dying: c.r.Debug().Str("worker", workerIDStr).Msg("stopping metadata provider") return nil case cb, ok := <-c.healthyWorkers: @@ -247,6 +249,7 @@ func (c *Component) dispatchIncomingRequest(request provider.Query) { requestsMap := map[netip.Addr][]uint{ request.ExporterIP: {request.IfIndex}, } + dying := c.t.Dying() for c.config.MaxBatchRequests > 0 { select { case request := <-c.dispatcherChannel: @@ -262,7 +265,7 @@ func (c *Component) dispatchIncomingRequest(request provider.Query) { if len(indexes) < c.config.MaxBatchRequests && len(requestsMap) < 4 { continue } - case <-c.t.Dying(): + case <-dying: return default: // No more requests in queue @@ -274,7 +277,7 @@ func (c *Component) dispatchIncomingRequest(request provider.Query) { c.metrics.providerBatchedCount.Add(float64(len(ifIndexes))) } select { - case <-c.t.Dying(): + case <-dying: return case c.providerChannel <- provider.BatchQuery{ExporterIP: exporterIP, IfIndexes: ifIndexes}: } diff --git a/inlet/metadata/root_test.go b/outlet/metadata/root_test.go similarity index 97% rename from inlet/metadata/root_test.go rename to outlet/metadata/root_test.go index 8481626d..939dcf2a 100644 --- a/inlet/metadata/root_test.go +++ b/outlet/metadata/root_test.go @@ -16,8 +16,8 @@ import ( "akvorado/common/daemon" "akvorado/common/helpers" "akvorado/common/reporter" - "akvorado/inlet/metadata/provider" - "akvorado/inlet/metadata/provider/static" + "akvorado/outlet/metadata/provider" + "akvorado/outlet/metadata/provider/static" ) func expectMockLookup(t *testing.T, c *Component, exporter string, ifIndex uint, expected provider.Answer) { @@ -133,7 +133,7 @@ func TestAutoRefresh(t *testing.T) { }, }) - gotMetrics := r.GetMetrics("akvorado_inlet_metadata_cache_") + gotMetrics := r.GetMetrics("akvorado_outlet_metadata_cache_") for _, runs := range []string{"29", "30", "31"} { // 63/2 expectedMetrics := map[string]string{ `expired_entries_total`: "0", @@ -229,7 +229,7 @@ func TestProviderBreaker(t *testing.T) { } time.Sleep(50 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_metadata_provider_", "breaker_opens_total") + gotMetrics := r.GetMetrics("akvorado_outlet_metadata_provider_", "breaker_opens_total") expectedMetrics := map[string]string{ `breaker_opens_total{exporter="127.0.0.1"}`: tc.ExpectedCount, } @@ -286,7 +286,7 @@ func TestBatching(t *testing.T) { }) t.Run("check", func(t *testing.T) { - gotMetrics := r.GetMetrics("akvorado_inlet_metadata_provider_", "batched_requests_total") + gotMetrics := r.GetMetrics("akvorado_outlet_metadata_provider_", "batched_requests_total") expectedMetrics := map[string]string{ `batched_requests_total`: "4", } diff --git a/inlet/metadata/tests.go b/outlet/metadata/tests.go similarity index 98% rename from inlet/metadata/tests.go rename to outlet/metadata/tests.go index 9be54c41..8a51660b 100644 --- a/inlet/metadata/tests.go +++ b/outlet/metadata/tests.go @@ -14,7 +14,7 @@ import ( "akvorado/common/helpers" "akvorado/common/reporter" "akvorado/common/schema" - "akvorado/inlet/metadata/provider" + "akvorado/outlet/metadata/provider" ) // mockProvider represents a mock provider. diff --git a/inlet/routing/config.go b/outlet/routing/config.go similarity index 90% rename from inlet/routing/config.go rename to outlet/routing/config.go index 2f0570cd..4bad69c3 100644 --- a/inlet/routing/config.go +++ b/outlet/routing/config.go @@ -5,9 +5,9 @@ package routing import ( "akvorado/common/helpers" - "akvorado/inlet/routing/provider" - "akvorado/inlet/routing/provider/bioris" - "akvorado/inlet/routing/provider/bmp" + "akvorado/outlet/routing/provider" + "akvorado/outlet/routing/provider/bioris" + "akvorado/outlet/routing/provider/bmp" ) // Configuration describes the configuration for the routing client. diff --git a/inlet/routing/config_test.go b/outlet/routing/config_test.go similarity index 100% rename from inlet/routing/config_test.go rename to outlet/routing/config_test.go diff --git a/inlet/routing/metrics.go b/outlet/routing/metrics.go similarity index 100% rename from inlet/routing/metrics.go rename to outlet/routing/metrics.go diff --git a/inlet/routing/provider/bioris/config.go b/outlet/routing/provider/bioris/config.go similarity index 97% rename from inlet/routing/provider/bioris/config.go rename to outlet/routing/provider/bioris/config.go index bae345e0..4b0dee4f 100644 --- a/inlet/routing/provider/bioris/config.go +++ b/outlet/routing/provider/bioris/config.go @@ -7,7 +7,7 @@ package bioris import ( "time" - "akvorado/inlet/routing/provider" + "akvorado/outlet/routing/provider" ) // Configuration describes the configuration for the BioRIS component. diff --git a/inlet/routing/provider/bioris/config_test.go b/outlet/routing/provider/bioris/config_test.go similarity index 100% rename from inlet/routing/provider/bioris/config_test.go rename to outlet/routing/provider/bioris/config_test.go diff --git a/inlet/routing/provider/bioris/metrics.go b/outlet/routing/provider/bioris/metrics.go similarity index 100% rename from inlet/routing/provider/bioris/metrics.go rename to outlet/routing/provider/bioris/metrics.go diff --git a/inlet/routing/provider/bioris/root.go b/outlet/routing/provider/bioris/root.go similarity index 98% rename from inlet/routing/provider/bioris/root.go rename to outlet/routing/provider/bioris/root.go index 550371da..38528fbc 100644 --- a/inlet/routing/provider/bioris/root.go +++ b/outlet/routing/provider/bioris/root.go @@ -26,8 +26,8 @@ import ( "gopkg.in/tomb.v2" "akvorado/common/reporter" - "akvorado/inlet/routing/provider" - "akvorado/inlet/routing/provider/bmp" + "akvorado/outlet/routing/provider" + "akvorado/outlet/routing/provider/bmp" ) var ( @@ -97,7 +97,7 @@ func (p *Provider) Start() error { p.Refresh(ctx) } refresh(context.Background()) - p.d.Daemon.Track(&p.t, "inlet/bmp") + p.d.Daemon.Track(&p.t, "outlet/bmp") p.t.Go(func() error { ticker := time.NewTicker(p.config.Refresh) defer ticker.Stop() diff --git a/inlet/routing/provider/bioris/root_test.go b/outlet/routing/provider/bioris/root_test.go similarity index 98% rename from inlet/routing/provider/bioris/root_test.go rename to outlet/routing/provider/bioris/root_test.go index ff957f23..9279205c 100644 --- a/inlet/routing/provider/bioris/root_test.go +++ b/outlet/routing/provider/bioris/root_test.go @@ -26,7 +26,7 @@ import ( "akvorado/common/daemon" "akvorado/common/helpers" "akvorado/common/reporter" - "akvorado/inlet/routing/provider" + "akvorado/outlet/routing/provider" ) func TestChooseRouter(t *testing.T) { @@ -462,7 +462,7 @@ func TestBioRIS(t *testing.T) { } for try := 2; try >= 0; try-- { - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bioris_") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bioris_") expectedMetrics := map[string]string{ // connection_up may take a bit of time fmt.Sprintf(`connection_up{ris="%s"}`, addr): "1", @@ -502,7 +502,7 @@ func TestNonWorkingBioRIS(t *testing.T) { } helpers.StartStop(t, p) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bioris_") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bioris_") expectedMetrics := map[string]string{ `connection_up{ris="ris.invalid:1000"}`: "0", `connection_up{ris="192.0.2.10:1000"}`: "0", diff --git a/inlet/routing/provider/bmp/config.go b/outlet/routing/provider/bmp/config.go similarity index 98% rename from inlet/routing/provider/bmp/config.go rename to outlet/routing/provider/bmp/config.go index 3ec8d065..1f9b0c16 100644 --- a/inlet/routing/provider/bmp/config.go +++ b/outlet/routing/provider/bmp/config.go @@ -6,7 +6,7 @@ package bmp import ( "time" - "akvorado/inlet/routing/provider" + "akvorado/outlet/routing/provider" ) // Configuration describes the configuration for the BMP server. diff --git a/inlet/routing/provider/bmp/config_test.go b/outlet/routing/provider/bmp/config_test.go similarity index 100% rename from inlet/routing/provider/bmp/config_test.go rename to outlet/routing/provider/bmp/config_test.go diff --git a/inlet/routing/provider/bmp/events.go b/outlet/routing/provider/bmp/events.go similarity index 100% rename from inlet/routing/provider/bmp/events.go rename to outlet/routing/provider/bmp/events.go diff --git a/inlet/routing/provider/bmp/hash.go b/outlet/routing/provider/bmp/hash.go similarity index 100% rename from inlet/routing/provider/bmp/hash.go rename to outlet/routing/provider/bmp/hash.go diff --git a/inlet/routing/provider/bmp/lookup.go b/outlet/routing/provider/bmp/lookup.go similarity index 98% rename from inlet/routing/provider/bmp/lookup.go rename to outlet/routing/provider/bmp/lookup.go index dc3f04b6..0b77ff2c 100644 --- a/inlet/routing/provider/bmp/lookup.go +++ b/outlet/routing/provider/bmp/lookup.go @@ -10,7 +10,7 @@ import ( "github.com/kentik/patricia" - "akvorado/inlet/routing/provider" + "akvorado/outlet/routing/provider" ) // LookupResult is the result of the Lookup() function. diff --git a/inlet/routing/provider/bmp/metrics.go b/outlet/routing/provider/bmp/metrics.go similarity index 100% rename from inlet/routing/provider/bmp/metrics.go rename to outlet/routing/provider/bmp/metrics.go diff --git a/inlet/routing/provider/bmp/rd.go b/outlet/routing/provider/bmp/rd.go similarity index 100% rename from inlet/routing/provider/bmp/rd.go rename to outlet/routing/provider/bmp/rd.go diff --git a/inlet/routing/provider/bmp/rd_test.go b/outlet/routing/provider/bmp/rd_test.go similarity index 98% rename from inlet/routing/provider/bmp/rd_test.go rename to outlet/routing/provider/bmp/rd_test.go index 9be331df..9b41b2d4 100644 --- a/inlet/routing/provider/bmp/rd_test.go +++ b/outlet/routing/provider/bmp/rd_test.go @@ -7,7 +7,7 @@ import ( "testing" "akvorado/common/helpers" - "akvorado/inlet/routing/provider/bmp" + "akvorado/outlet/routing/provider/bmp" "github.com/osrg/gobgp/v3/pkg/packet/bgp" ) diff --git a/inlet/routing/provider/bmp/release.go b/outlet/routing/provider/bmp/release.go similarity index 100% rename from inlet/routing/provider/bmp/release.go rename to outlet/routing/provider/bmp/release.go diff --git a/inlet/routing/provider/bmp/remove.go b/outlet/routing/provider/bmp/remove.go similarity index 96% rename from inlet/routing/provider/bmp/remove.go rename to outlet/routing/provider/bmp/remove.go index e30264c3..8c3bf060 100644 --- a/inlet/routing/provider/bmp/remove.go +++ b/outlet/routing/provider/bmp/remove.go @@ -9,9 +9,10 @@ import ( ) func (p *Provider) peerRemovalWorker() error { + dying := p.t.Dying() for { select { - case <-p.t.Dying(): + case <-dying: return nil case pkey := <-p.peerRemovalChan: exporterStr := pkey.exporter.Addr().Unmap().String() @@ -55,7 +56,7 @@ func (p *Provider) peerRemovalWorker() error { // Run is incomplete, update metrics and sleep a bit p.metrics.peerRemovalPartial.WithLabelValues(exporterStr).Inc() select { - case <-p.t.Dying(): + case <-dying: return nil case <-time.After(p.config.RIBPeerRemovalSleepInterval): } diff --git a/inlet/routing/provider/bmp/rib.go b/outlet/routing/provider/bmp/rib.go similarity index 100% rename from inlet/routing/provider/bmp/rib.go rename to outlet/routing/provider/bmp/rib.go diff --git a/inlet/routing/provider/bmp/rib_test.go b/outlet/routing/provider/bmp/rib_test.go similarity index 100% rename from inlet/routing/provider/bmp/rib_test.go rename to outlet/routing/provider/bmp/rib_test.go diff --git a/inlet/routing/provider/bmp/root.go b/outlet/routing/provider/bmp/root.go similarity index 97% rename from inlet/routing/provider/bmp/root.go rename to outlet/routing/provider/bmp/root.go index 6a80cda0..1e2e7d1d 100644 --- a/inlet/routing/provider/bmp/root.go +++ b/outlet/routing/provider/bmp/root.go @@ -16,7 +16,7 @@ import ( "gopkg.in/tomb.v2" "akvorado/common/reporter" - "akvorado/inlet/routing/provider" + "akvorado/outlet/routing/provider" ) // Provider represents the BMP provider. @@ -65,7 +65,7 @@ func (configuration Configuration) New(r *reporter.Reporter, dependencies Depend } p.staleTimer = p.d.Clock.AfterFunc(time.Hour, p.removeStalePeers) - p.d.Daemon.Track(&p.t, "inlet/bmp") + p.d.Daemon.Track(&p.t, "outlet/bmp") p.initMetrics() return &p, nil } diff --git a/inlet/routing/provider/bmp/root_test.go b/outlet/routing/provider/bmp/root_test.go similarity index 95% rename from inlet/routing/provider/bmp/root_test.go rename to outlet/routing/provider/bmp/root_test.go index 76f81edd..0ebed8ff 100644 --- a/inlet/routing/provider/bmp/root_test.go +++ b/outlet/routing/provider/bmp/root_test.go @@ -15,7 +15,7 @@ import ( "akvorado/common/helpers" "akvorado/common/reporter" - "akvorado/inlet/routing/provider" + "akvorado/outlet/routing/provider" "github.com/osrg/gobgp/v3/pkg/packet/bgp" ) @@ -86,7 +86,7 @@ func TestBMP(t *testing.T) { // Init+EOR send(t, conn, "bmp-init.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `opened_connections_total{exporter="127.0.0.1"}`: "1", @@ -99,7 +99,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-terminate.pcap") time.Sleep(30 * time.Millisecond) - gotMetrics = r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics = r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics = map[string]string{ `closed_connections_total{exporter="127.0.0.1"}`: "1", `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", @@ -124,7 +124,7 @@ func TestBMP(t *testing.T) { mockClock.Add(2 * time.Hour) for tries := 20; tries >= 0; tries-- { time.Sleep(5 * time.Millisecond) - gotMetrics = r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics = r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics = map[string]string{ `closed_connections_total{exporter="127.0.0.1"}`: "1", `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", @@ -153,7 +153,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-peers-up.pcap") send(t, conn, "bmp-eor.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "4", @@ -181,7 +181,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-reach.pcap") send(t, conn, "bmp-reach-addpath.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "4", @@ -239,7 +239,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-init.pcap") send(t, conn, "bmp-reach.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ // Same metrics as previously, except the AddPath peer. `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", @@ -265,7 +265,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-peers-up.pcap") send(t, conn, "bmp-eor.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "4", @@ -293,7 +293,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-reach.pcap") send(t, conn, "bmp-peer-down.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "4", @@ -349,7 +349,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-eor.pcap") send(t, conn, "bmp-reach.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "4", @@ -388,7 +388,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-eor.pcap") send(t, conn, "bmp-reach.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "4", @@ -441,7 +441,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-reach.pcap") send(t, conn, "bmp-unreach.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "4", @@ -472,7 +472,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-init.pcap") send(t, conn, "bmp-l3vpn.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "1", @@ -510,7 +510,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-init.pcap") send(t, conn, "bmp-l3vpn.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "routes") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "routes") expectedMetrics := map[string]string{ `routes_total{exporter="127.0.0.1"}`: "2", } @@ -531,7 +531,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-init.pcap") send(t, conn, "bmp-l3vpn.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "routes") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "routes") expectedMetrics := map[string]string{ `routes_total{exporter="127.0.0.1"}`: "0", } @@ -553,7 +553,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-init.pcap") send(t, conn, "bmp-l3vpn.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "routes") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "routes") expectedMetrics := map[string]string{ `routes_total{exporter="127.0.0.1"}`: "2", } @@ -586,7 +586,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-init.pcap") send(t, conn, "bmp-l3vpn.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "routes") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "routes") expectedMetrics := map[string]string{ `routes_total{exporter="127.0.0.1"}`: "2", } @@ -618,7 +618,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-eor.pcap") send(t, conn, "bmp-unreach.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "4", @@ -651,7 +651,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-unreach.pcap") send(t, conn, "bmp-unreach.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "4", @@ -691,7 +691,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-unreach.pcap") send(t, conn, "bmp-unreach.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "4", @@ -721,7 +721,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-reach.pcap") send(t, conn, "bmp-eor.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "4", @@ -780,7 +780,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-l3vpn.pcap") conn.Close() time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "1", @@ -808,7 +808,7 @@ func TestBMP(t *testing.T) { mockClock.Add(2 * time.Hour) time.Sleep(20 * time.Millisecond) - gotMetrics = r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics = r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics = map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "1", @@ -842,7 +842,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-l3vpn.pcap") send(t, conn, "bmp-reach-unknown-family.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") ignoredMetric := `ignored_updates_total{error="unknown route family. AFI: 57, SAFI: 65",exporter="127.0.0.1",reason="afi-safi"}` expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", @@ -881,7 +881,7 @@ func TestBMP(t *testing.T) { send(t, conn, "bmp-l3vpn.pcap") send(t, conn, "bmp-reach-vpls.pcap") time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "1", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "1", @@ -914,7 +914,7 @@ func TestBMP(t *testing.T) { send(t, conn2, "bmp-l3vpn.pcap") conn1.Close() time.Sleep(20 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics := map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "2", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "2", @@ -944,7 +944,7 @@ func TestBMP(t *testing.T) { mockClock.Add(2 * time.Hour) time.Sleep(20 * time.Millisecond) - gotMetrics = r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics = r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics = map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "2", `received_messages_total{exporter="127.0.0.1",type="peer-up-notification"}`: "2", @@ -973,7 +973,7 @@ func TestBMP(t *testing.T) { send(t, conn2, "bmp-terminate.pcap") time.Sleep(30 * time.Millisecond) - gotMetrics = r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics = r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics = map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "2", `received_messages_total{exporter="127.0.0.1",type="termination"}`: "1", @@ -996,7 +996,7 @@ func TestBMP(t *testing.T) { mockClock.Add(2 * time.Hour) time.Sleep(20 * time.Millisecond) - gotMetrics = r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics = r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") expectedMetrics = map[string]string{ `received_messages_total{exporter="127.0.0.1",type="initiation"}`: "2", `received_messages_total{exporter="127.0.0.1",type="termination"}`: "1", @@ -1039,7 +1039,7 @@ func TestBMP(t *testing.T) { mockClock.Add(2 * time.Hour) for tries := 20; tries >= 0; tries-- { time.Sleep(5 * time.Millisecond) - gotMetrics := r.GetMetrics("akvorado_inlet_routing_provider_bmp_", "-locked_duration") + gotMetrics := r.GetMetrics("akvorado_outlet_routing_provider_bmp_", "-locked_duration") // For removed_partial_peers_total, we have 18 routes, but only 14 routes // can be removed while keeping 1 route on each peer. 14 is the max, but // we rely on good-willing from the scheduler to get this number. diff --git a/inlet/routing/provider/bmp/serve.go b/outlet/routing/provider/bmp/serve.go similarity index 100% rename from inlet/routing/provider/bmp/serve.go rename to outlet/routing/provider/bmp/serve.go diff --git a/inlet/routing/provider/bmp/testdata/bmp-eor.pcap b/outlet/routing/provider/bmp/testdata/bmp-eor.pcap similarity index 100% rename from inlet/routing/provider/bmp/testdata/bmp-eor.pcap rename to outlet/routing/provider/bmp/testdata/bmp-eor.pcap diff --git a/inlet/routing/provider/bmp/testdata/bmp-init.pcap b/outlet/routing/provider/bmp/testdata/bmp-init.pcap similarity index 100% rename from inlet/routing/provider/bmp/testdata/bmp-init.pcap rename to outlet/routing/provider/bmp/testdata/bmp-init.pcap diff --git a/inlet/routing/provider/bmp/testdata/bmp-l3vpn.pcap b/outlet/routing/provider/bmp/testdata/bmp-l3vpn.pcap similarity index 100% rename from inlet/routing/provider/bmp/testdata/bmp-l3vpn.pcap rename to outlet/routing/provider/bmp/testdata/bmp-l3vpn.pcap diff --git a/inlet/routing/provider/bmp/testdata/bmp-peer-down.pcap b/outlet/routing/provider/bmp/testdata/bmp-peer-down.pcap similarity index 100% rename from inlet/routing/provider/bmp/testdata/bmp-peer-down.pcap rename to outlet/routing/provider/bmp/testdata/bmp-peer-down.pcap diff --git a/inlet/routing/provider/bmp/testdata/bmp-peer-up.pcap b/outlet/routing/provider/bmp/testdata/bmp-peer-up.pcap similarity index 100% rename from inlet/routing/provider/bmp/testdata/bmp-peer-up.pcap rename to outlet/routing/provider/bmp/testdata/bmp-peer-up.pcap diff --git a/inlet/routing/provider/bmp/testdata/bmp-peers-up.pcap b/outlet/routing/provider/bmp/testdata/bmp-peers-up.pcap similarity index 100% rename from inlet/routing/provider/bmp/testdata/bmp-peers-up.pcap rename to outlet/routing/provider/bmp/testdata/bmp-peers-up.pcap diff --git a/inlet/routing/provider/bmp/testdata/bmp-reach-addpath.pcap b/outlet/routing/provider/bmp/testdata/bmp-reach-addpath.pcap similarity index 100% rename from inlet/routing/provider/bmp/testdata/bmp-reach-addpath.pcap rename to outlet/routing/provider/bmp/testdata/bmp-reach-addpath.pcap diff --git a/inlet/routing/provider/bmp/testdata/bmp-reach-unknown-family.pcap b/outlet/routing/provider/bmp/testdata/bmp-reach-unknown-family.pcap similarity index 100% rename from inlet/routing/provider/bmp/testdata/bmp-reach-unknown-family.pcap rename to outlet/routing/provider/bmp/testdata/bmp-reach-unknown-family.pcap diff --git a/inlet/routing/provider/bmp/testdata/bmp-reach-vpls.pcap b/outlet/routing/provider/bmp/testdata/bmp-reach-vpls.pcap similarity index 100% rename from inlet/routing/provider/bmp/testdata/bmp-reach-vpls.pcap rename to outlet/routing/provider/bmp/testdata/bmp-reach-vpls.pcap diff --git a/inlet/routing/provider/bmp/testdata/bmp-reach.pcap b/outlet/routing/provider/bmp/testdata/bmp-reach.pcap similarity index 100% rename from inlet/routing/provider/bmp/testdata/bmp-reach.pcap rename to outlet/routing/provider/bmp/testdata/bmp-reach.pcap diff --git a/inlet/routing/provider/bmp/testdata/bmp-terminate.pcap b/outlet/routing/provider/bmp/testdata/bmp-terminate.pcap similarity index 100% rename from inlet/routing/provider/bmp/testdata/bmp-terminate.pcap rename to outlet/routing/provider/bmp/testdata/bmp-terminate.pcap diff --git a/inlet/routing/provider/bmp/testdata/bmp-unreach.pcap b/outlet/routing/provider/bmp/testdata/bmp-unreach.pcap similarity index 100% rename from inlet/routing/provider/bmp/testdata/bmp-unreach.pcap rename to outlet/routing/provider/bmp/testdata/bmp-unreach.pcap diff --git a/inlet/routing/provider/bmp/tests.go b/outlet/routing/provider/bmp/tests.go similarity index 99% rename from inlet/routing/provider/bmp/tests.go rename to outlet/routing/provider/bmp/tests.go index b18bb5b0..b29711d1 100644 --- a/inlet/routing/provider/bmp/tests.go +++ b/outlet/routing/provider/bmp/tests.go @@ -12,7 +12,7 @@ import ( "akvorado/common/daemon" "akvorado/common/reporter" - "akvorado/inlet/routing/provider" + "akvorado/outlet/routing/provider" "github.com/benbjohnson/clock" "github.com/osrg/gobgp/v3/pkg/packet/bgp" diff --git a/inlet/routing/provider/bmp/utils.go b/outlet/routing/provider/bmp/utils.go similarity index 100% rename from inlet/routing/provider/bmp/utils.go rename to outlet/routing/provider/bmp/utils.go diff --git a/inlet/routing/provider/bmp/utils_test.go b/outlet/routing/provider/bmp/utils_test.go similarity index 100% rename from inlet/routing/provider/bmp/utils_test.go rename to outlet/routing/provider/bmp/utils_test.go diff --git a/inlet/routing/provider/root.go b/outlet/routing/provider/root.go similarity index 100% rename from inlet/routing/provider/root.go rename to outlet/routing/provider/root.go diff --git a/inlet/routing/root.go b/outlet/routing/root.go similarity index 98% rename from inlet/routing/root.go rename to outlet/routing/root.go index 68e7d9bf..3d279c9d 100644 --- a/inlet/routing/root.go +++ b/outlet/routing/root.go @@ -12,7 +12,7 @@ import ( "time" "akvorado/common/reporter" - "akvorado/inlet/routing/provider" + "akvorado/outlet/routing/provider" ) // Component represents the metadata compomenent. diff --git a/inlet/routing/root_test.go b/outlet/routing/root_test.go similarity index 100% rename from inlet/routing/root_test.go rename to outlet/routing/root_test.go diff --git a/inlet/routing/tests.go b/outlet/routing/tests.go similarity index 95% rename from inlet/routing/tests.go rename to outlet/routing/tests.go index 1c521662..d64886c3 100644 --- a/inlet/routing/tests.go +++ b/outlet/routing/tests.go @@ -10,7 +10,7 @@ import ( "akvorado/common/daemon" "akvorado/common/reporter" - "akvorado/inlet/routing/provider/bmp" + "akvorado/outlet/routing/provider/bmp" ) // NewMock creates a new mock component using BMP as a provider (it's a real one