mirror of
https://github.com/akvorado/akvorado.git
synced 2025-12-12 06:24:10 +01:00
28783ff4f3
Fix #605 All MergeTree tables are now replicated. For some tables, a `_local` variant is added and the non-`_local` variant is now distributed. The distributed tables are the `flows` table, the `flows_DDDD` tables (where `DDDD` is a duration), as well as the `flows_raw_errors` table. The `exporters` table is not distributed and stays local. The data is following this schema: - data is coming from `flows_HHHH_raw` table, using the Kafka engine - the `flows_HHHH_raw_consumer` reads data from `flows_HHHH_raw` (local) and sends it to `flows` (distributed) when there is no error - the `flows_raw_errors_consumer` reads data from `flows_HHHH_raw` (local) and sends it to `flows_raw_errors` (distributed) - the `flows_DDDD_consumer` reads fata from `flows_local` (local) and sends it to `flow_DDDD_local` (local) - the `exporters_consumer` reads data from `flows` (distributed) and sends it to `exporters` (local) The reason for `flows_HHHH_raw_consumer` to send data to the distributed `flows` table, and not the local one is to ensure flows are balanced (for example, if there is not enough Kafka partitions). But sending it to `flows_local` would have been possible. On the other hand, it is important for `flows_DDDD_consumer` to read from local to avoid duplication. It could have sent to distributed, but the data is now balanced correctly and we just send it to local instead for better performance. The `exporters_consumer` is allowed to read from the distributed `flows` table because it writes the result to the local `exporters` table.
75 lines
2.8 KiB
Go
75 lines
2.8 KiB
Go
// SPDX-FileCopyrightText: 2024 Free Mobile
|
|
// SPDX-License-Identifier: AGPL-3.0-only
|
|
|
|
package clickhousedb
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"akvorado/common/helpers"
|
|
)
|
|
|
|
func TestTransformQueryOnCluster(t *testing.T) {
|
|
cases := []struct {
|
|
Input string
|
|
Expected string
|
|
}{
|
|
{"SYSTEM RELOAD DICTIONARIES", "SYSTEM RELOAD DICTIONARIES ON CLUSTER akvorado"},
|
|
{"system reload dictionaries", "system reload dictionaries ON CLUSTER akvorado"},
|
|
{" system reload dictionaries ", "system reload dictionaries ON CLUSTER akvorado"},
|
|
{"DROP DATABASE IF EXISTS 02028_db", "DROP DATABASE IF EXISTS 02028_db ON CLUSTER akvorado"},
|
|
{
|
|
"CREATE TABLE test_01148_atomic.rmt2 (n int, PRIMARY KEY n) ENGINE=ReplicatedMergeTree",
|
|
"CREATE TABLE test_01148_atomic.rmt2 ON CLUSTER akvorado (n int, PRIMARY KEY n) ENGINE=ReplicatedMergeTree",
|
|
},
|
|
{
|
|
"DROP TABLE IF EXISTS test_repl NO DELAY",
|
|
"DROP TABLE IF EXISTS test_repl ON CLUSTER akvorado NO DELAY",
|
|
},
|
|
{
|
|
"ALTER TABLE 02577_keepermap_delete_update UPDATE value2 = value2 * 10 + 2 WHERE value2 < 100",
|
|
"ALTER TABLE 02577_keepermap_delete_update ON CLUSTER akvorado UPDATE value2 = value2 * 10 + 2 WHERE value2 < 100",
|
|
},
|
|
{"ATTACH DICTIONARY db_01018.dict1", "ATTACH DICTIONARY db_01018.dict1 ON CLUSTER akvorado"},
|
|
{
|
|
`CREATE DICTIONARY default.asns
|
|
(
|
|
asn UInt32 INJECTIVE,
|
|
name String
|
|
)
|
|
PRIMARY KEY asn
|
|
SOURCE(HTTP(URL 'http://akvorado-orchestrator:8080/api/v0/orchestrator/clickhouse/asns.csv' FORMAT 'CSVWithNames'))
|
|
LIFETIME(MIN 0 MAX 3600)
|
|
LAYOUT(HASHED())
|
|
SETTINGS(format_csv_allow_single_quotes = 0)`,
|
|
`CREATE DICTIONARY default.asns ON CLUSTER akvorado ( asn UInt32 INJECTIVE, name String ) PRIMARY KEY asn SOURCE(HTTP(URL 'http://akvorado-orchestrator:8080/api/v0/orchestrator/clickhouse/asns.csv' FORMAT 'CSVWithNames')) LIFETIME(MIN 0 MAX 3600) LAYOUT(HASHED()) SETTINGS(format_csv_allow_single_quotes = 0)`,
|
|
},
|
|
{
|
|
`
|
|
CREATE TABLE queue (
|
|
timestamp UInt64,
|
|
level String,
|
|
message String
|
|
) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow')
|
|
`,
|
|
`CREATE TABLE queue ON CLUSTER akvorado ( timestamp UInt64, level String, message String ) ENGINE = Kafka('localhost:9092', 'topic', 'group1', 'JSONEachRow')`,
|
|
},
|
|
{
|
|
`
|
|
CREATE MATERIALIZED VIEW consumer TO daily
|
|
AS SELECT toDate(toDateTime(timestamp)) AS day, level, count() as total
|
|
FROM queue GROUP BY day, level
|
|
`,
|
|
`CREATE MATERIALIZED VIEW consumer ON CLUSTER akvorado TO daily AS SELECT toDate(toDateTime(timestamp)) AS day, level, count() as total FROM queue GROUP BY day, level`,
|
|
},
|
|
// Not modified
|
|
{"SELECT 1", "SELECT 1"},
|
|
}
|
|
for _, tc := range cases {
|
|
got := TransformQueryOnCluster(tc.Input, "akvorado")
|
|
if diff := helpers.Diff(got, tc.Expected); diff != "" {
|
|
t.Errorf("TransformQueryOnCluster(%q) (-got +want):\n%s", tc.Input, diff)
|
|
}
|
|
}
|
|
}
|