From c70f3b74bf69aa6e2b438dd06efff96c02b025b0 Mon Sep 17 00:00:00 2001 From: Vincent Bernat Date: Tue, 18 Jun 2024 18:50:34 +0200 Subject: [PATCH] orchestrator/clickhouse: remove init.sh endpoint Instead, just map configuration files inside the container. As we don't have to push the schema anymore, pushing some arbitrary configuration does not seem to be our job. --- console/data/docs/02-configuration.md | 5 -- console/data/docs/03-usage.md | 3 - console/data/docs/05-troubleshooting.md | 11 --- docker/clickhouse/akvorado.xml | 28 +++++++ .../cluster-1.xml} | 0 .../cluster-2.xml} | 0 .../cluster-3.xml} | 0 .../cluster-4.xml} | 0 .../cluster.xml} | 0 docker/docker-compose-clickhouse-cluster.yml | 16 ++-- docker/docker-compose-dev.yml | 16 ++-- docker/docker-compose.yml | 3 +- orchestrator/clickhouse/config.go | 11 ++- .../clickhouse/data/docker-entrypoint.sh | 19 ----- orchestrator/clickhouse/http.go | 75 +------------------ orchestrator/clickhouse/http_test.go | 20 +---- 16 files changed, 52 insertions(+), 155 deletions(-) create mode 100644 docker/clickhouse/akvorado.xml rename docker/{clickhouse-cluster-1.xml => clickhouse/cluster-1.xml} (100%) rename docker/{clickhouse-cluster-2.xml => clickhouse/cluster-2.xml} (100%) rename docker/{clickhouse-cluster-3.xml => clickhouse/cluster-3.xml} (100%) rename docker/{clickhouse-cluster-4.xml => clickhouse/cluster-4.xml} (100%) rename docker/{clickhouse-cluster.xml => clickhouse/cluster.xml} (100%) delete mode 100755 orchestrator/clickhouse/data/docker-entrypoint.sh diff --git a/console/data/docs/02-configuration.md b/console/data/docs/02-configuration.md index 4fc9d9ca..b4828081 100644 --- a/console/data/docs/02-configuration.md +++ b/console/data/docs/02-configuration.md @@ -820,11 +820,6 @@ provided inside `clickhouse`: - `resolutions` defines the various resolutions to keep data - `max-partitions` defines the number of partitions to use when creating consolidated tables -- `system-log-ttl` defines the TTL for system log tables. Set to 0 to disable. - As these tables are partitioned by month, it's useless to use a too low value. - The default value is 30 days. This requires a restart of ClickHouse. -- `prometheus-endpoint` defines the endpoint to configure to expose ClickHouse - metrics to Prometheus. When not defined, this is left unconfigured. - `networks` maps subnets to attributes. Attributes are `name`, `role`, `site`, `region`, and `tenant`. They are exposed as `SrcNetName`, `DstNetName`, `SrcNetRole`, `DstNetRole`, etc. It is also possible to override GeoIP diff --git a/console/data/docs/03-usage.md b/console/data/docs/03-usage.md index 676950df..cd74732a 100644 --- a/console/data/docs/03-usage.md +++ b/console/data/docs/03-usage.md @@ -67,9 +67,6 @@ services: The following endpoints are exposed for use by ClickHouse: -- `/api/v0/orchestrator/clickhouse/init.sh` contains the schemas in the form of a - script to execute during initialization to get them installed at the - proper location - `/api/v0/orchestrator/clickhouse/protocols.csv` contains a CSV with the mapping between protocol numbers and names - `/api/v0/orchestrator/clickhouse/asns.csv` contains a CSV with the mapping diff --git a/console/data/docs/05-troubleshooting.md b/console/data/docs/05-troubleshooting.md index 1f997c4b..73f70d3a 100644 --- a/console/data/docs/05-troubleshooting.md +++ b/console/data/docs/05-troubleshooting.md @@ -372,14 +372,3 @@ FROM system.errors ORDER BY last_error_time LIMIT 10 FORMAT Vertical ``` - -Notably, it may complain about a missing schema for a received -message. In this case, you need to ensure the schemas used by -*Akvorado* are available. When using `docker compose`, you can restart -the orchestrator and ClickHouse to ensure it downloads the latest -schemas. Otherwise, you can manually execute the script installing the -schemas on your ClickHouse server and restart: - -```sh -curl http://akvorado/api/v0/orchestrator/clickhouse/init.sh | sh -``` diff --git a/docker/clickhouse/akvorado.xml b/docker/clickhouse/akvorado.xml new file mode 100644 index 00000000..92415bb6 --- /dev/null +++ b/docker/clickhouse/akvorado.xml @@ -0,0 +1,28 @@ + + + + event_date + INTERVAL 30 DAY DELETE + + + event_date + INTERVAL 30 DAY DELETE + + + event_date + INTERVAL 30 DAY DELETE + + + event_date + INTERVAL 30 DAY DELETE + + + event_date + INTERVAL 30 DAY DELETE + + + event_date + INTERVAL 30 DAY DELETE + + + + /metrics + true + true + true + + diff --git a/docker/clickhouse-cluster-1.xml b/docker/clickhouse/cluster-1.xml similarity index 100% rename from docker/clickhouse-cluster-1.xml rename to docker/clickhouse/cluster-1.xml diff --git a/docker/clickhouse-cluster-2.xml b/docker/clickhouse/cluster-2.xml similarity index 100% rename from docker/clickhouse-cluster-2.xml rename to docker/clickhouse/cluster-2.xml diff --git a/docker/clickhouse-cluster-3.xml b/docker/clickhouse/cluster-3.xml similarity index 100% rename from docker/clickhouse-cluster-3.xml rename to docker/clickhouse/cluster-3.xml diff --git a/docker/clickhouse-cluster-4.xml b/docker/clickhouse/cluster-4.xml similarity index 100% rename from docker/clickhouse-cluster-4.xml rename to docker/clickhouse/cluster-4.xml diff --git a/docker/clickhouse-cluster.xml b/docker/clickhouse/cluster.xml similarity index 100% rename from docker/clickhouse-cluster.xml rename to docker/clickhouse/cluster.xml diff --git a/docker/docker-compose-clickhouse-cluster.yml b/docker/docker-compose-clickhouse-cluster.yml index 9d403a70..19c598a4 100644 --- a/docker/docker-compose-clickhouse-cluster.yml +++ b/docker/docker-compose-clickhouse-cluster.yml @@ -12,8 +12,8 @@ services: depends_on: - zookeeper volumes: - - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml - - ./clickhouse-cluster-1.xml:/etc/clickhouse-server/config.d/cluster-1.xml + - ./clickhouse/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml + - ./clickhouse/cluster-1.xml:/etc/clickhouse-server/config.d/cluster-1.xml clickhouse-2: extends: file: docker-compose.yml @@ -21,8 +21,8 @@ services: depends_on: - zookeeper volumes: - - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml - - ./clickhouse-cluster-2.xml:/etc/clickhouse-server/config.d/cluster-2.xml + - ./clickhouse/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml + - ./clickhouse/cluster-2.xml:/etc/clickhouse-server/config.d/cluster-2.xml - akvorado-clickhouse-2:/var/lib/clickhouse clickhouse-3: extends: @@ -31,8 +31,8 @@ services: depends_on: - zookeeper volumes: - - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml - - ./clickhouse-cluster-3.xml:/etc/clickhouse-server/config.d/cluster-3.xml + - ./clickhouse/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml + - ./clickhouse/cluster-3.xml:/etc/clickhouse-server/config.d/cluster-3.xml - akvorado-clickhouse-3:/var/lib/clickhouse clickhouse-4: extends: @@ -41,6 +41,6 @@ services: depends_on: - zookeeper volumes: - - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml - - ./clickhouse-cluster-4.xml:/etc/clickhouse-server/config.d/cluster-4.xml + - ./clickhouse/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml + - ./clickhouse/cluster-4.xml:/etc/clickhouse-server/config.d/cluster-4.xml - akvorado-clickhouse-4:/var/lib/clickhouse diff --git a/docker/docker-compose-dev.yml b/docker/docker-compose-dev.yml index 932ea3f0..5fe3d428 100644 --- a/docker/docker-compose-dev.yml +++ b/docker/docker-compose-dev.yml @@ -121,30 +121,30 @@ services: cap_add: - SYS_NICE volumes: - - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml - - ./clickhouse-cluster-1.xml:/etc/clickhouse-server/config.d/cluster-1.xml + - ./clickhouse/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml + - ./clickhouse/cluster-1.xml:/etc/clickhouse-server/config.d/cluster-1.xml ports: - 127.0.0.1:8123:8123/tcp - 127.0.0.1:9000:9000/tcp clickhouse-2: <<: *clickhouse volumes: - - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml - - ./clickhouse-cluster-2.xml:/etc/clickhouse-server/config.d/cluster-2.xml + - ./clickhouse/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml + - ./clickhouse/cluster-2.xml:/etc/clickhouse-server/config.d/cluster-2.xml ports: - 127.0.0.1:9001:9000/tcp clickhouse-3: <<: *clickhouse volumes: - - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml - - ./clickhouse-cluster-3.xml:/etc/clickhouse-server/config.d/cluster-3.xml + - ./clickhouse/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml + - ./clickhouse/cluster-3.xml:/etc/clickhouse-server/config.d/cluster-3.xml ports: - 127.0.0.1:9002:9000/tcp clickhouse-4: <<: *clickhouse volumes: - - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml - - ./clickhouse-cluster-4.xml:/etc/clickhouse-server/config.d/cluster-4.xml + - ./clickhouse/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml + - ./clickhouse/cluster-4.xml:/etc/clickhouse-server/config.d/cluster-4.xml ports: - 127.0.0.1:9003:9000/tcp diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 72c8636e..a6858bdb 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -167,9 +167,8 @@ services: file: versions.yml service: clickhouse volumes: - - ../orchestrator/clickhouse/data/docker-entrypoint.sh:/entrypoint-override.sh:ro - akvorado-clickhouse:/var/lib/clickhouse - entrypoint: /entrypoint-override.sh + - ./clickhouse/akvorado.xml:/etc/clickhouse-server/config.d/akvorado.xml environment: - CLICKHOUSE_INIT_TIMEOUT=60 - CLICKHOUSE_SKIP_USER_SETUP=1 diff --git a/orchestrator/clickhouse/config.go b/orchestrator/clickhouse/config.go index b7a21697..4524e055 100644 --- a/orchestrator/clickhouse/config.go +++ b/orchestrator/clickhouse/config.go @@ -27,11 +27,6 @@ type Configuration struct { // MaxPartitions define the number of partitions to have for a // consolidated flow tables when full. MaxPartitions int `validate:"isdefault|min=1"` - // SystemLogTTL is the TTL to set for system log tables. - SystemLogTTL time.Duration `validate:"isdefault|min=1m"` - // PrometheusEndpoint defines the endpoint ClickHouse can use to expose - // metrics to Prometheus. If not defined, this is not configured. - PrometheusEndpoint string // ASNs is a mapping from AS numbers to names. It replaces or // extends the builtin list of AS numbers. ASNs map[uint32]string @@ -52,6 +47,11 @@ type Configuration struct { // OrchestratorBasicAuth holds optional basic auth credentials to reach // orchestrator from ClickHouse OrchestratorBasicAuth *ConfigurationBasicAuth + + // SystemLogTTL is unused now. + SystemLogTTL time.Duration `yaml:"-"` + // PrometheusEndpoint is unused now. + PrometheusEndpoint string `yaml:"-"` } // ConfigurationBasicAuth holds Username and Password subfields @@ -100,7 +100,6 @@ func DefaultConfiguration() Configuration { }, MaxPartitions: 50, NetworkSourcesTimeout: 10 * time.Second, - SystemLogTTL: 30 * 24 * time.Hour, // 30 days } } diff --git a/orchestrator/clickhouse/data/docker-entrypoint.sh b/orchestrator/clickhouse/data/docker-entrypoint.sh deleted file mode 100755 index e997c289..00000000 --- a/orchestrator/clickhouse/data/docker-entrypoint.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: 2024 Free Mobile -# SPDX-License-Identifier: AGPL-3.0-only - -set -e - -if [[ $# -lt 1 ]] || [[ "$1" = "--"* ]]; then - rm -f /tmp/init.sh - while [[ ! -s /tmp/init.sh ]]; do - sleep 1 - echo "Downloading ClickHouse init script..." - wget --no-proxy -qO /tmp/init.sh \ - http://akvorado-orchestrator:8080/api/v0/orchestrator/clickhouse/init.sh || continue - done - sh /tmp/init.sh -fi - -# Use official entrypoint -exec /entrypoint.sh "$@" diff --git a/orchestrator/clickhouse/http.go b/orchestrator/clickhouse/http.go index 2b6cf80a..b6b9e24f 100644 --- a/orchestrator/clickhouse/http.go +++ b/orchestrator/clickhouse/http.go @@ -4,7 +4,6 @@ package clickhouse import ( - "bytes" "compress/gzip" "embed" "encoding/csv" @@ -13,7 +12,6 @@ import ( "net/http" "os" "strconv" - "text/template" "time" ) @@ -23,49 +21,9 @@ var ( //go:embed data/asns.csv //go:embed data/tcp.csv //go:embed data/udp.csv - data embed.FS - initShTemplate = template.Must(template.New("initsh").Parse(`#!/bin/sh - -# Install Protobuf schema -mkdir -p /var/lib/clickhouse/format_schemas -echo "Install flow schema flow-{{ .FlowSchemaHash }}.proto" -cat > /var/lib/clickhouse/format_schemas/flow-{{ .FlowSchemaHash }}.proto <<'EOPROTO' -{{ .FlowSchema }} -EOPROTO - -# Alter ClickHouse configuration -mkdir -p /etc/clickhouse-server/config.d -echo "Add Akvorado-specific configuration to ClickHouse" -cat > /etc/clickhouse-server/config.d/akvorado.xml <<'EOCONFIG' - -{{- if gt .SystemLogTTL 0 }} -{{- range $table := .SystemLogTables }} - <{{ $table }}> - event_date + INTERVAL {{ $.SystemLogTTL }} SECOND DELETE - -{{- end }} -{{- end }} -{{- if ne .PrometheusEndpoint "" }} - - {{ .PrometheusEndpoint }} - true - true - true - -{{- end }} - -EOCONFIG -`)) + data embed.FS ) -type initShVariables struct { - FlowSchemaHash string - FlowSchema string - SystemLogTTL int - SystemLogTables []string - PrometheusEndpoint string -} - func (c *Component) addHandlerEmbedded(url string, path string) { c.d.HTTP.AddHandler(url, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -83,37 +41,6 @@ func (c *Component) addHandlerEmbedded(url string, path string) { // registerHTTPHandler register some handlers that will be useful for // ClickHouse func (c *Component) registerHTTPHandlers() error { - // init.sh - c.d.HTTP.AddHandler("/api/v0/orchestrator/clickhouse/init.sh", - http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - var result bytes.Buffer - if err := initShTemplate.Execute(&result, initShVariables{ - FlowSchemaHash: c.d.Schema.ProtobufMessageHash(), - FlowSchema: c.d.Schema.ProtobufDefinition(), - SystemLogTTL: int(c.config.SystemLogTTL.Seconds()), - SystemLogTables: []string{ - "asynchronous_metric_log", - "error_log", - "metric_log", - "part_log", - "processors_profile_log", - "query_log", - "query_thread_log", - "query_views_log", - "session_log", - "text_log", - "trace_log", - }, - PrometheusEndpoint: c.config.PrometheusEndpoint, - }); err != nil { - c.r.Err(err).Msg("unable to serialize init.sh") - http.Error(w, fmt.Sprintf("Unable to serialize init.sh"), http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "text/x-shellscript") - w.Write(result.Bytes()) - })) - // Add handler for custom dicts for name, dict := range c.d.Schema.GetCustomDictConfig() { c.d.HTTP.AddHandler(fmt.Sprintf("/api/v0/orchestrator/clickhouse/custom_dict_%s.csv", name), http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { diff --git a/orchestrator/clickhouse/http_test.go b/orchestrator/clickhouse/http_test.go index c72379d4..c14ed6a0 100644 --- a/orchestrator/clickhouse/http_test.go +++ b/orchestrator/clickhouse/http_test.go @@ -4,7 +4,6 @@ package clickhouse import ( - "fmt" "testing" "akvorado/common/clickhousedb" @@ -75,30 +74,13 @@ func TestHTTPEndpoints(t *testing.T) { `192.0.2.0/24,infra,,,,,,,,`, }, }, { - URL: "/api/v0/orchestrator/clickhouse/init.sh", - ContentType: "text/x-shellscript", - FirstLines: []string{ - `#!/bin/sh`, - ``, - `# Install Protobuf schema`, - `mkdir -p /var/lib/clickhouse/format_schemas`, - fmt.Sprintf(`echo "Install flow schema flow-%s.proto"`, - c.d.Schema.ProtobufMessageHash()), - fmt.Sprintf(`cat > /var/lib/clickhouse/format_schemas/flow-%s.proto <<'EOPROTO'`, - c.d.Schema.ProtobufMessageHash()), - "", - `syntax = "proto3";`, - }, - }, - { URL: "/api/v0/orchestrator/clickhouse/custom_dict_none.csv", ContentType: "text/plain; charset=utf-8", StatusCode: 404, FirstLines: []string{ "unable to deliver custom dict csv file none.csv", }, - }, - { + }, { URL: "/api/v0/orchestrator/clickhouse/custom_dict_test.csv", ContentType: "text/csv; charset=utf-8", FirstLines: []string{