console: use INTERPOLATE to fill values when no value present

This dump the requirement on ClickHouse to 22.4. Use 22.8 LTS. This should also fix a memory leak (not yet fixed in 22.3).
2025-12-12 06:24:10 +01:00 · 2022-09-08 15:41:33 +02:00
parent 447638f784
commit 672d632a5a
8 changed files with 45 additions and 21 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -52,7 +52,7 @@ jobs:
        ports:
          - 9092:9093
      clickhouse:
-        image: clickhouse/clickhouse-server:22.3
+        image: clickhouse/clickhouse-server:22.8
        ports:
          - 9000:9000
    env:
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -33,7 +33,7 @@ run tests:
      alias: zookeeper
    - name: bitnami/kafka:2.8.1
      alias: kafka
-    - name: clickhouse/clickhouse-server:22.3
+    - name: clickhouse/clickhouse-server:22.8
      alias: clickhouse
  script:
    - time apk add --no-cache git make gcc musl-dev protoc shared-mime-info npm curl
--- a/console/data/docs/01-install.md
+++ b/console/data/docs/01-install.md
@@ -4,7 +4,8 @@
 single binary or Docker image. It also requires an installation of
 [Kafka](https://kafka.apache.org/quickstart) and
 [ClickHouse](https://clickhouse.com/docs/en/getting-started/install/).
-They have to be installed separately.
+They have to be installed separately. For ClickHouse, the minimal
+version is 22.4 (due to the use of the `INTERPOLATE` directive).

 ## Docker image

--- a/console/data/docs/99-changelog.md
+++ b/console/data/docs/99-changelog.md
@@ -13,6 +13,11 @@ identified with a specific icon:

 ## Unreleased

+This release bumps the minimal required version for ClickHouse to
+22.4. The `docker-compose` file has been updated to use ClickHouse
+22.8 (which is a long term version).
+
+- 💥 *console*: make ClickHouse interpolate missing values (ClickHouse 22.4+ is required)
 - 🩹 *orchestrator*: validate configuration of other services on start
 - 🩹 *inlet*: correctly parse `inlet.snmp.communities` when it is just a string
 - 🌱 *cmd*: print a shorter message when an internal error happens when parsing configuration
--- a/console/graph.go
+++ b/console/graph.go
@@ -119,6 +119,7 @@ func (input graphHandlerInput) toSQL1(axis int, options toSQL1Options) string {
 	}
 	selectFields := []string{}
 	dimensions := []string{}
+	dimensionsInterpolate := ""
 	others := []string{}
 	for _, column := range input.Dimensions {
 		field := column.toSQLSelect()
@@ -131,8 +132,10 @@ func (input graphHandlerInput) toSQL1(axis int, options toSQL1Options) string {
 			strings.Join(dimensions, ", "),
 			strings.Join(selectFields, ", "),
 			strings.Join(others, ", ")))
+		dimensionsInterpolate = fmt.Sprintf("[%s]", strings.Join(others, ", "))
 	} else {
 		fields = append(fields, "emptyArrayString() AS dimensions")
+		dimensionsInterpolate = "emptyArrayString()"
 	}

 	// With
@@ -161,7 +164,8 @@ GROUP BY time, dimensions
 ORDER BY time WITH FILL
 FROM {{ .TimefilterStart }}%s
 TO {{ .TimefilterEnd }} + INTERVAL 1 second%s
- STEP {{ .Interval }})
+ STEP {{ .Interval }}
+ INTERPOLATE (dimensions AS %s))
 {{ end }}`,
 		templateContext(inputContext{
 			Start:             input.Start,
@@ -171,7 +175,9 @@ ORDER BY time WITH FILL
 			Points:            input.Points,
 			Units:             input.Units,
 		}),
-		withStr, axis, strings.Join(fields, ",\n "), where, offsetShift, offsetShift)
+		withStr, axis, strings.Join(fields, ",\n "), where, offsetShift, offsetShift,
+		dimensionsInterpolate,
+	)
 	return strings.TrimSpace(sqlQuery)
 }

--- a/console/graph_test.go
+++ b/console/graph_test.go
@@ -166,7 +166,8 @@ GROUP BY time, dimensions
 ORDER BY time WITH FILL
 FROM {{ .TimefilterStart }}
 TO {{ .TimefilterEnd }} + INTERVAL 1 second
- STEP {{ .Interval }})
+ STEP {{ .Interval }}
+ INTERPOLATE (dimensions AS emptyArrayString()))
 {{ end }}`,
 		}, {
 			Description: "no dimensions, no filters, l2 bps",
@@ -191,7 +192,8 @@ GROUP BY time, dimensions
 ORDER BY time WITH FILL
 FROM {{ .TimefilterStart }}
 TO {{ .TimefilterEnd }} + INTERVAL 1 second
- STEP {{ .Interval }})
+ STEP {{ .Interval }}
+ INTERPOLATE (dimensions AS emptyArrayString()))
 {{ end }}
 `,
 		}, {
@@ -217,7 +219,8 @@ GROUP BY time, dimensions
 ORDER BY time WITH FILL
 FROM {{ .TimefilterStart }}
 TO {{ .TimefilterEnd }} + INTERVAL 1 second
- STEP {{ .Interval }})
+ STEP {{ .Interval }}
+ INTERPOLATE (dimensions AS emptyArrayString()))
 {{ end }}`,
 		}, {
 			Description: "no dimensions",
@@ -242,7 +245,8 @@ GROUP BY time, dimensions
 ORDER BY time WITH FILL
 FROM {{ .TimefilterStart }}
 TO {{ .TimefilterEnd }} + INTERVAL 1 second
- STEP {{ .Interval }})
+ STEP {{ .Interval }}
+ INTERPOLATE (dimensions AS emptyArrayString()))
 {{ end }}`,
 		}, {
 			Description: "no dimensions, escaped filter",
@@ -267,7 +271,8 @@ GROUP BY time, dimensions
 ORDER BY time WITH FILL
 FROM {{ .TimefilterStart }}
 TO {{ .TimefilterEnd }} + INTERVAL 1 second
- STEP {{ .Interval }})
+ STEP {{ .Interval }}
+ INTERPOLATE (dimensions AS emptyArrayString()))
 {{ end }}`,
 		}, {
 			Description: "no dimensions, reverse direction",
@@ -296,7 +301,8 @@ GROUP BY time, dimensions
 ORDER BY time WITH FILL
 FROM {{ .TimefilterStart }}
 TO {{ .TimefilterEnd }} + INTERVAL 1 second
- STEP {{ .Interval }})
+ STEP {{ .Interval }}
+ INTERPOLATE (dimensions AS emptyArrayString()))
 {{ end }}
 UNION ALL
 {{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
@@ -311,7 +317,8 @@ GROUP BY time, dimensions
 ORDER BY time WITH FILL
 FROM {{ .TimefilterStart }}
 TO {{ .TimefilterEnd }} + INTERVAL 1 second
- STEP {{ .Interval }})
+ STEP {{ .Interval }}
+ INTERPOLATE (dimensions AS emptyArrayString()))
 {{ end }}`,
 		}, {
 			Description: "no filters",
@@ -342,7 +349,8 @@ GROUP BY time, dimensions
 ORDER BY time WITH FILL
 FROM {{ .TimefilterStart }}
 TO {{ .TimefilterEnd }} + INTERVAL 1 second
- STEP {{ .Interval }})
+ STEP {{ .Interval }}
+ INTERPOLATE (dimensions AS ['Other', 'Other']))
 {{ end }}`,
 		}, {
 			Description: "no filters, reverse",
@@ -374,7 +382,8 @@ GROUP BY time, dimensions
 ORDER BY time WITH FILL
 FROM {{ .TimefilterStart }}
 TO {{ .TimefilterEnd }} + INTERVAL 1 second
- STEP {{ .Interval }})
+ STEP {{ .Interval }}
+ INTERPOLATE (dimensions AS ['Other', 'Other']))
 {{ end }}
 UNION ALL
 {{ with context @@{"start":"2022-04-10T15:45:10Z","end":"2022-04-11T15:45:10Z","points":100,"units":"l3bps"}@@ }}
@@ -389,7 +398,8 @@ GROUP BY time, dimensions
 ORDER BY time WITH FILL
 FROM {{ .TimefilterStart }}
 TO {{ .TimefilterEnd }} + INTERVAL 1 second
- STEP {{ .Interval }})
+ STEP {{ .Interval }}
+ INTERPOLATE (dimensions AS ['Other', 'Other']))
 {{ end }}`,
 		}, {
 			Description: "no filters, previous period",
@@ -421,7 +431,8 @@ GROUP BY time, dimensions
 ORDER BY time WITH FILL
 FROM {{ .TimefilterStart }}
 TO {{ .TimefilterEnd }} + INTERVAL 1 second
- STEP {{ .Interval }})
+ STEP {{ .Interval }}
+ INTERPOLATE (dimensions AS ['Other', 'Other']))
 {{ end }}
 UNION ALL
 {{ with context @@{"start":"2022-04-09T15:45:10Z","end":"2022-04-10T15:45:10Z","start-for-interval":"2022-04-10T15:45:10Z","points":100,"units":"l3bps"}@@ }}
@@ -436,7 +447,8 @@ GROUP BY time, dimensions
 ORDER BY time WITH FILL
 FROM {{ .TimefilterStart }} + INTERVAL 86400 second
 TO {{ .TimefilterEnd }} + INTERVAL 1 second + INTERVAL 86400 second
- STEP {{ .Interval }})
+ STEP {{ .Interval }}
+ INTERPOLATE (dimensions AS emptyArrayString()))
 {{ end }}`,
 		},
 	}
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -23,7 +23,7 @@ services:
      - 127.0.0.1:9092:9093/tcp

  clickhouse:
-    image: clickhouse/clickhouse-server:22.3
+    image: clickhouse/clickhouse-server:22.8
    ports:
      - 127.0.0.1:8123:8123/tcp
      - 127.0.0.1:9000:9000/tcp
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -60,8 +60,8 @@ services:
  # Choose one of the two options: build from source, or download
  # latest stable release from GitHub.
  akvorado-service: &akvorado-image
-    # build: {context: .}
-    image: ghcr.io/vincentbernat/akvorado:latest
+    build: {context: .}
+    # image: ghcr.io/vincentbernat/akvorado:latest
    command: version

  akvorado-orchestrator:
@@ -130,7 +130,7 @@ services:
    command: demo-exporter http://akvorado-orchestrator:8080#3

  clickhouse:
-    image: clickhouse/clickhouse-server:22.3
+    image: clickhouse/clickhouse-server:22.8
    volumes:
      - ./orchestrator/clickhouse/data/docker-entrypoint.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
      - akvorado-clickhouse:/var/lib/clickhouse