mirror of
https://github.com/akvorado/akvorado.git
synced 2025-12-12 06:24:10 +01:00
e5a625aecf
Inserting into ClickHouse should be done in large batches to minimize the number of parts created. This would require the user to tune the number of Kafka workers to match a target of around 50k-100k rows. Instead, we dynamically tune the number of workers depending on the load to reach this target. We keep using async if we are too low in number of flows. It is still possible to do better by consolidating batches from various workers, but that's something I wanted to avoid. Also, increase the maximum wait time to 5 seconds. It should be good enough for most people. Fix #1885
58 lines
1.1 KiB
Go
58 lines
1.1 KiB
Go
// SPDX-FileCopyrightText: 2024 Free Mobile
|
|
// SPDX-License-Identifier: AGPL-3.0-only
|
|
|
|
//go:build !release
|
|
|
|
package kafka
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
)
|
|
|
|
type mockComponent struct {
|
|
config Configuration
|
|
incoming chan []byte
|
|
}
|
|
|
|
// NewMock instantiates a fake Kafka consumer that will produce messages sent on
|
|
// the returned channel.
|
|
func NewMock(_ *testing.T, config Configuration) (Component, chan<- []byte) {
|
|
c := mockComponent{
|
|
config: config,
|
|
incoming: make(chan []byte),
|
|
}
|
|
return &c, c.incoming
|
|
}
|
|
|
|
// StartWorkers start a set of workers to produce received messages.
|
|
func (c *mockComponent) StartWorkers(workerBuilder WorkerBuilderFunc) error {
|
|
ch := make(chan ScaleRequest, 10)
|
|
go func() {
|
|
for {
|
|
// Ignore all incoming scaling requests
|
|
<-ch
|
|
}
|
|
}()
|
|
for i := range c.config.MinWorkers {
|
|
callback, shutdown := workerBuilder(i, ch)
|
|
defer shutdown()
|
|
go func() {
|
|
for {
|
|
message, ok := <-c.incoming
|
|
if !ok {
|
|
return
|
|
}
|
|
callback(context.Background(), message)
|
|
}
|
|
}()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Stop stops the mock component.
|
|
func (c *mockComponent) Stop() error {
|
|
close(c.incoming)
|
|
return nil
|
|
}
|