johanels/akvorado
1
0
Fork 0
mirror of https://github.com/akvorado/akvorado.git synced 2025-12-12 06:24:10 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

*/kafka: switch to franz-go

Browse Source 
The concurrency of this library is easier to handle than Sarama.
Notably, it is more compatible with the new model of "almost share
nothing" we use for the inlet and the outlet. The lock for workers in
outlet is removed. We can now use sync.Pool to allocate slice of bytes
in inlet.

It may also be more performant.

In the future, we may want to commit only when pushing data to
ClickHouse. However, this does not seem easy when there is a rebalance.
In case of rebalance, we need to do something when a partition is
revoked to avoid duplicating data. For example, we could flush the
current batch to ClickHouse. Have a look at the
`example/mark_offsets/main.go` file in franz-go repository for a
possible approach. In the meantime, we rely on autocommit.

Another contender could be https://github.com/segmentio/kafka-go. Also
see https://github.com/twmb/franz-go/pull/1064.
This commit is contained in:
Vincent Bernat
2025-07-20 07:33:06 +02:00
parent 872bda0501
commit 756e4a8fbd
33 changed files with 871 additions and 1107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
common/kafka/oauth_test.go
View File

@@ -12,7 +12,7 @@ import (
"testing"
"time"
"github.com/IBM/sarama"
"github.com/twmb/franz-go/pkg/kgo"
"golang.org/x/oauth2"
"golang.org/x/oauth2/clientcredentials"
)
@@ -77,8 +77,6 @@ func TestOAuth2ServerClientCredentials(t *testing.T) {
func TestOAuth2Broker(t *testing.T) {
r := reporter.NewMock(t)
GlobalKafkaLogger.Register(r)
defer GlobalKafkaLogger.Unregister()
// Ensure broker is ready.
SetupKafkaBroker(t)
@@ -90,25 +88,27 @@ func TestOAuth2Broker(t *testing.T) {
[]string{"kafka:9093", "127.0.0.1:9093"})
config := DefaultConfiguration()
config.Brokers = []string{broker}
config.SASL = SASLConfiguration{
Username: "kafka-client",
Password: "kafka-client-secret",
Mechanism: SASLOauth,
OAuthTokenURL: fmt.Sprintf("http://%s/default/token", oauthServer),
}
kafkaConfig, err := NewConfig(config)
opts, err := NewConfig(r, config)
if err != nil {
t.Fatalf("NewConfig() error:\n%+v", err)
}
if err := kafkaConfig.Validate(); err != nil {
t.Fatalf("Validate() error:\n%+v", err)
}
client, err := sarama.NewClient([]string{broker}, kafkaConfig)
client, err := kgo.NewClient(opts...)
if err != nil {
t.Fatalf("sarama.NewClient() error:\n%+v", err)
t.Fatalf("kgo.NewClient() error:\n%+v", err)
}
if err := client.RefreshMetadata(); err != nil {
t.Fatalf("client.RefreshMetadata() error:\n%+v", err)
defer client.Close()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := client.Ping(ctx); err != nil {
t.Fatalf("client.Ping() error:\n%+v", err)
}
}