akvorado/configure/clickhouse/migrations.go

package clickhouse

import (
	"akvorado/common/reporter"
	"akvorado/inlet/flow"
	"embed"
	"errors"
	"fmt"
	"net"
	"net/url"
	"strconv"
	"strings"

	"github.com/golang-migrate/migrate/v4"
	"github.com/golang-migrate/migrate/v4/database"
	"github.com/golang-migrate/migrate/v4/source/iofs"
)

//go:embed data/migrations/*.sql data/migrations/*.sql.tmpl
var migrations embed.FS

// migrateDatabase execute database migration. It tries on each
// server. This does not handle clustering. See
// https://github.com/golang-migrate/migrate/pull/568 for the day we
// want to handle clustering properly.
func (c *Component) migrateDatabase() error {
	errs := []error{}
	for _, server := range c.config.Servers {
		if err := c.migrateDatabaseOnServer(server); err != nil {
			errs = append(errs, err)
		}
	}
	if len(errs) == 0 {
		return nil
	}
	return errs[0]
}

// migrateDatabaseOnServer tries to attempt database migration on the provided server
func (c *Component) migrateDatabaseOnServer(server string) error {
	baseURL := c.config.AkvoradoURL
	if baseURL == "" {
		var err error
		if baseURL, err = c.getHTTPBaseURL(server); err != nil {
			return err
		}
	}
	data := map[string]string{
		"KafkaBrokers": strings.Join(c.config.Kafka.Brokers, ","),
		"KafkaTopic":   fmt.Sprintf("%s-v%d", c.config.Kafka.Topic, flow.CurrentSchemaVersion),
		"KafkaThreads": strconv.Itoa(c.config.KafkaThreads),
		"BaseURL":      baseURL,
	}

	l := c.r.With().
		Str("server", server).
		Str("database", c.config.Database).
		Str("username", c.config.Username).
		Logger()

	queryString := url.Values{}
	queryString.Set("database", c.config.Database)
	clickhouseURL := url.URL{
		Scheme:   "clickhouse",
		User:     url.UserPassword(c.config.Username, c.config.Password),
		Host:     server,
		RawQuery: queryString.Encode(),
	}

	templatedMigrations := &templatedFS{data, migrations}
	sourceDriver, err := iofs.New(templatedMigrations, "data/migrations")
	if err != nil {
		l.Err(err).Msg("unable to read migration data")
		return fmt.Errorf("unable to read migration data: %w", err)
	}
	// No call to sourceDriver.Open() (not needed), no defer to
	// sourceDrive.Close() either
	databaseDriver, err := database.Open(clickhouseURL.String())
	if err != nil {
		l.Err(err).Msg("unable to open ClickHouse database")
		return fmt.Errorf("unable to open ClickHouse database: %w", err)
	}
	defer func() {
		if err := databaseDriver.Close(); err != nil {
			l.Err(err).Msg("unable to close Clickhosue database")
		}
	}()

	m, err := migrate.NewWithInstance("iofs", sourceDriver, "clickhouse", databaseDriver)
	if err != nil {
		l.Err(err).Msg("unable to setup migration")
		return fmt.Errorf("unable to setup migration: %w", err)
	}
	m.Log = &migrateLogger{c.r}

	logCurrentVersion := func(why string) {
		currentVersion, dirty, err := m.Version()
		if err != nil && !errors.Is(err, migrate.ErrNilVersion) {
			l.Err(err).Msg("unable to get current version")
			return
		}
		if err != nil {
			currentVersion = 0
			dirty = false
		}
		l.Info().
			Uint("current_version", currentVersion).
			Bool("dirty", dirty).
			Msg(why)
	}

	logCurrentVersion("migration starting")
	err = m.Up()
	if err != nil && !errors.Is(err, migrate.ErrNoChange) {
		l.Err(err).Msg("unable to execute database migration")
		return fmt.Errorf("unable to execute database migration: %w", err)
	}
	if err == nil {
		logCurrentVersion("migration done")
	} else {
		logCurrentVersion("migration already done")
	}
	close(c.migrationsDone)
	return nil
}

// getHTTPBaseURL tries to guess the appropriate URL to access our
// HTTP daemon. It tries to get our IP address using an unconnected
// UDP socket.
func (c *Component) getHTTPBaseURL(address string) (string, error) {
	// Get IP address
	conn, err := net.Dial("udp", address)
	if err != nil {
		return "", fmt.Errorf("cannot get our IP address: %w", err)
	}
	defer conn.Close()
	localAddr := conn.LocalAddr().(*net.UDPAddr)

	// Combine with HTTP port
	_, port, err := net.SplitHostPort(c.d.HTTP.Address.String())
	if err != nil {
		return "", fmt.Errorf("cannot get HTTP port: %w", err)
	}
	base := fmt.Sprintf("http://%s",
		net.JoinHostPort(localAddr.IP.String(), port))
	c.r.Debug().Msgf("detected base URL is %s", base)
	return base, nil
}

type migrateLogger struct {
	r *reporter.Reporter
}

func (l *migrateLogger) Printf(format string, v ...interface{}) {
	if e := l.r.Info(); e.Enabled() {
		e.Msg(fmt.Sprintf(format, v...))
	}
}

func (l *migrateLogger) Verbose() bool {
	return false
}