console: implement "filter/saved" endpoints

cmd/console.go

@@ -11,6 +11,7 @@ import (
 	"akvorado/common/reporter"
 	"akvorado/console"
 	"akvorado/console/authentication"
+	"akvorado/console/database"
 )
 
 // ConsoleConfiguration represents the configuration file for the console command.
@@ -20,6 +21,7 @@ type ConsoleConfiguration struct {
 	Console    console.Configuration
 	ClickHouse clickhousedb.Configuration
 	Auth       authentication.Configuration
+	Database   database.Configuration
 }
 
 // DefaultConsoleConfiguration is the default configuration for the console command.
@@ -30,6 +32,7 @@ func DefaultConsoleConfiguration() ConsoleConfiguration {
 		Console:    console.DefaultConfiguration(),
 		ClickHouse: clickhousedb.DefaultConfiguration(),
 		Auth:       authentication.DefaultConfiguration(),
+		Database:   database.DefaultConfiguration(),
 	}
 }
 
@@ -92,10 +95,16 @@ func consoleStart(r *reporter.Reporter, config ConsoleConfiguration, checkOnly b
 	if err != nil {
 		return fmt.Errorf("unable to initialize authentication component: %w", err)
 	}
+	databaseComponent, err := database.New(r, config.Database)
+	if err != nil {
+		return fmt.Errorf("unable to initialize database component: %w", err)
+	}
 	consoleComponent, err := console.New(r, config.Console, console.Dependencies{
 		Daemon:       daemonComponent,
 		HTTP:         httpComponent,
 		ClickHouseDB: clickhouseComponent,
+		Auth:         authenticationComponent,
+		Database:     databaseComponent,
 	})
 	if err != nil {
 		return fmt.Errorf("unable to initialize console component: %w", err)
@@ -115,6 +124,7 @@ func consoleStart(r *reporter.Reporter, config ConsoleConfiguration, checkOnly b
 		httpComponent,
 		clickhouseComponent,
 		authenticationComponent,
+		databaseComponent,
 		consoleComponent,
 	}
 	return StartStopComponents(r, daemonComponent, components)

common/helpers/tests.go

@@ -39,6 +39,7 @@ func Diff(a, b interface{}) string {
 // HTTPEndpointCases describes case for TestHTTPEndpoints
 type HTTPEndpointCases []struct {
 	Description string
+	Method      string
 	URL         string
 	Header      http.Header
 	JSONInput   interface{}
@@ -64,8 +65,15 @@ func TestHTTPEndpoints(t *testing.T, serverAddr net.Addr, cases HTTPEndpointCase
 			}
 			var resp *http.Response
 			var err error
+			if tc.Method == "" {
+				if tc.JSONInput == nil {
+					tc.Method = "GET"
+				} else {
+					tc.Method = "POST"
+				}
+			}
 			if tc.JSONInput == nil {
-				req, _ := http.NewRequest("GET",
+				req, _ := http.NewRequest(tc.Method,
 					fmt.Sprintf("http://%s%s", serverAddr, tc.URL),
 					nil)
 				if tc.Header != nil {
@@ -73,7 +81,7 @@ func TestHTTPEndpoints(t *testing.T, serverAddr net.Addr, cases HTTPEndpointCase
 				}
 				resp, err = http.DefaultClient.Do(req)
 				if err != nil {
-					t.Fatalf("GET %s:\n%+v", tc.URL, err)
+					t.Fatalf("%s %s:\n%+v", tc.Method, tc.URL, err)
 				}
 			} else {
 				payload := new(bytes.Buffer)
@@ -81,7 +89,7 @@ func TestHTTPEndpoints(t *testing.T, serverAddr net.Addr, cases HTTPEndpointCase
 				if err != nil {
 					t.Fatalf("Encode() error:\n%+v", err)
 				}
-				req, _ := http.NewRequest("POST",
+				req, _ := http.NewRequest(tc.Method,
 					fmt.Sprintf("http://%s%s", serverAddr, tc.URL),
 					payload)
 				if tc.Header != nil {
@@ -90,7 +98,7 @@ func TestHTTPEndpoints(t *testing.T, serverAddr net.Addr, cases HTTPEndpointCase
 				req.Header.Add("Content-Type", "application/json")
 				resp, err = http.DefaultClient.Do(req)
 				if err != nil {
-					t.Fatalf("POST %s:\n%+v", tc.URL, err)
+					t.Fatalf("%s %s:\n%+v", tc.Method, tc.URL, err)
 				}
 			}
 

console/assets_test.go

@@ -18,8 +18,7 @@ func TestServeAssets(t *testing.T) {
 		t.Run(name, func(t *testing.T) {
 			conf := DefaultConfiguration()
 			conf.ServeLiveFS = live
-			c, h, _, _ := NewMock(t, conf)
-			helpers.StartStop(t, c)
+			_, h, _, _ := NewMock(t, conf)
 
 			helpers.TestHTTPEndpoints(t, h.Address, helpers.HTTPEndpointCases{
 				{

console/clickhouse_test.go

@@ -169,7 +169,6 @@ func TestQueryFlowsTables(t *testing.T) {
 	}
 
 	c, _, _, _ := NewMock(t, DefaultConfiguration())
-	helpers.StartStop(t, c)
 	for _, tc := range cases {
 		t.Run(tc.Description, func(t *testing.T) {
 			c.flowsTables = tc.Tables

console/data/docs/02-configuration.md

@@ -331,9 +331,9 @@ resolutions:
 
 ## Console service
 
-The main components of the console service are `http` and `console`.
-`http` accepts the [same configuration](#http) as for the inlet
-service.
+The main components of the console service are `http`, `console`,
+`authentication` and `database`. `http` accepts the [same
+configuration](#http) as for the inlet service.
 
 ### Authentication
 
@@ -352,16 +352,15 @@ is also possible to modify the default user (when no header is
 present) by tweaking the `default-user` key:
 
 ```yaml
-console:
-  authentication:
-    headers:
-      login: Remote-User
-      name: Remote-Name
-      email: Remote-Email
-      logout-url: X-Logout-URL
-    default-user:
-      login: default
-      name: Default User
+authentication:
+  headers:
+    login: Remote-User
+    name: Remote-Name
+    email: Remote-Email
+    logout-url: X-Logout-URL
+  default-user:
+    login: default
+    name: Default User
 ```
 
 To prevent access when not authenticated, the `login` field for the
@@ -382,3 +381,16 @@ There also exist simpler solutions only providing authentication:
 
 - [OAuth2 Proxy](https://oauth2-proxy.github.io/oauth2-proxy/), associated with [Dex](https://dexidp.io/)
 - [Ory](https://www.ory.sh), notably Hydra and Oathkeeper
+
+### Database
+
+The console stores some data, like per-user filters, into a relational
+database. When the database is not configured, data is only stored in
+memory and will be lost on restart. Currently, the only accepted
+driver is SQLite.
+
+```yaml
+database:
+  driver: sqlite
+  dsn: /var/lib/akvorado/console.sqlite
+```

console/database/config.go

@@ -0,0 +1,17 @@
+package database
+
+// Configuration describes the configuration for the authentication component.
+type Configuration struct {
+	// Driver defines the driver for the database
+	Driver string
+	// DSN defines the DSN to connect to the database
+	DSN string
+}
+
+// DefaultConfiguration represents the default configuration for the console component.
+func DefaultConfiguration() Configuration {
+	return Configuration{
+		Driver: "sqlite",
+		DSN:    "file::memory:?cache=shared",
+	}
+}

console/database/logs.go

@@ -0,0 +1,47 @@
+package database
+
+import (
+	"context"
+	"time"
+
+	gormlogger "gorm.io/gorm/logger"
+	"gorm.io/gorm/utils"
+
+	"akvorado/common/reporter"
+)
+
+type logger struct {
+	r *reporter.Reporter
+}
+
+func (l *logger) LogMode(gormlogger.LogLevel) gormlogger.Interface {
+	return l
+}
+
+func (l *logger) Info(ctx context.Context, s string, args ...interface{}) {
+	l.r.Info().Msgf(s, args...)
+}
+
+func (l *logger) Warn(ctx context.Context, s string, args ...interface{}) {
+	l.r.Warn().Msgf(s, args...)
+}
+
+func (l *logger) Error(ctx context.Context, s string, args ...interface{}) {
+	l.r.Error().Msgf(s, args...)
+}
+
+func (l *logger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) {
+	elapsed := time.Since(begin)
+	sql, _ := fc()
+	fields := map[string]interface{}{
+		"sql":      sql,
+		"duration": elapsed,
+		"source":   utils.FileWithLineNum(),
+	}
+	if err != nil {
+		l.r.Error().Err(err).Fields(fields).Msg("SQL query error")
+		return
+	}
+
+	l.r.Debug().Fields(fields).Msg("SQL query")
+}

console/database/root.go

@@ -0,0 +1,61 @@
+package database
+
+import (
+	"fmt"
+
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+
+	"akvorado/common/reporter"
+)
+
+// Component represents the database compomenent.
+type Component struct {
+	r      *reporter.Reporter
+	config Configuration
+
+	db *gorm.DB
+}
+
+// New creates a new database component.
+func New(r *reporter.Reporter, configuration Configuration) (*Component, error) {
+	c := Component{
+		r:      r,
+		config: configuration,
+	}
+	switch c.config.Driver {
+	case "sqlite":
+		db, err := gorm.Open(sqlite.Open(c.config.DSN), &gorm.Config{
+			Logger: &logger{r},
+		})
+		if err != nil {
+			return nil, fmt.Errorf("unable to open database: %w", err)
+		}
+		c.db = db
+	default:
+		return nil, fmt.Errorf("%q is not a supporter driver", c.config.Driver)
+	}
+	return &c, nil
+}
+
+// Start starts the database component
+func (c *Component) Start() error {
+	c.r.Info().Msg("starting database component")
+	if err := c.db.AutoMigrate(&SavedFilter{}); err != nil {
+		return fmt.Errorf("cannot migrate database: %w", err)
+	}
+	return nil
+}
+
+// Stop stops the database component.
+func (c *Component) Stop() error {
+	defer c.r.Info().Msg("database component stopped")
+	if c.db != nil {
+		sqlDB, err := c.db.DB()
+		if err != nil {
+			return err
+		}
+		return sqlDB.Close()
+	}
+	return nil
+}

console/database/saved_filters.go

@@ -0,0 +1,47 @@
+package database
+
+import (
+	"context"
+	"errors"
+	"fmt"
+)
+
+// SavedFilter represents a saved filter in database.
+type SavedFilter struct {
+	ID          uint   `json:"id"`
+	User        string `gorm:"index" json:"user"`
+	Shared      bool   `json:"shared"`
+	Description string `json:"description" binding:"required"`
+	Content     string `json:"content" binding:"required"`
+}
+
+// CreateSavedFilter creates a new saved filter in database.
+func (c *Component) CreateSavedFilter(ctx context.Context, f SavedFilter) error {
+	result := c.db.WithContext(ctx).Omit("ID").Create(&f)
+	if result.Error != nil {
+		return fmt.Errorf("unable to create new saved filter: %w", result.Error)
+	}
+	return nil
+}
+
+// ListSavedFilters list all saved filters for the provided user
+func (c *Component) ListSavedFilters(ctx context.Context, user string) ([]SavedFilter, error) {
+	var results []SavedFilter
+	result := c.db.WithContext(ctx).Where(&SavedFilter{User: user}).Find(&results)
+	if result.Error != nil {
+		return nil, fmt.Errorf("unable to retrieve saved filters: %w", result.Error)
+	}
+	return results, nil
+}
+
+// DeleteSavedFilter deletes the provided saved filter
+func (c *Component) DeleteSavedFilter(ctx context.Context, f SavedFilter) error {
+	result := c.db.WithContext(ctx).Where(&SavedFilter{User: f.User}).Delete(&f)
+	if result.Error != nil {
+		return fmt.Errorf("cannot delete saved filter: %w", result.Error)
+	}
+	if result.RowsAffected == 0 {
+		return errors.New("no matching saved filter to delete")
+	}
+	return nil
+}

console/database/saved_filters_test.go

@@ -0,0 +1,84 @@
+package database
+
+import (
+	"context"
+	"testing"
+
+	"akvorado/common/helpers"
+	"akvorado/common/reporter"
+)
+
+func TestSavedFilter(t *testing.T) {
+	r := reporter.NewMock(t)
+	c := NewMock(t, r)
+
+	// Create
+	if err := c.CreateSavedFilter(context.Background(), SavedFilter{
+		ID:          17,
+		User:        "marty",
+		Shared:      false,
+		Description: "marty's filter",
+		Content:     "SrcAS = 12322",
+	}); err != nil {
+		t.Fatalf("CreateSavedFilter() error:\n%+v", err)
+	}
+	if err := c.CreateSavedFilter(context.Background(), SavedFilter{
+		User:        "judith",
+		Shared:      true,
+		Description: "judith's filter",
+		Content:     "InIfBoundary = external",
+	}); err != nil {
+		t.Fatalf("CreateSavedFilter() error:\n%+v", err)
+	}
+	if err := c.CreateSavedFilter(context.Background(), SavedFilter{
+		User:        "marty",
+		Shared:      true,
+		Description: "marty's second filter",
+		Content:     "InIfBoundary = internal",
+	}); err != nil {
+		t.Fatalf("CreateSavedFilter() error:\n%+v", err)
+	}
+
+	// List
+	got, err := c.ListSavedFilters(context.Background(), "marty")
+	if err != nil {
+		t.Fatalf("ListSavedFilters() error:\n%+v", err)
+	}
+	if diff := helpers.Diff(got, []SavedFilter{
+		{
+			ID:          1,
+			User:        "marty",
+			Shared:      false,
+			Description: "marty's filter",
+			Content:     "SrcAS = 12322",
+		}, {
+			ID:          3,
+			User:        "marty",
+			Shared:      true,
+			Description: "marty's second filter",
+			Content:     "InIfBoundary = internal",
+		},
+	}); diff != "" {
+		t.Fatalf("ListSavedFilters() (-got, +want):\n%s", diff)
+	}
+
+	// Delete
+	if err := c.DeleteSavedFilter(context.Background(), SavedFilter{ID: 1}); err != nil {
+		t.Fatalf("DeleteSavedFilter() error:\n%+v", err)
+	}
+	got, _ = c.ListSavedFilters(context.Background(), "marty")
+	if diff := helpers.Diff(got, []SavedFilter{
+		{
+			ID:          3,
+			User:        "marty",
+			Shared:      true,
+			Description: "marty's second filter",
+			Content:     "InIfBoundary = internal",
+		},
+	}); diff != "" {
+		t.Fatalf("ListSavedFilters() (-got, +want):\n%s", diff)
+	}
+	if err := c.DeleteSavedFilter(context.Background(), SavedFilter{ID: 1}); err == nil {
+		t.Fatal("DeleteSavedFilter() no error")
+	}
+}

console/database/tests.go

@@ -0,0 +1,21 @@
+//go:build !release
+
+package database
+
+import (
+	"testing"
+
+	"akvorado/common/helpers"
+	"akvorado/common/reporter"
+)
+
+// NewMock instantiantes a new authentication component
+func NewMock(t *testing.T, r *reporter.Reporter) *Component {
+	t.Helper()
+	c, err := New(r, DefaultConfiguration())
+	if err != nil {
+		t.Fatalf("New() error:\n%+v", err)
+	}
+	helpers.StartStop(t, c)
+	return c
+}

console/docs_test.go

@@ -6,8 +6,6 @@ import (
 	netHTTP "net/http"
 	"strings"
 	"testing"
-
-	"akvorado/common/helpers"
 )
 
 func TestServeDocs(t *testing.T) {
@@ -27,8 +25,7 @@ func TestServeDocs(t *testing.T) {
 			t.Run(fmt.Sprintf("%s-%s", name, tc.Path), func(t *testing.T) {
 				conf := DefaultConfiguration()
 				conf.ServeLiveFS = live
-				c, h, _, _ := NewMock(t, conf)
-				helpers.StartStop(t, c)
+				_, h, _, _ := NewMock(t, conf)
 
 				resp, err := netHTTP.Get(fmt.Sprintf("http://%s/api/v0/console/docs/%s",
 					h.Address, tc.Path))

console/filter.go

@@ -3,11 +3,14 @@ package console
 import (
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 
 	"github.com/gin-gonic/gin"
 
 	"akvorado/common/helpers"
+	"akvorado/console/authentication"
+	"akvorado/console/database"
 	"akvorado/console/filter"
 )
 
@@ -268,3 +271,51 @@ LIMIT 20`, column, column, column, column, column)
 	gc.JSON(http.StatusOK, filterCompleteHandlerOutput{filteredCompletions})
 	return
 }
+
+func (c *Component) filterSavedListHandlerFunc(gc *gin.Context) {
+	ctx := c.t.Context(gc.Request.Context())
+	user := gc.MustGet("user").(authentication.UserInformation).Login
+	filters, err := c.d.Database.ListSavedFilters(ctx, user)
+	if err != nil {
+		c.r.Err(err).Msg("unable to list filters")
+		gc.JSON(http.StatusInternalServerError, gin.H{"message": "unable to list filters"})
+		return
+	}
+	gc.JSON(http.StatusOK, gin.H{"filters": filters})
+}
+
+func (c *Component) filterSavedDeleteHandlerFunc(gc *gin.Context) {
+	ctx := c.t.Context(gc.Request.Context())
+	user := gc.MustGet("user").(authentication.UserInformation).Login
+	id, err := strconv.ParseUint(gc.Param("id"), 10, 64)
+	if err != nil {
+		gc.JSON(http.StatusBadRequest, gin.H{"message": "bad ID format"})
+		return
+	}
+	if err := c.d.Database.DeleteSavedFilter(ctx, database.SavedFilter{
+		ID:   uint(id),
+		User: user,
+	}); err != nil {
+		// Assume this is because it is not found
+		gc.JSON(http.StatusNotFound, gin.H{"message": "filter not found"})
+		return
+	}
+	gc.JSON(http.StatusNoContent, nil)
+}
+
+func (c *Component) filterSavedAddHandlerFunc(gc *gin.Context) {
+	ctx := c.t.Context(gc.Request.Context())
+	user := gc.MustGet("user").(authentication.UserInformation).Login
+	var filter database.SavedFilter
+	if err := gc.ShouldBindJSON(&filter); err != nil {
+		gc.JSON(http.StatusBadRequest, gin.H{"message": helpers.Capitalize(err.Error())})
+		return
+	}
+	filter.User = user
+	if err := c.d.Database.CreateSavedFilter(ctx, filter); err != nil {
+		c.r.Err(err).Msg("cannot create saved filter")
+		gc.JSON(http.StatusInternalServerError, gin.H{"message": "cannot create new filter"})
+		return
+	}
+	gc.JSON(http.StatusNoContent, nil)
+}

console/filter_test.go

@@ -1,6 +1,7 @@
 package console
 
 import (
+	netHTTP "net/http"
 	"testing"
 
 	"github.com/gin-gonic/gin"
@@ -10,8 +11,7 @@ import (
 )
 
 func TestFilterHandlers(t *testing.T) {
-	c, h, mockConn, _ := NewMock(t, DefaultConfiguration())
-	helpers.StartStop(t, c)
+	_, h, mockConn, _ := NewMock(t, DefaultConfiguration())
 
 	mockConn.EXPECT().
 		Select(gomock.Any(), gomock.Any(), `
@@ -186,6 +186,69 @@ UNION DISTINCT
 				{"label": "customer-2", "detail": "network name", "quoted": true},
 				{"label": "customer-3", "detail": "network name", "quoted": true},
 			}},
+		}, {
+			Description: "list, no filters",
+			URL:         "/api/v0/console/filter/saved",
+			StatusCode:  200,
+			JSONOutput:  gin.H{"filters": []gin.H{}},
+		}, {
+			Description: "store one filter",
+			URL:         "/api/v0/console/filter/saved",
+			StatusCode:  204,
+			JSONInput: gin.H{
+				"description": "test 1",
+				"content":     "InIfBoundary = external",
+			},
+			ContentType: "application/json; charset=utf-8",
+		}, {
+			Description: "list stored filters",
+			URL:         "/api/v0/console/filter/saved",
+			JSONOutput: gin.H{"filters": []gin.H{
+				{
+					"id":          1,
+					"shared":      false,
+					"user":        "__default",
+					"description": "test 1",
+					"content":     "InIfBoundary = external",
+				},
+			}},
+		}, {
+			Description: "list stored filters as another user",
+			URL:         "/api/v0/console/filter/saved",
+			Header: func() netHTTP.Header {
+				headers := make(netHTTP.Header)
+				headers.Add("Remote-User", "alfred")
+				return headers
+			}(),
+			JSONOutput: gin.H{"filters": []gin.H{}},
+		}, {
+			Description: "delete stored filter as another user",
+			Method:      "DELETE",
+			URL:         "/api/v0/console/filter/saved/1",
+			Header: func() netHTTP.Header {
+				headers := make(netHTTP.Header)
+				headers.Add("Remote-User", "alfred")
+				return headers
+			}(),
+			StatusCode: 404,
+			JSONOutput: gin.H{"message": "filter not found"},
+		}, {
+			Description: "delete stored filter",
+			Method:      "DELETE",
+			URL:         "/api/v0/console/filter/saved/1",
+			StatusCode:  204,
+			ContentType: "application/json; charset=utf-8",
+		}, {
+			Description: "delete stored filter with invalid ID",
+			Method:      "DELETE",
+			URL:         "/api/v0/console/filter/saved/kjgdfhgh",
+			StatusCode:  400,
+			JSONOutput:  gin.H{"message": "bad ID format"},
+		}, {
+			Description: "list stored filter after delete",
+			URL:         "/api/v0/console/filter/saved",
+			StatusCode:  200,
+			JSONOutput:  gin.H{"filters": []gin.H{}},
 		},
 	})
 }

console/graph_test.go

@@ -140,8 +140,7 @@ ORDER BY time`,
 }
 
 func TestGraphHandler(t *testing.T) {
-	c, h, mockConn, _ := NewMock(t, DefaultConfiguration())
-	helpers.StartStop(t, c)
+	_, h, mockConn, _ := NewMock(t, DefaultConfiguration())
 
 	base := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
 	expectedSQL := []struct {

console/root.go

@@ -17,6 +17,7 @@ import (
 	"akvorado/common/http"
 	"akvorado/common/reporter"
 	"akvorado/console/authentication"
+	"akvorado/console/database"
 
 	"github.com/benbjohnson/clock"
 	"gopkg.in/tomb.v2"
@@ -47,6 +48,7 @@ type Dependencies struct {
 	ClickHouseDB *clickhousedb.Component
 	Clock        clock.Clock
 	Auth         *authentication.Component
+	Database     *database.Component
 }
 
 // New creates a new console component.
@@ -88,6 +90,9 @@ func (c *Component) Start() error {
 	endpoint.POST("/sankey", c.sankeyHandlerFunc)
 	endpoint.POST("/filter/validate", c.filterValidateHandlerFunc)
 	endpoint.POST("/filter/complete", c.filterCompleteHandlerFunc)
+	endpoint.GET("/filter/saved", c.filterSavedListHandlerFunc)
+	endpoint.DELETE("/filter/saved/:id", c.filterSavedDeleteHandlerFunc)
+	endpoint.POST("/filter/saved", c.filterSavedAddHandlerFunc)
 	endpoint.GET("/user/info", c.d.Auth.UserInfoHandlerFunc)
 	endpoint.GET("/user/avatar", c.d.Auth.UserAvatarHandlerFunc)
 

console/sankey_test.go

@@ -118,8 +118,7 @@ ORDER BY xps DESC`,
 }
 
 func TestSankeyHandler(t *testing.T) {
-	c, h, mockConn, _ := NewMock(t, DefaultConfiguration())
-	helpers.StartStop(t, c)
+	_, h, mockConn, _ := NewMock(t, DefaultConfiguration())
 
 	expectedSQL := []struct {
 		Xps        float64  `ch:"xps"`

console/tests.go

@@ -10,9 +10,11 @@ import (
 	"akvorado/common/clickhousedb"
 	"akvorado/common/clickhousedb/mocks"
 	"akvorado/common/daemon"
+	"akvorado/common/helpers"
 	"akvorado/common/http"
 	"akvorado/common/reporter"
 	"akvorado/console/authentication"
+	"akvorado/console/database"
 )
 
 // NewMock instantiantes a new authentication component
@@ -28,9 +30,11 @@ func NewMock(t *testing.T, config Configuration) (*Component, *http.Component, *
 		ClickHouseDB: ch,
 		Clock:        mockClock,
 		Auth:         authentication.NewMock(t, r),
+		Database:     database.NewMock(t, r),
 	})
 	if err != nil {
 		t.Fatalf("New() error:\n%+v", err)
 	}
+	helpers.StartStop(t, c)
 	return c, h, mockConn, mockClock
 }

console/widgets_test.go

@@ -15,8 +15,7 @@ import (
 )
 
 func TestWidgetLastFlow(t *testing.T) {
-	c, h, mockConn, _ := NewMock(t, DefaultConfiguration())
-	helpers.StartStop(t, c)
+	_, h, mockConn, _ := NewMock(t, DefaultConfiguration())
 
 	ctrl := gomock.NewController(t)
 	mockRows := mocks.NewMockRows(ctrl)
@@ -91,8 +90,7 @@ func TestWidgetLastFlow(t *testing.T) {
 }
 
 func TestFlowRate(t *testing.T) {
-	c, h, mockConn, _ := NewMock(t, DefaultConfiguration())
-	helpers.StartStop(t, c)
+	_, h, mockConn, _ := NewMock(t, DefaultConfiguration())
 
 	ctrl := gomock.NewController(t)
 	mockRow := mocks.NewMockRow(ctrl)
@@ -115,8 +113,7 @@ func TestFlowRate(t *testing.T) {
 }
 
 func TestWidgetExporters(t *testing.T) {
-	c, h, mockConn, _ := NewMock(t, DefaultConfiguration())
-	helpers.StartStop(t, c)
+	_, h, mockConn, _ := NewMock(t, DefaultConfiguration())
 
 	expected := []struct {
 		ExporterName string
@@ -146,8 +143,7 @@ func TestWidgetExporters(t *testing.T) {
 }
 
 func TestWidgetTop(t *testing.T) {
-	c, h, mockConn, _ := NewMock(t, DefaultConfiguration())
-	helpers.StartStop(t, c)
+	_, h, mockConn, _ := NewMock(t, DefaultConfiguration())
 
 	gomock.InOrder(
 		mockConn.EXPECT().
@@ -214,8 +210,7 @@ func TestWidgetTop(t *testing.T) {
 }
 
 func TestWidgetGraph(t *testing.T) {
-	c, h, mockConn, mockClock := NewMock(t, DefaultConfiguration())
-	helpers.StartStop(t, c)
+	_, h, mockConn, mockClock := NewMock(t, DefaultConfiguration())
 
 	base := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
 	mockClock.Set(base.Add(24 * time.Hour))

docker-compose.yml

@@ -24,8 +24,6 @@ services:
 
   clickhouse:
     image: clickhouse/clickhouse-server:22.2
-    depends_on:
-      - kafka
     ports:
       - 127.0.0.1:8123:8123/tcp
       - 127.0.0.1:9000:9000/tcp

go.mod

@@ -31,6 +31,8 @@ require (
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
 	gopkg.in/yaml.v2 v2.4.0
+	gorm.io/driver/sqlite v1.3.4
+	gorm.io/gorm v1.23.5
 )
 
 require (
@@ -59,9 +61,12 @@ require (
 	github.com/jcmturner/gofork v1.0.0 // indirect
 	github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.15.1 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
+	github.com/mattn/go-sqlite3 v1.14.12 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect

go.sum

@@ -327,6 +327,11 @@ github.com/jcmturner/gokrb5/v8 v8.4.2 h1:6ZIM6b/JJN0X8UM43ZOM6Z4SJzla+a/u7scXFJz
 github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc=
 github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -382,6 +387,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ1Z0=
+github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
@@ -1008,6 +1015,11 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/sqlite v1.3.4 h1:NnFOPVfzi4CPsJPH4wXr6rMkPb4ElHEqKMvrsx9c9Fk=
+gorm.io/driver/sqlite v1.3.4/go.mod h1:B+8GyC9K7VgzJAcrcXMRPdnMcck+8FgJynEehEPM16U=
+gorm.io/gorm v1.23.4/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
+gorm.io/gorm v1.23.5 h1:TnlF26wScKSvknUC/Rn8t0NLLM22fypYBlvj1+aH6dM=
+gorm.io/gorm v1.23.5/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=