common/embed: replace all go:embed use by an embedded archive

Some of the files were quite big:

- asns.csv ~ 3 MB
- index.js ~ 1.5 MB
- *.svg ~ 2 MB

Use a ZIP archive to put them all and embed it. This reduce the binary
size from 89 MB to 82 MB. 🤯

This also pulls some code modernization (use of http.ServeFileFS).

.github/workflows/ci.yml

@@ -57,7 +57,7 @@ jobs:
       - name: Setup
         uses: ./.github/actions/setup
       - name: Install dependencies
-        run: sudo apt-get install -qqy shared-mime-info curl
+        run: sudo apt-get install -qqy shared-mime-info curl zip
 
       # Regular tests
       - name: Go race tests

.gitignore

@@ -5,6 +5,7 @@
 /orchestrator/clickhouse/data/udp.csv
 /console/filter/parser.go
 /common/schema/definition_gen.go
+/common/embed/data/embed.zip
 mock_*.go
 *_enumer.go
 *.pb.go

.gitlab-ci.yml

@@ -22,7 +22,7 @@ run tests:
     # past but this is a slight burden to maintain in addition to GitHub CI.
     # Check commit ceaa6ebf8ef6 for the last version supporting functional
     # tests.
-    - time apk add --no-cache git make gcc musl-dev shared-mime-info npm curl
+    - time apk add --no-cache git make gcc musl-dev shared-mime-info npm curl zip
     - export GOMODCACHE=$PWD/.go-cache
     - npm config --user set cache $PWD/.npm-cache
     - time go mod download

Makefile

@@ -38,7 +38,8 @@ GENERATED_TEST_GO = \
 GENERATED = \
 	$(GENERATED_GO) \
 	$(GENERATED_JS) \
-	console/data/frontend
+	console/data/frontend \
+	common/embed/data/embed.zip
 
 .PHONY: all all-indep
 BUILD_ARGS =
@@ -153,6 +154,11 @@ default-%.pgo:
 	   [ -n $$ip ] ; \
 	   curl -so $@ "http://$$ip:8080/debug/pprof/profile?seconds=30"
 
+common/embed/data/embed.zip: console/data/frontend console/authentication/data/avatars console/data/docs
+common/embed/data/embed.zip: orchestrator/clickhouse/data/protocols.csv orchestrator/clickhouse/data/icmp.csv orchestrator/clickhouse/data/asns.csv orchestrator/clickhouse/data/tcp.csv orchestrator/clickhouse/data/udp.csv
+common/embed/data/embed.zip: ; $(info $(M) generate embed.zip…)
+	$Q mkdir -p common/embed/data && zip --quiet --recurse-paths --filesync $@ $^
+
 # Tests
 
 .PHONY: check test tests test-race test-short test-bench test-coverage

common/embed/fs.go

@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: 2025 Free Mobile
+// SPDX-License-Identifier: AGPL-3.0-only
+
+// Package embed provides access to the compressed archive containing all the
+// embedded files.
+package embed
+
+import (
+	"archive/zip"
+	"bytes"
+	_ "embed"
+	"fmt"
+	"io/fs"
+	"sync"
+)
+
+var (
+	//go:embed data/embed.zip
+	embeddedZip []byte
+	data        fs.FS
+	dataOnce    sync.Once
+	dataReady   chan struct{}
+)
+
+// Data returns a filesystem with the files contained in the embedded archive.
+func Data() fs.FS {
+	dataOnce.Do(func() {
+		r, err := zip.NewReader(bytes.NewReader(embeddedZip), int64(len(embeddedZip)))
+		if err != nil {
+			panic(fmt.Sprintf("cannot read embedded archive: %s", err))
+		}
+		data = r
+		close(dataReady)
+	})
+	<-dataReady
+	return data
+}
+
+func init() {
+	dataReady = make(chan struct{})
+}

common/embed/fs_test.go

@@ -0,0 +1,28 @@
+// SPDX-FileCopyrightText: 2025 Free Mobile
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package embed_test
+
+import (
+	"io"
+	"testing"
+
+	"akvorado/common/embed"
+	"akvorado/common/helpers"
+)
+
+func TestData(t *testing.T) {
+	f, err := embed.Data().Open("orchestrator/clickhouse/data/protocols.csv")
+	if err != nil {
+		t.Fatalf("Open() error:\n%+v", err)
+	}
+	expected := "proto,name,description"
+	got := make([]byte, len(expected))
+	_, err = io.ReadFull(f, got)
+	if err != nil {
+		t.Fatalf("ReadFull() error:\n%+v", err)
+	}
+	if diff := helpers.Diff(string(got), expected); diff != "" {
+		t.Fatalf("ReadFull() (-got, +want):\n%s", diff)
+	}
+}

console/assets.go

@@ -4,31 +4,20 @@
 package console
 
 import (
-	"embed"
 	"net/http"
-	"time"
 )
 
-//go:embed data/frontend
-var embeddedAssets embed.FS
-
 func (c *Component) defaultHandlerFunc(w http.ResponseWriter, req *http.Request) {
-	assets := c.embedOrLiveFS(embeddedAssets, "data/frontend")
+	assets := c.embedOrLiveFS("data/frontend")
-	f, err := http.FS(assets).Open("index.html")
+	http.ServeFileFS(w, req, assets, "index.html")
-	if err != nil {
-		http.Error(w, "Application not found.", http.StatusInternalServerError)
-		return
-	}
-	http.ServeContent(w, req, "index.html", time.Time{}, f)
-	f.Close()
 }
 
 func (c *Component) staticAssetsHandlerFunc(w http.ResponseWriter, req *http.Request) {
-	assets := c.embedOrLiveFS(embeddedAssets, "data/frontend/assets")
+	assets := c.embedOrLiveFS("data/frontend/assets")
 	http.FileServer(http.FS(assets)).ServeHTTP(w, req)
 }
 
 func (c *Component) docAssetsHandlerFunc(w http.ResponseWriter, req *http.Request) {
-	docs := c.embedOrLiveFS(embeddedDocs, "data/docs")
+	docs := c.embedOrLiveFS("data/docs")
 	http.FileServer(http.FS(docs)).ServeHTTP(w, req)
 }

console/authentication/handlers.go

@@ -5,7 +5,6 @@ package authentication
 
 import (
 	"bufio"
-	"embed"
 	"fmt"
 	"hash/fnv"
 	"image"
@@ -15,12 +14,11 @@ import (
 	"math/rand"
 	"net/http"
 
+	"akvorado/common/embed"
+
 	"github.com/gin-gonic/gin"
 )
 
-//go:embed data/avatars
-var avatarParts embed.FS
-
 // UserInfoHandlerFunc returns the information about the currently logged user.
 func (c *Component) UserInfoHandlerFunc(gc *gin.Context) {
 	info := gc.MustGet("user").(UserInformation)
@@ -44,6 +42,12 @@ func (c *Component) UserAvatarHandlerFunc(gc *gin.Context) {
 	}
 
 	// Grab list of parts
+	avatarParts, err := fs.Sub(embed.Data(), "console/authentication")
+	if err != nil {
+		c.r.Err(err).Msg("cannot open embedded archive")
+		gc.JSON(http.StatusInternalServerError, gin.H{"message": "Cannot build avatar."})
+		return
+	}
 	parts := []string{}
 	partList, err := avatarParts.Open("data/avatars/partlist.txt")
 	if err != nil {

console/docs.go

@@ -5,7 +5,6 @@ package console
 
 import (
 	"bytes"
-	"embed"
 	"fmt"
 	"io"
 	"io/fs"
@@ -24,8 +23,6 @@ import (
 )
 
 var (
-	//go:embed data/docs
-	embeddedDocs       embed.FS
 	internalLinkRegexp = regexp.MustCompile("^(([0-9]+)-([a-z]+).md)(#.*|$)")
 )
 
@@ -43,7 +40,7 @@ type DocumentTOC struct {
 }
 
 func (c *Component) docsHandlerFunc(gc *gin.Context) {
-	docs := c.embedOrLiveFS(embeddedDocs, "data/docs")
+	docs := c.embedOrLiveFS("data/docs")
 	requestedDocument := gc.Param("name")
 
 	var markdown []byte

console/root.go

@@ -19,6 +19,7 @@ import (
 
 	"akvorado/common/clickhousedb"
 	"akvorado/common/daemon"
+	"akvorado/common/embed"
 	"akvorado/common/httpserver"
 	"akvorado/common/reporter"
 	"akvorado/common/schema"
@@ -138,14 +139,14 @@ func (c *Component) Stop() error {
 // embedOrLiveFS returns a subset of the provided embedded filesystem,
 // except if the component is configured to use the live filesystem.
 // Then, it returns the provided tree.
-func (c *Component) embedOrLiveFS(embed fs.FS, p string) fs.FS {
+func (c *Component) embedOrLiveFS(prefix string) fs.FS {
 	var fileSystem fs.FS
 	if c.config.ServeLiveFS {
 		_, src, _, _ := runtime.Caller(0)
-		fileSystem = os.DirFS(filepath.Join(path.Dir(src), p))
+		fileSystem = os.DirFS(filepath.Join(path.Dir(src), prefix))
 	} else {
 		var err error
-		fileSystem, err = fs.Sub(embed, p)
+		fileSystem, err = fs.Sub(embed.Data(), path.Join("console", prefix))
 		if err != nil {
 			panic(err)
 		}

docker/Dockerfile

@@ -6,7 +6,7 @@ COPY Makefile .
 RUN make console/data/frontend
 
 FROM --platform=$BUILDPLATFORM golang:alpine AS build-go
-RUN apk add --no-cache make mailcap curl
+RUN apk add --no-cache make mailcap curl zip
 WORKDIR /build
 # Cache for modules
 COPY go.mod go.sum .

flake.nix

@@ -49,6 +49,7 @@
           src = ./.;
           vendorHash = l.readFile ./nix/vendorHash.txt;
           proxyVendor = true; # generated code may contain additional dependencies
+          nativeBuildInputs = [ pkgs.zip ];
           buildPhase = ''
             cp -r ${frontend}/node_modules console/frontend/node_modules
             cp -r ${frontend}/data console/data/frontend
@@ -145,6 +146,7 @@
             nodejs
             pkgs.git
             pkgs.curl
+            pkgs.zip
           ];
         };
       });

orchestrator/clickhouse/http.go

@@ -5,40 +5,31 @@ package clickhouse
 
 import (
 	"compress/gzip"
-	"embed"
 	"encoding/csv"
 	"fmt"
 	"io"
+	"io/fs"
 	"net/http"
 	"os"
 	"strconv"
 	"time"
+
+	"akvorado/common/embed"
 )
 
-//go:embed data/protocols.csv
-//go:embed data/icmp.csv
-//go:embed data/asns.csv
-//go:embed data/tcp.csv
-//go:embed data/udp.csv
-var data embed.FS
-
 func (c *Component) addHandlerEmbedded(url string, path string) {
+	data, _ := fs.Sub(embed.Data(), "orchestrator/clickhouse")
 	c.d.HTTP.AddHandler(url,
 		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			f, err := http.FS(data).Open(path)
+			http.ServeFileFS(w, r, data, path)
-			if err != nil {
-				c.r.Err(err).Msgf("unable to open %s", path)
-				http.Error(w, fmt.Sprintf("Unable to open %q.", path), http.StatusInternalServerError)
-				return
-			}
-			http.ServeContent(w, r, path, time.Time{}, f)
-			f.Close()
 		}))
 }
 
 // registerHTTPHandler register some handlers that will be useful for
 // ClickHouse
 func (c *Component) registerHTTPHandlers() error {
+	data, _ := fs.Sub(embed.Data(), "orchestrator/clickhouse")
+
 	// Add handler for custom dicts
 	for name, dict := range c.d.Schema.GetCustomDictConfig() {
 		c.d.HTTP.AddHandler(fmt.Sprintf("/api/v0/orchestrator/clickhouse/custom_dict_%s.csv", name), http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
@@ -163,7 +154,7 @@ func (c *Component) registerHTTPHandlers() error {
 	}
 
 	// Static CSV files
-	entries, err := data.ReadDir("data")
+	entries, err := data.(fs.ReadDirFS).ReadDir("data")
 	if err != nil {
 		return fmt.Errorf("unable to read data directory: %w", err)
 	}