diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f59accd..5d87bdc5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: - name: Setup uses: ./.github/actions/setup - name: Install dependencies - run: sudo apt-get install -qqy shared-mime-info curl + run: sudo apt-get install -qqy shared-mime-info curl zip # Regular tests - name: Go race tests diff --git a/.gitignore b/.gitignore index d1c35200..7ae84cc2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ /orchestrator/clickhouse/data/udp.csv /console/filter/parser.go /common/schema/definition_gen.go +/common/embed/data/embed.zip mock_*.go *_enumer.go *.pb.go diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 11c3c11f..2a40ef91 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,7 +22,7 @@ run tests: # past but this is a slight burden to maintain in addition to GitHub CI. # Check commit ceaa6ebf8ef6 for the last version supporting functional # tests. - - time apk add --no-cache git make gcc musl-dev shared-mime-info npm curl + - time apk add --no-cache git make gcc musl-dev shared-mime-info npm curl zip - export GOMODCACHE=$PWD/.go-cache - npm config --user set cache $PWD/.npm-cache - time go mod download diff --git a/Makefile b/Makefile index 6ae2429d..9bbea9c1 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,8 @@ GENERATED_TEST_GO = \ GENERATED = \ $(GENERATED_GO) \ $(GENERATED_JS) \ - console/data/frontend + console/data/frontend \ + common/embed/data/embed.zip .PHONY: all all-indep BUILD_ARGS = @@ -153,6 +154,11 @@ default-%.pgo: [ -n $$ip ] ; \ curl -so $@ "http://$$ip:8080/debug/pprof/profile?seconds=30" +common/embed/data/embed.zip: console/data/frontend console/authentication/data/avatars console/data/docs +common/embed/data/embed.zip: orchestrator/clickhouse/data/protocols.csv orchestrator/clickhouse/data/icmp.csv orchestrator/clickhouse/data/asns.csv orchestrator/clickhouse/data/tcp.csv orchestrator/clickhouse/data/udp.csv +common/embed/data/embed.zip: ; $(info $(M) generate embed.zip…) + $Q mkdir -p common/embed/data && zip --quiet --recurse-paths --filesync $@ $^ + # Tests .PHONY: check test tests test-race test-short test-bench test-coverage diff --git a/common/embed/fs.go b/common/embed/fs.go new file mode 100644 index 00000000..d6f85fdd --- /dev/null +++ b/common/embed/fs.go @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: 2025 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +// Package embed provides access to the compressed archive containing all the +// embedded files. +package embed + +import ( + "archive/zip" + "bytes" + _ "embed" + "fmt" + "io/fs" + "sync" +) + +var ( + //go:embed data/embed.zip + embeddedZip []byte + data fs.FS + dataOnce sync.Once + dataReady chan struct{} +) + +// Data returns a filesystem with the files contained in the embedded archive. +func Data() fs.FS { + dataOnce.Do(func() { + r, err := zip.NewReader(bytes.NewReader(embeddedZip), int64(len(embeddedZip))) + if err != nil { + panic(fmt.Sprintf("cannot read embedded archive: %s", err)) + } + data = r + close(dataReady) + }) + <-dataReady + return data +} + +func init() { + dataReady = make(chan struct{}) +} diff --git a/common/embed/fs_test.go b/common/embed/fs_test.go new file mode 100644 index 00000000..0ca75a37 --- /dev/null +++ b/common/embed/fs_test.go @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: 2025 Free Mobile +// SPDX-License-Identifier: AGPL-3.0-only + +package embed_test + +import ( + "io" + "testing" + + "akvorado/common/embed" + "akvorado/common/helpers" +) + +func TestData(t *testing.T) { + f, err := embed.Data().Open("orchestrator/clickhouse/data/protocols.csv") + if err != nil { + t.Fatalf("Open() error:\n%+v", err) + } + expected := "proto,name,description" + got := make([]byte, len(expected)) + _, err = io.ReadFull(f, got) + if err != nil { + t.Fatalf("ReadFull() error:\n%+v", err) + } + if diff := helpers.Diff(string(got), expected); diff != "" { + t.Fatalf("ReadFull() (-got, +want):\n%s", diff) + } +} diff --git a/console/assets.go b/console/assets.go index 28ac6c04..628ae8e6 100644 --- a/console/assets.go +++ b/console/assets.go @@ -4,31 +4,20 @@ package console import ( - "embed" "net/http" - "time" ) -//go:embed data/frontend -var embeddedAssets embed.FS - func (c *Component) defaultHandlerFunc(w http.ResponseWriter, req *http.Request) { - assets := c.embedOrLiveFS(embeddedAssets, "data/frontend") - f, err := http.FS(assets).Open("index.html") - if err != nil { - http.Error(w, "Application not found.", http.StatusInternalServerError) - return - } - http.ServeContent(w, req, "index.html", time.Time{}, f) - f.Close() + assets := c.embedOrLiveFS("data/frontend") + http.ServeFileFS(w, req, assets, "index.html") } func (c *Component) staticAssetsHandlerFunc(w http.ResponseWriter, req *http.Request) { - assets := c.embedOrLiveFS(embeddedAssets, "data/frontend/assets") + assets := c.embedOrLiveFS("data/frontend/assets") http.FileServer(http.FS(assets)).ServeHTTP(w, req) } func (c *Component) docAssetsHandlerFunc(w http.ResponseWriter, req *http.Request) { - docs := c.embedOrLiveFS(embeddedDocs, "data/docs") + docs := c.embedOrLiveFS("data/docs") http.FileServer(http.FS(docs)).ServeHTTP(w, req) } diff --git a/console/authentication/handlers.go b/console/authentication/handlers.go index 8fc1faed..025c86db 100644 --- a/console/authentication/handlers.go +++ b/console/authentication/handlers.go @@ -5,7 +5,6 @@ package authentication import ( "bufio" - "embed" "fmt" "hash/fnv" "image" @@ -15,12 +14,11 @@ import ( "math/rand" "net/http" + "akvorado/common/embed" + "github.com/gin-gonic/gin" ) -//go:embed data/avatars -var avatarParts embed.FS - // UserInfoHandlerFunc returns the information about the currently logged user. func (c *Component) UserInfoHandlerFunc(gc *gin.Context) { info := gc.MustGet("user").(UserInformation) @@ -44,6 +42,12 @@ func (c *Component) UserAvatarHandlerFunc(gc *gin.Context) { } // Grab list of parts + avatarParts, err := fs.Sub(embed.Data(), "console/authentication") + if err != nil { + c.r.Err(err).Msg("cannot open embedded archive") + gc.JSON(http.StatusInternalServerError, gin.H{"message": "Cannot build avatar."}) + return + } parts := []string{} partList, err := avatarParts.Open("data/avatars/partlist.txt") if err != nil { diff --git a/console/docs.go b/console/docs.go index 7142ccdb..e501f337 100644 --- a/console/docs.go +++ b/console/docs.go @@ -5,7 +5,6 @@ package console import ( "bytes" - "embed" "fmt" "io" "io/fs" @@ -24,8 +23,6 @@ import ( ) var ( - //go:embed data/docs - embeddedDocs embed.FS internalLinkRegexp = regexp.MustCompile("^(([0-9]+)-([a-z]+).md)(#.*|$)") ) @@ -43,7 +40,7 @@ type DocumentTOC struct { } func (c *Component) docsHandlerFunc(gc *gin.Context) { - docs := c.embedOrLiveFS(embeddedDocs, "data/docs") + docs := c.embedOrLiveFS("data/docs") requestedDocument := gc.Param("name") var markdown []byte diff --git a/console/root.go b/console/root.go index b0879fa6..5aeebea9 100644 --- a/console/root.go +++ b/console/root.go @@ -19,6 +19,7 @@ import ( "akvorado/common/clickhousedb" "akvorado/common/daemon" + "akvorado/common/embed" "akvorado/common/httpserver" "akvorado/common/reporter" "akvorado/common/schema" @@ -138,14 +139,14 @@ func (c *Component) Stop() error { // embedOrLiveFS returns a subset of the provided embedded filesystem, // except if the component is configured to use the live filesystem. // Then, it returns the provided tree. -func (c *Component) embedOrLiveFS(embed fs.FS, p string) fs.FS { +func (c *Component) embedOrLiveFS(prefix string) fs.FS { var fileSystem fs.FS if c.config.ServeLiveFS { _, src, _, _ := runtime.Caller(0) - fileSystem = os.DirFS(filepath.Join(path.Dir(src), p)) + fileSystem = os.DirFS(filepath.Join(path.Dir(src), prefix)) } else { var err error - fileSystem, err = fs.Sub(embed, p) + fileSystem, err = fs.Sub(embed.Data(), path.Join("console", prefix)) if err != nil { panic(err) } diff --git a/docker/Dockerfile b/docker/Dockerfile index 9b43a19b..c715cc9f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,7 +6,7 @@ COPY Makefile . RUN make console/data/frontend FROM --platform=$BUILDPLATFORM golang:alpine AS build-go -RUN apk add --no-cache make mailcap curl +RUN apk add --no-cache make mailcap curl zip WORKDIR /build # Cache for modules COPY go.mod go.sum . diff --git a/flake.nix b/flake.nix index 667ded8c..7ac7552e 100644 --- a/flake.nix +++ b/flake.nix @@ -49,6 +49,7 @@ src = ./.; vendorHash = l.readFile ./nix/vendorHash.txt; proxyVendor = true; # generated code may contain additional dependencies + nativeBuildInputs = [ pkgs.zip ]; buildPhase = '' cp -r ${frontend}/node_modules console/frontend/node_modules cp -r ${frontend}/data console/data/frontend @@ -145,6 +146,7 @@ nodejs pkgs.git pkgs.curl + pkgs.zip ]; }; }); diff --git a/orchestrator/clickhouse/http.go b/orchestrator/clickhouse/http.go index a5817627..9bd22070 100644 --- a/orchestrator/clickhouse/http.go +++ b/orchestrator/clickhouse/http.go @@ -5,40 +5,31 @@ package clickhouse import ( "compress/gzip" - "embed" "encoding/csv" "fmt" "io" + "io/fs" "net/http" "os" "strconv" "time" + + "akvorado/common/embed" ) -//go:embed data/protocols.csv -//go:embed data/icmp.csv -//go:embed data/asns.csv -//go:embed data/tcp.csv -//go:embed data/udp.csv -var data embed.FS - func (c *Component) addHandlerEmbedded(url string, path string) { + data, _ := fs.Sub(embed.Data(), "orchestrator/clickhouse") c.d.HTTP.AddHandler(url, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - f, err := http.FS(data).Open(path) - if err != nil { - c.r.Err(err).Msgf("unable to open %s", path) - http.Error(w, fmt.Sprintf("Unable to open %q.", path), http.StatusInternalServerError) - return - } - http.ServeContent(w, r, path, time.Time{}, f) - f.Close() + http.ServeFileFS(w, r, data, path) })) } // registerHTTPHandler register some handlers that will be useful for // ClickHouse func (c *Component) registerHTTPHandlers() error { + data, _ := fs.Sub(embed.Data(), "orchestrator/clickhouse") + // Add handler for custom dicts for name, dict := range c.d.Schema.GetCustomDictConfig() { c.d.HTTP.AddHandler(fmt.Sprintf("/api/v0/orchestrator/clickhouse/custom_dict_%s.csv", name), http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -163,7 +154,7 @@ func (c *Component) registerHTTPHandlers() error { } // Static CSV files - entries, err := data.ReadDir("data") + entries, err := data.(fs.ReadDirFS).ReadDir("data") if err != nil { return fmt.Errorf("unable to read data directory: %w", err) }