rcserver: implement prometheus metrics on a dedicated port - fixes #7940

fs/rc/rcserver/metrics.go

@@ -0,0 +1,97 @@
+// Package rcserver implements the HTTP endpoint to serve the remote control
+package rcserver
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/fshttp"
+	"github.com/rclone/rclone/fs/rc"
+	"github.com/rclone/rclone/fs/rc/jobs"
+	libhttp "github.com/rclone/rclone/lib/http"
+)
+
+const path = "/metrics"
+
+var promHandlerFunc http.HandlerFunc
+
+func init() {
+	rcloneCollector := accounting.NewRcloneCollector(context.Background())
+	prometheus.MustRegister(rcloneCollector)
+
+	m := fshttp.NewMetrics("rclone")
+	for _, c := range m.Collectors() {
+		prometheus.MustRegister(c)
+	}
+	fshttp.DefaultMetrics = m
+
+	promHandlerFunc = promhttp.Handler().ServeHTTP
+}
+
+// MetricsStart the remote control server if configured
+//
+// If the server wasn't configured the *Server returned may be nil
+func MetricsStart(ctx context.Context, opt *rc.Options) (*MetricsServer, error) {
+	jobs.SetOpt(opt) // set the defaults for jobs
+	if opt.MetricsHTTP.ListenAddr[0] != "" {
+		// Serve on the DefaultServeMux so can have global registrations appear
+		s, err := newMetricsServer(ctx, opt)
+		if err != nil {
+			return nil, err
+		}
+		return s, s.Serve()
+	}
+	return nil, nil
+}
+
+// MetricsServer contains everything to run the rc server
+type MetricsServer struct {
+	ctx             context.Context // for global config
+	server          *libhttp.Server
+	promHandlerFunc http.Handler
+	opt             *rc.Options
+}
+
+func newMetricsServer(ctx context.Context, opt *rc.Options) (*MetricsServer, error) {
+	s := &MetricsServer{
+		ctx:             ctx,
+		opt:             opt,
+		promHandlerFunc: promHandlerFunc,
+	}
+
+	var err error
+	s.server, err = libhttp.NewServer(ctx,
+		libhttp.WithConfig(opt.MetricsHTTP),
+		libhttp.WithAuth(opt.MetricsAuth),
+		libhttp.WithTemplate(opt.MetricsTemplate),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init server: %w", err)
+	}
+
+	router := s.server.Router()
+	router.Get(path, promHandlerFunc)
+	return s, nil
+}
+
+// Serve runs the http server in the background.
+//
+// Use s.Close() and s.Wait() to shutdown server
+func (s *MetricsServer) Serve() error {
+	s.server.Serve()
+	return nil
+}
+
+// Wait blocks while the server is serving requests
+func (s *MetricsServer) Wait() {
+	s.server.Wait()
+}
+
+// Shutdown gracefully shuts down the server
+func (s *MetricsServer) Shutdown() error {
+	return s.server.Shutdown()
+}

fs/rc/rcserver/metrics_test.go

@@ -0,0 +1,88 @@
+package rcserver
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"regexp"
+	"testing"
+
+	_ "github.com/rclone/rclone/backend/local"
+	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/config/configfile"
+	"github.com/rclone/rclone/fs/rc"
+	"github.com/stretchr/testify/require"
+)
+
+// Run a suite of tests
+func testMetricsServer(t *testing.T, tests []testRun, opt *rc.Options) {
+	t.Helper()
+	ctx := context.Background()
+	configfile.Install()
+	rcServer, err := newMetricsServer(ctx, opt)
+	require.NoError(t, err)
+	testURL := rcServer.server.URLs()[0]
+	mux := rcServer.server.Router()
+	emulateCalls(t, tests, mux, testURL)
+}
+
+// return an enabled rc
+func newMetricsTestOpt() rc.Options {
+	opt := rc.Opt
+	opt.MetricsHTTP.ListenAddr = []string{testBindAddress}
+	return opt
+}
+
+func TestMetrics(t *testing.T) {
+	stats := accounting.GlobalStats()
+	tests := makeMetricsTestCases(stats)
+	opt := newMetricsTestOpt()
+	testMetricsServer(t, tests, &opt)
+
+	// Test changing a couple options
+	stats.Bytes(500)
+	for i := 0; i < 30; i++ {
+		require.NoError(t, stats.DeleteFile(context.Background(), 0))
+	}
+	stats.Errors(2)
+	stats.Bytes(324)
+
+	tests = makeMetricsTestCases(stats)
+	testMetricsServer(t, tests, &opt)
+}
+
+func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) {
+	tests = []testRun{{
+		Name:     "Bytes Transferred Metric",
+		URL:      "metrics",
+		Method:   "GET",
+		Status:   http.StatusOK,
+		Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())),
+	}, {
+		Name:     "Checked Files Metric",
+		URL:      "metrics",
+		Method:   "GET",
+		Status:   http.StatusOK,
+		Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())),
+	}, {
+		Name:     "Errors Metric",
+		URL:      "metrics",
+		Method:   "GET",
+		Status:   http.StatusOK,
+		Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())),
+	}, {
+		Name:     "Deleted Files Metric",
+		URL:      "metrics",
+		Method:   "GET",
+		Status:   http.StatusOK,
+		Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.GetDeletes())),
+	}, {
+		Name:     "Files Transferred Metric",
+		URL:      "metrics",
+		Method:   "GET",
+		Status:   http.StatusOK,
+		Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())),
+	},
+	}
+	return
+}

fs/rc/rcserver/rcserver.go

@@ -18,13 +18,9 @@ import (
 	"time"
 
 	"github.com/go-chi/chi/v5/middleware"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/fs/rc"
 	"github.com/rclone/rclone/fs/rc/jobs"
@@ -35,21 +31,6 @@ import (
 	"github.com/skratchdot/open-golang/open"
 )
 
-var promHandler http.Handler
-
-func init() {
-	rcloneCollector := accounting.NewRcloneCollector(context.Background())
-	prometheus.MustRegister(rcloneCollector)
-
-	m := fshttp.NewMetrics("rclone")
-	for _, c := range m.Collectors() {
-		prometheus.MustRegister(c)
-	}
-	fshttp.DefaultMetrics = m
-
-	promHandler = promhttp.Handler()
-}
-
 // Start the remote control server if configured
 //
 // If the server wasn't configured the *Server returned may be nil
@@ -376,7 +357,7 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string)
 		s.serveRemote(w, r, fsMatchResult[2], fsMatchResult[1])
 		return
 	case path == "metrics" && s.opt.EnableMetrics:
-		promHandler.ServeHTTP(w, r)
+		promHandlerFunc(w, r)
 		return
 	case path == "*" && s.opt.Serve:
 		// Serve /* as the remote listing

fs/rc/rcserver/rcserver_test.go

@@ -15,9 +15,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/go-chi/chi/v5"
+
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/fs/rc"
 	"github.com/stretchr/testify/assert"
@@ -115,6 +116,10 @@ func testServer(t *testing.T, tests []testRun, opt *rc.Options) {
 	require.NoError(t, err)
 	testURL := rcServer.server.URLs()[0]
 	mux := rcServer.server.Router()
+	emulateCalls(t, tests, mux, testURL)
+}
+
+func emulateCalls(t *testing.T, tests []testRun, mux chi.Router, testURL string) {
 	for _, test := range tests {
 		t.Run(test.Name, func(t *testing.T) {
 			t.Helper()
@@ -568,61 +573,6 @@ Unknown command
 	testServer(t, tests, &opt)
 }
 
-func TestMetrics(t *testing.T) {
-	stats := accounting.GlobalStats()
-	tests := makeMetricsTestCases(stats)
-	opt := newTestOpt()
-	opt.EnableMetrics = true
-	testServer(t, tests, &opt)
-
-	// Test changing a couple options
-	stats.Bytes(500)
-	for i := 0; i < 30; i++ {
-		require.NoError(t, stats.DeleteFile(context.Background(), 0))
-	}
-	stats.Errors(2)
-	stats.Bytes(324)
-
-	tests = makeMetricsTestCases(stats)
-	testServer(t, tests, &opt)
-}
-
-func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) {
-	tests = []testRun{{
-		Name:     "Bytes Transferred Metric",
-		URL:      "/metrics",
-		Method:   "GET",
-		Status:   http.StatusOK,
-		Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())),
-	}, {
-		Name:     "Checked Files Metric",
-		URL:      "/metrics",
-		Method:   "GET",
-		Status:   http.StatusOK,
-		Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())),
-	}, {
-		Name:     "Errors Metric",
-		URL:      "/metrics",
-		Method:   "GET",
-		Status:   http.StatusOK,
-		Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())),
-	}, {
-		Name:     "Deleted Files Metric",
-		URL:      "/metrics",
-		Method:   "GET",
-		Status:   http.StatusOK,
-		Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.GetDeletes())),
-	}, {
-		Name:     "Files Transferred Metric",
-		URL:      "/metrics",
-		Method:   "GET",
-		Status:   http.StatusOK,
-		Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())),
-	},
-	}
-	return
-}
-
 var matchRemoteDirListing = regexp.MustCompile(`<title>Directory listing of /</title>`)
 
 func TestServingRoot(t *testing.T) {