console: dynamically fetch available dimensions

common/helpers/bimap.go

@@ -33,6 +33,24 @@ func (bi *Bimap[K, V]) LoadKey(v V) (K, bool) {
 	return k, ok
 }
 
+// Keys returns a slice of the keys in the bimap.
+func (bi *Bimap[K, V]) Keys() []K {
+	var keys []K
+	for k := range bi.forward {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+// Values returns a slice of the values in the bimap.
+func (bi *Bimap[K, V]) Values() []V {
+	var values []V
+	for v := range bi.inverse {
+		values = append(values, v)
+	}
+	return values
+}
+
 // String returns a string representation of the bimap.
 func (bi *Bimap[K, V]) String() string {
 	return fmt.Sprintf("Bi%v", bi.forward)

common/helpers/bimap_test.go

@@ -1,6 +1,7 @@
 package helpers_test
 
 import (
+	"sort"
 	"testing"
 
 	"akvorado/common/helpers"
@@ -59,3 +60,33 @@ func TestBimapLoadKey(t *testing.T) {
 		}
 	}
 }
+
+func TestBimapKeys(t *testing.T) {
+	input := helpers.NewBimap(map[int]string{
+		1: "hello",
+		2: "world",
+		3: "happy",
+	})
+	got := input.Keys()
+	expected := []int{1, 2, 3}
+	sort.Ints(got)
+	sort.Ints(expected)
+	if diff := helpers.Diff(got, expected); diff != "" {
+		t.Errorf("Keys() (-want, +got):\n%s", diff)
+	}
+}
+
+func TestBimapValues(t *testing.T) {
+	input := helpers.NewBimap(map[int]string{
+		1: "hello",
+		2: "world",
+		3: "happy",
+	})
+	got := input.Values()
+	expected := []string{"hello", "world", "happy"}
+	sort.Strings(got)
+	sort.Strings(expected)
+	if diff := helpers.Diff(got, expected); diff != "" {
+		t.Errorf("Values() (-want, +got):\n%s", diff)
+	}
+}

console/frontend/src/components/InputDimensions.vue

@@ -11,7 +11,14 @@
           class="peer block w-full appearance-none rounded-t-lg border-0 border-b-2 border-gray-300 bg-gray-50 px-2.5 pb-1.5 pt-4 text-left text-sm text-gray-900 focus:border-blue-600 focus:outline-none focus:ring-0 dark:border-gray-600 dark:bg-gray-700 dark:text-white dark:focus:border-blue-500"
         >
           <span class="block flex flex-wrap gap-2 pt-1">
-            <span v-if="selectedDimensions.length === 0">No dimensions</span>
+            <span
+              v-if="dimensionsError"
+              class="text-red-600 dark:text-red-400"
+              >{{ dimensionsError }}</span
+            >
+            <span v-if="selectedDimensions.length === 0 && !dimensionsError"
+              >No dimensions</span
+            >
             <span
               v-for="dimension in selectedDimensions"
               :key="dimension.id"
@@ -33,7 +40,12 @@
         </ListboxButton>
         <label
           for="dimensions"
-          class="z-5 absolute top-3 left-2.5 origin-[0] -translate-y-3 scale-75 transform text-sm text-gray-500 peer-focus:text-blue-600 dark:text-gray-400 dark:peer-focus:text-blue-500"
+          class="z-5 absolute top-3 left-2.5 origin-[0] -translate-y-3 scale-75 transform text-sm"
+          :class="{
+            'text-red-600 dark:text-red-500': dimensionsError,
+            'text-gray-500 peer-focus:text-blue-600 dark:text-gray-400 dark:peer-focus:text-blue-500':
+              !dimensionsError,
+          }"
         >
           Dimensions
         </label>
@@ -94,7 +106,7 @@ import {
   ListboxOption,
 } from "@headlessui/vue";
 import { XIcon, CheckIcon, SelectorIcon } from "@heroicons/vue/solid";
-import { dataColor } from "../utils";
+import { dataColor, compareFields } from "../utils";
 import InputString from "./InputString.vue";
 
 const props = defineProps({
@@ -109,45 +121,10 @@ const props = defineProps({
 const emit = defineEmits(["update:modelValue"]);
 
 // We don't fetch from server since error handling would be a tad compex.
-const dimensions = [
+const dimensions = ref([]);
-  { name: "ExporterAddress" },
+const dimensionsError = ref("");
-  { name: "ExporterName" },
-  { name: "ExporterGroup" },
-  { name: "InIfBoundary" },
-  { name: "InIfConnectivity" },
-  { name: "InIfDescription" },
-  { name: "InIfName" },
-  { name: "InIfProvider" },
-  { name: "InIfSpeed" },
-  { name: "OutIfBoundary" },
-  { name: "OutIfConnectivity" },
-  { name: "OutIfDescription" },
-  { name: "OutIfName" },
-  { name: "OutIfProvider" },
-  { name: "OutIfSpeed" },
-  { name: "SrcAS" },
-  { name: "SrcCountry" },
-  { name: "SrcPort" },
-  { name: "SrcAddr" },
-  { name: "DstAS" },
-  { name: "DstCountry" },
-  { name: "DstAddr" },
-  { name: "DstPort" },
-  { name: "EType" },
-  { name: "Proto" },
-  { name: "ForwardingStatus" },
-].map((v, idx) => ({
-  id: idx + 1,
-  color: dataColor(
-    ["Exporter", "Src", "Dst", "In", "Out", ""]
-      .map((p) => v.name.startsWith(p))
-      .indexOf(true)
-  ),
-  ...v,
-}));
-
 const selectedDimensions = ref([]);
-const limit = ref(10);
+const limit = ref("10");
 const limitError = computed(() => {
   const val = parseInt(limit.value);
   if (isNaN(val)) {
@@ -162,6 +139,28 @@ const limitError = computed(() => {
   return "";
 });
 
+// Fetch dimensions
+(async () => {
+  try {
+    const response = await fetch("/api/v0/console/graph/fields");
+    if (!response.ok) {
+      throw `Server returned ${response.status} status`;
+    }
+    const data = await response.json();
+    dimensions.value = data.sort(compareFields).map((v, idx) => ({
+      id: idx + 1,
+      name: v,
+      color: dataColor(
+        ["Exporter", "Src", "Dst", "In", "Out", ""]
+          .map((p) => v.startsWith(p))
+          .indexOf(true)
+      ),
+    }));
+  } catch (err) {
+    dimensionsError.value = "Cannot fetch dimensions";
+  }
+})();
+
 const removeDimension = (dimension) => {
   selectedDimensions.value = selectedDimensions.value.filter(
     (d) => d !== dimension
@@ -169,23 +168,26 @@ const removeDimension = (dimension) => {
 };
 
 watch(
-  () => props.modelValue,
+  () => [props.modelValue, dimensions],
-  (model) => {
+  ([model, dimensions]) => {
     limit.value = model.limit.toString();
     selectedDimensions.value = model.selected
-      .map((name) => dimensions.filter((d) => d.name === name)[0])
+      .map((name) => dimensions.value.filter((d) => d.name === name)[0])
       .filter((d) => d !== undefined);
   },
-  { immediate: true, deep: true }
+  { deep: true }
 );
-watch([selectedDimensions, limit], ([selected, limit]) => {
+watch(
-  const updated = {
+  [selectedDimensions, limit, limitError, dimensionsError],
-    selected: selected.map((d) => d.name),
+  ([selected, limit, limitError, dimensionsError]) => {
-    limit: parseInt(limit) || limit,
+    const updated = {
-    errors: !!limitError.value,
+      selected: selected.map((d) => d.name),
-  };
+      limit: parseInt(limit) || limit,
-  if (JSON.stringify(updated) !== JSON.stringify(props.modelValue)) {
+      errors: !!(limitError || dimensionsError),
-    emit("update:modelValue", updated);
+    };
+    if (JSON.stringify(updated) !== JSON.stringify(props.modelValue)) {
+      emit("update:modelValue", updated);
+    }
   }
-});
+);
 </script>

console/frontend/src/components/VisualizeOptions.vue

@@ -109,6 +109,7 @@ const options = computed(() => {
     options.end = timeRange.value.end;
     options.dimensions = dimensions.value.selected;
     options.limit = dimensions.value.limit;
+    options.points = props.state.points;
     return options;
   } catch (_) {
     return {};

console/frontend/src/components/WidgetLastFlow.vue

@@ -18,6 +18,7 @@
 
 <script setup>
 import { ref, watch } from "vue";
+import { compareFields } from "../utils";
 
 const props = defineProps({
   refresh: {
@@ -36,45 +37,9 @@ watch(
       return;
     }
     const data = await response.json();
-    lastFlow.value = Object.entries(data).sort(([f1], [f2]) => {
+    lastFlow.value = Object.entries(data).sort(([f1], [f2]) =>
-      const metric = {
+      compareFields(f1, f2)
-        Dat: 1,
+    );
-        Tim: 2,
-        Byt: 3,
-        Pac: 4,
-        ETy: 5,
-        Pro: 6,
-        Exp: 7,
-        Sam: 8,
-        Seq: 9,
-        Dst: 10,
-        Src: 10,
-        InI: 11,
-        Out: 11,
-      };
-      const m1 = metric[f1.substring(0, 3)] || 100;
-      const m2 = metric[f2.substring(0, 3)] || 100;
-      const cmp = m1 - m2;
-      if (cmp) {
-        return cmp;
-      }
-      if (m1 === 10) {
-        f1 = f1.substring(3);
-        f2 = f2.substring(3);
-      } else if (m1 === 11) {
-        if (f1.startsWith("InIf")) {
-          f1 = f1.substring(4);
-        } else {
-          f1 = f1.substring(5);
-        }
-        if (f2.startsWith("InIf")) {
-          f2 = f2.substring(4);
-        } else {
-          f2 = f2.substring(5);
-        }
-      }
-      return f1.localeCompare(f2);
-    });
   },
   { immediate: true }
 );

console/frontend/src/utils/index.js

@@ -9,4 +9,43 @@ export function formatBps(value) {
   return `${value}${suffixes[idx]}`;
 }
 
+// Order function for field names
+export function compareFields(f1, f2) {
+  const metric = {
+    Dat: 1,
+    Tim: 2,
+    Byt: 3,
+    Pac: 4,
+    Exp: 7,
+    Sam: 8,
+    Seq: 9,
+    Src: 10,
+    Dst: 12,
+    InI: 11,
+    Out: 13,
+  };
+  const m1 = metric[f1.substring(0, 3)] || 100;
+  const m2 = metric[f2.substring(0, 3)] || 100;
+  const cmp = m1 - m2;
+  if (cmp) {
+    return cmp;
+  }
+  if (m1 === 10) {
+    f1 = f1.substring(3);
+    f2 = f2.substring(3);
+  } else if (m1 === 11) {
+    if (f1.startsWith("InIf")) {
+      f1 = f1.substring(4);
+    } else {
+      f1 = f1.substring(5);
+    }
+    if (f2.startsWith("InIf")) {
+      f2 = f2.substring(4);
+    } else {
+      f2 = f2.substring(5);
+    }
+  }
+  return f1.localeCompare(f2);
+}
+
 export { dataColor, dataColorGrey } from "./palette.js";

console/frontend/src/views/VisualizePage.vue

@@ -65,7 +65,7 @@ const encodeState = (state) => {
 const defaultState = () => ({
   start: "6 hours ago",
   end: "now",
-  points: 400,
+  points: 200,
   limit: 10,
   dimensions: ["SrcAS", "ExporterName"],
   filter: {

console/graph.go

@@ -513,3 +513,7 @@ func (c *Component) graphHandlerFunc(gc *gin.Context) {
 
 	gc.JSON(http.StatusOK, output)
 }
+
+func (c *Component) graphFieldsHandlerFunc(gc *gin.Context) {
+	gc.JSON(http.StatusOK, graphColumnMap.Values())
+}

console/graph_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	netHTTP "net/http"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -520,3 +521,71 @@ func TestGraphHandler(t *testing.T) {
 		t.Fatalf("POST /api/v0/console/graph (-got, +want):\n%s", diff)
 	}
 }
+
+func TestGraphFieldsHandler(t *testing.T) {
+	r := reporter.NewMock(t)
+	ch, _ := clickhousedb.NewMock(t, r)
+	h := http.NewMock(t, r)
+	c, err := New(r, Configuration{}, Dependencies{
+		Daemon:       daemon.NewMock(t),
+		HTTP:         h,
+		ClickHouseDB: ch,
+	})
+	if err != nil {
+		t.Fatalf("New() error:\n%+v", err)
+	}
+	helpers.StartStop(t, c)
+
+	resp, err := netHTTP.Get(fmt.Sprintf("http://%s/api/v0/console/graph/fields", h.Address))
+	if err != nil {
+		t.Fatalf("POST /api/v0/console/graph/fields:\n%+v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		t.Errorf("POST /api/v0/console/graph/fields: got status code %d, not 200", resp.StatusCode)
+	}
+	gotContentType := resp.Header.Get("Content-Type")
+	if gotContentType != "application/json; charset=utf-8" {
+		t.Errorf("POST /api/v0/console/graph/fields Content-Type (-got, +want):\n-%s\n+%s",
+			gotContentType, "application/json; charset=utf-8")
+	}
+	decoder := json.NewDecoder(resp.Body)
+	var got []string
+	if err := decoder.Decode(&got); err != nil {
+		t.Fatalf("POST /api/v0/console/graph error:\n%+v", err)
+	}
+	expected := []string{
+		"ExporterAddress",
+		"ExporterName",
+		"ExporterGroup",
+		"SrcAddr",
+		"DstAddr",
+		"SrcAS",
+		"DstAS",
+		"SrcCountry",
+		"DstCountry",
+		"InIfName",
+		"OutIfName",
+		"InIfDescription",
+		"OutIfDescription",
+		"InIfSpeed",
+		"OutIfSpeed",
+		"InIfConnectivity",
+		"OutIfConnectivity",
+		"InIfProvider",
+		"OutIfProvider",
+		"InIfBoundary",
+		"OutIfBoundary",
+		"EType",
+		"Proto",
+		"SrcPort",
+		"DstPort",
+		"ForwardingStatus",
+	}
+	sort.Strings(expected)
+	sort.Strings(got)
+
+	if diff := helpers.Diff(got, expected); diff != "" {
+		t.Fatalf("POST /api/v0/console/graph/fields (-got, +want):\n%s", diff)
+	}
+}

console/root.go

@@ -82,6 +82,7 @@ func (c *Component) Start() error {
 	c.d.HTTP.GinRouter.GET("/api/v0/console/widget/top/:name", c.widgetTopHandlerFunc)
 	c.d.HTTP.GinRouter.GET("/api/v0/console/widget/graph", c.widgetGraphHandlerFunc)
 	c.d.HTTP.GinRouter.POST("/api/v0/console/graph", c.graphHandlerFunc)
+	c.d.HTTP.GinRouter.GET("/api/v0/console/graph/fields", c.graphFieldsHandlerFunc)
 
 	c.t.Go(func() error {
 		ticker := time.NewTicker(10 * time.Second)