johanels/akvorado
1
0
Fork 0
mirror of https://github.com/akvorado/akvorado.git synced 2025-12-12 06:24:10 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

console: use resolution when all tables are equally bad for old data

Browse Source
This commit is contained in:
Vincent Bernat
2022-07-09 08:30:12 +02:00
parent f5e2e85d0d
commit d5dcc885bc
2 changed files with 32 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
console/clickhouse.go
View File

@@ -38,7 +38,7 @@ func (c *Component) queryFlowsTable(query string, start, end time.Time, targetRe
// criteria. // criteria.
candidates := []int{} candidates := []int{}
for idx, table := range c.flowsTables { for idx, table := range c.flowsTables {
if start.After(table.Oldest) { if start.After(table.Oldest.Add(table.Resolution)) {
candidates = append(candidates, idx) candidates = append(candidates, idx)
} }
} }
@@ -46,12 +46,22 @@ func (c *Component) queryFlowsTable(query string, start, end time.Time, targetRe
// No candidate, fallback to the one with oldest data // No candidate, fallback to the one with oldest data
best := 0 best := 0
for idx, table := range c.flowsTables { for idx, table := range c.flowsTables {
if c.flowsTables[best].Oldest.After(table.Oldest) { if c.flowsTables[best].Oldest.After(table.Oldest.Add(table.Resolution)) {
best = idx best = idx
} }
} }
candidates = []int{best} candidates = []int{best}
} else if len(candidates) > 1 { // Add other candidates that are not far off in term of oldest data
for idx, table := range c.flowsTables {
if idx == best {
continue
}
if c.flowsTables[best].Oldest.After(table.Oldest) {
candidates = append(candidates, idx)
}
}
}
if len(candidates) > 1 {
// Use resolution to find the best one // Use resolution to find the best one
best := 0 best := 0
for _, idx := range candidates { for _, idx := range candidates {

19
console/clickhouse_test.go
View File

@@ -93,6 +93,13 @@ func TestQueryFlowsTables(t *testing.T) {
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC), Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC), End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')", Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')",
}, {
Description: "timefilter.Start and timefilter.Stop",
Tables: []flowsTable{{"flows", 0, time.Date(2022, 03, 10, 15, 45, 10, 0, time.UTC)}},
Query: "SELECT {timefilter.Start}, {timefilter.Stop}",
Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC),
Expected: "SELECT toDateTime('2022-04-10 15:45:10', 'UTC'), toDateTime('2022-04-11 15:45:10', 'UTC')",
}, { }, {
Description: "only flows table and out of range request", Description: "only flows table and out of range request",
Tables: []flowsTable{{"flows", 0, time.Date(2022, 04, 10, 22, 45, 10, 0, time.UTC)}}, Tables: []flowsTable{{"flows", 0, time.Date(2022, 04, 10, 22, 45, 10, 0, time.UTC)}},
@@ -168,6 +175,18 @@ func TestQueryFlowsTables(t *testing.T) {
End: time.Date(2022, 04, 11, 15, 46, 10, 0, time.UTC), End: time.Date(2022, 04, 11, 15, 46, 10, 0, time.UTC),
Resolution: 2 * time.Minute, Resolution: 2 * time.Minute,
Expected: "SELECT 1 FROM flows_5m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:00', 'UTC') AND toDateTime('2022-04-11 15:45:00', 'UTC')", Expected: "SELECT 1 FROM flows_5m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:00', 'UTC') AND toDateTime('2022-04-11 15:45:00', 'UTC')",
}, {
Description: "select best resolution when equality for oldest data",
Tables: []flowsTable{
{"flows", 0, time.Date(2022, 04, 10, 22, 40, 55, 0, time.UTC)},
{"flows_1m0s", time.Minute, time.Date(2022, 04, 10, 22, 40, 00, 0, time.UTC)},
{"flows_1h0m0s", time.Hour, time.Date(2022, 04, 10, 22, 00, 10, 0, time.UTC)},
},
Query: "SELECT 1 FROM {table} WHERE {timefilter}",
Start: time.Date(2022, 04, 10, 15, 46, 10, 0, time.UTC),
End: time.Date(2022, 04, 11, 15, 46, 10, 0, time.UTC),
Resolution: 2 * time.Minute,
Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:46:00', 'UTC') AND toDateTime('2022-04-11 15:46:00', 'UTC')",
}, },
} }