diff --git a/console/clickhouse.go b/console/clickhouse.go index af39f2ab..77b23f97 100644 --- a/console/clickhouse.go +++ b/console/clickhouse.go @@ -38,7 +38,7 @@ func (c *Component) queryFlowsTable(query string, start, end time.Time, targetRe // criteria. candidates := []int{} for idx, table := range c.flowsTables { - if start.After(table.Oldest) { + if start.After(table.Oldest.Add(table.Resolution)) { candidates = append(candidates, idx) } } @@ -46,12 +46,22 @@ func (c *Component) queryFlowsTable(query string, start, end time.Time, targetRe // No candidate, fallback to the one with oldest data best := 0 for idx, table := range c.flowsTables { - if c.flowsTables[best].Oldest.After(table.Oldest) { + if c.flowsTables[best].Oldest.After(table.Oldest.Add(table.Resolution)) { best = idx } } candidates = []int{best} - } else if len(candidates) > 1 { + // Add other candidates that are not far off in term of oldest data + for idx, table := range c.flowsTables { + if idx == best { + continue + } + if c.flowsTables[best].Oldest.After(table.Oldest) { + candidates = append(candidates, idx) + } + } + } + if len(candidates) > 1 { // Use resolution to find the best one best := 0 for _, idx := range candidates { diff --git a/console/clickhouse_test.go b/console/clickhouse_test.go index 04726a45..0c5cad84 100644 --- a/console/clickhouse_test.go +++ b/console/clickhouse_test.go @@ -93,6 +93,13 @@ func TestQueryFlowsTables(t *testing.T) { Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC), End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC), Expected: "SELECT 1 FROM flows WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:10', 'UTC') AND toDateTime('2022-04-11 15:45:10', 'UTC')", + }, { + Description: "timefilter.Start and timefilter.Stop", + Tables: []flowsTable{{"flows", 0, time.Date(2022, 03, 10, 15, 45, 10, 0, time.UTC)}}, + Query: "SELECT {timefilter.Start}, {timefilter.Stop}", + Start: time.Date(2022, 04, 10, 15, 45, 10, 0, time.UTC), + End: time.Date(2022, 04, 11, 15, 45, 10, 0, time.UTC), + Expected: "SELECT toDateTime('2022-04-10 15:45:10', 'UTC'), toDateTime('2022-04-11 15:45:10', 'UTC')", }, { Description: "only flows table and out of range request", Tables: []flowsTable{{"flows", 0, time.Date(2022, 04, 10, 22, 45, 10, 0, time.UTC)}}, @@ -168,6 +175,18 @@ func TestQueryFlowsTables(t *testing.T) { End: time.Date(2022, 04, 11, 15, 46, 10, 0, time.UTC), Resolution: 2 * time.Minute, Expected: "SELECT 1 FROM flows_5m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:45:00', 'UTC') AND toDateTime('2022-04-11 15:45:00', 'UTC')", + }, { + Description: "select best resolution when equality for oldest data", + Tables: []flowsTable{ + {"flows", 0, time.Date(2022, 04, 10, 22, 40, 55, 0, time.UTC)}, + {"flows_1m0s", time.Minute, time.Date(2022, 04, 10, 22, 40, 00, 0, time.UTC)}, + {"flows_1h0m0s", time.Hour, time.Date(2022, 04, 10, 22, 00, 10, 0, time.UTC)}, + }, + Query: "SELECT 1 FROM {table} WHERE {timefilter}", + Start: time.Date(2022, 04, 10, 15, 46, 10, 0, time.UTC), + End: time.Date(2022, 04, 11, 15, 46, 10, 0, time.UTC), + Resolution: 2 * time.Minute, + Expected: "SELECT 1 FROM flows_1m0s WHERE TimeReceived BETWEEN toDateTime('2022-04-10 15:46:00', 'UTC') AND toDateTime('2022-04-11 15:46:00', 'UTC')", }, }