From 3ada047e94c0281f2006371d78a41341138ea8c0 Mon Sep 17 00:00:00 2001 From: Parag Jain Date: Fri, 19 Dec 2025 18:34:07 +0530 Subject: [PATCH 1/9] use pushdown as default null filling implementation for timeseries API --- runtime/drivers/registry.go | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/drivers/registry.go b/runtime/drivers/registry.go index 1b7cd6079e8..ad7e9da4ef4 100644 --- a/runtime/drivers/registry.go +++ b/runtime/drivers/registry.go @@ -172,6 +172,7 @@ func (i *Instance) Config() (InstanceConfig, error) { MetricsApproximateComparisonsCTE: false, MetricsApproxComparisonTwoPhaseLimit: 250, MetricsExactifyDruidTopN: false, + MetricsNullFillingImplementation: "pushdown", AlertsDefaultStreamingRefreshCron: "0 0 * * *", // Every 24 hours AlertsFastStreamingRefreshCron: "*/10 * * * *", // Every 10 minutes } From 46c6b7ced32e0026daf890bea2d5d3517198ec26 Mon Sep 17 00:00:00 2001 From: Parag Jain Date: Sat, 20 Dec 2025 00:35:17 +0530 Subject: [PATCH 2/9] use first day, month --- runtime/drivers/olap.go | 4 ++-- runtime/metricsview/ast.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/drivers/olap.go b/runtime/drivers/olap.go index 2f0d8f51832..28cd27e7559 100644 --- a/runtime/drivers/olap.go +++ b/runtime/drivers/olap.go @@ -616,9 +616,9 @@ func (d Dialect) IntervalSubtract(tsExpr, unitExpr string, grain runtimev1.TimeG } } -func (d Dialect) SelectTimeRangeBins(start, end time.Time, grain runtimev1.TimeGrain, alias string, tz *time.Location) (string, []any, error) { +func (d Dialect) SelectTimeRangeBins(start, end time.Time, grain runtimev1.TimeGrain, alias string, tz *time.Location, firstDay, firstMonth int) (string, []any, error) { g := timeutil.TimeGrainFromAPI(grain) - start = timeutil.TruncateTime(start, g, tz, 1, 1) + start = timeutil.TruncateTime(start, g, tz, firstDay, firstMonth) var args []any switch d { case DialectDuckDB: diff --git a/runtime/metricsview/ast.go b/runtime/metricsview/ast.go index 7e0f1cc76e7..5d2a6cfee3e 100644 --- a/runtime/metricsview/ast.go +++ b/runtime/metricsview/ast.go @@ -1128,7 +1128,7 @@ func (a *AST) buildSpineSelect(alias string, spine *Spine, tr *TimeRange) (*Sele } } - sel, args, err := a.Dialect.SelectTimeRangeBins(spine.TimeRange.Start, spine.TimeRange.End, spine.TimeRange.Grain.ToProto(), timeAlias, tz) + sel, args, err := a.Dialect.SelectTimeRangeBins(spine.TimeRange.Start, spine.TimeRange.End, spine.TimeRange.Grain.ToProto(), timeAlias, tz, int(a.MetricsView.FirstDayOfWeek), int(a.MetricsView.FirstMonthOfYear)) if err != nil { return nil, fmt.Errorf("failed to generate time spine: %w", err) } From e59d6a1c6fa0232d00d76ad4ad867f7afce2eb8a Mon Sep 17 00:00:00 2001 From: Parag Jain Date: Mon, 22 Dec 2025 17:27:35 +0530 Subject: [PATCH 3/9] don't use range func to time bins for duckdb --- runtime/drivers/olap.go | 4 +--- runtime/queries/metricsview_timeseries_test.go | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/runtime/drivers/olap.go b/runtime/drivers/olap.go index 28cd27e7559..94254e3d522 100644 --- a/runtime/drivers/olap.go +++ b/runtime/drivers/olap.go @@ -621,8 +621,6 @@ func (d Dialect) SelectTimeRangeBins(start, end time.Time, grain runtimev1.TimeG start = timeutil.TruncateTime(start, g, tz, firstDay, firstMonth) var args []any switch d { - case DialectDuckDB: - return fmt.Sprintf("SELECT timezone('%s', range) AS %s FROM range('%s'::TIMESTAMP, '%s'::TIMESTAMP, INTERVAL '1 %s')", tz.String(), d.EscapeIdentifier(alias), start.In(tz).Format(time.DateTime), end.In(tz).Format(time.DateTime), d.ConvertToDateTruncSpecifier(grain)), nil, nil case DialectClickHouse: // format - SELECT c1 AS "alias" FROM VALUES(toDateTime('2021-01-01 00:00:00'), toDateTime('2021-01-01 00:00:00'),...) var sb strings.Builder @@ -636,7 +634,7 @@ func (d Dialect) SelectTimeRangeBins(start, end time.Time, grain runtimev1.TimeG } sb.WriteString(")") return sb.String(), args, nil - case DialectDruid, DialectPinot: + case DialectDuckDB, DialectDruid, DialectPinot: // generate select like - SELECT * FROM ( // VALUES // (CAST('2006-01-02T15:04:05Z' AS TIMESTAMP)), diff --git a/runtime/queries/metricsview_timeseries_test.go b/runtime/queries/metricsview_timeseries_test.go index a085e62a863..aa6c4a5ec32 100644 --- a/runtime/queries/metricsview_timeseries_test.go +++ b/runtime/queries/metricsview_timeseries_test.go @@ -169,10 +169,8 @@ func TestMetricsViewsTimeseries_quarter_grain_IST(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, q.Result) rows := q.Result.Data - require.Len(t, rows, 6) + require.Len(t, rows, 5) i := 0 - require.Equal(t, parseTime(t, "2022-10-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) - i++ require.Equal(t, parseTime(t, "2022-12-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) i++ require.Equal(t, parseTime(t, "2023-03-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) From db913fae4ec3fac70c8679018b03b226e17079f9 Mon Sep 17 00:00:00 2001 From: Parag Jain Date: Wed, 24 Dec 2025 16:20:35 +0530 Subject: [PATCH 4/9] change test as per duckdb range behaviour --- runtime/drivers/olap.go | 6 ++++- .../queries/metricsview_timeseries_test.go | 24 ++++++++++--------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/runtime/drivers/olap.go b/runtime/drivers/olap.go index 94254e3d522..dc96785d95c 100644 --- a/runtime/drivers/olap.go +++ b/runtime/drivers/olap.go @@ -621,6 +621,10 @@ func (d Dialect) SelectTimeRangeBins(start, end time.Time, grain runtimev1.TimeG start = timeutil.TruncateTime(start, g, tz, firstDay, firstMonth) var args []any switch d { + case DialectDuckDB: + // first convert start and end to the target timezone as the application sends UTC representation of the time, so it will send `2024-03-12T18:30:00Z` for the 13th day of March in Asia/Kolkata timezone (`2024-03-13T00:00:00Z`) + // then let duckdb range over it and then convert back to the target timezone + return fmt.Sprintf("SELECT timezone('%s', range) AS %s FROM range('%s'::TIMESTAMP, '%s'::TIMESTAMP, INTERVAL '1 %s')", tz.String(), d.EscapeIdentifier(alias), start.In(tz).Format(time.DateTime), end.In(tz).Format(time.DateTime), d.ConvertToDateTruncSpecifier(grain)), nil, nil case DialectClickHouse: // format - SELECT c1 AS "alias" FROM VALUES(toDateTime('2021-01-01 00:00:00'), toDateTime('2021-01-01 00:00:00'),...) var sb strings.Builder @@ -634,7 +638,7 @@ func (d Dialect) SelectTimeRangeBins(start, end time.Time, grain runtimev1.TimeG } sb.WriteString(")") return sb.String(), args, nil - case DialectDuckDB, DialectDruid, DialectPinot: + case DialectDruid, DialectPinot: // generate select like - SELECT * FROM ( // VALUES // (CAST('2006-01-02T15:04:05Z' AS TIMESTAMP)), diff --git a/runtime/queries/metricsview_timeseries_test.go b/runtime/queries/metricsview_timeseries_test.go index aa6c4a5ec32..6b38fd9e0a8 100644 --- a/runtime/queries/metricsview_timeseries_test.go +++ b/runtime/queries/metricsview_timeseries_test.go @@ -346,7 +346,7 @@ func TestMetricsViewTimeSeries_DayLightSavingsBackwards_Continuous_Second(t *tes rows := q.Result.Data require.Len(t, rows, 1) i := 0 - require.Equal(t, parseTime(t, "2023-11-05T05:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.Equal(t, parseTime(t, "2023-11-05T06:00:00Z").AsTime(), rows[i].Ts.AsTime()) q = &queries.MetricsViewTimeSeries{ MeasureNames: []string{"total_records"}, @@ -386,7 +386,7 @@ func TestMetricsViewTimeSeries_DayLightSavingsBackwards_Continuous_Minute(t *tes rows := q.Result.Data require.Len(t, rows, 1) i := 0 - require.Equal(t, parseTime(t, "2023-11-05T05:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.Equal(t, parseTime(t, "2023-11-05T06:00:00Z").AsTime(), rows[i].Ts.AsTime()) q = &queries.MetricsViewTimeSeries{ MeasureNames: []string{"total_records"}, @@ -424,14 +424,13 @@ func TestMetricsViewTimeSeries_DayLightSavingsBackwards_Continuous_Hourly(t *tes require.NoError(t, err) require.NotEmpty(t, q.Result) rows := q.Result.Data - require.Len(t, rows, 5) + require.Len(t, rows, 4) i := 0 require.Equal(t, parseTime(t, "2023-11-05T03:00:00Z").AsTime(), rows[i].Ts.AsTime()) i++ require.Equal(t, parseTime(t, "2023-11-05T04:00:00Z").AsTime(), rows[i].Ts.AsTime()) i++ - require.Equal(t, parseTime(t, "2023-11-05T05:00:00Z").AsTime(), rows[i].Ts.AsTime()) - i++ + // no 05:00 hour since 04:00 to 05:00 UTC are same because of DST fall back require.Equal(t, parseTime(t, "2023-11-05T06:00:00Z").AsTime(), rows[i].Ts.AsTime()) i++ require.Equal(t, parseTime(t, "2023-11-05T07:00:00Z").AsTime(), rows[i].Ts.AsTime()) @@ -458,7 +457,7 @@ func TestMetricsViewTimeSeries_DayLightSavingsBackwards_Sparse_Hourly(t *testing require.NoError(t, err) require.NotEmpty(t, q.Result) rows := q.Result.Data - require.Len(t, rows, 5) + require.Len(t, rows, 4) i := 0 require.Equal(t, parseTime(t, "2023-11-05T03:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.NotNil(t, q.Result.Data[i].Records.AsMap()["total_records"]) @@ -466,9 +465,7 @@ func TestMetricsViewTimeSeries_DayLightSavingsBackwards_Sparse_Hourly(t *testing require.Equal(t, parseTime(t, "2023-11-05T04:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.Nil(t, q.Result.Data[i].Records.AsMap()["total_records"]) i++ - require.Equal(t, parseTime(t, "2023-11-05T05:00:00Z").AsTime(), rows[i].Ts.AsTime()) - require.NotNil(t, q.Result.Data[i].Records.AsMap()["total_records"]) - i++ + // no 05:00 hour since 04:00 to 05:00 UTC are same because of DST fall back require.Equal(t, parseTime(t, "2023-11-05T06:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.Nil(t, q.Result.Data[i].Records.AsMap()["total_records"]) i++ @@ -585,7 +582,7 @@ func TestMetricsViewTimeSeries_DayLightSavingsForwards_Continuous_Hourly(t *test require.NoError(t, err) require.NotEmpty(t, q.Result) rows := q.Result.Data - require.Len(t, rows, 5) + require.Len(t, rows, 6) i := 0 require.Equal(t, parseTime(t, "2023-03-12T04:00:00Z").AsTime(), rows[i].Ts.AsTime()) i++ @@ -595,6 +592,8 @@ func TestMetricsViewTimeSeries_DayLightSavingsForwards_Continuous_Hourly(t *test i++ require.Equal(t, parseTime(t, "2023-03-12T07:00:00Z").AsTime(), rows[i].Ts.AsTime()) i++ + require.Equal(t, parseTime(t, "2023-03-12T07:00:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ require.Equal(t, parseTime(t, "2023-03-12T08:00:00Z").AsTime(), rows[i].Ts.AsTime()) } @@ -619,7 +618,7 @@ func TestMetricsViewTimeSeries_DayLightSavingsForwards_Sparse_Hourly(t *testing. require.NoError(t, err) require.NotEmpty(t, q.Result) rows := q.Result.Data - require.Len(t, rows, 5) + require.Len(t, rows, 6) i := 0 require.Equal(t, parseTime(t, "2023-03-12T04:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.Nil(t, q.Result.Data[i].Records.AsMap()["total_records"]) @@ -633,6 +632,9 @@ func TestMetricsViewTimeSeries_DayLightSavingsForwards_Sparse_Hourly(t *testing. require.Equal(t, parseTime(t, "2023-03-12T07:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.NotNil(t, q.Result.Data[i].Records.AsMap()["total_records"]) i++ + require.Equal(t, parseTime(t, "2023-03-12T07:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["total_records"]) + i++ require.Equal(t, parseTime(t, "2023-03-12T08:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.Nil(t, q.Result.Data[i].Records.AsMap()["total_records"]) } From 5152580b17c2577c7f662d103e5b59b855b1b120 Mon Sep 17 00:00:00 2001 From: Parag Jain Date: Mon, 29 Dec 2025 11:46:00 +0530 Subject: [PATCH 5/9] fix having timespine coalesce circular dependency --- runtime/metricsview/ast.go | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/runtime/metricsview/ast.go b/runtime/metricsview/ast.go index 5d2a6cfee3e..0edf9579605 100644 --- a/runtime/metricsview/ast.go +++ b/runtime/metricsview/ast.go @@ -304,6 +304,7 @@ func NewAST(mv *runtimev1.MetricsViewSpec, sec MetricsViewSecurity, qry *Query, // Handle Having. If the root node is grouped, we add it as a HAVING clause, otherwise wrap it in a SELECT and add it as a WHERE clause. if ast.Query.Having != nil { + spineSelect := ast.Root.SpineSelect // We need to wrap in a new SELECT because a WHERE/HAVING clause cannot apply directly to a field with a window function. // This also enables us to template the field name instead of the field expression into the expression. ast.WrapSelect(ast.Root, ast.GenerateIdentifier()) @@ -318,6 +319,22 @@ func NewAST(mv *runtimev1.MetricsViewSpec, sec MetricsViewSecurity, qry *Query, } ast.Root.Where = res + + // time spine needs to be applied before having so that treat_nulls_as works correctly for derived measures + // and treat_nulls_as should be applied before Having so that wrong bins are not included in final results + // however having also removes null filling bins that does not match criteria, so its depended on time spine + // this creates a cyclic dependency between time spine and having, so we re-apply time spine after having to restore removed time bins + if spineSelect != nil && ast.Query.Spine.TimeRange != nil { + ast.WrapSelect(ast.Root, ast.GenerateIdentifier()) + ast.Root.SpineSelect = ast.ShallowCopyWithAlias(spineSelect, ast.GenerateIdentifier()) + + // Update the dimension fields to derive from the SpineSelect instead of the FromSelect + // (since by definition, some dimension values in the spine might not be present in FromSelect). + for i, f := range ast.Root.DimFields { + f.Expr = ast.Dialect.EscapeMember(ast.Root.SpineSelect.Alias, f.Name) + ast.Root.DimFields[i] = f + } + } } // Incrementally add each sort criterion. @@ -692,6 +709,32 @@ func (a *AST) WrapSelect(s *SelectNode, innerAlias string) { s.CrossJoinSelects = nil } +func (a *AST) ShallowCopyWithAlias(s *SelectNode, newAlias string) *SelectNode { + cpy := &SelectNode{ + RawSelect: s.RawSelect, + Alias: newAlias, + IsCTE: s.IsCTE, + DimFields: s.DimFields, + MeasureFields: s.MeasureFields, + FromTable: s.FromTable, + FromSelect: s.FromSelect, + SpineSelect: s.SpineSelect, + LeftJoinSelects: s.LeftJoinSelects, + CrossJoinSelects: s.CrossJoinSelects, + JoinComparisonSelect: s.JoinComparisonSelect, + JoinComparisonType: s.JoinComparisonType, + Unnests: s.Unnests, + Group: s.Group, + Where: s.Where, + TimeWhere: s.TimeWhere, + Having: s.Having, + OrderBy: s.OrderBy, + Limit: s.Limit, + Offset: s.Offset, + } + return cpy +} + // ConvertToCTE util func that sets IsCTE and only adds to a.CTEs if IsCTE was false func (a *AST) ConvertToCTE(n *SelectNode) { if n.IsCTE { From 03f432ac978e542143f1034eb7cb8e7c26833ce2 Mon Sep 17 00:00:00 2001 From: Parag Jain Date: Mon, 29 Dec 2025 13:11:49 +0530 Subject: [PATCH 6/9] refactor --- runtime/metricsview/ast.go | 46 ++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/runtime/metricsview/ast.go b/runtime/metricsview/ast.go index 0edf9579605..0559ce75166 100644 --- a/runtime/metricsview/ast.go +++ b/runtime/metricsview/ast.go @@ -304,7 +304,6 @@ func NewAST(mv *runtimev1.MetricsViewSpec, sec MetricsViewSecurity, qry *Query, // Handle Having. If the root node is grouped, we add it as a HAVING clause, otherwise wrap it in a SELECT and add it as a WHERE clause. if ast.Query.Having != nil { - spineSelect := ast.Root.SpineSelect // We need to wrap in a new SELECT because a WHERE/HAVING clause cannot apply directly to a field with a window function. // This also enables us to template the field name instead of the field expression into the expression. ast.WrapSelect(ast.Root, ast.GenerateIdentifier()) @@ -320,19 +319,14 @@ func NewAST(mv *runtimev1.MetricsViewSpec, sec MetricsViewSecurity, qry *Query, ast.Root.Where = res - // time spine needs to be applied before having so that treat_nulls_as works correctly for derived measures - // and treat_nulls_as should be applied before Having so that wrong bins are not included in final results - // however having also removes null filling bins that does not match criteria, so its depended on time spine - // this creates a cyclic dependency between time spine and having, so we re-apply time spine after having to restore removed time bins - if spineSelect != nil && ast.Query.Spine.TimeRange != nil { - ast.WrapSelect(ast.Root, ast.GenerateIdentifier()) - ast.Root.SpineSelect = ast.ShallowCopyWithAlias(spineSelect, ast.GenerateIdentifier()) - - // Update the dimension fields to derive from the SpineSelect instead of the FromSelect - // (since by definition, some dimension values in the spine might not be present in FromSelect). - for i, f := range ast.Root.DimFields { - f.Expr = ast.Dialect.EscapeMember(ast.Root.SpineSelect.Alias, f.Name) - ast.Root.DimFields[i] = f + if ast.Query.Spine != nil && ast.Query.Spine.TimeRange != nil { + // time spine needs to be applied before having so that treat_nulls_as works correctly for derived measures + // and treat_nulls_as should be applied before Having so that wrong bins are not included in final results + // however having also removes null filling bins that does not match criteria, so its depended on time spine + // this creates a cyclic dependency between time spine and having, so we re-apply time spine after having to restore removed time bins + err = ast.addSpineSelect(ast.Root, ast.Query.TimeRange, false) + if err != nil { + return nil, err } } } @@ -1083,27 +1077,35 @@ func (a *AST) buildBaseSelect(alias string, comparison bool) (*SelectNode, error return nil, fmt.Errorf("failed to add time range: %w", err) } + err = a.addSpineSelect(n, tr, comparison) + if err != nil { + return nil, err + } + + return n, nil +} + +func (a *AST) addSpineSelect(baseSelect *SelectNode, timeRange *TimeRange, comparison bool) error { // If there is a spine, we wrap the base SELECT in a new SELECT that we add the spine to. // We do not join the spine directly to the FromTable because the join would be evaluated before the GROUP BY, // which would impact the measure aggregations (e.g. counts per group would be wrong). if a.Query.Spine != nil && !(a.Query.Spine.TimeRange != nil && comparison) { // Skip time range spines in the comparison select - sn, err := a.buildSpineSelect(a.GenerateIdentifier(), a.Query.Spine, tr) + sn, err := a.buildSpineSelect(a.GenerateIdentifier(), a.Query.Spine, timeRange) if err != nil { - return nil, err + return err } - a.WrapSelect(n, a.GenerateIdentifier()) - n.SpineSelect = sn + a.WrapSelect(baseSelect, a.GenerateIdentifier()) + baseSelect.SpineSelect = sn // Update the dimension fields to derive from the SpineSelect instead of the FromSelect // (since by definition, some dimension values in the spine might not be present in FromSelect). - for i, f := range n.DimFields { + for i, f := range baseSelect.DimFields { f.Expr = a.Dialect.EscapeMember(sn.Alias, f.Name) - n.DimFields[i] = f + baseSelect.DimFields[i] = f } } - - return n, nil + return nil } // buildSpineSelect constructs a SELECT node for the given spine of dimension values. From 5bc24904e3a26515564a1947f28a3c958dbcbcbf Mon Sep 17 00:00:00 2001 From: Parag Jain Date: Mon, 29 Dec 2025 13:41:09 +0530 Subject: [PATCH 7/9] check --- runtime/metricsview/ast.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/runtime/metricsview/ast.go b/runtime/metricsview/ast.go index 0559ce75166..480c863b65b 100644 --- a/runtime/metricsview/ast.go +++ b/runtime/metricsview/ast.go @@ -1114,6 +1114,10 @@ func (a *AST) buildSpineSelect(alias string, spine *Spine, tr *TimeRange) (*Sele return nil, nil } + if spine.Where != nil && spine.TimeRange != nil { + return nil, errors.New("spine cannot have both 'where' and 'time_range'") + } + if spine.Where != nil { // Using buildWhereForUnderlyingTable to include security filters. // Note that buildWhereForUnderlyingTable handles nil expressions gracefully. From 59886a3941e530a76a2e25ad51b8a8ab976fad5b Mon Sep 17 00:00:00 2001 From: Parag Jain Date: Mon, 29 Dec 2025 13:42:12 +0530 Subject: [PATCH 8/9] remove method --- runtime/metricsview/ast.go | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/runtime/metricsview/ast.go b/runtime/metricsview/ast.go index 480c863b65b..5ad6d60203d 100644 --- a/runtime/metricsview/ast.go +++ b/runtime/metricsview/ast.go @@ -703,32 +703,6 @@ func (a *AST) WrapSelect(s *SelectNode, innerAlias string) { s.CrossJoinSelects = nil } -func (a *AST) ShallowCopyWithAlias(s *SelectNode, newAlias string) *SelectNode { - cpy := &SelectNode{ - RawSelect: s.RawSelect, - Alias: newAlias, - IsCTE: s.IsCTE, - DimFields: s.DimFields, - MeasureFields: s.MeasureFields, - FromTable: s.FromTable, - FromSelect: s.FromSelect, - SpineSelect: s.SpineSelect, - LeftJoinSelects: s.LeftJoinSelects, - CrossJoinSelects: s.CrossJoinSelects, - JoinComparisonSelect: s.JoinComparisonSelect, - JoinComparisonType: s.JoinComparisonType, - Unnests: s.Unnests, - Group: s.Group, - Where: s.Where, - TimeWhere: s.TimeWhere, - Having: s.Having, - OrderBy: s.OrderBy, - Limit: s.Limit, - Offset: s.Offset, - } - return cpy -} - // ConvertToCTE util func that sets IsCTE and only adds to a.CTEs if IsCTE was false func (a *AST) ConvertToCTE(n *SelectNode) { if n.IsCTE { From 95e59dde51da1f867729a332ba4bbcfea84d02c2 Mon Sep 17 00:00:00 2001 From: Parag Jain Date: Mon, 29 Dec 2025 22:35:31 +0530 Subject: [PATCH 9/9] simplify --- runtime/metricsview/ast.go | 11 --- .../queries/metricsview_timeseries_test.go | 70 ++++++++++--------- 2 files changed, 38 insertions(+), 43 deletions(-) diff --git a/runtime/metricsview/ast.go b/runtime/metricsview/ast.go index 5ad6d60203d..1a4b124b083 100644 --- a/runtime/metricsview/ast.go +++ b/runtime/metricsview/ast.go @@ -318,17 +318,6 @@ func NewAST(mv *runtimev1.MetricsViewSpec, sec MetricsViewSecurity, qry *Query, } ast.Root.Where = res - - if ast.Query.Spine != nil && ast.Query.Spine.TimeRange != nil { - // time spine needs to be applied before having so that treat_nulls_as works correctly for derived measures - // and treat_nulls_as should be applied before Having so that wrong bins are not included in final results - // however having also removes null filling bins that does not match criteria, so its depended on time spine - // this creates a cyclic dependency between time spine and having, so we re-apply time spine after having to restore removed time bins - err = ast.addSpineSelect(ast.Root, ast.Query.TimeRange, false) - if err != nil { - return nil, err - } - } } // Incrementally add each sort criterion. diff --git a/runtime/queries/metricsview_timeseries_test.go b/runtime/queries/metricsview_timeseries_test.go index 6b38fd9e0a8..02a28dd1206 100644 --- a/runtime/queries/metricsview_timeseries_test.go +++ b/runtime/queries/metricsview_timeseries_test.go @@ -674,23 +674,11 @@ func TestMetricsViewTimeSeries_having_clause(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, q.Result) rows := q.Result.Data - require.Len(t, rows, 6) + require.Len(t, rows, 2) i := 0 require.Equal(t, parseTime(t, "2019-01-01T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.NotNil(t, q.Result.Data[i].Records.AsMap()["sum_imps"]) i++ - require.Equal(t, parseTime(t, "2019-01-02T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) - require.Nil(t, q.Result.Data[i].Records.AsMap()["sum_imps"]) - i++ - require.Equal(t, parseTime(t, "2019-01-03T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) - require.Nil(t, q.Result.Data[i].Records.AsMap()["sum_imps"]) - i++ - require.Equal(t, parseTime(t, "2019-01-04T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) - require.Nil(t, q.Result.Data[i].Records.AsMap()["sum_imps"]) - i++ - require.Equal(t, parseTime(t, "2019-01-05T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) - require.Nil(t, q.Result.Data[i].Records.AsMap()["sum_imps"]) - i++ require.Equal(t, parseTime(t, "2019-01-06T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.NotNil(t, q.Result.Data[i].Records.AsMap()["sum_imps"]) } @@ -738,16 +726,47 @@ func TestMetricsTimeseries_measure_filters_same_name(t *testing.T) { err = q.Resolve(context.Background(), rt, instanceID, 0) require.NoError(t, err) require.NotEmpty(t, q.Result) - outputResult(q.Result.Meta, q.Result.Data) + rows := q.Result.Data + require.Len(t, rows, 13) i := 0 - require.Equal(t, "null", fieldsToString(q.Result.Data[i].Records, "bid_price")) + require.Equal(t, parseTime(t, "2022-01-03T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) i++ - require.Equal(t, "null", fieldsToString(q.Result.Data[i].Records, "bid_price")) + require.Equal(t, parseTime(t, "2022-01-04T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) i++ - require.Equal(t, "3", fieldsToString(q.Result.Data[i].Records, "bid_price")) + require.Equal(t, parseTime(t, "2022-01-06T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) i++ - require.Equal(t, "3", fieldsToString(q.Result.Data[i].Records, "bid_price")) - + require.Equal(t, parseTime(t, "2022-01-07T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-08T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-09T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-11T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-12T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-13T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-15T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-18T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-21T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-23T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) } func toStructpbValue(t *testing.T, v any) *structpb.Value { @@ -755,16 +774,3 @@ func toStructpbValue(t *testing.T, v any) *structpb.Value { require.NoError(t, err) return sv } - -func outputResult(schema []*runtimev1.MetricsViewColumn, data []*runtimev1.TimeSeriesValue) { - for _, s := range schema { - fmt.Printf("%v,", s.Name) - } - fmt.Println() - for i, row := range data { - for _, s := range schema { - fmt.Printf("%s %v,", row.Ts.AsTime().Format(time.RFC3339), row.Records.Fields[s.Name].AsInterface()) - } - fmt.Printf(" %d \n", i) - } -}