diff --git a/runtime/drivers/registry.go b/runtime/drivers/registry.go index 1b7cd6079e8..ad7e9da4ef4 100644 --- a/runtime/drivers/registry.go +++ b/runtime/drivers/registry.go @@ -172,6 +172,7 @@ func (i *Instance) Config() (InstanceConfig, error) { MetricsApproximateComparisonsCTE: false, MetricsApproxComparisonTwoPhaseLimit: 250, MetricsExactifyDruidTopN: false, + MetricsNullFillingImplementation: "pushdown", AlertsDefaultStreamingRefreshCron: "0 0 * * *", // Every 24 hours AlertsFastStreamingRefreshCron: "*/10 * * * *", // Every 10 minutes } diff --git a/runtime/metricsview/ast.go b/runtime/metricsview/ast.go index 5d2a6cfee3e..1a4b124b083 100644 --- a/runtime/metricsview/ast.go +++ b/runtime/metricsview/ast.go @@ -1040,27 +1040,35 @@ func (a *AST) buildBaseSelect(alias string, comparison bool) (*SelectNode, error return nil, fmt.Errorf("failed to add time range: %w", err) } + err = a.addSpineSelect(n, tr, comparison) + if err != nil { + return nil, err + } + + return n, nil +} + +func (a *AST) addSpineSelect(baseSelect *SelectNode, timeRange *TimeRange, comparison bool) error { // If there is a spine, we wrap the base SELECT in a new SELECT that we add the spine to. // We do not join the spine directly to the FromTable because the join would be evaluated before the GROUP BY, // which would impact the measure aggregations (e.g. counts per group would be wrong). if a.Query.Spine != nil && !(a.Query.Spine.TimeRange != nil && comparison) { // Skip time range spines in the comparison select - sn, err := a.buildSpineSelect(a.GenerateIdentifier(), a.Query.Spine, tr) + sn, err := a.buildSpineSelect(a.GenerateIdentifier(), a.Query.Spine, timeRange) if err != nil { - return nil, err + return err } - a.WrapSelect(n, a.GenerateIdentifier()) - n.SpineSelect = sn + a.WrapSelect(baseSelect, a.GenerateIdentifier()) + baseSelect.SpineSelect = sn // Update the dimension fields to derive from the SpineSelect instead of the FromSelect // (since by definition, some dimension values in the spine might not be present in FromSelect). - for i, f := range n.DimFields { + for i, f := range baseSelect.DimFields { f.Expr = a.Dialect.EscapeMember(sn.Alias, f.Name) - n.DimFields[i] = f + baseSelect.DimFields[i] = f } } - - return n, nil + return nil } // buildSpineSelect constructs a SELECT node for the given spine of dimension values. @@ -1069,6 +1077,10 @@ func (a *AST) buildSpineSelect(alias string, spine *Spine, tr *TimeRange) (*Sele return nil, nil } + if spine.Where != nil && spine.TimeRange != nil { + return nil, errors.New("spine cannot have both 'where' and 'time_range'") + } + if spine.Where != nil { // Using buildWhereForUnderlyingTable to include security filters. // Note that buildWhereForUnderlyingTable handles nil expressions gracefully. diff --git a/runtime/queries/metricsview_timeseries_test.go b/runtime/queries/metricsview_timeseries_test.go index a085e62a863..02a28dd1206 100644 --- a/runtime/queries/metricsview_timeseries_test.go +++ b/runtime/queries/metricsview_timeseries_test.go @@ -169,10 +169,8 @@ func TestMetricsViewsTimeseries_quarter_grain_IST(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, q.Result) rows := q.Result.Data - require.Len(t, rows, 6) + require.Len(t, rows, 5) i := 0 - require.Equal(t, parseTime(t, "2022-10-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) - i++ require.Equal(t, parseTime(t, "2022-12-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) i++ require.Equal(t, parseTime(t, "2023-03-31T18:30:00Z").AsTime(), rows[i].Ts.AsTime()) @@ -348,7 +346,7 @@ func TestMetricsViewTimeSeries_DayLightSavingsBackwards_Continuous_Second(t *tes rows := q.Result.Data require.Len(t, rows, 1) i := 0 - require.Equal(t, parseTime(t, "2023-11-05T05:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.Equal(t, parseTime(t, "2023-11-05T06:00:00Z").AsTime(), rows[i].Ts.AsTime()) q = &queries.MetricsViewTimeSeries{ MeasureNames: []string{"total_records"}, @@ -388,7 +386,7 @@ func TestMetricsViewTimeSeries_DayLightSavingsBackwards_Continuous_Minute(t *tes rows := q.Result.Data require.Len(t, rows, 1) i := 0 - require.Equal(t, parseTime(t, "2023-11-05T05:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.Equal(t, parseTime(t, "2023-11-05T06:00:00Z").AsTime(), rows[i].Ts.AsTime()) q = &queries.MetricsViewTimeSeries{ MeasureNames: []string{"total_records"}, @@ -426,14 +424,13 @@ func TestMetricsViewTimeSeries_DayLightSavingsBackwards_Continuous_Hourly(t *tes require.NoError(t, err) require.NotEmpty(t, q.Result) rows := q.Result.Data - require.Len(t, rows, 5) + require.Len(t, rows, 4) i := 0 require.Equal(t, parseTime(t, "2023-11-05T03:00:00Z").AsTime(), rows[i].Ts.AsTime()) i++ require.Equal(t, parseTime(t, "2023-11-05T04:00:00Z").AsTime(), rows[i].Ts.AsTime()) i++ - require.Equal(t, parseTime(t, "2023-11-05T05:00:00Z").AsTime(), rows[i].Ts.AsTime()) - i++ + // no 05:00 hour since 04:00 to 05:00 UTC are same because of DST fall back require.Equal(t, parseTime(t, "2023-11-05T06:00:00Z").AsTime(), rows[i].Ts.AsTime()) i++ require.Equal(t, parseTime(t, "2023-11-05T07:00:00Z").AsTime(), rows[i].Ts.AsTime()) @@ -460,7 +457,7 @@ func TestMetricsViewTimeSeries_DayLightSavingsBackwards_Sparse_Hourly(t *testing require.NoError(t, err) require.NotEmpty(t, q.Result) rows := q.Result.Data - require.Len(t, rows, 5) + require.Len(t, rows, 4) i := 0 require.Equal(t, parseTime(t, "2023-11-05T03:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.NotNil(t, q.Result.Data[i].Records.AsMap()["total_records"]) @@ -468,9 +465,7 @@ func TestMetricsViewTimeSeries_DayLightSavingsBackwards_Sparse_Hourly(t *testing require.Equal(t, parseTime(t, "2023-11-05T04:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.Nil(t, q.Result.Data[i].Records.AsMap()["total_records"]) i++ - require.Equal(t, parseTime(t, "2023-11-05T05:00:00Z").AsTime(), rows[i].Ts.AsTime()) - require.NotNil(t, q.Result.Data[i].Records.AsMap()["total_records"]) - i++ + // no 05:00 hour since 04:00 to 05:00 UTC are same because of DST fall back require.Equal(t, parseTime(t, "2023-11-05T06:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.Nil(t, q.Result.Data[i].Records.AsMap()["total_records"]) i++ @@ -587,7 +582,7 @@ func TestMetricsViewTimeSeries_DayLightSavingsForwards_Continuous_Hourly(t *test require.NoError(t, err) require.NotEmpty(t, q.Result) rows := q.Result.Data - require.Len(t, rows, 5) + require.Len(t, rows, 6) i := 0 require.Equal(t, parseTime(t, "2023-03-12T04:00:00Z").AsTime(), rows[i].Ts.AsTime()) i++ @@ -597,6 +592,8 @@ func TestMetricsViewTimeSeries_DayLightSavingsForwards_Continuous_Hourly(t *test i++ require.Equal(t, parseTime(t, "2023-03-12T07:00:00Z").AsTime(), rows[i].Ts.AsTime()) i++ + require.Equal(t, parseTime(t, "2023-03-12T07:00:00Z").AsTime(), rows[i].Ts.AsTime()) + i++ require.Equal(t, parseTime(t, "2023-03-12T08:00:00Z").AsTime(), rows[i].Ts.AsTime()) } @@ -621,7 +618,7 @@ func TestMetricsViewTimeSeries_DayLightSavingsForwards_Sparse_Hourly(t *testing. require.NoError(t, err) require.NotEmpty(t, q.Result) rows := q.Result.Data - require.Len(t, rows, 5) + require.Len(t, rows, 6) i := 0 require.Equal(t, parseTime(t, "2023-03-12T04:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.Nil(t, q.Result.Data[i].Records.AsMap()["total_records"]) @@ -635,6 +632,9 @@ func TestMetricsViewTimeSeries_DayLightSavingsForwards_Sparse_Hourly(t *testing. require.Equal(t, parseTime(t, "2023-03-12T07:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.NotNil(t, q.Result.Data[i].Records.AsMap()["total_records"]) i++ + require.Equal(t, parseTime(t, "2023-03-12T07:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["total_records"]) + i++ require.Equal(t, parseTime(t, "2023-03-12T08:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.Nil(t, q.Result.Data[i].Records.AsMap()["total_records"]) } @@ -674,23 +674,11 @@ func TestMetricsViewTimeSeries_having_clause(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, q.Result) rows := q.Result.Data - require.Len(t, rows, 6) + require.Len(t, rows, 2) i := 0 require.Equal(t, parseTime(t, "2019-01-01T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.NotNil(t, q.Result.Data[i].Records.AsMap()["sum_imps"]) i++ - require.Equal(t, parseTime(t, "2019-01-02T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) - require.Nil(t, q.Result.Data[i].Records.AsMap()["sum_imps"]) - i++ - require.Equal(t, parseTime(t, "2019-01-03T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) - require.Nil(t, q.Result.Data[i].Records.AsMap()["sum_imps"]) - i++ - require.Equal(t, parseTime(t, "2019-01-04T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) - require.Nil(t, q.Result.Data[i].Records.AsMap()["sum_imps"]) - i++ - require.Equal(t, parseTime(t, "2019-01-05T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) - require.Nil(t, q.Result.Data[i].Records.AsMap()["sum_imps"]) - i++ require.Equal(t, parseTime(t, "2019-01-06T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) require.NotNil(t, q.Result.Data[i].Records.AsMap()["sum_imps"]) } @@ -738,16 +726,47 @@ func TestMetricsTimeseries_measure_filters_same_name(t *testing.T) { err = q.Resolve(context.Background(), rt, instanceID, 0) require.NoError(t, err) require.NotEmpty(t, q.Result) - outputResult(q.Result.Meta, q.Result.Data) + rows := q.Result.Data + require.Len(t, rows, 13) i := 0 - require.Equal(t, "null", fieldsToString(q.Result.Data[i].Records, "bid_price")) + require.Equal(t, parseTime(t, "2022-01-03T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) i++ - require.Equal(t, "null", fieldsToString(q.Result.Data[i].Records, "bid_price")) + require.Equal(t, parseTime(t, "2022-01-04T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) i++ - require.Equal(t, "3", fieldsToString(q.Result.Data[i].Records, "bid_price")) + require.Equal(t, parseTime(t, "2022-01-06T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) i++ - require.Equal(t, "3", fieldsToString(q.Result.Data[i].Records, "bid_price")) - + require.Equal(t, parseTime(t, "2022-01-07T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-08T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-09T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-11T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-12T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-13T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-15T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-18T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-21T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) + i++ + require.Equal(t, parseTime(t, "2022-01-23T00:00:00Z").AsTime(), rows[i].Ts.AsTime()) + require.NotNil(t, q.Result.Data[i].Records.AsMap()["bid_price"]) } func toStructpbValue(t *testing.T, v any) *structpb.Value { @@ -755,16 +774,3 @@ func toStructpbValue(t *testing.T, v any) *structpb.Value { require.NoError(t, err) return sv } - -func outputResult(schema []*runtimev1.MetricsViewColumn, data []*runtimev1.TimeSeriesValue) { - for _, s := range schema { - fmt.Printf("%v,", s.Name) - } - fmt.Println() - for i, row := range data { - for _, s := range schema { - fmt.Printf("%s %v,", row.Ts.AsTime().Format(time.RFC3339), row.Records.Fields[s.Name].AsInterface()) - } - fmt.Printf(" %d \n", i) - } -}