diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstByIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstByIT.java new file mode 100644 index 0000000000000..490f8f52262cf --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstByIT.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAggregationFirstByIT { + private static final String DATABASE_NAME = "test_first_by_agg"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table_a(" + + "device STRING TAG, " + + "s_int INT32 FIELD, " + + "s_long INT64 FIELD, " + + "s_float FLOAT FIELD, " + + "s_double DOUBLE FIELD, " + + "s_bool BOOLEAN FIELD, " + + "s_string STRING FIELD, " + + "time_type TIMESTAMP FIELD, " + + "y_criteria INT32 FIELD)", // Acts as s2 + "CREATE TABLE table_b(" + "time TIMESTAMP TIME, " + "s_back INT32 FIELD)", + + // Batch Insert + "INSERT INTO table_a(time, device, s_int, s_long, s_float, s_double, s_bool, s_string, time_type, y_criteria) VALUES " + + // Case 1: s2 (y_criteria) has NO NULLs in Valid Times. + // Device: d1 + + "(1, 'd1', 1, 1, 1.0, 1.0, true, '1s', NULL, 1)," + + "(200, 'd1', 200, 200, 200.0, 200.0, true, '200s', 200, 200)," + + "(100, 'd1', 100, 100, 100.0, 100.0, true, '100s', 100, 100)," + + "(10, 'd1', 10, 10, 10.0, 10.0, true, '10s', 10, 10)," + + "(5, 'd1', 5, 5, 5.0, 5.0, false, '5s', 5, 5)," // Target + + // Case 2: s2 (y_criteria) has NULLs + // Device: d2 + + "(2, 'd2', 2, 2, 2.0, 2.0, true, '2s', NULL, 2)," + + "(5, 'd2', 5, 5, 5.0, 5.0, false, '5s', 5, NULL)," + + "(8, 'd2', 8, 8, 8.0, 8.0, false, '8s', 8, NULL)," + + "(10, 'd2', 10, 10, 10.0, 10.0, true, '10s', 10, 10)," // Target + + "(20, 'd2', 20, 20, 20.0, 20.0, true, '20s', 20, 20)," + + // Case 3: s1 (value) has NULLs. + // Device: d3 + + "(3, 'd3', 3, 3, 3.0, 3.0, true, '3s', NULL, 3)," + + "(5, 'd3', 5, 5, NULL, NULL, NULL, NULL, 5, 5)," // Target + + "(10, 'd3', 10, 10, 10.0, 10.0, true, '10s', 10, NULL)," + + "(20, 'd3', 20, 20, 20.0, 20.0, true, '20s', 20, 20)," + + // Case 4: s2 (y_criteria) is ALL NULLs. + // Device: d4 + + "(4, 'd4', 66, 66, 66.0, 66.0, true, '66s', NULL, NULL)," + + "(5, 'd4', 5, 5, 5.0, 5.0, false, '5s', 5, NULL)," + + "(10, 'd4', 10, 10, 10.0, 10.0, true, '10s', 10, NULL)," + + "(20, 'd4', 20, 20, 20.0, 20.0, true, '20s', 20, NULL)," + + // Case 5: All time_type are NULL. + // Device: d5 + + "(1, 'd5', 10, 10, 10.0, 10.0, true, '10s', NULL, NULL)," + + "(2, 'd5', 50, 50, 50.0, 50.0, false, '50s', NULL, 50)" // target + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testFirstBy_d1_NoNulls() { + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + String[] retArray = {"5,5,5.0,5.0,false,5s,"}; + runTest("d1", expectedHeader, retArray); + } + + @Test + public void testFirstBy_d2_ForwardTracking() { + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + String[] retArray = {"10,10,10.0,10.0,true,10s,"}; + runTest("d2", expectedHeader, retArray); + } + + @Test + public void testFirstBy_d3_TargetNull() { + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + String[] retArray = {"5,5,null,null,null,null,"}; + runTest("d3", expectedHeader, retArray); + } + + @Test + public void testFirstBy_d4_AllNullCriteria() { + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + // Expected: No valid s2 found. + String[] retArray = {"null,null,null,null,null,null,"}; + runTest("d4", expectedHeader, retArray); + } + + @Test + public void testFirstBy_d5_AllTimeNull() { + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + // Expected: The row with y_criteria=NULL is skipped. The row with y_criteria=50 is picked. + String[] retArray = {"50,50,50.0,50.0,false,50s,"}; + runTest("d5", expectedHeader, retArray); + } + + private void runTest(String deviceId, String[] expectedHeader, String[] retArray) { + tableResultSetEqualTest( + "select " + + "first_by(s_int, y_criteria, time), " + + "first_by(s_long, y_criteria, time), " + + "first_by(s_float, y_criteria, time), " + + "first_by(s_double, y_criteria, time), " + + "first_by(s_bool, y_criteria, time), " + + "first_by(s_string, y_criteria, time) " + + "from " + + "(select time_type as time, s_int, s_long, s_float, s_double, s_bool, s_string, y_criteria " + + "from table_a left join table_b on table_a.time=table_b.time " + + "where table_a.device='" + + deviceId + + "') ", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstByInGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstByInGroupIT.java new file mode 100644 index 0000000000000..1045d76eba8fa --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstByInGroupIT.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAggregationFirstByInGroupIT { + + private static final String DATABASE_NAME = "test_first_by_in_group_agg"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table_a(" + + "s_int INT32 FIELD, " + + "s_long INT64 FIELD, " + + "s_float FLOAT FIELD, " + + "s_double DOUBLE FIELD, " + + "s_bool BOOLEAN FIELD, " + + "s_string STRING FIELD, " + + "time_type TIMESTAMP FIELD, " + + "y_criteria INT32 FIELD, " // Acts as s2 + + "partition STRING FIELD)", + "CREATE TABLE table_b(" + "time TIMESTAMP TIME, " + "s_back INT32 FIELD)", + + // Batch Insert: 5 Partitions (p1-p5) + "INSERT INTO table_a(time, s_int, s_long, s_float, s_double, s_bool, s_string, time_type, y_criteria, partition) VALUES " + + // Partition p1: s2 has NO NULLs in Valid Times. + + "(1, 99, 99, 99.0, 99.0, true, '99s', NULL, 99, 'p1')," + + "(2, 200, 200, 200.0, 200.0, true, '200s', 200, 200, 'p1')," + + "(3, 100, 100, 100.0, 100.0, true, '100s', 100, 100, 'p1')," + + "(4, 10, 10, 10.0, 10.0, true, '10s', 10, 10, 'p1')," + + "(5, 5, 5, 5.0, 5.0, false, '5s', 5, 5, 'p1')," // Target + + // Partition p2: s2 has NULLs + + "(6, 88, 88, 88.0, 88.0, true, '88s', NULL, 88, 'p2')," + + "(7, 5, 5, 5.0, 5.0, false, '5s', 5, NULL,'p2')," + + "(8, 8, 8, 8.0, 8.0, false, '8s', 8, NULL,'p2')," + + "(9, 10, 10, 10.0, 10.0, true, '10s', 10, 10, 'p2')," // Target + + "(10, 20, 20, 20.0, 20.0, true, '20s', 20, 20, 'p2')," + + // Partition p3: s1 (value) has NULLs. + + "(11, 77, 77, 77.0, 77.0, true, '77s', NULL, 77, 'p3')," + + "(12, NULL, NULL, 5.0, 5.0, NULL, NULL, 5, 5, 'p3')," // Target + // (Values null) + + "(13, 10, 10, 10.0, 10.0, true, '10s', 10, NULL,'p3')," + + "(14, 20, 20, 20.0, 20.0, true, '20s', 20, 20, 'p3')," + + // Partition p4: s2 is ALL NULLs. + // Logic: No row satisfies the criteria. Result is NULL. + + "(15, 66, 66, 66.0, 66.0, true, '66s', NULL, NULL,'p4')," + + "(16, 5, 5, 5.0, 5.0, false, '5s', 5, NULL,'p4')," + + "(17, 10, 10, 10.0, 10.0, true, '10s', 10, NULL,'p4')," + + "(18, 20, 20, 20.0, 20.0, true, '20s', 20, NULL,'p4')," + + // Partition p5: All time_type are NULL. + + "(19, 10, 10, 10.0, 10.0, true, '10s', NULL, NULL,'p5')," + + "(20, 50, 50, 50.0, 50.0, false, '50s', NULL, 50, 'p5')" // Target + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testGroupedFirstByAggregation() { + + // Expected Header: partition column + 6 aggregation results + String[] expectedHeader = {"partition", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6"}; + + // Expected Results: + String[] retArray = { + "p1,5,5,5.0,5.0,false,5s,", + "p2,10,10,10.0,10.0,true,10s,", + "p3,null,null,5.0,5.0,null,null,", + "p4,null,null,null,null,null,null,", + "p5,50,50,50.0,50.0,false,50s," + }; + + tableResultSetEqualTest( + "select " + + "partition, " + + "first_by(s_int, y_criteria, time), " + + "first_by(s_long, y_criteria, time), " + + "first_by(s_float, y_criteria, time), " + + "first_by(s_double, y_criteria, time), " + + "first_by(s_bool, y_criteria, time), " + + "first_by(s_string, y_criteria, time) " + + "from " + // SubQuery: Rename time_type to 'ts' to avoid ambiguity with physical 'time' + + "(select time_type as time, s_int, s_long, s_float, s_double, s_bool, s_string, y_criteria, partition " + + "from table_a left join table_b on table_a.time=table_b.time) " + + "group by partition " + + "order by partition", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstIT.java new file mode 100644 index 0000000000000..92fc15127b34c --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstIT.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAggregationFirstIT { + + private static final String DATABASE_NAME = "test_null_time_aggs_all_types_first"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table_a(" + + "device STRING TAG, " + + "s_int INT32 FIELD, " + + "s_long INT64 FIELD, " + + "s_float FLOAT FIELD, " + + "s_double DOUBLE FIELD, " + + "s_bool BOOLEAN FIELD, " + + "s_string STRING FIELD, " + + "time_type TIMESTAMP FIELD)", + "CREATE TABLE table_b(" + "time TIMESTAMP TIME, " + "s_back INT32 FIELD)", + + // batch insert：d1(5 rows) + d2(4 rows) + d3(2 rows) + // Values are changed to positive numbers + "INSERT INTO table_a(time, device, s_int, s_long, s_float, s_double, s_bool, s_string, time_type) VALUES " + + // --- Device d1 --- + + "(100, 'd1', 100, 100, 100.0, 100.0, true, '100s', 100)," + + "(50, 'd1', 50, 50, 50.0, 50.0, false, '50s', NULL)," + + "(10, 'd1', 10, 10, 10.0, 10.0, true, '10s', 10)," + + "(5, 'd1', 5, 5, 5.0, 5.0, false, '5s', 5)," + + "(-50, 'd1', -50, -50, -50.0, -50.0, false, '-50s', NULL)," + + // --- Device d2 --- + + "(80, 'd2', 80, 80, 80.0, 80.0, true, '80s', NULL)," + + "(9, 'd2', NULL, 9, 9.0, NULL, false, NULL, 9)," + + "(40, 'd2', 40, 40, 40.0, 40.0, false, '40s', 40)," + + "(-20, 'd2', -20, -20, -20.0, -20.0, true, '-20s', NULL)," + + "(10, 'd2', 10, NULL, NULL, 10.0, NULL, '10s', 10)," + + // --- Device d3 (Pure NULL test) --- + // d3: all time_type are values, but data cols are NULL (to test if first returns null + // value correctly) + + "(200, 'd3', null, null, null, null, null, null, 200)," + + "(150, 'd3', null, null, null, null, null, null, 150)" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testAggregation() { + + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + + // Expected Result for d1: + // Valid time_types: 100, 10, 5. + // FIRST(..., time) looks for MIN(time_type). + // MIN(time_type) is 5. + // Corresponding values at time_type=5: s_int=5, s_long=5, ..., s_bool=false, s_string='5s' + String[] retArray = {"5,5,5.0,5.0,false,5s,"}; + + tableResultSetEqualTest( + "select " + + "first(s_int, time), " + + "first(s_long, time), " + + "first(s_float, time), " + + "first(s_double, time), " + + "first(s_bool, time), " + + "first(s_string, time) " + + "from " + // subQuery: project all the column needed and rename the time_type to the time + + "(select " + + " time_type as time, " + + " s_int, s_long, s_float, s_double, s_bool, s_string " + + "from table_a " + + "left join table_b on table_a.time=table_b.time " + + "where table_a.device='d1') ", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testAggregationWithNullValue() { + + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + + // Expected Result for d2: + // Valid time_types: 40, 5, 4. + // MIN(time_type) is 4. + // Row at time_type=4: + // s_int=4, s_long=NULL, s_float=NULL, s_double=4.0, s_bool=NULL, s_string='4s' + String[] retArray = {"10,9,9.0,10.0,false,10s,"}; + + tableResultSetEqualTest( + "select " + + "first(s_int, time), " + + "first(s_long, time), " + + "first(s_float, time), " + + "first(s_double, time), " + + "first(s_bool, time), " + + "first(s_string, time) " + + "from " + // subQuery: project all the column needed and rename the time_type to the time + + "(select " + + " time_type as time, " + + " s_int, s_long, s_float, s_double, s_bool, s_string " + + "from table_a " + + "left join table_b on table_a.time=table_b.time " + + "where table_a.device='d2') ", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testAggregationWithAllNull() { + + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + String[] retArray = {"null,null,null,null,null,null,"}; + + tableResultSetEqualTest( + "select " + + "first(s_int, time), " + + "first(s_long, time), " + + "first(s_float, time), " + + "first(s_double, time), " + + "first(s_bool, time), " + + "first(s_string, time) " + + "from " + // subQuery: project all the column needed and rename the time_type to the time + + "(select " + + " time_type as time, " + + " s_int, s_long, s_float, s_double, s_bool, s_string " + + "from table_a " + + "left join table_b on table_a.time=table_b.time " + + "where table_a.device='d3') ", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstInGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstInGroupIT.java new file mode 100644 index 0000000000000..48decb65b8253 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationFirstInGroupIT.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAggregationFirstInGroupIT { + + private static final String DATABASE_NAME = "test_grouped_first_agg"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table_a(" + + "s_int INT32 FIELD, " + + "s_long INT64 FIELD, " + + "s_float FLOAT FIELD, " + + "s_double DOUBLE FIELD, " + + "s_bool BOOLEAN FIELD, " + + "s_string STRING FIELD, " + + "time_type TIMESTAMP FIELD, " + + "partition STRING FIELD)", + "CREATE TABLE table_b(" + "time TIMESTAMP TIME, " + "s_back INT32 FIELD)", + + // Batch Insert: Split into 3 partitions (using positive numbers) + // Physical 'time' column is kept distinct and out-of-order as requested. + "INSERT INTO table_a(time, s_int, s_long, s_float, s_double, s_bool, s_string, time_type, partition) VALUES " + + // Partition 'p1': Standard Mixed Scenario + // Valid time_types: 40, 5, 80. + // Logic: Min Valid Time is 5. + + "(1, 40, 40, 40.0, 40.0, false, '40s', 40, 'p1')," // Valid + + "(2, 20, 20, 20.0, 20.0, true, '20s', NULL, 'p1')," // Null Time + + "(-1, 5, 5, 5.0, 5.0, false, '5s', 5, 'p1')," // Min Valid Time + // (5) + + "(-2, 4, 4, 4.0, 4.0, true, '4s', NULL, 'p1')," // Null Time + + "(-100, 80, 80, 80.0, 80.0, true, '80s', 80, 'p1')," // Valid + + // Partition 'p2': Mixed Null Values Scenario + + "(11, 80, 80, 80.0, 80.0, true, '80s', NULL, 'p2')," // Null Time + + "(-21, 40, 40, 40.0, 40.0, false, '40s', 40, 'p2')," // Previous Valid + + "(-102, 20, 20, 20.0, 20.0, true, '20s', NULL, 'p2')," // Null Time + + "(33, NULL, 5, 5.0, NULL, false, NULL, 5, 'p2')," + + "(100, 100, 100, 100.0, 100, 100, 100, 100, 'p2')," + + // Partition 'p3': Only Null Time Scenario + // Logic: No Valid Time. "First Null Wins" strategy applies. + + "(68, 80, 80, 80.0, 80.0, true, '80s', NULL, 'p3')," + + "(288, 40, 40, 40.0, 40.0, false, '40s', NULL, 'p3')" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testGroupedFirstAggregation() { + + // Expected Header: partition column + 6 aggregation results + String[] expectedHeader = {"partition", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6"}; + + // Expected Results: + String[] retArray = { + // p1 + "p1,5,5,5.0,5.0,false,5s,", + // p2 + "p2,40,5,5.0,40.0,false,40s,", + // p3 + "p3,80,80,80.0,80.0,true,80s," + }; + + tableResultSetEqualTest( + "select " + + "partition, " + + "first(s_int, time), " + + "first(s_long, time), " + + "first(s_float, time), " + + "first(s_double, time), " + + "first(s_bool, time), " + + "first(s_string, time) " + + "from " + // SubQuery: Rename time_type to 'ts' to avoid ambiguity with physical 'time' + + "(select " + + " time_type as time, " + + " partition, " + + " s_int, s_long, s_float, s_double, s_bool, s_string " + + "from table_a " + + "left join table_b on table_a.time=table_b.time) " + + "group by partition " + + "order by partition", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastByIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastByIT.java new file mode 100644 index 0000000000000..42bef69eb44f4 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastByIT.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAggregationLastByIT { + + private static final String DATABASE_NAME = "test_last_by_agg"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table_a(" + + "device STRING TAG, " + + "s_int INT32 FIELD, " + + "s_long INT64 FIELD, " + + "s_float FLOAT FIELD, " + + "s_double DOUBLE FIELD, " + + "s_bool BOOLEAN FIELD, " + + "s_string STRING FIELD, " + + "time_type TIMESTAMP FIELD, " + + "y_criteria INT32 FIELD)", + "CREATE TABLE table_b(" + "time TIMESTAMP TIME, " + "s_back INT32 FIELD)", + + // Batch Insert + "INSERT INTO table_a(time, device, s_int, s_long, s_float, s_double, s_bool, s_string, time_type, y_criteria) VALUES " + + // Case 1: s2 (y_criteria) has NO NULLs in Valid Times. + // Device: d1 + + "(99, 'd1', 99, 99, 99.0, 99.0, true, '99s', NULL, 99)," + + "(-200, 'd1', -200, -200, -200.0, -200.0, true, '-200s', -200, -200)," + + "(-100, 'd1', -100, -100, -100.0, -100.0, true, '-100s', -100, -100)," + + "(-10, 'd1', -10, -10, -10.0, -10.0, true, '-10s', -10, -10)," + + "(-5, 'd1', -5, -5, -5.0, -5.0, false, '-5s', -5, -5)," // Target + + // Case 2: s2 (y_criteria) has NULLs (Backtracking). + // Device: d2 + + "(88, 'd2', 88, 88, 88.0, 88.0, true, '2s', NULL, 88)," + + "(-5, 'd2', -5, -5, -5.0, -5.0, false, '-5s', -5, NULL)," + + "(-8, 'd2', -8, -8, -8.0, -8.0, false, '-8s', -8, NULL)," + + "(-10, 'd2', -10, -10, -10.0, -10.0, true, '-10s', -10, -10)," // Target + + "(-20, 'd2', -20, -20, -20.0, -20.0, true, '-20s', -20, -20)," + + // Case 3: s1 (value) has NULLs. + // Device: d3 + + "(3, 'd3', 77, 77, 77.0, 77.0, true, '77s', NULL, 77)," + + "(-5, 'd3', -5, -5, NULL, NULL, NULL, NULL, -5, -5)," // Target + + "(-10, 'd3', -10, -10, -10.0, -10.0, true, '-10s', -10, NULL)," + + "(-20, 'd3', -20, -20, -20.0, -20.0, true, '-20s', -20, -20)," + + // Case 4: s2 (y_criteria) is ALL NULLs. + // Device: d4 + + "(4, 'd4', 66, 66, 66.0, 66.0, true, '66s', NULL, NULL)," + + "(-5, 'd4', -5, -5, -5.0, -5.0, false, '-5s', -5, NULL)," + + "(-10, 'd4', -10, -10, -10.0, -10.0, true, '-10s', -10, NULL)," + + "(-20, 'd4', -20, -20, -20.0, -20.0, true, '-20s', -20, NULL)," + + // Case 5: All time_type are NULL. + // Device: d5 + + "(1, 'd5', 10, 10, 10.0, 10.0, true, '10s', NULL, NULL)," + + "(2, 'd5', 50, 50, 50.0, 50.0, false, '50s', NULL, 50)" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testLastBy_d1_NoNulls() { + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + String[] retArray = {"-5,-5,-5.0,-5.0,false,-5s,"}; + runTest("d1", expectedHeader, retArray); + } + + @Test + public void testLastBy_d2_Backtracking() { + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + String[] retArray = {"-10,-10,-10.0,-10.0,true,-10s,"}; + runTest("d2", expectedHeader, retArray); + } + + @Test + public void testLastBy_d3_TargetNull() { + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + String[] retArray = {"-5,-5,null,null,null,null,"}; + runTest("d3", expectedHeader, retArray); + } + + @Test + public void testLastBy_d4_AllNullCriteria() { + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + // Expected: No valid s2 found. + String[] retArray = {"null,null,null,null,null,null,"}; + runTest("d4", expectedHeader, retArray); + } + + @Test + public void testLastBy_d5_AllTimeNull() { + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + // Expected: The row with y_criteria=NULL is skipped. The row with y_criteria=50 is picked. + String[] retArray = {"50,50,50.0,50.0,false,50s,"}; + runTest("d5", expectedHeader, retArray); + } + + private void runTest(String deviceId, String[] expectedHeader, String[] retArray) { + tableResultSetEqualTest( + "select " + + "last_by(s_int, y_criteria, time), " + + "last_by(s_long, y_criteria, time), " + + "last_by(s_float, y_criteria, time), " + + "last_by(s_double, y_criteria, time), " + + "last_by(s_bool, y_criteria, time), " + + "last_by(s_string, y_criteria, time) " + + "from " + + "(select time_type as time, s_int, s_long, s_float, s_double, s_bool, s_string, y_criteria " + + "from table_a left join table_b on table_a.time=table_b.time " + + "where table_a.device='" + + deviceId + + "') ", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastByInGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastByInGroupIT.java new file mode 100644 index 0000000000000..5b59e0875ed62 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastByInGroupIT.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IoTDBAggregationLastByInGroupIT { + private static final String DATABASE_NAME = "test_last_by_in_group_agg"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table_a(" + + "s_int INT32 FIELD, " + + "s_long INT64 FIELD, " + + "s_float FLOAT FIELD, " + + "s_double DOUBLE FIELD, " + + "s_bool BOOLEAN FIELD, " + + "s_string STRING FIELD, " + + "time_type TIMESTAMP FIELD, " + + "y_criteria INT32 FIELD, " // Acts as s2 + + "partition STRING FIELD)", + "CREATE TABLE table_b(" + "time TIMESTAMP TIME, " + "s_back INT32 FIELD)", + + // Batch Insert: 5 Partitions (p1-p5) using negative values + "INSERT INTO table_a(time, s_int, s_long, s_float, s_double, s_bool, s_string, time_type, y_criteria, partition) VALUES " + + // Partition p1: s2 has NO NULLs in Valid Times. + + "(1, -99, -99, -99.0, -99.0, true, '-99s', NULL, -99, 'p1')," + + "(2, -200, -200, -200.0, -200.0, true, '-200s', -200, -200, 'p1')," + + "(3, -100, -100, -100.0, -100.0, true, '-100s', -100, -100, 'p1')," + + "(4, -10, -10, -10.0, -10.0, true, '-10s', -10, -10, 'p1')," + + "(5, -5, -5, -5.0, -5.0, false, '-5s', -5, -5, 'p1')," // Target + + // Partition p2: s2 has NULLs (Backtracking). + + "(6, -88, -88, -88.0, -88.0, true, '-88s', NULL, -88, 'p2')," + + "(7, -5, -5, -5.0, -5.0, false, '-5s', -5, NULL, 'p2')," + + "(8, -8, -8, -8.0, -8.0, false, '-8s', -8, NULL, 'p2')," + + "(9, -10, -10, -10.0, -10.0, true, '-10s', -10, -10, 'p2')," // Target + + "(10, -20, -20, -20.0, -20.0, true, '-20s', -20, -20, 'p2')," + + // Partition p3: s1 (value) has NULLs. + + "(11, -77, -77, -77.0, -77.0, true, '-77s', NULL, -77, 'p3')," + + "(12, NULL, NULL, -5.0, -5.0, NULL, NULL, -5, -5, 'p3')," // Target + + "(13, -10, -10, -10.0, -10.0, true, '-10s', -10, NULL, 'p3')," + + "(14, -20, -20, -20.0, -20.0, true, '-20s', -20, -20, 'p3')," + + // Partition p4: s2 is ALL NULLs. + + "(15, -66, -66, -66.0, -66.0, true, '-66s', NULL, NULL, 'p4')," + + "(16, -5, -5, -5.0, -5.0, false, '-5s', -5, NULL, 'p4')," + + "(17, -10, -10, -10.0, -10.0, true, '-10s', -10, NULL, 'p4')," + + "(18, -20, -20, -20.0, -20.0, true, '-20s', -20, NULL, 'p4')," + + // Partition p5: All time_type are NULL. + + "(19, -10, -10, -10.0, -10.0, true, '-10s', NULL, NULL, 'p5')," + + "(20, -50, -50, -50.0, -50.0, false, '-50s', NULL, -50, 'p5')" // Target + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testGroupedLastByAggregation() { + + // Expected Header: partition column + 6 aggregation results + String[] expectedHeader = {"partition", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6"}; + + // Expected Results: + String[] retArray = { + "p1,-5,-5,-5.0,-5.0,false,-5s,", + "p2,-10,-10,-10.0,-10.0,true,-10s,", + "p3,null,null,-5.0,-5.0,null,null,", + "p4,null,null,null,null,null,null,", + "p5,-50,-50,-50.0,-50.0,false,-50s," + }; + + tableResultSetEqualTest( + "select " + + "partition, " + + "last_by(s_int, y_criteria, time), " + + "last_by(s_long, y_criteria, time), " + + "last_by(s_float, y_criteria, time), " + + "last_by(s_double, y_criteria, time), " + + "last_by(s_bool, y_criteria, time), " + + "last_by(s_string, y_criteria, time) " + + "from " + // SubQuery: Rename time_type to 'ts' to avoid ambiguity + + "(select time_type as time, s_int, s_long, s_float, s_double, s_bool, s_string, y_criteria, partition " + + "from table_a left join table_b on table_a.time=table_b.time) " + + "group by partition " + + "order by partition", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastIT.java new file mode 100644 index 0000000000000..e9e860010f51a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastIT.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAggregationLastIT { + + private static final String DATABASE_NAME = "test_null_time_aggs_all_types_last"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table_a(" + + "device STRING TAG, " + + "s_int INT32 FIELD, " + + "s_long INT64 FIELD, " + + "s_float FLOAT FIELD, " + + "s_double DOUBLE FIELD, " + + "s_bool BOOLEAN FIELD, " + + "s_string STRING FIELD, " + + "time_type TIMESTAMP FIELD)", + "CREATE TABLE table_b(" + "time TIMESTAMP TIME, " + "s_back INT32 FIELD)", + + // batch insert：d1(5 rows) + d2(3 rows) + "INSERT INTO table_a(time, device, s_int, s_long, s_float, s_double, s_bool, s_string, time_type) VALUES " + // --- Device d1 --- + + "(-50, 'd1', -50, -50, -50.0, -50.0, false, '-50s', NULL)," + + "(-10, 'd1', -10, -10, -10.0, -10.0, true, '-10s', -10)," + + "(-100, 'd1', -100, -100, -100.0, -100.0, true, '-100s', -100)," + + "(50, 'd1', 50, 50, 50.0, 50.0, false, '50s', NULL)," + + "(-5, 'd1', -5, -5, -5.0, -5.0, false, '-5s', -5)," + + // --- Device d2 --- + + "(-80, 'd2', -80, -80, -80.0, -80.0, true, '-80s', NULL)," + + "(-5, 'd2', NULL, -5, -5.0, NULL, false, NULL, -5)," + + "(-40, 'd2', -40, -40, -40.0, -40.0, false, '-40s', -40)," + + "(20, 'd2', 20, 20, 20.0, 20.0, true, '20s', NULL)," + + "(-4, 'd2', -4, NULL, NULL, -4.0, NULL, '-4s', -4)," + + // --- Device d3 (Pure NULL test) --- + // d3: all time_type are NULL + + "(-200, 'd3', null, null, null, null, null, null, -200)," + + "(-150, 'd3', null, null, null, null, null, null, -150)" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testAggregation() { + + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + + // Expected Result: + String[] retArray = {"-5,-5,-5.0,-5.0,false,-5s,"}; + + tableResultSetEqualTest( + "select " + + "last(s_int, time), " + + "last(s_long, time), " + + "last(s_float, time), " + + "last(s_double, time), " + + "last(s_bool, time), " + + "last(s_string, time) " + + "from " + // subQuery: project all the column needed and rename the time_type to the time + + "(select " + + " time_type as time, " + + " s_int, s_long, s_float, s_double, s_bool, s_string " + + "from table_a " + + "left join table_b on table_a.time=table_b.time " + + "where table_a.device='d1') ", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testAggregationWithNullValue() { + + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + + // Expected Result: + String[] retArray = {"-4,-5,-5.0,-4.0,false,-4s,"}; + + tableResultSetEqualTest( + "select " + + "last(s_int, time), " + + "last(s_long, time), " + + "last(s_float, time), " + + "last(s_double, time), " + + "last(s_bool, time), " + + "last(s_string, time) " + + "from " + // subQuery: project all the column needed and rename the time_type to the time + + "(select " + + " time_type as time, " + + " s_int, s_long, s_float, s_double, s_bool, s_string " + + "from table_a " + + "left join table_b on table_a.time=table_b.time " + + "where table_a.device='d2') ", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testAggregationWithAllNull() { + + String[] expectedHeader = {"_col0", "_col1", "_col2", "_col3", "_col4", "_col5"}; + + // Expected Result: + String[] retArray = {"null,null,null,null,null,null,"}; + + tableResultSetEqualTest( + "select " + + "last(s_int, time), " + + "last(s_long, time), " + + "last(s_float, time), " + + "last(s_double, time), " + + "last(s_bool, time), " + + "last(s_string, time) " + + "from " + // subQuery: project all the column needed and rename the time_type to the time + + "(select " + + " time_type as time, " + + " s_int, s_long, s_float, s_double, s_bool, s_string " + + "from table_a " + + "left join table_b on table_a.time=table_b.time " + + "where table_a.device='d3') ", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastInGroupIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastInGroupIT.java new file mode 100644 index 0000000000000..1f6b2c680ff13 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBAggregationLastInGroupIT.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAggregationLastInGroupIT { + + private static final String DATABASE_NAME = "test_grouped_last_agg"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table_a(" + + "s_int INT32 FIELD, " + + "s_long INT64 FIELD, " + + "s_float FLOAT FIELD, " + + "s_double DOUBLE FIELD, " + + "s_bool BOOLEAN FIELD, " + + "s_string STRING FIELD, " + + "time_type TIMESTAMP FIELD, " + + "partition STRING FIELD)", + "CREATE TABLE table_b(" + "time TIMESTAMP TIME, " + "s_back INT32 FIELD)", + + // Batch Insert: Split into 3 partitions (the time column is out of order) + "INSERT INTO table_a(time, s_int, s_long, s_float, s_double, s_bool, s_string, time_type, partition) VALUES " + + // Partition 'p1': Standard Mixed Scenario + // Logic: Max Valid Time is -5. Values are all valid. + + "(1, -40, -40, -40.0, -40.0, false, '-40s', -40, 'p1')," // Valid + + "(2, 20, 20, 20.0, 20.0, true, '20s', NULL, 'p1')," // Null Time + + "(-1, -5, -5, -5.0, -5.0, false, '-5s', -5, 'p1')," // Max Valid Time + + "(-2, -4, -4, -4.0, -4.0, true, '-4s', NULL, 'p1')," // Null Time + + "(-100, -80, -80, -80.0, -80.0, true, '-80s', -80, 'p1')," // Null Time + + // Partition 'p2': Mixed Null Values Scenario + // Logic: Max Valid Time is -5. Values contain mixed NULLs. + + "(11, -80, -80, -80.0, -80.0, true, '-80s', NULL, 'p2')," + + "(-21, -40, -40, -40.0, -40.0, false, '-40s', -40, 'p2')," // Previous Valid + + "(-102, 20, 20, 20.0, 20.0, true, '20s', NULL, 'p2')," + // Max Valid Time (-5) has partial NULLs: + + "(33, NULL, -5, -5.0, NULL, false, NULL, -5, 'p2')," + + // Partition 'p3': Only Null Time Scenario + + "(68, -80, -80, -80.0, -80.0, true, '-80s', NULL, 'p3')," + + "(288, -40, -40, -40.0, -40.0, false, '-40s', NULL, 'p3')" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testGroupedLastAggregation() { + + // Expected Header: partition column + 6 aggregation results + String[] expectedHeader = {"partition", "_col1", "_col2", "_col3", "_col4", "_col5", "_col6"}; + + // Expected Results: + String[] retArray = { + "p1,-5,-5,-5.0,-5.0,false,-5s,", + "p2,-40,-5,-5.0,-40.0,false,-40s,", + "p3,-80,-80,-80.0,-80.0,true,-80s," + }; + + tableResultSetEqualTest( + "select " + + "partition, " + + "last(s_int, time), " + + "last(s_long, time), " + + "last(s_float, time), " + + "last(s_double, time), " + + "last(s_bool, time), " + + "last(s_string, time) " + + "from " + // SubQuery: Rename time_type to time, include partition + + "(select " + + " time_type as time, " + + " partition, " + + " s_int, s_long, s_float, s_double, s_bool, s_string " + + "from table_a " + + "left join table_b on table_a.time=table_b.time) " + + "group by partition " + + "order by partition", + expectedHeader, + retArray, + DATABASE_NAME); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java index 7436bbf035b57..d1f04cfdc920f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java @@ -36,7 +36,7 @@ import java.nio.charset.StandardCharsets; import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Utils.serializeTimeValue; +import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Utils.serializeTimeValueWithNull; public class FirstAccumulator implements TableAccumulator { @@ -47,10 +47,11 @@ public class FirstAccumulator implements TableAccumulator { protected long minTime = Long.MAX_VALUE; protected boolean initResult = false; private final boolean canFinishAfterInit; + protected boolean initNullTimeValue = false; public FirstAccumulator(TSDataType seriesDataType, boolean canFinishAfterInit) { this.seriesDataType = seriesDataType; - firstValue = TsPrimitiveType.getByType(seriesDataType); + this.firstValue = TsPrimitiveType.getByType(seriesDataType); this.canFinishAfterInit = canFinishAfterInit; } @@ -112,26 +113,43 @@ public void addIntermediate(Column argument) { byte[] bytes = argument.getBinary(i).getValues(); long time = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 0); - int offset = 8; + boolean isTimeNull = BytesUtils.bytesToBool(bytes, 8); + int offset = 9; switch (seriesDataType) { case INT32: case DATE: int intVal = BytesUtils.bytesToInt(bytes, offset); - updateIntFirstValue(intVal, time); + if (!isTimeNull) { + updateIntFirstValue(intVal, time); + } else { + updateIntNullTimeValue(intVal); + } break; case INT64: case TIMESTAMP: long longVal = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - updateLongFirstValue(longVal, time); + if (!isTimeNull) { + updateLongFirstValue(longVal, time); + } else { + updateLongNullTimeValue(longVal); + } break; case FLOAT: float floatVal = BytesUtils.bytesToFloat(bytes, offset); - updateFloatFirstValue(floatVal, time); + if (!isTimeNull) { + updateFloatFirstValue(floatVal, time); + } else { + updateFloatNullTimeValue(floatVal); + } break; case DOUBLE: double doubleVal = BytesUtils.bytesToDouble(bytes, offset); - updateDoubleFirstValue(doubleVal, time); + if (!isTimeNull) { + updateDoubleFirstValue(doubleVal, time); + } else { + updateDoubleNullTimeValue(doubleVal); + } break; case TEXT: case BLOB: @@ -140,11 +158,19 @@ public void addIntermediate(Column argument) { int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - updateBinaryFirstValue(binaryVal, time); + if (!isTimeNull) { + updateBinaryFirstValue(binaryVal, time); + } else { + updateBinaryNullTimeValue(binaryVal); + } break; case BOOLEAN: boolean boolVal = BytesUtils.bytesToBool(bytes, offset); - updateBooleanFirstValue(boolVal, time); + if (!isTimeNull) { + updateBooleanFirstValue(boolVal, time); + } else { + updateBooleanNullTimeValue(boolVal); + } break; default: throw new UnSupportedDataTypeException( @@ -158,47 +184,56 @@ public void evaluateIntermediate(ColumnBuilder columnBuilder) { checkArgument( columnBuilder instanceof BinaryColumnBuilder, "intermediate input and output of FIRST should be BinaryColumn"); - if (!initResult) { - columnBuilder.appendNull(); - } else { + + // Case 1: Found a valid result with a valid timestamp (Highest Priority) + if (initResult || initNullTimeValue) { + // if the initResult is activated, the result must carry a not null time + boolean isOrderTimeNull = !initResult; columnBuilder.writeBinary( - new Binary(serializeTimeValue(seriesDataType, minTime, firstValue))); + new Binary( + serializeTimeValueWithNull(seriesDataType, minTime, isOrderTimeNull, firstValue))); + return; } + + columnBuilder.appendNull(); } @Override public void evaluateFinal(ColumnBuilder columnBuilder) { - if (!initResult) { + + // all the values are the null + if (!initResult && !initNullTimeValue) { columnBuilder.appendNull(); - } else { - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(firstValue.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(firstValue.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(firstValue.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(firstValue.getDouble()); - break; - case TEXT: - case BLOB: - case STRING: - case OBJECT: - columnBuilder.writeBinary(firstValue.getBinary()); - break; - case BOOLEAN: - columnBuilder.writeBoolean(firstValue.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException( - String.format("Unsupported data type in FIRST aggregation: %s", seriesDataType)); - } + return; + } + + switch (seriesDataType) { + case INT32: + case DATE: + columnBuilder.writeInt(firstValue.getInt()); + break; + case INT64: + case TIMESTAMP: + columnBuilder.writeLong(firstValue.getLong()); + break; + case FLOAT: + columnBuilder.writeFloat(firstValue.getFloat()); + break; + case DOUBLE: + columnBuilder.writeDouble(firstValue.getDouble()); + break; + case TEXT: + case BLOB: + case STRING: + case OBJECT: + columnBuilder.writeBinary(firstValue.getBinary()); + break; + case BOOLEAN: + columnBuilder.writeBoolean(firstValue.getBoolean()); + break; + default: + throw new UnSupportedDataTypeException( + String.format("Unsupported data type in FIRST aggregation: %s", seriesDataType)); } } @@ -265,216 +300,264 @@ public void addStatistics(Statistics[] statistics) { @Override public void reset() { initResult = false; + this.initNullTimeValue = false; this.minTime = Long.MAX_VALUE; this.firstValue.reset(); } + private boolean checkAndUpdateFirstTime(long curTime) { + if (!initResult || curTime < minTime) { + initResult = true; + minTime = curTime; + return true; + } + return false; + } + + private boolean checkAndUpdateNullTime() { + if (!initResult && !initNullTimeValue) { + initNullTimeValue = true; + return true; + } + return false; + } + + /** + * Updates the accumulator, prioritizing values with valid timestamps over those with null + * timestamps. + */ protected void addIntInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateIntFirstValue(valueColumn.getInt(i), timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateIntFirstValue(valueColumn.getInt(position), timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: Time is not null. Attempt to update the value with the minimum time. + updateIntFirstValue(valueColumn.getInt(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + // Case B: Time is NULL, the nullTimeValue should only be assigned once + updateIntNullTimeValue(valueColumn.getInt(position)); } } } protected void updateIntFirstValue(int value, long curTime) { - initResult = true; - if (curTime < minTime) { - minTime = curTime; + if (checkAndUpdateFirstTime(curTime)) { + firstValue.setInt(value); + } + } + + protected void updateIntNullTimeValue(int value) { + if (checkAndUpdateNullTime()) { firstValue.setInt(value); } } + /** + * Updates the accumulator, prioritizing values with valid timestamps over those with null + * timestamps. + */ protected void addLongInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateLongFirstValue(valueColumn.getLong(i), timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateLongFirstValue(valueColumn.getLong(position), timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: Time is not null. Attempt to update the value with the minimum time. + updateLongFirstValue(valueColumn.getLong(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + // Case B: Time is NULL, the nullTimeValue should only be assigned once + updateLongNullTimeValue(valueColumn.getLong(position)); } } } protected void updateLongFirstValue(long value, long curTime) { - initResult = true; - if (curTime < minTime) { - minTime = curTime; + if (checkAndUpdateFirstTime(curTime)) { firstValue.setLong(value); } } + protected void updateLongNullTimeValue(long value) { + if (checkAndUpdateNullTime()) { + firstValue.setLong(value); + } + } + + /** + * Updates the accumulator, prioritizing values with valid timestamps over those with null + * timestamps. + */ protected void addFloatInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateFloatFirstValue(valueColumn.getFloat(i), timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateFloatFirstValue(valueColumn.getFloat(position), timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + if (!timeColumn.isNull(position)) { + updateFloatFirstValue(valueColumn.getFloat(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + updateFloatNullTimeValue(valueColumn.getFloat(position)); } } } protected void updateFloatFirstValue(float value, long curTime) { - initResult = true; - if (curTime < minTime) { - minTime = curTime; + if (checkAndUpdateFirstTime(curTime)) { + firstValue.setFloat(value); + } + } + + protected void updateFloatNullTimeValue(float value) { + if (checkAndUpdateNullTime()) { firstValue.setFloat(value); } } + /** + * Updates the accumulator, prioritizing values with valid timestamps over those with null + * timestamps. + */ protected void addDoubleInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateDoubleFirstValue(valueColumn.getDouble(i), timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateDoubleFirstValue(valueColumn.getDouble(position), timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + if (!timeColumn.isNull(position)) { + updateDoubleFirstValue(valueColumn.getDouble(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + updateDoubleNullTimeValue(valueColumn.getDouble(position)); } } } protected void updateDoubleFirstValue(double value, long curTime) { - initResult = true; - if (curTime < minTime) { - minTime = curTime; + if (checkAndUpdateFirstTime(curTime)) { + firstValue.setDouble(value); + } + } + + protected void updateDoubleNullTimeValue(double value) { + if (checkAndUpdateNullTime()) { firstValue.setDouble(value); } } + /** + * Updates the accumulator, prioritizing values with valid timestamps over those with null + * timestamps. + */ protected void addBinaryInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateBinaryFirstValue(valueColumn.getBinary(i), timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateBinaryFirstValue(valueColumn.getBinary(position), timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + if (!timeColumn.isNull(position)) { + updateBinaryFirstValue(valueColumn.getBinary(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + updateBinaryNullTimeValue(valueColumn.getBinary(position)); } } } protected void updateBinaryFirstValue(Binary value, long curTime) { - initResult = true; - if (curTime < minTime) { - minTime = curTime; + if (checkAndUpdateFirstTime(curTime)) { + firstValue.setBinary(value); + } + } + + protected void updateBinaryNullTimeValue(Binary value) { + if (checkAndUpdateNullTime()) { firstValue.setBinary(value); } } + /** + * Updates the accumulator, prioritizing values with valid timestamps over those with null + * timestamps. + */ protected void addBooleanInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateBooleanFirstValue(valueColumn.getBoolean(i), timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateBooleanFirstValue(valueColumn.getBoolean(position), timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + if (!timeColumn.isNull(position)) { + updateBooleanFirstValue(valueColumn.getBoolean(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + updateBooleanNullTimeValue(valueColumn.getBoolean(position)); } } } protected void updateBooleanFirstValue(boolean value, long curTime) { - initResult = true; - if (curTime < minTime) { - minTime = curTime; + if (checkAndUpdateFirstTime(curTime)) { + firstValue.setBoolean(value); + } + } + + protected void updateBooleanNullTimeValue(boolean value) { + if (checkAndUpdateNullTime()) { firstValue.setBoolean(value); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java index 551b8f965ffbb..90057390a9b25 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java @@ -35,7 +35,7 @@ import java.nio.charset.StandardCharsets; import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Utils.serializeTimeValue; +import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Utils.serializeTimeValueWithNull; public class FirstByAccumulator implements TableAccumulator { @@ -54,6 +54,7 @@ public class FirstByAccumulator implements TableAccumulator { private boolean xIsNull = true; private boolean initResult = false; + protected boolean initNullTimeValue = false; private final boolean canFinishAfterInit; @@ -134,50 +135,80 @@ public void addIntermediate(Column argument) { byte[] bytes = argument.getBinary(i).getValues(); long curTime = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 0); int offset = Long.BYTES; - boolean isXNull = BytesUtils.bytesToBool(bytes, offset); + boolean isOrderTimeNull = BytesUtils.bytesToBool(bytes, offset); + offset += 1; + boolean isXValueNull = BytesUtils.bytesToBool(bytes, offset); offset += 1; - - if (isXNull) { - if (!initResult || curTime < yFirstTime) { - initResult = true; - yFirstTime = curTime; - xIsNull = true; - } - continue; - } switch (xDataType) { case INT32: case DATE: - int xIntVal = BytesUtils.bytesToInt(bytes, offset); - updateIntFirstValue(xIntVal, curTime); + int intVal = isXValueNull ? 0 : BytesUtils.bytesToInt(bytes, offset); + if (!isOrderTimeNull) { + updateIntFirstValue(isXValueNull, intVal, curTime); + } else { + updateIntNullTimeValue(isXValueNull, intVal); + } break; + case INT64: case TIMESTAMP: - long longVal = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - updateLongFirstValue(longVal, curTime); + long longVal = + isXValueNull ? 0 : BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); + if (!isOrderTimeNull) { + updateLongFirstValue(isXValueNull, longVal, curTime); + } else { + updateLongNullTimeValue(isXValueNull, longVal); + } break; + case FLOAT: - float floatVal = BytesUtils.bytesToFloat(bytes, offset); - updateFloatFirstValue(floatVal, curTime); + float floatVal = isXValueNull ? 0 : BytesUtils.bytesToFloat(bytes, offset); + if (!isOrderTimeNull) { + updateFloatFirstValue(isXValueNull, floatVal, curTime); + } else { + updateFloatNullTimeValue(isXValueNull, floatVal); + } break; + case DOUBLE: - double doubleVal = BytesUtils.bytesToDouble(bytes, offset); - updateDoubleFirstValue(doubleVal, curTime); + double doubleVal = isXValueNull ? 0 : BytesUtils.bytesToDouble(bytes, offset); + if (!isOrderTimeNull) { + updateDoubleFirstValue(isXValueNull, doubleVal, curTime); + } else { + updateDoubleNullTimeValue(isXValueNull, doubleVal); + } break; + case TEXT: case BLOB: case OBJECT: case STRING: - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - updateBinaryFirstValue(binaryVal, curTime); + Binary binaryVal = null; + if (!isXValueNull) { + int length = BytesUtils.bytesToInt(bytes, offset); + offset += Integer.BYTES; + binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); + } + if (!isOrderTimeNull) { + updateBinaryFirstValue(isXValueNull, binaryVal, curTime); + } else { + updateBinaryNullTimeValue(isXValueNull, binaryVal); + } break; + case BOOLEAN: - boolean boolVal = BytesUtils.bytesToBool(bytes, offset); - updateBooleanFirstValue(boolVal, curTime); + boolean boolVal = false; + if (!isXValueNull) { + boolVal = BytesUtils.bytesToBool(bytes, offset); + } + if (!isOrderTimeNull) { + updateBooleanFirstValue(isXValueNull, boolVal, curTime); + } else { + updateBooleanNullTimeValue(isXValueNull, boolVal); + } break; + default: throw new UnSupportedDataTypeException( String.format("Unsupported data type in FIRST_BY Aggregation: %s", yDataType)); @@ -191,17 +222,22 @@ public void evaluateIntermediate(ColumnBuilder columnBuilder) { columnBuilder instanceof BinaryColumnBuilder, "intermediate input and output of FIRST_BY should be BinaryColumn"); - if (!initResult) { - columnBuilder.appendNull(); - } else { + if (initResult || initNullTimeValue) { + // if the initResult is activated, the result must carry a not null time + boolean isOrderTimeNull = !initResult; columnBuilder.writeBinary( - new Binary(serializeTimeValue(xDataType, yFirstTime, xIsNull, xResult))); + new Binary( + serializeTimeValueWithNull( + xDataType, yFirstTime, xIsNull, isOrderTimeNull, xResult))); + return; } + + columnBuilder.appendNull(); } @Override public void evaluateFinal(ColumnBuilder columnBuilder) { - if (!initResult || xIsNull) { + if (xIsNull) { columnBuilder.appendNull(); return; } @@ -316,308 +352,274 @@ public void addStatistics(Statistics[] statistics) { @Override public void reset() { initResult = false; + initNullTimeValue = false; xIsNull = true; this.yFirstTime = Long.MAX_VALUE; this.xResult.reset(); } - protected void addIntInput( - Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateIntFirstValue(xColumn, i, timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateIntFirstValue(xColumn, position, timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } - } - } - } - } - - protected void updateIntFirstValue(Column xColumn, int xIdx, long curTime) { + private boolean checkAndUpdateFirstTime(boolean isXValueNull, long curTime) { if (!initResult || curTime < yFirstTime) { initResult = true; yFirstTime = curTime; - if (xColumn.isNull(xIdx)) { + + if (isXValueNull) { xIsNull = true; + return false; } else { xIsNull = false; - xResult.setInt(xColumn.getInt(xIdx)); + return true; } } + return false; } - protected void updateIntFirstValue(int val, long curTime) { - if (!initResult || curTime < yFirstTime) { - initResult = true; - yFirstTime = curTime; - xIsNull = false; - xResult.setInt(val); + private boolean checkAndUpdateNullTime(boolean isXValueNull) { + if (!initResult && !initNullTimeValue) { + initNullTimeValue = true; + + if (isXValueNull) { + xIsNull = true; + return false; + } else { + xIsNull = false; + return true; + } } + return false; } - protected void addLongInput( + protected void addIntInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateLongFirstValue(xColumn, i, timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateLongFirstValue(xColumn, position, timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: The order time is not null. Attempt to update the xResult. + updateIntFirstValue( + xColumn.isNull(position), xColumn.getInt(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + // Case B: The order time is null. Attempt to update the xNullTimeValue. + updateIntNullTimeValue(xColumn.isNull(position), xColumn.getInt(position)); } } } - protected void updateLongFirstValue(Column xColumn, int xIdx, long curTime) { - if (!initResult || curTime < yFirstTime) { - initResult = true; - yFirstTime = curTime; - if (xColumn.isNull(xIdx)) { - xIsNull = true; + protected void updateIntFirstValue(boolean isXValueNull, int xValue, long curTime) { + if (checkAndUpdateFirstTime(isXValueNull, curTime)) { + xResult.setInt(xValue); + } + } + + protected void updateIntNullTimeValue(boolean isXValueNull, int xValue) { + if (checkAndUpdateNullTime(isXValueNull)) { + xResult.setInt(xValue); + } + } + + protected void addLongInput( + Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; + } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: Valid Time + updateLongFirstValue( + xColumn.isNull(position), xColumn.getLong(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; + } } else { - xIsNull = false; - xResult.setLong(xColumn.getLong(xIdx)); + // Case B: Null Time + updateLongNullTimeValue(xColumn.isNull(position), xColumn.getLong(position)); } } } - protected void updateLongFirstValue(long value, long curTime) { - if (!initResult || curTime < yFirstTime) { - initResult = true; - yFirstTime = curTime; - xIsNull = false; - xResult.setLong(value); + protected void updateLongFirstValue(boolean isXValueNull, long xValue, long curTime) { + if (checkAndUpdateFirstTime(isXValueNull, curTime)) { + xResult.setLong(xValue); + } + } + + protected void updateLongNullTimeValue(boolean isXValueNull, long xValue) { + if (checkAndUpdateNullTime(isXValueNull)) { + xResult.setLong(xValue); } } protected void addFloatInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateFloatFirstValue(xColumn, i, timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateFloatFirstValue(xColumn, position, timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: Valid Time + updateFloatFirstValue( + xColumn.isNull(position), xColumn.getFloat(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + // Case B: Null Time + updateFloatNullTimeValue(xColumn.isNull(position), xColumn.getFloat(position)); } } } - protected void updateFloatFirstValue(Column xColumn, int xIdx, long curTime) { - if (!initResult || curTime < yFirstTime) { - initResult = true; - yFirstTime = curTime; - if (xColumn.isNull(xIdx)) { - xIsNull = true; - } else { - xIsNull = false; - xResult.setFloat(xColumn.getFloat(xIdx)); - } + protected void updateFloatFirstValue(boolean isXValueNull, float xValue, long curTime) { + if (checkAndUpdateFirstTime(isXValueNull, curTime)) { + xResult.setFloat(xValue); } } - protected void updateFloatFirstValue(float value, long curTime) { - if (!initResult || curTime < yFirstTime) { - initResult = true; - yFirstTime = curTime; - xIsNull = false; - xResult.setFloat(value); + protected void updateFloatNullTimeValue(boolean isXValueNull, float xValue) { + if (checkAndUpdateNullTime(isXValueNull)) { + xResult.setFloat(xValue); } } protected void addDoubleInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateDoubleFirstValue(xColumn, i, timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateDoubleFirstValue(xColumn, position, timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + updateDoubleFirstValue( + xColumn.isNull(position), xColumn.getDouble(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + updateDoubleNullTimeValue(xColumn.isNull(position), xColumn.getDouble(position)); } } } - protected void updateDoubleFirstValue(Column xColumn, int xIdx, long curTime) { - if (!initResult || curTime < yFirstTime) { - initResult = true; - yFirstTime = curTime; - if (xColumn.isNull(xIdx)) { - xIsNull = true; - } else { - xIsNull = false; - xResult.setDouble(xColumn.getDouble(xIdx)); - } + protected void updateDoubleFirstValue(boolean isXValueNull, double xValue, long curTime) { + if (checkAndUpdateFirstTime(isXValueNull, curTime)) { + xResult.setDouble(xValue); } } - protected void updateDoubleFirstValue(double val, long curTime) { - if (!initResult || curTime < yFirstTime) { - initResult = true; - yFirstTime = curTime; - xIsNull = false; - xResult.setDouble(val); + protected void updateDoubleNullTimeValue(boolean isXValueNull, double xValue) { + if (checkAndUpdateNullTime(isXValueNull)) { + xResult.setDouble(xValue); } } protected void addBinaryInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateBinaryFirstValue(xColumn, i, timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateBinaryFirstValue(xColumn, position, timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + updateBinaryFirstValue( + xColumn.isNull(position), xColumn.getBinary(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + updateBinaryNullTimeValue(xColumn.isNull(position), xColumn.getBinary(position)); } } } - protected void updateBinaryFirstValue(Column xColumn, int xIdx, long curTime) { - if (!initResult || curTime < yFirstTime) { - initResult = true; - yFirstTime = curTime; - if (xColumn.isNull(xIdx)) { - xIsNull = true; - } else { - xIsNull = false; - xResult.setBinary(xColumn.getBinary(xIdx)); - } + protected void updateBinaryFirstValue(boolean isXValueNull, Binary xValue, long curTime) { + if (checkAndUpdateFirstTime(isXValueNull, curTime)) { + xResult.setBinary(xValue); } } - protected void updateBinaryFirstValue(Binary val, long curTime) { - if (!initResult || curTime < yFirstTime) { - initResult = true; - yFirstTime = curTime; - xIsNull = false; - xResult.setBinary(val); + protected void updateBinaryNullTimeValue(boolean isXValueNull, Binary xValue) { + if (checkAndUpdateNullTime(isXValueNull)) { + xResult.setBinary(xValue); } } protected void addBooleanInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateBooleanFirstValue(xColumn, i, timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateBooleanFirstValue(xColumn, position, timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + updateBooleanFirstValue( + xColumn.isNull(position), xColumn.getBoolean(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + updateBooleanNullTimeValue(xColumn.isNull(position), xColumn.getBoolean(position)); } } } - protected void updateBooleanFirstValue(Column xColumn, int xIdx, long curTime) { - if (!initResult || curTime < yFirstTime) { - initResult = true; - yFirstTime = curTime; - if (xColumn.isNull(xIdx)) { - xIsNull = true; - } else { - xIsNull = false; - xResult.setBoolean(xColumn.getBoolean(xIdx)); - } + protected void updateBooleanFirstValue(boolean isXValueNull, boolean xValue, long curTime) { + if (checkAndUpdateFirstTime(isXValueNull, curTime)) { + xResult.setBoolean(xValue); } } - protected void updateBooleanFirstValue(boolean val, long curTime) { - if (!initResult || curTime < yFirstTime) { - initResult = true; - yFirstTime = curTime; - xIsNull = false; - xResult.setBoolean(val); + protected void updateBooleanNullTimeValue(boolean isXValueNull, boolean xValue) { + if (checkAndUpdateNullTime(isXValueNull)) { + xResult.setBoolean(xValue); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByDescAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByDescAccumulator.java index c6f5fdb9a763c..c6f310048adb0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByDescAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByDescAccumulator.java @@ -42,22 +42,25 @@ public TableAccumulator copy() { @Override protected void addIntInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateIntFirstValue(xColumn, i, timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateIntFirstValue(xColumn, position, timeColumn.getLong(position)); - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: The order time is not null. Attempt to update the xResult. + updateIntFirstValue( + xColumn.isNull(position), xColumn.getInt(position), timeColumn.getLong(position)); + } else { + // Case B: The order time is null. Attempt to update the xNullTimeValue. + updateIntNullTimeValue(xColumn.isNull(position), xColumn.getInt(position)); } } } @@ -65,22 +68,25 @@ protected void addIntInput( @Override protected void addLongInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateLongFirstValue(xColumn, i, timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateLongFirstValue(xColumn, position, timeColumn.getLong(position)); - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: Valid Time + updateLongFirstValue( + xColumn.isNull(position), xColumn.getLong(position), timeColumn.getLong(position)); + } else { + // Case B: Null Time + updateLongNullTimeValue(xColumn.isNull(position), xColumn.getLong(position)); } } } @@ -88,22 +94,25 @@ protected void addLongInput( @Override protected void addFloatInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateFloatFirstValue(xColumn, i, timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateFloatFirstValue(xColumn, position, timeColumn.getLong(position)); - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: Valid Time + updateFloatFirstValue( + xColumn.isNull(position), xColumn.getFloat(position), timeColumn.getLong(position)); + } else { + // Case B: Null Time + updateFloatNullTimeValue(xColumn.isNull(position), xColumn.getFloat(position)); } } } @@ -111,22 +120,23 @@ protected void addFloatInput( @Override protected void addDoubleInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateDoubleFirstValue(xColumn, i, timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateDoubleFirstValue(xColumn, position, timeColumn.getLong(position)); - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + updateDoubleFirstValue( + xColumn.isNull(position), xColumn.getDouble(position), timeColumn.getLong(position)); + } else { + updateDoubleNullTimeValue(xColumn.isNull(position), xColumn.getDouble(position)); } } } @@ -134,22 +144,23 @@ protected void addDoubleInput( @Override protected void addBinaryInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateBinaryFirstValue(xColumn, i, timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateBinaryFirstValue(xColumn, position, timeColumn.getLong(position)); - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + updateBinaryFirstValue( + xColumn.isNull(position), xColumn.getBinary(position), timeColumn.getLong(position)); + } else { + updateBinaryNullTimeValue(xColumn.isNull(position), xColumn.getBinary(position)); } } } @@ -157,22 +168,23 @@ protected void addBinaryInput( @Override protected void addBooleanInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateBooleanFirstValue(xColumn, i, timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateBooleanFirstValue(xColumn, position, timeColumn.getLong(position)); - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + updateBooleanFirstValue( + xColumn.isNull(position), xColumn.getBoolean(position), timeColumn.getLong(position)); + } else { + updateBooleanNullTimeValue(xColumn.isNull(position), xColumn.getBoolean(position)); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstDescAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstDescAccumulator.java index d8cc8ef0286d8..17103ab683898 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstDescAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstDescAccumulator.java @@ -40,132 +40,132 @@ public TableAccumulator copy() { @Override protected void addIntInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateIntFirstValue(valueColumn.getInt(i), timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateIntFirstValue(valueColumn.getInt(position), timeColumn.getLong(position)); - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: Time is not null. Attempt to update the value with the minimum time. + updateIntFirstValue(valueColumn.getInt(position), timeColumn.getLong(position)); + } else { + // Case B: Time is NULL, the nullTimeValue should only be assigned once + updateIntNullTimeValue(valueColumn.getInt(position)); } } } @Override protected void addLongInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateLongFirstValue(valueColumn.getLong(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateLongFirstValue(valueColumn.getLong(position), timeColumn.getLong(position)); - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: Time is not null. Attempt to update the value with the minimum time. + updateLongFirstValue(valueColumn.getLong(position), timeColumn.getLong(position)); + } else { + // Case B: Time is NULL, the nullTimeValue should only be assigned once + updateLongNullTimeValue(valueColumn.getLong(position)); } } } @Override protected void addFloatInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateFloatFirstValue(valueColumn.getFloat(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateFloatFirstValue(valueColumn.getFloat(position), timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateFloatFirstValue(valueColumn.getFloat(position), timeColumn.getLong(position)); + } else { + updateFloatNullTimeValue(valueColumn.getFloat(position)); } } } @Override protected void addDoubleInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateDoubleFirstValue(valueColumn.getDouble(i), timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateDoubleFirstValue(valueColumn.getDouble(position), timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateDoubleFirstValue(valueColumn.getDouble(position), timeColumn.getLong(position)); + } else { + updateDoubleNullTimeValue(valueColumn.getDouble(position)); } } } @Override protected void addBinaryInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateBinaryFirstValue(valueColumn.getBinary(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateBinaryFirstValue(valueColumn.getBinary(position), timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateBinaryFirstValue(valueColumn.getBinary(position), timeColumn.getLong(position)); + } else { + updateBinaryNullTimeValue(valueColumn.getBinary(position)); } } } @Override protected void addBooleanInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateBooleanFirstValue(valueColumn.getBoolean(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateBooleanFirstValue(valueColumn.getBoolean(position), timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateBooleanFirstValue(valueColumn.getBoolean(position), timeColumn.getLong(position)); + } else { + updateBooleanNullTimeValue(valueColumn.getBoolean(position)); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java index eb0d9fec5d350..4da324e8b0372 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java @@ -36,7 +36,7 @@ import java.nio.charset.StandardCharsets; import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Utils.serializeTimeValue; +import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Utils.serializeTimeValueWithNull; public class LastAccumulator implements TableAccumulator { @@ -46,10 +46,11 @@ public class LastAccumulator implements TableAccumulator { protected TsPrimitiveType lastValue; protected long maxTime = Long.MIN_VALUE; protected boolean initResult = false; + protected boolean initNullTimeValue = false; public LastAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; - lastValue = TsPrimitiveType.getByType(seriesDataType); + this.lastValue = TsPrimitiveType.getByType(seriesDataType); } public boolean hasInitResult() { @@ -122,26 +123,43 @@ public void addIntermediate(Column argument) { byte[] bytes = argument.getBinary(i).getValues(); long time = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 0); - int offset = Long.BYTES; + boolean isOrderTimeNull = BytesUtils.bytesToBool(bytes, Long.BYTES); + int offset = Long.BYTES + 1; switch (seriesDataType) { case INT32: case DATE: int intVal = BytesUtils.bytesToInt(bytes, offset); - updateIntLastValue(intVal, time); + if (!isOrderTimeNull) { + updateIntLastValue(intVal, time); + } else { + updateIntNullTimeValue(intVal); + } break; case INT64: case TIMESTAMP: long longVal = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - updateLongLastValue(longVal, time); + if (!isOrderTimeNull) { + updateLongLastValue(longVal, time); + } else { + updateLongNullTimeValue(longVal); + } break; case FLOAT: float floatVal = BytesUtils.bytesToFloat(bytes, offset); - updateFloatLastValue(floatVal, time); + if (!isOrderTimeNull) { + updateFloatLastValue(floatVal, time); + } else { + updateFloatNullTimeValue(floatVal); + } break; case DOUBLE: double doubleVal = BytesUtils.bytesToDouble(bytes, offset); - updateDoubleLastValue(doubleVal, time); + if (!isOrderTimeNull) { + updateDoubleLastValue(doubleVal, time); + } else { + updateDoubleNullTimeValue(doubleVal); + } break; case TEXT: case BLOB: @@ -150,11 +168,19 @@ public void addIntermediate(Column argument) { int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - updateBinaryLastValue(binaryVal, time); + if (!isOrderTimeNull) { + updateBinaryLastValue(binaryVal, time); + } else { + updateBinaryNullTimeValue(binaryVal); + } break; case BOOLEAN: boolean boolVal = BytesUtils.bytesToBool(bytes, offset); - updateBooleanLastValue(boolVal, time); + if (!isOrderTimeNull) { + updateBooleanLastValue(boolVal, time); + } else { + updateBooleanNullTimeValue(boolVal); + } break; default: throw new UnSupportedDataTypeException( @@ -169,46 +195,53 @@ public void evaluateIntermediate(ColumnBuilder columnBuilder) { columnBuilder instanceof BinaryColumnBuilder, "intermediate input and output of LAST should be BinaryColumn"); - if (!initResult) { - columnBuilder.appendNull(); - } else { - columnBuilder.writeBinary(new Binary(serializeTimeValue(seriesDataType, maxTime, lastValue))); + if (initResult || initNullTimeValue) { + // if the initResult is activated, the result must carry a not null time + boolean isOrderTimeNull = !initResult; + columnBuilder.writeBinary( + new Binary( + serializeTimeValueWithNull(seriesDataType, maxTime, isOrderTimeNull, lastValue))); + return; } + columnBuilder.appendNull(); } @Override public void evaluateFinal(ColumnBuilder columnBuilder) { - if (!initResult) { + + // all the values are the null + if (!initResult && !initNullTimeValue) { columnBuilder.appendNull(); - } else { - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(lastValue.getInt()); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(lastValue.getLong()); - break; - case FLOAT: - columnBuilder.writeFloat(lastValue.getFloat()); - break; - case DOUBLE: - columnBuilder.writeDouble(lastValue.getDouble()); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - columnBuilder.writeBinary(lastValue.getBinary()); - break; - case BOOLEAN: - columnBuilder.writeBoolean(lastValue.getBoolean()); - break; - default: - throw new UnSupportedDataTypeException( - String.format("Unsupported data type in LAST aggregation: %s", seriesDataType)); - } + return; + } + + switch (seriesDataType) { + case INT32: + case DATE: + columnBuilder.writeInt(lastValue.getInt()); + break; + case INT64: + case TIMESTAMP: + columnBuilder.writeLong(lastValue.getLong()); + break; + case FLOAT: + columnBuilder.writeFloat(lastValue.getFloat()); + break; + case DOUBLE: + columnBuilder.writeDouble(lastValue.getDouble()); + break; + case TEXT: + case BLOB: + case OBJECT: + case STRING: + columnBuilder.writeBinary(lastValue.getBinary()); + break; + case BOOLEAN: + columnBuilder.writeBoolean(lastValue.getBoolean()); + break; + default: + throw new UnSupportedDataTypeException( + String.format("Unsupported data type in LAST aggregation: %s", seriesDataType)); } } @@ -275,180 +308,230 @@ public void addStatistics(Statistics[] statistics) { @Override public void reset() { initResult = false; + this.initNullTimeValue = false; this.maxTime = Long.MIN_VALUE; this.lastValue.reset(); } + private boolean checkAndUpdateLastTime(long curTime) { + if (!initResult || curTime > maxTime) { + initResult = true; + maxTime = curTime; + return true; + } + return false; + } + + private boolean checkAndUpdateNullTime() { + if (!initResult && !initNullTimeValue) { + initNullTimeValue = true; + return true; + } + return false; + } + + /** + * Updates the accumulator, prioritizing values with valid timestamps over those with null + * timestamps. + */ protected void addIntInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateIntLastValue(valueColumn.getInt(i), timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateIntLastValue(valueColumn.getInt(position), timeColumn.getLong(position)); - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: Time is not null. Attempt to update the value with the minimum time. + updateIntLastValue(valueColumn.getInt(position), timeColumn.getLong(position)); + } else { + // Case B: Time is NULL, the nullTimeValue should only be assigned once + updateIntNullTimeValue(valueColumn.getInt(position)); } } } protected void updateIntLastValue(int value, long curTime) { - initResult = true; - if (curTime > maxTime) { - maxTime = curTime; + if (checkAndUpdateLastTime(curTime)) { + lastValue.setInt(value); + } + } + + protected void updateIntNullTimeValue(int value) { + if (checkAndUpdateNullTime()) { lastValue.setInt(value); } } + /** + * Updates the accumulator, prioritizing values with valid timestamps over those with null + * timestamps. + */ protected void addLongInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateLongLastValue(valueColumn.getLong(i), timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateLongLastValue(valueColumn.getLong(position), timeColumn.getLong(position)); - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: Time is not null. Attempt to update the value with the minimum time. + updateLongLastValue(valueColumn.getLong(position), timeColumn.getLong(position)); + } else { + // Case B: Time is NULL, the nullTimeValue should only be assigned once + updateLongNullTimeValue(valueColumn.getLong(position)); } } } protected void updateLongLastValue(long value, long curTime) { - initResult = true; - if (curTime > maxTime) { - maxTime = curTime; + if (checkAndUpdateLastTime(curTime)) { + lastValue.setLong(value); + } + } + + protected void updateLongNullTimeValue(long value) { + if (checkAndUpdateNullTime()) { lastValue.setLong(value); } } protected void addFloatInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateFloatLastValue(valueColumn.getFloat(i), timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateFloatLastValue(valueColumn.getFloat(position), timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateFloatLastValue(valueColumn.getFloat(position), timeColumn.getLong(position)); + } else { + updateFloatNullTimeValue(valueColumn.getFloat(position)); } } } protected void updateFloatLastValue(float value, long curTime) { - initResult = true; - if (curTime > maxTime) { - maxTime = curTime; + if (checkAndUpdateLastTime(curTime)) { + lastValue.setFloat(value); + } + } + + protected void updateFloatNullTimeValue(float value) { + if (checkAndUpdateNullTime()) { lastValue.setFloat(value); } } protected void addDoubleInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateDoubleLastValue(valueColumn.getDouble(i), timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateDoubleLastValue(valueColumn.getDouble(position), timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateDoubleLastValue(valueColumn.getDouble(position), timeColumn.getLong(position)); + } else { + updateDoubleNullTimeValue(valueColumn.getDouble(position)); } } } protected void updateDoubleLastValue(double value, long curTime) { - initResult = true; - if (curTime > maxTime) { - maxTime = curTime; + if (checkAndUpdateLastTime(curTime)) { + lastValue.setDouble(value); + } + } + + protected void updateDoubleNullTimeValue(double value) { + if (checkAndUpdateNullTime()) { lastValue.setDouble(value); } } protected void addBinaryInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateBinaryLastValue(valueColumn.getBinary(i), timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateBinaryLastValue(valueColumn.getBinary(position), timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateBinaryLastValue(valueColumn.getBinary(position), timeColumn.getLong(position)); + } else { + updateBinaryNullTimeValue(valueColumn.getBinary(position)); } } } protected void updateBinaryLastValue(Binary value, long curTime) { - initResult = true; - if (curTime > maxTime) { - maxTime = curTime; + if (checkAndUpdateLastTime(curTime)) { + lastValue.setBinary(value); + } + } + + protected void updateBinaryNullTimeValue(Binary value) { + if (checkAndUpdateNullTime()) { lastValue.setBinary(value); } } protected void addBooleanInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateBooleanLastValue(valueColumn.getBoolean(i), timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateBooleanLastValue(valueColumn.getBoolean(position), timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateBooleanLastValue(valueColumn.getBoolean(position), timeColumn.getLong(position)); + } else { + updateBooleanNullTimeValue(valueColumn.getBoolean(position)); } } } protected void updateBooleanLastValue(boolean value, long curTime) { - initResult = true; - if (curTime > maxTime) { - maxTime = curTime; + if (checkAndUpdateLastTime(curTime)) { + lastValue.setBoolean(value); + } + } + + protected void updateBooleanNullTimeValue(boolean value) { + if (checkAndUpdateNullTime()) { lastValue.setBoolean(value); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java index ddfff1ed2cfee..7309f9ef67de7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java @@ -33,7 +33,7 @@ import org.apache.tsfile.write.UnSupportedDataTypeException; import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Utils.serializeTimeValue; +import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Utils.serializeTimeValueWithNull; public class LastByAccumulator implements TableAccumulator { @@ -52,6 +52,7 @@ public class LastByAccumulator implements TableAccumulator { private boolean xIsNull = true; protected boolean initResult = false; + protected boolean initNullTimeValue = false; public LastByAccumulator( TSDataType xDataType, TSDataType yDataType, boolean xIsTimeColumn, boolean yIsTimeColumn) { @@ -59,7 +60,6 @@ public LastByAccumulator( this.yDataType = yDataType; this.xIsTimeColumn = xIsTimeColumn; this.yIsTimeColumn = yIsTimeColumn; - this.xResult = TsPrimitiveType.getByType(xDataType); } @@ -140,50 +140,81 @@ public void addIntermediate(Column argument) { byte[] bytes = argument.getBinary(i).getValues(); long curTime = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 0); int offset = Long.BYTES; - boolean isXNull = BytesUtils.bytesToBool(bytes, offset); + boolean isOrderTimeNull = BytesUtils.bytesToBool(bytes, offset); + offset += 1; + boolean isXValueNull = BytesUtils.bytesToBool(bytes, offset); offset += 1; - - if (isXNull) { - if (!initResult || curTime > yLastTime) { - initResult = true; - yLastTime = curTime; - xIsNull = true; - } - continue; - } switch (xDataType) { case INT32: case DATE: - int xIntVal = BytesUtils.bytesToInt(bytes, offset); - updateIntLastValue(xIntVal, curTime); + // if the x value is null, could not serialize the x value + int xIntVal = isXValueNull ? 0 : BytesUtils.bytesToInt(bytes, offset); + if (!isOrderTimeNull) { + updateIntLastValue(isXValueNull, xIntVal, curTime); + } else { + updateIntNullTimeValue(isXValueNull, xIntVal); + } break; + case INT64: case TIMESTAMP: - long longVal = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - updateLongLastValue(longVal, curTime); + long longVal = + isXValueNull ? 0 : BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); + if (!isOrderTimeNull) { + updateLongLastValue(isXValueNull, longVal, curTime); + } else { + updateLongNullTimeValue(isXValueNull, longVal); + } break; + case FLOAT: - float floatVal = BytesUtils.bytesToFloat(bytes, offset); - updateFloatLastValue(floatVal, curTime); + float floatVal = isXValueNull ? 0 : BytesUtils.bytesToFloat(bytes, offset); + if (!isOrderTimeNull) { + updateFloatLastValue(isXValueNull, floatVal, curTime); + } else { + updateFloatNullTimeValue(isXValueNull, floatVal); + } break; + case DOUBLE: - double doubleVal = BytesUtils.bytesToDouble(bytes, offset); - updateDoubleLastValue(doubleVal, curTime); + double doubleVal = isXValueNull ? 0 : BytesUtils.bytesToDouble(bytes, offset); + if (!isOrderTimeNull) { + updateDoubleLastValue(isXValueNull, doubleVal, curTime); + } else { + updateDoubleNullTimeValue(isXValueNull, doubleVal); + } break; + case TEXT: case BLOB: case STRING: case OBJECT: - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - updateBinaryLastValue(binaryVal, curTime); + Binary binaryVal = null; + if (!isXValueNull) { + int length = BytesUtils.bytesToInt(bytes, offset); + offset += Integer.BYTES; + binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); + } + if (!isOrderTimeNull) { + updateBinaryLastValue(isXValueNull, binaryVal, curTime); + } else { + updateBinaryNullTimeValue(isXValueNull, binaryVal); + } break; + case BOOLEAN: - boolean boolVal = BytesUtils.bytesToBool(bytes, offset); - updateBooleanLastValue(boolVal, curTime); + boolean boolVal = false; + if (!isXValueNull) { + boolVal = BytesUtils.bytesToBool(bytes, offset); + } + if (!isOrderTimeNull) { + updateBooleanLastValue(isXValueNull, boolVal, curTime); + } else { + updateBooleanNullTimeValue(isXValueNull, boolVal); + } break; + default: throw new UnSupportedDataTypeException( String.format("Unsupported data type in LAST_BY Aggregation: %s", yDataType)); @@ -197,17 +228,22 @@ public void evaluateIntermediate(ColumnBuilder columnBuilder) { columnBuilder instanceof BinaryColumnBuilder, "intermediate input and output of LAST_BY should be BinaryColumn"); - if (!initResult) { - columnBuilder.appendNull(); - } else { + if (initResult || initNullTimeValue) { + // if the initResult is activated, the result must carry a not null time + boolean isOrderTimeNull = !initResult; columnBuilder.writeBinary( - new Binary(serializeTimeValue(xDataType, yLastTime, xIsNull, xResult))); + new Binary( + serializeTimeValueWithNull(xDataType, yLastTime, xIsNull, isOrderTimeNull, xResult))); + return; } + + columnBuilder.appendNull(); } @Override public void evaluateFinal(ColumnBuilder columnBuilder) { - if (!initResult || xIsNull) { + + if (xIsNull) { columnBuilder.appendNull(); return; } @@ -316,273 +352,257 @@ public void addStatistics(Statistics[] statistics) { @Override public void reset() { initResult = false; + initNullTimeValue = false; xIsNull = true; this.yLastTime = Long.MIN_VALUE; this.xResult.reset(); } - // TODO can add last position optimization if last position is null ? - protected void addIntInput( - Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + private boolean checkAndUpdateLastTime(boolean isXValueNull, long curTime) { + if (!initResult || curTime > yLastTime) { + initResult = true; + yLastTime = curTime; - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateIntLastValue(xColumn, i, timeColumn.getLong(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateIntLastValue(xColumn, position, timeColumn.getLong(position)); - } + if (isXValueNull) { + xIsNull = true; + return false; + } else { + xIsNull = false; + return true; } } + return false; } - protected void updateIntLastValue(Column xColumn, int xIdx, long curTime) { - if (!initResult || curTime > yLastTime) { - initResult = true; - yLastTime = curTime; - if (xColumn.isNull(xIdx)) { + private boolean checkAndUpdateNullTime(boolean isXValueNull) { + if (!initResult && !initNullTimeValue) { + initNullTimeValue = true; + + if (isXValueNull) { xIsNull = true; + return false; } else { xIsNull = false; - xResult.setInt(xColumn.getInt(xIdx)); + return true; } } + return false; } - protected void updateIntLastValue(int val, long curTime) { - if (!initResult || curTime > yLastTime) { - initResult = true; - yLastTime = curTime; - xIsNull = false; - xResult.setInt(val); + // TODO can add last position optimization if last position is null ? + protected void addIntInput( + Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; + } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: The order time is not null. Attempt to update the xResult. + updateIntLastValue( + xColumn.isNull(position), xColumn.getInt(position), timeColumn.getLong(position)); + } else { + // Case B: The order time is null. Attempt to update the xNullTimeValue. + updateIntNullTimeValue(xColumn.isNull(position), xColumn.getInt(position)); + } + } + } + + protected void updateIntLastValue(boolean isXValueNull, int xValue, long curTime) { + if (checkAndUpdateLastTime(isXValueNull, curTime)) { + xResult.setInt(xValue); + } + } + + protected void updateIntNullTimeValue(boolean isXValueNull, int xValue) { + if (checkAndUpdateNullTime(isXValueNull)) { + xResult.setInt(xValue); } } protected void addLongInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateLongLastValue(xColumn, i, timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateLongLastValue(xColumn, position, timeColumn.getLong(position)); - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: The order time is not null. Attempt to update the xResult. + updateLongLastValue( + xColumn.isNull(position), xColumn.getLong(position), timeColumn.getLong(position)); + } else { + // Case A: The order time is null. Attempt to update the xNullTimeValue. + updateLongNullTimeValue(xColumn.isNull(position), xColumn.getLong(position)); } } } - protected void updateLongLastValue(Column xColumn, int xIdx, long curTime) { - if (!initResult || curTime > yLastTime) { - initResult = true; - yLastTime = curTime; - if (xColumn.isNull(xIdx)) { - xIsNull = true; - } else { - xIsNull = false; - xResult.setLong(xColumn.getLong(xIdx)); - } + protected void updateLongLastValue(boolean isXValueNull, long xValue, long curTime) { + if (checkAndUpdateLastTime(isXValueNull, curTime)) { + xResult.setLong(xValue); } } - protected void updateLongLastValue(long value, long curTime) { - if (!initResult || curTime > yLastTime) { - initResult = true; - yLastTime = curTime; - xIsNull = false; - xResult.setLong(value); + protected void updateLongNullTimeValue(boolean isXValueNull, long xValue) { + if (checkAndUpdateNullTime(isXValueNull)) { + xResult.setLong(xValue); } } protected void addFloatInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateFloatLastValue(xColumn, i, timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateFloatLastValue(xColumn, position, timeColumn.getLong(position)); - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: Valid Time + updateFloatLastValue( + xColumn.isNull(position), xColumn.getFloat(position), timeColumn.getLong(position)); + } else { + // Case B: Null Time + updateFloatNullTimeValue(xColumn.isNull(position), xColumn.getFloat(position)); } } } - protected void updateFloatLastValue(Column xColumn, int xIdx, long curTime) { - if (!initResult || curTime > yLastTime) { - initResult = true; - yLastTime = curTime; - if (xColumn.isNull(xIdx)) { - xIsNull = true; - } else { - xIsNull = false; - xResult.setFloat(xColumn.getFloat(xIdx)); - } + protected void updateFloatLastValue(boolean isXValueNull, float xValue, long curTime) { + if (checkAndUpdateLastTime(isXValueNull, curTime)) { + xResult.setFloat(xValue); } } - protected void updateFloatLastValue(float value, long curTime) { - if (!initResult || curTime > yLastTime) { - initResult = true; - yLastTime = curTime; - xIsNull = false; - xResult.setFloat(value); + protected void updateFloatNullTimeValue(boolean isXValueNull, float xValue) { + if (checkAndUpdateNullTime(isXValueNull)) { + xResult.setFloat(xValue); } } protected void addDoubleInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateDoubleLastValue(xColumn, i, timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateDoubleLastValue(xColumn, position, timeColumn.getLong(position)); - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + updateDoubleLastValue( + xColumn.isNull(position), xColumn.getDouble(position), timeColumn.getLong(position)); + } else { + updateDoubleNullTimeValue(xColumn.isNull(position), xColumn.getDouble(position)); } } } - protected void updateDoubleLastValue(Column xColumn, int xIdx, long curTime) { - if (!initResult || curTime > yLastTime) { - initResult = true; - yLastTime = curTime; - if (xColumn.isNull(xIdx)) { - xIsNull = true; - } else { - xIsNull = false; - xResult.setDouble(xColumn.getDouble(xIdx)); - } + protected void updateDoubleLastValue(boolean isXValueNull, double xValue, long curTime) { + if (checkAndUpdateLastTime(isXValueNull, curTime)) { + xResult.setDouble(xValue); } } - protected void updateDoubleLastValue(double val, long curTime) { - if (!initResult || curTime > yLastTime) { - initResult = true; - yLastTime = curTime; - xIsNull = false; - xResult.setDouble(val); + protected void updateDoubleNullTimeValue(boolean isXValueNull, double xValue) { + if (checkAndUpdateNullTime(isXValueNull)) { + xResult.setDouble(xValue); } } protected void addBinaryInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateBinaryLastValue(xColumn, i, timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateBinaryLastValue(xColumn, position, timeColumn.getLong(position)); - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + updateBinaryLastValue( + xColumn.isNull(position), xColumn.getBinary(position), timeColumn.getLong(position)); + } else { + updateBinaryNullTimeValue(xColumn.isNull(position), xColumn.getBinary(position)); } } } - protected void updateBinaryLastValue(Column xColumn, int xIdx, long curTime) { - if (!initResult || curTime > yLastTime) { - initResult = true; - yLastTime = curTime; - if (xColumn.isNull(xIdx)) { - xIsNull = true; - } else { - xIsNull = false; - xResult.setBinary(xColumn.getBinary(xIdx)); - } + protected void updateBinaryLastValue(boolean isXValueNull, Binary xValue, long curTime) { + if (checkAndUpdateLastTime(isXValueNull, curTime)) { + xResult.setBinary(xValue); } } - protected void updateBinaryLastValue(Binary val, long curTime) { - if (!initResult || curTime > yLastTime) { - initResult = true; - yLastTime = curTime; - xIsNull = false; - xResult.setBinary(val); + protected void updateBinaryNullTimeValue(boolean isXValueNull, Binary xValue) { + if (checkAndUpdateNullTime(isXValueNull)) { + xResult.setBinary(xValue); } } protected void addBooleanInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateBooleanLastValue(xColumn, i, timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateBooleanLastValue(xColumn, position, timeColumn.getLong(position)); - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + updateBooleanLastValue( + xColumn.isNull(position), xColumn.getBoolean(position), timeColumn.getLong(position)); + } else { + updateBooleanNullTimeValue(xColumn.isNull(position), xColumn.getBoolean(position)); } } } - protected void updateBooleanLastValue(Column xColumn, int xIdx, long curTime) { - if (!initResult || curTime > yLastTime) { - initResult = true; - yLastTime = curTime; - if (xColumn.isNull(xIdx)) { - xIsNull = true; - } else { - xIsNull = false; - xResult.setBoolean(xColumn.getBoolean(xIdx)); - } + protected void updateBooleanLastValue(boolean isXValueNull, boolean xValue, long curTime) { + if (checkAndUpdateLastTime(isXValueNull, curTime)) { + xResult.setBoolean(xValue); } } - protected void updateBooleanLastValue(boolean val, long curTime) { - if (!initResult || curTime > yLastTime) { - initResult = true; - yLastTime = curTime; - xIsNull = false; - xResult.setBoolean(val); + protected void updateBooleanNullTimeValue(boolean isXValueNull, boolean xValue) { + if (checkAndUpdateNullTime(isXValueNull)) { + xResult.setBoolean(xValue); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByDescAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByDescAccumulator.java index 37ac6c790363e..d664f2286d6a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByDescAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByDescAccumulator.java @@ -77,28 +77,28 @@ public boolean hasFinalResult() { @Override protected void addIntInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateIntLastValue(xColumn, i, timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateIntLastValue(xColumn, position, timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: The order time is not null. Attempt to update the xResult. + updateIntLastValue( + xColumn.isNull(position), xColumn.getInt(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + // Case B: The order time is null. Attempt to update the xNullTimeValue. + updateIntNullTimeValue(xColumn.isNull(position), xColumn.getInt(position)); } } } @@ -106,28 +106,28 @@ protected void addIntInput( @Override protected void addLongInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateLongLastValue(xColumn, i, timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateLongLastValue(xColumn, position, timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: The order time is not null. Attempt to update the xResult. + updateLongLastValue( + xColumn.isNull(position), xColumn.getLong(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + // Case B: The order time is null. Attempt to update the xNullTimeValue. + updateLongNullTimeValue(xColumn.isNull(position), xColumn.getLong(position)); } } } @@ -135,28 +135,28 @@ protected void addLongInput( @Override protected void addFloatInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateFloatLastValue(xColumn, i, timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateFloatLastValue(xColumn, position, timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: Valid Time + updateFloatLastValue( + xColumn.isNull(position), xColumn.getFloat(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + // Case B: Null Time + updateFloatNullTimeValue(xColumn.isNull(position), xColumn.getFloat(position)); } } } @@ -164,28 +164,26 @@ protected void addFloatInput( @Override protected void addDoubleInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateDoubleLastValue(xColumn, i, timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateDoubleLastValue(xColumn, position, timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + updateDoubleLastValue( + xColumn.isNull(position), xColumn.getDouble(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + updateDoubleNullTimeValue(xColumn.isNull(position), xColumn.getDouble(position)); } } } @@ -193,28 +191,26 @@ protected void addDoubleInput( @Override protected void addBinaryInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateBinaryLastValue(xColumn, i, timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateBinaryLastValue(xColumn, position, timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + updateBinaryLastValue( + xColumn.isNull(position), xColumn.getBinary(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + updateBinaryNullTimeValue(xColumn.isNull(position), xColumn.getBinary(position)); } } } @@ -222,28 +218,26 @@ protected void addBinaryInput( @Override protected void addBooleanInput( Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateBooleanLastValue(xColumn, i, timeColumn.getLong(i)); - if (canFinishAfterInit) { - return; - } - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateBooleanLastValue(xColumn, position, timeColumn.getLong(position)); - if (canFinishAfterInit) { - return; - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + updateBooleanLastValue( + xColumn.isNull(position), xColumn.getBoolean(position), timeColumn.getLong(position)); + if (canFinishAfterInit) { + return; } + } else { + updateBooleanNullTimeValue(xColumn.isNull(position), xColumn.getBoolean(position)); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastDescAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastDescAccumulator.java index d015b7599573c..80c1227056f49 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastDescAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastDescAccumulator.java @@ -59,144 +59,138 @@ public boolean hasFinalResult() { @Override protected void addIntInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateIntLastValue(valueColumn.getInt(i), timeColumn.getLong(i)); - return; - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateIntLastValue(valueColumn.getInt(position), timeColumn.getLong(position)); - return; - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: Time is not null. Attempt to update the value with the minimum time. + updateIntLastValue(valueColumn.getInt(position), timeColumn.getLong(position)); + return; + } else { + // Case B: Time is NULL, the nullTimeValue should only be assigned once + updateIntNullTimeValue(valueColumn.getInt(position)); } } } @Override protected void addLongInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateLongLastValue(valueColumn.getLong(i), timeColumn.getLong(i)); - return; - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateLongLastValue(valueColumn.getLong(position), timeColumn.getLong(position)); - return; - } + + // Check if the time is null + if (!timeColumn.isNull(position)) { + // Case A: Time is not null. Attempt to update the value with the minimum time. + updateLongLastValue(valueColumn.getLong(position), timeColumn.getLong(position)); + return; + } else { + // Case B: Time is NULL, the nullTimeValue should only be assigned once + updateLongNullTimeValue(valueColumn.getLong(position)); } } } @Override protected void addFloatInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateFloatLastValue(valueColumn.getFloat(i), timeColumn.getLong(i)); - return; - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateFloatLastValue(valueColumn.getFloat(position), timeColumn.getLong(position)); - return; - } + + if (!timeColumn.isNull(position)) { + updateFloatLastValue(valueColumn.getFloat(position), timeColumn.getLong(position)); + return; + } else { + updateFloatNullTimeValue(valueColumn.getFloat(position)); } } } @Override protected void addDoubleInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateDoubleLastValue(valueColumn.getDouble(i), timeColumn.getLong(i)); - return; - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateDoubleLastValue(valueColumn.getDouble(position), timeColumn.getLong(position)); - return; - } + + if (!timeColumn.isNull(position)) { + updateDoubleLastValue(valueColumn.getDouble(position), timeColumn.getLong(position)); + return; + } else { + updateDoubleNullTimeValue(valueColumn.getDouble(position)); } } } @Override protected void addBinaryInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateBinaryLastValue(valueColumn.getBinary(i), timeColumn.getLong(i)); - return; - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateBinaryLastValue(valueColumn.getBinary(position), timeColumn.getLong(position)); - return; - } + + if (!timeColumn.isNull(position)) { + updateBinaryLastValue(valueColumn.getBinary(position), timeColumn.getLong(position)); + return; + } else { + updateBinaryNullTimeValue(valueColumn.getBinary(position)); } } } @Override protected void addBooleanInput(Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); - - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateBooleanLastValue(valueColumn.getBoolean(i), timeColumn.getLong(i)); - return; - } + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateBooleanLastValue(valueColumn.getBoolean(position), timeColumn.getLong(position)); - return; - } + + if (!timeColumn.isNull(position)) { + updateBooleanLastValue(valueColumn.getBoolean(position), timeColumn.getLong(position)); + return; + } else { + updateBooleanNullTimeValue(valueColumn.getBoolean(position)); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java index 99d5fef12089f..1bd0e64e6e100 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java @@ -78,6 +78,16 @@ public static void serializeBinaryValue(Binary binary, byte[] valueBytes, int of System.arraycopy(binary.getValues(), 0, valueBytes, offset, binary.getValues().length); } + public static byte[] serializeTimeValueWithNull( + TSDataType seriesDataType, long time, boolean isOrderTimeNull, TsPrimitiveType value) { + + byte[] valueBytes = new byte[9 + calcTypeSize(seriesDataType, value)]; + BytesUtils.longToBytes(time, valueBytes, 0); + BytesUtils.boolToBytes(isOrderTimeNull, valueBytes, 8); + serializeValue(seriesDataType, value, valueBytes, 9); + return valueBytes; + } + public static byte[] serializeTimeValue( TSDataType seriesDataType, long time, TsPrimitiveType value) { byte[] valueBytes = new byte[8 + calcTypeSize(seriesDataType, value)]; @@ -103,6 +113,26 @@ public static byte[] serializeTimeValue( return valueBytes; } + public static byte[] serializeTimeValueWithNull( + TSDataType seriesDataType, + long time, + boolean valueIsNull, + boolean isOrderTimeNull, + TsPrimitiveType value) { + // Allocate buffer: fixed header size (10 bytes) + dynamic value size if present + byte[] valueBytes = + valueIsNull ? new byte[10] : new byte[10 + calcTypeSize(seriesDataType, value)]; + BytesUtils.longToBytes(time, valueBytes, 0); + BytesUtils.boolToBytes(isOrderTimeNull, valueBytes, 8); + BytesUtils.boolToBytes(valueIsNull, valueBytes, 9); + + // Serialize body: actual value if not null + if (!valueIsNull) { + serializeValue(seriesDataType, value, valueBytes, 10); + } + return valueBytes; + } + public static int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { switch (dataType) { case BOOLEAN: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java index b8fe8bbe1b224..456b46c24ac19 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java @@ -50,6 +50,8 @@ public class GroupedFirstAccumulator implements GroupedAccumulator { RamUsageEstimator.shallowSizeOfInstance(GroupedFirstAccumulator.class); private final TSDataType seriesDataType; private final LongBigArray minTimes = new LongBigArray(Long.MAX_VALUE); + private final BooleanBigArray inits = new BooleanBigArray(); + private final BooleanBigArray initNullTimeValues = new BooleanBigArray(); private LongBigArray longValues; private IntBigArray intValues; @@ -122,12 +124,18 @@ public long getEstimatedSize() { String.format("Unsupported data type in FIRST Aggregation: %s", seriesDataType)); } - return INSTANCE_SIZE + valuesSize; + return INSTANCE_SIZE + + inits.sizeOf() + + minTimes.sizeOf() + + initNullTimeValues.sizeOf() + + valuesSize; } @Override public void setGroupCount(long groupCount) { minTimes.ensureCapacity(groupCount); + inits.ensureCapacity(groupCount); + initNullTimeValues.ensureCapacity(groupCount); switch (seriesDataType) { case INT32: case DATE: @@ -207,25 +215,44 @@ public void addIntermediate(int[] groupIds, Column argument) { byte[] bytes = argument.getBinary(i).getValues(); long time = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 0); int offset = Long.BYTES; + boolean isOrderTimeNull = BytesUtils.bytesToBool(bytes, offset); + offset += 1; + int groupId = groupIds[i]; switch (seriesDataType) { case INT32: case DATE: int intVal = BytesUtils.bytesToInt(bytes, offset); - updateIntValue(groupIds[i], intVal, time); + if (!isOrderTimeNull) { + updateIntValue(groupId, intVal, time); + } else { + updateIntNullTimeValue(groupId, intVal); + } break; case INT64: case TIMESTAMP: long longVal = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - updateLongValue(groupIds[i], longVal, time); + if (!isOrderTimeNull) { + updateLongValue(groupId, longVal, time); + } else { + updateLongNullTimeValue(groupId, longVal); + } break; case FLOAT: float floatVal = BytesUtils.bytesToFloat(bytes, offset); - updateFloatValue(groupIds[i], floatVal, time); + if (!isOrderTimeNull) { + updateFloatValue(groupId, floatVal, time); + } else { + updateFloatNullTimeValue(groupId, floatVal); + } break; case DOUBLE: double doubleVal = BytesUtils.bytesToDouble(bytes, offset); - updateDoubleValue(groupIds[i], doubleVal, time); + if (!isOrderTimeNull) { + updateDoubleValue(groupId, doubleVal, time); + } else { + updateDoubleNullTimeValue(groupId, doubleVal); + } break; case TEXT: case BLOB: @@ -234,11 +261,19 @@ public void addIntermediate(int[] groupIds, Column argument) { int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - updateBinaryValue(groupIds[i], binaryVal, time); + if (!isOrderTimeNull) { + updateBinaryValue(groupId, binaryVal, time); + } else { + updateBinaryNullTimeValue(groupId, binaryVal); + } break; case BOOLEAN: boolean boolVal = BytesUtils.bytesToBool(bytes, offset); - updateBooleanValue(groupIds[i], boolVal, time); + if (!isOrderTimeNull) { + updateBooleanValue(groupId, boolVal, time); + } else { + updateBooleanNullTimeValue(groupId, boolVal); + } break; default: throw new UnSupportedDataTypeException( @@ -252,46 +287,47 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { checkArgument( columnBuilder instanceof BinaryColumnBuilder, "intermediate input and output of FIRST should be BinaryColumn"); - if (minTimes.get(groupId) == Long.MAX_VALUE) { - columnBuilder.appendNull(); - } else { + if (inits.get(groupId) || initNullTimeValues.get(groupId)) { columnBuilder.writeBinary(new Binary(serializeTimeWithValue(groupId))); + return; } + columnBuilder.appendNull(); } @Override public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { - if (minTimes.get(groupId) == Long.MAX_VALUE) { + if (!inits.get(groupId) && !initNullTimeValues.get(groupId)) { columnBuilder.appendNull(); - } else { - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(intValues.get(groupId)); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(longValues.get(groupId)); - break; - case FLOAT: - columnBuilder.writeFloat(floatValues.get(groupId)); - break; - case DOUBLE: - columnBuilder.writeDouble(doubleValues.get(groupId)); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - columnBuilder.writeBinary(binaryValues.get(groupId)); - break; - case BOOLEAN: - columnBuilder.writeBoolean(booleanValues.get(groupId)); - break; - default: - throw new UnSupportedDataTypeException( - String.format("Unsupported data type in FIRST Aggregation: %s", seriesDataType)); - } + return; + } + + switch (seriesDataType) { + case INT32: + case DATE: + columnBuilder.writeInt(intValues.get(groupId)); + break; + case INT64: + case TIMESTAMP: + columnBuilder.writeLong(longValues.get(groupId)); + break; + case FLOAT: + columnBuilder.writeFloat(floatValues.get(groupId)); + break; + case DOUBLE: + columnBuilder.writeDouble(doubleValues.get(groupId)); + break; + case TEXT: + case BLOB: + case OBJECT: + case STRING: + columnBuilder.writeBinary(binaryValues.get(groupId)); + break; + case BOOLEAN: + columnBuilder.writeBoolean(booleanValues.get(groupId)); + break; + default: + throw new UnSupportedDataTypeException( + String.format("Unsupported data type in FIRST Aggregation: %s", seriesDataType)); } } @@ -301,6 +337,8 @@ public void prepareFinal() {} @Override public void reset() { minTimes.reset(); + inits.reset(); + initNullTimeValues.reset(); switch (seriesDataType) { case INT32: case DATE: @@ -334,32 +372,39 @@ public void reset() { private byte[] serializeTimeWithValue(int groupId) { byte[] bytes; int length = Long.BYTES; + boolean isOrderTimeNull = !inits.get(groupId); + length += 1; + switch (seriesDataType) { case INT32: case DATE: length += Integer.BYTES; bytes = new byte[length]; longToBytes(minTimes.get(groupId), bytes, 0); - intToBytes(intValues.get(groupId), bytes, Long.BYTES); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES); + intToBytes(intValues.get(groupId), bytes, Long.BYTES + 1); return bytes; case INT64: case TIMESTAMP: length += Long.BYTES; bytes = new byte[length]; longToBytes(minTimes.get(groupId), bytes, 0); - longToBytes(longValues.get(groupId), bytes, Long.BYTES); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES); + longToBytes(longValues.get(groupId), bytes, Long.BYTES + 1); return bytes; case FLOAT: length += Float.BYTES; bytes = new byte[length]; longToBytes(minTimes.get(groupId), bytes, 0); - floatToBytes(floatValues.get(groupId), bytes, Long.BYTES); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES); + floatToBytes(floatValues.get(groupId), bytes, Long.BYTES + 1); return bytes; case DOUBLE: length += Double.BYTES; bytes = new byte[length]; longToBytes(minTimes.get(groupId), bytes, 0); - doubleToBytes(doubleValues.get(groupId), bytes, Long.BYTES); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES); + doubleToBytes(doubleValues.get(groupId), bytes, Long.BYTES + 1); return bytes; case TEXT: case BLOB: @@ -369,14 +414,16 @@ private byte[] serializeTimeWithValue(int groupId) { length += Integer.BYTES + values.length; bytes = new byte[length]; longToBytes(minTimes.get(groupId), bytes, 0); - BytesUtils.intToBytes(values.length, bytes, Long.BYTES); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES); + BytesUtils.intToBytes(values.length, bytes, Long.BYTES + 1); System.arraycopy(values, 0, bytes, length - values.length, values.length); return bytes; case BOOLEAN: length++; bytes = new byte[length]; longToBytes(minTimes.get(groupId), bytes, 0); - boolToBytes(booleanValues.get(groupId), bytes, Long.BYTES); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES); + boolToBytes(booleanValues.get(groupId), bytes, Long.BYTES + 1); return bytes; default: throw new UnSupportedDataTypeException( @@ -384,188 +431,223 @@ private byte[] serializeTimeWithValue(int groupId) { } } + private boolean checkAndUpdateFirstTime(int groupId, long curTime) { + if (!inits.get(groupId) || curTime < minTimes.get(groupId)) { + inits.set(groupId, true); + minTimes.set(groupId, curTime); + return true; + } + return false; + } + + private boolean checkAndUpdateNullTime(int groupId) { + if (!inits.get(groupId) && !initNullTimeValues.get(groupId)) { + initNullTimeValues.set(groupId, true); + return true; + } + return false; + } + private void addIntInput( int[] groupIds, Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateIntValue(groupIds[i], valueColumn.getInt(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateIntValue( - groupIds[position], valueColumn.getInt(position), timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateIntValue( + groupIds[position], valueColumn.getInt(position), timeColumn.getLong(position)); + } else { + updateIntNullTimeValue(groupIds[position], valueColumn.getInt(position)); } } } protected void updateIntValue(int groupId, int value, long curTime) { - long minTime = minTimes.get(groupId); - if (curTime < minTime) { - minTimes.set(groupId, curTime); + if (checkAndUpdateFirstTime(groupId, curTime)) { + intValues.set(groupId, value); + } + } + + protected void updateIntNullTimeValue(int groupId, int value) { + if (checkAndUpdateNullTime(groupId)) { intValues.set(groupId, value); } } private void addLongInput( int[] groupIds, Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateLongValue(groupIds[i], valueColumn.getLong(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateLongValue( - groupIds[position], valueColumn.getLong(position), timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateLongValue( + groupIds[position], valueColumn.getLong(position), timeColumn.getLong(position)); + } else { + updateLongNullTimeValue(groupIds[position], valueColumn.getLong(position)); } } } protected void updateLongValue(int groupId, long value, long curTime) { - long minTime = minTimes.get(groupId); - if (curTime < minTime) { - minTimes.set(groupId, curTime); + if (checkAndUpdateFirstTime(groupId, curTime)) { + longValues.set(groupId, value); + } + } + + protected void updateLongNullTimeValue(int groupId, long value) { + if (checkAndUpdateNullTime(groupId)) { longValues.set(groupId, value); } } private void addFloatInput( int[] groupIds, Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateFloatValue(groupIds[i], valueColumn.getFloat(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateFloatValue( - groupIds[position], valueColumn.getFloat(position), timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateFloatValue( + groupIds[position], valueColumn.getFloat(position), timeColumn.getLong(position)); + } else { + updateFloatNullTimeValue(groupIds[position], valueColumn.getFloat(position)); } } } protected void updateFloatValue(int groupId, float value, long curTime) { - long minTime = minTimes.get(groupId); - if (curTime < minTime) { - minTimes.set(groupId, curTime); + if (checkAndUpdateFirstTime(groupId, curTime)) { + floatValues.set(groupId, value); + } + } + + protected void updateFloatNullTimeValue(int groupId, float value) { + if (checkAndUpdateNullTime(groupId)) { floatValues.set(groupId, value); } } private void addDoubleInput( int[] groupIds, Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateDoubleValue(groupIds[i], valueColumn.getDouble(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateDoubleValue( - groupIds[position], valueColumn.getDouble(position), timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateDoubleValue( + groupIds[position], valueColumn.getDouble(position), timeColumn.getLong(position)); + } else { + updateDoubleNullTimeValue(groupIds[position], valueColumn.getDouble(position)); } } } protected void updateDoubleValue(int groupId, double value, long curTime) { - long minTime = minTimes.get(groupId); - if (curTime < minTime) { - minTimes.set(groupId, curTime); + if (checkAndUpdateFirstTime(groupId, curTime)) { + doubleValues.set(groupId, value); + } + } + + protected void updateDoubleNullTimeValue(int groupId, double value) { + if (checkAndUpdateNullTime(groupId)) { doubleValues.set(groupId, value); } } private void addBinaryInput( int[] groupIds, Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateBinaryValue(groupIds[i], valueColumn.getBinary(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateBinaryValue( - groupIds[position], valueColumn.getBinary(position), timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateBinaryValue( + groupIds[position], valueColumn.getBinary(position), timeColumn.getLong(position)); + } else { + updateBinaryNullTimeValue(groupIds[position], valueColumn.getBinary(position)); } } } protected void updateBinaryValue(int groupId, Binary value, long curTime) { - long minTime = minTimes.get(groupId); - if (curTime < minTime) { - minTimes.set(groupId, curTime); + if (checkAndUpdateFirstTime(groupId, curTime)) { + binaryValues.set(groupId, value); + } + } + + protected void updateBinaryNullTimeValue(int groupId, Binary value) { + if (checkAndUpdateNullTime(groupId)) { binaryValues.set(groupId, value); } } private void addBooleanInput( int[] groupIds, Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateBooleanValue(groupIds[i], valueColumn.getBoolean(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateBooleanValue( - groupIds[position], valueColumn.getBoolean(position), timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateBooleanValue( + groupIds[position], valueColumn.getBoolean(position), timeColumn.getLong(position)); + } else { + updateBooleanNullTimeValue(groupIds[position], valueColumn.getBoolean(position)); } } } protected void updateBooleanValue(int groupId, boolean value, long curTime) { - long minTime = minTimes.get(groupId); - if (curTime < minTime) { - minTimes.set(groupId, curTime); + if (checkAndUpdateFirstTime(groupId, curTime)) { + booleanValues.set(groupId, value); + } + } + + protected void updateBooleanNullTimeValue(int groupId, boolean value) { + if (checkAndUpdateNullTime(groupId)) { booleanValues.set(groupId, value); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java index d56ba59ef946b..38ff43f06df61 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java @@ -56,6 +56,7 @@ public class GroupedFirstByAccumulator implements GroupedAccumulator { private final LongBigArray yFirstTimes = new LongBigArray(Long.MAX_VALUE); private final BooleanBigArray inits = new BooleanBigArray(); + private final BooleanBigArray initNullTimeValues = new BooleanBigArray(); private LongBigArray xLongValues; private IntBigArray xIntValues; @@ -132,13 +133,19 @@ public long getEstimatedSize() { String.format("Unsupported data type in FIRST_BY Aggregation: %s", xDataType)); } - return INSTANCE_SIZE + valuesSize + yFirstTimes.sizeOf() + inits.sizeOf() + xNulls.sizeOf(); + return INSTANCE_SIZE + + valuesSize + + yFirstTimes.sizeOf() + + inits.sizeOf() + + initNullTimeValues.sizeOf() + + xNulls.sizeOf(); } @Override public void setGroupCount(long groupCount) { yFirstTimes.ensureCapacity(groupCount); inits.ensureCapacity(groupCount); + initNullTimeValues.ensureCapacity(groupCount); xNulls.ensureCapacity(groupCount); switch (xDataType) { case INT32: @@ -177,6 +184,7 @@ public void prepareFinal() {} public void reset() { yFirstTimes.reset(); inits.reset(); + initNullTimeValues.reset(); xNulls.reset(); switch (xDataType) { case INT32: @@ -259,50 +267,74 @@ public void addIntermediate(int[] groupIds, Column argument) { byte[] bytes = argument.getBinary(i).getValues(); long curTime = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 0); int offset = Long.BYTES; - boolean isXNull = BytesUtils.bytesToBool(bytes, offset); + boolean isXValueNull = BytesUtils.bytesToBool(bytes, offset); + offset += 1; + boolean isOrderTimeNull = BytesUtils.bytesToBool(bytes, offset); offset += 1; int groupId = groupIds[i]; - if (isXNull) { - if (!inits.get(groupId) || curTime < yFirstTimes.get(groupId)) { - inits.set(groupId, true); - yFirstTimes.set(groupId, curTime); - xNulls.set(groupId, true); - } - continue; - } - switch (xDataType) { case INT32: case DATE: - int xIntVal = BytesUtils.bytesToInt(bytes, offset); - updateIntFirstValue(groupId, xIntVal, curTime); + int intVal = isXValueNull ? 0 : BytesUtils.bytesToInt(bytes, offset); + if (!isOrderTimeNull) { + updateIntFirstValue(groupId, isXValueNull, intVal, curTime); + } else { + updateIntNullTimeValue(groupId, isXValueNull, intVal); + } break; case INT64: case TIMESTAMP: - long longVal = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - updateLongFirstValue(groupId, longVal, curTime); + long longVal = + isXValueNull ? 0 : BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); + if (!isOrderTimeNull) { + updateLongFirstValue(groupId, isXValueNull, longVal, curTime); + } else { + updateLongNullTimeValue(groupId, isXValueNull, longVal); + } break; case FLOAT: - float floatVal = BytesUtils.bytesToFloat(bytes, offset); - updateFloatFirstValue(groupId, floatVal, curTime); + float floatVal = isXValueNull ? 0 : BytesUtils.bytesToFloat(bytes, offset); + if (!isOrderTimeNull) { + updateFloatFirstValue(groupId, isXValueNull, floatVal, curTime); + } else { + updateFloatNullTimeValue(groupId, isXValueNull, floatVal); + } break; case DOUBLE: - double doubleVal = BytesUtils.bytesToDouble(bytes, offset); - updateDoubleFirstValue(groupId, doubleVal, curTime); + double doubleVal = isXValueNull ? 0 : BytesUtils.bytesToDouble(bytes, offset); + if (!isOrderTimeNull) { + updateDoubleFirstValue(groupId, isXValueNull, doubleVal, curTime); + } else { + updateDoubleNullTimeValue(groupId, isXValueNull, doubleVal); + } break; case TEXT: case BLOB: case OBJECT: case STRING: - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - updateBinaryFirstValue(groupId, binaryVal, curTime); + Binary binaryVal = null; + if (!isXValueNull) { + int length = BytesUtils.bytesToInt(bytes, offset); + offset += Integer.BYTES; + binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); + } + if (!isOrderTimeNull) { + updateBinaryFirstValue(groupId, isXValueNull, binaryVal, curTime); + } else { + updateBinaryNullTimeValue(groupId, isXValueNull, binaryVal); + } break; case BOOLEAN: - boolean boolVal = BytesUtils.bytesToBool(bytes, offset); - updateBooleanFirstValue(groupId, boolVal, curTime); + boolean boolVal = false; + if (!isXValueNull) { + boolVal = BytesUtils.bytesToBool(bytes, offset); + } + if (!isOrderTimeNull) { + updateBooleanFirstValue(groupId, isXValueNull, boolVal, curTime); + } else { + updateBooleanNullTimeValue(groupId, isXValueNull, boolVal); + } break; default: throw new UnSupportedDataTypeException( @@ -317,46 +349,51 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { columnBuilder instanceof BinaryColumnBuilder, "intermediate input and output of FIRST_BY should be BinaryColumn"); - if (!inits.get(groupId)) { - columnBuilder.appendNull(); - } else { + if (inits.get(groupId) || initNullTimeValues.get(groupId)) { columnBuilder.writeBinary(new Binary(serializeTimeWithValue(groupId))); + return; } + columnBuilder.appendNull(); } + /** Serializes group state: Time (8B) | xNull (1B) | OrderTimeNull (1B) | [Value] */ private byte[] serializeTimeWithValue(int groupId) { boolean xNull = xNulls.get(groupId); - int length = Long.BYTES + 1 + (xNull ? 0 : calculateValueLength(groupId)); + int length = Long.BYTES + 2 + (xNull ? 0 : calculateValueLength(groupId)); byte[] bytes = new byte[length]; + boolean isOrderTimeNull = !inits.get(groupId); longToBytes(yFirstTimes.get(groupId), bytes, 0); - boolToBytes(xNulls.get(groupId), bytes, Long.BYTES); + boolToBytes(xNull, bytes, Long.BYTES); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES + 1); + if (!xNull) { + int valueOffset = Long.BYTES + 2; switch (xDataType) { case INT32: case DATE: - intToBytes(xIntValues.get(groupId), bytes, Long.BYTES + 1); + intToBytes(xIntValues.get(groupId), bytes, valueOffset); return bytes; case INT64: case TIMESTAMP: - longToBytes(xLongValues.get(groupId), bytes, Long.BYTES + 1); + longToBytes(xLongValues.get(groupId), bytes, valueOffset); return bytes; case FLOAT: - floatToBytes(xFloatValues.get(groupId), bytes, Long.BYTES + 1); + floatToBytes(xFloatValues.get(groupId), bytes, valueOffset); return bytes; case DOUBLE: - doubleToBytes(xDoubleValues.get(groupId), bytes, Long.BYTES + 1); + doubleToBytes(xDoubleValues.get(groupId), bytes, valueOffset); return bytes; case TEXT: case BLOB: case OBJECT: case STRING: byte[] values = xBinaryValues.get(groupId).getValues(); - intToBytes(values.length, bytes, Long.BYTES + 1); + intToBytes(values.length, bytes, valueOffset); System.arraycopy(values, 0, bytes, length - values.length, values.length); return bytes; case BOOLEAN: - boolToBytes(xBooleanValues.get(groupId), bytes, Long.BYTES + 1); + boolToBytes(xBooleanValues.get(groupId), bytes, valueOffset); return bytes; default: throw new UnSupportedDataTypeException( @@ -393,7 +430,7 @@ private int calculateValueLength(int groupId) { @Override public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { - if (!inits.get(groupId) || xNulls.get(groupId)) { + if (xNulls.get(groupId)) { columnBuilder.appendNull(); return; } @@ -428,271 +465,267 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { } } - private void addIntInput( - int[] groupIds, Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + private boolean checkAndUpdateFirstTime(int groupId, boolean isXValueNull, long curTime) { + if (!inits.get(groupId) || curTime < yFirstTimes.get(groupId)) { + inits.set(groupId, true); + yFirstTimes.set(groupId, curTime); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateIntFirstValue(groupIds[i], xColumn, i, timeColumn.getLong(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateIntFirstValue(groupIds[position], xColumn, position, timeColumn.getLong(position)); - } + if (isXValueNull) { + xNulls.set(groupId, true); + return false; + } else { + xNulls.set(groupId, false); + return true; } } + return false; } - protected void updateIntFirstValue(int groupId, Column xColumn, int xIdx, long curTime) { - if (!inits.get(groupId) || curTime < yFirstTimes.get(groupId)) { - inits.set(groupId, true); - yFirstTimes.set(groupId, curTime); - if (xColumn.isNull(xIdx)) { + private boolean checkAndUpdateNullTime(int groupId, boolean isXValueNull) { + if (!inits.get(groupId) && !initNullTimeValues.get(groupId)) { + initNullTimeValues.set(groupId, true); + + if (isXValueNull) { xNulls.set(groupId, true); + return false; } else { xNulls.set(groupId, false); - xIntValues.set(groupId, xColumn.getInt(xIdx)); + return true; } } + return false; } - protected void updateIntFirstValue(int groupId, int val, long curTime) { - if (!inits.get(groupId) || curTime < yFirstTimes.get(groupId)) { - inits.set(groupId, true); - yFirstTimes.set(groupId, curTime); - xNulls.set(groupId, false); - xIntValues.set(groupId, val); + private void addIntInput( + int[] groupIds, Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; + } + + if (!timeColumn.isNull(position)) { + updateIntFirstValue( + groupIds[position], + xColumn.isNull(position), + xColumn.getInt(position), + timeColumn.getLong(position)); + } else { + updateIntNullTimeValue( + groupIds[position], xColumn.isNull(position), xColumn.getInt(position)); + } + } + } + + protected void updateIntFirstValue(int groupId, boolean isXValueNull, int xValue, long curTime) { + if (checkAndUpdateFirstTime(groupId, isXValueNull, curTime)) { + xIntValues.set(groupId, xValue); + } + } + + protected void updateIntNullTimeValue(int groupId, boolean isXValueNull, int xValue) { + if (checkAndUpdateNullTime(groupId, isXValueNull)) { + xIntValues.set(groupId, xValue); } } private void addLongInput( int[] groupIds, Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateLongFirstValue(groupIds[i], xColumn, i, timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateLongFirstValue(groupIds[position], xColumn, position, timeColumn.getLong(position)); - } + + int groupId = groupIds[position]; + if (!timeColumn.isNull(position)) { + updateLongFirstValue( + groupId, + xColumn.isNull(position), + xColumn.getLong(position), + timeColumn.getLong(position)); + } else { + updateLongNullTimeValue(groupId, xColumn.isNull(position), xColumn.getLong(position)); } } } - protected void updateLongFirstValue(int groupId, Column xColumn, int xIdx, long curTime) { - if (!inits.get(groupId) || curTime < yFirstTimes.get(groupId)) { - inits.set(groupId, true); - yFirstTimes.set(groupId, curTime); - if (xColumn.isNull(xIdx)) { - xNulls.set(groupId, true); - } else { - xNulls.set(groupId, false); - xLongValues.set(groupId, xColumn.getLong(xIdx)); - } + protected void updateLongFirstValue( + int groupId, boolean isXValueNull, long xValue, long curTime) { + if (checkAndUpdateFirstTime(groupId, isXValueNull, curTime)) { + xLongValues.set(groupId, xValue); } } - protected void updateLongFirstValue(int groupId, long val, long curTime) { - if (!inits.get(groupId) || curTime < yFirstTimes.get(groupId)) { - inits.set(groupId, true); - yFirstTimes.set(groupId, curTime); - xNulls.set(groupId, false); - xLongValues.set(groupId, val); + protected void updateLongNullTimeValue(int groupId, boolean isXValueNull, long xValue) { + if (checkAndUpdateNullTime(groupId, isXValueNull)) { + xLongValues.set(groupId, xValue); } } private void addFloatInput( int[] groupIds, Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateFloatFirstValue(groupIds[i], xColumn, i, timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateFloatFirstValue( - groupIds[position], xColumn, position, timeColumn.getLong(position)); - } + + int groupId = groupIds[position]; + if (!timeColumn.isNull(position)) { + updateFloatFirstValue( + groupId, + xColumn.isNull(position), + xColumn.getFloat(position), + timeColumn.getLong(position)); + } else { + updateFloatNullTimeValue(groupId, xColumn.isNull(position), xColumn.getFloat(position)); } } } - protected void updateFloatFirstValue(int groupId, Column xColumn, int xIdx, long curTime) { - if (!inits.get(groupId) || curTime < yFirstTimes.get(groupId)) { - inits.set(groupId, true); - yFirstTimes.set(groupId, curTime); - if (xColumn.isNull(xIdx)) { - xNulls.set(groupId, true); - } else { - xNulls.set(groupId, false); - xFloatValues.set(groupId, xColumn.getFloat(xIdx)); - } + protected void updateFloatFirstValue( + int groupId, boolean isXValueNull, float xValue, long curTime) { + if (checkAndUpdateFirstTime(groupId, isXValueNull, curTime)) { + xFloatValues.set(groupId, xValue); } } - protected void updateFloatFirstValue(int groupId, float val, long curTime) { - if (!inits.get(groupId) || curTime < yFirstTimes.get(groupId)) { - inits.set(groupId, true); - yFirstTimes.set(groupId, curTime); - xNulls.set(groupId, false); - xFloatValues.set(groupId, val); + protected void updateFloatNullTimeValue(int groupId, boolean isXValueNull, float xValue) { + if (checkAndUpdateNullTime(groupId, isXValueNull)) { + xFloatValues.set(groupId, xValue); } } private void addDoubleInput( int[] groupIds, Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateDoubleFirstValue(groupIds[i], xColumn, i, timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateDoubleFirstValue( - groupIds[position], xColumn, position, timeColumn.getLong(position)); - } + + int groupId = groupIds[position]; + if (!timeColumn.isNull(position)) { + updateDoubleFirstValue( + groupId, + xColumn.isNull(position), + xColumn.getDouble(position), + timeColumn.getLong(position)); + } else { + updateDoubleNullTimeValue(groupId, xColumn.isNull(position), xColumn.getDouble(position)); } } } - protected void updateDoubleFirstValue(int groupId, Column xColumn, int xIdx, long curTime) { - if (!inits.get(groupId) || curTime < yFirstTimes.get(groupId)) { - inits.set(groupId, true); - yFirstTimes.set(groupId, curTime); - if (xColumn.isNull(xIdx)) { - xNulls.set(groupId, true); - } else { - xNulls.set(groupId, false); - xDoubleValues.set(groupId, xColumn.getDouble(xIdx)); - } + protected void updateDoubleFirstValue( + int groupId, boolean isXValueNull, double xValue, long curTime) { + if (checkAndUpdateFirstTime(groupId, isXValueNull, curTime)) { + xDoubleValues.set(groupId, xValue); } } - protected void updateDoubleFirstValue(int groupId, double val, long curTime) { - if (!inits.get(groupId) || curTime < yFirstTimes.get(groupId)) { - inits.set(groupId, true); - yFirstTimes.set(groupId, curTime); - xNulls.set(groupId, false); - xDoubleValues.set(groupId, val); + protected void updateDoubleNullTimeValue(int groupId, boolean isXValueNull, double xValue) { + if (checkAndUpdateNullTime(groupId, isXValueNull)) { + xDoubleValues.set(groupId, xValue); } } private void addBinaryInput( int[] groupIds, Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateBinaryFirstValue(groupIds[i], xColumn, i, timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateBinaryFirstValue( - groupIds[position], xColumn, position, timeColumn.getLong(position)); - } + + int groupId = groupIds[position]; + if (!timeColumn.isNull(position)) { + updateBinaryFirstValue( + groupId, + xColumn.isNull(position), + xColumn.getBinary(position), + timeColumn.getLong(position)); + } else { + updateBinaryNullTimeValue(groupId, xColumn.isNull(position), xColumn.getBinary(position)); } } } - protected void updateBinaryFirstValue(int groupId, Column xColumn, int xIdx, long curTime) { - if (!inits.get(groupId) || curTime < yFirstTimes.get(groupId)) { - inits.set(groupId, true); - yFirstTimes.set(groupId, curTime); - if (xColumn.isNull(xIdx)) { - xNulls.set(groupId, true); - } else { - xNulls.set(groupId, false); - xBinaryValues.set(groupId, xColumn.getBinary(xIdx)); - } + protected void updateBinaryFirstValue( + int groupId, boolean isXValueNull, Binary xValue, long curTime) { + if (checkAndUpdateFirstTime(groupId, isXValueNull, curTime)) { + xBinaryValues.set(groupId, xValue); } } - protected void updateBinaryFirstValue(int groupId, Binary val, long curTime) { - if (!inits.get(groupId) || curTime < yFirstTimes.get(groupId)) { - inits.set(groupId, true); - yFirstTimes.set(groupId, curTime); - xNulls.set(groupId, false); - xBinaryValues.set(groupId, val); + protected void updateBinaryNullTimeValue(int groupId, boolean isXValueNull, Binary xValue) { + if (checkAndUpdateNullTime(groupId, isXValueNull)) { + xBinaryValues.set(groupId, xValue); } } private void addBooleanInput( int[] groupIds, Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateBooleanFirstValue(groupIds[i], xColumn, i, timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateBooleanFirstValue( - groupIds[position], xColumn, position, timeColumn.getLong(position)); - } + + int groupId = groupIds[position]; + if (!timeColumn.isNull(position)) { + updateBooleanFirstValue( + groupId, + xColumn.isNull(position), + xColumn.getBoolean(position), + timeColumn.getLong(position)); + } else { + updateBooleanNullTimeValue(groupId, xColumn.isNull(position), xColumn.getBoolean(position)); } } } - protected void updateBooleanFirstValue(int groupId, Column xColumn, int xIdx, long curTime) { - if (!inits.get(groupId) || curTime < yFirstTimes.get(groupId)) { - inits.set(groupId, true); - yFirstTimes.set(groupId, curTime); - if (xColumn.isNull(xIdx)) { - xNulls.set(groupId, true); - } else { - xNulls.set(groupId, false); - xBooleanValues.set(groupId, xColumn.getBoolean(xIdx)); - } + protected void updateBooleanFirstValue( + int groupId, boolean isXValueNull, boolean xValue, long curTime) { + if (checkAndUpdateFirstTime(groupId, isXValueNull, curTime)) { + xBooleanValues.set(groupId, xValue); } } - protected void updateBooleanFirstValue(int groupId, boolean val, long curTime) { - if (!inits.get(groupId) || curTime < yFirstTimes.get(groupId)) { - inits.set(groupId, true); - yFirstTimes.set(groupId, curTime); - xNulls.set(groupId, false); - xBooleanValues.set(groupId, val); + protected void updateBooleanNullTimeValue(int groupId, boolean isXValueNull, boolean xValue) { + if (checkAndUpdateNullTime(groupId, isXValueNull)) { + xBooleanValues.set(groupId, xValue); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java index b21a683ab0a1a..632e9fdb50ee0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java @@ -50,6 +50,8 @@ public class GroupedLastAccumulator implements GroupedAccumulator { RamUsageEstimator.shallowSizeOfInstance(GroupedLastAccumulator.class); private final TSDataType seriesDataType; private final LongBigArray maxTimes = new LongBigArray(Long.MIN_VALUE); + private final BooleanBigArray inits = new BooleanBigArray(); + private final BooleanBigArray initNullTimeValues = new BooleanBigArray(); private LongBigArray longValues; private IntBigArray intValues; @@ -122,12 +124,18 @@ public long getEstimatedSize() { String.format("Unsupported data type in LAST Aggregation: %s", seriesDataType)); } - return INSTANCE_SIZE + valuesSize; + return INSTANCE_SIZE + + inits.sizeOf() + + maxTimes.sizeOf() + + initNullTimeValues.sizeOf() + + valuesSize; } @Override public void setGroupCount(long groupCount) { maxTimes.ensureCapacity(groupCount); + inits.ensureCapacity(groupCount); + initNullTimeValues.ensureCapacity(groupCount); switch (seriesDataType) { case INT32: case DATE: @@ -207,25 +215,44 @@ public void addIntermediate(int[] groupIds, Column argument) { byte[] bytes = argument.getBinary(i).getValues(); long time = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 0); int offset = Long.BYTES; + boolean isOrderTimeNull = BytesUtils.bytesToBool(bytes, offset); + offset += 1; + int groupId = groupIds[i]; switch (seriesDataType) { case INT32: case DATE: int intVal = BytesUtils.bytesToInt(bytes, offset); - updateIntValue(groupIds[i], intVal, time); + if (!isOrderTimeNull) { + updateIntValue(groupId, intVal, time); + } else { + updateIntNullTimeValue(groupId, intVal); + } break; case INT64: case TIMESTAMP: long longVal = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - updateLongValue(groupIds[i], longVal, time); + if (!isOrderTimeNull) { + updateLongValue(groupId, longVal, time); + } else { + updateLongNullTimeValue(groupId, longVal); + } break; case FLOAT: float floatVal = BytesUtils.bytesToFloat(bytes, offset); - updateFloatValue(groupIds[i], floatVal, time); + if (!isOrderTimeNull) { + updateFloatValue(groupId, floatVal, time); + } else { + updateFloatNullTimeValue(groupId, floatVal); + } break; case DOUBLE: double doubleVal = BytesUtils.bytesToDouble(bytes, offset); - updateDoubleValue(groupIds[i], doubleVal, time); + if (!isOrderTimeNull) { + updateDoubleValue(groupId, doubleVal, time); + } else { + updateDoubleNullTimeValue(groupId, doubleVal); + } break; case TEXT: case BLOB: @@ -234,11 +261,19 @@ public void addIntermediate(int[] groupIds, Column argument) { int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - updateBinaryValue(groupIds[i], binaryVal, time); + if (!isOrderTimeNull) { + updateBinaryValue(groupId, binaryVal, time); + } else { + updateBinaryNullTimeValue(groupId, binaryVal); + } break; case BOOLEAN: boolean boolVal = BytesUtils.bytesToBool(bytes, offset); - updateBooleanValue(groupIds[i], boolVal, time); + if (!isOrderTimeNull) { + updateBooleanValue(groupId, boolVal, time); + } else { + updateBooleanNullTimeValue(groupId, boolVal); + } break; default: throw new UnSupportedDataTypeException( @@ -252,46 +287,47 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { checkArgument( columnBuilder instanceof BinaryColumnBuilder, "intermediate input and output of LAST should be BinaryColumn"); - if (maxTimes.get(groupId) == Long.MIN_VALUE) { - columnBuilder.appendNull(); - } else { + if (inits.get(groupId) || initNullTimeValues.get(groupId)) { columnBuilder.writeBinary(new Binary(serializeTimeWithValue(groupId))); + return; } + columnBuilder.appendNull(); } @Override public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { - if (maxTimes.get(groupId) == Long.MIN_VALUE) { + if (!inits.get(groupId) && !initNullTimeValues.get(groupId)) { columnBuilder.appendNull(); - } else { - switch (seriesDataType) { - case INT32: - case DATE: - columnBuilder.writeInt(intValues.get(groupId)); - break; - case INT64: - case TIMESTAMP: - columnBuilder.writeLong(longValues.get(groupId)); - break; - case FLOAT: - columnBuilder.writeFloat(floatValues.get(groupId)); - break; - case DOUBLE: - columnBuilder.writeDouble(doubleValues.get(groupId)); - break; - case TEXT: - case BLOB: - case OBJECT: - case STRING: - columnBuilder.writeBinary(binaryValues.get(groupId)); - break; - case BOOLEAN: - columnBuilder.writeBoolean(booleanValues.get(groupId)); - break; - default: - throw new UnSupportedDataTypeException( - String.format("Unsupported data type in LAST Aggregation: %s", seriesDataType)); - } + return; + } + + switch (seriesDataType) { + case INT32: + case DATE: + columnBuilder.writeInt(intValues.get(groupId)); + break; + case INT64: + case TIMESTAMP: + columnBuilder.writeLong(longValues.get(groupId)); + break; + case FLOAT: + columnBuilder.writeFloat(floatValues.get(groupId)); + break; + case DOUBLE: + columnBuilder.writeDouble(doubleValues.get(groupId)); + break; + case TEXT: + case BLOB: + case OBJECT: + case STRING: + columnBuilder.writeBinary(binaryValues.get(groupId)); + break; + case BOOLEAN: + columnBuilder.writeBoolean(booleanValues.get(groupId)); + break; + default: + throw new UnSupportedDataTypeException( + String.format("Unsupported data type in LAST Aggregation: %s", seriesDataType)); } } @@ -301,6 +337,8 @@ public void prepareFinal() {} @Override public void reset() { maxTimes.reset(); + inits.reset(); + initNullTimeValues.reset(); switch (seriesDataType) { case INT32: case DATE: @@ -334,32 +372,39 @@ public void reset() { private byte[] serializeTimeWithValue(int groupId) { byte[] bytes; int length = Long.BYTES; + boolean isOrderTimeNull = !inits.get(groupId); + length += 1; + switch (seriesDataType) { case INT32: case DATE: length += Integer.BYTES; bytes = new byte[length]; longToBytes(maxTimes.get(groupId), bytes, 0); - intToBytes(intValues.get(groupId), bytes, Long.BYTES); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES); + intToBytes(intValues.get(groupId), bytes, Long.BYTES + 1); return bytes; case INT64: case TIMESTAMP: length += Long.BYTES; bytes = new byte[length]; longToBytes(maxTimes.get(groupId), bytes, 0); - longToBytes(longValues.get(groupId), bytes, Long.BYTES); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES); + longToBytes(longValues.get(groupId), bytes, Long.BYTES + 1); return bytes; case FLOAT: length += Float.BYTES; bytes = new byte[length]; longToBytes(maxTimes.get(groupId), bytes, 0); - floatToBytes(floatValues.get(groupId), bytes, Long.BYTES); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES); + floatToBytes(floatValues.get(groupId), bytes, Long.BYTES + 1); return bytes; case DOUBLE: length += Double.BYTES; bytes = new byte[length]; longToBytes(maxTimes.get(groupId), bytes, 0); - doubleToBytes(doubleValues.get(groupId), bytes, Long.BYTES); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES); + doubleToBytes(doubleValues.get(groupId), bytes, Long.BYTES + 1); return bytes; case TEXT: case BLOB: @@ -369,14 +414,16 @@ private byte[] serializeTimeWithValue(int groupId) { length += Integer.BYTES + values.length; bytes = new byte[length]; longToBytes(maxTimes.get(groupId), bytes, 0); - BytesUtils.intToBytes(values.length, bytes, Long.BYTES); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES); + BytesUtils.intToBytes(values.length, bytes, Long.BYTES + 1); System.arraycopy(values, 0, bytes, length - values.length, values.length); return bytes; case BOOLEAN: length++; bytes = new byte[length]; longToBytes(maxTimes.get(groupId), bytes, 0); - boolToBytes(booleanValues.get(groupId), bytes, Long.BYTES); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES); + boolToBytes(booleanValues.get(groupId), bytes, Long.BYTES + 1); return bytes; default: throw new UnSupportedDataTypeException( @@ -384,188 +431,223 @@ private byte[] serializeTimeWithValue(int groupId) { } } + private boolean checkAndUpdateLastTime(int groupId, long curTime) { + if (!inits.get(groupId) || curTime > maxTimes.get(groupId)) { + inits.set(groupId, true); + maxTimes.set(groupId, curTime); + return true; + } + return false; + } + + private boolean checkAndUpdateNullTime(int groupId) { + if (!inits.get(groupId) && !initNullTimeValues.get(groupId)) { + initNullTimeValues.set(groupId, true); + return true; + } + return false; + } + private void addIntInput( int[] groupIds, Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateIntValue(groupIds[i], valueColumn.getInt(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateIntValue( - groupIds[position], valueColumn.getInt(position), timeColumn.getLong(position)); - } + + int groupId = groupIds[position]; + if (!timeColumn.isNull(position)) { + updateIntValue(groupId, valueColumn.getInt(position), timeColumn.getLong(position)); + } else { + updateIntNullTimeValue(groupId, valueColumn.getInt(position)); } } } protected void updateIntValue(int groupId, int value, long curTime) { - long maxTime = maxTimes.get(groupId); - if (curTime > maxTime) { - maxTimes.set(groupId, curTime); + if (checkAndUpdateLastTime(groupId, curTime)) { + intValues.set(groupId, value); + } + } + + protected void updateIntNullTimeValue(int groupId, int value) { + if (checkAndUpdateNullTime(groupId)) { intValues.set(groupId, value); } } private void addLongInput( int[] groupIds, Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateLongValue(groupIds[i], valueColumn.getLong(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateLongValue( - groupIds[position], valueColumn.getLong(position), timeColumn.getLong(position)); - } + + int groupId = groupIds[position]; + if (!timeColumn.isNull(position)) { + updateLongValue(groupId, valueColumn.getLong(position), timeColumn.getLong(position)); + } else { + updateLongNullTimeValue(groupId, valueColumn.getLong(position)); } } } protected void updateLongValue(int groupId, long value, long curTime) { - long maxTime = maxTimes.get(groupId); - if (curTime > maxTime) { - maxTimes.set(groupId, curTime); + if (checkAndUpdateLastTime(groupId, curTime)) { + longValues.set(groupId, value); + } + } + + protected void updateLongNullTimeValue(int groupId, long value) { + if (checkAndUpdateNullTime(groupId)) { longValues.set(groupId, value); } } private void addFloatInput( int[] groupIds, Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateFloatValue(groupIds[i], valueColumn.getFloat(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateFloatValue( - groupIds[position], valueColumn.getFloat(position), timeColumn.getLong(position)); - } + + int groupId = groupIds[position]; + if (!timeColumn.isNull(position)) { + updateFloatValue(groupId, valueColumn.getFloat(position), timeColumn.getLong(position)); + } else { + updateFloatNullTimeValue(groupId, valueColumn.getFloat(position)); } } } protected void updateFloatValue(int groupId, float value, long curTime) { - long maxTime = maxTimes.get(groupId); - if (curTime > maxTime) { - maxTimes.set(groupId, curTime); + if (checkAndUpdateLastTime(groupId, curTime)) { + floatValues.set(groupId, value); + } + } + + protected void updateFloatNullTimeValue(int groupId, float value) { + if (checkAndUpdateNullTime(groupId)) { floatValues.set(groupId, value); } } private void addDoubleInput( int[] groupIds, Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateDoubleValue(groupIds[i], valueColumn.getDouble(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateDoubleValue( - groupIds[position], valueColumn.getDouble(position), timeColumn.getLong(position)); - } + + int groupId = groupIds[position]; + if (!timeColumn.isNull(position)) { + updateDoubleValue(groupId, valueColumn.getDouble(position), timeColumn.getLong(position)); + } else { + updateDoubleNullTimeValue(groupId, valueColumn.getDouble(position)); } } } protected void updateDoubleValue(int groupId, double value, long curTime) { - long maxTime = maxTimes.get(groupId); - if (curTime > maxTime) { - maxTimes.set(groupId, curTime); + if (checkAndUpdateLastTime(groupId, curTime)) { + doubleValues.set(groupId, value); + } + } + + protected void updateDoubleNullTimeValue(int groupId, double value) { + if (checkAndUpdateNullTime(groupId)) { doubleValues.set(groupId, value); } } private void addBinaryInput( int[] groupIds, Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateBinaryValue(groupIds[i], valueColumn.getBinary(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateBinaryValue( - groupIds[position], valueColumn.getBinary(position), timeColumn.getLong(position)); - } + + int groupId = groupIds[position]; + if (!timeColumn.isNull(position)) { + updateBinaryValue(groupId, valueColumn.getBinary(position), timeColumn.getLong(position)); + } else { + updateBinaryNullTimeValue(groupId, valueColumn.getBinary(position)); } } } protected void updateBinaryValue(int groupId, Binary value, long curTime) { - long maxTime = maxTimes.get(groupId); - if (curTime > maxTime) { - maxTimes.set(groupId, curTime); + if (checkAndUpdateLastTime(groupId, curTime)) { + binaryValues.set(groupId, value); + } + } + + protected void updateBinaryNullTimeValue(int groupId, Binary value) { + if (checkAndUpdateNullTime(groupId)) { binaryValues.set(groupId, value); } } private void addBooleanInput( int[] groupIds, Column valueColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!valueColumn.isNull(i)) { - updateBooleanValue(groupIds[i], valueColumn.getBoolean(i), timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (valueColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!valueColumn.isNull(position)) { - updateBooleanValue( - groupIds[position], valueColumn.getBoolean(position), timeColumn.getLong(position)); - } + + int groupId = groupIds[position]; + if (!timeColumn.isNull(position)) { + updateBooleanValue(groupId, valueColumn.getBoolean(position), timeColumn.getLong(position)); + } else { + updateBooleanNullTimeValue(groupId, valueColumn.getBoolean(position)); } } } protected void updateBooleanValue(int groupId, boolean value, long curTime) { - long maxTime = maxTimes.get(groupId); - if (curTime > maxTime) { - maxTimes.set(groupId, curTime); + if (checkAndUpdateLastTime(groupId, curTime)) { + booleanValues.set(groupId, value); + } + } + + protected void updateBooleanNullTimeValue(int groupId, boolean value) { + if (checkAndUpdateNullTime(groupId)) { booleanValues.set(groupId, value); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java index db48d221f18b2..6b8288995142d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java @@ -56,6 +56,7 @@ public class GroupedLastByAccumulator implements GroupedAccumulator { private final LongBigArray yLastTimes = new LongBigArray(Long.MIN_VALUE); private final BooleanBigArray inits = new BooleanBigArray(); + private final BooleanBigArray initNullTimeValues = new BooleanBigArray(); private LongBigArray xLongValues; private IntBigArray xIntValues; @@ -132,13 +133,19 @@ public long getEstimatedSize() { String.format("Unsupported data type in LAST_BY Aggregation: %s", xDataType)); } - return INSTANCE_SIZE + valuesSize + yLastTimes.sizeOf() + inits.sizeOf() + xNulls.sizeOf(); + return INSTANCE_SIZE + + valuesSize + + yLastTimes.sizeOf() + + inits.sizeOf() + + initNullTimeValues.sizeOf() + + xNulls.sizeOf(); } @Override public void setGroupCount(long groupCount) { yLastTimes.ensureCapacity(groupCount); inits.ensureCapacity(groupCount); + initNullTimeValues.ensureCapacity(groupCount); xNulls.ensureCapacity(groupCount); switch (xDataType) { case INT32: @@ -177,6 +184,7 @@ public void prepareFinal() {} public void reset() { yLastTimes.reset(); inits.reset(); + initNullTimeValues.reset(); xNulls.reset(); switch (xDataType) { case INT32: @@ -239,7 +247,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { return; default: throw new UnSupportedDataTypeException( - String.format("Unsupported data type in LAST_BY Aggregation: %s", yDataType)); + String.format("Unsupported data type in LAST_BY Aggregation: %s", xDataType)); } } @@ -259,54 +267,78 @@ public void addIntermediate(int[] groupIds, Column argument) { byte[] bytes = argument.getBinary(i).getValues(); long curTime = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, 0); int offset = Long.BYTES; - boolean isXNull = BytesUtils.bytesToBool(bytes, offset); + boolean isXValueNull = BytesUtils.bytesToBool(bytes, offset); + offset += 1; + boolean isOrderTimeNull = BytesUtils.bytesToBool(bytes, offset); offset += 1; int groupId = groupIds[i]; - if (isXNull) { - if (!inits.get(groupId) || curTime > yLastTimes.get(groupId)) { - inits.set(groupId, true); - yLastTimes.set(groupId, curTime); - xNulls.set(groupId, true); - } - continue; - } - switch (xDataType) { case INT32: case DATE: - int xIntVal = BytesUtils.bytesToInt(bytes, offset); - updateIntLastValue(groupId, xIntVal, curTime); + int intVal = isXValueNull ? 0 : BytesUtils.bytesToInt(bytes, offset); + if (!isOrderTimeNull) { + updateIntLastValue(groupId, isXValueNull, intVal, curTime); + } else { + updateIntNullTimeValue(groupId, isXValueNull, intVal); + } break; case INT64: case TIMESTAMP: - long longVal = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); - updateLongLastValue(groupId, longVal, curTime); + long longVal = + isXValueNull ? 0 : BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); + if (!isOrderTimeNull) { + updateLongLastValue(groupId, isXValueNull, longVal, curTime); + } else { + updateLongNullTimeValue(groupId, isXValueNull, longVal); + } break; case FLOAT: - float floatVal = BytesUtils.bytesToFloat(bytes, offset); - updateFloatLastValue(groupId, floatVal, curTime); + float floatVal = isXValueNull ? 0 : BytesUtils.bytesToFloat(bytes, offset); + if (!isOrderTimeNull) { + updateFloatLastValue(groupId, isXValueNull, floatVal, curTime); + } else { + updateFloatNullTimeValue(groupId, isXValueNull, floatVal); + } break; case DOUBLE: - double doubleVal = BytesUtils.bytesToDouble(bytes, offset); - updateDoubleLastValue(groupId, doubleVal, curTime); + double doubleVal = isXValueNull ? 0 : BytesUtils.bytesToDouble(bytes, offset); + if (!isOrderTimeNull) { + updateDoubleLastValue(groupId, isXValueNull, doubleVal, curTime); + } else { + updateDoubleNullTimeValue(groupId, isXValueNull, doubleVal); + } break; case TEXT: case BLOB: case OBJECT: case STRING: - int length = BytesUtils.bytesToInt(bytes, offset); - offset += Integer.BYTES; - Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); - updateBinaryLastValue(groupId, binaryVal, curTime); + Binary binaryVal = null; + if (!isXValueNull) { + int length = BytesUtils.bytesToInt(bytes, offset); + offset += Integer.BYTES; + binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); + } + if (!isOrderTimeNull) { + updateBinaryLastValue(groupId, isXValueNull, binaryVal, curTime); + } else { + updateBinaryNullTimeValue(groupId, isXValueNull, binaryVal); + } break; case BOOLEAN: - boolean boolVal = BytesUtils.bytesToBool(bytes, offset); - updateBooleanLastValue(groupId, boolVal, curTime); + boolean boolVal = false; + if (!isXValueNull) { + boolVal = BytesUtils.bytesToBool(bytes, offset); + } + if (!isOrderTimeNull) { + updateBooleanLastValue(groupId, isXValueNull, boolVal, curTime); + } else { + updateBooleanNullTimeValue(groupId, isXValueNull, boolVal); + } break; default: throw new UnSupportedDataTypeException( - String.format("Unsupported data type in LAST_BY Aggregation: %s", yDataType)); + String.format("Unsupported data type in LAST_BY Aggregation: %s", xDataType)); } } } @@ -317,50 +349,54 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { columnBuilder instanceof BinaryColumnBuilder, "intermediate input and output of LAST_BY should be BinaryColumn"); - if (!inits.get(groupId)) { - columnBuilder.appendNull(); - } else { + if (inits.get(groupId) || initNullTimeValues.get(groupId)) { columnBuilder.writeBinary(new Binary(serializeTimeWithValue(groupId))); + return; } + columnBuilder.appendNull(); } private byte[] serializeTimeWithValue(int groupId) { boolean xNull = xNulls.get(groupId); - int length = Long.BYTES + 1 + (xNull ? 0 : calculateValueLength(groupId)); + int length = Long.BYTES + 2 + (xNull ? 0 : calculateValueLength(groupId)); byte[] bytes = new byte[length]; + boolean isOrderTimeNull = !inits.get(groupId); longToBytes(yLastTimes.get(groupId), bytes, 0); boolToBytes(xNulls.get(groupId), bytes, Long.BYTES); + boolToBytes(isOrderTimeNull, bytes, Long.BYTES + 1); + if (!xNull) { + int valueOffset = Long.BYTES + 2; switch (xDataType) { case INT32: case DATE: - intToBytes(xIntValues.get(groupId), bytes, Long.BYTES + 1); + intToBytes(xIntValues.get(groupId), bytes, valueOffset); return bytes; case INT64: case TIMESTAMP: - longToBytes(xLongValues.get(groupId), bytes, Long.BYTES + 1); + longToBytes(xLongValues.get(groupId), bytes, valueOffset); return bytes; case FLOAT: - floatToBytes(xFloatValues.get(groupId), bytes, Long.BYTES + 1); + floatToBytes(xFloatValues.get(groupId), bytes, valueOffset); return bytes; case DOUBLE: - doubleToBytes(xDoubleValues.get(groupId), bytes, Long.BYTES + 1); + doubleToBytes(xDoubleValues.get(groupId), bytes, valueOffset); return bytes; case TEXT: case BLOB: case OBJECT: case STRING: byte[] values = xBinaryValues.get(groupId).getValues(); - intToBytes(values.length, bytes, Long.BYTES + 1); + intToBytes(values.length, bytes, valueOffset); System.arraycopy(values, 0, bytes, length - values.length, values.length); return bytes; case BOOLEAN: - boolToBytes(xBooleanValues.get(groupId), bytes, Long.BYTES + 1); + boolToBytes(xBooleanValues.get(groupId), bytes, valueOffset); return bytes; default: throw new UnSupportedDataTypeException( - String.format("Unsupported data type in LAST_BY Aggregation: %s", yDataType)); + String.format("Unsupported data type in LAST_BY Aggregation: %s", xDataType)); } } return bytes; @@ -393,7 +429,7 @@ private int calculateValueLength(int groupId) { @Override public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { - if (!inits.get(groupId) || xNulls.get(groupId)) { + if (xNulls.get(groupId)) { columnBuilder.appendNull(); return; } @@ -428,270 +464,266 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { } } - private void addIntInput( - int[] groupIds, Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + private boolean checkAndUpdateLastTime(int groupId, boolean isXValueNull, long curTime) { + if (!inits.get(groupId) || curTime > yLastTimes.get(groupId)) { + inits.set(groupId, true); + yLastTimes.set(groupId, curTime); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateIntLastValue(groupIds[i], xColumn, i, timeColumn.getLong(i)); - } - } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateIntLastValue(groupIds[position], xColumn, position, timeColumn.getLong(position)); - } + if (isXValueNull) { + xNulls.set(groupId, true); + return false; + } else { + xNulls.set(groupId, false); + return true; } } + return false; } - protected void updateIntLastValue(int groupId, Column xColumn, int xIdx, long curTime) { - if (!inits.get(groupId) || curTime > yLastTimes.get(groupId)) { - inits.set(groupId, true); - yLastTimes.set(groupId, curTime); - if (xColumn.isNull(xIdx)) { + private boolean checkAndUpdateNullTime(int groupId, boolean isXValueNull) { + if (!inits.get(groupId) && !initNullTimeValues.get(groupId)) { + initNullTimeValues.set(groupId, true); + + if (isXValueNull) { xNulls.set(groupId, true); + return false; } else { xNulls.set(groupId, false); - xIntValues.set(groupId, xColumn.getInt(xIdx)); + return true; } } + return false; } - protected void updateIntLastValue(int groupId, int val, long curTime) { - if (!inits.get(groupId) || curTime > yLastTimes.get(groupId)) { - inits.set(groupId, true); - yLastTimes.set(groupId, curTime); - xNulls.set(groupId, false); - xIntValues.set(groupId, val); + private void addIntInput( + int[] groupIds, Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; + } + + if (!timeColumn.isNull(position)) { + updateIntLastValue( + groupIds[position], + xColumn.isNull(position), + xColumn.getInt(position), + timeColumn.getLong(position)); + } else { + updateIntNullTimeValue( + groupIds[position], xColumn.isNull(position), xColumn.getInt(position)); + } + } + } + + protected void updateIntLastValue(int groupId, boolean isXValueNull, int xValue, long curTime) { + if (checkAndUpdateLastTime(groupId, isXValueNull, curTime)) { + xIntValues.set(groupId, xValue); + } + } + + protected void updateIntNullTimeValue(int groupId, boolean isXValueNull, int xValue) { + if (checkAndUpdateNullTime(groupId, isXValueNull)) { + xIntValues.set(groupId, xValue); } } private void addLongInput( int[] groupIds, Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateLongLastValue(groupIds[i], xColumn, i, timeColumn.getLong(i)); - } + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); + + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateLongLastValue(groupIds[position], xColumn, position, timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateLongLastValue( + groupIds[position], + xColumn.isNull(position), + xColumn.getLong(position), + timeColumn.getLong(position)); + } else { + updateLongNullTimeValue( + groupIds[position], xColumn.isNull(position), xColumn.getLong(position)); } } } - protected void updateLongLastValue(int groupId, Column xColumn, int xIdx, long curTime) { - if (!inits.get(groupId) || curTime > yLastTimes.get(groupId)) { - inits.set(groupId, true); - yLastTimes.set(groupId, curTime); - if (xColumn.isNull(xIdx)) { - xNulls.set(groupId, true); - } else { - xNulls.set(groupId, false); - xLongValues.set(groupId, xColumn.getLong(xIdx)); - } + protected void updateLongLastValue(int groupId, boolean isXValueNull, long xValue, long curTime) { + if (checkAndUpdateLastTime(groupId, isXValueNull, curTime)) { + xLongValues.set(groupId, xValue); } } - protected void updateLongLastValue(int groupId, long val, long curTime) { - if (!inits.get(groupId) || curTime > yLastTimes.get(groupId)) { - inits.set(groupId, true); - yLastTimes.set(groupId, curTime); - xNulls.set(groupId, false); - xLongValues.set(groupId, val); + protected void updateLongNullTimeValue(int groupId, boolean isXValueNull, long xValue) { + if (checkAndUpdateNullTime(groupId, isXValueNull)) { + xLongValues.set(groupId, xValue); } } private void addFloatInput( int[] groupIds, Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateFloatLastValue(groupIds[i], xColumn, i, timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateFloatLastValue(groupIds[position], xColumn, position, timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateFloatLastValue( + groupIds[position], + xColumn.isNull(position), + xColumn.getFloat(position), + timeColumn.getLong(position)); + } else { + updateFloatNullTimeValue( + groupIds[position], xColumn.isNull(position), xColumn.getFloat(position)); } } } - protected void updateFloatLastValue(int groupId, Column xColumn, int xIdx, long curTime) { - if (!inits.get(groupId) || curTime > yLastTimes.get(groupId)) { - inits.set(groupId, true); - yLastTimes.set(groupId, curTime); - if (xColumn.isNull(xIdx)) { - xNulls.set(groupId, true); - } else { - xNulls.set(groupId, false); - xFloatValues.set(groupId, xColumn.getFloat(xIdx)); - } + protected void updateFloatLastValue( + int groupId, boolean isXValueNull, float xValue, long curTime) { + if (checkAndUpdateLastTime(groupId, isXValueNull, curTime)) { + xFloatValues.set(groupId, xValue); } } - protected void updateFloatLastValue(int groupId, float val, long curTime) { - if (!inits.get(groupId) || curTime > yLastTimes.get(groupId)) { - inits.set(groupId, true); - yLastTimes.set(groupId, curTime); - xNulls.set(groupId, false); - xFloatValues.set(groupId, val); + protected void updateFloatNullTimeValue(int groupId, boolean isXValueNull, float xValue) { + if (checkAndUpdateNullTime(groupId, isXValueNull)) { + xFloatValues.set(groupId, xValue); } } private void addDoubleInput( int[] groupIds, Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateDoubleLastValue(groupIds[i], xColumn, i, timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateDoubleLastValue( - groupIds[position], xColumn, position, timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateDoubleLastValue( + groupIds[position], + xColumn.isNull(position), + xColumn.getDouble(position), + timeColumn.getLong(position)); + } else { + updateDoubleNullTimeValue( + groupIds[position], xColumn.isNull(position), xColumn.getDouble(position)); } } } - protected void updateDoubleLastValue(int groupId, Column xColumn, int xIdx, long curTime) { - if (!inits.get(groupId) || curTime > yLastTimes.get(groupId)) { - inits.set(groupId, true); - yLastTimes.set(groupId, curTime); - if (xColumn.isNull(xIdx)) { - xNulls.set(groupId, true); - } else { - xNulls.set(groupId, false); - xDoubleValues.set(groupId, xColumn.getDouble(xIdx)); - } + protected void updateDoubleLastValue( + int groupId, boolean isXValueNull, double xValue, long curTime) { + if (checkAndUpdateLastTime(groupId, isXValueNull, curTime)) { + xDoubleValues.set(groupId, xValue); } } - protected void updateDoubleLastValue(int groupId, double val, long curTime) { - if (!inits.get(groupId) || curTime > yLastTimes.get(groupId)) { - inits.set(groupId, true); - yLastTimes.set(groupId, curTime); - xNulls.set(groupId, false); - xDoubleValues.set(groupId, val); + protected void updateDoubleNullTimeValue(int groupId, boolean isXValueNull, double xValue) { + if (checkAndUpdateNullTime(groupId, isXValueNull)) { + xDoubleValues.set(groupId, xValue); } } private void addBinaryInput( int[] groupIds, Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateBinaryLastValue(groupIds[i], xColumn, i, timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateBinaryLastValue( - groupIds[position], xColumn, position, timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateBinaryLastValue( + groupIds[position], + xColumn.isNull(position), + xColumn.getBinary(position), + timeColumn.getLong(position)); + } else { + updateBinaryNullTimeValue( + groupIds[position], xColumn.isNull(position), xColumn.getBinary(position)); } } } - protected void updateBinaryLastValue(int groupId, Column xColumn, int xIdx, long curTime) { - if (!inits.get(groupId) || curTime > yLastTimes.get(groupId)) { - inits.set(groupId, true); - yLastTimes.set(groupId, curTime); - if (xColumn.isNull(xIdx)) { - xNulls.set(groupId, true); - } else { - xNulls.set(groupId, false); - xBinaryValues.set(groupId, xColumn.getBinary(xIdx)); - } + protected void updateBinaryLastValue( + int groupId, boolean isXValueNull, Binary xValue, long curTime) { + if (checkAndUpdateLastTime(groupId, isXValueNull, curTime)) { + xBinaryValues.set(groupId, xValue); } } - protected void updateBinaryLastValue(int groupId, Binary val, long curTime) { - if (!inits.get(groupId) || curTime > yLastTimes.get(groupId)) { - inits.set(groupId, true); - yLastTimes.set(groupId, curTime); - xNulls.set(groupId, false); - xBinaryValues.set(groupId, val); + protected void updateBinaryNullTimeValue(int groupId, boolean isXValueNull, Binary xValue) { + if (checkAndUpdateNullTime(groupId, isXValueNull)) { + xBinaryValues.set(groupId, xValue); } } private void addBooleanInput( int[] groupIds, Column xColumn, Column yColumn, Column timeColumn, AggregationMask mask) { - int positionCount = mask.getSelectedPositionCount(); + int selectPositionCount = mask.getSelectedPositionCount(); + + boolean isSelectAll = mask.isSelectAll(); + int[] selectedPositions = isSelectAll ? null : mask.getSelectedPositions(); - if (mask.isSelectAll()) { - for (int i = 0; i < positionCount; i++) { - if (!yColumn.isNull(i)) { - updateBooleanLastValue(groupIds[i], xColumn, i, timeColumn.getLong(i)); - } + for (int i = 0; i < selectPositionCount; i++) { + int position = isSelectAll ? i : selectedPositions[i]; + if (yColumn.isNull(position)) { + continue; } - } else { - int[] selectedPositions = mask.getSelectedPositions(); - int position; - for (int i = 0; i < positionCount; i++) { - position = selectedPositions[i]; - if (!yColumn.isNull(position)) { - updateBooleanLastValue( - groupIds[position], xColumn, position, timeColumn.getLong(position)); - } + + if (!timeColumn.isNull(position)) { + updateBooleanLastValue( + groupIds[position], + xColumn.isNull(position), + xColumn.getBoolean(position), + timeColumn.getLong(position)); + } else { + updateBooleanNullTimeValue( + groupIds[position], xColumn.isNull(position), xColumn.getBoolean(position)); } } } - protected void updateBooleanLastValue(int groupId, Column xColumn, int xIdx, long curTime) { - if (!inits.get(groupId) || curTime > yLastTimes.get(groupId)) { - inits.set(groupId, true); - yLastTimes.set(groupId, curTime); - if (xColumn.isNull(xIdx)) { - xNulls.set(groupId, true); - } else { - xNulls.set(groupId, false); - xBooleanValues.set(groupId, xColumn.getBoolean(xIdx)); - } + protected void updateBooleanLastValue( + int groupId, boolean isXValueNull, boolean xValue, long curTime) { + if (checkAndUpdateLastTime(groupId, isXValueNull, curTime)) { + xBooleanValues.set(groupId, xValue); } } - protected void updateBooleanLastValue(int groupId, boolean val, long curTime) { - if (!inits.get(groupId) || curTime > yLastTimes.get(groupId)) { - inits.set(groupId, true); - yLastTimes.set(groupId, curTime); - xNulls.set(groupId, false); - xBooleanValues.set(groupId, val); + protected void updateBooleanNullTimeValue(int groupId, boolean isXValueNull, boolean xValue) { + if (checkAndUpdateNullTime(groupId, isXValueNull)) { + xBooleanValues.set(groupId, xValue); } } }